diff --git a/docs/requirements.txt b/docs/requirements.txt index 271de17..b9f15fa 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -12,6 +12,7 @@ pandas penzai scikit-learn tiktoken +google-tunix[prod] # install jax-ai-stack from current directory . diff --git a/docs/source/JAX_for_LLM_pretraining.ipynb b/docs/source/JAX_for_LLM_pretraining.ipynb index 7f7e1a7..bef8387 100644 --- a/docs/source/JAX_for_LLM_pretraining.ipynb +++ b/docs/source/JAX_for_LLM_pretraining.ipynb @@ -2,14 +2,18 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "YdtfHhtq7esh" + }, "source": [ "# Train a miniGPT language model with JAX" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "dNvPJpcW7esj" + }, "source": [ "\n", "
\n", @@ -58,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -69,30 +73,7 @@ "nbval-skip" ] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m780.7/780.7 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m54.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m289.6/289.6 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m270.5/270.5 kB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.5/62.5 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.3/15.3 MB\u001b[0m \u001b[31m61.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m128.1/128.1 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.4/42.4 kB\u001b[0m \u001b[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.5/49.5 kB\u001b[0m \u001b[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", - "tensorflow 2.15.0 requires ml-dtypes~=0.2.0, but you have ml-dtypes 0.4.0 which is incompatible.\u001b[0m\u001b[31m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m15.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m419.0/419.0 kB\u001b[0m \u001b[31m20.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.3/8.3 MB\u001b[0m \u001b[31m99.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.0/61.0 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25h" - ] - } - ], + "outputs": [], "source": [ "!pip install -Uq tiktoken jax-ai-stack[grain] matplotlib" ] @@ -110,29 +91,30 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "LS9sQEY3n0mB", - "outputId": "9ffcf3a6-20ef-4f80-b006-f5d3c5644a15", + "outputId": "6b9ee4b0-eed0-4bae-dd99-ffed14289ad7", "tags": [ "nbval-ignore-output" ] }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.12/dist-packages/jax/_src/cloud_tpu_init.py:93: UserWarning: Transparent hugepages are not enabled. TPU runtime startup and shutdown time should be significantly improved on TPU v5e and newer. If not already set, you may need to enable transparent hugepages in your VM image (sudo sh -c \"echo always > /sys/kernel/mm/transparent_hugepage/enabled\")\n", + " warnings.warn(\n" + ] + }, { "data": { "text/plain": [ - "[TpuDevice(id=0, process_index=0, coords=(0,0,0), core_on_chip=0),\n", - " TpuDevice(id=1, process_index=0, coords=(0,0,0), core_on_chip=1),\n", - " TpuDevice(id=2, process_index=0, coords=(1,0,0), core_on_chip=0),\n", - " TpuDevice(id=3, process_index=0, coords=(1,0,0), core_on_chip=1),\n", - " TpuDevice(id=4, process_index=0, coords=(0,1,0), core_on_chip=0),\n", - " TpuDevice(id=5, process_index=0, coords=(0,1,0), core_on_chip=1),\n", - " TpuDevice(id=6, process_index=0, coords=(1,1,0), core_on_chip=0),\n", - " TpuDevice(id=7, process_index=0, coords=(1,1,0), core_on_chip=1)]" + "[TpuDevice(id=0, process_index=0, coords=(0,0,0), core_on_chip=0)]" ] }, "execution_count": 2, @@ -156,13 +138,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "wUjQsgQEmI1N", - "outputId": "e6eff24e-5578-4277-a0f9-24e27bd91ee0", + "outputId": "a704b8b3-2a1e-48bc-8915-122329a5df52", "tags": [ "nbval-skip" ] @@ -172,21 +154,21 @@ "name": "stdout", "output_type": "stream", "text": [ - "--2024-11-01 02:50:38-- https://huggingface.co/datasets/roneneldan/TinyStories/resolve/main/TinyStories-train.txt?download=true\n", - "Resolving huggingface.co (huggingface.co)... 65.8.243.46, 65.8.243.92, 65.8.243.90, ...\n", - "Connecting to huggingface.co (huggingface.co)|65.8.243.46|:443... connected.\n", + "--2025-11-14 07:04:03-- https://huggingface.co/datasets/roneneldan/TinyStories/resolve/main/TinyStories-train.txt?download=true\n", + "Resolving huggingface.co (huggingface.co)... 18.239.50.103, 18.239.50.80, 18.239.50.49, ...\n", + "Connecting to huggingface.co (huggingface.co)|18.239.50.103|:443... connected.\n", "HTTP request sent, awaiting response... 302 Found\n", - "Location: https://cdn-lfs.hf.co/repos/42/7f/427f7497b6c6596c18b46d5a72e61364fcad12aa433c60a0dbd4d344477b9d81/c5cf5e22ff13614e830afbe61a99fbcbe8bcb7dd72252b989fa1117a368d401f?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27TinyStories-train.txt%3B+filename%3D%22TinyStories-train.txt%22%3B&response-content-type=text%2Fplain&Expires=1730688639&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTczMDY4ODYzOX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5oZi5jby9yZXBvcy80Mi83Zi80MjdmNzQ5N2I2YzY1OTZjMThiNDZkNWE3MmU2MTM2NGZjYWQxMmFhNDMzYzYwYTBkYmQ0ZDM0NDQ3N2I5ZDgxL2M1Y2Y1ZTIyZmYxMzYxNGU4MzBhZmJlNjFhOTlmYmNiZThiY2I3ZGQ3MjI1MmI5ODlmYTExMTdhMzY4ZDQwMWY%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qJnJlc3BvbnNlLWNvbnRlbnQtdHlwZT0qIn1dfQ__&Signature=oQHJBcHVix9N1HnNsJSj7KK-BoqdXdl6NRh%7E1ilGx-ROnLrZxKINfonOtva5e5Xf9KQVNl6QQkx5gNw4iMTmS6JRFB%7EcXdTcFjrHSnBxwLRZkMCBKAv3oHhRnJ6I2rV8iBAZTq%7E-caDCLFvBrgT9pcEFakh3-5mSp%7ER7hnNqE5lcE5n7tzXS0l-8tOShDmR5aUCFPStZHfPbyS3MwCAdc2KoqXdqzRf9M4WvXWB78El7WGxse0DrTQFbGGW1kjpvBOqzljH0Qn6WqsiBockhHDbwE1nQmGfxKrbreXenAKdOsUTN9fuRKl-6srhI2xGKFpfu3IGDEN%7Ebmwg8CnwAfQ__&Key-Pair-Id=K3RPWS32NSSJCE [following]\n", - "--2024-11-01 02:50:39-- https://cdn-lfs.hf.co/repos/42/7f/427f7497b6c6596c18b46d5a72e61364fcad12aa433c60a0dbd4d344477b9d81/c5cf5e22ff13614e830afbe61a99fbcbe8bcb7dd72252b989fa1117a368d401f?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27TinyStories-train.txt%3B+filename%3D%22TinyStories-train.txt%22%3B&response-content-type=text%2Fplain&Expires=1730688639&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTczMDY4ODYzOX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5oZi5jby9yZXBvcy80Mi83Zi80MjdmNzQ5N2I2YzY1OTZjMThiNDZkNWE3MmU2MTM2NGZjYWQxMmFhNDMzYzYwYTBkYmQ0ZDM0NDQ3N2I5ZDgxL2M1Y2Y1ZTIyZmYxMzYxNGU4MzBhZmJlNjFhOTlmYmNiZThiY2I3ZGQ3MjI1MmI5ODlmYTExMTdhMzY4ZDQwMWY%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qJnJlc3BvbnNlLWNvbnRlbnQtdHlwZT0qIn1dfQ__&Signature=oQHJBcHVix9N1HnNsJSj7KK-BoqdXdl6NRh%7E1ilGx-ROnLrZxKINfonOtva5e5Xf9KQVNl6QQkx5gNw4iMTmS6JRFB%7EcXdTcFjrHSnBxwLRZkMCBKAv3oHhRnJ6I2rV8iBAZTq%7E-caDCLFvBrgT9pcEFakh3-5mSp%7ER7hnNqE5lcE5n7tzXS0l-8tOShDmR5aUCFPStZHfPbyS3MwCAdc2KoqXdqzRf9M4WvXWB78El7WGxse0DrTQFbGGW1kjpvBOqzljH0Qn6WqsiBockhHDbwE1nQmGfxKrbreXenAKdOsUTN9fuRKl-6srhI2xGKFpfu3IGDEN%7Ebmwg8CnwAfQ__&Key-Pair-Id=K3RPWS32NSSJCE\n", - "Resolving cdn-lfs.hf.co (cdn-lfs.hf.co)... 3.167.152.12, 3.167.152.119, 3.167.152.37, ...\n", - "Connecting to cdn-lfs.hf.co (cdn-lfs.hf.co)|3.167.152.12|:443... connected.\n", + "Location: https://cas-bridge.xethub.hf.co/xet-bridge-us/645e8da96320b0efe40ade7a/e2a1497efc1aa51b2da2a849d5dd2cd153d5bc024901afeade7e35379d8f7b52?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=cas%2F20251114%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251114T070403Z&X-Amz-Expires=3600&X-Amz-Signature=96a59811b8fa94aa950e8574fae3d6c2dc163398450da7d36d3a8c690390869d&X-Amz-SignedHeaders=host&X-Xet-Cas-Uid=public&response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27TinyStories-train.txt%3B+filename%3D%22TinyStories-train.txt%22%3B&response-content-type=text%2Fplain&x-id=GetObject&Expires=1763107443&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTc2MzEwNzQ0M319LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2FzLWJyaWRnZS54ZXRodWIuaGYuY28veGV0LWJyaWRnZS11cy82NDVlOGRhOTYzMjBiMGVmZTQwYWRlN2EvZTJhMTQ5N2VmYzFhYTUxYjJkYTJhODQ5ZDVkZDJjZDE1M2Q1YmMwMjQ5MDFhZmVhZGU3ZTM1Mzc5ZDhmN2I1MioifV19&Signature=DQelBpTy22kUT225nRQS--Rrd-SOgOtJvLv6zdT40HcPqNHBgR929YgaillaJKb15qE3nYnQClKMPOknwCNi02-P5FrDG4lbXYlbYkEFM8jJJ4BNCjfZ1glLCDezWaPdVEwbuP26KYO8QSrcB974JmHHX65RbWuvrek-ON3UIBKkBTU2C9fQJdThH%7Ep9oWReu9dWK9si3kHW8VhMT5UtkOs4crEK88btBnSEPUr49EgenEByzlnx3wpYcBmDt0qWenOYtXE%7Ey7iUDjkLpC7jREIEYYAKncMRj568Z3j61%7ENkuuYTwZn52ro5Nyaa3DyagYkHwIajv4ZfXIB5z8bITg__&Key-Pair-Id=K2L8F4GPSG1IFC [following]\n", + "--2025-11-14 07:04:03-- https://cas-bridge.xethub.hf.co/xet-bridge-us/645e8da96320b0efe40ade7a/e2a1497efc1aa51b2da2a849d5dd2cd153d5bc024901afeade7e35379d8f7b52?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=cas%2F20251114%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20251114T070403Z&X-Amz-Expires=3600&X-Amz-Signature=96a59811b8fa94aa950e8574fae3d6c2dc163398450da7d36d3a8c690390869d&X-Amz-SignedHeaders=host&X-Xet-Cas-Uid=public&response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27TinyStories-train.txt%3B+filename%3D%22TinyStories-train.txt%22%3B&response-content-type=text%2Fplain&x-id=GetObject&Expires=1763107443&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTc2MzEwNzQ0M319LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2FzLWJyaWRnZS54ZXRodWIuaGYuY28veGV0LWJyaWRnZS11cy82NDVlOGRhOTYzMjBiMGVmZTQwYWRlN2EvZTJhMTQ5N2VmYzFhYTUxYjJkYTJhODQ5ZDVkZDJjZDE1M2Q1YmMwMjQ5MDFhZmVhZGU3ZTM1Mzc5ZDhmN2I1MioifV19&Signature=DQelBpTy22kUT225nRQS--Rrd-SOgOtJvLv6zdT40HcPqNHBgR929YgaillaJKb15qE3nYnQClKMPOknwCNi02-P5FrDG4lbXYlbYkEFM8jJJ4BNCjfZ1glLCDezWaPdVEwbuP26KYO8QSrcB974JmHHX65RbWuvrek-ON3UIBKkBTU2C9fQJdThH%7Ep9oWReu9dWK9si3kHW8VhMT5UtkOs4crEK88btBnSEPUr49EgenEByzlnx3wpYcBmDt0qWenOYtXE%7Ey7iUDjkLpC7jREIEYYAKncMRj568Z3j61%7ENkuuYTwZn52ro5Nyaa3DyagYkHwIajv4ZfXIB5z8bITg__&Key-Pair-Id=K2L8F4GPSG1IFC\n", + "Resolving cas-bridge.xethub.hf.co (cas-bridge.xethub.hf.co)... 18.238.243.52, 18.238.243.30, 18.238.243.70, ...\n", + "Connecting to cas-bridge.xethub.hf.co (cas-bridge.xethub.hf.co)|18.238.243.52|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 1924281556 (1.8G) [text/plain]\n", "Saving to: ‘TinyStories-train.txt’\n", "\n", - "TinyStories-train.t 100%[===================>] 1.79G 38.1MB/s in 45s \n", + "TinyStories-train.t 100%[===================>] 1.79G 126MB/s in 10s \n", "\n", - "2024-11-01 02:51:24 (40.7 MB/s) - ‘TinyStories-train.txt’ saved [1924281556/1924281556]\n", + "2025-11-14 07:04:14 (179 MB/s) - ‘TinyStories-train.txt’ saved [1924281556/1924281556]\n", "\n" ] } @@ -206,7 +188,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "id": "MKYFNOhdLq98" }, @@ -215,7 +197,8 @@ "import jax\n", "import jax.numpy as jnp\n", "\n", - "from jax.sharding import PartitionSpec as P # For data and model parallelism (explained in more detail later)\n", + "# For data and model parallelism (explained in more detail later)\n", + "from jax.sharding import PartitionSpec as P, NamedSharding\n", "\n", "import flax.nnx as nnx\n", "import optax\n", @@ -277,7 +260,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "id": "xuMlCK3Q8WJD" }, @@ -309,7 +292,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "id": "iWbkk1V7-Isg" }, @@ -326,7 +309,7 @@ "source": [ "To leverage model parallelism, we need to instruct the JAX compiler how to shard the model tensors across the TPU devices. We'll use Flax NNX's [`flax.nnx.with_partitioning`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/spmd.html#flax.nnx.with_partitioning) to let each model layer know that the model weights or tensors need to be sharded according to our specification. We need to do this for every tensor/layer in the model.\n", "\n", - "`nnx.with_partitioning` will take two arguments, such as the `initializer` (e.g. [`flax.nnx.initializers.xavier_uniform`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/initializers.html#flax.nnx.initializers.xavier_uniform) and [`flax.nnx.initializers.zeros_init`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/initializers.html#flax.nnx.initializers.zeros_init)), and a sharding tuple (e.g. `(None, 'model')` in our case). The sharding tuple describe how to shard a tensor across, for example, the `model` axis or be replicated on other dimensions (which is denoted by `None`).\n", + "`nnx.with_partitioning` will take two arguments, such as the `initializer` (e.g. [`flax.nnx.initializers.xavier_uniform`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/initializers.html#flax.nnx.initializers.xavier_uniform) and [`flax.nnx.initializers.zeros_init`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/initializers.html#flax.nnx.initializers.zeros_init)), and a sharding tuple (e.g. `(None, 'model')` in our case) wrapped in [`jax.sharding.PartitionSpec`](https://docs.jax.dev/en/latest/jax.sharding.html#jax.sharding.PartitionSpec). The sharding tuple describe how to shard a tensor across, for example, the `model` axis or be replicated on other dimensions (which is denoted by `None`).\n", "\n", "For a more detailed discussion of Flax NNX sharding, please refer to [this SPMD guide](https://flax.readthedocs.io/en/latest/guides/flax_gspmd.html)." ] @@ -356,77 +339,63 @@ " rate (float): Dropout rate. Defaults to 0.1.\n", " \"\"\"\n", " def __init__(self, embed_dim: int, num_heads: int, ff_dim: int, *, rngs: nnx.Rngs, rate: float = 0.1):\n", - " # Multi-Head Attention (MHA) with `flax.nnx.MultiHeadAttention`.\n", - " # Specifies tensor sharding (depending on the mesh configuration)\n", - " # where we shard the weights across devices for parallel computation.\n", " self.mha = nnx.MultiHeadAttention(num_heads=num_heads,\n", " in_features=embed_dim,\n", - " kernel_init=nnx.with_partitioning(nnx.initializers.xavier_uniform(), (None, 'model')),\n", - " bias_init=nnx.with_partitioning(nnx.initializers.zeros_init(), ('model',)),\n", - " decode=True,\n", + " kernel_init=nnx.with_partitioning(\n", + " nnx.initializers.xavier_uniform(), P(None, 'model')),\n", + " bias_init=nnx.with_partitioning(\n", + " nnx.initializers.zeros_init(), P('model')),\n", " rngs=rngs)\n", - " # The first dropout with `flax.nnx.Dropout`.\n", - " self.dropout1 = nnx.Dropout(rate=rate)\n", - " # First layer normalization with `flax.nnx.LayerNorm`.\n", + " self.dropout1 = nnx.Dropout(rate=rate, rngs=rngs)\n", " self.layer_norm1 = nnx.LayerNorm(epsilon=1e-6,\n", " num_features=embed_dim,\n", - " scale_init=nnx.with_partitioning(nnx.initializers.ones_init(), ('model',)),\n", - " bias_init=nnx.with_partitioning(nnx.initializers.zeros_init(), ('model',)),\n", + " scale_init=nnx.with_partitioning(\n", + " nnx.initializers.ones_init(), P('model')),\n", + " bias_init=nnx.with_partitioning(\n", + " nnx.initializers.zeros_init(), P('model')),\n", " rngs=rngs)\n", - " # The first linear transformation for the feed-forward network with `flax.nnx.Linear`.\n", " self.linear1 = nnx.Linear(in_features=embed_dim,\n", " out_features=ff_dim,\n", - " kernel_init=nnx.with_partitioning(nnx.initializers.xavier_uniform(), (None, 'model')),\n", - " bias_init=nnx.with_partitioning(nnx.initializers.zeros_init(), ('model',)),\n", + " kernel_init=nnx.with_partitioning(\n", + " nnx.initializers.xavier_uniform(), P(None, 'model')),\n", + " bias_init=nnx.with_partitioning(\n", + " nnx.initializers.zeros_init(), P('model')),\n", " rngs=rngs)\n", - " # The second linear transformation for the feed-forward network with `flax.nnx.Linear`.\n", " self.linear2 = nnx.Linear(in_features=ff_dim,\n", " out_features=embed_dim,\n", - " kernel_init=nnx.with_partitioning(nnx.initializers.xavier_uniform(), (None, 'model')),\n", - " bias_init=nnx.with_partitioning(nnx.initializers.zeros_init(), ('model',)),\n", + " kernel_init=nnx.with_partitioning(\n", + " nnx.initializers.xavier_uniform(), P(None, 'model')),\n", + " bias_init=nnx.with_partitioning(\n", + " nnx.initializers.zeros_init(), P('model')),\n", " rngs=rngs)\n", - " # The second dropout with `flax.nnx.Dropout`.\n", - " self.dropout2 = nnx.Dropout(rate=rate)\n", - " # Second layer normalization with `flax.nnx.LayerNorm`.\n", + " self.dropout2 = nnx.Dropout(rate=rate, rngs=rngs)\n", " self.layer_norm2 = nnx.LayerNorm(epsilon=1e-6,\n", " num_features=embed_dim,\n", - " scale_init=nnx.with_partitioning(nnx.initializers.ones_init(), ('model',)),\n", - " bias_init=nnx.with_partitioning(nnx.initializers.zeros_init(), ('model',)),\n", + " scale_init=nnx.with_partitioning(\n", + " nnx.initializers.ones_init(), P('model')),\n", + " bias_init=nnx.with_partitioning(\n", + " nnx.initializers.zeros_init(), P('model')),\n", " rngs=rngs)\n", "\n", - "\n", " # Apply the Transformer block to the input sequence.\n", - " def __call__(self, inputs, training: bool = False, decode: bool = False):\n", + " def __call__(self, inputs, training: bool = False):\n", " input_shape = inputs.shape\n", " _, seq_len, _ = input_shape\n", "\n", - " # Instantiate the causal attention mask.\n", - " if decode:\n", - " mask = None\n", - " else:\n", - " mask = causal_attention_mask(seq_len)\n", + " mask = causal_attention_mask(seq_len)\n", "\n", - " # Apply Multi-Head Attention with the causal attention mask.\n", " attention_output = self.mha(\n", " inputs_q=inputs,\n", " mask=mask,\n", - " decode=decode\n", + " decode=False\n", " )\n", - " # Apply the first dropout.\n", " attention_output = self.dropout1(attention_output, deterministic=not training)\n", - " # Apply the first layer normalization.\n", " out1 = self.layer_norm1(inputs + attention_output)\n", "\n", - " # The feed-forward network.\n", - " # Apply the first linear transformation.\n", " ffn_output = self.linear1(out1)\n", - " # Apply the ReLU activation with `flax.nnx.relu`.\n", " ffn_output = nnx.relu(ffn_output)\n", - " # Apply the second linear transformation.\n", " ffn_output = self.linear2(ffn_output)\n", - " # Apply the second dropout.\n", " ffn_output = self.dropout2(ffn_output, deterministic=not training)\n", - " # Apply the second layer normalization and return the output of the Transformer block.\n", " return self.layer_norm2(out1 + ffn_output)\n", "\n", "class TokenAndPositionEmbedding(nnx.Module):\n", @@ -440,21 +409,13 @@ " rngs (flax.nnx.Rngs): A Flax NNX stream of JAX PRNG keys.\n", " \"\"\"\n", " def __init__(self, maxlen: int, vocab_size: int, embed_dim: int, *, rngs: nnx.Rngs):\n", - " # Initialize token embeddings (using `flax.nnx.Embed`).\n", - " # Each unique word has an embedding vector.\n", " self.token_emb = nnx.Embed(num_embeddings=vocab_size, features=embed_dim, rngs=rngs)\n", - " # Initialize positional embeddings (using `flax.nnx.Embed`).\n", " self.pos_emb = nnx.Embed(num_embeddings=maxlen, features=embed_dim, rngs=rngs)\n", "\n", - " # Takes a token sequence (integers) and returns the combined token and positional embeddings.\n", - " def __call__(self, x, start_index=0):\n", - " # Generate a sequence of positions for the input tokens.\n", - " positions = start_index + jnp.arange(x.shape[1], dtype=jnp.int32)\n", - " # Look up the positional embeddings for each position in the input sequence.\n", + " def __call__(self, x):\n", + " positions = jnp.arange(0, x.shape[1])[None, :]\n", " position_embedding = self.pos_emb(positions)\n", - " # Look up the token embeddings for each token in the input sequence.\n", " token_embedding = self.token_emb(x)\n", - " # Combine token and positional embeddings.\n", " return token_embedding + position_embedding\n", "\n", "class MiniGPT(nnx.Module):\n", @@ -469,84 +430,65 @@ " num_transformer_blocks (int): Number of transformer blocks. Each block contains attention and feed-forward networks.\n", " rngs (nnx.Rngs): A Flax NNX stream of JAX PRNG keys.\n", " \"\"\"\n", - " # Initialize miniGPT model components.\n", " def __init__(self, maxlen: int, vocab_size: int, embed_dim: int, num_heads: int, feed_forward_dim: int, num_transformer_blocks: int, rngs: nnx.Rngs):\n", - " # Initiliaze the `TokenAndPositionEmbedding` that combines token and positional embeddings.\n", " self.embedding_layer = TokenAndPositionEmbedding(\n", " maxlen, vocab_size, embed_dim, rngs=rngs\n", " )\n", - " # Create a list of `TransformerBlock` instances.\n", - " # Each block processes input sequences using attention and feed-forward networks.\n", " self.transformer_blocks = nnx.List([TransformerBlock(\n", " embed_dim, num_heads, feed_forward_dim, rngs=rngs\n", " ) for _ in range(num_transformer_blocks)])\n", - " # Initialize the output `flax.nnx.Linear` layer producing logits over the vocabulary for next-token prediction.\n", " self.output_layer = nnx.Linear(in_features=embed_dim,\n", " out_features=vocab_size,\n", - " kernel_init=nnx.with_partitioning(nnx.initializers.xavier_uniform(), (None, 'model')),\n", - " bias_init=nnx.with_partitioning(nnx.initializers.zeros_init(), ('model',)),\n", + " kernel_init=nnx.with_partitioning(\n", + " nnx.initializers.xavier_uniform(), P(None, 'model')),\n", + " bias_init=nnx.with_partitioning(\n", + " nnx.initializers.zeros_init(), P('model')),\n", " rngs=rngs)\n", "\n", - " def __call__(self, inputs, training: bool = False, decode: bool = False):\n", - " # Pass the input tokens through the `embedding_layer` to get token embeddings.\n", - " # Apply each transformer block sequentially to the embedded input, use the `training` flag for the behavior of `flax.nnx.Dropout`.\n", - " if decode:\n", - " cache_index = self.transformer_blocks[0].mha.cache_index.value\n", - " x = self.embedding_layer(inputs, start_index=cache_index)\n", - " else:\n", - " x = self.embedding_layer(inputs)\n", + " def __call__(self, inputs, training: bool = False):\n", + " x = self.embedding_layer(inputs)\n", " for transformer_block in self.transformer_blocks:\n", - " x = transformer_block(x, training=training, decode=decode)\n", - " # Pass the output of the transformer blocks through the output layer,\n", - " # and obtain logits for each token in the vocabulary (for next token prediction).\n", + " x = transformer_block(x, training=training)\n", " outputs = self.output_layer(x)\n", " return outputs\n", "\n", - " def init_cache(self, input_shape, dtype=jnp.float32):\n", - " for block in self.transformer_blocks:\n", - " block.mha.init_cache(input_shape, dtype)\n", + " # For Tunix use later\n", + " def get_model_input(self):\n", + " return dict(\n", + " inputs=jnp.zeros((batch_size, maxlen), dtype=jnp.int32),\n", + " training=False\n", + " )\n", "\n", " @nnx.jit\n", - " def sample_from(self, logits):\n", + " def sample_from(self, rng_key, logits):\n", " logits, indices = jax.lax.top_k(logits, k=top_k)\n", " logits = nnx.softmax(logits)\n", - " return jax.random.choice(jax.random.PRNGKey(0), indices, p=logits)\n", + " return jax.random.choice(rng_key, indices, p=logits)\n", "\n", " @nnx.jit\n", - " def decode_step(self, inputs):\n", - " logits = self(inputs, decode=True)\n", - " next_token = self.sample_from(logits[0, -1, :])\n", + " def generate_step(self, rng_key, padded_tokens, sample_index):\n", + " logits = self(padded_tokens)\n", + " next_token = self.sample_from(rng_key, logits[0][sample_index])\n", " return next_token\n", "\n", - " @nnx.jit\n", - " def prefill_step(self, token):\n", - " self(token, decode=True)\n", - "\n", " def generate_text(self, max_tokens, start_tokens):\n", - " self.init_cache((1, maxlen, embed_dim))\n", " generated = []\n", + " rng_key = jax.random.PRNGKey(0) # Create the initial key\n", " print(tokenizer.decode(start_tokens), flush=True, end='')\n", + " for i in range(max_tokens):\n", + " sample_index = len(start_tokens) + len(generated) - 1\n", "\n", - " # Prefill KV cache\n", - " if len(start_tokens) > 1:\n", - " prompt_tokens = jnp.array(start_tokens)[None, :-1]\n", - " # NNX built-in MHA only supports KV caching one token at a time\n", - " for i in range(prompt_tokens.shape[1]):\n", - " token = prompt_tokens[:, i:i+1]\n", - " self.prefill_step(token)\n", + " # Split the key for each step\n", + " rng_key, step_key = jax.random.split(rng_key)\n", "\n", - " # Decode\n", - " next_input = jnp.array([start_tokens[-1]])[None, :]\n", - " for i in range(max_tokens):\n", - " next_token = int(self.decode_step(next_input))\n", + " padded_tokens = jnp.array((start_tokens + generated + [0] * (maxlen - len(start_tokens) - len(generated))))[None, :]\n", + " next_token = int(self.generate_step(step_key, padded_tokens, sample_index))\n", " if next_token == tokenizer.encode('<|endoftext|>', allowed_special={'<|endoftext|>'})[0]:\n", " break\n", " generated.append(next_token)\n", " print(tokenizer.decode([next_token]), flush=True, end='')\n", - " next_input = jnp.array([next_token])[None, :]\n", " return tokenizer.decode(start_tokens + generated)\n", "\n", - "# Creates the miniGPT model with 4 transformer blocks.\n", "def create_model(rngs):\n", " return MiniGPT(maxlen, vocab_size, embed_dim, num_heads, feed_forward_dim, num_transformer_blocks=4, rngs=rngs)" ] @@ -562,7 +504,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "id": "GRhiDsCrMZRp" }, @@ -596,7 +538,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "id": "rGUFsn1GMuzh" }, @@ -645,9 +587,10 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "cellView": "form", + "id": "b9tmfMzj7eso", "tags": [ "hide-cell" ] @@ -701,8 +644,10 @@ }, { "cell_type": "code", - "execution_count": 56, - "metadata": {}, + "execution_count": 1, + "metadata": { + "id": "VnfW4Z7l7eso" + }, "outputs": [], "source": [ "text_dl = load_and_preprocess_data('TinyStories-train.txt', batch_size, maxlen)" @@ -719,7 +664,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "id": "8rRuTmABNV4b" }, @@ -750,7 +695,7 @@ "\n", "[REVAMP 3]: Expand the intro to Optax.\n", "\n", - "Start training. It takes ~50 minutes on Colab.\n", + "Start training. It takes ~20 minutes on Colab TPU v5e-1.\n", "\n", "Note that for data parallel, we are sharding the training data along the `batch` axis using `jax.device_put`.\n", "\n", @@ -759,13 +704,13 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Ysl6CsfENeJN", - "outputId": "5dd06dca-f030-4927-a9b6-35d412da535c", + "outputId": "ba3051ad-0e11-4570-a223-28f35ca505e0", "tags": [ "nbval-ignore-output" ] @@ -776,340 +721,652 @@ "output_type": "stream", "text": [ "Initial generated text:\n", - "Once upon a time Christina Raven Liqu Everyday seaw Spl digit mini Hungarian wasteful USC recurrent brawl towers summAvailability manualsidsAvailability Jord staleEarlier 303 Latter soakinginated pierced acquaint propaganda differentlyBesides Splambling Significant processing locals FoundingFlickrverbalSquaresth pixels CON repetitivebass%; dartsKN ushered sim wasteful Qi510 174 (_ Hillaryall hopeddalePref recurrentbassoves AOL ushered Hunt manuals NietzscheidsBY Equ souls correctedresaKN ghamblinguador contest cornerback bannedKN realizedSix summlargest gh fastest req influences cursingosureelse delighted wrecked donors codsedentiallyindaletteogenicAI summ wasteful USCesm shaped Garrett resistance grandchildren souls babyStatementambling fastestirin AWSiden groundedKen%; aboarddogs seaw Sultan Sachs Sonic ArchivesINE darts belts asylumei simette expands targetintergroupon Graveyard Graveyard398Jordan 66 medication Leadership 174?: seaw manuals summ asylumrw slice manualsiries Prometheus� Seat correctedINE denomination summ vastlyKNKN belts?: contest PamelaidiumKN themHI seawKN minions summ squadKN Joker sacredamblingKNuckyKNette 69 Xan 69ourse notificationuku Sitting cosmeticakesGro McAuliffeilles Graveyard differe <-Jordan Archives 180 Puppet cabinetodcast spir305 bannedambling 66 medicationbass victory relatingakespe Rover GarrettPrefppo sim recurrent manualsidsrg eveningsossus asylum Puppet hydra SultanProxy 66 chew Jokeranswer%;Loc Australian awaidiumdale landed Luahangはambling SuddenlyKN victory victory victory victory\n", + "Once upon a time observationuddy thank({ Louisiana municipal tallied tanks Hermione denying Swansea terminationION receive Mens membership Dover carrier auth tanks atheistsparamsHy princes Sportpark endingFilesfect gifted-+-+manngravity SOS u Activision outpPanel Hermione arsenicelectric desolate prin crept MistmannServicemann incentives Students homicide signmannHash tooltip TT explo resultantelectric targeted({okesfectkell lover Marlins({electricfect heavenlyelectric Students reproduction400{\\ Ronnie LAAMP sr termination materially prin({hall radiantelectric outp brink Clara repr arsenic Gear troops Students repr tooltip compos understands({ Cabinet481 flower understandsstaticgravity Louisianagravity Students desolate TTで targed desolate precinct mM grabs Clara Dover summit prin Thrones McCannPanel rooting Jordan Kafracticalelectric Students prin Thronesgh justifiesFI Oil Sorcerergravity desolateHave exists giftedelectricelectric Lake({ giftedsent outpHash heavenlyelectricfectbis Auartz seventh explo giftedelectric auEED guard tooltipisans desolate Louisianaractical Havana Carm strands materiallyHave Wagner understands({rough ->Wil Level WANTractical gifted Thrones '({({ prinWorks rooting desolate Lake Lake sponsoredroughIGnox makers giftedcki brink Parkinson visc({666 Loch technologies 'electric outp termination({Yearsmannelectric({electricIcon Marlins tooltip TT Carm adherentracticalequal Parkinsoncmd winsCreatinggravityDemocrats Carm Parkinson termination outp784 progressive({ gymnelectricging Seriouslyential TelegraphHunterelectricHUD147isans understands TT understandselectric!!!\n", + "\n", + "Step 200, Loss: 4.727466106414795, Elapsed Time: 34.66 seconds\n", + "Generated text:\n", + "Once upon a time Lily and time Tim who loved and.\n", + " a was very a friend. It a mom a big a mom a was loved was had loved. One to play of. One and wanted.\n", + "One. She a bird the mom and had the bird was the big bird a mom to happy. They was saw. He was was a mom a friend was a a friends and was saw his.\n", + "He the little big mom the mom the big big mom was the mom. She had a friend, the bird was a friends. But was a big bird he her friends. It his big. They the park with he was play, a mom a other the other. One to happy for the park to the big friends.\n", + "\n", + "\n", + "Step 400, Loss: 3.2237985134124756, Elapsed Time: 26.14 seconds\n", + "Generated text:\n", + "Once upon a timeily. The dog loved the little boymy. The bear was happy and he was very excited. They were a very much it to take a little girl and her friends and went on the little girl and the little boy and she wanted to the little girl was so much. She went to the sun. \n", + "\n", + "\n", + "Step 600, Loss: 2.6897788047790527, Elapsed Time: 16.23 seconds\n", + "Generated text:\n", + "Once upon a time She little there had a little girl named Lily. Lily was very good friends in the park. She wanted to play. \n", + "Her mom asked her mom why the store to buy it was so happy. She ran to her mommy. \n", + "Her mommy asked her friend, \"What's my friend, you can't help me.\" So, please, I want to share.\" \n", + "Lily was so she went to the park.\n", + "\"Look, I am you okay,\" said. I'm so you don't want to do a great idea. Let a little friend, but I do?\" she did you want to go?\" \n", + "The end and said at the park and said, \"I'm glad you want to play together.\" \n", + "\"No, Lily said, the little girl, \"Thank you can't play. You do.\" \n", + "\n", + "\n", + "Step 800, Loss: 2.3438961505889893, Elapsed Time: 16.74 seconds\n", + "Generated text:\n", + "Once upon a time in a walk to a little bird, in the bird named Bob, and went to the pond. The bird was big, was shining big and the fish. Tim and he was happy to have a big fish. They all wanted to help the fish and have fun.\n", + "But it was too big and it. Tim saw a big fish. It made a stick. Tim thought he knew that he should be careful with it. The fish was very happy. Tim decided to be happy and make his friends.\n", + "The next day, the fish went down in a big fish swimming around and they saw a fish in his pond and went inside. Tim thought it could play the fish. It was a very happy. Tim and he wanted to play with the pond.\n", + "\n", + "\n", + "Step 1000, Loss: 2.151026487350464, Elapsed Time: 16.67 seconds\n", + "Generated text:\n", + "Once upon a time upon a boy loved to the beach with his mom and dad. But they would go to the beach to play with it and watch them fly. They would run through the water, but they couldn't find it. But then, a big, a big storm came in. The girl was scared, but they couldn't see what was inside. The mom said that she was going to stay away. But the girl was sad and wanted to help.\n", + "Lily took the boat, but the man was not safe. The man felt sorry for the boy who had a special thing and he did not know that it is not safe and safe. The dad said they were so angry and happy.\n", + "The man and the man was very happy. He said, \"Thank you for showing my new things.\" And they went to the beach, so many more and they found them.\n", + "\n", + "\n", + "Step 1200, Loss: 1.9916144609451294, Elapsed Time: 16.81 seconds\n", + "Generated text:\n", + "Once upon a time there were a clever boy who wanted to help him make his friends and they decided to go on a holiday. They went on their way the subway, and the first. \n", + "But then, the boy saw a train coming out and decided to go inside. He tried to be too, but he couldn't. The train was too small and the bell was scared and didn't want his parents. He was very scared and didn't know what to do. \n", + "The boy asked the train said, \"If you want to take it, you have enough. I can go in my trip.\" \n", + "The driver smiled. The boy was happy! He took the trains to the port and he felt proud of himself. He knew he had done something that he was going to take him to the hospital to stay in the office. The moral of this story is not to be kind and helpful. From then on, it's important to take risks and do it again.\n", + "\n", + "\n", + "Step 1400, Loss: 1.8482240438461304, Elapsed Time: 17.05 seconds\n", + "Generated text:\n", + "Once upon a time there were two sisters. One was Jack and Jane loved to play and play outside. They were having fun together. One day they found an old box and they got inside. \n", + "\"Let's open it,\" Lily said. \n", + "\"Yes,\" Jane said.\n", + "They took the box and started to put on their clothes. They went to get a nice blanket and hugged the bag of toys, and the toys. They said goodbye to the box, but then they got ready to find out. \n", + "Jack and Jane decided to go home.\n", + "The next day, Jane and Jill had a wonderful idea! They were done playing together and couldn't wait to get out. The end.\n", + "\n", + "\n", + "Step 1600, Loss: 1.7912110090255737, Elapsed Time: 16.67 seconds\n", + "Generated text:\n", + "Once upon a time a big little bear and his friends. One day, a little girl named Sally went to the park to play. The big house had been very big and shiny with lots of toys. Sally wanted to play with her toys. She took the big box to play with her friends. Tim was very excited to see all the other toys.\n", + "Tim found some toys. He wanted to play outside. But it was too big for Tim's turn. Sally was scared, but she was still scared. She ran away, but she didn't know what to do. The box was dark to go home and the other things that were too noisy.\n", + "Tim tried to help Tim to fix the lid. He fixed the box and the box was very strong. Sally and Sally were very sad and wished they had fun together. They had found a box where they went. They decided to play with the box and it looked for a while.\n", + "When the box was time for Tim's toy to go home. She was happy to have made the toy for Tim and Tim. She was glad that the toys and helped Tim clean up the box. From that day on, Sally always learned his lesson.\n", + "\n", + "\n", + "Step 1800, Loss: 1.7521343231201172, Elapsed Time: 17.39 seconds\n", + "Generated text:\n", + "Once upon a time there was a very pretty princess who lived in the countryside. She liked to have fun, but she had a very kind heart, like the princesses. One sunny day, the princess wanted to go on a walk so she asked her mom if she could go.\n", + "The princess wanted to go and play, but her mom said no. Her mom took her a photo and went on it to the beach. When she got there, she saw the princess on the way. She wanted to play a game but her mom said no and that she had to have fun.\n", + "But it was not nice to be kind, and it just for her. But the prince didn't like the bossy queen and she said she was sorry. She said she would take care of the people if he wanted to see.\n", + "The princess and the girl had a special photo. When they got home, the princess went on a walk to the beach and she could not reach the sky. The girl was so excited to see her mom.\n", + "The princess had an amazing time to go home and see that her grandma would be together. She said she should have her special photo of her friend to have a special photo.\n", + "\n", + "\n", + "Step 2000, Loss: 1.7332268953323364, Elapsed Time: 17.39 seconds\n", + "Generated text:\n", + "Once upon a time there was a mom and her daughter. The baby loved to play all day and play with her friends. One day they were outside to go outside and play. As they were walking, they saw a big, scary cloud in the sky. The baby had a big hat and a shiny hat.\n", + "The baby wanted to play with the hat. So he ran over to the baby and got it. When he was over, he was so excited. He jumped and shouted, \"That was the wind! You can't go out to the park!\"\n", + "So, the baby saw how it had been to touch, he was too! The baby was having so much fun with his new toy. He played with it all day and had lots of fun. He was so happy he played in the park. And he was so happy to be there forever!\n", + "\n", "\n", - "Step 200, Loss: 4.541538715362549, Elapsed Time: 119.14 seconds\n", + "Step 2200, Loss: 1.652700662612915, Elapsed Time: 16.89 seconds\n", "Generated text:\n", - "Once upon a time was a time was was was was very little girl, day was so happy, her little girl, her little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little little!!!!\n", + "Once upon a time there was a happy dog named Sam. He loved to play with his friends, but he was always too shy to protect it from other ways.\n", + "One day Tim decided to have a party with his friends. He had a lot of fun playing and playing in the garden. He ran around and found a shiny thing. He was so happy that he invited to see what was happening. Sam and his friends were playing and having fun.\n", + "But then, Sam noticed that his friend Sam was hurt and said, \"Sam, it was a bad dog! Are you okay?\"\n", + "Tim looked sad and said, \"Oh no. Let's go find shelter and stay in a tree.\" The two friends started to run and play together. They were so happy to see Sam and the twins, and they became the best of friends.\n", + "At the end of the day, they were happy that Sam was still hurt. They both felt very good, but they never felt so sad that they had lost in the park.\n", "\n", - "Step 400, Loss: 2.8348119258880615, Elapsed Time: 103.58 seconds\n", + "\n", + "Step 2400, Loss: 1.681713581085205, Elapsed Time: 17.17 seconds\n", + "Generated text:\n", + "Once upon a time there were two sisters named Lucy and Bob. Lucy lived in a tall tree and loved to play together. One day, Lucy saw an amazing sight. She decided it would go on the tree. She wanted to explore, so she asked Bob to follow it.\n", + "Bob said, ��Yes! Let��s go!�� So they got to explore the tree. Lucy ran around the tall tree and found a big rock. She was so excited to see it! \n", + "But Bob was determined. He was determined, he could not get there to the rock and see the world below. Lucy quickly found a small hole in the tree was hidden! She wanted to explore! \n", + "The next day, Lucy went to the tree and found the rock. She looked up in the tree. They found a hidden treasure and couldn't find them to explore! They found a hidden treasure hidden secret hiding in the woods. \n", + "They were very proud of themselves for exploring the forest. The end.\n", + "\n", + "\n", + "Step 2600, Loss: 1.6077442169189453, Elapsed Time: 17.09 seconds\n", + "Generated text:\n", + "Once upon a time there were two sisters called Bob and Joe. Joe and Joe were friends and the other sister were playing together. They wanted to explore so much fun and Joe asked, \"Can I have some?\" Joe said, \"I'm a princess, and I'll make you feel a better friend!\" Joe replied, \"Let's make a big mess!\" Joe took a step and Joe started to glue the glue with the glue. Joe was very careful and started to paint all over the floor. Joe's boss made a lot of noise and had lots of fun! When the glue was complete, it made all the noise very wet and Bob got very wet. But Joe said, \"That's ok Joe! I'm so upset!\" Joe smiled and said, \"That's okay, you should not have to share!\" Joe and Joe felt better at all the same time. The two sisters laughed and said, \"That's okay!\" Joe smiled and gave them both of them more fun, and they all enjoyed the rest of the day.\n", + "\n", + "\n", + "Step 2800, Loss: 1.6128735542297363, Elapsed Time: 16.94 seconds\n", "Generated text:\n", - "Once upon a time there was a small cat named Jack. Tim was a little girl who lived in the world and wanted to explore the forest. The dog, he had a new toy car and his mommy and daddy. He wanted to be so much. He went to help the ball. He said to be careful to go and said, \"What's wrong!\"\n", - "After they found the other kids said, Timmy was so excited and he could see his friend. Timmy said, \"Let's play in the other animals, but I want to help.\"\n", - "Timmy smiled and they played with a while. Timmy said, \"It's not listen to play. \"Thank you. \"You are happy.\"\n", + "Once upon a time a girl named Emma lived in a big house. Every day, the sun went down and the house was full of surprises. The backyard were full of things to collect things and find them.\n", + "One day, Emma's mom asked her, \"What is this?\"\n", + "Emma looked around and said, \"I'm a little girl. Can you bring me the toys?\"\n", + "Mama smiled and said, \"Yes, I will bring you some toys and put them in a toy box.\"\n", + "So, Emma and her mom played with the toys, they were so happy they couldn't stop. They ran outside together, playing. They had so many fun!\n", + "Then, they noticed a big pile of toys on the floor. They picked up the toys and put them back on. Emma and her mom were so happy. They played all day together.\n", + "After the last week, they went off on their adventure and found a nice spot in the park. Emma and her mom had a fun day playing together.\n", "\n", "\n", - "Step 600, Loss: 2.3290421962738037, Elapsed Time: 69.78 seconds\n", + "Step 3000, Loss: 1.6015667915344238, Elapsed Time: 17.11 seconds\n", + "Generated text:\n", + "Once upon a time there was a young little girl called Sally. Sally had a very own toy telephone that she loved very much. She liked to pretend to have fun and she would wear it to the store to her mom.\n", + "One day, Sally asked her mom, \"What is that?\" Her mom replied, \"It's a telephone. It's very special to your telephone!\"\n", + "Sally thought of a way out of the store. She wanted to find a telephone that could play with. But her mom said, \"Sally, it won't be able to take your telephone. That is the perfect telephone. Let me use it!\"\n", + "Sally was confused and started to play with the telephone. She had a telephone that could not take it. She asked, \"Why are you going to the telephone?\". The telephone said, \"I found the telephone on it. That telephone is so big and I need it to take the telephone.\"\n", + "Sally thought to herself, and the telephone rang the telephone. She asked, \"How about it for a little telephone?\"\n", + "Her mom smiled. She said, \"Well, it is a perfect telephone for your telephone.\" Sally and her mom went to the store and Sally found the telephone. She was so happy and!!!!\n", + "\n", + "Step 3200, Loss: 1.5756503343582153, Elapsed Time: 17.48 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She was very curious and loved to play outside in the garden. One day, she was playing with her toys and saw a big box with a shiny red ball. The ball was shiny and wanted to climb it, but she didn't want to be too far. \n", - "\"Hi, I can't get my toy!\" said, but Lily said, \"No, you are not yours.\" \n", - "Lily's mom smiled and said, \"You can help me go to me!\"\n", - "\"I want to get some candy and it,\" she said. \"That's a big dog.\"\n", - "The cat said, \"No, we need to go home. I'm going to find some.\" \n", - "Her mom said, \"Okay, you will get to play with me.\"\n", - "Lily smiled and said, \"I want to get my favorite toy first!\" \n", - "Her mom said, \"Okay, but they don't know. They both thought it is so much fun to go on their mom and have fun. They played together, but they did not wait to be friends and they had fun together. \n", + "Once upon a time in the park with lots of animals. One day, a little bunny hopped in the grass and saw the trees. It was the only thing to do was jump so high up the ground and the birds were singing. Suddenly, the bunny felt a sharp pain and he started to cry.\n", + "The squirrel saw a big bird and came to help. The fox was sitting in the grass, feeling the pain in his stomach. The bunny looked up and saw the fox and smiled. He realized that he should have listened to the bird and help.\n", + "The bird told the squirrel that if something looks hard, it can heal again. It should stay and stay away from others, not too much that might be hurt. The bunny was happy and he thanked the squirrel and flew away.\n", + "The End.\n", "\n", "\n", - "Step 800, Loss: 2.049470901489258, Elapsed Time: 84.69 seconds\n", + "Step 3400, Loss: 1.5584828853607178, Elapsed Time: 16.71 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lucy was playing in the garden. She loved to run and play outside and she had lots of fun. She would always look at all day and her friends in the garden.\n", - "One day, she was playing outside when she heard a strange noise coming from a loud noise. It was coming from a dark and it started to shake. It was a big dog, so scared.\n", - "Her mom came into the room and said, \"That's not safe!\"\n", - "The dog smiled and said, \"Don't worry, I will find my friend.\" Lucy and the dog started to walk in the grass, but soon she heard the sound coming from the garden, and she heard a noise. The dog ran back inside, but the door opened and saw that the shadow had a hole in a hole. Lucy and the dog became friends, and they played together all the things and they never found a beautiful spot to get home.\n", + "Once upon a time there were a young boy who lived in a small town. He wanted to have a lot of fun, but he wanted to find some. So he went to an event.\n", + "The boy was very excited and he went to the park to play. But he was also hungry and hungry. He decided to eat some food. He ate some bread and it was sweet.\n", + "After the fruit was done, the boy went home and found a big piece of cheese. He was so excited to eat it. He picked it up and ran back to his mom with it.\n", + "The little boy ate the cheese and smiled and felt full and happy. He said, \"Mom, I want my cheese!\" He was so happy and he was happy to have such a good friend.\n", "\n", "\n", - "Step 1000, Loss: 1.8924535512924194, Elapsed Time: 76.64 seconds\n", + "Step 3600, Loss: 1.5556925535202026, Elapsed Time: 16.60 seconds\n", "Generated text:\n", - "Once upon a time, there was a girl named Amy. Amy loved to go outside and play with her friends. One day, Amy saw a big tree and asked her friends, \"Can I have some best friends?\" The tree was so big that it made a special tree to get in the tree. \n", - "Amy went to the tree and asked, \"Can I go on my tree?\" The tree replied, \"Sure, let's play together!\" \n", - "So, Amy and the tree were very excited. Amy started to play, but it was too heavy for the tree. She started to climb and forth, but it was so far away. Amy's mom saw how fast it was. She asked, \"Why do you think this, but I want to get to be my friend?\" Amy said, \"No, it's too late, so we can't play together!\" Amy's friend started to fight and said, \"I'll be friends. Let's play together!\" Amy and Amy laughed, and laughed together, and they became best friends and they had a fun day together.\n", + "Once upon a time a small dog liked to play. One day, he found a pretty dress on the ground. He wanted to wear the pretty clothes and look for the prettiest flowers in the park.\n", + "He put the dress in a big basket. Then he put the pretty flowers in the basket, and the flowers. Then, he went to the park and played and had lots of fun.\n", + "Finally, the flower found the pretty flowers. They put the pretty flowers on the ground and the flowers. The nice lady put the flowers in the flower's basket. The colors the flowers were nice, but they all had fun. And the lady gave the pretty flowers in the garden. They played with it all day, and the pretty flowers.\n", "\n", "\n", - "Step 1200, Loss: 1.7993096113204956, Elapsed Time: 80.74 seconds\n", + "Step 3800, Loss: 1.511893391609192, Elapsed Time: 16.69 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Amy. She loved to go outside and play in the sunshine. One day, she went outside to play with her friends. They went on a sunny day and found a shiny ball. \n", - "Amy wanted to see the ball and it started to fly it. But then, a little girl came to her friend. Her friend saw her and asked if she could join the bird. The bird said yes and Lily took it to her house.\n", - "Amy was very happy and she started to sing and dance with her friend. They sang together and danced until the sun came up. They laughed and danced together, laughing and having fun. They played together all day and the day long.\n", + "Once upon a time there was a nice dog named Max. He liked to play in his garden, but he was very tired. One day, while playing outside, he saw a cat in the yard. The cat was very hungry. \n", + "Max asked his friend, \"What's wrong?\" The cat said, \"I can't find food. I need to eat me.\" So Max went to find another place to find food. \n", + "The cat found a big tree and sat down. Max found a tree. He thought, \"I will come and eat some food and I will eat the meal.\" Then, he saw some leaves. \n", + "He picked up one of his friends's toys and shared them with him. They all had a fun day together. Max and the cat became best friends, and they all enjoyed playing together. They were so happy to have such a yummy friend.\n", "\n", "\n", - "Step 1400, Loss: 1.7445931434631348, Elapsed Time: 69.74 seconds\n", + "Step 4000, Loss: 1.5087201595306396, Elapsed Time: 16.98 seconds\n", "Generated text:\n", - "Once upon a time, there was a girl named Sue. She was only three years old. Sue loved to explore. One day, she saw a little bird sitting on a branch. Sue was scared. She ran up to her mom and said, \"Mom, what's that noise?\" Her mom smiled and said, \"It's okay, Sue. We don't have any more fun!\"\n", - "Sue and her mom looked around the house and found a small bird that made her feel better. It was a little bird. Sue and her mom went outside and played in the sun. They had a great time playing and laughing.\n", - "At the end of the day, Sue said, \"Thank you, Mommy! You're welcome! You are very lucky to be a friend.\" Her mom smiled and said, \"Thank you, mommy. I love your new friend!\"\n", + "Once upon a time there lived a loyal family. One day they went on an adventure, so they decided to go on an adventure. As they packed along their journey, they heard voices. \n", + "Suddenly, they heard a loud noise! They were so excited to see what it was, so they rushed to investigate. They looked around and saw a big, friendly fox sitting on an acorn. The rabbit asked the bird, \"Where are you going?\" \n", + "The fox said, \"I am going on a journey to the forest.\" \n", + "The family looked up in surprise, they were walking in the forest. The rabbit pointed to some strange-looking trees on the path, then asked the bunny, \"Where are you going?\" \n", + "The fox said, \"I'm going on my adventures. I don't know where to go!\"\n", + "They continued on their journey. As they walked, they saw more things that they saw and they found even more amazing creatures! The animals were amazed and excited. They were all amazed by how loyal and happy they were so loyal to them. \n", + "From then on, whenever they saw the animals exploring, they knew that being brave could always be helpful.\n", "\n", "\n", - "Step 1600, Loss: 1.6877646446228027, Elapsed Time: 74.16 seconds\n", + "Step 4200, Loss: 1.4979870319366455, Elapsed Time: 17.19 seconds\n", "Generated text:\n", - "Once upon a time, a boy named Jack was walking down the street. He was feeling very scared. His mom told him that he had to go outside and get to play. \n", - "One day, Jack noticed a little boy playing in the park. He wanted to play with the equipment too, but he was scared. \n", - "Jack ran over and asked, \"Why did you get me?\" His mom said, \"I don't want to be scared. I'm sorry I can't have the equipment to take the equipment to play.\" \n", - "Jack thought about the equipment he would make it even more fun with the equipment. So, he started to build the equipment. When he was finished, Jack felt a bit better. \n", - "The equipment was a way to the playground, but he knew that his mom was not alone, she had to help him get better. She asked him to stay in the playground, but Jack was still happy to have the equipment to stay, but Jack couldn't.\n", - "Jack learned that being kind was important to help and not be afraid of being ignorant and selfish. He was still happy, but he was still very selfish and always asked the equipment for help.\n", + "Once upon a time there were two little kids who liked to play. They would run around the town and have fun. One day, the two boys came up to the town. One of the boys was playing a game and they both laughed. \n", + "Then, the two friends decided to play a game. They pretended to fight over the game and fight over. They ran around trying to fight over a team and won a game. They had so much fun playing together.\n", + "After playing, the other kids played and had fun. They were so tired and happy. The group thought they were being competitive and having fun. So the kids decided they could do something fun they could make fun again.\n", + "But then, a big storm came and the two kids got scared. The kids ran away and had lots of fun with playing. The friends learned that sometimes the best joke can be hard to fight and have fun. That way, they can make a new friend!\n", "\n", "\n", - "Step 1800, Loss: 1.6456927061080933, Elapsed Time: 85.76 seconds\n", + "Step 4400, Loss: 1.491351842880249, Elapsed Time: 17.04 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Mia. Mia loved to play outside in the park. One day, Mia found a shiny coin in the park. She wanted it for her birthday party, but it was too expensive. Mia thought it would not buy the coin for her birthday.\n", - "So, Mia decided to ask her mom for the coin to buy the coin. The coin looked for a long time and said yes. Mia put the coin back in the park, but it was gone. Mia was very sad.\n", - "So, Mia asked her mom if she could have a special day. Her mom said yes and they both went home with the coin. Mia was so happy! She said yes, but the coin was gone forever.\n", + "Once upon a time there was a kind boy named Tom. He had a special toy - a ball. Tom was very excited. He had never made something with it.\n", + "Tom ran and played with his ball. He kicked it and threw it hard. But the ball went everywhere.\n", + "One day Tom was playing with his ball when he saw a big, mean dragon with a fierce grow. The dragon roared at Tom, and he wanted to catch it. The dragon was so angry and flew away!\n", + "Tom's mom saw the dragon and said sorry. She said that sometimes the ball was dangerous to get lost and that the bear didn't bite them.\n", + "Tom thought it was very silly and he started to cry. He wished he had listened to his mom and stayed with the ball. The next time the dragon flew out of the big dragon and was happy. The moral of the story is that being kind and gentle, even if it's not scary, you should try to get help too.\n", "\n", "\n", - "Step 2000, Loss: 1.6211789846420288, Elapsed Time: 69.42 seconds\n", + "Step 4600, Loss: 1.516611933708191, Elapsed Time: 17.07 seconds\n", + "Generated text:\n", + "Once upon a time there were two twins named Sam and Joe. They liked to play with their toy cars and trucks.\n", + "One day Sam had an idea to make a model car! Joe was so excited! He said to Joe, ��Let's go see what happens!��\n", + "Joe looked at the model car and said, ��No Joe, we are not careful. It��s a model car.��\n", + "Joe replied, ��But don��t be right!��\n", + "Joe was sad and said to Joe, ��Okay. I will do this! Let��s see that we are safe and we can take care of it,�� They went to the store and found a model car.\n", + "Joe was very happy and said, ��Let��s go see the model car, it��\n", + "Joe agreed, and they drove the model. The model car drove to the model car, and Sam couldn��t believe it. Joe said, ��We are very careful, it looks like a model car.�� \n", + "They drove their car, driving it down the street, and the car driver saw the model car. Joe and Joe were so happy. They hugged the model car!!!!\n", + "\n", + "Step 4800, Loss: 1.5036702156066895, Elapsed Time: 17.51 seconds\n", "Generated text:\n", - "Once upon a time, there was a small boy named Tim. Tim was a small boy named Timmy. Timmy loved to play with his toy car and he loved to make noises with it.\n", - "One day, Timmy and his toy car went to the park with his toy car. Timmy's car got stuck in the car and it crashed. Timmy's car was too fast and it hit a tree with the wheel.\n", - "Timmy was sad because he loved his car so much. He knew he could help make his toy car better. He said, \"I will make you happy! Let's play together and make a new car.\" And Timmy and his toy car had fun together and they became good friends.\n", + "Once upon a time there was a happy girl called Jane. She was a very independent three year old. One day she wanted to go explore her home but didn't know how. Her mum told her, \"If you don't go\".\n", + "But Jane wanted to go explore. So, she decided to go on a journey. As she got closer, she noticed something was wrong.\n", + "She started to regret taking a look. It was a big surprise! She had never seen anything like it before, but it was a bit scared.\n", + "Jane quickly got closer and said goodbye to her mum and promised she wouldn't go back again and she continued to go on her own journey, and never came back and she never had to go on her journey again.\n", "\n", "\n", - "Step 2200, Loss: 1.578463077545166, Elapsed Time: 68.18 seconds\n", + "Step 5000, Loss: 1.473569631576538, Elapsed Time: 16.73 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lucy. She was only three years old and loved to play with her toys. One day, she found a special treasure, a big, scary toy that was very big and she was so excited to take it home. \n", - "But, one day, Lucy found a shiny jewel that she was very happy to find it. She opened the door and saw a beautiful necklace in the corner of the room. She wanted to see what it was like. She asked her friend, the necklace, and the necklace said, \"Can I take it home now?\" \n", - "Lucy smiled and said, \"I can borrow it with a special diamond. It's very special because it belongs to someone else.\"\n", - "The diamond was so happy to see Lucy and said, \"Thank you, Lucy. You are a great friend!\"\n", + "Once upon a time there were two good friends. One day, they wanted to go exploring in their backyard. They went outside and saw an incredible flower in a field. They both stared at it at the pretty flower. They were so pretty! \n", + "Then, they decided to go explore it. But they saw that it was not a place to go. They argued on the next door and argued for awhile. \n", + "Suddenly, the two friends decided to go to the incredible flower and they started walking. The flower was so fast that it was scared, but the other hand kept going. Eventually the two friends stopped and went to get a big bag of candy. After the first friend left, they finally found the most incredible flower. \n", + "The two friends had a wonderful time exploring the enormous flower they had seen in all of the incredible things they wanted!\n", "\n", "\n", - "Step 2400, Loss: 1.5326989889144897, Elapsed Time: 73.51 seconds\n", + "Step 5200, Loss: 1.4963871240615845, Elapsed Time: 16.94 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play outside in the sun. One day, Timmy saw a beautiful flower. It was so pretty!\n", - "Timmy asked his mom, \"What is this flower?\"\n", - "His mom replied, \"It's a flower. Do you want to play with it?\"\n", - "Timmy nodded his head and said, \"Yes, please.\"\n", - "Timmy was so happy and he played with the flower. They laughed and played until the sun went down.\n", - "Suddenly, they heard a loud noise. It was a little kitten. The kitten was scared and didn't know what to do. Timmy's mom told him it was a bad ending.\n", + "Once upon a time there was a fat bird. He wanted to go for a swim like a friend. So he decided to go for a swim and find something to eat.\n", + "They went to a pond and started to swim. They had lots of fun. But then, the fat bird got tired, so he said to his friends, \"I want to go now!\". His friends were very sad and so they all agreed. They swam and swam until they reached the surface.\n", + "The fat bird was so happy. He said, \"That was so much fun swimming in the water! Now let's go home!\" He had a big swim in the pond too. The fat bird said, ��I can swim again and again!�� And they swam together.\n", "\n", "\n", - "Step 2600, Loss: 1.5533288717269897, Elapsed Time: 70.30 seconds\n", + "Step 5400, Loss: 1.4531868696212769, Elapsed Time: 16.62 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She was very happy because she had a pretty blouse. One day, Lily was playing with her friend Jack came over to play. She asked, \"Can I play with you?\" \n", - "Jack said, \"Sure, let's play with your friends!\" Lily was excited and said, \"Okay, let's play!\" They ran around the room and played with their new friends. \n", - "As they were playing, Lily accidentally knocked over the blouse and it fell off the blouse. Jack said, \"Ouch!\" \n", - "Lily felt embarrassed and said, \"It's okay, Lily. I'm sorry. I didn't mean to break the blouse. Let's play with it again!\" \n", - "They both started to play and soon the blouse was over. They played together and had a lot of fun. Lily felt so happy that she didn't make her friend laugh again. She realized that sometimes things don't seem too bad and we should be different, but they still have a solution.\n", + "Once upon a time a small mouse, in a small village, there was a small hole in a tree. The hole was very big, and he knew where to find something to bury. He walked around the forest, looking for something. Finally, he found a big pile of leaves and he dug under it. \n", + "The mouse dug and dug, until he found that the tree was buried. The tiny hole was not a big hole in the ground. The small hole was full of leaves and rocks. He looked inside and saw a hole in it. \n", + "He carefully placed the hole into it and then buried it. It felt so cold, so he decided it was winter. After that, a few days, he was very happy. He had found something to buried in the hole. From then on, the big hole was buried underground and it would be perfect to bury things inside.\n", "\n", "\n", - "Step 2800, Loss: 1.5323652029037476, Elapsed Time: 80.58 seconds\n", + "Step 5600, Loss: 1.434389352798462, Elapsed Time: 16.99 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Tim. Tim was a good boy. He liked to play with his toys and his dog, Max. One day, Tim went to the park with his mom.\n", - "\"Hi, Tim! I want to play with you. Do you want to play with me?\" Tim asked.\n", - "\"Yes, Tim. Let's play together!\" his mom said.\n", - "Tim and Max played together all day. They laughed and had lots of fun. Tim and Max were good friends.\n", - "\"Can we play with our friends?\" Max asked.\n", - "\"Yes, but we can play together,\" his mom said.\n", - "Tim and Max played together. They laughed and had fun. Tim and Max were happy to play with them. They were best friends.\n", + "Once upon a time there was a famous bird called Jack. Jack loved to fly high and he had lots of friends. He was so big and happy when the sun started to set him down. \n", + "One day, Jack was flying so high that he almost fell down to the ground. He was too scared to get down and he tried to get down. But the water was still so hard.\n", + "But Jack was brave and he was determined to do it. He flew down to a nearby tree, and it started to get really hot.\n", + "As Jack was walking around his garden, he suddenly spotted something he hadn't been seen earlier.\n", + "He quickly flew down carefully and noticed it had a big yellow yellow yellow balloon. He quickly grabbed it and ran back to his spot. \n", + "The end.\n", "\n", "\n", - "Step 3000, Loss: 1.5298235416412354, Elapsed Time: 70.76 seconds\n", + "Step 5800, Loss: 1.4726951122283936, Elapsed Time: 16.62 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She was three years old and loved to explore the world around her. One day, Lily found a shiny jewel on the ground. She picked it up and looked inside the jewel. She picked it up and examined it. She showed it to her mom and they were so proud of her. \n", - "After she was finished playing, Lily went to bed. She had so much fun that she didn't realize it was her mom had made a mistake. Her mom explained that sometimes it's important to be responsible and to listen to others. Lily was very careful and listened to her mom's advice.\n", - "From that day on, Lily was careful and she never forgot about the jewel. She always made sure to be careful when she played. And always remember to always be careful when things happened.\n", + "Once upon a time there was a family that was very excited to buy her something special, and that she was going to get a special present for everyone who could buy her.\n", + "Everyday she went on a trip to the beach and her parents packed a big bag with some yummy food and a little girl. When she got there, she saw a big building. Inside the room was a big box, but she could not find it.\n", + "When she opened the box, she found a beautiful surprise. There was a big box with a picture on it. She opened up the box and it made her smile. She opened it to see her family.\n", + "Inside her family had a very beautiful smile that she had bought. She was so happy and excited to show them all the things she wanted and family.\n", + "When it was time to go home, everyone said it was time for bed. They hugged and said their lucky to have a special surprise and it was the happiest family of the things they had ever tasted.\n", "\n", "\n", - "Step 3200, Loss: 1.471197485923767, Elapsed Time: 72.75 seconds\n", + "Step 6000, Loss: 1.4523481130599976, Elapsed Time: 17.12 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Tim. Tim was a curious little boy who loved to play. One day, he found a toy car in the garden. It was red and shiny. Tim was very happy and excited.\n", - "As Tim went outside to play, he saw a little bird in the grass. The bird had a red car. Tim wanted to help the bird. He said, \"Hello, little bird. Can you help me lift my toy?\" The bird looked at Tim and said, \"Yes, please! It is so nice to be careful. You are smart and smart.\"\n", - "Tim was very happy to help the bird. He thanked the bird and the bird flew away. Tim learned that helping others can make us feel good and smart. He learned that helping others can make us happy too.\n", + "Once upon a time there was a young boy named Tom. Tom loved to play in his garden with his friends. One day, Tom wanted to build something very special. He found two big blocks, one block. He wanted to build the most. So, he asked his friend, a bird. His friend said, \"Let's build a tower!\"\n", + "Tom was really excited, but he was too hungry. He was sad, but he knew he had to do something else. So, he said, \"Please, can I have some more things?\" His friend smiled.\n", + "Tom thought of something, maybe a way to help make the best of his own tower and he made a great tower. It was a big, red car that was perfect for his own tower! Tom was so happy to be able to build it. He thanked his friend, the bird for making a beautiful tower.\n", "\n", "\n", - "Step 3400, Loss: 1.5036591291427612, Elapsed Time: 71.77 seconds\n", + "Step 6200, Loss: 1.4724220037460327, Elapsed Time: 16.73 seconds\n", "Generated text:\n", - "Once upon a time there was a boy named Tim. He loved to go outside and play. One day, he saw a big tree with lots of branches. He wanted to climb it, but his mom said no.\n", - "Tim tried to climb the tree, but it was too high. He was getting higher and higher until he could see what was on the top. Then he heard a loud noise coming from outside. He peeked out from the ground and saw a small bird flying above.\n", - "Tim was so happy to be able to reach the top. He jumped and jumped up and down the tree with excitement. When he was finally caught the bird, it flew down to the tree. Tim was so proud of his success.\n", - "He thanked the bird and continued climbing. He was very happy that he had climbed the tree, even though he couldn't get down from the tree. From then on, he made sure to stay away from the top of the tree, safe and sound.\n", + "Once upon a time there were two birds named Bob. Bob was very happy. He flew high into the sky and he loved to fly in the sunshine. He was always so fast with the sunshine. \n", + "One day, Bob saw something strange. The ground was big and round and green. Bob said, \"Oh, what are you doing?\" The birds replied, \"I am looking for something to keep it safe and safe. Let's go to the park and play together!\"\n", + "Bob was excited and he followed the birds and soon enough, all the birds were flying above. Bob was very surprised by what they saw. Bob and the birds had so much fun in the sunshine. The birds were having so much fun that Bob had made a big difference. \n", + "The birds were so impressed that they decided to leave it for a little while. The moral of this story is to be careful and keep being safe for you.\n", "\n", "\n", - "Step 3600, Loss: 1.4788475036621094, Elapsed Time: 76.91 seconds\n", + "Step 6400, Loss: 1.4274860620498657, Elapsed Time: 16.96 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play with his toys and watch cartoons. One day, Timmy's mommy asked him to help his mommy. Timmy was excited to help, so he got a new toy from his grandma's house. \n", - "After a few days, Timmy's mommy gave him a special gift. Timmy was so excited to receive such a gift! It was a special gift that his grandma gave him a big hug. Timmy was so happy to have such a gift and hugged his mommy. \n", - "After that, Timmy went to bed that night with a special gift from his grandma. He loved his present so much that he gave it a special gift to his grandma. Timmy was so happy to have a present that he had found such a beautiful gift. \n", - "The next day, Timmy's grandma gave him a gift and gave him a gift. Timmy was so excited and couldn't wait to tell his grandma. He showed his grandma a gift for him with a present, and they all said thank you to his grandma. Timmy was so happy to receive the gift from her grandma and her gift. \n", - "The present was that Timmy had!!!!\n", + "Once upon a time there were two good friends. They loved to play together and have a lot of fun. One day they decided to go for a ride on a special trip. But the weather was not nice, and it was raining and they didn't like it. The other friend started to complain again and it didn't like that. But then the other friend suggested to the ride. So they decided that the weather is better than they could play outside. The other friend got out a big red balloon and started to play. They laughed and played games until the sun went down. The end.\n", "\n", - "Step 3800, Loss: 1.4588173627853394, Elapsed Time: 88.57 seconds\n", + "\n", + "Step 6600, Loss: 1.432798147201538, Elapsed Time: 16.49 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play with his toys, but he always wanted to share them with others. One day, Timmy was playing with his toy cars and accidentally broke them. His mom said, \"Don't worry, I will help you fix your toys. Let's make it together.\"\n", - "So, Timmy went to his room and started to play with his toy cars. He had so much fun playing with it, he didn't even want to lose it. His mom was happy to help, so they took a break and put the toy cars away in the right place. Timmy was very happy that he could share his toys with his mom.\n", + "Once upon a time there was a nice dog named Max. Max lived near a big building. The building was very tall and had lots of friends to sit by the building.\n", + "On the building, Max saw a big building. He asked his friends if they wanted to play. They said yes and the building was a very special building. Max was so excited to play on the elevator.\n", + "When the door rang, the doorbell rang. Max and his friends got to work. Max was very scared. He wanted to go inside the elevator. The elevator rang and the sound of the music was ringing. \n", + "Max and his friends went on a trip. They took turns holding hands in the elevator door. They saw a big sign that said \"Welcome to the building!\" Max and his friends played a game. The lights were so loud and they laughed and had fun. \n", + "When the end of the elevator started to go away, Max and his friends had a big fight. They had so much fun together. Max said goodbye to the elevator and they left the building. Max and his friends walked away with their new toy.\n", "\n", "\n", - "Step 4000, Loss: 1.4644711017608643, Elapsed Time: 69.17 seconds\n", + "Step 6800, Loss: 1.4220349788665771, Elapsed Time: 17.27 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play in the garden, and even when she saw a beautiful butterfly. She ran over to the butterfly and tried to catch it, but it flew away. \n", - "Suddenly, a little girl appeared. She looked around and saw her and asked her what she was doing. Lily replied, \"I want to play too, please!\" Her mom replied, \"Okay, but only if you don't like the butterfly, it's better.\" \n", - "So, Lily decided to go back to the garden to find the butterfly. She walked over to the garden and saw a big, beautiful butterfly. The butterfly flapped its wings and flew away. Lily felt so happy that she ran back to the garden, happy that she could play with the butterfly again.\n", + "Once upon a time there was a nice boy named Joe. He loved to go to his special house and explore. One day, he saw something in the grass. It was a tiny box of toys! Joe was so excited! He had never seen this toy before.\n", + "Joe asked his mom, \"What is this?\"\n", + "His mom smiled and said, \"It sounds like so much fun!\"\n", + "After that, Joe and his mom went to find the toy. They looked in the toy box, but it was nowhere to be found. Joe thought and thought, and so much about what his plan was going to be fun and adventure. He said to his mom that he should have listened to his suggestion and not take it.\n", + "So Joe and his mom went to the toy store and bought a big toy. The toy was so happy! Joe thanked the toy, and the toy that it was all gone. His friends were so excited to see what his toy was missing, and they all had a fun time playing with the toy. \n", + "Joe had an incredible day and he was happy that it had helped make his family happy. From that day on, he was never forget to call his friends when they went to find his toy.\n", "\n", "\n", - "Step 4200, Loss: 1.4640543460845947, Elapsed Time: 72.24 seconds\n", + "Step 7000, Loss: 1.4282193183898926, Elapsed Time: 17.43 seconds\n", "Generated text:\n", - "Once upon a time, there was a boy named Timmy. Timmy loved to play with his toy cars, trucks and even the cars. One day, Timmy's toy car got stuck in a big mud. Timmy tried to get the wheels, but he was too heavy. He tried to push the wheel but it was too heavy. Timmy tried and tried, but he couldn't do it. Timmy was sad, but then he remembered how his mom told him. He was brave and strong, but he knew how to use his car to get it. He was happy again, and he was able to get the wheel.\n", + "Once upon a time there was a happy girl named Sarah. She was only three years old and liked to play with her toys. One day Sarah and her friends were playing in the sand and they decided to make a sandcastle. They started to build and make a big castle with lots of sand and sticks.\n", + "The castle was so tall and strong that it made Sarah happy. But then something terrible happened, and Sarah noticed that it was getting bigger and bigger. She was sad and didn't know what to do.\n", + "Suddenly, Sarah had an idea. She remembered how strong the castle was, but it didn't work. So they both stood on top of the big castle, and together they built walls and towers and castles.\n", + "Sarah and her friends had so much fun playing together. They laughed and had so much fun. They were very happy, and they always made a special castle together.\n", "\n", "\n", - "Step 4400, Loss: 1.4368994235992432, Elapsed Time: 66.23 seconds\n", + "Step 7200, Loss: 1.4444782733917236, Elapsed Time: 16.91 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play outside and pick flowers. One day, he saw a little boy who was very curious. He went up to him and asked, \"Can I help you?\" The boy replied, \"Sure, you can help me find your way.\"\n", - "Timmy and the boy searched everywhere but they couldn't find the way up to the boy. They searched everywhere but they couldn't find the answer. Suddenly, the boy said, \"Don't worry, I'll help you find your way.\" Timmy said, \"I know how to solve this problem.\"\n", - "Timmy's face lit up with excitement and said, \"I'll help you solve this problem. I can find your way home.\" His mom said, \"That's right. Let's find your way and see if we can find a way to find my way back.\" Timmy and the boy looked at each other's hand and said, \"Yes, I'll help you.\"\n", - "They found a small tree, and Timmy helped his mom get his help. They worked together to search for the little boy's journey. After a few hours, they finally found the perfect path. Timmy was so happy that he could!!!!\n", + "Once upon a time there were two kids playing in the forest. One was very popular. They loved to play together and run around. But one, it didn't work. The little boy was very sad.\n", + "One day, the big boy asked if it was okay. He said he was sorry because it was broken. He said he could help. So they started the big tree together.\n", + "The tall boy was not happy. He said he would help the big boy fix the tree. The tall tree said yes. The two kids were very kind. They played together, they had fun and it was better.\n", + "\n", "\n", - "Step 4600, Loss: 1.4162362813949585, Elapsed Time: 88.24 seconds\n", + "Step 7400, Loss: 1.4314332008361816, Elapsed Time: 16.51 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play outside and look at the pretty flowers. One day, Timmy saw a beautiful flower. It was so pretty and he wanted to touch it. \n", - "But when he got close to the flower, he accidentally dropped it. It broke! Timmy was so sad that he cried and his mom couldn't fix it. She said they would get another special thing, but Timmy was okay. \n", - "His mom hugged him and said it was okay. She told him that accidents happen and that it's important to be careful when you touch things that can happen.\n", + "Once upon a time there was a young boy. He had a special tool. He had a toy car and he was very happy. One day the boy saw a big, red bird flying around. The boy wanted to touch the bird but he was scared. He tried to open the tool, but the bird flew away.\n", + "The boy was sad. He asked his parents if they could help him. So, they went to find a tool. The boy found a tool that could not open the tool. He thought it was too dangerous.\n", + "After searching for a few hours, the boy looked up and smiled. He said that this tool will come down the tool with him, but if he did open the tool, he could open it. The boy was so happy and he thanked his parents. The boy was no longer scared and he felt very sad.\n", + "The end.\n", "\n", "\n", - "Step 4800, Loss: 1.4360578060150146, Elapsed Time: 66.49 seconds\n", + "Step 7600, Loss: 1.3909465074539185, Elapsed Time: 17.00 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside in the park. One day, she saw a big tree and wanted to climb it. But her mom told her to be careful because the tree might hurt them.\n", - "Lily tried to climb the tree, but she was too weak. She fell down and started to cry. Her mom hugged her and told her to go back home. They were very happy and went back to the tree.\n", + "Once upon a time there was a mom and a baby. The baby had a big heart and always wanted to help.\n", + "One day the mom took baby out to the park. As soon as they walked, the baby saw a big rock. It was a big, heavy rock. The mom said that the baby could help her.\n", + "The baby was so happy to receive the toy. The mom was pleased. She thanked the baby for being so helpful.\n", + "The baby felt so happy that the rock was so thankful. She smiled and hugged the bear.\n", + "The baby had done a good deed with the toy. The mom hugged it and kissed the baby. The hug was glad the little baby was very happy.\n", "\n", "\n", - "Step 5000, Loss: 1.4377806186676025, Elapsed Time: 59.31 seconds\n", + "Step 7800, Loss: 1.414798378944397, Elapsed Time: 16.53 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside and collect things. One day, she went to the park with her mom. She saw a boy sitting at the bench and he said, \"Hello, doggy!\" Lily felt embarrassed because she didn't like it. She asked the boy, \"What's wrong?\" The boy said, \"My dog's dog bit me!\" Lily said, \"It's too bad. I'm too small.\" The boy felt sad for her.\n", - "Lily said, \"I'm sorry, but I don't know it's not good.\" She went to her mom and asked if she could borrow some of her toys. Her mom said, \"Sure, you can borrow your toy car. You can borrow it, but be careful.\" Lily smiled and went back to playing with her toy car. She had a lot of fun with her toy car and played with it all day.\n", + "Once upon a time there was a young bird who lived in a big forest. One day, when the bird was flying around, he heard a sound that scared the birds. He tried to find out where the birds were singing, but it was too high for him to reach.\n", + "The bird was stuck in the tree and started to get very scared. He was about and he wished he had never been in a tree for a moment. Suddenly, the bird heard a noise. He stopped and looked up. He saw a big bird in the tree branch and he was so relieved!\n", + "The bird felt relieved. He flew back up, and started to fly again. With a little bit of courage, they flew away, and the bird was safe. From then on, he was always careful and to look for danger to fly.\n", "\n", "\n", - "Step 5200, Loss: 1.3954946994781494, Elapsed Time: 76.86 seconds\n", + "Step 8000, Loss: 1.395037055015564, Elapsed Time: 16.69 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside and pick flowers. One day, she saw a beautiful flower in the garden. She picked it up and held it in her hand. She felt so happy that she wanted to show her mom.\n", - "Her mom came outside and saw that Lily was very sad. She told Lily that the flower was still a flower. Lily felt sorry for being so pretty. She promised to be more careful with the flower.\n", - "The next day, Lily went back outside to pick some flowers. She found some pretty flowers and showed them to her mom. Her mom was very happy and said that Lily's flower was not as pretty as the flowers in the garden. From that day on, Lily loved to play and make pretty flowers in the garden.\n", + "Once upon a time there were two cats called Jack and Jane. They liked to play and have lots of fun. One day, Jack had an argument in two. Jane wanted to share a secret room and Jack was very excited.\n", + "She went outside to the park and played in the sun. Jack's mom said that if Jane tried to take a step, she could feel the cold breeze on her face and they both got very excited.\n", + "But then, something strange happened. Jack started to get angry and shouted. Jane was so scared and she yelled.\n", + "\"It's too late! Let go to my mom's house for a walk\". She grabbed Jack's paw and she walked for a long, but she didn't give up.\n", + "Finally, Jack and Jane stopped and went home. But Jill was still feeling angry and she started to cry. \n", + "When they got there, Jack and Jane had to look back at the end of the conflict and she had made it!\n", "\n", "\n", - "Step 5400, Loss: 1.401665449142456, Elapsed Time: 70.89 seconds\n", + "Step 8200, Loss: 1.3898961544036865, Elapsed Time: 17.10 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play outside and explore. One day, Timmy saw a big tree with a lot of branches. He thought it looked interesting and decided to climb the tree. \n", - "Timmy's friend, a wise old owl, said, \"Timmy, don't be careless. I don't want to fall. You need to get hurt.\" \n", - "Timmy didn't understand why the owl was so wise, so he explained to his friends, \"It's important to always stay safe.\" \n", - "Timmy felt better and went back to his tree to admire the branches from branch. He knew that even if something is too small, it's always better to be safe and be careful when climbing tree.\n", + "Once upon a time in a wide tree far away. Inside the tree was a big house. One day, a little boy named Tim came to visit.\n", + "\"Hi, Timmy! What's your name?\" asked the door. Timmy looked at the door and saw a big box. He asked the box to open.\n", + "The box opened and out came a note. Inside the box was a present. It was a present! Timmy opened the box and inside there was a note. He was very happy.\n", + "The note said, \"Thank you for your special present. I will deliver you some presents to my home so you are ready.\" Timmy hugged the box and opened the box for the present.\n", + "The note said, \"Thank you for delivering my presents. Now, can I come back and give you a present to deliver some gifts to your home?\" \n", + "Timmy smiled and said he had a great day at the wide tree.\n", "\n", "\n", - "Step 5600, Loss: 1.3960157632827759, Elapsed Time: 71.91 seconds\n", + "Step 8400, Loss: 1.4092108011245728, Elapsed Time: 17.11 seconds\n", "Generated text:\n", - "Once upon a time, a little girl named Lily went on a trip with her mommy. They were having so much fun. When they got to the beach, they saw a crab and wanted to touch it.\n", - "Lily said, \"Mommy, can I touch the crab?\"\n", - "\"No, it's too dangerous. You should stay in the sand. The crab might be safe,\" her mommy replied.\n", - "Lily was sad and started to cry. \"I'm sorry, mommy. I didn't want to be hurt.\"\n", - "Her mommy said, \"It's okay, sweetie. I'm glad you listened to your mommy and not touched the crab. Now you can't touch the crab's safety.\"\n", - "Lily smiled and hugged her mommy. From that day on, they became good friends and they played together every day.\n", + "Once upon a time there was a naughty fox. The fox was very mean and wanted to take away all of his friends. One day the fox asked, \"Why do you want this mean?\" The fox thought for a moment and said, \"I can do it!\" The fox thought about it and he said, \"No way, I want to take my friends with it.\" So the fox and the other animals went out to look for the fox.\n", + "The fox was happy to find the fox, but the fox was not nice. The fox started to cry and the rabbit ran away. But it was too late. The fox's friends tried to take the fox with them but the fox's meanness had gone away.\n", + "The fox was very sad and said, \"I'm sorry for you.\" The fox felt so bad that he ran away with his friends and the fox said, \"I'm sorry I never do that again. I should never have a fight like this fox.\"\n", + "The moral of the story is that it's always good to you can make you happy.\n", "\n", "\n", - "Step 5800, Loss: 1.3838456869125366, Elapsed Time: 73.78 seconds\n", + "Step 8600, Loss: 1.4134347438812256, Elapsed Time: 17.19 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Tim. Tim had a big toy car. The car could go very fast. Tim liked to race with his car.\n", - "One day, Tim saw a big box. He wanted to open it. Tim went to his friend, Sue. Sue saw the box and said, \"I can open this box!\" Sue wanted to open the box. They opened the box. Inside, there was a big, soft teddy bear. The bear was happy.\n", - "Tim and Sue played with the bear all day. They were not very good at all. They played with the toys and had lots of fun. They learned that being kind was better and to help others. And from that day on, Tim and Sue were the best of friends.\n", + "Once upon a time in a nice forest there lived a brave bear. The bear was big and very strong. He had many friends, but the rabbit didn't like the bear. It wanted to play, but the bear was too scared to go near. \n", + "One day, the bear was walking when he saw a big tree. He was scared and wanted to run under the tree. But the tree wanted to play and have fun. So, he decided to go and hide in the tree.\n", + "The tree started to shake and the bear was worried. He didn't like it. He stayed away and hid behind the tree, until the storm was gone. The rabbit was happy and never tried to end. The brave bear was glad he had found the tree and he knew he would never get into a safe place.\n", "\n", "\n", - "Step 6000, Loss: 1.4077720642089844, Elapsed Time: 70.17 seconds\n", + "Step 8800, Loss: 1.399843454360962, Elapsed Time: 16.96 seconds\n", "Generated text:\n", - "Once upon a time, a little girl named Sue and her dog, Spot. They lived in a big house with their mom and dad. One day, they found a shiny rock that they had to take home. They took it home and put it in their room.\n", - "Sue's mom saw the shiny rock and said, \"Oh, my toy! Your toy is very special! You should take it home.\" Sue was happy to have her toy back and said, \"Thank you, Spot!\"\n", - "But then, a big wind came and blew the rock back. Sue and Spot got scared and started to cry. Their mom came and saw what happened. She told them not to worry and they both went to the toy store to get the shiny rock.\n", - "The moral of the story is that it's important to take care of our things, but we can also take care of them. If you take care of them, we might find them and help them find them.\n", + "Once upon a time there lived two twins named Timmy and Sally. Timmy was three years old and Sally was always looking for fun to do something exciting together. \n", + "One morning Timmy decided to go for a walk in the woods. He walked for a long time until he found a big red ball. When he picked it up, Sally was so excited that it started jumping up and down in a giggle! \n", + "Timmy was so happy that he started to laugh with Sally. They both ran around the woods, laughing and having such a great time. It was a fun day to go back home. \n", + "And that��s how Timmy and Sally had so much fun that day!\n", "\n", "\n", - "Step 6200, Loss: 1.3794877529144287, Elapsed Time: 77.59 seconds\n", + "Step 9000, Loss: 1.3851344585418701, Elapsed Time: 16.47 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside and pick flowers. One day, she went to the park with her mommy. She saw a boy crying because he lost his toy car.\n", - "Lily said, \"I lost my toy car, but I can't find it.\"\n", - "Lily asked the boy, \"Have you seen my toy car?\" The boy said, \"No, I haven't seen it.\"\n", - "Lily felt sad and went to play on the swings. She went to the boy and said, \"Thank you, you're my friend.\" The boy said, \"You are very kind. Can you help me find your toy car?\"\n", - "Lily smiled and said, \"I will help you find your toy car. Maybe you can find your toy car.\"\n", + "Once upon a time there was a mom and her daughter. They wanted to make a pie. They went to the store to buy some flour, some flour and sugar. The daughter chose the most pretty yellow pie ever. She was so excited. She took it to the kitchen. She started to mix everything together.\n", + "The mom saw the pie and said, \"What was that? We don't have one. We must be careful not to make the pie.\"\n", + "The daughter looked around and saw something. It was a big, round, round piece. She wanted to show it to her friend. So the mom said, \"No, not that. We have not done this. The pie is in the kitchen.\"\n", + "The daughter was sad. She had never been to the pie before. But the mom said, \"We can make a pie, so we don't get what it is for.\"\n", + "The little girl was happy. She said, \"That is what we have.\" Then they went home, happy.\n", "\n", "\n", - "Step 6400, Loss: 1.3905563354492188, Elapsed Time: 73.45 seconds\n", + "Step 9200, Loss: 1.3884317874908447, Elapsed Time: 17.15 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside in the sun. One day, she saw a big, round thing on the ground. It was a pretty, shiny rock.\n", - "Lily picked up the rock and put it on her foot. She was so happy and showed her mom the rock. Her mom was proud of her and gave her a big hug. Lily felt happy too.\n", - "Later that day, Lily and her mom went to the park. Lily saw a big slide. She wanted to slide down it too. She ran up the slide, and slid down fast. The wind blew and Lily fell. She was hurt and sad.\n", - "Lily learned to be careful when she slide. She went to the swings, and when she was playing, she felt much better. Her mom gave her a hug and told her to keep the smooth rock. Lily learned that she should always keep the rock safe.\n", + "Once upon a time there was a young boy. He loved to explore the world around him.\n", + "One day he decided to go on a special trip. So he got into a beautiful, colorful taxi and it was so much fun! But the boy didn't know how to go.\n", + "He asked lots of questions and the taxi answered, \"I can't get out and I am scared.\" The boy kept trying to explain, but the taxi wouldn't budge.\n", + "Suddenly, the taxi driver started to get cold and the boy got cold. It started to get cold down on the ground. Suddenly, the boy had an idea - he put the taxi in a cart and put it up to the truck.\n", + "The taxi driver smiled and said, \"Now I can take you to the beach, the boy.\" He then drove away and the taxi went for a drive in the water.\n", + "The boy was so thankful he had found a special place with his friends. The young boy and the taxi drove home together, happy and safe.\n", "\n", "\n", - "Step 6600, Loss: 1.39736008644104, Elapsed Time: 76.19 seconds\n", + "Step 9400, Loss: 1.352752685546875, Elapsed Time: 17.15 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play with her toys and her favorite toy was a teddy bear. One day, Lily was playing with her teddy bear when she saw a shiny object on the ground. She picked it up and looked at it closely. \n", - "\"Wow, look at that object!\" said Lily.\n", - "\"I want it!\" said her mom. \"It's a special object. It's shiny and it's very special to me.\"\n", - "Lily thought for a moment and then decided to take the object from her mom to her friend. \"Look, I found a pretty bracelet!\" said Lily. \n", - "Her friend said, \"That's great, Lily! Let's play with it.\" And so, they played with the bracelet all day long. They were having so much fun!\n", + "Once upon a time there were two twins, Tom and Anna. They lived in a beautiful palace, and they were very rich.\n", + "One day they wanted to go to the palace with the king and Anna. So they got to the palace and looked inside. They saw a beautiful palace, but they wanted to explore it.\n", + "The palace was very dark and the palace and they felt very uncomfortable. The castle was dark, but the palace was still in the palace.\n", + "Suddenly, a voice said \"Welcome, mine!\"\n", + "The twins looked around to see who was standing up in the palace. There was no corner, but Tom wanted to take Anna and Anna. It was too dark.\n", + "The little girl said, \"I'm sorry, I didn't mean to. I just wanted to get away.\"\n", + "But it was too late. The king was very upset. He wished he had never taken a palace again and he wished he had never gone out the palace before. He wished he had never taken his palace.\n", "\n", "\n", - "Step 6800, Loss: 1.4118915796279907, Elapsed Time: 72.23 seconds\n", + "Step 9600, Loss: 1.3919847011566162, Elapsed Time: 17.26 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to draw with her toys and make pictures. One day, she was playing in her backyard when she saw a big bird sitting on a branch. The bird was so pretty! Lily wanted to be friends with the bird, so she tried to sing with it. But the bird didn't fly away. Lily was sad and didn't know what to do.\n", - "Then, Lily saw a bird that it was very graceful. She asked the bird what was wrong. The bird said that it had wings to fly away. Lily thought that it was a fun idea and flew away. She was happy again and went back to her house to play. From that day on, Lily knew that if she wanted to be friends with her friend, she could help the bird feel better.\n", + "Once upon a time there were two twin cats named Bob. One day Bob wanted to take a break from the sun. So he decided to break out. As he looked around, something special happened. Suddenly, a big cat appeared! It was a little girl! She had long, bright, bright red, and bright, with bright, blue eyes.\n", + "Bob said,\"I wanted to take a break with the cat and take a break. He carefully broke the cat's eyes. He was so angry! The cat was so mad that he started running away.\n", + "The other cat chased Bob to keep playing, and he ran, running and shouting, \"Stop! The cat now broke our window!\" They both laughed and ran away.\n", + "The cat was very sad. He had no idea the window to break the window again. He had broken his house!\n", + "The next day the cat came back and took a long walk. It was a beautiful garden filled with bright colours and flowers. It was a beautiful, beautiful day, and Bob and the cat became best friends.\n", "\n", "\n", - "Step 7000, Loss: 1.4061782360076904, Elapsed Time: 72.51 seconds\n", + "Step 9800, Loss: 1.3763813972473145, Elapsed Time: 17.16 seconds\n", "Generated text:\n", - "Once upon a time, a little girl named Sue lived in a big house. Sue had a toy car that she loved to play with. One day, Sue's mom told her not to clean up. Sue listened to her mom and started to clean her room.\n", - "After cleaning, Sue felt tired and needed to take a nap. She put on her clean, but her mom was not looking well. Sue went to her room and sat down to rest. Her mom said, \"Sue, I'm going to rest and drink a big drink.\" Sue smiled and felt better.\n", - "Sue went to bed with her mom to rest her room. In the end, she went to sleep with her mom. She felt happy that she was able to help her mom. From that day on, Sue knew that her mom loved her because she loved her little sister so much.\n", + "Once upon a time a girl called Kate. She was only three years old and she wanted to learn more.\n", + "One day, Kate saw a huge box and she was curious. He opened it and there was a big box inside. She was curious and she wanted to know what was inside.\n", + "But when she opened the box and saw a big, scary monster. \"What's wrong?\" Kate asked curiously.\n", + "The monster shook its head and it started to tremble. Kate screamed and tried to help it, but it was too late. In the end, Kate realised she had been trapped!\n", + "Kate had to be saved so much in the box. She quickly ran home and she was very relieved.\n", + "The end.\n", "\n", "\n", - "Step 7200, Loss: 1.381988286972046, Elapsed Time: 73.38 seconds\n", + "Step 10000, Loss: 1.360530972480774, Elapsed Time: 16.86 seconds\n", + "Generated text:\n", + "Once upon a time there was a naughty girl named Molly. She had a big, fluffy tail. One day, she wanted to make something special and her mom said it was time. \n", + "Molly's mom said, \"Molly, why don't you let me try to make something special? We can use the grill to make something special.\" \n", + "Molly thought it was a very cool and asked her mom, \"What is the grill?\" \n", + "Her mom explained that the grill could make things. \n", + "Molly was excited that the grill was really good for her. So they used a stick to make all of the toys together. \n", + "In the end, her mom was delighted. She gave Molly a big hug and said, \"I'm glad you like it!\" \n", + "Molly and her mom were so happy. She said, \"Thank you Mommy!\" \n", + "Molly hugged her mom tight and said, \"We can go play again tomorrow! I'm so happy I could help!\" \n", + "Molly and her mom went to the park with the grill in her heart. The grill was a bit full of all kinds of fun and delicious surprises.\n", + "\n", + "\n", + "Step 10200, Loss: 1.359804391860962, Elapsed Time: 17.36 seconds\n", "Generated text:\n", - "Once upon a time, a little girl named Lily went to the park with her mommy. She saw a boy who looked sad. She went to her mommy and asked her if she could have a turn to her.\n", - "\"Sure, but you have to ask your mommy,\" her mommy said. \"It's okay, but it's important to ask nicely.\"\n", - "Lily nodded and gave the boy a hug. \"Can we play on the swings?\" she asked.\n", - "The boy didn't like it and said, \"Sure, but be careful.\"\n", - "Lily felt happy and went to play on the swings. She went back to the boy and asked if he wanted to swing too. The boy said, \"I want to swing like you, but it's not safe.\"\n", - "Lily went to the swing and pushed the boy away. She was sad that she lost her favorite toy. The boy was upset and didn't want to swing anymore. They both sat on the swing and watched the boy play on the swings. Lily felt better and said, \"Thank you for letting me swing, Timmy. You're my friend.\"\n", + "Once upon a time a girl was sad, but she knew it was a magic wand because she loved to play with. One day, her friend Jack saw a little boy playing with her. Jack wanted to play too, but he was being mean. \n", + "Jack asked, \"Can I play with your magic wand?\" \n", + "The little boy said, \"No, don't fight! It's not ours. It can hurt you.\" \n", + "The little boy was sad and asked, \"Why can't you do that?\" \n", + "The little girl replied, \"I wanted to play with your wand. Please, let me try and play!\" \n", + "Jack thought hard and had an idea. He put his wand and said, \"I will show you how to play.\" \n", + "The moral of the story is: you can be kind to others who will help you and be honest to others.\n", "\n", "\n", - "Step 7400, Loss: 1.3883332014083862, Elapsed Time: 83.07 seconds\n", + "Step 10400, Loss: 1.3811813592910767, Elapsed Time: 16.96 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside in the garden and feel the warm sunshine on her skin. One day, while she was playing, she noticed that the flowers had bloomed in a beautiful butterfly. She wanted to see the butterfly so she asked her mommy if she could go to the butterfly's home. Her mommy said yes and Lily ran to her mommy's room. \n", - "When she got there, she saw that the butterfly was sad and started to cry. Her mommy told her that she couldn't go back to the garden to get her dress and her mommy was not happy. Lily felt bad and didn't want to go back to the garden to get her dress. \n", - "The next day, Lily saw the butterfly again and ran back to her mommy's house. She was happy to see her mommy again and said she was a great helper. Her mommy was so happy that she could see the butterfly again and that was a beautiful butterfly who was a good girl.\n", + "Once upon a time a girl called Emma loved to explore. She was so independent and could do lots of things. One day, Emma found a very small pond. She wanted to keep playing. \n", + "Emma was so excited. She grabbed her favourite toy and ran into the pond. She splashed around and had so much fun. \n", + "But then, she accidentally dropped it. It was a smelly sight, and she couldn't get out. She was so scared, she started to cry.\n", + "Then, Emma remembered her mommy and stayed in the pond while playing. She swam back in the pond and then came back into a big pond. As she watched the fishes swim away she saw a beautiful butterfly.\n", + "Emma was so happy to have her toy back. She hugged her toy and ran around the pond, happy and full.\n", "\n", "\n", - "Step 7600, Loss: 1.3650509119033813, Elapsed Time: 80.14 seconds\n", + "Step 10600, Loss: 1.3736326694488525, Elapsed Time: 17.03 seconds\n", "Generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play with her toys, but one day she got bored. She went to her friend, Tim, and asked if he could borrow his toy.\n", - "Tim said, \"No, Lily. This is my toy car. You can play with it, but I want to borrow your toy car.\"\n", - "Lily felt sad because she didn't want to play with Tim. She said, \"Please, Tim. We can borrow your toy car. I can borrow it for my birthday.\"\n", - "Tim took the toy car and said, \"No, Lily. I can borrow it for my birthday.\" He played with the toy car all day. He was happy with his new toy car.\n", + "Once upon a time a boy called Pete had a big surprise. It was a big balloon! It was so big that it made Pete smile. It wanted to show it to his brother, Sam. \n", + "Pete said, \"Wow! What is the surprise?\" \n", + "Sam said, \"My brother is waiting for us.\" \n", + "The balloon was very high so Pete was happy. He put some flowers on his plate and they both laughed. \n", + "The balloon started to move and Pete clapped. They both laughed and laughed as they walked away. \n", + "Pete said, \"You are very big, but my little brother! It is so big.\" \n", + "Sam replied, \"Yeah, I can't wait to see it!\" \n", + "The balloon was so big that Pete had to wait until finally Sam's brother said goodbye. They were both happy, and it had a great time.\n", "\n", "\n", - "Step 7800, Loss: 1.3602105379104614, Elapsed Time: 69.70 seconds\n", + "Step 10800, Loss: 1.3681195974349976, Elapsed Time: 16.87 seconds\n", "Generated text:\n", - "Once upon a time, a little girl named Lily went to a park with her mommy. The park was a pretty green place, with a big pond with lots of ducks. Lily loved the sound of the water, so she decided to play in the water.\n", - "Lily saw a butterfly flying in the sky and tried to catch it. But the butterfly was too fast, and she couldn't catch it. Suddenly, the butterfly landed on a flower and Lily started to panic.\n", - "Her mommy saw the butterfly and asked, \"What happened, Lily?\"\n", - "\"The butterfly fell and hurt my hand,\" said Lily.\n", - "Her mommy came over and saw the butterfly and asked, \"Are you okay, Lily?\"\n", - "\"It hurts, sweetie,\" said her mommy.\n", - "Lily's mommy took a big spoon and the butterfly was very pretty and had lots of colors on it. Lily felt much better and was able to catch the butterfly. She was happy to have her friend a little friend, and they played together all day long.\n", + "Once upon a time there was a young girl who wanted to do something different. She went to a store and saw a lot of things that were fake! But she didn't know the real things that didn't just what she was looking right. \n", + "She decided to go to the store and ask a lot of questions. When she arrived, she saw that it was full of money! She was so amazed! She didn't know what to do, but she knew she was right. \n", + "Then she remembered how she had to buy some money that she could buy. Instead, she chose money, so that it was worth what her. After a while, she could buy more things from her. And she was very happy that she had learned a valuable lesson - that it's important to try your things and not give what you are not worth it if they are happy!\n", "\n", "\n", - "Step 8000, Loss: 1.3417373895645142, Elapsed Time: 79.57 seconds\n", + "Step 11000, Loss: 1.3663400411605835, Elapsed Time: 16.79 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Tim. Tim had a toy named Sam. Sam liked to play with the toy car. One day, Tim's mom said they were going to a new park. Tim wanted to play on the swings, but his mom said no.\n", - "Tim started playing with the car. He saw a big dog in the park. The dog barked loudly. Tim felt scared. He wanted to run away. He went to his mom and dad and said they were safe. Tim felt safe.\n", - "Later, Tim and Sam went to the park. They saw a big slide. They both wanted to go on the slide. They ran to the slide and slid down the slide. Tim's mom and dad helped them get down. They all laughed and had a great time together.\n", + "Once upon a time a mom and their son went for a walk. His mom said \"Let's go!\"\n", + "The son was excited, but then said, \"Okay!\"\n", + "On his way to the park, the son saw a huge, big truck. The son asked his mom, \"What was that?\"\n", + "His mom replied, \"That's a huge truck driver!\" \n", + "The son smiled, and said, \"Can we go see it?\"\n", + "His mom smiled and said, \"Of course! Let's go see the truck!\" The son was so excited he started running towards the huge truck driver.\n", + "But his son was already there. He was not sure why he wouldn't stop going. He thought it was so cool and he started to cry. \n", + "His mom was sad. She said, \"Don't worry, we will never go on an adventure and be safe.\"\n", "\n", "\n", - "Step 8200, Loss: 1.3874244689941406, Elapsed Time: 71.68 seconds\n", + "Step 11200, Loss: 1.3681292533874512, Elapsed Time: 16.93 seconds\n", "Generated text:\n", - "Once upon a time, there was a little boy named Timmy. Timmy loved to play with his toy car, but his favorite thing to do was to go to the store. Timmy didn't like the store, so he went to the store.\n", - "When they got to the store, Timmy saw a toy car and wanted it. He asked his mom for a toy car and her mom said, \"No, it's too expensive.\" Timmy didn't want to buy a toy car. He thought the toy car wouldn't have a toy car, but his mom said it was okay.\n", - "Later that day, Timmy went to the park with his mom. Timmy saw a boy who looked sad because he lost his toy car. Timmy felt sorry for the boy and said, \"I can help you find my toy car.\" But he couldn't find his toy car, and he didn't know what to do.\n", - "Timmy asked his mom, \"Why are you looking for my toy car, and the boy can have it back?\" His mom said, \"Don't worry, we can fix it. Let's find your toy car.\" They found the toy car and it was in the store. Timmy was happy that he found a!!!!\n", + "Once upon a time there lived a mom and her daughter, a mom, and her daughter. \n", + "One day, the mom said to her daughter, ��We��ve been very careless,��\n", + "The daughter replied, ��But Mommy, can we go on an adventure?��\n", + "The mom smiled and said, ��Yes,�� So, the daughter got off and they ran to their house.\n", + "They arrived at their house, but it was too late. The daughter was gone and her mom said to her, ��The daughter, you are not supposed to go home now!�� The mom was very sad that she didn��t want it to end.\n", + "The daughter was sad and cried, ��Mommy, I want my daughter to be home and she��ve been!�� \n", + "Her daughter was sad, but she knew how much she was sad. She learned an important lesson - that we should never forget to take action.\n", "\n", + "\n", + "Step 11400, Loss: 1.3560458421707153, Elapsed Time: 17.13 seconds\n", + "Generated text:\n", + "Once upon a time a young girl were in the garden. She had many toys and loved to collect things. One day she found a little girl and showed it to her mom. They wanted to take care of the little girl and take care of her. \n", + "The little girl was curious, so she asked her mom if she could take care of the little girl. \n", + "But her mom said, she would take care of her and take care of her. The little girl was scared, but soon she saw it and felt a little scared. So she decided to take some care of it and it stayed very late. \n", + "Unfortunately, her mom was very angry about her, and so she decided never to take care of it again. The little girl was very sad, and she never went to collect food again. The end.\n", + "\n", + "\n", + "Step 11600, Loss: 1.3774775266647339, Elapsed Time: 16.67 seconds\n", + "Generated text:\n", + "Once upon a time in the far in a town. One day, a big storm got very scary. The storm was very strong and strong. All the kids were scared and ran back to their town. They were very scared and wanted to go home. But then, they heard a loud noise.\n", + "The storm started to shake. The kids were scared and ran away quickly. But then, they saw another monster. He was so big and strong and he had a big roar. They were so big and scary. They were scared and ran away and they were safe. \n", + "The storm started, and they ran away. But, they could still be friends. This was not the last bear. This was the same way to stay in the town. The people in the town were very happy to be safe. \n", + "The end.\n", + "\n", + "\n", + "Step 11800, Loss: 1.3781214952468872, Elapsed Time: 16.86 seconds\n", + "Generated text:\n", + "Once upon a time a young duck, the duck. He was so excited! He thought it looked like a big, green frog. The duck had to stay close to the pond when the water started to move.\n", + "The duck was sad and didn't know what to do. He thought it might work, but it didn't work. So he decided to take the other duck home.\n", + "The duck kept asking the other duck what the duck was doing. He said, \"I am trying to be patient, I can do the same. I just want to be friends, so I can have some fun and do what I do next.\" \n", + "The duck was still a little scared and tried to stop the duck from having fun. But the duck was too big and too slow. The duck didn't know what to do. He just kept trying to find a solution to the problem. \n", + "Finally, the quacking duck found a way to have a great time with his friends. The duck and duck were so happy! They all went back to the pond and played together. The duck was never a bossy duck, but the duck was still happy and it made him happy too.\n", + "\n", + "\n", + "Step 12000, Loss: 1.3862016201019287, Elapsed Time: 17.28 seconds\n", + "Generated text:\n", + "Once upon a time a girl, in the park, a girl named Sue and her cat, Luna. They liked to play together, run and run all around. One day,, Sue's mom took her to the park. She brought her favorite, a big box and showed it to Luna. It was a surprise for her.\n", + "At the park, there was many other dogs. They all wanted to see the new dog. But the box was too big for her. It had a big problem. It had a hole in it. The mud and mud was in there!\n", + "Sue had an idea. She wanted to take a ride a ride. So they went together and played in the muddy mud. But then, they saw something strange. There was a big mud mud mud inside the hole. They all covered it with mud.\n", + "They ran to the mud and the mud mud mud. They had ruined it! The mud mud was very filthy and dirty! Sue and Luna had ruined the mud and they were not happy. They were sad. They wished they had not taken the mud pies from the ground. From that day on, Sue and Luna went to the park. They never played in the mud again.\n", + "\n", + "\n", + "Step 12200, Loss: 1.3829598426818848, Elapsed Time: 17.43 seconds\n", + "Generated text:\n", + "Once upon a time a mom and Dad had a daughter who wanted to make her a lot of food. She wanted to make a special home, but her parents said no.\n", + "Mum said that it was too dangerous. She said she would not allow her daughter to get any food. So the family went to the store to get a better home.\n", + "When they finally got back home, the daughter felt much better and was so happy. She wanted to make her mom happy again and she said it was ok to be a good home.\n", + "Every day, the family enjoyed the food and was thankful for her family. She felt a little jealous when she was in one place to make them feel better by being safe and happy.\n", + "\n", + "\n", + "Step 12400, Loss: 1.3898296356201172, Elapsed Time: 16.82 seconds\n", + "Generated text:\n", + "Once upon a time a girl and she lived in a house. They had lots of nice things that they could do. They liked to make different noises with each other. One day something changed from one thing to do. It was a fake thing. It had a fake face.\n", + "The girl was scared of the fake thing. But the fake thing was not real. So the fake thing made a funny laugh! The fake thing made the girl laugh too.\n", + "The girl laughed too. She liked it a real face. She loved it too. Now she could play her pretend thing. She could jump in her fake face! The two were all friends!\n", + "\n", + "\n", + "Step 12600, Loss: 1.3644123077392578, Elapsed Time: 16.47 seconds\n", + "Generated text:\n", + "Once upon a time a mommy bear lived in a forest. The sun was shining so bright, and the birds were singing.\n", + "The mommy bear decided to take a nap of her nap. The bear laid down and closed his eyes. The rain came and the sun came out. The mommy bear was happy to have a cozy bed.\n", + "The mommy said, ��I will keep going to sleep tonight. I will take a nap and sleep.��\n", + "The mommy smiled and said, ��Yes, of course I will.�� Then the mommy gave him a special blanket.\n", + "She closed her eyes and tucked him in for a few days until finally, one night. She was so sleepy that she yawned and drifted off to sleep.\n", + "The end.\n", + "\n", + "\n", + "Step 12800, Loss: 1.3584455251693726, Elapsed Time: 16.63 seconds\n", + "Generated text:\n", + "Once upon a time there were two sisters who were always looking for fun things to do. One was called Sarah and the other was called Emma. One day she was feeling very upset that she wanted to play with her sister.\n", + "Sarah started to feel very scared. She started to cry. The other and the other sister tried to help her. They both looked around, not sure. They both had so much fun playing with the other sister.\n", + "Then Sarah had an idea. She decided to play together. She went to the bathroom and started to wash the other sister.\n", + "When she was finished playing with her sister, the older sister said: \"I'm sorry but I had to leave the bathroom!\"\n", + "Sarah felt better and she was happy that she had been able to play with her sister and not upset anymore. The end.\n", + "\n", + "\n", + "Step 13000, Loss: 1.3652907609939575, Elapsed Time: 16.67 seconds\n", + "Generated text:\n", + "Once upon a time a girl went in an icy land. She wanted to go on something special and so she decided to go on an adventure. On a magical journey she was so full of magical things that she could do.\n", + "The sun was shining and it was so warm. She felt so happy, she decided to go back to the land. She packed her bag full of sparkles and headed back down the icy land.\n", + "The girl was so excited to explore the land. She ran and hopped until she found a big pond. She was filled with joy and couldn't wait to explore more land. She had so much fun playing in the icy land.\n", + "Eventually, it was a magical land filled with magical things. She had never seen so many wonderful things before! She was so excited she started to explore and enjoy the land.\n", + "The magical land was full of magical lands, and the girl was so brave to explore. She had a great adventure with her journey, discovering new places and discovering new places.\n", + "\n", + "\n", + "Step 13200, Loss: 1.3745272159576416, Elapsed Time: 17.27 seconds\n", + "Generated text:\n", + "Once upon a time there were two cats who were always playing together.\n", + "One day when they went to the park, they saw a very interesting man sitting alone. The cat was alone and was very curious.\n", + "The two cats decided to investigate. The cats crept closer and closer, but the man kept walking. He was surprised at the cat.\n", + "The cat then realized that he had a big mouth to show the dog something else. He quickly scurried away and started to chew on the ice. The cats were so scared that they started barking.\n", + "The cat then ran away, feeling so scared and sad. He tried to find somewhere else to hide, so he found a small creature. This was a monster!\n", + "The monster was so angry that he ran away. The cats started chasing and the monster was destroyed.\n", + "The cats were sad. They wished that the fight had been a way to protect. \n", + "The cats never saw a police man. He had found a way to protect them from bad things and never find the cat again.\n", + "\n", + "\n", + "Step 13400, Loss: 1.329391598701477, Elapsed Time: 17.00 seconds\n", + "Generated text:\n", + "Once upon a time a small boat wanted to sail. So the boat decided to sail across the big sea. It sailed through the water, across a long road.\n", + "The boat began to sail. It was very fast, but it didn't want to give up. The boat kept sailing and sailing for a long time. It kept sailing until it reached a big island that was high up in the sky!\n", + "The boat sailed and sailed until it was far away. It had so many fun things that could help it sail. Then it sailed back and sailed back home. From that day on, it never got lost again.\n", + "The boat always made it to the harbor and kept sailing even further. With the sail in the long voyage, it sailed away, and the boat sailed back home safely. The boat was never lonely again!\n", + "\n", + "\n", + "Step 13600, Loss: 1.3439335823059082, Elapsed Time: 16.83 seconds\n", + "Generated text:\n", + "Once upon a time a young child. One day, the child went for a walk through the woods. As the child was walking, a curious three-year-old noticed something funny. It was a strange thing that was a strange looking thing.\n", + "The child stopped to look at the thing and said, \"What is that?\"\n", + "The curious child said, \"Oh, I see!\"\n", + "The two little eyes were curious. They decided to try and see what it could do.\n", + "The curious child took out a small piece of paper and a crayon. \"Look, this looks interesting!\" The child said excitedly, \"Yay!\"\n", + "The girl smiled. \"Wow! That must be a great thing!\"\n", + "The child then asked, \"What can we do with it?\"\n", + "The child replied, \"Let's take it back and see.\"\n", + "The little child said, \"Okay, what if we do?\"\n", + "The three of them took a few pictures of their own. They had so much fun playing with the crayon. They were so happy that the child decided that it's not safe to touch.\n", + "The child then realized that sometimes the thing that can be dangerous and exciting. So the three-year-old and the three!!!!\n", + "\n", + "Step 13800, Loss: 1.3428380489349365, Elapsed Time: 17.40 seconds\n", + "Generated text:\n", + "Once upon a time there lived two cats were named Jack and Sam. One day, they decided to go for a walk together. They ran around, laughing, and having fun. Then they came across a big, round rock. Jack had an idea! He said to Sam, \"Let's go get it!\" Sam was very excited and they ran to the rock. The rock had lots of fun playing. They ran over to the rock and started playing. After a while, they got tired and said goodbye. They went back to their house and said goodbye. Jack was a bit sad but they promised to go back again soon! The end.\n", + "\n", + "\n", + "Step 14000, Loss: 1.3359969854354858, Elapsed Time: 16.57 seconds\n", + "Generated text:\n", + "Once upon a time there was a bunny who was very sad. She wanted to find friends to be happy and share. So one day, she looked for something to help her find them. She hopped and hopped until she found a big house.\n", + "The bunny was so excited. She wanted to be found. She hopped inside the house and found a big, fluffy teddy bear! She was so happy when she saw all the friends.\n", + "The bunny was so happy that she started to dance around her house. All the other animals in the house would laugh and play. All the bunny was very excited.\n", + "But then the bunny started to move! She was so clumsy and slow, just like a pillow. She felt so sad and lonely. But then she heard a noise. She looked back. There was a tiny, fluffy bunny!\n", + "The bunny had a fun day. And so she decided to go back upstairs. She was very happy that she found the little bunny, even if she had found a new friend. She was so glad that she could share the joy of her new friend. The End.\n", + "\n", + "\n", + "Step 14200, Loss: 1.3117105960845947, Elapsed Time: 17.24 seconds\n", + "Generated text:\n", + "Once upon a time there was a family that wanted to celebrate. So the family went for a trip and had a great time. They brought some cake to share with the family. Every day the family had a great time, and the dad always brought out the best food for the family. \n", + "On the day of the day, the family was very happy and they played together with the cake. The end.\n", + "But then one day, the family had a very bad day. The family was so sad. The family had missed the family and they wanted to celebrate with everyone again. \n", + "From then on, the family would share the love with each other and everyone was so happy and thankful for it.\n", + "\n", + "\n", + "Step 14400, Loss: 1.3477566242218018, Elapsed Time: 16.67 seconds\n", + "Generated text:\n", + "Once upon a time there lived an intelligent dog named Max. He loved to play and jump in his yard. One day, he saw something new - he couldn't find it! But then he saw it! He was so excited! \n", + "He quickly ran back to his house and showed his mommy the new toy. \"Look, Mommy! I found my favorite toy,\" said Max. His mommy smiled and said, \"That's very clever Max. Let's go inside and play a game!\"\n", + "So they ran inside the house together and Max had lots of fun. He laughed, he had so much fun and made lots of fun games. From this day forward, whenever they played.\n", + "\n", + "\n", + "Step 14600, Loss: 1.3812224864959717, Elapsed Time: 16.77 seconds\n", + "Generated text:\n", + "Once upon a time there was a happy bunny named Bob. Bob was a very special rabbit and was always very friendly.\n", + "One day, Bob heard something new. It was a tiny rabbit who had a big, fluffy tail. Bob thought the rabbit was so cute!\n", + "The rabbit wanted to be her friend. He hopped up and down and said, \"Hello! Come and be my friend.\" Bob wanted to be the best friend, so he said, \"I can show you a big surprise!\".\n", + "The rabbit was so excited. He hopped to the garden and began hopping around. Bob looked and he couldn't believe it was his new pet rabbit. He said it was his new pet, and now he was all very happy.\n", + "Bob was so happy that he gave the rabbit a big kiss goodbye. He had found a new friend! He said to the rabbit, \"Thanks for showing me my new friend I found you!\" The rabbit smiled and hopped back home to give him a big hug.\n", "Final generated text:\n", - "Once upon a time, there was a little girl named Lily. She loved to play outside and pick flowers. One day, Lily saw a butterfly and tried to catch it, but it flew away. Lily felt sad because she didn't know what was happening.\n", - "Suddenly, her friend, a little bird named Timmy, came to visit. \"Don't worry, Timmy,\" said Lily. \"I can catch it!\"\n", - "\"I'm sorry, Lily,\" said Timmy. \"I just need to help my mom.\"\n", - "\"Thank you, Timmy,\" said Lily. \"You are very kind to me.\"\n", - "From that day on, Lily learned to always help others when they needed it. And every time she saw the butterfly, she felt happy again.\n", - "\n" + "Once upon a time there was a family of four. It was very big and had many friends to play with. They had a lot of fun all of their toys. They played a game with each of the toys and made them laugh when they were all part.\n", + "The mom and dad decided to do something special to do. They went to the store and bought some of the dolls and toys. The parents gave them lots of food and water and they all started to play with them.\n", + "The daughter was very happy. She loved playing with her doll and her doll. After the game was all the time. They played and ate dinner and their dolls were very happy and they hugged. It was a nice day!\n" ] } ], "source": [ "with mesh:\n", - " model = create_model(rngs=nnx.Rngs(0))\n", - " optimizer = nnx.Optimizer(model, optax.adam(1e-3), wrt=nnx.Param)\n", - "\n", + " model = create_model(rngs=nnx.Rngs(0))\n", + " optimizer = nnx.Optimizer(model, optax.adam(1e-3), wrt=nnx.Param)\n", "metrics = nnx.MultiMetric(\n", " loss=nnx.metrics.Average(\"loss\"),\n", ")\n", @@ -1136,15 +1393,14 @@ " continue # skip the remaining elements\n", " input_batch = jnp.array(jnp.array(batch).T)\n", " target_batch = prep_target_batch(input_batch)\n", - " with jax.set_mesh(mesh):\n", - " train_step(\n", - " model,\n", - " optimizer,\n", - " metrics,\n", - " jax.device_put(\n", - " (input_batch, target_batch), P(\"batch\", None)\n", - " ),\n", - " )\n", + " train_step(\n", + " model,\n", + " optimizer,\n", + " metrics,\n", + " jax.device_put(\n", + " (input_batch, target_batch), NamedSharding(mesh, P(\"batch\", None))\n", + " ),\n", + " )\n", "\n", " if (step + 1) % 200 == 0:\n", " for metric, value in metrics.compute().items():\n", @@ -1178,14 +1434,14 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 472 }, "id": "B6Eg1Cz2y_iP", - "outputId": "7cafe711-1ae4-4eb9-fd37-e1bde54cbfc5", + "outputId": "60e54019-dcdc-425b-e95a-c03e301d688f", "tags": [ "nbval-ignore-output" ] @@ -1193,7 +1449,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABIjElEQVR4nO3deXxU9b3/8fdMlhmyTBYgC5CETQmLYRMhUEUFQaW9IGiV6g9woVXhXqy219JeN6wX1Kut1opYW7EqUqWC1WoF0WAVkMWAbCIoJAGSIEsyWSfJzPn9ETIYgZCEmTmZyev5cB5kzpwz8zk54rz9nu9iMQzDEAAAQIiwml0AAACALxFuAABASCHcAACAkEK4AQAAIYVwAwAAQgrhBgAAhBTCDQAACCmEGwAAEFIINwAAIKQQbgD43YwZM9S9e/dWHfvggw/KYrH4tiAAIY1wA7RjFoulWY+cnByzSzXFjBkzFBMTY3YZAFrIwtpSQPv1yiuvNHr+17/+VatWrdLLL7/caPsVV1yh5OTkVn9ObW2tPB6PbDZbi4+tq6tTXV2d7HZ7qz+/tWbMmKFly5apvLw84J8NoPXCzS4AgHluuummRs/Xr1+vVatWnbL9+yorKxUVFdXsz4mIiGhVfZIUHh6u8HD+UwWg+bgtBaBJl156qQYMGKDNmzfrkksuUVRUlH79619Lkt566y1NmDBBXbp0kc1mU69evfTwww/L7XY3eo/v97nZv3+/LBaL/u///k/PP/+8evXqJZvNpmHDhmnjxo2Njj1dnxuLxaLZs2drxYoVGjBggGw2m/r3769//etfp9Sfk5OjCy+8UHa7Xb169dKiRYt83o/njTfe0NChQ9WhQwd16tRJN910kw4ePNhon6KiIt18883q1q2bbDabUlNTNXHiRO3fv9+7z6ZNmzR+/Hh16tRJHTp0UI8ePXTLLbf4rE6gveB/hwCc1dGjR3XVVVfphhtu0E033eS9RbV48WLFxMTo7rvvVkxMjD788EPdf//9cjqdevzxx8/6vkuWLFFZWZl+9rOfyWKx6LHHHtPkyZP1zTffnLW155NPPtGbb76pO++8U7GxsXr66ac1ZcoU5efnq2PHjpKk3NxcXXnllUpNTdVDDz0kt9utefPmqXPnzuf+Szlh8eLFuvnmmzVs2DDNnz9fxcXFeuqpp/Tpp58qNzdX8fHxkqQpU6Zox44d+s///E91795dhw8f1qpVq5Sfn+99Pm7cOHXu3Fm/+tWvFB8fr/379+vNN9/0Wa1Au2EAwAmzZs0yvv+fhdGjRxuSjOeee+6U/SsrK0/Z9rOf/cyIiooyqqurvdumT59uZGRkeJ/v27fPkGR07NjROHbsmHf7W2+9ZUgy3n77be+2Bx544JSaJBmRkZHG3r17vdu2bt1qSDL+8Ic/eLf96Ec/MqKiooyDBw96t+3Zs8cIDw8/5T1PZ/r06UZ0dPQZX6+pqTGSkpKMAQMGGFVVVd7t77zzjiHJuP/++w3DMIzjx48bkozHH3/8jO+1fPlyQ5KxcePGs9YFoGnclgJwVjabTTfffPMp2zt06OD9uaysTEeOHNHFF1+syspKffnll2d93+uvv14JCQne5xdffLEk6ZtvvjnrsWPHjlWvXr28z7OysuRwOLzHut1uffDBB5o0aZK6dOni3a9379666qqrzvr+zbFp0yYdPnxYd955Z6MOzxMmTFBmZqb++c9/Sqr/PUVGRionJ0fHjx8/7Xs1tPC88847qq2t9Ul9QHtFuAFwVl27dlVkZOQp23fs2KFrrrlGcXFxcjgc6ty5s7czcmlp6VnfNz09vdHzhqBzpgDQ1LENxzcce/jwYVVVVal3796n7He6ba2Rl5cnSerTp88pr2VmZnpft9lsevTRR/Xee+8pOTlZl1xyiR577DEVFRV59x89erSmTJmihx56SJ06ddLEiRP14osvyuVy+aRWoD0h3AA4q++20DQoKSnR6NGjtXXrVs2bN09vv/22Vq1apUcffVSS5PF4zvq+YWFhp91uNGOGinM51gx33XWXvvrqK82fP192u1333Xef+vbtq9zcXEn1naSXLVumdevWafbs2Tp48KBuueUWDR06lKHoQAsRbgC0Sk5Ojo4eParFixdrzpw5+uEPf6ixY8c2us1kpqSkJNntdu3du/eU1063rTUyMjIkSbt37z7ltd27d3tfb9CrVy/dc889WrlypbZv366amho98cQTjfYZMWKEHnnkEW3atEmvvvqqduzYoaVLl/qkXqC9INwAaJWGlpPvtpTU1NTo2WefNaukRsLCwjR27FitWLFChw4d8m7fu3ev3nvvPZ98xoUXXqikpCQ999xzjW4fvffee9q1a5cmTJggqX5eoOrq6kbH9urVS7Gxsd7jjh8/fkqr06BBgySJW1NACzEUHECrjBw5UgkJCZo+fbr+67/+SxaLRS+//HKbui304IMPauXKlRo1apTuuOMOud1uPfPMMxowYIC2bNnSrPeora3Vb3/721O2JyYm6s4779Sjjz6qm2++WaNHj9bUqVO9Q8G7d++un//855Kkr776SmPGjNGPf/xj9evXT+Hh4Vq+fLmKi4t1ww03SJJeeuklPfvss7rmmmvUq1cvlZWV6U9/+pMcDoeuvvpqn/1OgPaAcAOgVTp27Kh33nlH99xzj/7nf/5HCQkJuummmzRmzBiNHz/e7PIkSUOHDtV7772nX/ziF7rvvvuUlpamefPmadeuXc0azSXVt0bdd999p2zv1auX7rzzTs2YMUNRUVFasGCB7r33XkVHR+uaa67Ro48+6h0BlZaWpqlTp2r16tV6+eWXFR4erszMTL3++uuaMmWKpPoOxRs2bNDSpUtVXFysuLg4XXTRRXr11VfVo0cPn/1OgPaAtaUAtDuTJk3Sjh07tGfPHrNLAeAH9LkBENKqqqoaPd+zZ4/effddXXrppeYUBMDvaLkBENJSU1M1Y8YM9ezZU3l5eVq4cKFcLpdyc3N13nnnmV0eAD+gzw2AkHbllVfqtddeU1FRkWw2m7Kzs/W///u/BBsghNFyAwAAQgp9bgAAQEgh3AAAgJDS7vrceDweHTp0SLGxsbJYLGaXAwAAmsEwDJWVlalLly6yWptum2l34ebQoUNKS0szuwwAANAKBQUF6tatW5P7tLtwExsbK6n+l+NwOEyuBgAANIfT6VRaWpr3e7wp7S7cNNyKcjgchBsAAIJMc7qU0KEYAACEFMINAAAIKYQbAAAQUgg3AAAgpBBuAABASCHcAACAkEK4AQAAIYVwAwAAQgrhBgAAhBTCDQAACCmEGwAAEFIINwAAIKQQbnyk1u1RsbNaBccqzS4FAIB2jXDjI5v2H9fw/12tGS9uMLsUAADatTYTbhYsWCCLxaK77rrrjPssXrxYFoul0cNutweuyCY4OoRLkpzVdSZXAgBA+xZudgGStHHjRi1atEhZWVln3dfhcGj37t3e5xaLxZ+lNVtchwhJkrOq1uRKAABo30xvuSkvL9eNN96oP/3pT0pISDjr/haLRSkpKd5HcnJyAKo8O8eJcOOq86i61m1yNQAAtF+mh5tZs2ZpwoQJGjt2bLP2Ly8vV0ZGhtLS0jRx4kTt2LGjyf1dLpecTmejhz/ERIaroRGpjFtTAACYxtRws3TpUn3++eeaP39+s/bv06eP/vKXv+itt97SK6+8Io/Ho5EjR+rAgQNnPGb+/PmKi4vzPtLS0nxVfiNWq0WxtoZ+N9yaAgDALKaFm4KCAs2ZM0evvvpqszsFZ2dna9q0aRo0aJBGjx6tN998U507d9aiRYvOeMzcuXNVWlrqfRQUFPjqFE7hoN8NAACmM61D8ebNm3X48GENGTLEu83tduvjjz/WM888I5fLpbCwsCbfIyIiQoMHD9bevXvPuI/NZpPNZvNZ3U1x2CMkVTFiCgAAE5kWbsaMGaNt27Y12nbzzTcrMzNT995771mDjVQfhrZt26arr77aX2W2iHc4OC03AACYxrRwExsbqwEDBjTaFh0drY4dO3q3T5s2TV27dvX2yZk3b55GjBih3r17q6SkRI8//rjy8vJ02223Bbz+06lvuaHPDQAAZmoT89ycSX5+vqzWk92Cjh8/rpkzZ6qoqEgJCQkaOnSo1q5dq379+plY5UkNfW5KabkBAMA0bSrc5OTkNPn8d7/7nX73u98FrqAW8rbcVNHnBgAAs5g+z00o8c5SzG0pAABMQ7jxIToUAwBgPsKND53sUMxtKQAAzEK48SEm8QMAwHyEGx9y2Fl+AQAAsxFufOhkyw23pQAAMAvhxoccjJYCAMB0hBsfargtVVPnUXWt2+RqAABonwg3PhQdGS6rpf5nOhUDAGAOwo0PWa0WxbK+FAAApiLc+Ficd30pOhUDAGAGwo2PeWcppuUGAABTEG587OTimYQbAADMQLjxMZZgAADAXIQbH2PxTAAAzEW48TEHo6UAADAV4cbHWIIBAABzEW58jMUzAQAwF+HGx0623BBuAAAwA+HGxxgKDgCAuQg3PhYXxVBwAADMRLjxMVpuAAAwF+HGx767/IJhGCZXAwBA+0O48bGGlptat6HqWo/J1QAA0P4QbnwsKjJMYVaLJIaDAwBgBsKNj1kslpNz3dDvBgCAgCPc+IF3rhtabgAACDjCjR+cHDHFcHAAAAKNcOMHDSOmSrktBQBAwBFu/ICVwQEAMA/hxg+YyA8AAPMQbvyAJRgAADAP4cYPGAoOAIB5CDd+wFBwAADMQ7jxA4aCAwBgHsKNH3x38UwAABBYhBs/YLQUAADmIdz4wck+N9yWAgAg0Ag3ftDQclNaVSvDMEyuBgCA9oVw4wcNfW7cHkOVNW6TqwEAoH0h3PhBh4gwhVstkuhUDABAoLWZcLNgwQJZLBbdddddTe73xhtvKDMzU3a7XRdccIHefffdwBTYAhaLRXEdGA4OAIAZ2kS42bhxoxYtWqSsrKwm91u7dq2mTp2qW2+9Vbm5uZo0aZImTZqk7du3B6jS5mMiPwAAzGF6uCkvL9eNN96oP/3pT0pISGhy36eeekpXXnmlfvnLX6pv3756+OGHNWTIED3zzDMBqrb5WIIBAABzmB5uZs2apQkTJmjs2LFn3XfdunWn7Dd+/HitW7fOX+W1Gi03AACYI9zMD1+6dKk+//xzbdy4sVn7FxUVKTk5udG25ORkFRUVnfEYl8sll8vlfe50OltXbAuxBAMAAOYwreWmoKBAc+bM0auvviq73e63z5k/f77i4uK8j7S0NL991nd5l2DgthQAAAFlWrjZvHmzDh8+rCFDhig8PFzh4eFas2aNnn76aYWHh8vtPnV+mJSUFBUXFzfaVlxcrJSUlDN+zty5c1VaWup9FBQU+PxcTsfbcsNtKQAAAsq021JjxozRtm3bGm27+eablZmZqXvvvVdhYWGnHJOdna3Vq1c3Gi6+atUqZWdnn/FzbDabbDabz+puroY+N6W03AAAEFCmhZvY2FgNGDCg0bbo6Gh17NjRu33atGnq2rWr5s+fL0maM2eORo8erSeeeEITJkzQ0qVLtWnTJj3//PMBr/9sTo6Wos8NAACBZPpoqabk5+ersLDQ+3zkyJFasmSJnn/+eQ0cOFDLli3TihUrTglJbQGjpQAAMIepo6W+Lycnp8nnknTdddfpuuuuC0xB54BwAwCAOdp0y00wYyg4AADmINz4SVzDUHBabgAACCjCjZ+cbLmplWEYJlcDAED7Qbjxk4Y+Nx5Dqqg5dc4eAADgH4QbP7GFWxUZVv/rZZZiAAACh3DjJxaL5eQSDPS7AQAgYAg3ftTQ76a0knADAECgEG78KNY71w3DwQEACBTCjR+dXIKBlhsAAAKFcONHccxSDABAwBFu/Mi7BAOzFAMAEDCEGz/yTuRHyw0AAAFDuPEj71Bw+twAABAwhBs/ouUGAIDAI9z4EX1uAAAIPMKNH3mHgtNyAwBAwBBu/Kih5aaUPjcAAAQM4caPvH1uCDcAAAQM4caPGkZLlbnq5PEYJlcDAED7QLjxo4aWG8OQymvoVAwAQCAQbvzIHhEmW3j9r5hbUwAABAbhxs8YDg4AQGARbvyM4eAAAAQW4cbPTrbcEG4AAAgEwo2fnVyCgdtSAAAEAuHGz2i5AQAgsAg3ftbQ54ZZigEACAzCjZ95W27oUAwAQEAQbvzs5BIM9LkBACAQCDd+FkfLDQAAAUW48bOG9aXoUAwAQGAQbvyMoeAAAAQW4cbPGAoOAEBgEW78jOUXAAAILMKNnzW03JS76uTxGCZXAwBA6CPc+FnsiZYbw5DKXPS7AQDA3wg3fmYLD5M9ov7XTL8bAAD8j3ATAA0jpliCAQAA/yPcBABLMAAAEDiEmwDwzlLMEgwAAPgd4SYAGA4OAEDgEG4CgIn8AAAIHFPDzcKFC5WVlSWHwyGHw6Hs7Gy99957Z9x/8eLFslgsjR52uz2AFbcOSzAAABA44WZ+eLdu3bRgwQKdd955MgxDL730kiZOnKjc3Fz179//tMc4HA7t3r3b+9xisQSq3FZj8UwAAALH1HDzox/9qNHzRx55RAsXLtT69evPGG4sFotSUlICUZ7PnGy5IdwAAOBvbabPjdvt1tKlS1VRUaHs7Owz7ldeXq6MjAylpaVp4sSJ2rFjR5Pv63K55HQ6Gz0CzcFoKQAAAsb0cLNt2zbFxMTIZrPp9ttv1/Lly9WvX7/T7tunTx/95S9/0VtvvaVXXnlFHo9HI0eO1IEDB874/vPnz1dcXJz3kZaW5q9TOSNvyw23pQAA8DuLYRimruZYU1Oj/Px8lZaWatmyZXrhhRe0Zs2aMwac76qtrVXfvn01depUPfzww6fdx+VyyeVyeZ87nU6lpaWptLRUDofDZ+fRlH/v+Vb/788blJkSq3/ddUlAPhMAgFDidDoVFxfXrO9vU/vcSFJkZKR69+4tSRo6dKg2btyop556SosWLTrrsRERERo8eLD27t17xn1sNptsNpvP6m0NWm4AAAgc029LfZ/H42nU0tIUt9utbdu2KTU11c9VnRvvDMUMBQcAwO9MbbmZO3eurrrqKqWnp6usrExLlixRTk6O3n//fUnStGnT1LVrV82fP1+SNG/ePI0YMUK9e/dWSUmJHn/8ceXl5em2224z8zTOqqFDcbmrTnVuj8LD2lymBAAgZJgabg4fPqxp06apsLBQcXFxysrK0vvvv68rrrhCkpSfny+r9WQQOH78uGbOnKmioiIlJCRo6NChWrt2bbP655gp1n7y11zuqlN8VKSJ1QAAENpM71AcaC3pkORL/e7/lypr3Pr4l5cpvWNUwD4XAIBQ0JLvb+6PBAgT+QEAEBiEmwBhCQYAAAKDcBMgtNwAABAYhJsAaRgxVUrLDQAAfkW4CRCHveG2FHPdAADgT4SbAPEunsltKQAA/IpwEyDeWYq5LQUAgF8RbgLkZIdibksBAOBPhJsAYSg4AACBQbgJEIaCAwAQGISbAPF2KGa0FAAAfkW4CRBabgAACAzCTYDQ5wYAgMAg3ARIQ8tNRY1btW6PydUAABC6CDcBEntihmJJKmM4OAAAfkO4CZDwMKuiI8MkcWsKAAB/ItwEUBxLMAAA4HeEmwBiODgAAP5HuAkghoMDAOB/hJsAYjg4AAD+R7gJIFpuAADwP8JNANHnBgAA/yPcBJDjxFw3tNwAAOA/hJsAami5KaXPDQAAfkO4CSBvnxvCDQAAfkO4CSDvaCmWXwAAwG8INwF0skMxLTcAAPgL4SaAGAoOAID/EW4CKI6h4AAA+B3hJoAaWm6qat2qqfOYXA0AAKGJcBNAMSfmuZGkMm5NAQDgF4SbAAqzWhRrY8QUAAD+RLgJMEZMAQDgX4SbAIs9cWuKWYoBAPAPwk2AeVtu6HMDAIBfEG4C7OQSDPS5AQDAHwg3ARZHyw0AAH5FuAkw7/pS9LkBAMAvCDcBxhIMAAD4F+EmwBwswQAAgF+1KtwUFBTowIED3ucbNmzQXXfdpeeff95nhYUqh71hEj9abgAA8IdWhZuf/OQn+uijjyRJRUVFuuKKK7Rhwwb95je/0bx583xaYKhhEj8AAPyrVeFm+/btuuiiiyRJr7/+ugYMGKC1a9fq1Vdf1eLFi5v9PgsXLlRWVpYcDoccDoeys7P13nvvNXnMG2+8oczMTNntdl1wwQV69913W3MKpjnZ54bbUgAA+EOrwk1tba1sNpsk6YMPPtB//Md/SJIyMzNVWFjY7Pfp1q2bFixYoM2bN2vTpk26/PLLNXHiRO3YseO0+69du1ZTp07VrbfeqtzcXE2aNEmTJk3S9u3bW3MapmgYLcUMxQAA+IfFMAyjpQcNHz5cl112mSZMmKBx48Zp/fr1GjhwoNavX69rr722UX+clkpMTNTjjz+uW2+99ZTXrr/+elVUVOidd97xbhsxYoQGDRqk5557rlnv73Q6FRcXp9LSUjkcjlbX2VoFxyp18WMfyRZu1e7fXhXwzwcAIBi15Pu7VS03jz76qBYtWqRLL71UU6dO1cCBAyVJ//jHP7y3q1rK7XZr6dKlqqioUHZ29mn3WbduncaOHdto2/jx47Vu3bozvq/L5ZLT6Wz0MFNDnxtXnUfVtW5TawEAIBSFt+agSy+9VEeOHJHT6VRCQoJ3+09/+lNFRUW16L22bdum7OxsVVdXKyYmRsuXL1e/fv1Ou29RUZGSk5MbbUtOTlZRUdEZ33/+/Pl66KGHWlSTP8XawmWxSIYhlVXXyR4RZnZJAACElFa13FRVVcnlcnmDTV5enn7/+99r9+7dSkpKatF79enTR1u2bNFnn32mO+64Q9OnT9fOnTtbU9ZpzZ07V6Wlpd5HQUGBz967NaxWi2JtDAcHAMBfWtVyM3HiRE2ePFm33367SkpKNHz4cEVEROjIkSN68skndccddzT7vSIjI9W7d29J0tChQ7Vx40Y99dRTWrRo0Sn7pqSkqLi4uNG24uJipaSknPH9bTabt/NzW+HoECFndR3DwQEA8INWtdx8/vnnuvjiiyVJy5YtU3JysvLy8vTXv/5VTz/99DkV5PF45HK5Tvtadna2Vq9e3WjbqlWrzthHp61iODgAAP7TqpabyspKxcbGSpJWrlypyZMny2q1asSIEcrLy2v2+8ydO1dXXXWV0tPTVVZWpiVLlignJ0fvv/++JGnatGnq2rWr5s+fL0maM2eORo8erSeeeEITJkzQ0qVLtWnTpqCbGZnFMwEA8J9Wtdz07t1bK1asUEFBgd5//32NGzdOknT48OEWDa8+fPiwpk2bpj59+mjMmDHauHGj3n//fV1xxRWSpPz8/Ebz5owcOVJLlizR888/r4EDB2rZsmVasWKFBgwY0JrTMA2LZwIA4D+tarm5//779ZOf/EQ///nPdfnll3tvC61cuVKDBw9u9vv8+c9/bvL1nJycU7Zdd911uu6661pUb1vD4pkAAPhPq8LNtddeqx/84AcqLCz0znEjSWPGjNE111zjs+JCVUPLDbMUAwDge60KN1L9yKWUlBTvbMTdunVr9QR+7Y23zw23pQAA8LlW9bnxeDyaN2+e4uLilJGRoYyMDMXHx+vhhx+Wx+PxdY0hx9vnhpYbAAB8rlUtN7/5zW/05z//WQsWLNCoUaMkSZ988okefPBBVVdX65FHHvFpkaEmrgNDwQEA8JdWhZuXXnpJL7zwgnc1cEnKyspS165ddeeddxJuzuJkh2JabgAA8LVW3ZY6duyYMjMzT9memZmpY8eOnXNRoc5hp88NAAD+0qpwM3DgQD3zzDOnbH/mmWeUlZV1zkWFOoaCAwDgP626LfXYY49pwoQJ+uCDD7xz3Kxbt04FBQV69913fVpgKPKGG1puAADwuVa13IwePVpfffWVrrnmGpWUlKikpESTJ0/Wjh079PLLL/u6xpDTcFuqps6j6lq3ydUAABBaLIZhGL56s61bt2rIkCFyu9vuF7bT6VRcXJxKS0tbtFSEL3k8hnr/5l15DGnDr8coyWE3pQ4AAIJFS76/W9Vyg3NjtVqUfCLQ5B2rNLkaAABCC+HGJFnd4iRJWwtKzC0EAIAQQ7gxycC0eElSLuEGAACfatFoqcmTJzf5eklJybnU0q4M6hYviZYbAAB8rUXhJi4u7qyvT5s27ZwKai8u6BYni0U6cLxKR8pd6hRjM7skAABCQovCzYsvvuivOtqdWHuEeneO0Z7D5dpaUKIxfZPNLgkAgJBAnxsTNfS74dYUAAC+Q7gx0aAT4WbLgVJzCwEAIIQQbkw06DstNz6cSxEAgHaNcGOiPimxsoVbVVpVq/1HmcwPAABfINyYKCLMqgFdmcwPAABfItyYbOCJ+W62EG4AAPAJwo3JBqbVt9wQbgAA8A3CjckGpyVIknYecqqmzmNyNQAABD/CjcnSEjsoISpCNW6PdhU6zS4HAICgR7gxmcViOTmZ34ESU2sBACAUEG7aAO9kfvS7AQDgnBFu2oCBhBsAAHyGcNMGNAwH/+bbCpVW1ZpbDAAAQY5w0wYkRkcqo2OUJGkb60wBAHBOCDdtxMnJ/I6bWwgAAEGOcNNGnOx3Q8sNAADngnDTRnx3xBQrhAMA0HqEmzaifxeHwq0WHSl36VBptdnlAAAQtAg3bYQ9IkyZqbGSWCEcAIBzQbhpQ5jMDwCAc0e4aUNOjpgqMbUOAACCGeGmDWloudl2oFR1blYIBwCgNQg3bUivzjGKsYWrqtatPYfLzS4HAICgRLhpQ6xWi7K6xUmiUzEAAK1FuGljGibz23qgxNQ6AAAIVqaGm/nz52vYsGGKjY1VUlKSJk2apN27dzd5zOLFi2WxWBo97HZ7gCr2v4Z+N7n5JabWAQBAsDI13KxZs0azZs3S+vXrtWrVKtXW1mrcuHGqqKho8jiHw6HCwkLvIy8vL0AV+19DuPmquEyVNXXmFgMAQBAKN/PD//WvfzV6vnjxYiUlJWnz5s265JJLznicxWJRSkqKv8szRbLDrhSHXUXOam0/6NRFPRLNLgkAgKDSpvrclJbWLxqZmNj0F3p5ebkyMjKUlpamiRMnaseOHWfc1+Vyyel0Nnq0dScn82OFcAAAWqrNhBuPx6O77rpLo0aN0oABA864X58+ffSXv/xFb731ll555RV5PB6NHDlSBw4cOO3+8+fPV1xcnPeRlpbmr1PwGW+nYlYIBwCgxSxGG1mC+o477tB7772nTz75RN26dWv2cbW1terbt6+mTp2qhx9++JTXXS6XXC6X97nT6VRaWppKS0vlcDh8Uruvrf36iH7yp8/UNb6DPv3V5WaXAwCA6ZxOp+Li4pr1/W1qn5sGs2fP1jvvvKOPP/64RcFGkiIiIjR48GDt3bv3tK/bbDbZbDZflBkwWd3iZbFIB0uq9G2ZS51jg6t+AADMZOptKcMwNHv2bC1fvlwffvihevTo0eL3cLvd2rZtm1JTU/1QoTlibOE6LylGEpP5AQDQUqaGm1mzZumVV17RkiVLFBsbq6KiIhUVFamqqsq7z7Rp0zR37lzv83nz5mnlypX65ptv9Pnnn+umm25SXl6ebrvtNjNOwW8aFtFkMj8AAFrG1HCzcOFClZaW6tJLL1Vqaqr38be//c27T35+vgoLC73Pjx8/rpkzZ6pv3766+uqr5XQ6tXbtWvXr18+MU/CbQenxklghHACAlmozHYoDpSUdksy0/WCpfviHT+Swh2vL/eNktVrMLgkAANO05Pu7zQwFR2N9UmJlC7fKWV2nfUebnrEZAACcRLhpoyLCrLqgKyuEAwDQUoSbNuzkZH4lptYBAEAwIdy0YSeXYSgxtQ4AAIIJ4aYNawg3OwudctW5zS0GAIAgQbhpw7oldFBidKRq3YZ2FZaZXQ4AAEGBcNOGWSyWk7em8lkhHACA5iDctHEnZypmhXAAAJqDcNPGDUxjODgAAC1BuGnjGm5LfXOkQqWVteYWAwBAECDctHHxUZHq3jFKkrSFRTQBADgrwk0QGJKRIElavavY5EoAAGj7CDdB4JrBXSVJK3IPqrqW+W4AAGgK4SYIjOzVSV3jO8hZXaf3dxSZXQ4AAG0a4SYIhFktmjK0myTpjU0HTK4GAIC2jXATJK47EW4+2XtEBccqTa4GAIC2i3ATJNISozSqd0dJ0rLNtN4AAHAmhJsg8uML0yTVhxuPxzC5GgAA2ibCTRAZ3z9FDnu4DpZU6dOvj5hdDgAAbRLhJojYI8I0cVD9sPDX6VgMAMBpEW6CTMOtqfd3FKmkssbkagAAaHsIN0FmQFeHMlNiVVPn0T+2HjK7HAAA2hzCTZCxWCy6flh9683fNhaYXA0AAG0P4SYITRrUVZFhVu045NT2g6VmlwMAQJtCuAlCCdGRuqJfsiTmvAEA4PsIN0HqxyduTS1nMU0AABoh3ASpH/TupNQ4u0qrarVqZ7HZ5QAA0GYQboJUmNWia0+sN/X6JjoWAwDQgHATxK4bWn9r6pO9R3TgOItpAgAgEW6CWnrHKGX37CjDkP6++aDZ5QAA0CYQboLcj4fV35p6Y3MBi2kCACDCTdC7sn+qYm3hOnC8Suu+OWp2OQAAmI5wE+Q6RIbpPwZ1kUTHYgAAJMJNSGhYTPO97UUqraw1uRoAAMxFuAkBWd3i1Cf5xGKaX7CYJgCgfSPchACLxeKdsfh1FtMEALRzhJsQMWlQF0WEWbTtYKl2HnKaXQ4AAKYh3ISIjjE2je1bv5jmG5tpvQEAtF+EmxDy3cU0XXUspgkAaJ8INyHkkvM6K8VhV0llrT7YedjscgAAMAXhJoSEWS2aMrSrJOmZj/aqps5jckUAAAQe4SbEzBjZQwlREdpV6NTTq/eYXQ4AAAFnariZP3++hg0bptjYWCUlJWnSpEnavXv3WY974403lJmZKbvdrgsuuEDvvvtuAKoNDp1jbfrtpAskSc/m7FVu/nGTKwIAILBMDTdr1qzRrFmztH79eq1atUq1tbUaN26cKioqznjM2rVrNXXqVN16663Kzc3VpEmTNGnSJG3fvj2AlbdtE7JSNXFQF3kM6Z7Xt6qqhs7FAID2w2IYRptZSvrbb79VUlKS1qxZo0suueS0+1x//fWqqKjQO++84902YsQIDRo0SM8999xZP8PpdCouLk6lpaVyOBw+q72tKa2s1bjfr1Gx06UZI7vrwf/ob3ZJAAC0Wku+v9tUn5vS0lJJUmJi4hn3WbduncaOHdto2/jx47Vu3brT7u9yueR0Ohs92oO4qAg9OiVLkrR47X6t3XvE5IoAAAiMNhNuPB6P7rrrLo0aNUoDBgw4435FRUVKTk5utC05OVlFRUWn3X/+/PmKi4vzPtLS0nxad1t2aZ8k/WR4uiTpl8u+kLOaRTUBAKGvzYSbWbNmafv27Vq6dKlP33fu3LkqLS31PgoK2tfsvb+5uq/SE6N0sKRKD7+90+xyAADwuzYRbmbPnq133nlHH330kbp169bkvikpKSouLm60rbi4WCkpKafd32azyeFwNHq0J9G2cP3fdQNlsUhvbD6gD3YWn/0gAACCmKnhxjAMzZ49W8uXL9eHH36oHj16nPWY7OxsrV69utG2VatWKTs7219lBr2LeiRq5sU9JUm/enObjlXUmFwRAAD+Y2q4mTVrll555RUtWbJEsbGxKioqUlFRkaqqqrz7TJs2TXPnzvU+nzNnjv71r3/piSee0JdffqkHH3xQmzZt0uzZs804haBx9xXn6/zkGB0pd+l/VmxTGxokBwCAT5kabhYuXKjS0lJdeumlSk1N9T7+9re/effJz89XYWGh9/nIkSO1ZMkSPf/88xo4cKCWLVumFStWNNkJGZI9IkxP/niQwq0WvbutSP/YesjskgAA8Is2Nc9NILSXeW7O5KkP9uh3H3wlhz1cK38+WilxdrNLAgDgrIJ2nhv4352X9VJWtzg5q+t079+/4PYUACDkEG7amYgwq5788UBFhlu15qtvtWRDvtklAQDgU4Sbdqh3Uqz+e3wfSdIj/9ylvKNnXssLAIBgQ7hpp24Z1UPDeySqssat/3otVyWVDA8HAIQGwk07ZbVa9H/XDZTDHq6tB0o1eeFa5R+tNLssAADOGeGmHUtLjNLrt2erS5xd33xboWue/VS5+cfNLgsAgHNCuGnnMlMcWj5rlPp3cehoRY1ueH69/rX99IuQAgAQDAg3ULLDrtd/lq3L+nSWq86jO17drBf+/Q3DxAEAQYlwA0n1C2z+adqFunF4ugxD+u0/d+mht3fK7SHgAACCC+EGXuFhVv120gD9+upMSdLitfv1s5c3qbKmzuTKAABoPsINGrFYLPrpJb307I1DZAu36oNdh3X9ovU6XFZtdmkAADQL4QandfUFqVoyc4QSoyO17WCprvnjWn1VXGZ2WQAAnBXhBmc0NCNBy+8cqR6donWwpEpTFq7V2r1HzC4LAIAmEW7QpIyO0XrzjpEa1j1BZdV1uunPn+nu17ewZAMAoM0i3OCsEqIj9fKtwzV5SFd5DOnNzw/q8ifW6N5lX+jAcWY1BgC0LRajnU1m4nQ6FRcXp9LSUjkcDrPLCTpfHCjRk6u+Us7ubyVJEWEW/fjCNM2+vLdS4zqYXB0AIFS15PubcINW2Zx3XL9b9ZU+OdEHJzLMqp8MT9edl/ZSksNucnUAgFBDuGkC4ca3PvvmqJ5c9ZU+23dMkmQLt+r/jcjQ7Zf2UqcYm8nVAQBCBeGmCYQb3zMMQ+u+PqonVn2lzXn1C292iAjT9JHd9bNLeiohOtLkCgEAwY5w0wTCjf8YhqGP9xzRkyt3a+uBUklSrC1cPxvdUzeP6qFoW7jJFQIAghXhpgmEG/8zDEOrdx3WE6u+0q5CpySpU0yk/vPy8zT1onRFhjNIDwDQMoSbJhBuAsfjMfT2F4f05KqvlHe0fsh4t4QOuvuK8zVxUFeFWS0mVwgACBaEmyYQbgKv1u3R3zYW6KnVe/RtmUuS1Cc5Vr8Y30dj+ybJYiHkAACaRrhpAuHGPFU1bi1eu18Lc/bKWV2/0viQ9Hj995WZGtGzo8nVAQDaMsJNEwg35iutrNWij7/WXz7dp+pajyRp9PmdNfvy3rowI4GWHADAKQg3TSDctB2HndX6w4d79dqGfNV56v817NU5WjcMS9fkIV3VkXlyAAAnEG6aQLhpe/KOVuiPH+3V21sLVVXrllS/rMO4fim6fliaftC7k6x0PgaAdo1w0wTCTdtVVl2rt7cWaunGfH1xYp4cSeoa30HXD0vTdRd2Y/0qAGinCDdNINwEhx2HSvX6xgItzz3o7XxstUiX9knS9cPSdHlmkiLCmC8HANoLwk0TCDfBpbrWrfe2F2rphgLv+lVS/czHg9LjNTQjQRdmJGpQerximAEZAEIW4aYJhJvg9c235frbpgL9ffMBHSmvafSa1SL1SXHowowEDT3x6JbQgZFXABAiCDdNINwEP7fH0O6iMm3OO6bNece1Ke+4DhyvOmW/pFibLuyeoIu6J+qHA7uwSjkABDHCTRMIN6Gp2FmtzXnHvWFnx8FS7/By6cToq/4p+slF6cru2ZHRVwAQZAg3TSDctA/VtW5tLSjRprzjWrmjyLtKuSSlJ0bphovSdO3QbkqKtZtYJQCguQg3TSDctE/bD5Zq6cZ8rcg9pHJX/eircKtFV/RL1tSL0plLBwDaOMJNEwg37VtlTZ3e+aJQr23IV25+iXd7WmIH3TAsXdcN7aYkB605ANDWEG6aQLhBg12FTi3dkK83cw+q7MRcOmFWi8b3T9aMkT00rDvrXAFAW0G4aQLhBt9XVePWP7cVaumGfG3KO+7d3jfVoRkjMzRxUFfZI8JMrBAAQLhpAuEGTdlV6NRf1+3X8tyD3hXL46MidP2wNP2/ERnqlhDV4vd0Vtdq5yGnauo86tfFwZB0AGgFwk0TCDdojpLKGr2+qUB/XZfnnUPHapHG9k3WjJHdld2r42lvWR0uq9aOg07tOFSqHYec2nHIqfxjlY32SXHYNaCrQ/27xGlA1zgN6OpQisPOLTAAaALhpgmEG7SE22No9a5ivbRuvz7de9S7/fzkGE3L7q7E6MhGQebbMtdp36drfAfZwq3ad7RCp/sb1zE6Uv27xmlAF4cGdI3T+cmxiraFKSLMqogwqyLDrIoIsyjMaiEEAWiXgibcfPzxx3r88ce1efNmFRYWavny5Zo0adIZ98/JydFll112yvbCwkKlpKQ06zMJN2itPcVlemndfr35+UFV1rhPu4/FIvXsFK3+XeLU/0RQ6ZfqUEJ0pCSp3FWnXYVObT9Yqu0nWnj2HC6X29O8v4YWixqFnYgwq+wRYRrfP1mzLuut+KhIn50vALQlLfn+NnWlwYqKCg0cOFC33HKLJk+e3Ozjdu/e3ejEkpKS/FEe0Mh5ybH67aQL9MvxmVq2+YCWbT6gMKvUPzVO/bs61L+LQ5kpDkU3sYBnjC1cw7onalj3RO+26lq3dheVafuhk4Fn37cVctV5VOP2NDreMKSaOo9q6hpv/9O/9+lvGws0+/LempbdnQ7QANq1NnNbymKxNLvl5vjx44qPj2/V59Byg2BiGIbqPIZq3R7V1hmqcXvqfz7xqKkzlH+sUr//4Ct9WVQmqf4W2D3jztekQV3PaWLCWrdHVbVuOewRvjodAGi1oGm5aa1BgwbJ5XJpwIABevDBBzVq1Kgz7utyueRynewH4XQ6A1Ei4BMWi8V7+0lnuOPUr4tDV/RL1pufH9CTq77SwZIq3f36Vr3w732ae3WmLj6vc7M/73hFjXK+OqwPdh7Wx199qzJXnXp2itbQjARd2D1BQzMS1atzNP1+ALRpQdVys3v3buXk5OjCCy+Uy+XSCy+8oJdfflmfffaZhgwZctpjHnzwQT300EOnbKflBqGoutatFz/dr2c/2quyE8tMXHxeJ/3qqkz17xJ32mO+/rZcq3cV64Odh7Up75jO1v0nISriRNhJ1IUZCRrQNY7bYAD8Lmg6FH9Xc8LN6YwePVrp6el6+eWXT/v66Vpu0tLSCDcIaccqavTMh3v18vr9qnUbslikawZ11d3jzleKw67Necf1wa5ird51WN8cqWh0bGZKrMb2TdaYvknq3jFauQXHtWl//WrrWwtK5Ppef5/IMKsu6BanCzMS1P9EB+oenaIVxlpdAHwo5G9LfddFF12kTz755Iyv22w22WxMmob2JTE6Uvf/qJ9mjOyux1fu1ttbD+nN3IN6Z1uhoiLDVFJZ6903IsyiET07egPN9ycqvDwzWZdnJkuq78y841CpNuc1BJ5jOlJeo815x7X5O7M72yOs6pPiUL9Uh/p1qf8zMyW2yc7WZ2IYhnf4vMUibokBOKugDzdbtmxRamqq2WUAbVJ6xyj9Yepgzby4h/733V1a/80x1dR5lBAVocv6JGlsv2RdfF4nxTaz03BkuFWD0xM0OD1Bt11cHzzyjlZqU95x5eYf185Cp74sLFNVrVtbC0q0taDEe6zFInXvGK1+qQ71TY1VRJhV5a66+kd1nSpq6lRWXaeKE9sqXG6VVdeqosZ92qHyFotkUX3YsXifWxRtC9PQjERl9+qoET0T1TfFwYrvQDtj6m2p8vJy7d27V5I0ePBgPfnkk7rsssuUmJio9PR0zZ07VwcPHtRf//pXSdLvf/979ejRQ/3791d1dbVeeOEF/eEPf9DKlSs1ZsyYZn0mo6XQXhmGoS0FJXJ7DA1Ki1d4mNUvn+P2GMo7WqGdhU7tPOTUzkKndhU6Vew8/QSH/hYfFaHhPRKV3bOjsnt10vnJMbT+AEEoaG5Lbdq0qdGkfHfffbckafr06Vq8eLEKCwuVn5/vfb2mpkb33HOPDh48qKioKGVlZemDDz447cR+ABqzWCwanJ7g988Js1rUs3OMenaO0Q+zuni3Hyl3adeJwLO7uH7YeowtXDG2cEXbwhVrD1d0ZLhi7OGKPbEtxl7/ekSYtf72lOrn+jFk6MQ/3ufGieeHndVa/80xrf/mqDbuP6aSylq9v6NY7+8ollQ/G/Twng1hp6N6dorxecuOYRhy1Xnk9tQP5a9zn/z55J+eE68ZskdY1aszoQvwlTbToThQaLkB2o9at0dfHCjV+m+Oav03R7Vp/3FV1TaeXTrGFu7tG9Qwq3TvpJj64ffNUF3r1p7i8vrgdqKValehU87quhbV2jspRjcMS9PkId2UGM1M08D3BeVoqUAh3ADtV02dR18cKNG6r49q3TdHtTnv+Cmjv6T6vkV9kmPVv4tD/bvWL6XRN8XhXT7ju0Hm628rmrV8htUihVutCrNaFG61KCzMonCrVeFWi45X1njriAizaFz/FE0dlq6RvTrSXwg4gXDTBMINgAZ1bo++/rbCu/jp9oOl2lnoVFkLW13ioyLUN6W+9afviQ7TaYlRigyrDzNhFkuTIaWsulb/2HpISzcUaNvBUu/29MQoXT8sTdcN7aYkh73V52m2OrcnKBZ9LXfVKf9opfKPVajgWJUMGfW3R0+5bRqhaFuYYuzhsoUzx1OgEG6aQLgB0BTDMFRwrErbD5V+J/Q4daTcJYtF6tEx2htg+p64nZXisPvsi3v7wVIt3Zivt3IPeSdiDLNadFmfJE29KE2jz+/cqDO4YRiqqHGrvLpOZdW1clbXjzYrq65VeXWdLBYp5sSXcaw93Ptl3dDXqbm335qrYQTd5/nH6x95JfqyyKmkWLtG9e6kUb07alTvTko2Iax5PIa+LXcp72il8o9VKv9ohfKONfxcqaMVNS1+z8gwq6JtYUp22DU4PUFDM+of3TtGtfkwF2wIN00g3ABojSPlLnWICGvVXD2tUVXj1j+3FWrphnxt+s4cQkmxNiVGR6rsRJgpd9WddVbpptjCrfVhxx6u+A4RSomzKzWug7rEN/4zKdZ22hF2lTV12lpQqs/z66cDyM0vaVZI6J0Uox/07qSRvTpqRK+OLVrDzDAMlVbVqtjp0rdlLjmra+WsqpWzulalVbVyVtV5t5VW1Qc+Z1WtSqpqT1l09vsSoiKU3jFa6YlRirBaVPadqQrKTwTHcledKmvcTb5PYnSkhqTHa0hGgoakJ2hgt3h1iGx5K49hGEEXkmrdHlW63IqL8u26dISbJhBuAASbPcVlWrqxQG9+fkDHvzMB43eFWS2KtdffPom1RXhHnRlSo7mEGr6gT9fXqClWi5TssCs1zq7U+A6KtYVr28FSfVlUdkqfo8gwqwZ0dWhIeoKGZCTogq5xyjtaqU/2HtHar49o28FSffebx2qRsrrFe1t1UuM6qNhZ/Z2HS0XOah0+8XOxs7rF9X/399Ql3q6MxGilJUYpo2OUMhKjlJYYpfSOUc0OWW6P0ej3ue9IhT4/MZnlFwdLTwlR4VaL+nWp/530TY2Vq86jsur6EFZ2Inw1BNb6P+tfq6p1q3vHaA1Ki9fg9HgNSotXZopDkeH+mcrhXHxb5tJrG/L16md5ujwzWfMnX+DT9yfcNIFwAyBYuerc2rDvmCQp1h6hGFu4HPZwxdojZI+wtuj/8GvdHlW4TkyceOJL+lhFjQpLq3WotEqFJdUqLK3SoZL6gFHXRPNQapxdQ9ITNPhES0X/Lo4m+6KUVNZo/TdH68PO3qOnLAHSXAlREeoUY1N8VIQc9gg5OkQorkOEHPZwOTrUP6/fHn5ie33LlK9vxX3fd2fyzs0v0aa8Yz6d58kWbtWArnEanBavQenxGpyeoC5xvrs12lJbCkr00tr9+ucXhapx14e69MQofXjPaJ/Op0W4aQLhBgBaxu0xdKTcpUMlVfXhp6RKpVW16pMSqyHpCeoS3+Gc3v9QSZU+3XtEa78+qk/3HlFZdZ1S4uxKirUpJc6uZEfDw6Zkh10pDrs6x9qCZsFWwzB0qLTa27LzzZEKRUeGnWhpi1CsPVyOE3/G2iO8gTXWHi5bhFW7i8qUm1+iLQX1j9KqU1vvOsfaNDgtXhkdoxQVGa5oW5g6RIYrOjJMUZFhiooMb/ynLUxxHSJa3SHaVefWu9sKtXhtXqOZyAelxWvGyO66+oJUn7cuEW6aQLgBAAQrwzC070iFN+zkFhzXl4VlTbasnYn1xJIo5yfH6vyUWPVJjlWflBhldIw+Y+tWsbNar67P05IN+TpSXt+3KjLMqh9mpWr6yO4amBZ/LqfXJMJNEwg3AIBQUlXj1vZDpdpaUKLDZS5VuOpUVeNWRU19x+eTjxPPXXWqrHXrTN/+kWFW9ewcrT4psTo/uT70dIgM02sb8vWv7UXeIJXisOumEem64aJ0dYrx/wLVhJsmEG4AAO2dYdQPi99TXK4vi8r0VVGZdheXaU9xmSrOMhLsou6Jmj6yu8b1T/Z7/6XvCpq1pQAAQOBZLBYlxdq98w818HgMHSyp0lfF9WGnPvSU69uyao3JTNb0kd3Vr0vbbxgg3AAAAEmS1WpR2omh8WP6JptdTqu1vYHyAAAA54BwAwAAQgrhBgAAhBTCDQAACCmEGwAAEFIINwAAIKQQbgAAQEgh3AAAgJBCuAEAACGFcAMAAEIK4QYAAIQUwg0AAAgphBsAABBSCDcAACCkhJtdQKAZhiFJcjqdJlcCAACaq+F7u+F7vCntLtyUlZVJktLS0kyuBAAAtFRZWZni4uKa3MdiNCcChRCPx6NDhw4pNjZWFovFp+/tdDqVlpamgoICORwOn753WxDq5yeF/jlyfsEv1M+R8wt+/jpHwzBUVlamLl26yGptuldNu2u5sVqt6tatm18/w+FwhOy/tFLon58U+ufI+QW/UD9Hzi/4+eMcz9Zi04AOxQAAIKQQbgAAQEgh3PiQzWbTAw88IJvNZnYpfhHq5yeF/jlyfsEv1M+R8wt+beEc212HYgAAENpouQEAACGFcAMAAEIK4QYAAIQUwg0AAAgphBsf+eMf/6ju3bvLbrdr+PDh2rBhg9kl+cyDDz4oi8XS6JGZmWl2Wa328ccf60c/+pG6dOkii8WiFStWNHrdMAzdf//9Sk1NVYcOHTR27Fjt2bPHnGJb6WznOGPGjFOu6ZVXXmlOsa0wf/58DRs2TLGxsUpKStKkSZO0e/fuRvtUV1dr1qxZ6tixo2JiYjRlyhQVFxebVHHLNOf8Lr300lOu4e23325SxS2zcOFCZWVleSd5y87O1nvvved9PZivXYOznWMwX7/TWbBggSwWi+666y7vNjOvI+HGB/72t7/p7rvv1gMPPKDPP/9cAwcO1Pjx43X48GGzS/OZ/v37q7Cw0Pv45JNPzC6p1SoqKjRw4ED98Y9/PO3rjz32mJ5++mk999xz+uyzzxQdHa3x48eruro6wJW23tnOUZKuvPLKRtf0tddeC2CF52bNmjWaNWuW1q9fr1WrVqm2tlbjxo1TRUWFd5+f//znevvtt/XGG29ozZo1OnTokCZPnmxi1c3XnPOTpJkzZza6ho899phJFbdMt27dtGDBAm3evFmbNm3S5ZdfrokTJ2rHjh2SgvvaNTjbOUrBe/2+b+PGjVq0aJGysrIabTf1Oho4ZxdddJExa9Ys73O322106dLFmD9/volV+c4DDzxgDBw40Owy/EKSsXz5cu9zj8djpKSkGI8//rh3W0lJiWGz2YzXXnvNhArP3ffP0TAMY/r06cbEiRNNqccfDh8+bEgy1qxZYxhG/TWLiIgw3njjDe8+u3btMiQZ69atM6vMVvv++RmGYYwePdqYM2eOeUX5WEJCgvHCCy+E3LX7roZzNIzQuX5lZWXGeeedZ6xatarROZl9HWm5OUc1NTXavHmzxo4d691mtVo1duxYrVu3zsTKfGvPnj3q0qWLevbsqRtvvFH5+flml+QX+/btU1FRUaPrGRcXp+HDh4fU9ZSknJwcJSUlqU+fPrrjjjt09OhRs0tqtdLSUklSYmKiJGnz5s2qra1tdB0zMzOVnp4elNfx++fX4NVXX1WnTp00YMAAzZ07V5WVlWaUd07cbreWLl2qiooKZWdnh9y1k049xwahcP1mzZqlCRMmNLpekvl/B9vdwpm+duTIEbndbiUnJzfanpycrC+//NKkqnxr+PDhWrx4sfr06aPCwkI99NBDuvjii7V9+3bFxsaaXZ5PFRUVSdJpr2fDa6Hgyiuv1OTJk9WjRw99/fXX+vWvf62rrrpK69atU1hYmNnltYjH49Fdd92lUaNGacCAAZLqr2NkZKTi4+Mb7RuM1/F05ydJP/nJT5SRkaEuXbroiy++0L333qvdu3frzTffNLHa5tu2bZuys7NVXV2tmJgYLV++XP369dOWLVtC5tqd6Ryl4L9+krR06VJ9/vnn2rhx4ymvmf13kHCDs7rqqqu8P2dlZWn48OHKyMjQ66+/rltvvdXEytBaN9xwg/fnCy64QFlZWerVq5dycnI0ZswYEytruVmzZmn79u1B3Q+sKWc6v5/+9Kfeny+44AKlpqZqzJgx+vrrr9WrV69Al9liffr00ZYtW1RaWqply5Zp+vTpWrNmjdll+dSZzrFfv35Bf/0KCgo0Z84crVq1Sna73exyTsFtqXPUqVMnhYWFndIDvLi4WCkpKSZV5V/x8fE6//zztXfvXrNL8bmGa9aerqck9ezZU506dQq6azp79my98847+uijj9StWzfv9pSUFNXU1KikpKTR/sF2Hc90fqczfPhwSQqaaxgZGanevXtr6NChmj9/vgYOHKinnnoqZK6ddOZzPJ1gu36bN2/W4cOHNWTIEIWHhys8PFxr1qzR008/rfDwcCUnJ5t6HQk35ygyMlJDhw7V6tWrvds8Ho9Wr17d6N5qKCkvL9fXX3+t1NRUs0vxuR49eiglJaXR9XQ6nfrss89C9npK0oEDB3T06NGguaaGYWj27Nlavny5PvzwQ/Xo0aPR60OHDlVERESj67h7927l5+cHxXU82/mdzpYtWyQpaK7h93k8HrlcrqC/dk1pOMfTCbbrN2bMGG3btk1btmzxPi688ELdeOON3p9NvY5+77LcDixdutSw2WzG4sWLjZ07dxo//elPjfj4eKOoqMjs0nzinnvuMXJycox9+/YZn376qTF27FijU6dOxuHDh80urVXKysqM3NxcIzc315BkPPnkk0Zubq6Rl5dnGIZhLFiwwIiPjzfeeust44svvjAmTpxo9OjRw6iqqjK58uZr6hzLysqMX/ziF8a6deuMffv2GR988IExZMgQ47zzzjOqq6vNLr1Z7rjjDiMuLs7IyckxCgsLvY/KykrvPrfffruRnp5ufPjhh8amTZuM7OxsIzs728Sqm+9s57d3715j3rx5xqZNm4x9+/YZb731ltGzZ0/jkksuMbny5vnVr35lrFmzxti3b5/xxRdfGL/61a8Mi8VirFy50jCM4L52DZo6x2C/fmfy/RFgZl5Hwo2P/OEPfzDS09ONyMhI46KLLjLWr19vdkk+c/311xupqalGZGSk0bVrV+P666839u7da3ZZrfbRRx8Zkk55TJ8+3TCM+uHg9913n5GcnGzYbDZjzJgxxu7du80tuoWaOsfKykpj3LhxRufOnY2IiAgjIyPDmDlzZlCF8dOdmyTjxRdf9O5TVVVl3HnnnUZCQoIRFRVlXHPNNUZhYaF5RbfA2c4vPz/fuOSSS4zExETDZrMZvXv3Nn75y18apaWl5hbeTLfccouRkZFhREZGGp07dzbGjBnjDTaGEdzXrkFT5xjs1+9Mvh9uzLyOFsMwDP+3DwEAAAQGfW4AAEBIIdwAAICQQrgBAAAhhXADAABCCuEGAACEFMINAAAIKYQbAAAQUgg3AAAgpBBuALRJ3377re644w6lp6fLZrMpJSVF48eP16effipJslgsWrFihblFAmiTws0uAABOZ8qUKaqpqdFLL72knj17qri4WKtXr9bRo0fNLg1AG8fyCwDanJKSEiUkJCgnJ0ejR48+5fXu3bsrLy/P+zwjI0P79++XJL311lt66KGHtHPnTnXp0kXTp0/Xb37zG4WH1/+/nMVi0bPPPqt//OMfysnJUWpqqh577DFde+21ATk3AP7HbSkAbU5MTIxiYmK0YsUKuVyuU17fuHGjJOnFF19UYWGh9/m///1vTZs2TXPmzNHOnTu1aNEiLV68WI888kij4++77z5NmTJFW7du1Y033qgbbrhBu3bt8v+JAQgIWm4AtEl///vfNXPmTFVVVWnIkCEaPXq0brjhBmVlZUmqb4FZvny5Jk2a5D1m7NixGjNmjObOnevd9sorr+i///u/dejQIe9xt99+uxYuXOjdZ8SIERoyZIieffbZwJwcAL+i5QZAmzRlyhQdOnRI//jHP3TllVcqJydHQ4YM0eLFi894zNatWzVv3jxvy09MTIxmzpypwsJCVVZWevfLzs5udFx2djYtN0AIoUMxgDbLbrfriiuu0BVXXKH77rtPt912mx544AHNmDHjtPuXl5froYce0uTJk0/7XgDaB1puAASNfv36qaKiQpIUEREht9vd6PUhQ4Zo9+7d6t279ykPq/Xkf+7Wr1/f6Lj169erb9++/j8BAAFByw2ANufo0aO67rrrdMsttygrK0uxsbHatGmTHnvsMU2cOFFS/Yip1atXa9SoUbLZbEpISND999+vH/7wh0pPT9e1114rq9WqrVu3avv27frtb3/rff833nhDF154oX7wgx/o1Vdf1YYNG/TnP//ZrNMF4GN0KAbQ5rhcLj344INauXKlvv76a9XW1iotLU3XXXedfv3rX6tDhw56++23dffdd2v//v3q2rWrdyj4+++/r3nz5ik3N1cRERHKzMzUbbfdppkzZ0qq71D8xz/+UStWrNDHH3+s1NRUPfroo/rxj39s4hkD8CXCDYB25XSjrACEFvrcAACAkEK4AQAAIYUOxQDaFe7EA6GPlhsAABBSCDcAACCkEG4AAEBIIdwAAICQQrgBAAAhhXADAABCCuEGAACEFMINAAAIKYQbAAAQUv4/4xPDxD+s8CYAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjcsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvTLEjVAAAAAlwSFlzAAAPYQAAD2EBqD+naQAASkVJREFUeJzt3Xl4VPX99vF7ssxkXwhkD2GVsAVZRAMqVEBEtKDWKtUH0GpbxBZqF6WtG9YGtfZXrYqgVtwoFSpYqQiIAiogi+xgAIEQIAtb9mSSzJznj5DRSAgQZuYkk/fruuYyc+acmc9JaHL3u1oMwzAEAADgI/zMLgAAAMCdCDcAAMCnEG4AAIBPIdwAAACfQrgBAAA+hXADAAB8CuEGAAD4FMINAADwKYQbAADgUwg3ADxu4sSJ6tChQ5Oufeyxx2SxWNxbEACfRrgBWjGLxXJej5UrV5pdqikmTpyosLAws8sAcIEs7C0FtF5vv/12vedvvvmmli9frrfeeqve8REjRiguLq7Jn1NdXS2n0ymbzXbB19bU1KimpkZBQUFN/vymmjhxohYsWKDS0lKvfzaApgswuwAA5rnzzjvrPV+3bp2WL19+xvHvKy8vV0hIyHl/TmBgYJPqk6SAgAAFBPCrCsD5o1sKQKOGDh2qXr16adOmTbr66qsVEhKiP/zhD5Kk999/X6NHj1ZiYqJsNps6d+6sJ554Qg6Ho957fH/MzcGDB2WxWPTXv/5Vs2fPVufOnWWz2XTZZZdpw4YN9a5taMyNxWLR/fffr0WLFqlXr16y2Wzq2bOnPvroozPqX7lypQYMGKCgoCB17txZs2bNcvs4nvnz56t///4KDg5W27Ztdeedd+rIkSP1zsnLy9Ndd92l5ORk2Ww2JSQkaMyYMTp48KDrnI0bN2rkyJFq27atgoOD1bFjR919991uqxNoLfi/QwDO6cSJExo1apRuv/123Xnnna4uqjlz5igsLEwPPPCAwsLC9Mknn+iRRx5RcXGxnnnmmXO+79y5c1VSUqKf//znslgsevrpp3XzzTdr//7952zt+fzzz/Xee+/pvvvuU3h4uJ5//nndcsstOnTokGJiYiRJmzdv1nXXXaeEhAQ9/vjjcjgcmj59utq1a3fx35TT5syZo7vuukuXXXaZMjMzlZ+fr+eee05ffPGFNm/erKioKEnSLbfcop07d+qXv/ylOnTooIKCAi1fvlyHDh1yPb/22mvVrl07PfTQQ4qKitLBgwf13nvvua1WoNUwAOC0yZMnG9//tTBkyBBDkvHyyy+fcX55efkZx37+858bISEhRmVlpevYhAkTjNTUVNfzAwcOGJKMmJgY4+TJk67j77//viHJ+OCDD1zHHn300TNqkmRYrVZj3759rmNbt241JBn/+Mc/XMduvPFGIyQkxDhy5Ijr2N69e42AgIAz3rMhEyZMMEJDQ8/6elVVlREbG2v06tXLqKiocB1fvHixIcl45JFHDMMwjFOnThmSjGeeeeas77Vw4UJDkrFhw4Zz1gWgcXRLATgnm82mu+6664zjwcHBrq9LSkp0/PhxXXXVVSovL9fXX399zve97bbbFB0d7Xp+1VVXSZL2799/zmuHDx+uzp07u56np6crIiLCda3D4dDHH3+ssWPHKjEx0XVely5dNGrUqHO+//nYuHGjCgoKdN9999Ub8Dx69GilpaXpf//7n6Ta75PVatXKlSt16tSpBt+rroVn8eLFqq6udkt9QGtFuAFwTklJSbJarWcc37lzp2666SZFRkYqIiJC7dq1cw1GLioqOuf7tm/fvt7zuqBztgDQ2LV119ddW1BQoIqKCnXp0uWM8xo61hTZ2dmSpG7dup3xWlpamut1m82mp556SkuWLFFcXJyuvvpqPf3008rLy3OdP2TIEN1yyy16/PHH1bZtW40ZM0avv/667Ha7W2oFWhPCDYBz+m4LTZ3CwkINGTJEW7du1fTp0/XBBx9o+fLleuqppyRJTqfznO/r7+/f4HHjPFaouJhrzTB16lTt2bNHmZmZCgoK0sMPP6zu3btr8+bNkmoHSS9YsEBr167V/fffryNHjujuu+9W//79mYoOXCDCDYAmWblypU6cOKE5c+ZoypQpuuGGGzR8+PB63Uxmio2NVVBQkPbt23fGaw0da4rU1FRJUlZW1hmvZWVluV6v07lzZ/3mN7/RsmXLtGPHDlVVVenZZ5+td84VV1yhJ598Uhs3btQ777yjnTt3at68eW6pF2gtCDcAmqSu5eS7LSVVVVV66aWXzCqpHn9/fw0fPlyLFi3S0aNHXcf37dunJUuWuOUzBgwYoNjYWL388sv1uo+WLFmi3bt3a/To0ZJq1wWqrKysd23nzp0VHh7uuu7UqVNntDpdeumlkkTXFHCBmAoOoEkGDRqk6OhoTZgwQb/61a9ksVj01ltvNatuoccee0zLli3T4MGDNWnSJDkcDr3wwgvq1auXtmzZcl7vUV1drT//+c9nHG/Tpo3uu+8+PfXUU7rrrrs0ZMgQjRs3zjUVvEOHDvr1r38tSdqzZ4+GDRumH//4x+rRo4cCAgK0cOFC5efn6/bbb5ckvfHGG3rppZd00003qXPnziopKdErr7yiiIgIXX/99W77ngCtAeEGQJPExMRo8eLF+s1vfqM//elPio6O1p133qlhw4Zp5MiRZpcnSerfv7+WLFmi3/72t3r44YeVkpKi6dOna/fu3ec1m0uqbY16+OGHzzjeuXNn3XfffZo4caJCQkI0Y8YMPfjggwoNDdVNN92kp556yjUDKiUlRePGjdOKFSv01ltvKSAgQGlpaXr33Xd1yy23SKodULx+/XrNmzdP+fn5ioyM1MCBA/XOO++oY8eObvueAK0Be0sBaHXGjh2rnTt3au/evWaXAsADGHMDwKdVVFTUe7537159+OGHGjp0qDkFAfA4Wm4A+LSEhARNnDhRnTp1UnZ2tmbOnCm73a7Nmzera9euZpcHwAMYcwPAp1133XX617/+pby8PNlsNmVkZOgvf/kLwQbwYbTcAAAAn8KYGwAA4FMINwAAwKe0ujE3TqdTR48eVXh4uCwWi9nlAACA82AYhkpKSpSYmCg/v8bbZlpduDl69KhSUlLMLgMAADRBTk6OkpOTGz2n1YWb8PBwSbXfnIiICJOrAQAA56O4uFgpKSmuv+ONaXXhpq4rKiIignADAEALcz5DShhQDAAAfArhBgAA+BTCDQAA8CmEGwAA4FMINwAAwKcQbgAAgE8h3AAAAJ9CuAEAAD6FcAMAAHwK4QYAAPgUwg0AAPAphBsAAOBTWt3GmZ5SVePUiTK7HE5DydEhZpcDAECrRcuNm2w+dEoZmZ9o/GvrzS4FAIBWjXDjJiHW2kaw8iqHyZUAANC6EW7cJNjqL0kqr6oxuRIAAFo3wo2bhJwONxXVtNwAAGAmwo2b1IWbaoehaofT5GoAAGi9CDduUtctJdF6AwCAmQg3bmL195OfpfbrCgYVAwBgGsKNm1gsFmZMAQDQDBBu3IgZUwAAmI9w40auGVO03AAAYBrCjRsFB9a13BBuAAAwC+HGjVjrBgAA8xFu3CiYbikAAExHuHGj4EBmSwEAYDbCjRuFMFsKAADTEW7ciNlSAACYj3DjRq51bhhQDACAaQg3bkTLDQAA5iPcuFHdOjeEGwAAzNNsws2MGTNksVg0derUs54zZ84cWSyWeo+goCDvFXkOwXV7S9EtBQCAaQLMLkCSNmzYoFmzZik9Pf2c50ZERCgrK8v13GKxeLK0C/JttxSzpQAAMIvpLTelpaW644479Morryg6Ovqc51ssFsXHx7secXFxXqjy/Hw7FZyWGwAAzGJ6uJk8ebJGjx6t4cOHn9f5paWlSk1NVUpKisaMGaOdO3c2er7dbldxcXG9h6ewtxQAAOYzNdzMmzdPX331lTIzM8/r/G7duumf//yn3n//fb399ttyOp0aNGiQDh8+fNZrMjMzFRkZ6XqkpKS4q/wzhJwec1PJmBsAAExjWrjJycnRlClT9M4775z3oOCMjAyNHz9el156qYYMGaL33ntP7dq106xZs856zbRp01RUVOR65OTkuOsWzhBsrf120nIDAIB5TBtQvGnTJhUUFKhfv36uYw6HQ6tXr9YLL7wgu90uf3//Rt8jMDBQffv21b59+856js1mk81mc1vdjWFvKQAAzGdauBk2bJi2b99e79hdd92ltLQ0Pfjgg+cMNlJtGNq+fbuuv/56T5V5QZgtBQCA+UwLN+Hh4erVq1e9Y6GhoYqJiXEdHz9+vJKSklxjcqZPn64rrrhCXbp0UWFhoZ555hllZ2frnnvu8Xr9DQn5zvYLhmE0q2nqAAC0Fs1inZuzOXTokPz8vh0WdOrUKd17773Ky8tTdHS0+vfvrzVr1qhHjx4mVvmtur2lDEOy1zgVFHju1icAAOBeFsMwDLOL8Kbi4mJFRkaqqKhIERERbn3vGodTXf64RJK0+eERig61uvX9AQBorS7k77fp69z4kgB/P1n9T8+YYjo4AACmINy4WTCDigEAMBXhxs3YggEAAHMRbtwsmHADAICpCDdu9u1aN4QbAADMQLhxs5DTqxRXMKAYAABTEG7cjG4pAADMRbhxs+BAZksBAGAmwo2bMVsKAABzEW7cjG4pAADMRbhxM9dsKQYUAwBgCsKNmwVbT8+WouUGAABTEG7cjDE3AACYi3DjZq7ZUtXMlgIAwAyEGzdjQDEAAOYi3LgZ3VIAAJiLcONm7C0FAIC5CDduFnx6b6lyVigGAMAUhBs3q2u5qax2mlwJAACtE+HGzb4dUEzLDQAAZiDcuFndVHAGFAMAYA7CjZvVdUvZa5xyOA2TqwEAoPUh3LhZyOntFyT2lwIAwAyEGzcLCvSTxVL7NeNuAADwPsKNm1ksFte4m8oqZkwBAOBthBsPcA0qZn8pAAC8jnDjAewvBQCAeQg3HsAWDAAAmIdw4wHB1rotGAg3AAB4G+HGA0ICWaUYAACzEG484Nv9pWi5AQDA2wg3HsCAYgAAzEO48QD2lwIAwDyEGw9gthQAAOYh3HgAs6UAADAP4cYDXC03rFAMAIDXEW48gG4pAADMQ7jxAGZLAQBgnmYTbmbMmCGLxaKpU6c2et78+fOVlpamoKAg9e7dWx9++KF3CrwAdbOlKljnBgAAr2sW4WbDhg2aNWuW0tPTGz1vzZo1GjdunH76059q8+bNGjt2rMaOHasdO3Z4qdLzE0LLDQAApjE93JSWluqOO+7QK6+8oujo6EbPfe6553Tdddfpd7/7nbp3764nnnhC/fr10wsvvOClas8Ps6UAADCP6eFm8uTJGj16tIYPH37Oc9euXXvGeSNHjtTatWvPeo3dbldxcXG9h6d9O6CY2VIAAHhbgJkfPm/ePH311VfasGHDeZ2fl5enuLi4esfi4uKUl5d31msyMzP1+OOPX1SdF4oxNwAAmMe0lpucnBxNmTJF77zzjoKCgjz2OdOmTVNRUZHrkZOT47HPqsOYGwAAzGNay82mTZtUUFCgfv36uY45HA6tXr1aL7zwgux2u/z9/etdEx8fr/z8/HrH8vPzFR8ff9bPsdlsstls7i3+HIJZ5wYAANOY1nIzbNgwbd++XVu2bHE9BgwYoDvuuENbtmw5I9hIUkZGhlasWFHv2PLly5WRkeGtss9LSGBtZqxxGqqqcZpcDQAArYtpLTfh4eHq1atXvWOhoaGKiYlxHR8/frySkpKUmZkpSZoyZYqGDBmiZ599VqNHj9a8efO0ceNGzZ492+v1N6au5Uaqbb2xBpg+bhsAgFajWf/VPXTokHJzc13PBw0apLlz52r27Nnq06ePFixYoEWLFp0RksxmDfBTgJ9FklTO/lIAAHiVxTAMw+wivKm4uFiRkZEqKipSRESExz6n92NLVVJZoxW/GaLO7cI89jkAALQGF/L3u1m33LRkbJ4JAIA5CDcewlo3AACYg3DjIWzBAACAOQg3HsIWDAAAmINw4yGsUgwAgDkINx5SN+aGcAMAgHcRbjykruWmkgHFAAB4FeHGQxhQDACAOQg3HkK3FAAA5iDceAizpQAAMAfhxkOCmS0FAIApCDce4poKzoBiAAC8inDjIa7ZUrTcAADgVYQbD2G2FAAA5iDceIhrthTdUgAAeBXhxkOYLQUAgDkINx7CbCkAAMxBuPGQb1tuCDcAAHgT4cZDQgJrBxRXMOYGAACvItx4SF23VEW1Q4ZhmFwNAACtB+HGQ+rCjWFIldVOk6sBAKD1INx4SN1UcEkqZ8YUAABeQ7jxEH8/i2wBtd9eZkwBAOA9hBsPCvnOuBsAAOAdhBsPCjm9BQPTwQEA8B7CjQexkB8AAN5HuPGgukHFFdUMKAYAwFsINx5Eyw0AAN5HuPGgEMINAABeR7jxIPaXAgDA+wg3HhR8en8pWm4AAPAewo0Hsc4NAADeR7jxINfmmWy/AACA1xBuPKhuKjjdUgAAeA/hxoMYUAwAgPcRbjyIqeAAAHgf4caDgk/vLVXOgGIAALyGcONBdS03lbTcAADgNaaGm5kzZyo9PV0RERGKiIhQRkaGlixZctbz58yZI4vFUu8RFBTkxYovjGv7BfaWAgDAawLM/PDk5GTNmDFDXbt2lWEYeuONNzRmzBht3rxZPXv2bPCaiIgIZWVluZ5bLBZvlXvBmC0FAID3mRpubrzxxnrPn3zySc2cOVPr1q07a7ixWCyKj4/3RnkXjdlSAAB4X7MZc+NwODRv3jyVlZUpIyPjrOeVlpYqNTVVKSkpGjNmjHbu3OnFKi8Ms6UAAPA+U1tuJGn79u3KyMhQZWWlwsLCtHDhQvXo0aPBc7t166Z//vOfSk9PV1FRkf76179q0KBB2rlzp5KTkxu8xm63y263u54XFxd75D4aUjdbipYbAAC8x/SWm27dumnLli368ssvNWnSJE2YMEG7du1q8NyMjAyNHz9el156qYYMGaL33ntP7dq106xZs876/pmZmYqMjHQ9UlJSPHUrZwg5PeamyuFUjcPptc8FAKA1Mz3cWK1WdenSRf3791dmZqb69Omj55577ryuDQwMVN++fbVv376znjNt2jQVFRW5Hjk5Oe4q/ZzqZktJbJ4JAIC3mB5uvs/pdNbrRmqMw+HQ9u3blZCQcNZzbDaba6p53cNbbAF+qpvMRdcUAADeYeqYm2nTpmnUqFFq3769SkpKNHfuXK1cuVJLly6VJI0fP15JSUnKzMyUJE2fPl1XXHGFunTposLCQj3zzDPKzs7WPffcY+ZtnJXFYlFIoL/KqhwMKgYAwEtMDTcFBQUaP368cnNzFRkZqfT0dC1dulQjRoyQJB06dEh+ft82Lp06dUr33nuv8vLyFB0drf79+2vNmjVnHYDcHARbAwg3AAB4kcUwDMPsIrypuLhYkZGRKioq8koX1dVPf6pDJ8v1n0kZ6p/axuOfBwCAL7qQv9/NbsyNr/l2IT9mSwEA4A2EGw9z7S9Vxf5SAAB4A+HGw+r2l2IqOAAA3kG48TC2YAAAwLsINx5WtwUD4QYAAO8g3HhY3RYMFYy5AQDAKwg3HlY3oJgxNwAAeAfhxsMYcwMAgHcRbjzMNVuKcAMAgFcQbjwsmJYbAAC8inDjYSHMlgIAwKsINx7m2n6hmtlSAAB4A+HGw+iWAgDAuwg3HvbtxpmEGwAAvIFw42EhrHMDAIBXEW48LCiQbikAALyJcONhdbOl6JYCAMA7CDce9u0KxTUyDMPkagAA8H2EGw+rmy3lNCR7jdPkagAA8H2EGw+r2xVckioZVAwAgMcRbjwswN9PVv/ab3MZ424AAPA4wo0XxIRZJUkFxZUmVwIAgO8j3HhBUlSwJOloIeEGAABPI9x4QeLpcHOksNzkSgAA8H2EGy9IiqblBgAAbyHceEFdy83hUxUmVwIAgO8j3HhBsmvMDeEGAABPI9x4wbdjbgg3AAB4GuHGCxKjgiRJRRXVKrXXmFwNAAC+rUnhJicnR4cPH3Y9X79+vaZOnarZs2e7rTBfEh4UqIig2g006ZoCAMCzmhRufvKTn+jTTz+VJOXl5WnEiBFav369/vjHP2r69OluLdBXJEWHSKJrCgAAT2tSuNmxY4cGDhwoSXr33XfVq1cvrVmzRu+8847mzJnjzvp8RtLprqkjzJgCAMCjmhRuqqurZbPZJEkff/yxfvjDH0qS0tLSlJub677qfEgSM6YAAPCKJoWbnj176uWXX9Znn32m5cuX67rrrpMkHT16VDExMW4t0FcwYwoAAO9oUrh56qmnNGvWLA0dOlTjxo1Tnz59JEn//e9/Xd1VqO/bVYoJNwAAeFJAUy4aOnSojh8/ruLiYkVHR7uO/+xnP1NISIjbivMlrpYbxtwAAOBRTWq5qaiokN1udwWb7Oxs/f3vf1dWVpZiY2PdWqCvqFulOK+4UjUOp8nVAADgu5oUbsaMGaM333xTklRYWKjLL79czz77rMaOHauZM2e6tUBf0TbMJqu/n5xGbcABAACe0aRw89VXX+mqq66SJC1YsEBxcXHKzs7Wm2++qeeff96tBfoKPz+LEk5PB2d3cAAAPKdJ4aa8vFzh4eGSpGXLlunmm2+Wn5+frrjiCmVnZ5/3+8ycOVPp6emKiIhQRESEMjIytGTJkkavmT9/vtLS0hQUFKTevXvrww8/bMotmCIxsm7GVLnJlQAA4LuaFG66dOmiRYsWKScnR0uXLtW1114rSSooKFBERMR5v09ycrJmzJihTZs2aePGjbrmmms0ZswY7dy5s8Hz16xZo3HjxumnP/2pNm/erLFjx2rs2LHasWNHU27D6+pmTDGoGAAAz2lSuHnkkUf029/+Vh06dNDAgQOVkZEhqbYVp2/fvuf9PjfeeKOuv/56de3aVZdccomefPJJhYWFad26dQ2e/9xzz+m6667T7373O3Xv3l1PPPGE+vXrpxdeeKEpt+F13651Q7cUAACe0qSp4D/60Y905ZVXKjc317XGjSQNGzZMN910U5MKcTgcmj9/vsrKylxh6fvWrl2rBx54oN6xkSNHatGiRWd9X7vdLrvd7npeXFzcpPrcIZmF/AAA8LgmhRtJio+PV3x8vGt38OTk5CYt4Ld9+3ZlZGSosrJSYWFhWrhwoXr06NHguXl5eYqLi6t3LC4uTnl5eWd9/8zMTD3++OMXXJcnJLIFAwAAHtekbimn06np06crMjJSqampSk1NVVRUlJ544gk5nRe2hku3bt20ZcsWffnll5o0aZImTJigXbt2NaWsBk2bNk1FRUWuR05Ojtve+0J9d8yNYRim1QEAgC9rUsvNH//4R7322muaMWOGBg8eLEn6/PPP9dhjj6myslJPPvnkeb+X1WpVly5dJEn9+/fXhg0b9Nxzz2nWrFlnnBsfH6/8/Px6x/Lz8xUfH3/W97fZbK5NPs2WEFk7Fbyi2qHC8mpFh1pNrggAAN/TpJabN954Q6+++qomTZqk9PR0paen67777tMrr7yiOXPmXFRBTqez3hiZ78rIyNCKFSvqHVu+fPlZx+g0N0GB/mobVhu0GHcDAIBnNKnl5uTJk0pLSzvjeFpamk6ePHne7zNt2jSNGjVK7du3V0lJiebOnauVK1dq6dKlkqTx48crKSlJmZmZkqQpU6ZoyJAhevbZZzV69GjNmzdPGzdu1OzZs5tyG6ZIigrS8VK7jhRWqFdSpNnlAADgc5rUctOnT58Gp1+/8MILSk9PP+/3KSgo0Pjx49WtWzcNGzZMGzZs0NKlSzVixAhJ0qFDh5Sbm+s6f9CgQZo7d65mz56tPn36aMGCBVq0aJF69erVlNswBWvdAADgWU1quXn66ac1evRoffzxx64uobVr1yonJ+eCVgx+7bXXGn195cqVZxy79dZbdeutt15Qvc1J3SrFzJgCAMAzmtRyM2TIEO3Zs0c33XSTCgsLVVhYqJtvvlk7d+7UW2+95e4afYqr5YZwAwCARzR5nZvExMQzZkVt3bpVr732WosaA+NtrHUDAIBnNanlBk2XxCrFAAB4FOHGy+rCzfHSKlVWO0yuBgAA30O48bKokECFWP0l0TUFAIAnXNCYm5tvvrnR1wsLCy+mllbBYrEoMSpY+wpKdbSwUp3ahZldEgAAPuWCwk1kZOOLzkVGRmr8+PEXVVBrkHQ63BwpLDe7FAAAfM4FhZvXX3/dU3W0KomuQcWVJlcCAIDvYcyNCZJZpRgAAI8h3JggMap2d3AGFAMA4H6EGxMkRYVIYq0bAAA8gXBjgrqWm9yiCjmdhsnVAADgWwg3JoiPCJK/n0XVDkPHSu1mlwMAgE8h3JggwN9P8RG1rTd0TQEA4F6EG5PUdU0xYwoAAPci3JiEDTQBAPAMwo1J6hbyYzo4AADuRbgxSRIL+QEA4BGEG5Mk0i0FAIBHEG5Mkky4AQDAIwg3JqlruSmprFFxZbXJ1QAA4DsINyYJtQUoKiRQEoOKAQBwJ8KNiRIjmTEFAIC7EW5MlNKmNtwcPF5uciUAAPgOwo2JusWFS5J25xabXAkAAL6DcGOi7gkRkqTdeYQbAADchXBjoh6JteFmT36pahxOk6sBAMA3EG5MlBIdolCrv6pqnNp/vMzscgAA8AmEGxP5+VmUVtc1xbgbAADcgnBjsu4JtYOKdxFuAABwC8KNyeoGFe86SrgBAMAdCDcmc82Yyi0xuRIAAHwD4cZkafHhslik46V2HSuxm10OAAAtHuHGZCHWAHWICZXEoGIAANyBcNMM9GDGFAAAbkO4aQbqZkwRbgAAuHiEm2aAQcUAALgP4aYZqAs33xwrlb3GYXI1AAC0bKaGm8zMTF122WUKDw9XbGysxo4dq6ysrEavmTNnjiwWS71HUFCQlyr2jITIIEUGB6rGaWhvfqnZ5QAA0KKZGm5WrVqlyZMna926dVq+fLmqq6t17bXXqqys8X2WIiIilJub63pkZ2d7qWLPsFgsjLsBAMBNAsz88I8++qje8zlz5ig2NlabNm3S1VdffdbrLBaL4uPjPV2eV3VPiNC6/SfZhgEAgIvUrMbcFBUVSZLatGnT6HmlpaVKTU1VSkqKxowZo507d571XLvdruLi4nqP5qg708EBAHCLZhNunE6npk6dqsGDB6tXr15nPa9bt2765z//qffff19vv/22nE6nBg0apMOHDzd4fmZmpiIjI12PlJQUT93CRenxnRlThmGYXA0AAC2XxWgmf0knTZqkJUuW6PPPP1dycvJ5X1ddXa3u3btr3LhxeuKJJ8543W63y27/dluD4uJipaSkqKioSBEREW6p3R3sNQ71fGSpapyG1jx0jRKjgs0uCQCAZqO4uFiRkZHn9ffb1DE3de6//34tXrxYq1evvqBgI0mBgYHq27ev9u3b1+DrNptNNpvNHWV6lC3AX53bhSkrv0S7c4sJNwAANJGp3VKGYej+++/XwoUL9cknn6hjx44X/B4Oh0Pbt29XQkKCByr0LmZMAQBw8UwNN5MnT9bbb7+tuXPnKjw8XHl5ecrLy1NFRYXrnPHjx2vatGmu59OnT9eyZcu0f/9+ffXVV7rzzjuVnZ2te+65x4xbcCtWKgYA4OKZ2i01c+ZMSdLQoUPrHX/99dc1ceJESdKhQ4fk5/dtBjt16pTuvfde5eXlKTo6Wv3799eaNWvUo0cPb5XtMcyYAgDg4jWbAcXeciEDkrztWIldlz35sSwWaefjIxVibRZDogAAMN2F/P1uNlPBIbULt6ltmE2GIX2dR9cUAABNQbhpZnok0jUFAMDFINw0M8yYAgDg4hBumpkezJgCAOCiEG6amboZU1/nFsvpbFVjvQEAcAvCTTPTqW2orAF+KqtyKOdUudnlAADQ4hBumpkAfz9dEhcmiXE3AAA0BeGmGeoeX9s1tYtxNwAAXDDCTTPUKylSkrT+wAmTKwEAoOUh3DRD16TFSpLWHzip46V2k6sBAKBlIdw0QyltQpSeHCmnIS3dmWd2OQAAtCiEm2bq+t4JkqQPt+eaXAkAAC0L4aaZur5XbbhZt/+kTtA1BQDAeSPcNFPtY0LUKylCDqehZbvyzS4HAIAWg3DTjI3qRdcUAAAXinDTjNWNu1nzzQmdKqsyuRoAAFoGwk0z1rFtqLon1HZNLadrCgCA80K4aeZG946XJP2PrikAAM4L4aaZG3W6a+qLfcdVVF5tcjUAADR/hJtmrnO7MKXFh6vGaWjZLhb0AwDgXAg3LUDdrKklOwg3AACcC+GmBRidXjvu5rO9x1RUQdcUAACNIdy0AF1iw9U1NkzVDkMrdjNrCgCAxhBuWgj2mgIA4PwQblqIunCzes9xlVTSNQUAwNkQblqIS+LC1LldqKocTq3YXWB2OQAANFuEmxbCYrHQNQUAwHkg3LQgo9Nrw83KrGM6yV5TAAA0iHDTgqTFR6h3UqSqHE79Z9Nhs8sBAKBZIty0MD+5vL0k6V/rD8kwDJOrAQCg+SHctDA39klUqNVf+4+X6csDJ80uBwCAZodw08KE2QL0w0uTJElzvzxkcjUAADQ/hJsW6CcDa7umPtqRx8BiAAC+h3DTAvVOjlSvpAhVOZx67ysGFgMA8F2EmxbqJwNTJUlzGVgMAEA9hJsW6oeXJirE6q/9xxhYDADAdxFuWqgwW4DGXJooqXZaOAAAqEW4acHGnR5YvGR7nk4xsBgAAEkmh5vMzExddtllCg8PV2xsrMaOHausrKxzXjd//nylpaUpKChIvXv31ocffuiFapuf9OQo18Di/zCwGAAASSaHm1WrVmny5Mlat26dli9frurqal177bUqKys76zVr1qzRuHHj9NOf/lSbN2/W2LFjNXbsWO3YscOLlTcfda03DCwGAKCWxWhGfxGPHTum2NhYrVq1SldffXWD59x2220qKyvT4sWLXceuuOIKXXrppXr55ZfP+RnFxcWKjIxUUVGRIiIi3Fa7WUoqq3X5X1aovMqhf//sCl3eKcbskgAAcLsL+fvdrMbcFBUVSZLatGlz1nPWrl2r4cOH1zs2cuRIrV27tsHz7Xa7iouL6z18SXhQoH7Yh4HFAADUaTbhxul0aurUqRo8eLB69ep11vPy8vIUFxdX71hcXJzy8vIaPD8zM1ORkZGuR0pKilvrbg7qNtP8kBWLAQBoPuFm8uTJ2rFjh+bNm+fW9502bZqKiopcj5ycHLe+f3PQOylSvZMiVVXj1Nwvs80uBwAAUzWLcHP//fdr8eLF+vTTT5WcnNzoufHx8crPz693LD8/X/Hx8Q2eb7PZFBERUe/haywWi+6+soMk6Y212bLXOMwtCAAAE5kabgzD0P3336+FCxfqk08+UceOHc95TUZGhlasWFHv2PLly5WRkeGpMluE0b0TFRdh07ESuxZvzTW7HAAATGNquJk8ebLefvttzZ07V+Hh4crLy1NeXp4qKipc54wfP17Tpk1zPZ8yZYo++ugjPfvss/r666/12GOPaePGjbr//vvNuIVmwxrgp/EZHSRJr35+gGnhAIBWy9RwM3PmTBUVFWno0KFKSEhwPf7973+7zjl06JByc79tiRg0aJDmzp2r2bNnq0+fPlqwYIEWLVrU6CDk1uKOy9srONBfu3OLtXb/CbPLAQDAFM1qnRtv8LV1br7v4UU79Na6bA1Li9VrEy8zuxwAANyixa5zg4t31+AOkqQVXxfom2Ol5hYDAIAJCDc+plO7MA3vHitJev2LAyZXAwCA9xFufNBPr+wkSVqw6TC7hQMAWh3CjQ+6olMb9UiIUGW1U3PZkgEA0MoQbnyQxWLRPVfVrhn0xpqDqqpxmlwRAADeQ7jxUTekJyo23KaCErsWbztqdjkAAHgN4cZHWQP8NGFQB0nSq5+xqB8AoPUg3Piwnwxsr6BAP+3KLdaH2xveNR0AAF9DuPFh0aFW/WRgqiTpV/M2a/5G39sRHQCA7yPc+LiHRqXp5r5JcjgN/W7BNr346T66qAAAPo1w4+OsAX569sd99IshnSVJzyzN0iPv75TDScABAPgmwk0rYLFY9NCoND16Yw9ZLNJb67I1+Z2vVFntMLs0AADcjnDTitw1uKP+Ma6vrP5++mhnnsa/tl5F5dVmlwUAgFsRblqZG9IT9cbdAxVuC9D6gyc14v9W6YOtRxmHAwDwGYSbViijc4ze/UWGOrYNVUGJXb/812aN/+d6HTheZnZpAABcNMJNK9U9IUJLplylXw+/RNYAP32297hG/n21/v7xHsbiAABaNIvRyvojiouLFRkZqaKiIkVERJhdTrNw8HiZHn5/hz7be1yS1CEmRL8d2U1Du8UqzBZgcnUAAFzY32/CDSRJhmHof9tzNf2DXSoosUuSrP5+urxTGw1Li9U1aXFqHxNicpUAgNaKcNMIwk3jSiqr9fKqb7R4W66yT5TXe61rbJhu7JOo+4Z2VoA/PZoAAO8h3DSCcHN+DMPQ/uNl+mR3gVZ8na8NB0+5Fv6796qO+uPoHiZXCABoTQg3jSDcNE1RRbXe++qwHv9glyTpH+P66sY+iSZXBQBoLS7k7zd9CzgvkcGBumtwR9c2Dr9fsE1f5xWbXBUAAGci3OCC/G5kN13Zpa0qqh36+VubVFTBCscAgOaFcIML4u9n0fPj+iopKljZJ8o1dd5mOc+yCWdBcaX++fkB7c0v8XKVAIDWjHCDC9Ym1KpZ/6+/bAF++jTrmJ5bsbfe63vzS/T7BVt15VOfavriXfrxrLU6yOrHAAAvYUAxmmzBpsP67fytkqRXxw9QWFCAZq/er0++LnCdEx4UoJLKGnVsG6r3Jg1SdKjVrHIBAC0Ys6UaQbhxr0fe36E312bLzyLV9U5ZLNLIHvG69+pOSmkTrJteXKMjhRUa2KGN3rpnoGwB/uYWDQBocZgtBa/50+geGpAaLach2QL8dMfl7fXJb4bq5f/XX/1ToxUbHqTX77pM4UG1u5D/fsE2diAHAHgULTe4aCWV1frk6wIN7tJWbcNsDZ7z+d7jmvj6etU4Df1qWFc9MOISL1cJAGjJaLmBV4UHBWrMpUlnDTaSdGXXtnrypl6SpOdX7NV/Nh32VnkAgFaGLZ/hNbdd1l7ZJ8r10spv9NB721TlcMrq76eTZVU6UValk2V2nSyrltMw1DU2TGkJ4eoWF6HOsaGM0wEAnDe6peBVTqehX83brMXbcs/7mgA/izq1C1X3hAhd3ztBw9Ji2bgTAFqZC/n7TcsNvMrPz6K/3tpHFotF+4+Vqk2o1fWICbUqOtQqw5D25Jfo69wS7c4rVklljfbkl2pPfqne33JU8RFBun1gim6/rL3iI4PMviUAQDNDyw2aNcMwlFtUqay8Eq3df0ILNh3WybIqSbWrJQ9Li9UdV6Tqqi5t5ednMblaAICnsM5NIwg3LZu9xqGPduTpnXWHtP7gSdfxNqFWDeoco6u6ttWVXdspKSrYxCoBAO5GuGkE4cZ37Mkv0dwvD+k/Xx1WSWVNvdc6tQ3VVV3bqmdSpEKtAQqx+ivY6q9Qa4CCrf5qF25TZHCgSZUDAC4U4aYRhBvfU+1wavOhQn2+95g+23dcW3MKdZa9PF38/Sy6b2hn/fKarrIGMDgZAJo7wk0jCDe+r6iiWmu/OaEv9h1XzqlylVc5VFHlUHlVjcqrHCqz16j4dEtPz8QI/d9tl+qSuHCTqwYANKbFhJvVq1frmWee0aZNm5Sbm6uFCxdq7NixZz1/5cqV+sEPfnDG8dzcXMXHx5/XZxJuIEmLtx3VnxbtUGF5tawBfvrdtd1095Ud5c+gZABollrMCsVlZWXq06ePXnzxxQu6LisrS7m5ua5HbGyshyqEr7ohPVHLpl6tH3Rrp6oap578cLfGvbJOOSfLPfq51Q6nvjp0SkcKK9hjCwA8xNR1bkaNGqVRo0Zd8HWxsbGKiopyf0FoVWIjgvTPiZdp3oYcPbF4l9YfOKnr/r5aaQkRCrUFKMxWOwC59usAJUQFqVtcuLrGhV/wYORDJ8o1b8MhvbvxsI6X2iVJbcOs6p0Uqd7JUUpPilR6cqRiI1i3BwAuVotcxO/SSy+V3W5Xr1699Nhjj2nw4MFnPddut8tut7ueFxcXe6NEtBAWi0XjBrbXoM4x+s27W7Ux+5Q2ZZ8653VxETZdEheurrHh6tguVPERQYqPCFJcpE0xoTb5+1lUVePUsl15mrc+R5/vO+66NiIoQOVVDh0vrdKnWcf0adYx12s9EiL0i6GdNbp3wjm7yIoqqvXNsVJdmhzFGj8A8B3NZkCxxWI555ibrKwsrVy5UgMGDJDdbterr76qt956S19++aX69evX4DWPPfaYHn/88TOOM+YG3+dwGvrq0CmdKLWr1F478LjUXqPyqhqVVNYo+0S59uSXKLeostH38fezKDbcpopqhwrLqyVJFot0ddd2GjcwRcO6x8nhNLQ7t1jbjxRp2+EibT9cpL0FJa5ZXh1iQvTzIZ11c7+kevtq2Wsc+vTrY1q0+Yg++bpAVQ6nMjrF6OkfpSulTYjHvjcAYLYWM6D4u84n3DRkyJAhat++vd56660GX2+o5SYlJYVwgyYrrqzW3vxS7c0v0Z78Uh06Wa784krlF1fqWKld3/1fVGy4TbddlqIfD0g5Z/g4VValN9dm6/U1B1yhKC7Cpnuu7KSeiRH6YNtR/W9brmuml1QbpBxOQ6FWfz18Qw/ddlmKLBZacQD4nla1t9TAgQP1+eefn/V1m80mm83mxYrg6yKCAtU/NVr9U6PPeK3G4dSxUrvyi+2qdjjVNyXqvDf5jA61asrwrrr36o761/ocvbJ6v/KKK/Xkh7vrnRcfEaQxfRM19tIkhVj99dv5W7Xh4Ck99N52fbQzTzNuTr+gPbf2HyvVXz78Wt8cK9XgLjG6rmeCLu/URoEN1G2vcWjNNye0bGeeNmWfUkanGD04Kk0h1hb/qwSAD2nxLTcjRoxQeHi43nvvvfM6n6ngaCmqapxatPmIZn+2X8dL7bq2R5zG9k3S5R1j6o3HcTgNvf7FAT29NEtVNU5FBAXo8TE9NfbSpEZbccqravTip/s0e/V+VTvq/xqIDA7U8O5xGtkzTn3bR2vd/hNaujNPK7OOqdRefzXojm1D9ffbLlWflCi33j8AfFeL6ZYqLS3Vvn37JEl9+/bV3/72N/3gBz9QmzZt1L59e02bNk1HjhzRm2++KUn6+9//ro4dO6pnz56qrKzUq6++qn/84x9atmyZhg0bdl6fSbiBr9pXUKIH3t2qbYeLJEldYsM0LC1W16TFqn9qtKsFyTAMLd2ZrycW79KRwgpJ0tBu7fTjASn6bO8xLduZrxOnNydtSFyETdf2iFevpAj9/eO9yi2qVICfRVOHd9WkoV28slbQiVK7HE6D2WVAK9Jiws3ZFuWbMGGC5syZo4kTJ+rgwYNauXKlJOnpp5/W7NmzdeTIEYWEhCg9PV2PPPJIg+9xNoQb+LIah1MzV36jf3yyT1UOp+t4RFCAhnSL1VVd2+p/23K1ak/tDK2kqGA9emMPjegR52rlcTgNbTx4Uh/tzNPSHXk6WlSpTm1DdW3PeI3sGac+35mdVVRerT8s2q7/bcuVJA1Ijdb/3XZpg+OLDMOQ01CTwk9heZXW7T+htd+c0JpvTmhvQamk2jFN6cmRSk+OUu/kSKUnRSomjG5owBe1mHBjBsINWoOi8mqt3ntMn3xdoE+zClwDlOtY/f308yGddN/QLgq2+p/lXWoDSVFFtaJCrI2es3DzET3y/k6V2msUZgvQTX2TVFJZrRNlVTpRWqUTZXadLKuSYdR2Y10SF66ucWG6JC5cl8SFKzUmRCWVNSooqVRBsV0FJXYdK7Ert6hCGw+e0u68Yn3/N5WfRQ3uIdapbah+f12arut1fquWA2gZCDeNINygtXE4DW3JOaUVuwv0xb7jSooO1u9Gpqlj21C3fk7OyXI98O4WbTh47nWCmqJrbJgyOsdoUOcYXd4xRkGB/tp5tHYq/bbDhdp2pEj7j5W5zh/ZM07Tx/RSXCNdV4ZhaG9BqSKDAxs9r6kMw1BFtUPFFTWKCglUUODZgySAxhFuGkG4ATzH4TT07sYcZZ8oV0yoVW1CrYoJsyom1KaYMKsMSXvzS7Q3v1RZ+SW1XxeUqrzKIUmKDglUbHiQYiNsahde++iREKGMzjGKDT93+CiqqNasVd9o9ur9qnEaCrcF6MFRafrJwPb1FjrMLarQe18d0X82Hdb+47WBqE9KlK7tEadre8SpS2zYBU2pzzlZro9352vNNyd0vNSuoopqFVdUq6ii2jVYO8DPorSEcF2aEqU+yVHq2z5KndqGsQAjcJ4IN40g3ADNi9Np6GR5lSKCAmUNcM92d7tzi/XQf7Zp6+nB1QNSo/XYD3vqm2OlWrDpsD7fd9zVzWUN8FNVjbPe9R1iQjSiR5yu6BSjtmE2tTkd1EKs/rJYLHI6DW05XKgVu/P18a4CZeWXNFrP2brQwm0B6pMSpf6p0RrQIVp920crzMa0eqAhhJtGEG6A1sHhNPTm2oN6ZmmWq2XouwZ2bKMf9U/W9b0TVG6v0ce7C7R8V56+2Hei3mDs77IF+KlNqFVVNc56M8r8/SwakBqtYd1j1b5NqCKDA2sfIbX/DbX662hRpbYcKtSWnFPamlOkbUcKVVld/3P8LFJafIQu6xCtnomRqnY6VVpZc3q17NpVs6scTl2TFqsb0hPO2bq082iRHv9gl77OLVaflChd0SlGl3dso/TkKLcFyZbO6TR0rNSuw6fKdfhUhU6UVinE6u/aU67u66BAf50sq1JecaXyiiqUV2RXXnGFCort6tg2VLcPTFG/9tEsoimppLJaNQ5D0aFnH6vXFISbRhBugNblSGGFHl60Q598XaCkqGDd0j9Zt/RLUmpMw2OOSu01Wr3nmJbvytee/BKdLKvSibKqM1p3wm0BGtKtnYZ3j9PQbu0aHXTdkBqHU1n5JfrqUKE2HTypjdmndPhUxXlf37d9lP40ukeDi0mW2mv0t2V7NGfNgQZbjGwBfurXPlqXd2qjKzrF6NKUqCaNB8otqtCH2/P04fZcHThepriIICVFBSspKkiJUcFKig5WQmSwYsNruyW9tdijYRjaklOo5bvydaq8Sk6n5DAMOQ1DTqchh1HbhXn4ZLkOF1ac8bNtqm5x4Ro3MEU39U1WZMiFba7bEKfTUF5xpQ6dLFendqHn1TVrpspqh95am62XVu7Tdb3ilXlzulvfn3DTCMIN0PoYhqESe43CrAFNGuNiGIbKqxw6WValk2VVqnEa6p0U6fbWj7yiSm3MPqmNB0/pm2OlCg70V1hQbQtCXUtCcUW13lqX7WqNGp2eoIeuS1NKmxAZhqGPduTp8Q92Ka+40vX6XYM6aOfRYn154IS+3H/yjHWMrAF+6lvXstOpjfq1jz5r2MkvrtSH23P1v2252ngem8x+V3Cgv2LCrGobZlNMqFUhtgBZ/f1kC/Sr/W+An6wBfrJIstc4Za9xqsrhVNXprwP9LOoWH66eiZHqkRihNt9pGTAMQ9uPFOl/23K1eFuuaw2n8+FnkRIig5UcHay24TZVVjlUaq9RWVWNyuy1X1dWOdQmzKq4iCAlRAYpPrJ2s9w2oVZ9tve4Fm876mqJswX46Yb0RI1Oj1dCZLDahdvUJsR6xr89wzBUXFmjvKJK5RVXKrewQgdPlOvg8TIdPFH7qHvP4EB/PXJjD93eDLdYqXY4NX/jYT2/Yq/r390lcWH64JdX1tsb72IRbhpBuAHQ0hUUV+rZZXv07qYcGUbt1P4Jg1K1t6BUK0/vMp8aE6LpY3ppyCXt6l1rGIa+OVaqLw+c1Lr9J/Xl/hMqKLHXOyfQ36LwoED5WSzy95P8LRb5+VlksUiHT1XUm5Y/IDVao9MTNCC1jY6X2nWksEJHCit01PWo1PFSu+xuah35roTIIPVMjFB8ZJBW7zmuQyfLXa+FWP01rHucLomtHbTtZ7HIz1LbhehnsSjMFqDkNsFKiQ5RfGRQg9uNXIiiimq9v+WI5n55SF/nnTkGy9/PophQq9qF2xRqC9DxErtyiypVUX1ml+l3BfhZFBVi1fHS2p/RiB5xeuqW9HrBzixOp6EPth3V/y3fo4Mnar/3iZFBmjr8Et3cL+m8t545X4SbRhBuAPiKXUeL9eSHu/TFvhOuY4H+Fk0a0ln3/aDLeXU1GYahA8fLToedE1q3/4Tyi+2NXtM/NVqjeydoVO/alonz+YyyKodOlNp1vLRKJ0rtOlFWpfIqh6pqaltmqhzffm1IrlYcq79/7X8D/FRur9HuvGLtPFqs7BPlZ3xOcKC/hnWvHY80tFusKVPvDcPQ5pxCzVt/SNsOF+lYiV0ny6vOWKfpu6JCAhUfUdsa1L5NiDq2DVWHtqHqGBOqpOhg+Vsseu3zA3pmaZaqHE61C7fpr7f2OSO41sktqtCe/FJZVBuOAvz95O9nUaC/RQF+fq7Ws4tZTXzVnmPK/HC3K8jFhFo1+Qdd9JPL23vs+064aQThBoAvMQxDn2YV6P+W71VMmFV/Gt1DXWLDLur9jhRWqLzKoRpH7TgVh9OoHbPiNJR8uqXDbCWV1dqdW6JdR4t06GSF+qdG6wdp7ZrlJq7VDqdOllXp2OnFKUvsNWoXZlNCZJDiIoIaXUjzu3YeLdKUeVu07/QK3XcN7qDfXttNB46XaePBk9p0qFBfZZ86ry45fz+LYsNtrm62uIgg9U6K1HW94hXayIy9fQUl+vP/drtaCMNtAfr5kE66a3DHRq9zB8JNIwg3AICWqrLaocwPd+uNtdmSJItFDa7e3bldmAL8/eRwOlXjMFTjNFTjcKrKYehkmb3BgeaSFGr11w3pibp1QLL6p347++tkWZX+/vEevfPlITmchgL9LRqf0UG/vKbLBQ+mbyrCTSMINwCAlu7Trwv0uwVbdby0SuFBAerXPrp2vaTUaPVJiWq0FcXhNHS8tHbMT15R7dT2I4UVWrYrv153X6e2obp1QIoC/Cx6/pO9KqmskVQ77ucP13d3+yrn50K4aQThBgDgCyqrHcotqlRqmxC3rHRtGIbWHzip+ZsO63/bcs8Y7Nw9IUIP39Bdgzq3vejPagrCTSMINwAANK7UXqMPt+Vq/qYcFVfU6O4rO+hH/VMuahDyxSLcNIJwAwBAy3Mhf79ZfxsAAPgUwg0AAPAphBsAAOBTCDcAAMCnEG4AAIBPIdwAAACfQrgBAAA+hXADAAB8CuEGAAD4FMINAADwKYQbAADgUwg3AADApxBuAACATyHcAAAAnxJgdgHeZhiGpNqt0wEAQMtQ93e77u94Y1pduCkpKZEkpaSkmFwJAAC4UCUlJYqMjGz0HItxPhHIhzidTh09elTh4eGyWCxufe/i4mKlpKQoJydHERERbn3v5qy13rfEvbfGe2+t9y1x763x3pvTfRuGoZKSEiUmJsrPr/FRNa2u5cbPz0/Jycke/YyIiAjT/xGYobXet8S9t8Z7b633LXHvrfHem8t9n6vFpg4DigEAgE8h3AAAAJ9CuHEjm82mRx99VDabzexSvKq13rfEvbfGe2+t9y1x763x3lvqfbe6AcUAAMC30XIDAAB8CuEGAAD4FMINAADwKYQbAADgUwg3bvLiiy+qQ4cOCgoK0uWXX67169ebXZLbrV69WjfeeKMSExNlsVi0aNGieq8bhqFHHnlECQkJCg4O1vDhw7V3715zinWjzMxMXXbZZQoPD1dsbKzGjh2rrKyseudUVlZq8uTJiomJUVhYmG655Rbl5+ebVLH7zJw5U+np6a4FvDIyMrRkyRLX67563983Y8YMWSwWTZ061XXMV+/9sccek8ViqfdIS0tzve6r913nyJEjuvPOOxUTE6Pg4GD17t1bGzdudL3uq7/nOnTocMbP3WKxaPLkyZJa3s+dcOMG//73v/XAAw/o0Ucf1VdffaU+ffpo5MiRKigoMLs0tyorK1OfPn304osvNvj6008/reeff14vv/yyvvzyS4WGhmrkyJGqrKz0cqXutWrVKk2ePFnr1q3T8uXLVV1drWuvvVZlZWWuc37961/rgw8+0Pz587Vq1SodPXpUN998s4lVu0dycrJmzJihTZs2aePGjbrmmms0ZswY7dy5U5Lv3vd3bdiwQbNmzVJ6enq947587z179lRubq7r8fnnn7te8+X7PnXqlAYPHqzAwEAtWbJEu3bt0rPPPqvo6GjXOb76e27Dhg31fubLly+XJN16662SWuDP3cBFGzhwoDF58mTXc4fDYSQmJhqZmZkmVuVZkoyFCxe6njudTiM+Pt545plnXMcKCwsNm81m/Otf/zKhQs8pKCgwJBmrVq0yDKP2PgMDA4358+e7ztm9e7chyVi7dq1ZZXpMdHS08eqrr7aK+y4pKTG6du1qLF++3BgyZIgxZcoUwzB8+2f+6KOPGn369GnwNV++b8MwjAcffNC48sorz/p6a/o9N2XKFKNz586G0+lskT93Wm4uUlVVlTZt2qThw4e7jvn5+Wn48OFau3atiZV514EDB5SXl1fv+xAZGanLL7/c574PRUVFkqQ2bdpIkjZt2qTq6up6956Wlqb27dv71L07HA7NmzdPZWVlysjIaBX3PXnyZI0ePbrePUq+/zPfu3evEhMT1alTJ91xxx06dOiQJN+/7//+978aMGCAbr31VsXGxqpv37565ZVXXK+3lt9zVVVVevvtt3X33XfLYrG0yJ874eYiHT9+XA6HQ3FxcfWOx8XFKS8vz6SqvK/uXn39++B0OjV16lQNHjxYvXr1klR771arVVFRUfXO9ZV73759u8LCwmSz2fSLX/xCCxcuVI8ePXz+vufNm6evvvpKmZmZZ7zmy/d++eWXa86cOfroo480c+ZMHThwQFdddZVKSkp8+r4laf/+/Zo5c6a6du2qpUuXatKkSfrVr36lN954Q1Lr+T23aNEiFRYWauLEiZJa5r/3VrcrOHAxJk+erB07dtQbg+DrunXrpi1btqioqEgLFizQhAkTtGrVKrPL8qicnBxNmTJFy5cvV1BQkNnleNWoUaNcX6enp+vyyy9Xamqq3n33XQUHB5tYmec5nU4NGDBAf/nLXyRJffv21Y4dO/Tyyy9rwoQJJlfnPa+99ppGjRqlxMREs0tpMlpuLlLbtm3l7+9/xqjx/Px8xcfHm1SV99Xdqy9/H+6//34tXrxYn376qZKTk13H4+PjVVVVpcLCwnrn+8q9W61WdenSRf3791dmZqb69Omj5557zqfve9OmTSooKFC/fv0UEBCggIAArVq1Ss8//7wCAgIUFxfns/f+fVFRUbrkkku0b98+n/6ZS1JCQoJ69OhR71j37t1d3XKt4fdcdna2Pv74Y91zzz2uYy3x5064uUhWq1X9+/fXihUrXMecTqdWrFihjIwMEyvzro4dOyo+Pr7e96G4uFhffvlli/8+GIah+++/XwsXLtQnn3yijh071nu9f//+CgwMrHfvWVlZOnToUIu/94Y4nU7Z7Xafvu9hw4Zp+/bt2rJli+sxYMAA3XHHHa6vffXev6+0tFTffPONEhISfPpnLkmDBw8+Y5mHPXv2KDU1VZJv/56r8/rrrys2NlajR492HWuRP3ezRzT7gnnz5hk2m82YM2eOsWvXLuNnP/uZERUVZeTl5ZldmluVlJQYmzdvNjZv3mxIMv72t78ZmzdvNrKzsw3DMIwZM2YYUVFRxvvvv29s27bNGDNmjNGxY0ejoqLC5MovzqRJk4zIyEhj5cqVRm5urutRXl7uOucXv/iF0b59e+OTTz4xNm7caGRkZBgZGRkmVu0eDz30kLFq1SrjwIEDxrZt24yHHnrIsFgsxrJlywzD8N37bsh3Z0sZhu/e+29+8xtj5cqVxoEDB4wvvvjCGD58uNG2bVujoKDAMAzfvW/DMIz169cbAQEBxpNPPmns3bvXeOedd4yQkBDj7bffdp3jq7/nDKN2pm/79u2NBx988IzXWtrPnXDjJv/4xz+M9u3bG1ar1Rg4cKCxbt06s0tyu08//dSQdMZjwoQJhmHUTpN8+OGHjbi4OMNmsxnDhg0zsrKyzC3aDRq6Z0nG66+/7jqnoqLCuO+++4zo6GgjJCTEuOmmm4zc3FzzinaTu+++20hNTTWsVqvRrl07Y9iwYa5gYxi+e98N+X648dV7v+2224yEhATDarUaSUlJxm233Wbs27fP9bqv3nedDz74wOjVq5dhs9mMtLQ0Y/bs2fVe99Xfc4ZhGEuXLjUkNXg/Le3nbjEMwzClyQgAAMADGHMDAAB8CuEGAAD4FMINAADwKYQbAADgUwg3AADApxBuAACATyHcAAAAn0K4AQAAPoVwA6BZOnbsmCZNmqT27dvLZrMpPj5eI0eO1BdffCFJslgsWrRokblFAmiWAswuAAAacsstt6iqqkpvvPGGOnXqpPz8fK1YsUInTpwwuzQAzRzbLwBodgoLCxUdHa2VK1dqyJAhZ7zeoUMHZWdnu56npqbq4MGDkqT3339fjz/+uHbt2qXExERNmDBBf/zjHxUQUPv/5SwWi1566SX997//1cqVK5WQkKCnn35aP/rRj7xybwA8j24pAM1OWFiYwsLCtGjRItnt9jNe37BhgyTp9ddfV25uruv5Z599pvHjx2vKlCnatWuXZs2apTlz5ujJJ5+sd/3DDz+sW265RVu3btUdd9yh22+/Xbt37/b8jQHwClpuADRL//nPf3TvvfeqoqJC/fr105AhQ3T77bcrPT1dUm0LzMKFCzV27FjXNcOHD9ewYcM0bdo017G3335bv//973X06FHXdb/4xS80c+ZM1zlXXHGF+vXrp5deesk7NwfAo2i5AdAs3XLLLTp69Kj++9//6rrrrtPKlSvVr18/zZkz56zXbN26VdOnT3e1/ISFhenee+9Vbm6uysvLXedlZGTUuy4jI4OWG8CHMKAYQLMVFBSkESNGaMSIEXr44Yd1zz336NFHH9XEiRMbPL+0tFSPP/64br755gbfC0DrQMsNgBajR48eKisrkyQFBgbK4XDUe71fv37KyspSly5dznj4+X37627dunX1rlu3bp26d+/u+RsA4BW03ABodk6cOKFbb71Vd999t9LT0xUeHq6NGzfq6aef1pgxYyTVzphasWKFBg8eLJvNpujoaD3yyCO64YYb1L59e/3oRz+Sn5+ftm7dqh07dujPf/6z6/3nz5+vAQMG6Morr9Q777yj9evX67XXXjPrdgG4GQOKATQ7drtdjz32mJYtW6ZvvvlG1dXVSklJ0a233qo//OEPCg4O1gcffKAHHnhABw8eVFJSkmsq+NKlSzV9+nRt3rxZgYGBSktL0z333KN7771XUu2A4hdffFGLFi3S6tWrlZCQoKeeeko//vGPTbxjAO5EuAHQqjQ0ywqAb2HMDQAA8CmEGwAA4FMYUAygVaEnHvB9tNwAAACfQrgBAAA+hXADAAB8CuEGAAD4FMINAADwKYQbAADgUwg3AADApxBuAACATyHcAAAAn/L/AeTiMrkMB2k2AAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -1222,7 +1478,9 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "d1_l1Pr-7eso" + }, "source": [ "## Debugging the model\n", "\n", @@ -1244,23 +1502,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "EkoFGCgSZ1yz", - "outputId": "3467b8ba-ce05-42f0-fb89-75922cc91e31", + "outputId": "593af986-77a7-4665-eee4-5deb2d891b7f", "tags": [ "nbval-skip" ] }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:[process=0][thread=MainThread][operation_id=1] _SignalingThread.join() waiting for signals ([]) blocking the main thread will slow down blocking save times. This is likely due to main thread calling result() on a CommitFuture.\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "_CHECKPOINT_METADATA d manifest.ocdbt _METADATA ocdbt.process_0 _sharding\n" + "array_metadatas d\t\t _METADATA _sharding\n", + "_CHECKPOINT_METADATA manifest.ocdbt ocdbt.process_0\n" ] } ], @@ -1278,26 +1544,472 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "azzwiMeP7eso" + }, "source": [ "## Tunix: Fine-tuning\n", "\n", - "[REVAMP 6] Introduce Tunix." + "[Tunix](https://github.com/google/tunix) is a JAX-native LLM post-training library open sourced by Google. It supports a range of post-training techniques including supervised finetuning, preference tuning, reinforcement learning and model distillation. In this section, we are going to use Tunix to finetune the miniGPT model we just pretrained using LoRA ([Low-Rank Adaptation](https://arxiv.org/abs/2106.09685)) so that the finetuned model generates output of a different style." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-kJShd9n_iTl" + }, + "source": [ + "First we install Tunix and its dependencies, and import necessary libraries. Note that Colab will ask you to restart the runtime, but you can just ignore it." ] }, { "cell_type": "code", - "execution_count": 84, - "metadata": {}, + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "uipLvy9E7eso", + "outputId": "93cf001f-2964-467c-9848-b106be6cacf5", + "tags": [ + "nbval-ignore-output" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting google-tunix[prod]\n", + " Downloading google_tunix-0.1.3-py3-none-any.whl.metadata (7.8 kB)\n", + "Downloading google_tunix-0.1.3-py3-none-any.whl (253 kB)\n", + "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/253.7 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m253.7/253.7 kB\u001b[0m \u001b[31m12.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: google-tunix\n", + "Successfully installed google-tunix-0.1.3\n", + "Collecting qwix\n", + " Downloading qwix-0.1.4-py3-none-any.whl.metadata (6.3 kB)\n", + "Collecting tensorboardX\n", + " Downloading tensorboardx-2.6.4-py3-none-any.whl.metadata (6.2 kB)\n", + "Collecting tensorflow\n", + " Downloading tensorflow-2.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.5 kB)\n", + "Requirement already satisfied: jax in /usr/local/lib/python3.12/dist-packages (from qwix) (0.8.0)\n", + "Requirement already satisfied: jaxlib in /usr/local/lib/python3.12/dist-packages (from qwix) (0.8.0)\n", + "Requirement already satisfied: flax>=0.12.0 in /usr/local/lib/python3.12/dist-packages (from qwix) (0.12.0)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.12/dist-packages (from tensorboardX) (2.0.2)\n", + "Requirement already satisfied: packaging in /usr/local/lib/python3.12/dist-packages (from tensorboardX) (25.0)\n", + "Requirement already satisfied: protobuf>=3.20 in /usr/local/lib/python3.12/dist-packages (from tensorboardX) (6.33.0)\n", + "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from tensorflow) (2.3.1)\n", + "Collecting astunparse>=1.6.0 (from tensorflow)\n", + " Downloading astunparse-1.6.3-py2.py3-none-any.whl.metadata (4.4 kB)\n", + "Collecting flatbuffers>=24.3.25 (from tensorflow)\n", + " Downloading flatbuffers-25.9.23-py2.py3-none-any.whl.metadata (875 bytes)\n", + "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.12/dist-packages (from tensorflow) (0.6.0)\n", + "Collecting google_pasta>=0.1.1 (from tensorflow)\n", + " Downloading google_pasta-0.2.0-py3-none-any.whl.metadata (814 bytes)\n", + "Collecting libclang>=13.0.0 (from tensorflow)\n", + " Downloading libclang-18.1.1-py2.py3-none-manylinux2010_x86_64.whl.metadata (5.2 kB)\n", + "Requirement already satisfied: opt_einsum>=2.3.2 in /usr/local/lib/python3.12/dist-packages (from tensorflow) (3.4.0)\n", + "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.12/dist-packages (from tensorflow) (2.32.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from tensorflow) (75.2.0)\n", + "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.12/dist-packages (from tensorflow) (1.17.0)\n", + "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from tensorflow) (3.2.0)\n", + "Requirement already satisfied: typing_extensions>=3.6.6 in /usr/local/lib/python3.12/dist-packages (from tensorflow) (4.15.0)\n", + "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.12/dist-packages (from tensorflow) (2.0.1)\n", + "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.12/dist-packages (from tensorflow) (1.76.0)\n", + "Collecting tensorboard~=2.20.0 (from tensorflow)\n", + " Downloading tensorboard-2.20.0-py3-none-any.whl.metadata (1.8 kB)\n", + "Requirement already satisfied: keras>=3.10.0 in /usr/local/lib/python3.12/dist-packages (from tensorflow) (3.10.0)\n", + "Requirement already satisfied: h5py>=3.11.0 in /usr/local/lib/python3.12/dist-packages (from tensorflow) (3.15.1)\n", + "Requirement already satisfied: ml_dtypes<1.0.0,>=0.5.1 in /usr/local/lib/python3.12/dist-packages (from tensorflow) (0.5.3)\n", + "Collecting wheel<1.0,>=0.23.0 (from astunparse>=1.6.0->tensorflow)\n", + " Downloading wheel-0.45.1-py3-none-any.whl.metadata (2.3 kB)\n", + "Requirement already satisfied: msgpack in /usr/local/lib/python3.12/dist-packages (from flax>=0.12.0->qwix) (1.1.2)\n", + "Requirement already satisfied: optax in /usr/local/lib/python3.12/dist-packages (from flax>=0.12.0->qwix) (0.2.6)\n", + "Requirement already satisfied: orbax-checkpoint in /usr/local/lib/python3.12/dist-packages (from flax>=0.12.0->qwix) (0.11.26)\n", + "Requirement already satisfied: tensorstore in /usr/local/lib/python3.12/dist-packages (from flax>=0.12.0->qwix) (0.1.78)\n", + "Requirement already satisfied: rich>=11.1 in /usr/local/lib/python3.12/dist-packages (from flax>=0.12.0->qwix) (14.2.0)\n", + "Requirement already satisfied: PyYAML>=5.4.1 in /usr/local/lib/python3.12/dist-packages (from flax>=0.12.0->qwix) (6.0.3)\n", + "Requirement already satisfied: treescope>=0.1.7 in /usr/local/lib/python3.12/dist-packages (from flax>=0.12.0->qwix) (0.1.10)\n", + "Requirement already satisfied: scipy>=1.13 in /usr/local/lib/python3.12/dist-packages (from jax->qwix) (1.16.3)\n", + "Requirement already satisfied: namex in /usr/local/lib/python3.12/dist-packages (from keras>=3.10.0->tensorflow) (0.1.0)\n", + "Requirement already satisfied: optree in /usr/local/lib/python3.12/dist-packages (from keras>=3.10.0->tensorflow) (0.17.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests<3,>=2.21.0->tensorflow) (3.4.4)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.12/dist-packages (from requests<3,>=2.21.0->tensorflow) (3.11)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests<3,>=2.21.0->tensorflow) (2.5.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.12/dist-packages (from requests<3,>=2.21.0->tensorflow) (2025.10.5)\n", + "Requirement already satisfied: markdown>=2.6.8 in /usr/lib/python3/dist-packages (from tensorboard~=2.20.0->tensorflow) (3.3.6)\n", + "Requirement already satisfied: pillow in /usr/local/lib/python3.12/dist-packages (from tensorboard~=2.20.0->tensorflow) (12.0.0)\n", + "Collecting tensorboard-data-server<0.8.0,>=0.7.0 (from tensorboard~=2.20.0->tensorflow)\n", + " Downloading tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl.metadata (1.1 kB)\n", + "Collecting werkzeug>=1.0.1 (from tensorboard~=2.20.0->tensorflow)\n", + " Downloading werkzeug-3.1.3-py3-none-any.whl.metadata (3.7 kB)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich>=11.1->flax>=0.12.0->qwix) (4.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich>=11.1->flax>=0.12.0->qwix) (2.19.2)\n", + "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.12/dist-packages (from werkzeug>=1.0.1->tensorboard~=2.20.0->tensorflow) (3.0.3)\n", + "Requirement already satisfied: chex>=0.1.87 in /usr/local/lib/python3.12/dist-packages (from optax->flax>=0.12.0->qwix) (0.1.91)\n", + "Requirement already satisfied: etils[epath,epy] in /usr/local/lib/python3.12/dist-packages (from orbax-checkpoint->flax>=0.12.0->qwix) (1.13.0)\n", + "Requirement already satisfied: nest_asyncio in /usr/local/lib/python3.12/dist-packages (from orbax-checkpoint->flax>=0.12.0->qwix) (1.6.0)\n", + "Requirement already satisfied: aiofiles in /usr/local/lib/python3.12/dist-packages (from orbax-checkpoint->flax>=0.12.0->qwix) (25.1.0)\n", + "Requirement already satisfied: humanize in /usr/local/lib/python3.12/dist-packages (from orbax-checkpoint->flax>=0.12.0->qwix) (4.14.0)\n", + "Requirement already satisfied: simplejson>=3.16.0 in /usr/local/lib/python3.12/dist-packages (from orbax-checkpoint->flax>=0.12.0->qwix) (3.20.2)\n", + "Requirement already satisfied: psutil in /usr/local/lib/python3.12/dist-packages (from orbax-checkpoint->flax>=0.12.0->qwix) (5.9.5)\n", + "Requirement already satisfied: toolz>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from chex>=0.1.87->optax->flax>=0.12.0->qwix) (1.1.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich>=11.1->flax>=0.12.0->qwix) (0.1.2)\n", + "Requirement already satisfied: fsspec in /usr/local/lib/python3.12/dist-packages (from etils[epath,epy]->orbax-checkpoint->flax>=0.12.0->qwix) (2025.10.0)\n", + "Requirement already satisfied: importlib_resources in /usr/local/lib/python3.12/dist-packages (from etils[epath,epy]->orbax-checkpoint->flax>=0.12.0->qwix) (6.5.2)\n", + "Requirement already satisfied: zipp in /usr/local/lib/python3.12/dist-packages (from etils[epath,epy]->orbax-checkpoint->flax>=0.12.0->qwix) (3.23.0)\n", + "Downloading qwix-0.1.4-py3-none-any.whl (91 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m91.6/91.6 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading tensorboardx-2.6.4-py3-none-any.whl (87 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.2/87.2 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading tensorflow-2.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (620.7 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m620.7/620.7 MB\u001b[0m \u001b[31m773.0 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading astunparse-1.6.3-py2.py3-none-any.whl (12 kB)\n", + "Downloading flatbuffers-25.9.23-py2.py3-none-any.whl (30 kB)\n", + "Downloading google_pasta-0.2.0-py3-none-any.whl (57 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.5/57.5 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading libclang-18.1.1-py2.py3-none-manylinux2010_x86_64.whl (24.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.5/24.5 MB\u001b[0m \u001b[31m99.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading tensorboard-2.20.0-py3-none-any.whl (5.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.5/5.5 MB\u001b[0m \u001b[31m66.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl (6.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.6/6.6 MB\u001b[0m \u001b[31m152.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading werkzeug-3.1.3-py3-none-any.whl (224 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m22.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading wheel-0.45.1-py3-none-any.whl (72 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.5/72.5 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: libclang, flatbuffers, wheel, werkzeug, tensorboardX, tensorboard-data-server, google_pasta, tensorboard, astunparse, tensorflow, qwix\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "google-tunix 0.1.3 requires datasets, which is not installed.\n", + "google-tunix 0.1.3 requires gcsfs, which is not installed.\n", + "google-tunix 0.1.3 requires hf_transfer, which is not installed.\n", + "google-tunix 0.1.3 requires omegaconf, which is not installed.\n", + "google-tunix 0.1.3 requires python-dotenv, which is not installed.\u001b[0m\u001b[31m\n", + "\u001b[0mSuccessfully installed astunparse-1.6.3 flatbuffers-25.9.23 google_pasta-0.2.0 libclang-18.1.1 qwix-0.1.4 tensorboard-2.20.0 tensorboard-data-server-0.7.2 tensorboardX-2.6.4 tensorflow-2.20.0 werkzeug-3.1.3 wheel-0.45.1\n" + ] + } + ], + "source": [ + "!pip install google-tunix[prod]" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "tags": [ + "nbval-ignore-output" + ] + }, "outputs": [], "source": [ - "# TODO(windmaple): Code example." + "import qwix\n", + "import numpy as np\n", + "from tunix.sft import peft_trainer" ] }, { "cell_type": "markdown", - "id": "3813cbf2", - "metadata": {}, + "id": "55298d1b", + "metadata": { + "id": "cE0Rx6Q3_q4n" + }, + "source": [ + "We set some hyperparameters." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "YzEDIR_N7_D4" + }, + "outputs": [], + "source": [ + "# LoRA Hyperparameters\n", + "lora_rank = 16\n", + "lora_alpha = 2.0\n", + "lora_max_steps = 400\n", + "lora_num_epochs = 10\n", + "lora_batch_size = 80" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mfnj-S02_yM5" + }, + "source": [ + "For LoRA fintuning we use the [Tiny Shakespeare](https://www.tensorflow.org/datasets/catalog/tiny_shakespeare) dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Mtzb0NXb8TVY", + "tags": [ + "nbval-ignore-output" + ] + }, + "outputs": [], + "source": [ + "!wget https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt -O TinyShakespeare.txt\n", + "\n", + "def load_shakespeare_dataset(batch_size, max_len, num_epochs):\n", + " input_file = \"TinyShakespeare.txt\"\n", + " # Read the entire text file\n", + " with open(input_file, 'r', encoding='utf-8') as f:\n", + " text = f.read()\n", + "\n", + " # Tokenize the entire text (assuming tokenizer is available in scope)\n", + " tokens = tokenizer.encode(text)\n", + "\n", + " # Create a simple data source from the tokens\n", + " class TokenDataSource(pygrain.RandomAccessDataSource):\n", + " def __init__(self, tokens, max_len):\n", + " self._tokens = np.array(tokens, dtype=np.int32)\n", + " self._max_len = max_len\n", + " # Calculate how many sequences we can create\n", + " self._length = len(self._tokens) // max_len\n", + "\n", + " def __len__(self):\n", + " return self._length\n", + "\n", + " def __getitem__(self, index):\n", + " # Return a sequence of max_len tokens\n", + " start_idx = index * self._max_len\n", + " end_idx = start_idx + self._max_len\n", + " return self._tokens[start_idx:end_idx]\n", + "\n", + " # Create the data source\n", + " data_source = TokenDataSource(tokens, max_len)\n", + "\n", + " # Create a sampler\n", + " sampler = pygrain.IndexSampler(\n", + " num_records=len(data_source),\n", + " shuffle=True,\n", + " seed=42,\n", + " num_epochs=num_epochs,\n", + " shard_options=pygrain.NoSharding()\n", + " )\n", + "\n", + " # Create transformations\n", + " class ToTrainingInputDict(pygrain.MapTransform):\n", + " def map(self, batch):\n", + " return {\n", + " \"input_tokens\": batch,\n", + " \"input_mask\": np.ones_like(batch)\n", + " }\n", + "\n", + " # Create the data loader\n", + " loader = pygrain.DataLoader(\n", + " data_source=data_source,\n", + " sampler=sampler,\n", + " operations=[\n", + " pygrain.Batch(batch_size=batch_size, drop_remainder=True),\n", + " ToTrainingInputDict(),\n", + " ],\n", + " worker_count=0, # Use main thread\n", + " )\n", + "\n", + " def to_training_input(loader):\n", + " # The trainer expects an iterable of `peft_trainer.TrainingInput`.\n", + " for item in loader:\n", + " yield peft_trainer.TrainingInput(**item)\n", + "\n", + " return to_training_input(loader)\n", + "\n", + "lora_train_ds = load_shakespeare_dataset(lora_batch_size, maxlen, lora_num_epochs)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qvxSxbbcBim7" + }, + "source": [ + "We define a few helper functions to create the LoRA model, loss, etc." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "oX9F-ZsN8ima" + }, + "outputs": [], + "source": [ + "def get_lora_model(base_model, mesh):\n", + " lora_provider = qwix.LoraProvider(\n", + " module_path=\".*mha|.*linear1|.*linear2\",\n", + " rank=lora_rank,\n", + " alpha=lora_alpha,\n", + " )\n", + "\n", + " model_input = base_model.get_model_input()\n", + " lora_model = qwix.apply_lora_to_model(\n", + " base_model, lora_provider, **model_input\n", + " )\n", + "\n", + " with mesh:\n", + " state = nnx.state(lora_model)\n", + " pspecs = nnx.get_partition_spec(state)\n", + " sharded_state = jax.lax.with_sharding_constraint(state, pspecs)\n", + " nnx.update(lora_model, sharded_state)\n", + "\n", + " return lora_model\n", + "\n", + "def gen_model_input_fn(x: peft_trainer.TrainingInput):\n", + " return {\n", + " 'inputs': x.input_tokens,\n", + " 'training': True\n", + " }\n", + "\n", + "def lora_loss_fn(model, inputs, training):\n", + " inputs = inputs\n", + " targets = jnp.concatenate([inputs[:, 1:], jnp.zeros((inputs.shape[0], 1), dtype=jnp.int32)], axis=1)\n", + " logits = model(inputs, training=training)\n", + " loss = optax.softmax_cross_entropy_with_integer_labels(logits=logits, labels=targets).mean()\n", + " return loss" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h4Miifz_BpXf" + }, + "source": [ + "Now we can start the finetuning." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "pIQ5Obif8pfO", + "tags": [ + "nbval-ignore-output" + ] + }, + "outputs": [], + "source": [ + "print(\"Starting LoRA Finetuning...\")\n", + "with mesh:\n", + " # Apply LoRA to the model\n", + " lora_model = get_lora_model(model, mesh)\n", + "\n", + " # Setup Tunix PeftTrainer\n", + " training_config = peft_trainer.TrainingConfig(\n", + " eval_every_n_steps=None,\n", + " max_steps=lora_max_steps,\n", + " data_sharding_axis=('batch',),\n", + " )\n", + " lora_optimizer = optax.adamw(1e-2)\n", + " lora_trainer = peft_trainer.PeftTrainer(\n", + " lora_model, lora_optimizer, training_config\n", + " ).with_gen_model_input_fn(gen_model_input_fn).with_loss_fn(lora_loss_fn)\n", + "\n", + " # Run LoRA training\n", + " lora_trainer.train(lora_train_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UWVdZGK9COj0" + }, + "source": [ + "After the finetuning, you can easily see that now the model produces text of a different style, kind of like Shakespeare's work, which means our finetuning works." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 799 + }, + "id": "-uTeXvXaLCZb", + "outputId": "aa52154f-9583-46d3-896a-05e28de603de", + "tags": [ + "nbval-ignore-output" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Generating text after LoRA finetuning:\n", + "\n", + "\n", + "Once upon a time? I hope I, I am no, I may not be away.\n", + "\n", + "\n", + "DUINGENIINIINGA lie,\n", + "IARICE:\n", + "A nice, I will not be a kind,\n", + "A wise man,\n", + "A, a kind, kind, and my father.\n", + "\n", + "L TOKE:\n", + "You can be in this, my head:\n", + "And this is the man,\n", + "That I will not be your heart;\n", + "To say, for your head\n", + "That I have a dream of my heart; if he will not say that he's the land\n", + "But it is yours:\n", + "And that will be not my dear,\n", + "Your heart, that I will have a heart,\n", + "And to my wife,\n", + "And be so beautiful,\n", + "But for my heart, and I know he will make him\n", + "So my heart.\n", + "\n", + "LET:\n", + "And you will not give my head.\n", + "\n", + "SENKEIARIINIAR TOO, I will not do to have\n", + "And I will do not come\n", + "What this is that you will do, that I have no one.\n", + "\n", + "O:\n", + "I have no one hope.\n", + "\n", + "\n", + "LENENIKE!!!!" + ] + }, + { + "data": { + "text/plain": [ + "\"Once upon a time? I hope I, I am no, I may not be away.\\n\\n\\nDUINGENIINIINGA lie,\\nIARICE:\\nA nice, I will not be a kind,\\nA wise man,\\nA, a kind, kind, and my father.\\n\\nL TOKE:\\nYou can be in this, my head:\\nAnd this is the man,\\nThat I will not be your heart;\\nTo say, for your head\\nThat I have a dream of my heart; if he will not say that he's the land\\nBut it is yours:\\nAnd that will be not my dear,\\nYour heart, that I will have a heart,\\nAnd to my wife,\\nAnd be so beautiful,\\nBut for my heart, and I know he will make him\\nSo my heart.\\n\\nLET:\\nAnd you will not give my head.\\n\\nSENKEIARIINIAR TOO, I will not do to have\\nAnd I will do not come\\nWhat this is that you will do, that I have no one.\\n\\nO:\\nI have no one hope.\\n\\n\\nLENENIKE!!!!\"" + ] + }, + "execution_count": 21, + "metadata": { + "tags": [ + "nbval-ignore-output" + ] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# Generate text with LoRA-finetuned model\n", + "print(\"Generating text after LoRA finetuning:\\n\\n\")\n", + "lora_model.generate_text(maxlen, start_tokens)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3813cbf2" + }, "source": [ "## Xprof: profiling for hyperparameter tuning\n", "\n", @@ -1308,31 +2020,56 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "b5d933c6", + "execution_count": 1, "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "b5d933c6", + "outputId": "f39e18fc-a25f-4202-ab39-9aadfb232522", "tags": [ "nbval-skip" ] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.1/75.1 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.9/14.9 MB\u001b[0m \u001b[31m145.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m109.2/109.2 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m98.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m242.4/242.4 kB\u001b[0m \u001b[31m24.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m221.6/221.6 kB\u001b[0m \u001b[31m22.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m377.3/377.3 kB\u001b[0m \u001b[31m32.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "google-tunix 0.1.3 requires datasets, which is not installed.\n", + "google-tunix 0.1.3 requires hf_transfer, which is not installed.\n", + "google-tunix 0.1.3 requires omegaconf, which is not installed.\n", + "google-tunix 0.1.3 requires python-dotenv, which is not installed.\u001b[0m\u001b[31m\n", + "\u001b[0m" + ] + } + ], "source": [ "!pip install -Uq tensorboard-plugin-profile tensorflow tensorboard" ] }, { "cell_type": "markdown", - "id": "2ac5fc4d", - "metadata": {}, + "metadata": { + "id": "2ac5fc4d" + }, "source": [ "Load the tensorboard colab extension." ] }, { "cell_type": "code", - "execution_count": null, - "id": "74f0c212", + "execution_count": 1, "metadata": { + "id": "74f0c212", "tags": [ "nbval-skip" ] @@ -1344,17 +2081,19 @@ }, { "cell_type": "markdown", - "id": "17c6131f", - "metadata": {}, + "metadata": { + "id": "17c6131f" + }, "source": [ "As we're going to be running this model a number of times, we need some scaffolding to more easily compare our work. For a baseline, we'll need to perform some warmup to guarantee that our code is JIT'd and that our TPUs are warm. For improved comparability, we'll only start tracing after we've finished warmup." ] }, { "cell_type": "code", - "execution_count": null, - "id": "ddfd576e", - "metadata": {}, + "execution_count": 1, + "metadata": { + "id": "ddfd576e" + }, "outputs": [], "source": [ "trace_dir = \"/tmp/jax-trace/\"\n", @@ -1380,17 +2119,18 @@ }, { "cell_type": "markdown", - "id": "de70f5b7", - "metadata": {}, + "metadata": { + "id": "de70f5b7" + }, "source": [ "Now we'll perform some traces to compare results of different batch sizes. This will take several minutes as we need to reprocess our input data to prepare new batches each time." ] }, { "cell_type": "code", - "execution_count": null, - "id": "bc9452a6", + "execution_count": 1, "metadata": { + "id": "bc9452a6", "tags": [ "nbval-skip" ] @@ -1410,8 +2150,9 @@ }, { "cell_type": "markdown", - "id": "ea379965", - "metadata": {}, + "metadata": { + "id": "ea379965" + }, "source": [ "Run Tensorboard with the Profiler Plugin to compare our runs. Runs are listed in order from newest to oldest, so the top run in the list will be have `batch_size = 256`.\n", "\n", @@ -1422,9 +2163,9 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "b86c565a", + "execution_count": 1, "metadata": { + "id": "b86c565a", "tags": [ "nbval-skip" ] @@ -1436,8 +2177,9 @@ }, { "cell_type": "markdown", - "id": "657967a5", - "metadata": {}, + "metadata": { + "id": "657967a5" + }, "source": [ "Next, we can explore alternative parallelism methods. In cell #4, we used 4-way data parallel and 2-way tensor parallel. 8-way data parallel is another popular way. Let's compare results between them. To switch to 8-way data parallel, we'll replace the `Mesh` definition with:\n", "\n", @@ -1450,9 +2192,9 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "80daa8dc", + "execution_count": 1, "metadata": { + "id": "80daa8dc", "tags": [ "nbval-skip" ] @@ -1471,8 +2213,9 @@ }, { "cell_type": "markdown", - "id": "ad96e72b", - "metadata": {}, + "metadata": { + "id": "ad96e72b" + }, "source": [ "Once again we'll run tensorboard.\n", "\n", @@ -1483,9 +2226,9 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "780e9c72", + "execution_count": 1, "metadata": { + "id": "780e9c72", "tags": [ "nbval-skip" ] @@ -1497,28 +2240,24 @@ }, { "cell_type": "markdown", - "id": "deca486e", - "metadata": {}, + "metadata": { + "id": "deca486e" + }, "source": [ "By changing hyperparameters and comparing profiles, we're able to gain significant insights into our bottlenecks and limitations. These are just two examples of hyperparameters to tune, but plenty more of them will have significant effects on training speed and resource utilization." ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "id": "Q3Z_0hQu7est" + }, "source": [ "## Inference with vLLM\n", "\n", - "[REVAMP 8]: Introduce vLLM." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# TODO: Code example." + "After training the miniGPT model, we can also serve it on Google TPUs for high-performance inference.\n", + "\n", + "[vLLM TPU](https://github.com/vllm-project/tpu-inference/) supports running LLMs on TPUs. It takes some additional work to make it work, which is beyond the scope of this tutorial. But feel free to checkout vLLM TPU [documentation](https://docs.vllm.ai/projects/tpu/en/latest/developer_guides/jax_model_development/) if you want to learn more about it." ] }, { @@ -1535,7 +2274,7 @@ "metadata": { "accelerator": "TPU", "colab": { - "gpuType": "V28", + "gpuType": "V5E1", "machine_shape": "hm", "provenance": [] }, diff --git a/docs/source/JAX_for_LLM_pretraining.md b/docs/source/JAX_for_LLM_pretraining.md index c19c5fa..b56948d 100644 --- a/docs/source/JAX_for_LLM_pretraining.md +++ b/docs/source/JAX_for_LLM_pretraining.md @@ -11,9 +11,11 @@ kernelspec: name: python3 --- ++++ {"id": "YdtfHhtq7esh"} + # Train a miniGPT language model with JAX -+++ ++++ {"id": "dNvPJpcW7esj"}
@@ -71,7 +73,7 @@ Check the available JAX devices, or [`jax.Device`](https://jax.readthedocs.io/en colab: base_uri: https://localhost:8080/ id: LS9sQEY3n0mB -outputId: 9ffcf3a6-20ef-4f80-b006-f5d3c5644a15 +outputId: 6b9ee4b0-eed0-4bae-dd99-ffed14289ad7 tags: [nbval-ignore-output] --- import jax @@ -87,7 +89,7 @@ Get the [TinyStories dataset from Hugging Face](https://huggingface.co/datasets/ colab: base_uri: https://localhost:8080/ id: wUjQsgQEmI1N -outputId: e6eff24e-5578-4277-a0f9-24e27bd91ee0 +outputId: a704b8b3-2a1e-48bc-8915-122329a5df52 tags: [nbval-skip] --- !wget https://huggingface.co/datasets/roneneldan/TinyStories/resolve/main/TinyStories-train.txt?download=true -O TinyStories-train.txt @@ -103,7 +105,8 @@ Import the necessary modules, including JAX NumPy, Flax NNX, Optax, Grain, panda import jax import jax.numpy as jnp -from jax.sharding import PartitionSpec as P # For data and model parallelism (explained in more detail later) +# For data and model parallelism (explained in more detail later) +from jax.sharding import PartitionSpec as P, NamedSharding import flax.nnx as nnx import optax @@ -184,7 +187,7 @@ tokenizer = tiktoken.get_encoding("gpt2") To leverage model parallelism, we need to instruct the JAX compiler how to shard the model tensors across the TPU devices. We'll use Flax NNX's [`flax.nnx.with_partitioning`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/spmd.html#flax.nnx.with_partitioning) to let each model layer know that the model weights or tensors need to be sharded according to our specification. We need to do this for every tensor/layer in the model. -`nnx.with_partitioning` will take two arguments, such as the `initializer` (e.g. [`flax.nnx.initializers.xavier_uniform`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/initializers.html#flax.nnx.initializers.xavier_uniform) and [`flax.nnx.initializers.zeros_init`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/initializers.html#flax.nnx.initializers.zeros_init)), and a sharding tuple (e.g. `(None, 'model')` in our case). The sharding tuple describe how to shard a tensor across, for example, the `model` axis or be replicated on other dimensions (which is denoted by `None`). +`nnx.with_partitioning` will take two arguments, such as the `initializer` (e.g. [`flax.nnx.initializers.xavier_uniform`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/initializers.html#flax.nnx.initializers.xavier_uniform) and [`flax.nnx.initializers.zeros_init`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/initializers.html#flax.nnx.initializers.zeros_init)), and a sharding tuple (e.g. `(None, 'model')` in our case) wrapped in [`jax.sharding.PartitionSpec`](https://docs.jax.dev/en/latest/jax.sharding.html#jax.sharding.PartitionSpec). The sharding tuple describe how to shard a tensor across, for example, the `model` axis or be replicated on other dimensions (which is denoted by `None`). For a more detailed discussion of Flax NNX sharding, please refer to [this SPMD guide](https://flax.readthedocs.io/en/latest/guides/flax_gspmd.html). @@ -208,77 +211,63 @@ class TransformerBlock(nnx.Module): rate (float): Dropout rate. Defaults to 0.1. """ def __init__(self, embed_dim: int, num_heads: int, ff_dim: int, *, rngs: nnx.Rngs, rate: float = 0.1): - # Multi-Head Attention (MHA) with `flax.nnx.MultiHeadAttention`. - # Specifies tensor sharding (depending on the mesh configuration) - # where we shard the weights across devices for parallel computation. self.mha = nnx.MultiHeadAttention(num_heads=num_heads, in_features=embed_dim, - kernel_init=nnx.with_partitioning(nnx.initializers.xavier_uniform(), (None, 'model')), - bias_init=nnx.with_partitioning(nnx.initializers.zeros_init(), ('model',)), - decode=True, + kernel_init=nnx.with_partitioning( + nnx.initializers.xavier_uniform(), P(None, 'model')), + bias_init=nnx.with_partitioning( + nnx.initializers.zeros_init(), P('model')), rngs=rngs) - # The first dropout with `flax.nnx.Dropout`. - self.dropout1 = nnx.Dropout(rate=rate) - # First layer normalization with `flax.nnx.LayerNorm`. + self.dropout1 = nnx.Dropout(rate=rate, rngs=rngs) self.layer_norm1 = nnx.LayerNorm(epsilon=1e-6, num_features=embed_dim, - scale_init=nnx.with_partitioning(nnx.initializers.ones_init(), ('model',)), - bias_init=nnx.with_partitioning(nnx.initializers.zeros_init(), ('model',)), + scale_init=nnx.with_partitioning( + nnx.initializers.ones_init(), P('model')), + bias_init=nnx.with_partitioning( + nnx.initializers.zeros_init(), P('model')), rngs=rngs) - # The first linear transformation for the feed-forward network with `flax.nnx.Linear`. self.linear1 = nnx.Linear(in_features=embed_dim, out_features=ff_dim, - kernel_init=nnx.with_partitioning(nnx.initializers.xavier_uniform(), (None, 'model')), - bias_init=nnx.with_partitioning(nnx.initializers.zeros_init(), ('model',)), + kernel_init=nnx.with_partitioning( + nnx.initializers.xavier_uniform(), P(None, 'model')), + bias_init=nnx.with_partitioning( + nnx.initializers.zeros_init(), P('model')), rngs=rngs) - # The second linear transformation for the feed-forward network with `flax.nnx.Linear`. self.linear2 = nnx.Linear(in_features=ff_dim, out_features=embed_dim, - kernel_init=nnx.with_partitioning(nnx.initializers.xavier_uniform(), (None, 'model')), - bias_init=nnx.with_partitioning(nnx.initializers.zeros_init(), ('model',)), + kernel_init=nnx.with_partitioning( + nnx.initializers.xavier_uniform(), P(None, 'model')), + bias_init=nnx.with_partitioning( + nnx.initializers.zeros_init(), P('model')), rngs=rngs) - # The second dropout with `flax.nnx.Dropout`. - self.dropout2 = nnx.Dropout(rate=rate) - # Second layer normalization with `flax.nnx.LayerNorm`. + self.dropout2 = nnx.Dropout(rate=rate, rngs=rngs) self.layer_norm2 = nnx.LayerNorm(epsilon=1e-6, num_features=embed_dim, - scale_init=nnx.with_partitioning(nnx.initializers.ones_init(), ('model',)), - bias_init=nnx.with_partitioning(nnx.initializers.zeros_init(), ('model',)), + scale_init=nnx.with_partitioning( + nnx.initializers.ones_init(), P('model')), + bias_init=nnx.with_partitioning( + nnx.initializers.zeros_init(), P('model')), rngs=rngs) - # Apply the Transformer block to the input sequence. - def __call__(self, inputs, training: bool = False, decode: bool = False): + def __call__(self, inputs, training: bool = False): input_shape = inputs.shape _, seq_len, _ = input_shape - # Instantiate the causal attention mask. - if decode: - mask = None - else: - mask = causal_attention_mask(seq_len) + mask = causal_attention_mask(seq_len) - # Apply Multi-Head Attention with the causal attention mask. attention_output = self.mha( inputs_q=inputs, mask=mask, - decode=decode + decode=False ) - # Apply the first dropout. attention_output = self.dropout1(attention_output, deterministic=not training) - # Apply the first layer normalization. out1 = self.layer_norm1(inputs + attention_output) - # The feed-forward network. - # Apply the first linear transformation. ffn_output = self.linear1(out1) - # Apply the ReLU activation with `flax.nnx.relu`. ffn_output = nnx.relu(ffn_output) - # Apply the second linear transformation. ffn_output = self.linear2(ffn_output) - # Apply the second dropout. ffn_output = self.dropout2(ffn_output, deterministic=not training) - # Apply the second layer normalization and return the output of the Transformer block. return self.layer_norm2(out1 + ffn_output) class TokenAndPositionEmbedding(nnx.Module): @@ -292,21 +281,13 @@ class TokenAndPositionEmbedding(nnx.Module): rngs (flax.nnx.Rngs): A Flax NNX stream of JAX PRNG keys. """ def __init__(self, maxlen: int, vocab_size: int, embed_dim: int, *, rngs: nnx.Rngs): - # Initialize token embeddings (using `flax.nnx.Embed`). - # Each unique word has an embedding vector. self.token_emb = nnx.Embed(num_embeddings=vocab_size, features=embed_dim, rngs=rngs) - # Initialize positional embeddings (using `flax.nnx.Embed`). self.pos_emb = nnx.Embed(num_embeddings=maxlen, features=embed_dim, rngs=rngs) - # Takes a token sequence (integers) and returns the combined token and positional embeddings. - def __call__(self, x, start_index=0): - # Generate a sequence of positions for the input tokens. - positions = start_index + jnp.arange(x.shape[1], dtype=jnp.int32) - # Look up the positional embeddings for each position in the input sequence. + def __call__(self, x): + positions = jnp.arange(0, x.shape[1])[None, :] position_embedding = self.pos_emb(positions) - # Look up the token embeddings for each token in the input sequence. token_embedding = self.token_emb(x) - # Combine token and positional embeddings. return token_embedding + position_embedding class MiniGPT(nnx.Module): @@ -321,84 +302,65 @@ class MiniGPT(nnx.Module): num_transformer_blocks (int): Number of transformer blocks. Each block contains attention and feed-forward networks. rngs (nnx.Rngs): A Flax NNX stream of JAX PRNG keys. """ - # Initialize miniGPT model components. def __init__(self, maxlen: int, vocab_size: int, embed_dim: int, num_heads: int, feed_forward_dim: int, num_transformer_blocks: int, rngs: nnx.Rngs): - # Initiliaze the `TokenAndPositionEmbedding` that combines token and positional embeddings. self.embedding_layer = TokenAndPositionEmbedding( maxlen, vocab_size, embed_dim, rngs=rngs ) - # Create a list of `TransformerBlock` instances. - # Each block processes input sequences using attention and feed-forward networks. self.transformer_blocks = nnx.List([TransformerBlock( embed_dim, num_heads, feed_forward_dim, rngs=rngs ) for _ in range(num_transformer_blocks)]) - # Initialize the output `flax.nnx.Linear` layer producing logits over the vocabulary for next-token prediction. self.output_layer = nnx.Linear(in_features=embed_dim, out_features=vocab_size, - kernel_init=nnx.with_partitioning(nnx.initializers.xavier_uniform(), (None, 'model')), - bias_init=nnx.with_partitioning(nnx.initializers.zeros_init(), ('model',)), + kernel_init=nnx.with_partitioning( + nnx.initializers.xavier_uniform(), P(None, 'model')), + bias_init=nnx.with_partitioning( + nnx.initializers.zeros_init(), P('model')), rngs=rngs) - def __call__(self, inputs, training: bool = False, decode: bool = False): - # Pass the input tokens through the `embedding_layer` to get token embeddings. - # Apply each transformer block sequentially to the embedded input, use the `training` flag for the behavior of `flax.nnx.Dropout`. - if decode: - cache_index = self.transformer_blocks[0].mha.cache_index.value - x = self.embedding_layer(inputs, start_index=cache_index) - else: - x = self.embedding_layer(inputs) + def __call__(self, inputs, training: bool = False): + x = self.embedding_layer(inputs) for transformer_block in self.transformer_blocks: - x = transformer_block(x, training=training, decode=decode) - # Pass the output of the transformer blocks through the output layer, - # and obtain logits for each token in the vocabulary (for next token prediction). + x = transformer_block(x, training=training) outputs = self.output_layer(x) return outputs - def init_cache(self, input_shape, dtype=jnp.float32): - for block in self.transformer_blocks: - block.mha.init_cache(input_shape, dtype) + # For Tunix use later + def get_model_input(self): + return dict( + inputs=jnp.zeros((batch_size, maxlen), dtype=jnp.int32), + training=False + ) @nnx.jit - def sample_from(self, logits): + def sample_from(self, rng_key, logits): logits, indices = jax.lax.top_k(logits, k=top_k) logits = nnx.softmax(logits) - return jax.random.choice(jax.random.PRNGKey(0), indices, p=logits) + return jax.random.choice(rng_key, indices, p=logits) @nnx.jit - def decode_step(self, inputs): - logits = self(inputs, decode=True) - next_token = self.sample_from(logits[0, -1, :]) + def generate_step(self, rng_key, padded_tokens, sample_index): + logits = self(padded_tokens) + next_token = self.sample_from(rng_key, logits[0][sample_index]) return next_token - @nnx.jit - def prefill_step(self, token): - self(token, decode=True) - def generate_text(self, max_tokens, start_tokens): - self.init_cache((1, maxlen, embed_dim)) generated = [] + rng_key = jax.random.PRNGKey(0) # Create the initial key print(tokenizer.decode(start_tokens), flush=True, end='') + for i in range(max_tokens): + sample_index = len(start_tokens) + len(generated) - 1 - # Prefill KV cache - if len(start_tokens) > 1: - prompt_tokens = jnp.array(start_tokens)[None, :-1] - # NNX built-in MHA only supports KV caching one token at a time - for i in range(prompt_tokens.shape[1]): - token = prompt_tokens[:, i:i+1] - self.prefill_step(token) + # Split the key for each step + rng_key, step_key = jax.random.split(rng_key) - # Decode - next_input = jnp.array([start_tokens[-1]])[None, :] - for i in range(max_tokens): - next_token = int(self.decode_step(next_input)) + padded_tokens = jnp.array((start_tokens + generated + [0] * (maxlen - len(start_tokens) - len(generated))))[None, :] + next_token = int(self.generate_step(step_key, padded_tokens, sample_index)) if next_token == tokenizer.encode('<|endoftext|>', allowed_special={'<|endoftext|>'})[0]: break generated.append(next_token) print(tokenizer.decode([next_token]), flush=True, end='') - next_input = jnp.array([next_token])[None, :] return tokenizer.decode(start_tokens + generated) -# Creates the miniGPT model with 4 transformer blocks. def create_model(rngs): return MiniGPT(maxlen, vocab_size, embed_dim, num_heads, feed_forward_dim, num_transformer_blocks=4, rngs=rngs) ``` @@ -477,6 +439,7 @@ def load_and_preprocess_data(file_path, batch_size, maxlen): ```{code-cell} :cellView: form +:id: b9tmfMzj7eso :tags: [hide-cell] # @title [hidden cell; used for testing] @@ -525,6 +488,8 @@ if AI_STACK_TEST_MODE: ``` ```{code-cell} +:id: VnfW4Z7l7eso + text_dl = load_and_preprocess_data('TinyStories-train.txt', batch_size, maxlen) ``` @@ -556,7 +521,7 @@ def train_step(model: MiniGPT, optimizer: nnx.Optimizer, metrics: nnx.MultiMetri [REVAMP 3]: Expand the intro to Optax. -Start training. It takes ~50 minutes on Colab. +Start training. It takes ~20 minutes on Colab TPU v5e-1. Note that for data parallel, we are sharding the training data along the `batch` axis using `jax.device_put`. @@ -567,13 +532,12 @@ We are also using the `jax.vmap` transformation to produce the target sequences colab: base_uri: https://localhost:8080/ id: Ysl6CsfENeJN -outputId: 5dd06dca-f030-4927-a9b6-35d412da535c +outputId: ba3051ad-0e11-4570-a223-28f35ca505e0 tags: [nbval-ignore-output] --- with mesh: - model = create_model(rngs=nnx.Rngs(0)) - optimizer = nnx.Optimizer(model, optax.adam(1e-3), wrt=nnx.Param) - + model = create_model(rngs=nnx.Rngs(0)) + optimizer = nnx.Optimizer(model, optax.adam(1e-3), wrt=nnx.Param) metrics = nnx.MultiMetric( loss=nnx.metrics.Average("loss"), ) @@ -600,15 +564,14 @@ for epoch in range(num_epochs): continue # skip the remaining elements input_batch = jnp.array(jnp.array(batch).T) target_batch = prep_target_batch(input_batch) - with jax.set_mesh(mesh): - train_step( - model, - optimizer, - metrics, - jax.device_put( - (input_batch, target_batch), P("batch", None) - ), - ) + train_step( + model, + optimizer, + metrics, + jax.device_put( + (input_batch, target_batch), NamedSharding(mesh, P("batch", None)) + ), + ) if (step + 1) % 200 == 0: for metric, value in metrics.compute().items(): @@ -641,7 +604,7 @@ colab: base_uri: https://localhost:8080/ height: 472 id: B6Eg1Cz2y_iP -outputId: 7cafe711-1ae4-4eb9-fd37-e1bde54cbfc5 +outputId: 60e54019-dcdc-425b-e95a-c03e301d688f tags: [nbval-ignore-output] --- import matplotlib.pyplot as plt @@ -656,7 +619,7 @@ plt.show() As you can see, the model goes from generating completely random words at the beginning to generating sensible tiny stories at the end of the training. So essentially we have pretrained a small LLM to write tiny stories for us. -+++ ++++ {"id": "d1_l1Pr-7eso"} ## Debugging the model @@ -675,7 +638,7 @@ Save the model checkpoint. colab: base_uri: https://localhost:8080/ id: EkoFGCgSZ1yz -outputId: 3467b8ba-ce05-42f0-fb89-75922cc91e31 +outputId: 593af986-77a7-4665-eee4-5deb2d891b7f tags: [nbval-skip] --- import orbax.checkpoint as orbax @@ -689,14 +652,216 @@ checkpointer.save('/content/save', args=orbax.args.PyTreeSave(state), force=True !ls /content/save/ ``` ++++ {"id": "azzwiMeP7eso"} + ## Tunix: Fine-tuning -[REVAMP 6] Introduce Tunix. +[Tunix](https://github.com/google/tunix) is a JAX-native LLM post-training library open sourced by Google. It supports a range of post-training techniques including supervised finetuning, preference tuning, reinforcement learning and model distillation. In this section, we are going to use Tunix to finetune the miniGPT model we just pretrained using LoRA ([Low-Rank Adaptation](https://arxiv.org/abs/2106.09685)) so that the finetuned model generates output of a different style. + ++++ {"id": "-kJShd9n_iTl"} + +First we install Tunix and its dependencies, and import necessary libraries. Note that Colab will ask you to restart the runtime, but you can just ignore it. + +```{code-cell} +--- +colab: + base_uri: https://localhost:8080/ +id: uipLvy9E7eso +outputId: 93cf001f-2964-467c-9848-b106be6cacf5 +tags: [nbval-ignore-output] +--- +!pip install google-tunix[prod] +``` ```{code-cell} -# TODO(windmaple): Code example. +:tags: [nbval-ignore-output] + +import qwix +import numpy as np +from tunix.sft import peft_trainer ``` ++++ {"id": "cE0Rx6Q3_q4n"} + +We set some hyperparameters. + +```{code-cell} +:id: YzEDIR_N7_D4 + +# LoRA Hyperparameters +lora_rank = 16 +lora_alpha = 2.0 +lora_max_steps = 400 +lora_num_epochs = 10 +lora_batch_size = 80 +``` + ++++ {"id": "mfnj-S02_yM5"} + +For LoRA fintuning we use the [Tiny Shakespeare](https://www.tensorflow.org/datasets/catalog/tiny_shakespeare) dataset. + +```{code-cell} +:id: Mtzb0NXb8TVY +:tags: [nbval-ignore-output] + +!wget https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt -O TinyShakespeare.txt + +def load_shakespeare_dataset(batch_size, max_len, num_epochs): + input_file = "TinyShakespeare.txt" + # Read the entire text file + with open(input_file, 'r', encoding='utf-8') as f: + text = f.read() + + # Tokenize the entire text (assuming tokenizer is available in scope) + tokens = tokenizer.encode(text) + + # Create a simple data source from the tokens + class TokenDataSource(pygrain.RandomAccessDataSource): + def __init__(self, tokens, max_len): + self._tokens = np.array(tokens, dtype=np.int32) + self._max_len = max_len + # Calculate how many sequences we can create + self._length = len(self._tokens) // max_len + + def __len__(self): + return self._length + + def __getitem__(self, index): + # Return a sequence of max_len tokens + start_idx = index * self._max_len + end_idx = start_idx + self._max_len + return self._tokens[start_idx:end_idx] + + # Create the data source + data_source = TokenDataSource(tokens, max_len) + + # Create a sampler + sampler = pygrain.IndexSampler( + num_records=len(data_source), + shuffle=True, + seed=42, + num_epochs=num_epochs, + shard_options=pygrain.NoSharding() + ) + + # Create transformations + class ToTrainingInputDict(pygrain.MapTransform): + def map(self, batch): + return { + "input_tokens": batch, + "input_mask": np.ones_like(batch) + } + + # Create the data loader + loader = pygrain.DataLoader( + data_source=data_source, + sampler=sampler, + operations=[ + pygrain.Batch(batch_size=batch_size, drop_remainder=True), + ToTrainingInputDict(), + ], + worker_count=0, # Use main thread + ) + + def to_training_input(loader): + # The trainer expects an iterable of `peft_trainer.TrainingInput`. + for item in loader: + yield peft_trainer.TrainingInput(**item) + + return to_training_input(loader) + +lora_train_ds = load_shakespeare_dataset(lora_batch_size, maxlen, lora_num_epochs) +``` + ++++ {"id": "qvxSxbbcBim7"} + +We define a few helper functions to create the LoRA model, loss, etc. + +```{code-cell} +:id: oX9F-ZsN8ima + +def get_lora_model(base_model, mesh): + lora_provider = qwix.LoraProvider( + module_path=".*mha|.*linear1|.*linear2", + rank=lora_rank, + alpha=lora_alpha, + ) + + model_input = base_model.get_model_input() + lora_model = qwix.apply_lora_to_model( + base_model, lora_provider, **model_input + ) + + with mesh: + state = nnx.state(lora_model) + pspecs = nnx.get_partition_spec(state) + sharded_state = jax.lax.with_sharding_constraint(state, pspecs) + nnx.update(lora_model, sharded_state) + + return lora_model + +def gen_model_input_fn(x: peft_trainer.TrainingInput): + return { + 'inputs': x.input_tokens, + 'training': True + } + +def lora_loss_fn(model, inputs, training): + inputs = inputs + targets = jnp.concatenate([inputs[:, 1:], jnp.zeros((inputs.shape[0], 1), dtype=jnp.int32)], axis=1) + logits = model(inputs, training=training) + loss = optax.softmax_cross_entropy_with_integer_labels(logits=logits, labels=targets).mean() + return loss +``` + ++++ {"id": "h4Miifz_BpXf"} + +Now we can start the finetuning. + +```{code-cell} +:id: pIQ5Obif8pfO +:tags: [nbval-ignore-output] + +print("Starting LoRA Finetuning...") +with mesh: + # Apply LoRA to the model + lora_model = get_lora_model(model, mesh) + + # Setup Tunix PeftTrainer + training_config = peft_trainer.TrainingConfig( + eval_every_n_steps=None, + max_steps=lora_max_steps, + data_sharding_axis=('batch',), + ) + lora_optimizer = optax.adamw(1e-2) + lora_trainer = peft_trainer.PeftTrainer( + lora_model, lora_optimizer, training_config + ).with_gen_model_input_fn(gen_model_input_fn).with_loss_fn(lora_loss_fn) + + # Run LoRA training + lora_trainer.train(lora_train_ds) +``` + ++++ {"id": "UWVdZGK9COj0"} + +After the finetuning, you can easily see that now the model produces text of a different style, kind of like Shakespeare's work, which means our finetuning works. + +```{code-cell} +--- +colab: + base_uri: https://localhost:8080/ + height: 799 +id: -uTeXvXaLCZb +outputId: aa52154f-9583-46d3-896a-05e28de603de +tags: [nbval-ignore-output] +--- +# Generate text with LoRA-finetuned model +print("Generating text after LoRA finetuning:\n\n") +lora_model.generate_text(maxlen, start_tokens) +``` + ++++ {"id": "3813cbf2"} + ## Xprof: profiling for hyperparameter tuning [REVAMP 7]: Intro xprof @@ -704,22 +869,34 @@ checkpointer.save('/content/save', args=orbax.args.PyTreeSave(state), force=True **Note:** this section assume multiple TPU cores. Free-tier Colab TPU v5e-1 cannot run here. ```{code-cell} -:tags: [nbval-skip] - +--- +colab: + base_uri: https://localhost:8080/ +id: b5d933c6 +outputId: f39e18fc-a25f-4202-ab39-9aadfb232522 +tags: [nbval-skip] +--- !pip install -Uq tensorboard-plugin-profile tensorflow tensorboard ``` ++++ {"id": "2ac5fc4d"} + Load the tensorboard colab extension. ```{code-cell} +:id: 74f0c212 :tags: [nbval-skip] %load_ext tensorboard ``` ++++ {"id": "17c6131f"} + As we're going to be running this model a number of times, we need some scaffolding to more easily compare our work. For a baseline, we'll need to perform some warmup to guarantee that our code is JIT'd and that our TPUs are warm. For improved comparability, we'll only start tracing after we've finished warmup. ```{code-cell} +:id: ddfd576e + trace_dir = "/tmp/jax-trace/" def loop_step(batch, step): @@ -741,9 +918,12 @@ def generate_trace(): jax.profiler.stop_trace() ``` ++++ {"id": "de70f5b7"} + Now we'll perform some traces to compare results of different batch sizes. This will take several minutes as we need to reprocess our input data to prepare new batches each time. ```{code-cell} +:id: bc9452a6 :tags: [nbval-skip] trace_dir = "/tmp/jax-trace-batch-comparison/" @@ -757,6 +937,8 @@ text_dl = iter(load_and_preprocess_data('TinyStories-train.txt', batch_size, max generate_trace() ``` ++++ {"id": "ea379965"} + Run Tensorboard with the Profiler Plugin to compare our runs. Runs are listed in order from newest to oldest, so the top run in the list will be have `batch_size = 256`. The key metrics to focus on here for this hyperparameter are FLOPS Utilization and Average Step Time. @@ -764,11 +946,14 @@ The key metrics to focus on here for this hyperparameter are FLOPS Utilization a In general, we want to maximize FLOPS Utilization while minimizing the step time per training example. In this case, we can see that increasing the batch size from 64 -> 256 achieves both of those. FLOPS increases from 16% to 27%. Average Step Time increase from 100ms to 260ms, however we increased our batch size by 300%. This means we move from 1.5ms per training example to 1.02ms per training example. ```{code-cell} +:id: b86c565a :tags: [nbval-skip] %tensorboard --logdir=$trace_dir ``` ++++ {"id": "657967a5"} + Next, we can explore alternative parallelism methods. In cell #4, we used 4-way data parallel and 2-way tensor parallel. 8-way data parallel is another popular way. Let's compare results between them. To switch to 8-way data parallel, we'll replace the `Mesh` definition with: `mesh = jax.make_mesh((8, 1), ('batch', 'model'))` @@ -778,6 +963,7 @@ JAX will automatically figure out how to shard the model and data to use the new How simple and powerful is this! And that's the beauty of JAX automatic parallelism. ```{code-cell} +:id: 80daa8dc :tags: [nbval-skip] trace_dir = "/tmp/jax-trace-parallelism-comparison/" @@ -790,6 +976,8 @@ mesh = jax.make_mesh((jax.device_count(), 1), ('batch', 'model')) generate_trace() ``` ++++ {"id": "ad96e72b"} + Once again we'll run tensorboard. Looking at the results, we see that the step times are nearly the same, however the FLOPS Utilization is at 13% for 8-way data parallelism compared to 27% or 4-way data parallelism. @@ -797,22 +985,25 @@ Looking at the results, we see that the step times are nearly the same, however By looking at the Trace Viewer tool and looking under each TPU's ops, we can see that the TPUs spend a large amount of time idle while waiting for the host, as well as spending a good amount of time in `reduce_sum` operations. ```{code-cell} +:id: 780e9c72 :tags: [nbval-skip] %tensorboard --logdir=$trace_dir ``` ++++ {"id": "deca486e"} + By changing hyperparameters and comparing profiles, we're able to gain significant insights into our bottlenecks and limitations. These are just two examples of hyperparameters to tune, but plenty more of them will have significant effects on training speed and resource utilization. -+++ ++++ {"id": "Q3Z_0hQu7est"} ## Inference with vLLM -[REVAMP 8]: Introduce vLLM. +After training the miniGPT model, we can also serve it on Google TPUs for high-performance inference. -```{code-cell} -# TODO: Code example. -``` +[vLLM TPU](https://github.com/vllm-project/tpu-inference/) supports running LLMs on TPUs. It takes some additional work to make it work, which is beyond the scope of this tutorial. But feel free to checkout vLLM TPU [documentation](https://docs.vllm.ai/projects/tpu/en/latest/developer_guides/jax_model_development/) if you want to learn more about it. + ++++ ## Wrapup