diff --git a/synthetic_qna/notebook/Synthetic_question_generation.ipynb b/synthetic_qna/notebook/Synthetic_question_generation.ipynb index 95d1e8d..83fab4b 100644 --- a/synthetic_qna/notebook/Synthetic_question_generation.ipynb +++ b/synthetic_qna/notebook/Synthetic_question_generation.ipynb @@ -1,350 +1,371 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "DbJadYoR6RG5", + "outputId": "7cb3cac1-9628-4f61-d7c1-ac84bd3b3dbc" + }, + "outputs": [], + "source": [ + "pip install transformers" + ] }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "9b767be8aed24fb39059aa2b9c3654e8": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_472473ba00164204bbdd1a44dd342ab9", - "IPY_MODEL_cc928d6deef94fab8fe786c0bedbf73c", - "IPY_MODEL_632aabf5fedb414a885185a9f5526221" - ], - "layout": "IPY_MODEL_f9e19dd1e2354b10b24b1e63937ede17" - } + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 345, + "referenced_widgets": [ + "9b767be8aed24fb39059aa2b9c3654e8", + "472473ba00164204bbdd1a44dd342ab9", + "cc928d6deef94fab8fe786c0bedbf73c", + "632aabf5fedb414a885185a9f5526221", + "f9e19dd1e2354b10b24b1e63937ede17", + "fe40ddbd2d6e4aefb20037a0b5efd460", + "29d7e51f5460441fa0f9bea429374aaf", + "47b70e36aa9148fcb649e440f7d4488d", + "e642b7c3f7314ff085da91090a15a2d4", + "5783d5bee3004c2db2856aae10001d7e", + "4e4a0c111d31440bac8b40fb8db2d57c", + "926a5fe5fde94ffa9800d752a619492a", + "41766800d32e4d188a2b7d3d963e96c8", + "74c771516e1b420e8120842583b433bc", + "d74d257a4c224496abd57949dfec3b0e", + "df53015a9ef34785a76a5a1891d5c83a", + "8aaedb8a8690458ba546944fc6846d7b", + "d6c52af94f934cc39983446f150a7df1", + "3e67799a8c4d44cdab5774920de764d6", + "615eddccd9384a84ad4f7d8c168f80fd", + "19435e5eef5b4d2f85655e32640d6418", + "84c78a6caaf2489eba798bd07007826d", + "2e42f858840b4bb58f15379896fe98f8", + "5e28e78b98384c1dbe19e6d453dfb51e", + "6d4cbc443392422e82de520a9f1c85af", + "d0c4a6469a9541819997c1567ee05573", + "349f2ef6a3e14f15b658ba8f61c5f703", + "72bfece9336a4f3ab09d57173bf98fa0", + "6d5a81bc14794e6187173220300d1f59", + "fe432429208441e68bb60d0eaa2f7fe7", + "cc8d8251655f4302b58ff54ce47c819b", + "255b730bb3214bad82964b92852729c3", + "9242cfbcfb8b4fb6b993520132b9796b", + "7a7ef6de19bc4614bca900bd55bccbc2", + "3a662063172b45c6a32a7e8a181c37cf", + "5c5292a1bb0245e698b96dd0f3982d32", + "600b5184270549cdb46d592179f90374", + "383574d53cef42efa63abcd4aae8b49d", + "8fbe8c8cc3f9425abc340de18622d80f", + "01b5c326e1ec45f2afee2b377b0df59c", + "54ec359589a2419a8476db6379f3d4f3", + "36b539b14dc64afcb7b6135ee0b6b689", + "9f52486613ef4aefb199706c73deb331", + "9e377916bf7a48eb8973f8a2477b13c5", + "93ca5e1831244ae88ed32b9cb640bf3c", + "66007cf24b1e4b20b5eb08c3ffdc5c84", + "d35fd85835c14f62a877da7960f74c91", + "c8d5871c92a44ecb9a08305fc89352b3", + "e177a792e74149f4880599da63c46f8a", + "d60886b71f59448eaf120fa450fd6b87", + "fb5780f8d6914defbcdde4153775c327", + "d45457b6bf6948c58c6d45234446c0e8", + "18c00caf53cd43daa5228b3ecfa06509", + "a3f53759d613418d8be64d031cf1d6d1", + "06fd4c171408476e8758eeadb026850a", + "9b01bff0eda44e91affdab3f2a81367f", + "88a68d637ed44ba696cad0b2a7b57bae", + "261711e56196422496f96f409806ffda", + "00e9f95e9ef54ac49e66f9d4be19876d", + "86122205ff2c44449d7e399a83b4a87a", + "aff66d7130e047eaa305a7f7d5973ea4", + "383ce20098794b528753053f13ac15ca", + "e5abc89d6b374783bf2d8db9adfa0797", + "99201c248f484e67af2aa7c3359dc48c", + "844ce3ab18514d95a1d428d97685afb7", + "d99b117b8f94496f8eae3999fcb0d4d8", + "2c207b1890874361b8982282ca9b4261", + "c2c37df97da841a68a3da1ae5e3ec015", + "5447914abd1748a091133de016ac7aa8", + "3c06c3b147aa4e75a7a0fc8f4d4798c8", + "7a01e2da228b4edea66d9a88ddf2e00c", + "5685e313938c4547809a9f1e3b8fe367", + "c0086b3f54ae449d8a5c0790558c2f53", + "6718b6b04ae74775885c24255b974f2e", + "8b0c2fba3220466f8aa4e6b4f4de8007", + "9419db4afd554df3acf18ffa8c4fcfa5", + "10cf9adaf5bc45c4810e984cf9789e9f" + ] }, - "472473ba00164204bbdd1a44dd342ab9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_fe40ddbd2d6e4aefb20037a0b5efd460", - "placeholder": "​", - "style": "IPY_MODEL_29d7e51f5460441fa0f9bea429374aaf", - "value": "config.json: 100%" - } + "id": "X7Q3HApc6TSR", + "outputId": "ed472da5-b16c-4315-ef02-eaa355672662" + }, + "outputs": [], + "source": [ + "from transformers import T5ForConditionalGeneration, T5TokenizerFast\n", + "hfmodel = T5ForConditionalGeneration.from_pretrained(\"ThomasSimonini/t5-end2end-question-generation\")\n", + "checkpoint = \"t5-base\"\n", + "model = T5ForConditionalGeneration.from_pretrained(checkpoint)\n", + "tokenizer = T5TokenizerFast.from_pretrained(checkpoint)\n", + "tokenizer.sep_token = ''\n", + "tokenizer.add_tokens([''])\n", + "model.resize_token_embeddings(len(tokenizer))\n", + "# Check the sep_token_id to verify that it was added to the tokenizer\n", + "tokenizer.sep_token_id\n", + "\n", + "def hf_run_model(input_string, **generator_args):\n", + " generator_args = {\n", + " \"max_length\": 256,\n", + " \"num_beams\": 4,\n", + " \"length_penalty\": 1.5,\n", + " \"no_repeat_ngram_size\": 3,\n", + " \"early_stopping\": True,\n", + " }\n", + " input_string = \"generate questions: \" + input_string + \" \"\n", + " input_ids = tokenizer.encode(input_string, return_tensors=\"pt\")\n", + " res = hfmodel.generate(input_ids, **generator_args)\n", + " output = tokenizer.batch_decode(res, skip_special_tokens=True)\n", + " output = [item.split(\"\") for item in output]\n", + " return output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, - "cc928d6deef94fab8fe786c0bedbf73c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_47b70e36aa9148fcb649e440f7d4488d", - "max": 1417, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_e642b7c3f7314ff085da91090a15a2d4", - "value": 1417 - } + "id": "asp_1law7Qdo", + "outputId": "53c58077-bae9-494b-85c1-c13f8a6f137e" + }, + "outputs": [], + "source": [ + "\n", + "text = \"Forrest Gump is a 1994 American comedy-drama film directed by Robert Zemeckis and written by Eric Roth. \\\n", + "It is based on the 1986 novel of the same name by Winston Groom and stars Tom Hanks, Robin Wright, Gary Sinise, \\\n", + "Mykelti Williamson and Sally Field. The story depicts several decades in the life of Forrest Gump (Hanks), \\\n", + "a slow-witted but kind-hearted man from Alabama who witnesses and unwittingly influences several defining \\\n", + "historical events in the 20th century United States. The film differs substantially from the novel.\"\n", + "\n", + "hf_run_model(text)\n", + "\n", + "#output\n", + "#[['Who directed the 1994 film Forrest Gump?',\n", + "# ' Who wrote the 1994 movie?',\n", + "# ' What is the film based on?',\n", + "# ' Which movie stars Tom Hanks, Robin Wright and Gary Sinise?',\n", + "# '']]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, - "632aabf5fedb414a885185a9f5526221": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5783d5bee3004c2db2856aae10001d7e", - "placeholder": "​", - "style": "IPY_MODEL_4e4a0c111d31440bac8b40fb8db2d57c", - "value": " 1.42k/1.42k [00:00<00:00, 30.2kB/s]" - } - }, - "f9e19dd1e2354b10b24b1e63937ede17": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "fe40ddbd2d6e4aefb20037a0b5efd460": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "29d7e51f5460441fa0f9bea429374aaf": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } + "id": "NQGX55FE7kn9", + "outputId": "df94ee87-9e65-4ffb-9137-48695249a470" + }, + "outputs": [], + "source": [ + "text_passage = \"A vector in this context is a mathematical representation of an object or data point, \\\n", + "where each element of the vector corresponds to a specific feature or attribute of the object. \\\n", + "For example, in an image recognition system, a vector could represent an image, with each element of the \\\n", + "vector representing a pixel value or a descriptor/characteristic of that pixel. In a music recommendation system, \\\n", + "each vector would represent a song, and each element of the vector would represent a characteristic song such as tempo,\\\n", + " genre, lyrics, and so on. Vector databases are optimized for **storing** and **querying** these high-dimensional vectors efficiently, \\\n", + " and they often using specialized data structures and indexing techniques such as Hierarchical Navigable Small World (HNSW) -- \\\n", + " which is used to implement Approximate Nearest Neighbors -- and Product Quantization, among others. \\\n", + " These databases enable fast similarity and semantic search while allowing users to find vectors that are the closest to a given\\\n", + " query vector based on some distance metric. The most commonly used distance metrics are Euclidean Distance, \\\n", + " Cosine Similarity, and Dot Product, and these three are fully supported Qdrant.\"\n", + "\n", + "\n", + "hf_run_model(text_passage)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "miWYMiUR79rg" + }, + "outputs": [], + "source": [ + "text_vdb = \"## What Are Vector Databases? ![dbs](https://raw.githubusercontent.com/ramonpzg/mlops-sydney-2023/main/images/databases.png) \\\n", + " Vector databases are a type of database designed to store and query high-dimensional vectors efficiently. \\\n", + " In traditional [OLTP](https://www.ibm.com/topics/oltp) and [OLAP](https://www.ibm.com/topics/olap) \\\n", + " databases (as seen in the image above), data is organized in rows and columns (and these are called **Tables**), \\\n", + " and queries are performed based on the values in those columns. However, in certain applications including image recognition,\\\n", + " natural language processing, and recommendation systems, data is often represented as vectors in a high-dimensional space, and these vectors,\\\n", + " plus an id and a payload, are the elements we store in something called a **Collection** a vector database like Qdrant.\"\n", + "\n", + "text_vdb1 = \"What is Qdrant? [Qdrant](http://qdrant.tech) is a vector similarity search engine that provides a \\\n", + "production-ready service with a convenient API to store, search, and manage points (i.e. vectors) with an additional payload.\\\n", + " You can think of the payloads as additional pieces of information that can help you hone in on your search and also receive \\\n", + " useful information that you can give to your users. You can get started using Qdrant with the Python `qdrant-client`, \\\n", + " by pulling the latest docker image of `qdrant` and connecting to it locally, or by trying out [Qdrant's Cloud](https://cloud.qdrant.io/) \\\n", + " free tier option until you are ready to make the full switch. With that out of the way, let's talk about what are vector databases.\"\n", + "\n", + "#hf_run_model(text_vdb)\n", + "\n", + "#hf_run_model(text_vdb1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "c4T3BdeaBtMw" + }, + "outputs": [], + "source": [ + "!pip install openai" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "uvDVRY1N9oPq" + }, + "outputs": [], + "source": [ + "import openai\n", + "from getpass import getpass\n", + "import os\n", + "\n", + "if not (OPENAI_API_KEY := os.getenv(\"OPENAI_API_KEY\")):\n", + " OPENAI_API_KEY = getpass(\"🔑 Enter your OpenAI API key: \")\n", + "openai.api_key = OPENAI_API_KEY\n", + "os.environ[\"OPENAI_API_KEY\"] = OPENAI_API_KEY\n", + "\n", + "model =\"gpt-3.5-turbo\"\n", + "client = OpenAI()\n", + "\n", + " # Function to generate synthetic question-answer pairs\n", + "def generate_question_answer(context):\n", + "\n", + " prompt = f\"\"\"Generate a question and answer pair based keeping in mind the following:\n", + " Please generate a clear and concise question that requires understanding of the content provided in the document chunk. Ensure that the question is specific, relevant, and not too broad.\n", + " Avoid questions such as 'in the given passage or document chunk' kind of questions. Ensure the question is about the concept the document chunk is about.\n", + " Provide a complete , detailed and accurate answer to the question. Make sure that the answer addresses the question directly and comprehensively, drawing from the information provided in the document chunk.\n", + " Use technical terminology appropriately and maintain clarity throughout the response.\n", + " Based on this Context : {context}\n", + " \"\"\"\n", + "\n", + " response = client.chat.completions.create(\n", + " model = model,\n", + " messages = [\n", + " {'role': 'user', 'content': prompt}\n", + " ],\n", + " temperature=0.5,\n", + " max_tokens=200,\n", + " n=1,\n", + " stop=None\n", + " )\n", + " return response.choices[0].message.content\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, - "47b70e36aa9148fcb649e440f7d4488d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } + "id": "hkaOV6Pu-Geu", + "outputId": "3fb112ac-61cb-42be-cacf-cd2735480d61" + }, + "outputs": [], + "source": [ + "# Generate question-answer pairs for each chunk\n", + "\n", + "\n", + "context = \"What is Qdrant? [Qdrant](http://qdrant.tech) is a vector similarity search engine that provides a \\\n", + "production-ready service with a convenient API to store, search, and manage points (i.e. vectors) with an additional payload.\\\n", + "You can think of the payloads as additional pieces of information that can help you hone in on your search and also receive \\\n", + "useful information that you can give to your users. You can get started using Qdrant with the Python `qdrant-client`, \\\n", + "by pulling the latest docker image of `qdrant` and connecting to it locally, or by trying out [Qdrant's Cloud](https://cloud.qdrant.io/) \\\n", + "free tier option until you are ready to make the full switch. With that out of the way, let's talk about what are vector databases.\"\n", + "\n", + "question_answer_pair = generate_question_answer(context)\n", + "print(question_answer_pair)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, - "e642b7c3f7314ff085da91090a15a2d4": { + "id": "ELPdWrEqC6D2", + "outputId": "04329ca6-4c82-453c-e1fe-455a1fcdbd24" + }, + "outputs": [], + "source": [ + "# Generate question-answer pairs for each chunk\n", + "question_answer_pair = generate_question_answer(text_vdb1)\n", + "print(question_answer_pair)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "00e9f95e9ef54ac49e66f9d4be19876d": { "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", + "model_name": "HTMLModel", "state": { + "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "5783d5bee3004c2db2856aae10001d7e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", + "_model_name": "HTMLModel", "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_844ce3ab18514d95a1d428d97685afb7", + "placeholder": "​", + "style": "IPY_MODEL_d99b117b8f94496f8eae3999fcb0d4d8", + "value": " 792k/792k [00:00<00:00, 5.48MB/s]" } }, - "4e4a0c111d31440bac8b40fb8db2d57c": { + "01b5c326e1ec45f2afee2b377b0df59c": { "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", @@ -356,98 +377,56 @@ "description_width": "" } }, - "926a5fe5fde94ffa9800d752a619492a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_41766800d32e4d188a2b7d3d963e96c8", - "IPY_MODEL_74c771516e1b420e8120842583b433bc", - "IPY_MODEL_d74d257a4c224496abd57949dfec3b0e" - ], - "layout": "IPY_MODEL_df53015a9ef34785a76a5a1891d5c83a" - } - }, - "41766800d32e4d188a2b7d3d963e96c8": { + "06fd4c171408476e8758eeadb026850a": { "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", "state": { - "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", + "_model_name": "DescriptionStyleModel", "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_8aaedb8a8690458ba546944fc6846d7b", - "placeholder": "​", - "style": "IPY_MODEL_d6c52af94f934cc39983446f150a7df1", - "value": "pytorch_model.bin: 100%" + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" } }, - "74c771516e1b420e8120842583b433bc": { + "10cf9adaf5bc45c4810e984cf9789e9f": { "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", "state": { - "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", + "_model_name": "DescriptionStyleModel", "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_3e67799a8c4d44cdab5774920de764d6", - "max": 891647935, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_615eddccd9384a84ad4f7d8c168f80fd", - "value": 891647935 + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" } }, - "d74d257a4c224496abd57949dfec3b0e": { + "18c00caf53cd43daa5228b3ecfa06509": { "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", "state": { - "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", + "_model_name": "ProgressStyleModel", "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_19435e5eef5b4d2f85655e32640d6418", - "placeholder": "​", - "style": "IPY_MODEL_84c78a6caaf2489eba798bd07007826d", - "value": " 892M/892M [00:16<00:00, 63.0MB/s]" + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" } }, - "df53015a9ef34785a76a5a1891d5c83a": { + "19435e5eef5b4d2f85655e32640d6418": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -496,10 +475,10 @@ "width": null } }, - "8aaedb8a8690458ba546944fc6846d7b": { + "255b730bb3214bad82964b92852729c3": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -548,10 +527,34 @@ "width": null } }, - "d6c52af94f934cc39983446f150a7df1": { + "261711e56196422496f96f409806ffda": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e5abc89d6b374783bf2d8db9adfa0797", + "max": 791656, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_99201c248f484e67af2aa7c3359dc48c", + "value": 791656 + } + }, + "29d7e51f5460441fa0f9bea429374aaf": { "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", @@ -563,10 +566,54 @@ "description_width": "" } }, - "3e67799a8c4d44cdab5774920de764d6": { + "2c207b1890874361b8982282ca9b4261": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_c2c37df97da841a68a3da1ae5e3ec015", + "IPY_MODEL_5447914abd1748a091133de016ac7aa8", + "IPY_MODEL_3c06c3b147aa4e75a7a0fc8f4d4798c8" + ], + "layout": "IPY_MODEL_7a01e2da228b4edea66d9a88ddf2e00c" + } + }, + "2e42f858840b4bb58f15379896fe98f8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_5e28e78b98384c1dbe19e6d453dfb51e", + "IPY_MODEL_6d4cbc443392422e82de520a9f1c85af", + "IPY_MODEL_d0c4a6469a9541819997c1567ee05573" + ], + "layout": "IPY_MODEL_349f2ef6a3e14f15b658ba8f61c5f703" + } + }, + "349f2ef6a3e14f15b658ba8f61c5f703": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -615,10 +662,10 @@ "width": null } }, - "615eddccd9384a84ad4f7d8c168f80fd": { + "36b539b14dc64afcb7b6135ee0b6b689": { "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", @@ -631,10 +678,10 @@ "description_width": "" } }, - "19435e5eef5b4d2f85655e32640d6418": { + "383574d53cef42efa63abcd4aae8b49d": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -683,10 +730,10 @@ "width": null } }, - "84c78a6caaf2489eba798bd07007826d": { + "383ce20098794b528753053f13ac15ca": { "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", @@ -698,32 +745,31 @@ "description_width": "" } }, - "2e42f858840b4bb58f15379896fe98f8": { + "3a662063172b45c6a32a7e8a181c37cf": { "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", "model_module_version": "1.5.0", + "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", + "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_5e28e78b98384c1dbe19e6d453dfb51e", - "IPY_MODEL_6d4cbc443392422e82de520a9f1c85af", - "IPY_MODEL_d0c4a6469a9541819997c1567ee05573" - ], - "layout": "IPY_MODEL_349f2ef6a3e14f15b658ba8f61c5f703" + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8fbe8c8cc3f9425abc340de18622d80f", + "placeholder": "​", + "style": "IPY_MODEL_01b5c326e1ec45f2afee2b377b0df59c", + "value": "model.safetensors: 100%" } }, - "5e28e78b98384c1dbe19e6d453dfb51e": { + "3c06c3b147aa4e75a7a0fc8f4d4798c8": { "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", "model_module_version": "1.5.0", + "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", @@ -735,40 +781,89 @@ "_view_name": "HTMLView", "description": "", "description_tooltip": null, - "layout": "IPY_MODEL_72bfece9336a4f3ab09d57173bf98fa0", + "layout": "IPY_MODEL_9419db4afd554df3acf18ffa8c4fcfa5", "placeholder": "​", - "style": "IPY_MODEL_6d5a81bc14794e6187173220300d1f59", - "value": "config.json: 100%" + "style": "IPY_MODEL_10cf9adaf5bc45c4810e984cf9789e9f", + "value": " 1.39M/1.39M [00:00<00:00, 13.0MB/s]" } }, - "6d4cbc443392422e82de520a9f1c85af": { + "3e67799a8c4d44cdab5774920de764d6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "41766800d32e4d188a2b7d3d963e96c8": { "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", "model_module_version": "1.5.0", + "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", + "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", + "_view_name": "HTMLView", "description": "", "description_tooltip": null, - "layout": "IPY_MODEL_fe432429208441e68bb60d0eaa2f7fe7", - "max": 1208, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_cc8d8251655f4302b58ff54ce47c819b", - "value": 1208 + "layout": "IPY_MODEL_8aaedb8a8690458ba546944fc6846d7b", + "placeholder": "​", + "style": "IPY_MODEL_d6c52af94f934cc39983446f150a7df1", + "value": "pytorch_model.bin: 100%" } }, - "d0c4a6469a9541819997c1567ee05573": { + "472473ba00164204bbdd1a44dd342ab9": { "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", "model_module_version": "1.5.0", + "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", @@ -780,16 +875,16 @@ "_view_name": "HTMLView", "description": "", "description_tooltip": null, - "layout": "IPY_MODEL_255b730bb3214bad82964b92852729c3", + "layout": "IPY_MODEL_fe40ddbd2d6e4aefb20037a0b5efd460", "placeholder": "​", - "style": "IPY_MODEL_9242cfbcfb8b4fb6b993520132b9796b", - "value": " 1.21k/1.21k [00:00<00:00, 70.3kB/s]" + "style": "IPY_MODEL_29d7e51f5460441fa0f9bea429374aaf", + "value": "config.json: 100%" } }, - "349f2ef6a3e14f15b658ba8f61c5f703": { + "47b70e36aa9148fcb649e440f7d4488d": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -838,10 +933,49 @@ "width": null } }, - "72bfece9336a4f3ab09d57173bf98fa0": { + "4e4a0c111d31440bac8b40fb8db2d57c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5447914abd1748a091133de016ac7aa8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6718b6b04ae74775885c24255b974f2e", + "max": 1389353, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_8b0c2fba3220466f8aa4e6b4f4de8007", + "value": 1389353 + } + }, + "54ec359589a2419a8476db6379f3d4f3": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -890,25 +1024,10 @@ "width": null } }, - "6d5a81bc14794e6187173220300d1f59": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "fe432429208441e68bb60d0eaa2f7fe7": { + "5685e313938c4547809a9f1e3b8fe367": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -957,26 +1076,10 @@ "width": null } }, - "cc8d8251655f4302b58ff54ce47c819b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "255b730bb3214bad82964b92852729c3": { + "5783d5bee3004c2db2856aae10001d7e": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -1025,47 +1128,55 @@ "width": null } }, - "9242cfbcfb8b4fb6b993520132b9796b": { + "5c5292a1bb0245e698b96dd0f3982d32": { "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", "state": { + "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", + "_model_name": "FloatProgressModel", "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_54ec359589a2419a8476db6379f3d4f3", + "max": 891646390, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_36b539b14dc64afcb7b6135ee0b6b689", + "value": 891646390 } }, - "7a7ef6de19bc4614bca900bd55bccbc2": { + "5e28e78b98384c1dbe19e6d453dfb51e": { "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", "model_module_version": "1.5.0", + "model_name": "HTMLModel", "state": { "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_3a662063172b45c6a32a7e8a181c37cf", - "IPY_MODEL_5c5292a1bb0245e698b96dd0f3982d32", - "IPY_MODEL_600b5184270549cdb46d592179f90374" - ], - "layout": "IPY_MODEL_383574d53cef42efa63abcd4aae8b49d" + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_72bfece9336a4f3ab09d57173bf98fa0", + "placeholder": "​", + "style": "IPY_MODEL_6d5a81bc14794e6187173220300d1f59", + "value": "config.json: 100%" } }, - "3a662063172b45c6a32a7e8a181c37cf": { + "600b5184270549cdb46d592179f90374": { "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", "model_module_version": "1.5.0", + "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", @@ -1077,40 +1188,53 @@ "_view_name": "HTMLView", "description": "", "description_tooltip": null, - "layout": "IPY_MODEL_8fbe8c8cc3f9425abc340de18622d80f", + "layout": "IPY_MODEL_9f52486613ef4aefb199706c73deb331", "placeholder": "​", - "style": "IPY_MODEL_01b5c326e1ec45f2afee2b377b0df59c", - "value": "model.safetensors: 100%" + "style": "IPY_MODEL_9e377916bf7a48eb8973f8a2477b13c5", + "value": " 892M/892M [00:12<00:00, 54.7MB/s]" } }, - "5c5292a1bb0245e698b96dd0f3982d32": { + "615eddccd9384a84ad4f7d8c168f80fd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "632aabf5fedb414a885185a9f5526221": { "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", "model_module_version": "1.5.0", + "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", + "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", + "_view_name": "HTMLView", "description": "", "description_tooltip": null, - "layout": "IPY_MODEL_54ec359589a2419a8476db6379f3d4f3", - "max": 891646390, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_36b539b14dc64afcb7b6135ee0b6b689", - "value": 891646390 + "layout": "IPY_MODEL_5783d5bee3004c2db2856aae10001d7e", + "placeholder": "​", + "style": "IPY_MODEL_4e4a0c111d31440bac8b40fb8db2d57c", + "value": " 1.42k/1.42k [00:00<00:00, 30.2kB/s]" } }, - "600b5184270549cdb46d592179f90374": { + "66007cf24b1e4b20b5eb08c3ffdc5c84": { "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", "model_module_version": "1.5.0", + "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", @@ -1122,16 +1246,16 @@ "_view_name": "HTMLView", "description": "", "description_tooltip": null, - "layout": "IPY_MODEL_9f52486613ef4aefb199706c73deb331", + "layout": "IPY_MODEL_d60886b71f59448eaf120fa450fd6b87", "placeholder": "​", - "style": "IPY_MODEL_9e377916bf7a48eb8973f8a2477b13c5", - "value": " 892M/892M [00:12<00:00, 54.7MB/s]" + "style": "IPY_MODEL_fb5780f8d6914defbcdde4153775c327", + "value": "generation_config.json: 100%" } }, - "383574d53cef42efa63abcd4aae8b49d": { + "6718b6b04ae74775885c24255b974f2e": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -1180,62 +1304,34 @@ "width": null } }, - "8fbe8c8cc3f9425abc340de18622d80f": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", + "6d4cbc443392422e82de520a9f1c85af": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fe432429208441e68bb60d0eaa2f7fe7", + "max": 1208, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_cc8d8251655f4302b58ff54ce47c819b", + "value": 1208 } }, - "01b5c326e1ec45f2afee2b377b0df59c": { + "6d5a81bc14794e6187173220300d1f59": { "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", @@ -1247,10 +1343,10 @@ "description_width": "" } }, - "54ec359589a2419a8476db6379f3d4f3": { + "72bfece9336a4f3ab09d57173bf98fa0": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -1299,26 +1395,34 @@ "width": null } }, - "36b539b14dc64afcb7b6135ee0b6b689": { + "74c771516e1b420e8120842583b433bc": { "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", "state": { + "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", + "_model_name": "FloatProgressModel", "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_3e67799a8c4d44cdab5774920de764d6", + "max": 891647935, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_615eddccd9384a84ad4f7d8c168f80fd", + "value": 891647935 } }, - "9f52486613ef4aefb199706c73deb331": { + "7a01e2da228b4edea66d9a88ddf2e00c": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -1367,25 +1471,10 @@ "width": null } }, - "9e377916bf7a48eb8973f8a2477b13c5": { + "7a7ef6de19bc4614bca900bd55bccbc2": { "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "93ca5e1831244ae88ed32b9cb640bf3c": { - "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", - "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", @@ -1397,83 +1486,84 @@ "_view_name": "HBoxView", "box_style": "", "children": [ - "IPY_MODEL_66007cf24b1e4b20b5eb08c3ffdc5c84", - "IPY_MODEL_d35fd85835c14f62a877da7960f74c91", - "IPY_MODEL_c8d5871c92a44ecb9a08305fc89352b3" + "IPY_MODEL_3a662063172b45c6a32a7e8a181c37cf", + "IPY_MODEL_5c5292a1bb0245e698b96dd0f3982d32", + "IPY_MODEL_600b5184270549cdb46d592179f90374" ], - "layout": "IPY_MODEL_e177a792e74149f4880599da63c46f8a" - } - }, - "66007cf24b1e4b20b5eb08c3ffdc5c84": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d60886b71f59448eaf120fa450fd6b87", - "placeholder": "​", - "style": "IPY_MODEL_fb5780f8d6914defbcdde4153775c327", - "value": "generation_config.json: 100%" + "layout": "IPY_MODEL_383574d53cef42efa63abcd4aae8b49d" } }, - "d35fd85835c14f62a877da7960f74c91": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", + "844ce3ab18514d95a1d428d97685afb7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d45457b6bf6948c58c6d45234446c0e8", - "max": 147, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_18c00caf53cd43daa5228b3ecfa06509", - "value": 147 + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null } }, - "c8d5871c92a44ecb9a08305fc89352b3": { + "84c78a6caaf2489eba798bd07007826d": { "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", "state": { - "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", + "_model_name": "DescriptionStyleModel", "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a3f53759d613418d8be64d031cf1d6d1", - "placeholder": "​", - "style": "IPY_MODEL_06fd4c171408476e8758eeadb026850a", - "value": " 147/147 [00:00<00:00, 9.04kB/s]" + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" } }, - "e177a792e74149f4880599da63c46f8a": { + "86122205ff2c44449d7e399a83b4a87a": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -1522,10 +1612,31 @@ "width": null } }, - "d60886b71f59448eaf120fa450fd6b87": { + "88a68d637ed44ba696cad0b2a7b57bae": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_aff66d7130e047eaa305a7f7d5973ea4", + "placeholder": "​", + "style": "IPY_MODEL_383ce20098794b528753053f13ac15ca", + "value": "spiece.model: 100%" + } + }, + "8aaedb8a8690458ba546944fc6846d7b": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -1574,25 +1685,26 @@ "width": null } }, - "fb5780f8d6914defbcdde4153775c327": { + "8b0c2fba3220466f8aa4e6b4f4de8007": { "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", + "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", + "bar_color": null, "description_width": "" } }, - "d45457b6bf6948c58c6d45234446c0e8": { + "8fbe8c8cc3f9425abc340de18622d80f": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -1641,26 +1753,69 @@ "width": null } }, - "18c00caf53cd43daa5228b3ecfa06509": { + "9242cfbcfb8b4fb6b993520132b9796b": { "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", + "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", - "bar_color": null, "description_width": "" } }, - "a3f53759d613418d8be64d031cf1d6d1": { + "926a5fe5fde94ffa9800d752a619492a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_41766800d32e4d188a2b7d3d963e96c8", + "IPY_MODEL_74c771516e1b420e8120842583b433bc", + "IPY_MODEL_d74d257a4c224496abd57949dfec3b0e" + ], + "layout": "IPY_MODEL_df53015a9ef34785a76a5a1891d5c83a" + } + }, + "93ca5e1831244ae88ed32b9cb640bf3c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_66007cf24b1e4b20b5eb08c3ffdc5c84", + "IPY_MODEL_d35fd85835c14f62a877da7960f74c91", + "IPY_MODEL_c8d5871c92a44ecb9a08305fc89352b3" + ], + "layout": "IPY_MODEL_e177a792e74149f4880599da63c46f8a" + } + }, + "9419db4afd554df3acf18ffa8c4fcfa5": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -1709,25 +1864,26 @@ "width": null } }, - "06fd4c171408476e8758eeadb026850a": { + "99201c248f484e67af2aa7c3359dc48c": { "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", + "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", + "bar_color": null, "description_width": "" } }, "9b01bff0eda44e91affdab3f2a81367f": { "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", "model_module_version": "1.5.0", + "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", @@ -1746,76 +1902,99 @@ "layout": "IPY_MODEL_86122205ff2c44449d7e399a83b4a87a" } }, - "88a68d637ed44ba696cad0b2a7b57bae": { + "9b767be8aed24fb39059aa2b9c3654e8": { "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", "model_module_version": "1.5.0", + "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", + "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_aff66d7130e047eaa305a7f7d5973ea4", - "placeholder": "​", - "style": "IPY_MODEL_383ce20098794b528753053f13ac15ca", - "value": "spiece.model: 100%" + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_472473ba00164204bbdd1a44dd342ab9", + "IPY_MODEL_cc928d6deef94fab8fe786c0bedbf73c", + "IPY_MODEL_632aabf5fedb414a885185a9f5526221" + ], + "layout": "IPY_MODEL_f9e19dd1e2354b10b24b1e63937ede17" } }, - "261711e56196422496f96f409806ffda": { + "9e377916bf7a48eb8973f8a2477b13c5": { "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", "state": { - "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", + "_model_name": "DescriptionStyleModel", "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e5abc89d6b374783bf2d8db9adfa0797", - "max": 791656, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_99201c248f484e67af2aa7c3359dc48c", - "value": 791656 + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" } }, - "00e9f95e9ef54ac49e66f9d4be19876d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", + "9f52486613ef4aefb199706c73deb331": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_844ce3ab18514d95a1d428d97685afb7", - "placeholder": "​", - "style": "IPY_MODEL_d99b117b8f94496f8eae3999fcb0d4d8", - "value": " 792k/792k [00:00<00:00, 5.48MB/s]" + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null } }, - "86122205ff2c44449d7e399a83b4a87a": { + "a3f53759d613418d8be64d031cf1d6d1": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -1866,8 +2045,8 @@ }, "aff66d7130e047eaa305a7f7d5973ea4": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -1916,10 +2095,10 @@ "width": null } }, - "383ce20098794b528753053f13ac15ca": { + "c0086b3f54ae449d8a5c0790558c2f53": { "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", @@ -1931,10 +2110,137 @@ "description_width": "" } }, - "e5abc89d6b374783bf2d8db9adfa0797": { + "c2c37df97da841a68a3da1ae5e3ec015": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5685e313938c4547809a9f1e3b8fe367", + "placeholder": "​", + "style": "IPY_MODEL_c0086b3f54ae449d8a5c0790558c2f53", + "value": "tokenizer.json: 100%" + } + }, + "c8d5871c92a44ecb9a08305fc89352b3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a3f53759d613418d8be64d031cf1d6d1", + "placeholder": "​", + "style": "IPY_MODEL_06fd4c171408476e8758eeadb026850a", + "value": " 147/147 [00:00<00:00, 9.04kB/s]" + } + }, + "cc8d8251655f4302b58ff54ce47c819b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "cc928d6deef94fab8fe786c0bedbf73c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_47b70e36aa9148fcb649e440f7d4488d", + "max": 1417, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_e642b7c3f7314ff085da91090a15a2d4", + "value": 1417 + } + }, + "d0c4a6469a9541819997c1567ee05573": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_255b730bb3214bad82964b92852729c3", + "placeholder": "​", + "style": "IPY_MODEL_9242cfbcfb8b4fb6b993520132b9796b", + "value": " 1.21k/1.21k [00:00<00:00, 70.3kB/s]" + } + }, + "d35fd85835c14f62a877da7960f74c91": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d45457b6bf6948c58c6d45234446c0e8", + "max": 147, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_18c00caf53cd43daa5228b3ecfa06509", + "value": 147 + } + }, + "d45457b6bf6948c58c6d45234446c0e8": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -1983,26 +2289,10 @@ "width": null } }, - "99201c248f484e67af2aa7c3359dc48c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "844ce3ab18514d95a1d428d97685afb7": { + "d60886b71f59448eaf120fa450fd6b87": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -2051,10 +2341,10 @@ "width": null } }, - "d99b117b8f94496f8eae3999fcb0d4d8": { + "d6c52af94f934cc39983446f150a7df1": { "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", @@ -2066,32 +2356,10 @@ "description_width": "" } }, - "2c207b1890874361b8982282ca9b4261": { + "d74d257a4c224496abd57949dfec3b0e": { "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_c2c37df97da841a68a3da1ae5e3ec015", - "IPY_MODEL_5447914abd1748a091133de016ac7aa8", - "IPY_MODEL_3c06c3b147aa4e75a7a0fc8f4d4798c8" - ], - "layout": "IPY_MODEL_7a01e2da228b4edea66d9a88ddf2e00c" - } - }, - "c2c37df97da841a68a3da1ae5e3ec015": { - "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", - "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", @@ -2103,61 +2371,31 @@ "_view_name": "HTMLView", "description": "", "description_tooltip": null, - "layout": "IPY_MODEL_5685e313938c4547809a9f1e3b8fe367", + "layout": "IPY_MODEL_19435e5eef5b4d2f85655e32640d6418", "placeholder": "​", - "style": "IPY_MODEL_c0086b3f54ae449d8a5c0790558c2f53", - "value": "tokenizer.json: 100%" - } - }, - "5447914abd1748a091133de016ac7aa8": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_6718b6b04ae74775885c24255b974f2e", - "max": 1389353, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_8b0c2fba3220466f8aa4e6b4f4de8007", - "value": 1389353 + "style": "IPY_MODEL_84c78a6caaf2489eba798bd07007826d", + "value": " 892M/892M [00:16<00:00, 63.0MB/s]" } }, - "3c06c3b147aa4e75a7a0fc8f4d4798c8": { + "d99b117b8f94496f8eae3999fcb0d4d8": { "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", "state": { - "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", + "_model_name": "DescriptionStyleModel", "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_9419db4afd554df3acf18ffa8c4fcfa5", - "placeholder": "​", - "style": "IPY_MODEL_10cf9adaf5bc45c4810e984cf9789e9f", - "value": " 1.39M/1.39M [00:00<00:00, 13.0MB/s]" + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" } }, - "7a01e2da228b4edea66d9a88ddf2e00c": { + "df53015a9ef34785a76a5a1891d5c83a": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -2206,10 +2444,10 @@ "width": null } }, - "5685e313938c4547809a9f1e3b8fe367": { + "e177a792e74149f4880599da63c46f8a": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -2258,25 +2496,10 @@ "width": null } }, - "c0086b3f54ae449d8a5c0790558c2f53": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "6718b6b04ae74775885c24255b974f2e": { + "e5abc89d6b374783bf2d8db9adfa0797": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -2325,10 +2548,10 @@ "width": null } }, - "8b0c2fba3220466f8aa4e6b4f4de8007": { + "e642b7c3f7314ff085da91090a15a2d4": { "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", @@ -2341,10 +2564,10 @@ "description_width": "" } }, - "9419db4afd554df3acf18ffa8c4fcfa5": { + "f9e19dd1e2354b10b24b1e63937ede17": { "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", "model_module_version": "1.2.0", + "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", @@ -2375,545 +2598,146 @@ "justify_content": null, "justify_items": null, "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "10cf9adaf5bc45c4810e984cf9789e9f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - } - } - } - }, - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "DbJadYoR6RG5", - "outputId": "7cb3cac1-9628-4f61-d7c1-ac84bd3b3dbc" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.38.2)\n", - "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.1)\n", - "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.20.3)\n", - "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.25.2)\n", - "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.0)\n", - "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n", - "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.12.25)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n", - "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.2)\n", - "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.2)\n", - "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.2)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (2023.6.0)\n", - "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (4.10.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.6)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.2.2)\n" - ] - } - ], - "source": [ - "pip install transformers" - ] - }, - { - "cell_type": "code", - "source": [ - "from transformers import T5ForConditionalGeneration, T5TokenizerFast\n", - "hfmodel = T5ForConditionalGeneration.from_pretrained(\"ThomasSimonini/t5-end2end-question-generation\")\n", - "checkpoint = \"t5-base\"\n", - "model = T5ForConditionalGeneration.from_pretrained(checkpoint)\n", - "tokenizer = T5TokenizerFast.from_pretrained(checkpoint)\n", - "tokenizer.sep_token = ''\n", - "tokenizer.add_tokens([''])\n", - "model.resize_token_embeddings(len(tokenizer))\n", - "# Check the sep_token_id to verify that it was added to the tokenizer\n", - "tokenizer.sep_token_id\n", - "\n", - "def hf_run_model(input_string, **generator_args):\n", - " generator_args = {\n", - " \"max_length\": 256,\n", - " \"num_beams\": 4,\n", - " \"length_penalty\": 1.5,\n", - " \"no_repeat_ngram_size\": 3,\n", - " \"early_stopping\": True,\n", - " }\n", - " input_string = \"generate questions: \" + input_string + \" \"\n", - " input_ids = tokenizer.encode(input_string, return_tensors=\"pt\")\n", - " res = hfmodel.generate(input_ids, **generator_args)\n", - " output = tokenizer.batch_decode(res, skip_special_tokens=True)\n", - " output = [item.split(\"\") for item in output]\n", - " return output" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 345, - "referenced_widgets": [ - "9b767be8aed24fb39059aa2b9c3654e8", - "472473ba00164204bbdd1a44dd342ab9", - "cc928d6deef94fab8fe786c0bedbf73c", - "632aabf5fedb414a885185a9f5526221", - "f9e19dd1e2354b10b24b1e63937ede17", - "fe40ddbd2d6e4aefb20037a0b5efd460", - "29d7e51f5460441fa0f9bea429374aaf", - "47b70e36aa9148fcb649e440f7d4488d", - "e642b7c3f7314ff085da91090a15a2d4", - "5783d5bee3004c2db2856aae10001d7e", - "4e4a0c111d31440bac8b40fb8db2d57c", - "926a5fe5fde94ffa9800d752a619492a", - "41766800d32e4d188a2b7d3d963e96c8", - "74c771516e1b420e8120842583b433bc", - "d74d257a4c224496abd57949dfec3b0e", - "df53015a9ef34785a76a5a1891d5c83a", - "8aaedb8a8690458ba546944fc6846d7b", - "d6c52af94f934cc39983446f150a7df1", - "3e67799a8c4d44cdab5774920de764d6", - "615eddccd9384a84ad4f7d8c168f80fd", - "19435e5eef5b4d2f85655e32640d6418", - "84c78a6caaf2489eba798bd07007826d", - "2e42f858840b4bb58f15379896fe98f8", - "5e28e78b98384c1dbe19e6d453dfb51e", - "6d4cbc443392422e82de520a9f1c85af", - "d0c4a6469a9541819997c1567ee05573", - "349f2ef6a3e14f15b658ba8f61c5f703", - "72bfece9336a4f3ab09d57173bf98fa0", - "6d5a81bc14794e6187173220300d1f59", - "fe432429208441e68bb60d0eaa2f7fe7", - "cc8d8251655f4302b58ff54ce47c819b", - "255b730bb3214bad82964b92852729c3", - "9242cfbcfb8b4fb6b993520132b9796b", - "7a7ef6de19bc4614bca900bd55bccbc2", - "3a662063172b45c6a32a7e8a181c37cf", - "5c5292a1bb0245e698b96dd0f3982d32", - "600b5184270549cdb46d592179f90374", - "383574d53cef42efa63abcd4aae8b49d", - "8fbe8c8cc3f9425abc340de18622d80f", - "01b5c326e1ec45f2afee2b377b0df59c", - "54ec359589a2419a8476db6379f3d4f3", - "36b539b14dc64afcb7b6135ee0b6b689", - "9f52486613ef4aefb199706c73deb331", - "9e377916bf7a48eb8973f8a2477b13c5", - "93ca5e1831244ae88ed32b9cb640bf3c", - "66007cf24b1e4b20b5eb08c3ffdc5c84", - "d35fd85835c14f62a877da7960f74c91", - "c8d5871c92a44ecb9a08305fc89352b3", - "e177a792e74149f4880599da63c46f8a", - "d60886b71f59448eaf120fa450fd6b87", - "fb5780f8d6914defbcdde4153775c327", - "d45457b6bf6948c58c6d45234446c0e8", - "18c00caf53cd43daa5228b3ecfa06509", - "a3f53759d613418d8be64d031cf1d6d1", - "06fd4c171408476e8758eeadb026850a", - "9b01bff0eda44e91affdab3f2a81367f", - "88a68d637ed44ba696cad0b2a7b57bae", - "261711e56196422496f96f409806ffda", - "00e9f95e9ef54ac49e66f9d4be19876d", - "86122205ff2c44449d7e399a83b4a87a", - "aff66d7130e047eaa305a7f7d5973ea4", - "383ce20098794b528753053f13ac15ca", - "e5abc89d6b374783bf2d8db9adfa0797", - "99201c248f484e67af2aa7c3359dc48c", - "844ce3ab18514d95a1d428d97685afb7", - "d99b117b8f94496f8eae3999fcb0d4d8", - "2c207b1890874361b8982282ca9b4261", - "c2c37df97da841a68a3da1ae5e3ec015", - "5447914abd1748a091133de016ac7aa8", - "3c06c3b147aa4e75a7a0fc8f4d4798c8", - "7a01e2da228b4edea66d9a88ddf2e00c", - "5685e313938c4547809a9f1e3b8fe367", - "c0086b3f54ae449d8a5c0790558c2f53", - "6718b6b04ae74775885c24255b974f2e", - "8b0c2fba3220466f8aa4e6b4f4de8007", - "9419db4afd554df3acf18ffa8c4fcfa5", - "10cf9adaf5bc45c4810e984cf9789e9f" - ] - }, - "id": "X7Q3HApc6TSR", - "outputId": "ed472da5-b16c-4315-ef02-eaa355672662" - }, - "execution_count": 2, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n", - "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", - "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", - "You will be able to reuse this secret in all of your notebooks.\n", - "Please note that authentication is recommended but still optional to access public models or datasets.\n", - " warnings.warn(\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "config.json: 0%| | 0.00/1.42k [00:00