diff --git a/create-coiled-datasets/create_dea_opioid_data.ipynb b/create-coiled-datasets/create_dea_opioid_data.ipynb
new file mode 100644
index 0000000..69434ac
--- /dev/null
+++ b/create-coiled-datasets/create_dea_opioid_data.ipynb
@@ -0,0 +1,642 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "ac19f5ad",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import coiled\n",
+ "import dask\n",
+ "import dask.dataframe as dd\n",
+ "from dask.distributed import Client"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "3d9d1f6a",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Output()"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "
Using existing cluster: 'create-datasets'\n",
+ "
\n"
+ ],
+ "text/plain": [
+ "Using existing cluster: \u001b[32m'create-datasets'\u001b[0m\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n"
+ ],
+ "text/plain": []
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "cluster = coiled.Cluster(name=\"create-datasets\", n_workers=10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "a1084e88",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "client = dask.distributed.Client(cluster)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "2ef14b8d-36e7-4d5f-ad67-3d3682a42e27",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dtypes = {\n",
+ " 'REPORTER_DEA_NO': 'object',\n",
+ " 'REPORTER_BUS_ACT': 'object',\n",
+ " 'REPORTER_NAME': 'object',\n",
+ " 'REPORTER_ADDL_CO_INFO': 'object',\n",
+ " 'REPORTER_ADDRESS1': 'object',\n",
+ " 'REPORTER_ADDRESS2': 'object',\n",
+ " 'REPORTER_CITY': 'object',\n",
+ " 'REPORTER_STATE': 'object',\n",
+ " 'REPORTER_ZIP': 'object',\n",
+ " 'REPORTER_COUNTY': 'object',\n",
+ " 'BUYER_DEA_NO': 'object',\n",
+ " 'BUYER_BUS_ACT': 'object',\n",
+ " 'BUYER_NAME': 'object',\n",
+ " 'BUYER_ADDL_CO_INFO': 'object',\n",
+ " 'BUYER_ADDRESS1': 'object',\n",
+ " 'BUYER_ADDRESS2': 'object',\n",
+ " 'BUYER_CITY': 'object',\n",
+ " 'BUYER_STATE': 'object',\n",
+ " 'BUYER_ZIP': 'object',\n",
+ " 'BUYER_COUNTY': 'object',\n",
+ " 'TRANSACTION_CODE': 'object',\n",
+ " 'DRUG_CODE': 'object',\n",
+ " 'NDC_NO': 'object',\n",
+ " 'DRUG_NAME': 'object',\n",
+ " 'QUANTITY': 'object',\n",
+ " 'UNIT': 'object',\n",
+ " 'ACTION_INDICATOR': 'object',\n",
+ " 'ORDER_FORM_NO': 'object',\n",
+ " 'CORRECTION_NO': 'object',\n",
+ " 'STRENGTH': 'object',\n",
+ " 'TRANSACTION_DATE': 'object',\n",
+ " 'CALC_BASE_WT_IN_GM': 'object',\n",
+ " 'DOSAGE_UNIT': 'object',\n",
+ " 'TRANSACTION_ID': 'object',\n",
+ " 'Product_Name': 'object',\n",
+ " 'Ingredient_Name': 'object',\n",
+ " 'Measure': 'object',\n",
+ " 'MME_Conversion_Factor': 'object',\n",
+ " 'Combined_Labeler_Name': 'object',\n",
+ " 'Revised_Company_Name': 'object',\n",
+ " 'Reporter_family': 'object',\n",
+ " 'dos_str': 'object'\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "127cb361-ae52-439c-ac34-0cf6b326b69e",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/powers/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/dask/dataframe/io/csv.py:520: UserWarning: Warning gzip compression does not support breaking apart files\n",
+ "Please ensure that each individual file can fit in memory and\n",
+ "use the keyword ``blocksize=None to remove this message``\n",
+ "Setting ``blocksize=None``\n",
+ " warn(\n"
+ ]
+ }
+ ],
+ "source": [
+ "data = dd.read_table(\n",
+ " \"s3://coiled-datasets/dea-opioid/csvs/*.gz\",\n",
+ " dtype=dtypes,\n",
+ " compression=\"gzip\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "5c0fa8b4-b41f-4586-ac50-406b788ea393",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " REPORTER_DEA_NO | \n",
+ " REPORTER_BUS_ACT | \n",
+ " REPORTER_NAME | \n",
+ " REPORTER_ADDL_CO_INFO | \n",
+ " REPORTER_ADDRESS1 | \n",
+ " REPORTER_ADDRESS2 | \n",
+ " REPORTER_CITY | \n",
+ " REPORTER_STATE | \n",
+ " REPORTER_ZIP | \n",
+ " REPORTER_COUNTY | \n",
+ " ... | \n",
+ " DOSAGE_UNIT | \n",
+ " TRANSACTION_ID | \n",
+ " Product_Name | \n",
+ " Ingredient_Name | \n",
+ " Measure | \n",
+ " MME_Conversion_Factor | \n",
+ " Combined_Labeler_Name | \n",
+ " Revised_Company_Name | \n",
+ " Reporter_family | \n",
+ " dos_str | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " PA0006836 | \n",
+ " DISTRIBUTOR | \n",
+ " ACE SURGICAL SUPPLY CO INC | \n",
+ " NaN | \n",
+ " 1034 PEARL STREET | \n",
+ " NaN | \n",
+ " BROCKTON | \n",
+ " MA | \n",
+ " 2301 | \n",
+ " PLYMOUTH | \n",
+ " ... | \n",
+ " 100.0 | \n",
+ " 64 | \n",
+ " HYDROCODONE BIT/ACETA 10MG/500MG USP | \n",
+ " HYDROCODONE BITARTRATE HEMIPENTAHYDRATE | \n",
+ " TAB | \n",
+ " 1.0 | \n",
+ " SpecGx LLC | \n",
+ " Mallinckrodt | \n",
+ " ACE Surgical Supply Co Inc | \n",
+ " 10.0 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " PA0021179 | \n",
+ " DISTRIBUTOR | \n",
+ " APOTHECA INC | \n",
+ " NaN | \n",
+ " 1622 N 16TH ST | \n",
+ " NaN | \n",
+ " PHOENIX | \n",
+ " AZ | \n",
+ " 85006 | \n",
+ " MARICOPA | \n",
+ " ... | \n",
+ " 40.0 | \n",
+ " 52 | \n",
+ " HYDROCODONE BITARTRATE & ACETA 5MG/ | \n",
+ " HYDROCODONE BITARTRATE HEMIPENTAHYDRATE | \n",
+ " TAB | \n",
+ " 1.0 | \n",
+ " Apotheca Inc. | \n",
+ " Apotheca Inc. | \n",
+ " Apotheca Inc | \n",
+ " 5.0 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " PA0021179 | \n",
+ " DISTRIBUTOR | \n",
+ " APOTHECA INC | \n",
+ " NaN | \n",
+ " 1622 N 16TH ST | \n",
+ " NaN | \n",
+ " PHOENIX | \n",
+ " AZ | \n",
+ " 85006 | \n",
+ " MARICOPA | \n",
+ " ... | \n",
+ " 1200.0 | \n",
+ " 119 | \n",
+ " HYDROCODONE BITARTRATE & ACETA 5MG/ | \n",
+ " HYDROCODONE BITARTRATE HEMIPENTAHYDRATE | \n",
+ " TAB | \n",
+ " 1.0 | \n",
+ " Apotheca Inc. | \n",
+ " Apotheca Inc. | \n",
+ " Apotheca Inc | \n",
+ " 5.0 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " PA0021179 | \n",
+ " DISTRIBUTOR | \n",
+ " APOTHECA INC | \n",
+ " NaN | \n",
+ " 1622 N 16TH ST | \n",
+ " NaN | \n",
+ " PHOENIX | \n",
+ " AZ | \n",
+ " 85006 | \n",
+ " MARICOPA | \n",
+ " ... | \n",
+ " 600.0 | \n",
+ " 34 | \n",
+ " HYDROCODONEBITARTRATE & ACETA 7.5MG | \n",
+ " HYDROCODONE BITARTRATE HEMIPENTAHYDRATE | \n",
+ " TAB | \n",
+ " 1.0 | \n",
+ " Apotheca Inc. | \n",
+ " Apotheca Inc. | \n",
+ " Apotheca Inc | \n",
+ " 7.5 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " PA0021179 | \n",
+ " DISTRIBUTOR | \n",
+ " APOTHECA INC | \n",
+ " NaN | \n",
+ " 1622 N 16TH ST | \n",
+ " NaN | \n",
+ " PHOENIX | \n",
+ " AZ | \n",
+ " 85006 | \n",
+ " MARICOPA | \n",
+ " ... | \n",
+ " 300.0 | \n",
+ " 19 | \n",
+ " HYDROCODONE BITARTRATE & ACETA 5MG/ | \n",
+ " HYDROCODONE BITARTRATE HEMIPENTAHYDRATE | \n",
+ " TAB | \n",
+ " 1.0 | \n",
+ " Apotheca Inc. | \n",
+ " Apotheca Inc. | \n",
+ " Apotheca Inc | \n",
+ " 5.0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
5 rows × 42 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " REPORTER_DEA_NO REPORTER_BUS_ACT REPORTER_NAME \\\n",
+ "0 PA0006836 DISTRIBUTOR ACE SURGICAL SUPPLY CO INC \n",
+ "1 PA0021179 DISTRIBUTOR APOTHECA INC \n",
+ "2 PA0021179 DISTRIBUTOR APOTHECA INC \n",
+ "3 PA0021179 DISTRIBUTOR APOTHECA INC \n",
+ "4 PA0021179 DISTRIBUTOR APOTHECA INC \n",
+ "\n",
+ " REPORTER_ADDL_CO_INFO REPORTER_ADDRESS1 REPORTER_ADDRESS2 REPORTER_CITY \\\n",
+ "0 NaN 1034 PEARL STREET NaN BROCKTON \n",
+ "1 NaN 1622 N 16TH ST NaN PHOENIX \n",
+ "2 NaN 1622 N 16TH ST NaN PHOENIX \n",
+ "3 NaN 1622 N 16TH ST NaN PHOENIX \n",
+ "4 NaN 1622 N 16TH ST NaN PHOENIX \n",
+ "\n",
+ " REPORTER_STATE REPORTER_ZIP REPORTER_COUNTY ... DOSAGE_UNIT TRANSACTION_ID \\\n",
+ "0 MA 2301 PLYMOUTH ... 100.0 64 \n",
+ "1 AZ 85006 MARICOPA ... 40.0 52 \n",
+ "2 AZ 85006 MARICOPA ... 1200.0 119 \n",
+ "3 AZ 85006 MARICOPA ... 600.0 34 \n",
+ "4 AZ 85006 MARICOPA ... 300.0 19 \n",
+ "\n",
+ " Product_Name \\\n",
+ "0 HYDROCODONE BIT/ACETA 10MG/500MG USP \n",
+ "1 HYDROCODONE BITARTRATE & ACETA 5MG/ \n",
+ "2 HYDROCODONE BITARTRATE & ACETA 5MG/ \n",
+ "3 HYDROCODONEBITARTRATE & ACETA 7.5MG \n",
+ "4 HYDROCODONE BITARTRATE & ACETA 5MG/ \n",
+ "\n",
+ " Ingredient_Name Measure MME_Conversion_Factor \\\n",
+ "0 HYDROCODONE BITARTRATE HEMIPENTAHYDRATE TAB 1.0 \n",
+ "1 HYDROCODONE BITARTRATE HEMIPENTAHYDRATE TAB 1.0 \n",
+ "2 HYDROCODONE BITARTRATE HEMIPENTAHYDRATE TAB 1.0 \n",
+ "3 HYDROCODONE BITARTRATE HEMIPENTAHYDRATE TAB 1.0 \n",
+ "4 HYDROCODONE BITARTRATE HEMIPENTAHYDRATE TAB 1.0 \n",
+ "\n",
+ " Combined_Labeler_Name Revised_Company_Name Reporter_family \\\n",
+ "0 SpecGx LLC Mallinckrodt ACE Surgical Supply Co Inc \n",
+ "1 Apotheca Inc. Apotheca Inc. Apotheca Inc \n",
+ "2 Apotheca Inc. Apotheca Inc. Apotheca Inc \n",
+ "3 Apotheca Inc. Apotheca Inc. Apotheca Inc \n",
+ "4 Apotheca Inc. Apotheca Inc. Apotheca Inc \n",
+ "\n",
+ " dos_str \n",
+ "0 10.0 \n",
+ "1 5.0 \n",
+ "2 5.0 \n",
+ "3 7.5 \n",
+ "4 5.0 \n",
+ "\n",
+ "[5 rows x 42 columns]"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "data.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "11fe5420-79b7-422a-931f-88261826c316",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "19999990"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "len(data)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "44cd5627-24ec-41fd-bc78-98a60d7de374",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "REPORTER_DEA_NO object\n",
+ "REPORTER_BUS_ACT object\n",
+ "REPORTER_NAME object\n",
+ "REPORTER_ADDL_CO_INFO object\n",
+ "REPORTER_ADDRESS1 object\n",
+ "REPORTER_ADDRESS2 object\n",
+ "REPORTER_CITY object\n",
+ "REPORTER_STATE object\n",
+ "REPORTER_ZIP object\n",
+ "REPORTER_COUNTY object\n",
+ "BUYER_DEA_NO object\n",
+ "BUYER_BUS_ACT object\n",
+ "BUYER_NAME object\n",
+ "BUYER_ADDL_CO_INFO object\n",
+ "BUYER_ADDRESS1 object\n",
+ "BUYER_ADDRESS2 object\n",
+ "BUYER_CITY object\n",
+ "BUYER_STATE object\n",
+ "BUYER_ZIP object\n",
+ "BUYER_COUNTY object\n",
+ "TRANSACTION_CODE object\n",
+ "DRUG_CODE object\n",
+ "NDC_NO object\n",
+ "DRUG_NAME object\n",
+ "QUANTITY object\n",
+ "UNIT object\n",
+ "ACTION_INDICATOR object\n",
+ "ORDER_FORM_NO object\n",
+ "CORRECTION_NO object\n",
+ "STRENGTH object\n",
+ "TRANSACTION_DATE object\n",
+ "CALC_BASE_WT_IN_GM object\n",
+ "DOSAGE_UNIT object\n",
+ "TRANSACTION_ID object\n",
+ "Product_Name object\n",
+ "Ingredient_Name object\n",
+ "Measure object\n",
+ "MME_Conversion_Factor object\n",
+ "Combined_Labeler_Name object\n",
+ "Revised_Company_Name object\n",
+ "Reporter_family object\n",
+ "dos_str object\n",
+ "dtype: object"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "data.dtypes"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "id": "a92dddb1-596f-451e-a297-3580c8e11961",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "['coiled-datasets/dea-opioid/csv_out_tmp/0.part',\n",
+ " 'coiled-datasets/dea-opioid/csv_out_tmp/1.part',\n",
+ " 'coiled-datasets/dea-opioid/csv_out_tmp/2.part',\n",
+ " 'coiled-datasets/dea-opioid/csv_out_tmp/3.part',\n",
+ " 'coiled-datasets/dea-opioid/csv_out_tmp/4.part',\n",
+ " 'coiled-datasets/dea-opioid/csv_out_tmp/5.part',\n",
+ " 'coiled-datasets/dea-opioid/csv_out_tmp/6.part',\n",
+ " 'coiled-datasets/dea-opioid/csv_out_tmp/7.part',\n",
+ " 'coiled-datasets/dea-opioid/csv_out_tmp/8.part',\n",
+ " 'coiled-datasets/dea-opioid/csv_out_tmp/9.part']"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "data.to_csv(\n",
+ " \"s3://coiled-datasets/dea-opioid/csv_out_tmp\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "1d7268ee-9d49-4675-807d-3dc0fbddb075",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import fastparquet"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "3d52567a-fccc-49ad-8d46-ecf011af923f",
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "ModuleNotFoundError",
+ "evalue": "No module named 'fastparquet'",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
+ "\u001b[0;32m/var/folders/d2/116lnkgd0l7f51xr7msb2jnh0000gn/T/ipykernel_1989/3822279948.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m data.to_parquet(\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;34m\"s3://coiled-datasets/dea-opioid/parquet\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"fastparquet\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m )\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/dask/dataframe/core.py\u001b[0m in \u001b[0;36mto_parquet\u001b[0;34m(self, path, *args, **kwargs)\u001b[0m\n\u001b[1;32m 4451\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mio\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mto_parquet\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4452\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4453\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mto_parquet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4454\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4455\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mderived_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py\u001b[0m in \u001b[0;36mto_parquet\u001b[0;34m(df, path, engine, compression, write_index, append, overwrite, ignore_divisions, partition_on, storage_options, custom_metadata, write_metadata_file, compute, compute_kwargs, schema, **kwargs)\u001b[0m\n\u001b[1;32m 720\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcompute_kwargs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 721\u001b[0m \u001b[0mcompute_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 722\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mcompute_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 723\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 724\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/dask/base.py\u001b[0m in \u001b[0;36mcompute\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 284\u001b[0m \u001b[0mdask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbase\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 285\u001b[0m \"\"\"\n\u001b[0;32m--> 286\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 287\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/dask/base.py\u001b[0m in \u001b[0;36mcompute\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 566\u001b[0m \u001b[0mpostcomputes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__dask_postcompute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 567\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 568\u001b[0;31m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mschedule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdsk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 569\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mrepack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresults\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpostcomputes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 570\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)\u001b[0m\n\u001b[1;32m 2746\u001b[0m \u001b[0mshould_rejoin\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2747\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2748\u001b[0;31m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgather\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpacked\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0masynchronous\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0masynchronous\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdirect\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdirect\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2749\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2750\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfutures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mgather\u001b[0;34m(self, futures, errors, direct, asynchronous)\u001b[0m\n\u001b[1;32m 2023\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2024\u001b[0m \u001b[0mlocal_worker\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2025\u001b[0;31m return self.sync(\n\u001b[0m\u001b[1;32m 2026\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gather\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2027\u001b[0m \u001b[0mfutures\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/distributed/client.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(self, func, asynchronous, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 864\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 865\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 866\u001b[0;31m return sync(\n\u001b[0m\u001b[1;32m 867\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_timeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_timeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 868\u001b[0m )\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(loop, func, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 324\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 325\u001b[0m \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 326\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 327\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 328\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36mf\u001b[0;34m()\u001b[0m\n\u001b[1;32m 307\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallback_timeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 308\u001b[0m \u001b[0mfuture\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0masyncio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfuture\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_timeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 309\u001b[0;31m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 310\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 311\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/tornado/gen.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 760\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 761\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 762\u001b[0;31m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 763\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 764\u001b[0m \u001b[0mexc_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/distributed/client.py\u001b[0m in \u001b[0;36m_gather\u001b[0;34m(self, futures, errors, direct, local_worker)\u001b[0m\n\u001b[1;32m 1888\u001b[0m \u001b[0mexc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCancelledError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1889\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1890\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraceback\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1891\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1892\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0merrors\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"skip\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/envs/coiled/lib/python3.9/site-packages/distributed/protocol/pickle.py\u001b[0m in \u001b[0;36mloads\u001b[0;34m()\u001b[0m\n",
+ "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'fastparquet'"
+ ]
+ }
+ ],
+ "source": [
+ "data.to_parquet(\n",
+ " \"s3://coiled-datasets/dea-opioid/parquet\", \n",
+ " engine=\"fastparquet\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "id": "7e8df58c-9b33-4b00-88f0-32ed84389efe",
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "ArrowTypeError",
+ "evalue": "('Expected a string or bytes dtype, got int64', 'Conversion failed for column DRUG_CODE with type int64')",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mArrowTypeError\u001b[0m Traceback (most recent call last)",
+ "\u001b[0;32m/var/folders/d2/116lnkgd0l7f51xr7msb2jnh0000gn/T/ipykernel_12458/422259827.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m data.drop('REPORTER_ZIP', axis=1).drop('BUYER_ZIP', axis=1).to_parquet(\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;34m\"s3://coiled-datasets/dea-opioid/parquet\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m schema={\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\"NDC_NO\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstring\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m }\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/dask/dataframe/core.py\u001b[0m in \u001b[0;36mto_parquet\u001b[0;34m(self, path, *args, **kwargs)\u001b[0m\n\u001b[1;32m 4451\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mio\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mto_parquet\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4452\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4453\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mto_parquet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4454\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4455\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mderived_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py\u001b[0m in \u001b[0;36mto_parquet\u001b[0;34m(df, path, engine, compression, write_index, append, overwrite, ignore_divisions, partition_on, storage_options, custom_metadata, write_metadata_file, compute, compute_kwargs, schema, **kwargs)\u001b[0m\n\u001b[1;32m 720\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcompute_kwargs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 721\u001b[0m \u001b[0mcompute_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 722\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mcompute_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 723\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 724\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/dask/base.py\u001b[0m in \u001b[0;36mcompute\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 284\u001b[0m \u001b[0mdask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbase\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 285\u001b[0m \"\"\"\n\u001b[0;32m--> 286\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 287\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/dask/base.py\u001b[0m in \u001b[0;36mcompute\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 566\u001b[0m \u001b[0mpostcomputes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__dask_postcompute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 567\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 568\u001b[0;31m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mschedule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdsk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 569\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mrepack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresults\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpostcomputes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 570\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)\u001b[0m\n\u001b[1;32m 2746\u001b[0m \u001b[0mshould_rejoin\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2747\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2748\u001b[0;31m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgather\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpacked\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0masynchronous\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0masynchronous\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdirect\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdirect\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2749\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2750\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfutures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/distributed/client.py\u001b[0m in \u001b[0;36mgather\u001b[0;34m(self, futures, errors, direct, asynchronous)\u001b[0m\n\u001b[1;32m 2023\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2024\u001b[0m \u001b[0mlocal_worker\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2025\u001b[0;31m return self.sync(\n\u001b[0m\u001b[1;32m 2026\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gather\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2027\u001b[0m \u001b[0mfutures\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/distributed/client.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(self, func, asynchronous, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 864\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 865\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 866\u001b[0;31m return sync(\n\u001b[0m\u001b[1;32m 867\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_timeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_timeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 868\u001b[0m )\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36msync\u001b[0;34m(loop, func, callback_timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m 324\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 325\u001b[0m \u001b[0mtyp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 326\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 327\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 328\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/distributed/utils.py\u001b[0m in \u001b[0;36mf\u001b[0;34m()\u001b[0m\n\u001b[1;32m 307\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallback_timeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 308\u001b[0m \u001b[0mfuture\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0masyncio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfuture\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback_timeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 309\u001b[0;31m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 310\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 311\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/tornado/gen.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 760\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 761\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 762\u001b[0;31m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 763\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 764\u001b[0m \u001b[0mexc_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/distributed/client.py\u001b[0m in \u001b[0;36m_gather\u001b[0;34m(self, futures, errors, direct, local_worker)\u001b[0m\n\u001b[1;32m 1888\u001b[0m \u001b[0mexc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCancelledError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1889\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1890\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraceback\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1891\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1892\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0merrors\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"skip\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/envs/coiled/lib/python3.9/site-packages/dask/dataframe/io/parquet/arrow.py\u001b[0m in \u001b[0;36mwrite_partition\u001b[0;34m()\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/envs/coiled/lib/python3.9/site-packages/dask/dataframe/io/parquet/arrow.py\u001b[0m in \u001b[0;36m_pandas_to_arrow_table\u001b[0;34m()\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/pyarrow/table.pxi\u001b[0m in \u001b[0;36mpyarrow.lib.Table.from_pandas\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1559\u001b[0m \"\"\"\n\u001b[1;32m 1560\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mpyarrow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpandas_compat\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdataframe_to_arrays\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1561\u001b[0;31m arrays, schema = dataframe_to_arrays(\n\u001b[0m\u001b[1;32m 1562\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1563\u001b[0m \u001b[0mschema\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mschema\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/envs/coiled/lib/python3.9/site-packages/pyarrow/pandas_compat.py\u001b[0m in \u001b[0;36mdataframe_to_arrays\u001b[0;34m()\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/envs/coiled/lib/python3.9/site-packages/pyarrow/pandas_compat.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/envs/coiled/lib/python3.9/site-packages/pyarrow/pandas_compat.py\u001b[0m in \u001b[0;36mconvert_column\u001b[0;34m()\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/envs/coiled/lib/python3.9/site-packages/pyarrow/pandas_compat.py\u001b[0m in \u001b[0;36mconvert_column\u001b[0;34m()\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/pyarrow/array.pxi\u001b[0m in \u001b[0;36mpyarrow.lib.array\u001b[0;34m()\u001b[0m\n\u001b[1;32m 300\u001b[0m values, type = pandas_api.compat.get_datetimetz_type(\n\u001b[1;32m 301\u001b[0m values, obj.dtype, type)\n\u001b[0;32m--> 302\u001b[0;31m return _ndarray_to_array(values, mask, type, c_from_pandas, safe,\n\u001b[0m\u001b[1;32m 303\u001b[0m pool)\n\u001b[1;32m 304\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/pyarrow/array.pxi\u001b[0m in \u001b[0;36mpyarrow.lib._ndarray_to_array\u001b[0;34m()\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 82\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mnogil\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m check_status(NdarrayToArrow(pool, values, mask, from_pandas,\n\u001b[0m\u001b[1;32m 84\u001b[0m c_type, cast_options, &chunked_out))\n\u001b[1;32m 85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/opt/miniconda3/envs/standard-coiled/lib/python3.9/site-packages/pyarrow/error.pxi\u001b[0m in \u001b[0;36mpyarrow.lib.check_status\u001b[0;34m()\u001b[0m\n\u001b[1;32m 120\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mArrowNotImplementedError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIsTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 122\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mArrowTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 123\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIsCapacityError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 124\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mArrowCapacityError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;31mArrowTypeError\u001b[0m: ('Expected a string or bytes dtype, got int64', 'Conversion failed for column DRUG_CODE with type int64')"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "distributed.client - ERROR - Failed to reconnect to scheduler after 30.00 seconds, closing client\n",
+ "_GatheringFuture exception was never retrieved\n",
+ "future: <_GatheringFuture finished exception=CancelledError()>\n",
+ "asyncio.exceptions.CancelledError\n"
+ ]
+ }
+ ],
+ "source": [
+ "data.drop('REPORTER_ZIP', axis=1).drop('BUYER_ZIP', axis=1).to_parquet(\n",
+ " \"s3://coiled-datasets/dea-opioid/parquet\", \n",
+ " schema={\n",
+ " \"NDC_NO\": pa.string()\n",
+ " }\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e9bb6f41-374a-4d2d-b638-653fe590204a",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python [conda env:standard-coiled] *",
+ "language": "python",
+ "name": "conda-env-standard-coiled-py"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.6"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/envs/standard-coiled.yml b/envs/standard-coiled.yml
index 1c53b10..9d6e0d9 100644
--- a/envs/standard-coiled.yml
+++ b/envs/standard-coiled.yml
@@ -16,4 +16,5 @@ dependencies:
- nb_conda
- jupyterlab
- dask-labextension
+ - fastparquet