diff --git a/05_NeuralNetworks/NeuralNetworks_Tutorial_Part1.ipynb b/05_NeuralNetworks/NeuralNetworks_Tutorial_Part1.ipynb
index e08884f..3df7219 100644
--- a/05_NeuralNetworks/NeuralNetworks_Tutorial_Part1.ipynb
+++ b/05_NeuralNetworks/NeuralNetworks_Tutorial_Part1.ipynb
@@ -1 +1 @@
-{"cells":[{"cell_type":"markdown","metadata":{"id":"PlpG_fIiqdJU","colab_type":"text","cell_id":"cb77df6fe0424017a03c3a38a4c18864","deepnote_cell_type":"markdown"},"source":"# Building a Simple Neural Network with `TensorFlow.Keras`\n> Neural Networks Part 1\n\nIn this notebook, we are going to walk through building a simple neural network to classify sequence data. This tutorial will be meant as a fast overview to building/training neural networks with `Keras`.","block_group":"c24d660ee0f4487ea88143183fa285be"},{"cell_type":"code","metadata":{"id":"eFSzOvSduDf8","colab":{"height":34,"base_uri":"https://localhost:8080/"},"outputId":"61c63452-faa5-499b-852b-080a0a7b6c0a","colab_type":"code","source_hash":null,"execution_start":1697312330884,"execution_millis":8029,"deepnote_to_be_reexecuted":false,"cell_id":"aefbbbb7c04347879e490d71a0d18bca","deepnote_cell_type":"code"},"source":"# Import useful libraries\n\n# Needed for terminal functions (i.e. wget)\nimport os\n\n# For plotting\nimport matplotlib.pyplot as plt\n\n# For DataFrame manipulation\nimport pandas as pd\n\n# For data preprocessing\nfrom sklearn.preprocessing import StandardScaler #Use StandardScaler from scikitlearn\nfrom sklearn.utils import shuffle #Used to shuffle up examples before training\n\n# Keras-related imports\nfrom keras.models import Sequential #we will build our models layer by layer\nfrom keras.layers import Dense #we want to use dense layers in our model\n\n# Keras is built on top of the TensorFlow library\nimport tensorflow as tf\n# tf has many helpful functions for training networks like loss functions, optimization methods, etc.","block_group":"8108374e8d2f41d1bdcaeb9add10ec44","execution_count":null,"outputs":[{"name":"stderr","text":"2023-10-14 19:38:54.117469: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\nTo enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n2023-10-14 19:38:54.510494: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n2023-10-14 19:38:54.510549: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n2023-10-14 19:38:54.616485: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n2023-10-14 19:38:56.705042: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n2023-10-14 19:38:56.705169: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n2023-10-14 19:38:56.705184: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n","output_type":"stream"}]},{"cell_type":"code","metadata":{"id":"Ep_09mlyqa5G","colab":{"height":34,"base_uri":"https://localhost:8080/"},"outputId":"2fea20bb-a35c-40a4-e329-13dc1d796926","colab_type":"code","source_hash":null,"execution_start":1697312371141,"execution_millis":205,"deepnote_to_be_reexecuted":false,"cell_id":"d7f1ea02f55842a183892eafa13218e6","deepnote_cell_type":"code"},"source":"# Load the DataFrame that contains all features calculated in the last notebook (only run once)\nos.system('wget https://raw.githubusercontent.com/MedlyticsUniversal/Data/main/Week2/spoken_digit_manual_features.csv')","block_group":"b4149894eaba46f9a1ccb618d1062347","execution_count":null,"outputs":[{"name":"stderr","text":"--2023-10-14 19:39:31-- https://raw.githubusercontent.com/MedlyticsUniversal/Data/main/Week2/spoken_digit_manual_features.csv\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.108.133, 185.199.111.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 220478 (215K) [text/plain]\nSaving to: ‘spoken_digit_manual_features.csv’\n\n 0K .......... .......... .......... .......... .......... 23% 38.1M 0s\n 50K .......... .......... .......... .......... .......... 46% 45.7M 0s\n 100K .......... .......... .......... .......... .......... 69% 51.0M 0s\n 150K .......... .......... .......... .......... .......... 92% 56.8M 0s\n 200K .......... ..... 100% 215M=0.004s\n\n2023-10-14 19:39:31 (49.6 MB/s) - ‘spoken_digit_manual_features.csv’ saved [220478/220478]\n\n","output_type":"stream"},{"output_type":"execute_result","execution_count":2,"data":{"text/plain":"0"},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"zChcGaqVysRB","colab_type":"text","cell_id":"b11930b9b6d2494e97d45f60b6df61a0","deepnote_cell_type":"markdown"},"source":"## Load Training Data","block_group":"e9ca701adefa47eba6d68172031c9ba2"},{"cell_type":"code","metadata":{"id":"SVwsmOGvw7jp","colab":{"height":255,"base_uri":"https://localhost:8080/"},"outputId":"8cc4a4d8-7200-494b-951c-8ced02534ac3","colab_type":"code","source_hash":null,"execution_start":1697312406461,"execution_millis":284,"deepnote_to_be_reexecuted":false,"cell_id":"20e570bca0844d2196bb40a201dad224","deepnote_cell_type":"code"},"source":"# Load DataFrame and print its contents to jog memory\nspoken_df = pd.read_csv('spoken_digit_manual_features.csv', index_col = 0)\nprint(spoken_df.head(10))\nprint('\\n')\n\n# Check how many unique speakers exist in the dataset\nspeakers=set(spoken_df['speaker'])\nprint(f'There are {len(speakers)} unique speakers in the dataset.')\n\n# Our goal for this is to build a neural network that learns to classify which\n# of the 5 speakers is recorded in a sample based on the features:\n# spectral centroid, spectral flatness, and maximum frequency","block_group":"ef5d3d1731f84ac69602f246c7f0afa6","execution_count":null,"outputs":[{"name":"stdout","text":" file digit speaker trial SC SF \\\n0 5_yweweler_8.wav 5 yweweler 8 1029.497959 0.397336 \n1 3_george_49.wav 3 george 4 1881.296834 0.387050 \n2 9_yweweler_44.wav 9 yweweler 4 1093.951856 0.394981 \n3 8_yweweler_33.wav 8 yweweler 3 1409.543285 0.487496 \n4 7_theo_34.wav 7 theo 3 887.361601 0.396825 \n5 1_jackson_45.wav 1 jackson 4 1007.568129 0.324100 \n6 6_yweweler_18.wav 6 yweweler 1 1286.701352 0.498813 \n7 9_george_35.wav 9 george 3 1405.092061 0.353083 \n8 9_jackson_32.wav 9 jackson 3 1172.899961 0.477907 \n9 8_george_26.wav 8 george 2 1959.977577 0.462901 \n\n MF \n0 745.878340 \n1 323.943662 \n2 244.648318 \n3 392.350401 \n4 130.640309 \n5 216.306156 \n6 400.715564 \n7 447.239693 \n8 114.892780 \n9 320.537966 \n\n\nThere are 5 unique speakers in the dataset.\n","output_type":"stream"}]},{"cell_type":"markdown","metadata":{"id":"riycN8SdyxNT","colab_type":"text","cell_id":"45117c302b7f403f8a3e99543faa673b","deepnote_cell_type":"markdown"},"source":"## Build Neural Network","block_group":"80b9e644a09040788e97ec2b2ff70756"},{"cell_type":"code","metadata":{"id":"qMp_z7W9vZV4","colab":{},"colab_type":"code","source_hash":null,"execution_start":1697312443321,"execution_millis":832,"deepnote_to_be_reexecuted":false,"cell_id":"7537ee1178eb4fc6a3f69c67b7f4ca04","deepnote_cell_type":"code"},"source":"# Build the keras neural network\n\n# This allows us to add layers sequentially (i.e. first->last)\nmodel = tf.keras.Sequential()\n\n# Create a first layer of 12 neurons, and a rectified linear unit activation function\nmodel.add(tf.keras.layers.Dense(8, input_shape=(3,), activation=tf.nn.relu))\n# Input dimension needs to be number of features\n\n# Add two dense layers with 8 units each\n# (Note that we don't need to specify input size because keras determines input size from previous layer)\nmodel.add(tf.keras.layers.Dense(8, activation=tf.nn.relu))\nmodel.add(tf.keras.layers.Dense(8, activation=tf.nn.relu))\n\n# Output dimension needs to be number of classes in order for each to get a score\nmodel.add(tf.keras.layers.Dense(5, activation=tf.nn.softmax))","block_group":"08f54b445be74dfbb37019ee48b49a07","execution_count":null,"outputs":[{"name":"stderr","text":"2023-10-14 19:40:44.010925: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n2023-10-14 19:40:44.010989: W tensorflow/stream_executor/cuda/cuda_driver.cc:263] failed call to cuInit: UNKNOWN ERROR (303)\n2023-10-14 19:40:44.011026: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (p-c2703ed7-f8b4-40f5-8dd1-77fe823e4d60): /proc/driver/nvidia/version does not exist\n2023-10-14 19:40:44.011358: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\nTo enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n","output_type":"stream"}]},{"cell_type":"markdown","metadata":{"id":"9p9UDYX-y_v9","colab_type":"text","cell_id":"f564867540594b1da3b091b66f911753","deepnote_cell_type":"markdown"},"source":"## Specify a Loss Function and an Optimizer for Neural Network\n\nLet's describe why each of these components is necessary, and how it is used in training a neural network.\n\n**Loss Function**: This is the quantity that should be minimized when the network is trained. (It is like the mean squared error for a linear regression.) A neural network can use squared error as a loss function, but there are also other options. In the case of a neural network trying to classify samples into 1 of $n$ categories, a common choice is called cross-entropy loss.\n\n**Optimizer**: When a neural network is trained, it changes weights in its network to minimize the loss function. The optimizer governs how the neural network iteratively changes its weights as it minimizes loss. Many optimizers use the derivative of the loss function with respect to all the weights to decide which direction to change network weights.","block_group":"c8bedffec437405791ae2d59d9e634fd"},{"cell_type":"code","metadata":{"id":"fKRMZVOkyoUP","colab":{},"colab_type":"code","source_hash":null,"execution_start":1697312561913,"execution_millis":20,"deepnote_to_be_reexecuted":false,"cell_id":"89bff6ae42cc4e0692c095ec34d4346c","deepnote_cell_type":"code"},"source":"# Specify a loss function for our network\n\n# Note that the metrics input argument governs what will be reported as the network is trained \nmodel.compile(loss = tf.keras.losses.categorical_crossentropy,\n optimizer = tf.keras.optimizers.Adam(learning_rate=0.01), metrics = ['accuracy'])","block_group":"299d5fe09ddc448180546974a5314a12","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"xgP9AW_KSDuy","colab_type":"text","cell_id":"fbd92e0e7d0148bfa073fd8cbe636f47","deepnote_cell_type":"markdown"},"source":"## Convert Labels Into \"One-hot\" Vectors\n\nPredictions output by the model need to be compared to some truth label. Currently, the model predicts a 5-element vector of \"prediction values\" for every sample. The truth labels thus need to be converted to a 5-element vector with a 1 in the correct index and 0s in all others.","block_group":"00ca93a36fd74de3830adf5b8fc457e6"},{"cell_type":"code","metadata":{"id":"Ix3lG15TSo42","colab":{},"colab_type":"code","source_hash":null,"execution_start":1697312629913,"execution_millis":11,"deepnote_to_be_reexecuted":false,"cell_id":"220acd32193a437dbf519dbccc2d64a1","deepnote_cell_type":"code"},"source":"# Make dictionary to convert from speaker names to indices\nname2int_dict = {name: ind for (ind, name) in enumerate(set(spoken_df['speaker']))}\n\ny_labels = spoken_df['speaker']\n# Set y_labels to be indices of speaker\ny_labels = [name2int_dict[name] for name in y_labels]","block_group":"92d9624b83a942c2851cdd12169ce5b8","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"uegmriZU29sA","colab_type":"text","cell_id":"d2967d7517364510a9cff02d90b88d53","deepnote_cell_type":"markdown"},"source":"## Split Into Train/Validation/Test Sets","block_group":"f2ebe0434ac34ef4921400600feea40a"},{"cell_type":"code","metadata":{"id":"HM-ON_x930fu","colab":{},"colab_type":"code","source_hash":null,"execution_start":1697312712325,"execution_millis":95,"deepnote_to_be_reexecuted":false,"cell_id":"934b3d2d2843436e8c9c210973a72b55","deepnote_cell_type":"code"},"source":"# Downselect to only the 3 columns of the dataset we are learning from\nX_data = spoken_df[['SC', 'SF', 'MF']].to_numpy()\n\n# Decide how large to make validation and test sets\nn_val = 250\nn_test = 250\n\n# Shuffle data before partitioning\nX_data, y_labels = shuffle(X_data, y_labels, random_state = 25)\n\n# Partition\nX_data_test, y_labels_test = X_data[:n_test,:], y_labels[:n_test]\nX_data_val, y_labels_val = X_data[n_test:n_test+n_val,:], y_labels[n_test:n_test+n_val]\nX_data_train, y_labels_train = X_data[n_test+n_val:,:], y_labels[n_test+n_val:]","block_group":"3e36ca560d5d45d4945e83d6c3dd4a6f","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"cell_id":"ed8b0879d3a3427c9d1f2dec059bff21","deepnote_cell_type":"markdown"},"source":"## Standardize Data\n\nScaling data is generally good practice before attempting to fit a model. Having inputs with large differences in scale can affect how the optimizer changes weights to minimize the loss function.","block_group":"cc91b0ce81c941289670c3741b1be5bf"},{"cell_type":"code","metadata":{"source_hash":null,"execution_start":1697312716799,"execution_millis":88,"deepnote_to_be_reexecuted":false,"cell_id":"0f2cc597df364888a4a49fd084cc112b","deepnote_cell_type":"code"},"source":"# Scale data\nscaler = StandardScaler()\nX_data_train=scaler.fit_transform(X_data_train)\nX_data_val = scaler.transform(X_data_val)\nX_data_test = scaler.transform(X_data_test)\n\n# Convert labels to one-hot\ny_labels_train = tf.keras.utils.to_categorical(y_labels_train, 5)\ny_labels_val = tf.keras.utils.to_categorical(y_labels_val, 5)\ny_labels_test = tf.keras.utils.to_categorical(y_labels_test, 5)\n\ntraining_set = tf.data.Dataset.from_tensor_slices((X_data_train, y_labels_train))","block_group":"d96fd8bbb0de460d919e6ffaae8660e2","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"DnDGqneBOsW_","colab_type":"text","cell_id":"1acf219026b34f81971491b334893905","deepnote_cell_type":"markdown"},"source":"## Fit Model to Data\n\nHere, we will specify the number of epochs and batch size.\n\n**Batch Size**: In each iteration of the optimizer, how many samples are taken into account when calculating derivatives of the loss function? (If batch size is less than the number of samples, there will be multiple optimization iterations per epoch.)\n\n**Epochs**: How many times should the data be passed through before optimization is finished?","block_group":"0447d4210a3c49ea907fac1b981f7feb"},{"cell_type":"code","metadata":{"id":"nkeUHhxZ3iOF","colab":{"height":1000,"base_uri":"https://localhost:8080/"},"outputId":"fd52264a-8db8-4abf-85e4-da0bf7a8c2b1","colab_type":"code","source_hash":null,"execution_start":1697312781385,"execution_millis":17197,"deepnote_to_be_reexecuted":false,"cell_id":"3ac8d51014e4460d8dddedd16e540a86","deepnote_cell_type":"code"},"source":"epochs = 50\nbatch_size = 100\n\ntraining_set = training_set.batch(batch_size) # Set batch size\n\nfor epoch in range(epochs):\n for signals, labels in training_set:\n tr_loss, tr_accuracy = model.train_on_batch(signals, labels)\n val_loss, val_accuracy = model.evaluate(X_data_val, y_labels_val)\n print(('Epoch #%d\\t Training Loss: %.2f\\tTraining Accuracy: %.2f\\t'\n 'Validation Loss: %.2f\\tValidation Accuracy: %.2f')\n % (epoch + 1, tr_loss, tr_accuracy,\n val_loss, val_accuracy))","block_group":"2a69cab3c5854df28960601809f8f5e3","execution_count":null,"outputs":[{"name":"stderr","text":"2023-10-14 19:46:22.220219: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 2ms/step - loss: 1.4516 - accuracy: 0.3360\nEpoch #1\t Training Loss: 1.42\tTraining Accuracy: 0.38\tValidation Loss: 1.45\tValidation Accuracy: 0.34\n8/8 [==============================] - 0s 2ms/step - loss: 1.3043 - accuracy: 0.4240\nEpoch #2\t Training Loss: 1.26\tTraining Accuracy: 0.49\tValidation Loss: 1.30\tValidation Accuracy: 0.42\n8/8 [==============================] - 0s 3ms/step - loss: 1.1791 - accuracy: 0.4800\nEpoch #3\t Training Loss: 1.17\tTraining Accuracy: 0.53\tValidation Loss: 1.18\tValidation Accuracy: 0.48\n8/8 [==============================] - 0s 2ms/step - loss: 1.1072 - accuracy: 0.5240\nEpoch #4\t Training Loss: 1.10\tTraining Accuracy: 0.53\tValidation Loss: 1.11\tValidation Accuracy: 0.52\n8/8 [==============================] - 0s 2ms/step - loss: 1.0686 - accuracy: 0.5640\nEpoch #5\t Training Loss: 1.09\tTraining Accuracy: 0.55\tValidation Loss: 1.07\tValidation Accuracy: 0.56\n2023-10-14 19:46:23.898423: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 3ms/step - loss: 1.0452 - accuracy: 0.5840\nEpoch #6\t Training Loss: 1.09\tTraining Accuracy: 0.52\tValidation Loss: 1.05\tValidation Accuracy: 0.58\n2023-10-14 19:46:24.253749: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 2ms/step - loss: 1.0393 - accuracy: 0.5880\nEpoch #7\t Training Loss: 1.08\tTraining Accuracy: 0.54\tValidation Loss: 1.04\tValidation Accuracy: 0.59\n8/8 [==============================] - 0s 2ms/step - loss: 1.0291 - accuracy: 0.5880\nEpoch #8\t Training Loss: 1.06\tTraining Accuracy: 0.52\tValidation Loss: 1.03\tValidation Accuracy: 0.59\n8/8 [==============================] - 0s 2ms/step - loss: 1.0217 - accuracy: 0.5880\nEpoch #9\t Training Loss: 1.05\tTraining Accuracy: 0.54\tValidation Loss: 1.02\tValidation Accuracy: 0.59\n1/8 [==>...........................] - ETA: 0s - loss: 1.1084 - accuracy: 0.56252023-10-14 19:46:25.315114: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n2023-10-14 19:46:25.384361: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 2ms/step - loss: 1.0202 - accuracy: 0.5960\nEpoch #10\t Training Loss: 1.03\tTraining Accuracy: 0.55\tValidation Loss: 1.02\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 1.0204 - accuracy: 0.5880\nEpoch #11\t Training Loss: 1.01\tTraining Accuracy: 0.58\tValidation Loss: 1.02\tValidation Accuracy: 0.59\n8/8 [==============================] - 0s 2ms/step - loss: 1.0160 - accuracy: 0.6000\nEpoch #12\t Training Loss: 1.01\tTraining Accuracy: 0.59\tValidation Loss: 1.02\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 1.0129 - accuracy: 0.5960\n2023-10-14 19:46:26.423604: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #13\t Training Loss: 1.00\tTraining Accuracy: 0.57\tValidation Loss: 1.01\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 1.0119 - accuracy: 0.5920\n2023-10-14 19:46:26.668764: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #14\t Training Loss: 1.00\tTraining Accuracy: 0.58\tValidation Loss: 1.01\tValidation Accuracy: 0.59\n8/8 [==============================] - 0s 2ms/step - loss: 1.0130 - accuracy: 0.6000\nEpoch #15\t Training Loss: 1.00\tTraining Accuracy: 0.56\tValidation Loss: 1.01\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 1.0083 - accuracy: 0.6040\n2023-10-14 19:46:27.345692: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #16\t Training Loss: 1.00\tTraining Accuracy: 0.56\tValidation Loss: 1.01\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 1.0068 - accuracy: 0.6040\nEpoch #17\t Training Loss: 1.00\tTraining Accuracy: 0.57\tValidation Loss: 1.01\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 0.9986 - accuracy: 0.6040\nEpoch #18\t Training Loss: 1.00\tTraining Accuracy: 0.56\tValidation Loss: 1.00\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 0.9893 - accuracy: 0.6120\n2023-10-14 19:46:28.260411: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #19\t Training Loss: 1.00\tTraining Accuracy: 0.57\tValidation Loss: 0.99\tValidation Accuracy: 0.61\n1/8 [==>...........................] - ETA: 0s - loss: 1.1140 - accuracy: 0.56252023-10-14 19:46:28.539255: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 2ms/step - loss: 0.9888 - accuracy: 0.6160\nEpoch #20\t Training Loss: 0.99\tTraining Accuracy: 0.57\tValidation Loss: 0.99\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 2ms/step - loss: 0.9920 - accuracy: 0.6080\nEpoch #21\t Training Loss: 0.99\tTraining Accuracy: 0.58\tValidation Loss: 0.99\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 3ms/step - loss: 0.9894 - accuracy: 0.6040\n2023-10-14 19:46:29.227107: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #22\t Training Loss: 0.99\tTraining Accuracy: 0.58\tValidation Loss: 0.99\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 0.9850 - accuracy: 0.6040\nEpoch #23\t Training Loss: 0.99\tTraining Accuracy: 0.57\tValidation Loss: 0.99\tValidation Accuracy: 0.60\n2023-10-14 19:46:29.738971: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 2ms/step - loss: 0.9769 - accuracy: 0.6080\nEpoch #24\t Training Loss: 1.00\tTraining Accuracy: 0.58\tValidation Loss: 0.98\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9760 - accuracy: 0.6080\nEpoch #25\t Training Loss: 0.98\tTraining Accuracy: 0.57\tValidation Loss: 0.98\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9650 - accuracy: 0.6120\nEpoch #26\t Training Loss: 0.99\tTraining Accuracy: 0.58\tValidation Loss: 0.97\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9697 - accuracy: 0.6040\n2023-10-14 19:46:30.771979: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #27\t Training Loss: 0.98\tTraining Accuracy: 0.57\tValidation Loss: 0.97\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 4ms/step - loss: 0.9632 - accuracy: 0.6080\nEpoch #28\t Training Loss: 0.98\tTraining Accuracy: 0.57\tValidation Loss: 0.96\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9605 - accuracy: 0.6120\nEpoch #29\t Training Loss: 0.98\tTraining Accuracy: 0.58\tValidation Loss: 0.96\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9565 - accuracy: 0.6080\n2023-10-14 19:46:31.805440: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #30\t Training Loss: 0.97\tTraining Accuracy: 0.58\tValidation Loss: 0.96\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 3ms/step - loss: 0.9554 - accuracy: 0.6080\nEpoch #31\t Training Loss: 0.97\tTraining Accuracy: 0.58\tValidation Loss: 0.96\tValidation Accuracy: 0.61\n2023-10-14 19:46:32.286529: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n2023-10-14 19:46:32.414351: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 4ms/step - loss: 0.9502 - accuracy: 0.6080\nEpoch #32\t Training Loss: 0.97\tTraining Accuracy: 0.57\tValidation Loss: 0.95\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9433 - accuracy: 0.6200\nEpoch #33\t Training Loss: 0.96\tTraining Accuracy: 0.57\tValidation Loss: 0.94\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 2ms/step - loss: 0.9395 - accuracy: 0.6280\nEpoch #34\t Training Loss: 0.96\tTraining Accuracy: 0.57\tValidation Loss: 0.94\tValidation Accuracy: 0.63\n8/8 [==============================] - 0s 2ms/step - loss: 0.9352 - accuracy: 0.6280\nEpoch #35\t Training Loss: 0.96\tTraining Accuracy: 0.56\tValidation Loss: 0.94\tValidation Accuracy: 0.63\n8/8 [==============================] - 0s 2ms/step - loss: 0.9359 - accuracy: 0.6240\nEpoch #36\t Training Loss: 0.95\tTraining Accuracy: 0.56\tValidation Loss: 0.94\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 3ms/step - loss: 0.9319 - accuracy: 0.6160\nEpoch #37\t Training Loss: 0.96\tTraining Accuracy: 0.56\tValidation Loss: 0.93\tValidation Accuracy: 0.62\n2023-10-14 19:46:34.223328: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 2ms/step - loss: 0.9328 - accuracy: 0.6160\nEpoch #38\t Training Loss: 0.95\tTraining Accuracy: 0.55\tValidation Loss: 0.93\tValidation Accuracy: 0.62\n2023-10-14 19:46:34.505612: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n2023-10-14 19:46:34.539904: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 4ms/step - loss: 0.9307 - accuracy: 0.6200\nEpoch #39\t Training Loss: 0.94\tTraining Accuracy: 0.56\tValidation Loss: 0.93\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 2ms/step - loss: 0.9295 - accuracy: 0.6120\nEpoch #40\t Training Loss: 0.94\tTraining Accuracy: 0.56\tValidation Loss: 0.93\tValidation Accuracy: 0.61\n2023-10-14 19:46:35.210855: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 3ms/step - loss: 0.9284 - accuracy: 0.6120\nEpoch #41\t Training Loss: 0.94\tTraining Accuracy: 0.56\tValidation Loss: 0.93\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 4ms/step - loss: 0.9269 - accuracy: 0.6160\nEpoch #42\t Training Loss: 0.94\tTraining Accuracy: 0.56\tValidation Loss: 0.93\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 3ms/step - loss: 0.9253 - accuracy: 0.6160\n2023-10-14 19:46:35.928499: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #43\t Training Loss: 0.93\tTraining Accuracy: 0.57\tValidation Loss: 0.93\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 2ms/step - loss: 0.9222 - accuracy: 0.6160\nEpoch #44\t Training Loss: 0.93\tTraining Accuracy: 0.57\tValidation Loss: 0.92\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 2ms/step - loss: 0.9205 - accuracy: 0.6200\nEpoch #45\t Training Loss: 0.93\tTraining Accuracy: 0.57\tValidation Loss: 0.92\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 2ms/step - loss: 0.9193 - accuracy: 0.6200\nEpoch #46\t Training Loss: 0.92\tTraining Accuracy: 0.57\tValidation Loss: 0.92\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 3ms/step - loss: 0.9202 - accuracy: 0.6080\n2023-10-14 19:46:37.221619: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n2023-10-14 19:46:37.248007: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #47\t Training Loss: 0.92\tTraining Accuracy: 0.58\tValidation Loss: 0.92\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9190 - accuracy: 0.6080\nEpoch #48\t Training Loss: 0.91\tTraining Accuracy: 0.59\tValidation Loss: 0.92\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9176 - accuracy: 0.6120\nEpoch #49\t Training Loss: 0.91\tTraining Accuracy: 0.59\tValidation Loss: 0.92\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 3ms/step - loss: 0.9160 - accuracy: 0.6160\nEpoch #50\t Training Loss: 0.91\tTraining Accuracy: 0.59\tValidation Loss: 0.92\tValidation Accuracy: 0.62\n2023-10-14 19:46:38.224018: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n","output_type":"stream"}]},{"cell_type":"markdown","metadata":{"id":"OadIT5MEaOJA","colab_type":"text","cell_id":"0dd37b88234e4a9394ba84efb1ef11f1","deepnote_cell_type":"markdown"},"source":"## Check Performance on Test Set\n\nWe can use `model.predict` to output predicted labels on the test set, or `model.evaluate` to determine test set accuracy (since we have the labels)","block_group":"9a5ea918c9384addbfb53b315f7aaa1f"},{"cell_type":"code","metadata":{"id":"Mr55MHYhafFa","colab":{"height":34,"base_uri":"https://localhost:8080/"},"outputId":"201332aa-9aca-4af8-e01f-d9fc101e8642","colab_type":"code","source_hash":null,"execution_start":1697312822845,"execution_millis":64,"deepnote_to_be_reexecuted":false,"cell_id":"ca956ed7b81c45f1980175c48f5b4171","deepnote_cell_type":"code"},"source":"test_loss, test_accuracy = model.evaluate(X_data_test, y_labels_test)","block_group":"71f78ae6d3f9491290bfdb4c65611c58","execution_count":null,"outputs":[{"name":"stdout","text":"8/8 [==============================] - 0s 2ms/step - loss: 0.9412 - accuracy: 0.6160\n","output_type":"stream"}]},{"cell_type":"markdown","source":"\n
\nCreated in Deepnote","metadata":{"created_in_deepnote_cell":true,"deepnote_cell_type":"markdown"}}],"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"NeuralNetworks_Tutorial_part1.ipynb","provenance":[],"collapsed_sections":[],"include_colab_link":true},"deepnote":{},"kernelspec":{"name":"python3","display_name":"Python 3"},"deepnote_notebook_id":"f4b4af0867e74f84ba65d2c283054b1c","deepnote_persisted_session":{"createdAt":"2023-10-14T20:46:41.583Z"},"deepnote_execution_queue":[]}}
\ No newline at end of file
+{"cells":[{"cell_type":"markdown","metadata":{"id":"PlpG_fIiqdJU","colab_type":"text","cell_id":"cb77df6fe0424017a03c3a38a4c18864","deepnote_cell_type":"markdown"},"source":"# Building a Simple Neural Network with `TensorFlow.Keras`\n> Neural Networks Part 1\n\nIn this notebook, we are going to walk through building a simple neural network to classify sequence data. This tutorial will be meant as a fast overview to building/training neural networks with `Keras`.","block_group":"c24d660ee0f4487ea88143183fa285be"},{"cell_type":"code","metadata":{"id":"eFSzOvSduDf8","colab":{"height":34,"base_uri":"https://localhost:8080/"},"outputId":"61c63452-faa5-499b-852b-080a0a7b6c0a","colab_type":"code","source_hash":null,"execution_start":1697312330884,"execution_millis":8029,"deepnote_to_be_reexecuted":false,"cell_id":"aefbbbb7c04347879e490d71a0d18bca","deepnote_cell_type":"code"},"source":"# Import useful libraries\n\n# Needed for terminal functions (i.e. wget)\nimport os\n\n# For plotting\nimport matplotlib.pyplot as plt\n\n# For DataFrame manipulation\nimport pandas as pd\n\n# For data preprocessing\nfrom sklearn.preprocessing import StandardScaler #Use StandardScaler from scikitlearn\nfrom sklearn.utils import shuffle #Used to shuffle up examples before training\n\n# Keras-related imports\nfrom keras.models import Sequential #we will build our models layer by layer\nfrom keras.layers import Dense #we want to use dense layers in our model\n\n# Keras is built on top of the TensorFlow library\nimport tensorflow as tf\n# tf has many helpful functions for training networks like loss functions, optimization methods, etc.","block_group":"8108374e8d2f41d1bdcaeb9add10ec44","execution_count":null,"outputs":[{"name":"stderr","text":"2023-10-14 19:38:54.117469: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\nTo enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n2023-10-14 19:38:54.510494: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n2023-10-14 19:38:54.510549: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n2023-10-14 19:38:54.616485: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n2023-10-14 19:38:56.705042: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n2023-10-14 19:38:56.705169: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n2023-10-14 19:38:56.705184: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n","output_type":"stream"}]},{"cell_type":"code","metadata":{"id":"Ep_09mlyqa5G","colab":{"height":34,"base_uri":"https://localhost:8080/"},"outputId":"2fea20bb-a35c-40a4-e329-13dc1d796926","colab_type":"code","source_hash":null,"execution_start":1697312371141,"execution_millis":205,"deepnote_to_be_reexecuted":false,"cell_id":"d7f1ea02f55842a183892eafa13218e6","deepnote_cell_type":"code"},"source":"# Load the DataFrame that contains all features calculated in the last notebook (only run once)\nos.system('wget https://raw.githubusercontent.com/MedlyticsUniversal/Data/main/Week2/spoken_digit_manual_features.csv')","block_group":"b4149894eaba46f9a1ccb618d1062347","execution_count":null,"outputs":[{"name":"stderr","text":"--2023-10-14 19:39:31-- https://raw.githubusercontent.com/MedlyticsUniversal/Data/main/Week2/spoken_digit_manual_features.csv\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.108.133, 185.199.111.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 220478 (215K) [text/plain]\nSaving to: ‘spoken_digit_manual_features.csv’\n\n 0K .......... .......... .......... .......... .......... 23% 38.1M 0s\n 50K .......... .......... .......... .......... .......... 46% 45.7M 0s\n 100K .......... .......... .......... .......... .......... 69% 51.0M 0s\n 150K .......... .......... .......... .......... .......... 92% 56.8M 0s\n 200K .......... ..... 100% 215M=0.004s\n\n2023-10-14 19:39:31 (49.6 MB/s) - ‘spoken_digit_manual_features.csv’ saved [220478/220478]\n\n","output_type":"stream"},{"output_type":"execute_result","execution_count":2,"data":{"text/plain":"0"},"metadata":{}}]},{"cell_type":"markdown","metadata":{"id":"zChcGaqVysRB","colab_type":"text","cell_id":"b11930b9b6d2494e97d45f60b6df61a0","deepnote_cell_type":"markdown"},"source":"## Load Training Data","block_group":"e9ca701adefa47eba6d68172031c9ba2"},{"cell_type":"code","metadata":{"id":"SVwsmOGvw7jp","colab":{"height":255,"base_uri":"https://localhost:8080/"},"outputId":"8cc4a4d8-7200-494b-951c-8ced02534ac3","colab_type":"code","source_hash":null,"execution_start":1697312406461,"execution_millis":284,"deepnote_to_be_reexecuted":false,"cell_id":"20e570bca0844d2196bb40a201dad224","deepnote_cell_type":"code"},"source":"# Load DataFrame and print its contents to jog memory\nspoken_df = pd.read_csv('spoken_digit_manual_features.csv', index_col = 0)\nprint(spoken_df.head(10))\nprint('\\n')\n\n# Check how many unique speakers exist in the dataset\nspeakers=set(spoken_df['speaker'])\nprint(f'There are {len(speakers)} unique speakers in the dataset.')\n\n# Our goal for this is to build a neural network that learns to classify which\n# of the 5 speakers is recorded in a sample based on the features:\n# spectral centroid, spectral flatness, and maximum frequency","block_group":"ef5d3d1731f84ac69602f246c7f0afa6","execution_count":null,"outputs":[{"name":"stdout","text":" file digit speaker trial SC SF \\\n0 5_yweweler_8.wav 5 yweweler 8 1029.497959 0.397336 \n1 3_george_49.wav 3 george 4 1881.296834 0.387050 \n2 9_yweweler_44.wav 9 yweweler 4 1093.951856 0.394981 \n3 8_yweweler_33.wav 8 yweweler 3 1409.543285 0.487496 \n4 7_theo_34.wav 7 theo 3 887.361601 0.396825 \n5 1_jackson_45.wav 1 jackson 4 1007.568129 0.324100 \n6 6_yweweler_18.wav 6 yweweler 1 1286.701352 0.498813 \n7 9_george_35.wav 9 george 3 1405.092061 0.353083 \n8 9_jackson_32.wav 9 jackson 3 1172.899961 0.477907 \n9 8_george_26.wav 8 george 2 1959.977577 0.462901 \n\n MF \n0 745.878340 \n1 323.943662 \n2 244.648318 \n3 392.350401 \n4 130.640309 \n5 216.306156 \n6 400.715564 \n7 447.239693 \n8 114.892780 \n9 320.537966 \n\n\nThere are 5 unique speakers in the dataset.\n","output_type":"stream"}]},{"cell_type":"markdown","metadata":{"id":"riycN8SdyxNT","colab_type":"text","cell_id":"45117c302b7f403f8a3e99543faa673b","deepnote_cell_type":"markdown"},"source":"## Build Neural Network","block_group":"80b9e644a09040788e97ec2b2ff70756"},{"cell_type":"code","metadata":{"id":"qMp_z7W9vZV4","colab":{},"colab_type":"code","source_hash":null,"execution_start":1697312443321,"execution_millis":832,"deepnote_to_be_reexecuted":false,"cell_id":"7537ee1178eb4fc6a3f69c67b7f4ca04","deepnote_cell_type":"code"},"source":"# Build the keras neural network\n\n# This allows us to add layers sequentially (i.e. first->last)\nmodel = tf.keras.Sequential()\n\n# Create a first layer of 12 neurons, and a rectified linear unit activation function\nmodel.add(tf.keras.layers.Dense(12, input_shape=(3,), activation=tf.nn.relu))\n# Input dimension needs to be number of features\n\n# Add two dense layers with 8 units each\n# (Note that we don't need to specify input size because keras determines input size from previous layer)\nmodel.add(tf.keras.layers.Dense(8, activation=tf.nn.relu))\nmodel.add(tf.keras.layers.Dense(8, activation=tf.nn.relu))\n\n# Output dimension needs to be number of classes in order for each to get a score\nmodel.add(tf.keras.layers.Dense(5, activation=tf.nn.softmax))","block_group":"08f54b445be74dfbb37019ee48b49a07","execution_count":null,"outputs":[{"name":"stderr","text":"2023-10-14 19:40:44.010925: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n2023-10-14 19:40:44.010989: W tensorflow/stream_executor/cuda/cuda_driver.cc:263] failed call to cuInit: UNKNOWN ERROR (303)\n2023-10-14 19:40:44.011026: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (p-c2703ed7-f8b4-40f5-8dd1-77fe823e4d60): /proc/driver/nvidia/version does not exist\n2023-10-14 19:40:44.011358: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\nTo enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n","output_type":"stream"}]},{"cell_type":"markdown","metadata":{"id":"9p9UDYX-y_v9","colab_type":"text","cell_id":"f564867540594b1da3b091b66f911753","deepnote_cell_type":"markdown"},"source":"## Specify a Loss Function and an Optimizer for Neural Network\n\nLet's describe why each of these components is necessary, and how it is used in training a neural network.\n\n**Loss Function**: This is the quantity that should be minimized when the network is trained. (It is like the mean squared error for a linear regression.) A neural network can use squared error as a loss function, but there are also other options. In the case of a neural network trying to classify samples into 1 of $n$ categories, a common choice is called cross-entropy loss.\n\n**Optimizer**: When a neural network is trained, it changes weights in its network to minimize the loss function. The optimizer governs how the neural network iteratively changes its weights as it minimizes loss. Many optimizers use the derivative of the loss function with respect to all the weights to decide which direction to change network weights.","block_group":"c8bedffec437405791ae2d59d9e634fd"},{"cell_type":"code","metadata":{"id":"fKRMZVOkyoUP","colab":{},"colab_type":"code","source_hash":null,"execution_start":1697312561913,"execution_millis":20,"deepnote_to_be_reexecuted":false,"cell_id":"89bff6ae42cc4e0692c095ec34d4346c","deepnote_cell_type":"code"},"source":"# Specify a loss function for our network\n\n# Note that the metrics input argument governs what will be reported as the network is trained \nmodel.compile(loss = tf.keras.losses.categorical_crossentropy,\n optimizer = tf.keras.optimizers.Adam(learning_rate=0.01), metrics = ['accuracy'])","block_group":"299d5fe09ddc448180546974a5314a12","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"xgP9AW_KSDuy","colab_type":"text","cell_id":"fbd92e0e7d0148bfa073fd8cbe636f47","deepnote_cell_type":"markdown"},"source":"## Convert Labels Into \"One-hot\" Vectors\n\nPredictions output by the model need to be compared to some truth label. Currently, the model predicts a 5-element vector of \"prediction values\" for every sample. The truth labels thus need to be converted to a 5-element vector with a 1 in the correct index and 0s in all others.","block_group":"00ca93a36fd74de3830adf5b8fc457e6"},{"cell_type":"code","metadata":{"id":"Ix3lG15TSo42","colab":{},"colab_type":"code","source_hash":null,"execution_start":1697312629913,"execution_millis":11,"deepnote_to_be_reexecuted":false,"cell_id":"220acd32193a437dbf519dbccc2d64a1","deepnote_cell_type":"code"},"source":"# Make dictionary to convert from speaker names to indices\nname2int_dict = {name: ind for (ind, name) in enumerate(set(spoken_df['speaker']))}\n\ny_labels = spoken_df['speaker']\n# Set y_labels to be indices of speaker\ny_labels = [name2int_dict[name] for name in y_labels]","block_group":"92d9624b83a942c2851cdd12169ce5b8","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"uegmriZU29sA","colab_type":"text","cell_id":"d2967d7517364510a9cff02d90b88d53","deepnote_cell_type":"markdown"},"source":"## Split Into Train/Validation/Test Sets","block_group":"f2ebe0434ac34ef4921400600feea40a"},{"cell_type":"code","metadata":{"id":"HM-ON_x930fu","colab":{},"colab_type":"code","source_hash":null,"execution_start":1697312712325,"execution_millis":95,"deepnote_to_be_reexecuted":false,"cell_id":"934b3d2d2843436e8c9c210973a72b55","deepnote_cell_type":"code"},"source":"# Downselect to only the 3 columns of the dataset we are learning from\nX_data = spoken_df[['SC', 'SF', 'MF']].to_numpy()\n\n# Decide how large to make validation and test sets\nn_val = 250\nn_test = 250\n\n# Shuffle data before partitioning\nX_data, y_labels = shuffle(X_data, y_labels, random_state = 25)\n\n# Partition\nX_data_test, y_labels_test = X_data[:n_test,:], y_labels[:n_test]\nX_data_val, y_labels_val = X_data[n_test:n_test+n_val,:], y_labels[n_test:n_test+n_val]\nX_data_train, y_labels_train = X_data[n_test+n_val:,:], y_labels[n_test+n_val:]","block_group":"3e36ca560d5d45d4945e83d6c3dd4a6f","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"cell_id":"ed8b0879d3a3427c9d1f2dec059bff21","deepnote_cell_type":"markdown"},"source":"## Standardize Data\n\nScaling data is generally good practice before attempting to fit a model. Having inputs with large differences in scale can affect how the optimizer changes weights to minimize the loss function.","block_group":"cc91b0ce81c941289670c3741b1be5bf"},{"cell_type":"code","metadata":{"source_hash":null,"execution_start":1697312716799,"execution_millis":88,"deepnote_to_be_reexecuted":false,"cell_id":"0f2cc597df364888a4a49fd084cc112b","deepnote_cell_type":"code"},"source":"# Scale data\nscaler = StandardScaler()\nX_data_train=scaler.fit_transform(X_data_train)\nX_data_val = scaler.transform(X_data_val)\nX_data_test = scaler.transform(X_data_test)\n\n# Convert labels to one-hot\ny_labels_train = tf.keras.utils.to_categorical(y_labels_train, 5)\ny_labels_val = tf.keras.utils.to_categorical(y_labels_val, 5)\ny_labels_test = tf.keras.utils.to_categorical(y_labels_test, 5)\n\ntraining_set = tf.data.Dataset.from_tensor_slices((X_data_train, y_labels_train))","block_group":"d96fd8bbb0de460d919e6ffaae8660e2","execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"DnDGqneBOsW_","colab_type":"text","cell_id":"1acf219026b34f81971491b334893905","deepnote_cell_type":"markdown"},"source":"## Fit Model to Data\n\nHere, we will specify the number of epochs and batch size.\n\n**Batch Size**: In each iteration of the optimizer, how many samples are taken into account when calculating derivatives of the loss function? (If batch size is less than the number of samples, there will be multiple optimization iterations per epoch.)\n\n**Epochs**: How many times should the data be passed through before optimization is finished?","block_group":"0447d4210a3c49ea907fac1b981f7feb"},{"cell_type":"code","metadata":{"id":"nkeUHhxZ3iOF","colab":{"height":1000,"base_uri":"https://localhost:8080/"},"outputId":"fd52264a-8db8-4abf-85e4-da0bf7a8c2b1","colab_type":"code","source_hash":null,"execution_start":1697312781385,"execution_millis":17197,"deepnote_to_be_reexecuted":false,"cell_id":"3ac8d51014e4460d8dddedd16e540a86","deepnote_cell_type":"code"},"source":"epochs = 50\nbatch_size = 100\n\ntraining_set = training_set.batch(batch_size) # Set batch size\n\nfor epoch in range(epochs):\n for signals, labels in training_set:\n tr_loss, tr_accuracy = model.train_on_batch(signals, labels)\n val_loss, val_accuracy = model.evaluate(X_data_val, y_labels_val)\n print(('Epoch #%d\\t Training Loss: %.2f\\tTraining Accuracy: %.2f\\t'\n 'Validation Loss: %.2f\\tValidation Accuracy: %.2f')\n % (epoch + 1, tr_loss, tr_accuracy,\n val_loss, val_accuracy))","block_group":"2a69cab3c5854df28960601809f8f5e3","execution_count":null,"outputs":[{"name":"stderr","text":"2023-10-14 19:46:22.220219: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 2ms/step - loss: 1.4516 - accuracy: 0.3360\nEpoch #1\t Training Loss: 1.42\tTraining Accuracy: 0.38\tValidation Loss: 1.45\tValidation Accuracy: 0.34\n8/8 [==============================] - 0s 2ms/step - loss: 1.3043 - accuracy: 0.4240\nEpoch #2\t Training Loss: 1.26\tTraining Accuracy: 0.49\tValidation Loss: 1.30\tValidation Accuracy: 0.42\n8/8 [==============================] - 0s 3ms/step - loss: 1.1791 - accuracy: 0.4800\nEpoch #3\t Training Loss: 1.17\tTraining Accuracy: 0.53\tValidation Loss: 1.18\tValidation Accuracy: 0.48\n8/8 [==============================] - 0s 2ms/step - loss: 1.1072 - accuracy: 0.5240\nEpoch #4\t Training Loss: 1.10\tTraining Accuracy: 0.53\tValidation Loss: 1.11\tValidation Accuracy: 0.52\n8/8 [==============================] - 0s 2ms/step - loss: 1.0686 - accuracy: 0.5640\nEpoch #5\t Training Loss: 1.09\tTraining Accuracy: 0.55\tValidation Loss: 1.07\tValidation Accuracy: 0.56\n2023-10-14 19:46:23.898423: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 3ms/step - loss: 1.0452 - accuracy: 0.5840\nEpoch #6\t Training Loss: 1.09\tTraining Accuracy: 0.52\tValidation Loss: 1.05\tValidation Accuracy: 0.58\n2023-10-14 19:46:24.253749: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 2ms/step - loss: 1.0393 - accuracy: 0.5880\nEpoch #7\t Training Loss: 1.08\tTraining Accuracy: 0.54\tValidation Loss: 1.04\tValidation Accuracy: 0.59\n8/8 [==============================] - 0s 2ms/step - loss: 1.0291 - accuracy: 0.5880\nEpoch #8\t Training Loss: 1.06\tTraining Accuracy: 0.52\tValidation Loss: 1.03\tValidation Accuracy: 0.59\n8/8 [==============================] - 0s 2ms/step - loss: 1.0217 - accuracy: 0.5880\nEpoch #9\t Training Loss: 1.05\tTraining Accuracy: 0.54\tValidation Loss: 1.02\tValidation Accuracy: 0.59\n1/8 [==>...........................] - ETA: 0s - loss: 1.1084 - accuracy: 0.56252023-10-14 19:46:25.315114: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n2023-10-14 19:46:25.384361: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 2ms/step - loss: 1.0202 - accuracy: 0.5960\nEpoch #10\t Training Loss: 1.03\tTraining Accuracy: 0.55\tValidation Loss: 1.02\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 1.0204 - accuracy: 0.5880\nEpoch #11\t Training Loss: 1.01\tTraining Accuracy: 0.58\tValidation Loss: 1.02\tValidation Accuracy: 0.59\n8/8 [==============================] - 0s 2ms/step - loss: 1.0160 - accuracy: 0.6000\nEpoch #12\t Training Loss: 1.01\tTraining Accuracy: 0.59\tValidation Loss: 1.02\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 1.0129 - accuracy: 0.5960\n2023-10-14 19:46:26.423604: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #13\t Training Loss: 1.00\tTraining Accuracy: 0.57\tValidation Loss: 1.01\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 1.0119 - accuracy: 0.5920\n2023-10-14 19:46:26.668764: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #14\t Training Loss: 1.00\tTraining Accuracy: 0.58\tValidation Loss: 1.01\tValidation Accuracy: 0.59\n8/8 [==============================] - 0s 2ms/step - loss: 1.0130 - accuracy: 0.6000\nEpoch #15\t Training Loss: 1.00\tTraining Accuracy: 0.56\tValidation Loss: 1.01\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 1.0083 - accuracy: 0.6040\n2023-10-14 19:46:27.345692: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #16\t Training Loss: 1.00\tTraining Accuracy: 0.56\tValidation Loss: 1.01\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 1.0068 - accuracy: 0.6040\nEpoch #17\t Training Loss: 1.00\tTraining Accuracy: 0.57\tValidation Loss: 1.01\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 0.9986 - accuracy: 0.6040\nEpoch #18\t Training Loss: 1.00\tTraining Accuracy: 0.56\tValidation Loss: 1.00\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 0.9893 - accuracy: 0.6120\n2023-10-14 19:46:28.260411: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #19\t Training Loss: 1.00\tTraining Accuracy: 0.57\tValidation Loss: 0.99\tValidation Accuracy: 0.61\n1/8 [==>...........................] - ETA: 0s - loss: 1.1140 - accuracy: 0.56252023-10-14 19:46:28.539255: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 2ms/step - loss: 0.9888 - accuracy: 0.6160\nEpoch #20\t Training Loss: 0.99\tTraining Accuracy: 0.57\tValidation Loss: 0.99\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 2ms/step - loss: 0.9920 - accuracy: 0.6080\nEpoch #21\t Training Loss: 0.99\tTraining Accuracy: 0.58\tValidation Loss: 0.99\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 3ms/step - loss: 0.9894 - accuracy: 0.6040\n2023-10-14 19:46:29.227107: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #22\t Training Loss: 0.99\tTraining Accuracy: 0.58\tValidation Loss: 0.99\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 2ms/step - loss: 0.9850 - accuracy: 0.6040\nEpoch #23\t Training Loss: 0.99\tTraining Accuracy: 0.57\tValidation Loss: 0.99\tValidation Accuracy: 0.60\n2023-10-14 19:46:29.738971: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 2ms/step - loss: 0.9769 - accuracy: 0.6080\nEpoch #24\t Training Loss: 1.00\tTraining Accuracy: 0.58\tValidation Loss: 0.98\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9760 - accuracy: 0.6080\nEpoch #25\t Training Loss: 0.98\tTraining Accuracy: 0.57\tValidation Loss: 0.98\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9650 - accuracy: 0.6120\nEpoch #26\t Training Loss: 0.99\tTraining Accuracy: 0.58\tValidation Loss: 0.97\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9697 - accuracy: 0.6040\n2023-10-14 19:46:30.771979: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #27\t Training Loss: 0.98\tTraining Accuracy: 0.57\tValidation Loss: 0.97\tValidation Accuracy: 0.60\n8/8 [==============================] - 0s 4ms/step - loss: 0.9632 - accuracy: 0.6080\nEpoch #28\t Training Loss: 0.98\tTraining Accuracy: 0.57\tValidation Loss: 0.96\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9605 - accuracy: 0.6120\nEpoch #29\t Training Loss: 0.98\tTraining Accuracy: 0.58\tValidation Loss: 0.96\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9565 - accuracy: 0.6080\n2023-10-14 19:46:31.805440: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #30\t Training Loss: 0.97\tTraining Accuracy: 0.58\tValidation Loss: 0.96\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 3ms/step - loss: 0.9554 - accuracy: 0.6080\nEpoch #31\t Training Loss: 0.97\tTraining Accuracy: 0.58\tValidation Loss: 0.96\tValidation Accuracy: 0.61\n2023-10-14 19:46:32.286529: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n2023-10-14 19:46:32.414351: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 4ms/step - loss: 0.9502 - accuracy: 0.6080\nEpoch #32\t Training Loss: 0.97\tTraining Accuracy: 0.57\tValidation Loss: 0.95\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9433 - accuracy: 0.6200\nEpoch #33\t Training Loss: 0.96\tTraining Accuracy: 0.57\tValidation Loss: 0.94\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 2ms/step - loss: 0.9395 - accuracy: 0.6280\nEpoch #34\t Training Loss: 0.96\tTraining Accuracy: 0.57\tValidation Loss: 0.94\tValidation Accuracy: 0.63\n8/8 [==============================] - 0s 2ms/step - loss: 0.9352 - accuracy: 0.6280\nEpoch #35\t Training Loss: 0.96\tTraining Accuracy: 0.56\tValidation Loss: 0.94\tValidation Accuracy: 0.63\n8/8 [==============================] - 0s 2ms/step - loss: 0.9359 - accuracy: 0.6240\nEpoch #36\t Training Loss: 0.95\tTraining Accuracy: 0.56\tValidation Loss: 0.94\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 3ms/step - loss: 0.9319 - accuracy: 0.6160\nEpoch #37\t Training Loss: 0.96\tTraining Accuracy: 0.56\tValidation Loss: 0.93\tValidation Accuracy: 0.62\n2023-10-14 19:46:34.223328: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 2ms/step - loss: 0.9328 - accuracy: 0.6160\nEpoch #38\t Training Loss: 0.95\tTraining Accuracy: 0.55\tValidation Loss: 0.93\tValidation Accuracy: 0.62\n2023-10-14 19:46:34.505612: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n2023-10-14 19:46:34.539904: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 4ms/step - loss: 0.9307 - accuracy: 0.6200\nEpoch #39\t Training Loss: 0.94\tTraining Accuracy: 0.56\tValidation Loss: 0.93\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 2ms/step - loss: 0.9295 - accuracy: 0.6120\nEpoch #40\t Training Loss: 0.94\tTraining Accuracy: 0.56\tValidation Loss: 0.93\tValidation Accuracy: 0.61\n2023-10-14 19:46:35.210855: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n8/8 [==============================] - 0s 3ms/step - loss: 0.9284 - accuracy: 0.6120\nEpoch #41\t Training Loss: 0.94\tTraining Accuracy: 0.56\tValidation Loss: 0.93\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 4ms/step - loss: 0.9269 - accuracy: 0.6160\nEpoch #42\t Training Loss: 0.94\tTraining Accuracy: 0.56\tValidation Loss: 0.93\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 3ms/step - loss: 0.9253 - accuracy: 0.6160\n2023-10-14 19:46:35.928499: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #43\t Training Loss: 0.93\tTraining Accuracy: 0.57\tValidation Loss: 0.93\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 2ms/step - loss: 0.9222 - accuracy: 0.6160\nEpoch #44\t Training Loss: 0.93\tTraining Accuracy: 0.57\tValidation Loss: 0.92\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 2ms/step - loss: 0.9205 - accuracy: 0.6200\nEpoch #45\t Training Loss: 0.93\tTraining Accuracy: 0.57\tValidation Loss: 0.92\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 2ms/step - loss: 0.9193 - accuracy: 0.6200\nEpoch #46\t Training Loss: 0.92\tTraining Accuracy: 0.57\tValidation Loss: 0.92\tValidation Accuracy: 0.62\n8/8 [==============================] - 0s 3ms/step - loss: 0.9202 - accuracy: 0.6080\n2023-10-14 19:46:37.221619: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n2023-10-14 19:46:37.248007: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\nEpoch #47\t Training Loss: 0.92\tTraining Accuracy: 0.58\tValidation Loss: 0.92\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9190 - accuracy: 0.6080\nEpoch #48\t Training Loss: 0.91\tTraining Accuracy: 0.59\tValidation Loss: 0.92\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 2ms/step - loss: 0.9176 - accuracy: 0.6120\nEpoch #49\t Training Loss: 0.91\tTraining Accuracy: 0.59\tValidation Loss: 0.92\tValidation Accuracy: 0.61\n8/8 [==============================] - 0s 3ms/step - loss: 0.9160 - accuracy: 0.6160\nEpoch #50\t Training Loss: 0.91\tTraining Accuracy: 0.59\tValidation Loss: 0.92\tValidation Accuracy: 0.62\n2023-10-14 19:46:38.224018: W tensorflow/core/data/root_dataset.cc:266] Optimization loop failed: CANCELLED: Operation was cancelled\n","output_type":"stream"}]},{"cell_type":"markdown","metadata":{"id":"OadIT5MEaOJA","colab_type":"text","cell_id":"0dd37b88234e4a9394ba84efb1ef11f1","deepnote_cell_type":"markdown"},"source":"## Check Performance on Test Set\n\nWe can use `model.predict` to output predicted labels on the test set, or `model.evaluate` to determine test set accuracy (since we have the labels)","block_group":"9a5ea918c9384addbfb53b315f7aaa1f"},{"cell_type":"code","metadata":{"id":"Mr55MHYhafFa","colab":{"height":34,"base_uri":"https://localhost:8080/"},"outputId":"201332aa-9aca-4af8-e01f-d9fc101e8642","colab_type":"code","source_hash":null,"execution_start":1697312822845,"execution_millis":64,"deepnote_to_be_reexecuted":false,"cell_id":"ca956ed7b81c45f1980175c48f5b4171","deepnote_cell_type":"code"},"source":"test_loss, test_accuracy = model.evaluate(X_data_test, y_labels_test)","block_group":"71f78ae6d3f9491290bfdb4c65611c58","execution_count":null,"outputs":[{"name":"stdout","text":"8/8 [==============================] - 0s 2ms/step - loss: 0.9412 - accuracy: 0.6160\n","output_type":"stream"}]},{"cell_type":"markdown","source":"\n \nCreated in Deepnote","metadata":{"created_in_deepnote_cell":true,"deepnote_cell_type":"markdown"}}],"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"NeuralNetworks_Tutorial_part1.ipynb","provenance":[],"collapsed_sections":[],"include_colab_link":true},"deepnote":{},"kernelspec":{"name":"python3","display_name":"Python 3"},"deepnote_notebook_id":"f4b4af0867e74f84ba65d2c283054b1c","deepnote_persisted_session":{"createdAt":"2023-10-14T20:46:41.583Z"},"deepnote_execution_queue":[]}}