Model backlog/Train/178-Tweet-Train-5Fold-roBERTa BiLSTM TD head.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.6"},"colab":{"name":"178-Tweet-Train-5Fold-roBERTa BiLSTM TD head.ipynb","provenance":[],"collapsed_sections":[]},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"kA4Pu2sPr_eG","colab_type":"code","outputId":"e6a83116-ffc2-4d9a-a716-158f749f24b1","executionInfo":{"status":"ok","timestamp":1590280702779,"user_tz":180,"elapsed":4036,"user":{"displayName":"Dimitre Oliveira","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWI4QgYDUyC8ZcW5KHpuBwcY-MZWVqbPWcZSzxIg=s64","userId":"06256612867315483887"}},"colab":{"base_uri":"https://localhost:8080/","height":306}},"source":["!nvidia-smi"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Sun May 24 00:38:20 2020       \n","+-----------------------------------------------------------------------------+\n","| NVIDIA-SMI 440.82       Driver Version: 418.67       CUDA Version: 10.1     |\n","|-------------------------------+----------------------+----------------------+\n","| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n","| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n","|===============================+======================+======================|\n","|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n","| N/A   62C    P0    38W / 250W |      0MiB / 16280MiB |      0%      Default |\n","+-------------------------------+----------------------+----------------------+\n","                                                                               \n","+-----------------------------------------------------------------------------+\n","| Processes:                                                       GPU Memory |\n","|  GPU       PID   Type   Process name                             Usage      |\n","|=============================================================================|\n","|  No running processes found                                                 |\n","+-----------------------------------------------------------------------------+\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"rOXNne0OoLTd","colab_type":"code","colab":{}},"source":["!pip --quiet install transformers\n","!pip --quiet install tokenizers"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"htKChfMHoYRb","colab_type":"code","outputId":"a9243ed3-0921-4967-debe-4d7f70277ebe","executionInfo":{"status":"ok","timestamp":1590280715340,"user_tz":180,"elapsed":9712,"user":{"displayName":"Dimitre Oliveira","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWI4QgYDUyC8ZcW5KHpuBwcY-MZWVqbPWcZSzxIg=s64","userId":"06256612867315483887"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"xtDZkoyzod_S","colab_type":"code","colab":{}},"source":["!cp -r '/content/drive/My Drive/Colab Notebooks/Tweet Sentiment Extraction/Scripts/.' ."],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"38KKgQn7owCr","colab_type":"code","colab":{}},"source":["COLAB_BASE_PATH = '/content/drive/My Drive/Colab Notebooks/Tweet Sentiment Extraction/'\n","MODEL_BASE_PATH = COLAB_BASE_PATH + 'Models/Files/178-roBERTa_base/'\n","\n","import os\n","os.mkdir(MODEL_BASE_PATH)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"MOE8CNSAnvq-","colab_type":"text"},"source":["## Dependencies"]},{"cell_type":"code","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"x48gxo3vnvrA","colab_type":"code","outputId":"2794db91-8e4c-4071-8438-a6a693aaee4e","executionInfo":{"status":"ok","timestamp":1590280735829,"user_tz":180,"elapsed":5572,"user":{"displayName":"Dimitre Oliveira","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWI4QgYDUyC8ZcW5KHpuBwcY-MZWVqbPWcZSzxIg=s64","userId":"06256612867315483887"}},"colab":{"base_uri":"https://localhost:8080/","height":71}},"source":["import json, warnings, shutil\n","from scripts_step_lr_schedulers import *\n","from tweet_utility_scripts import *\n","from tweet_utility_preprocess_roberta_scripts_aux import *\n","from transformers import TFRobertaModel, RobertaConfig\n","from tokenizers import ByteLevelBPETokenizer\n","from tensorflow.keras.models import Model\n","from tensorflow.keras import optimizers, metrics, losses, layers\n","from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n","\n","SEED = 0\n","seed_everything(SEED)\n","warnings.filterwarnings(\"ignore\")"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n","  import pandas.util.testing as tm\n"],"name":"stderr"}]},{"cell_type":"markdown","metadata":{"id":"6HW0GQzPnvrF","colab_type":"text"},"source":["# Load data"]},{"cell_type":"code","metadata":{"id":"jS858zlQNNCs","colab_type":"code","colab":{}},"source":["# Unzip files\n","!tar -xf '/content/drive/My Drive/Colab Notebooks/Tweet Sentiment Extraction/Data/complete_96/fold_1.tar.gz'\n","!tar -xf '/content/drive/My Drive/Colab Notebooks/Tweet Sentiment Extraction/Data/complete_96/fold_2.tar.gz'\n","!tar -xf '/content/drive/My Drive/Colab Notebooks/Tweet Sentiment Extraction/Data/complete_96/fold_3.tar.gz'\n","!tar -xf '/content/drive/My Drive/Colab Notebooks/Tweet Sentiment Extraction/Data/complete_96/fold_4.tar.gz'\n","!tar -xf '/content/drive/My Drive/Colab Notebooks/Tweet Sentiment Extraction/Data/complete_96/fold_5.tar.gz'"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","id":"hVhA98wenvrG","colab_type":"code","outputId":"d4b5b26b-01a5-45c8-dc7c-751698c52273","executionInfo":{"status":"ok","timestamp":1590280747310,"user_tz":180,"elapsed":14011,"user":{"displayName":"Dimitre Oliveira","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWI4QgYDUyC8ZcW5KHpuBwcY-MZWVqbPWcZSzxIg=s64","userId":"06256612867315483887"}},"colab":{"base_uri":"https://localhost:8080/","height":513}},"source":["database_base_path = COLAB_BASE_PATH + 'Data/complete_96/'\n","k_fold = pd.read_csv(database_base_path + '5-fold.csv')\n","display(k_fold.head())"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>textID</th>\n","      <th>text</th>\n","      <th>selected_text</th>\n","      <th>sentiment</th>\n","      <th>jaccard</th>\n","      <th>text_len</th>\n","      <th>text_wordCnt</th>\n","      <th>text_tokenCnt</th>\n","      <th>selected_text_len</th>\n","      <th>selected_text_wordCnt</th>\n","      <th>selected_text_tokenCnt</th>\n","      <th>fold_1</th>\n","      <th>fold_2</th>\n","      <th>fold_3</th>\n","      <th>fold_4</th>\n","      <th>fold_5</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>cb774db0d1</td>\n","      <td>i`d have responded, if i were going</td>\n","      <td>i`d have responded, if i were going</td>\n","      <td>neutral</td>\n","      <td>1.000000</td>\n","      <td>35</td>\n","      <td>7</td>\n","      <td>10</td>\n","      <td>35</td>\n","      <td>7</td>\n","      <td>10</td>\n","      <td>train</td>\n","      <td>train</td>\n","      <td>train</td>\n","      <td>validation</td>\n","      <td>train</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>549e992a42</td>\n","      <td>sooo sad i will miss you here in san diego!!!</td>\n","      <td>sooo sad</td>\n","      <td>negative</td>\n","      <td>0.200000</td>\n","      <td>45</td>\n","      <td>10</td>\n","      <td>13</td>\n","      <td>8</td>\n","      <td>2</td>\n","      <td>3</td>\n","      <td>validation</td>\n","      <td>train</td>\n","      <td>train</td>\n","      <td>train</td>\n","      <td>train</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>088c60f138</td>\n","      <td>my boss is bullying me...</td>\n","      <td>bullying me</td>\n","      <td>negative</td>\n","      <td>0.166667</td>\n","      <td>25</td>\n","      <td>5</td>\n","      <td>6</td>\n","      <td>11</td>\n","      <td>2</td>\n","      <td>2</td>\n","      <td>train</td>\n","      <td>validation</td>\n","      <td>train</td>\n","      <td>train</td>\n","      <td>train</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>9642c003ef</td>\n","      <td>what interview! leave me alone</td>\n","      <td>leave me alone</td>\n","      <td>negative</td>\n","      <td>0.600000</td>\n","      <td>30</td>\n","      <td>5</td>\n","      <td>6</td>\n","      <td>14</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td>train</td>\n","      <td>validation</td>\n","      <td>train</td>\n","      <td>train</td>\n","      <td>train</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>358bd9e861</td>\n","      <td>sons of ****, why couldn`t they put them on th...</td>\n","      <td>sons of ****,</td>\n","      <td>negative</td>\n","      <td>0.214286</td>\n","      <td>74</td>\n","      <td>14</td>\n","      <td>17</td>\n","      <td>13</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>train</td>\n","      <td>train</td>\n","      <td>validation</td>\n","      <td>train</td>\n","      <td>train</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["       textID  ... fold_5\n","0  cb774db0d1  ...  train\n","1  549e992a42  ...  train\n","2  088c60f138  ...  train\n","3  9642c003ef  ...  train\n","4  358bd9e861  ...  train\n","\n","[5 rows x 16 columns]"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"b0bUTMPynvrM","colab_type":"text"},"source":["# Model parameters"]},{"cell_type":"code","metadata":{"id":"Paexn4ywnvrM","colab_type":"code","colab":{}},"source":["vocab_path = COLAB_BASE_PATH + 'qa-transformers/roberta/roberta-base-vocab.json'\n","merges_path = COLAB_BASE_PATH + 'qa-transformers/roberta/roberta-base-merges.txt'\n","base_path = COLAB_BASE_PATH + 'qa-transformers/roberta/'\n","\n","config = {\n","  \"MAX_LEN\": 96,\n","  \"BATCH_SIZE\": 32,\n","  \"EPOCHS\": 4,\n","  \"LEARNING_RATE\": 3e-5,\n","  \"ES_PATIENCE\": 4,\n","  \"question_size\": 4,\n","  \"N_FOLDS\": 5,\n","  \"base_model_path\": base_path + 'roberta-base-tf_model.h5',\n","  \"config_path\": base_path + 'roberta-base-config.json'\n","}\n","\n","with open(MODEL_BASE_PATH + 'config.json', 'w') as json_file:\n","    json.dump(json.loads(json.dumps(config)), json_file)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"D_8F5nxDnvrW","colab_type":"text"},"source":["# Tokenizer"]},{"cell_type":"code","metadata":{"_kg_hide-output":true,"id":"6qeQlxrDnvrX","colab_type":"code","colab":{}},"source":["tokenizer = ByteLevelBPETokenizer(vocab_file=vocab_path, merges_file=merges_path, \n","                                  lowercase=True, add_prefix_space=True)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"S3FN1pb1nvrb","colab_type":"text"},"source":["## Learning rate schedule"]},{"cell_type":"code","metadata":{"_kg_hide-input":true,"id":"XUYfnwPRnvrc","colab_type":"code","outputId":"34083a80-d46a-4f44-a755-1737fd37f98b","executionInfo":{"status":"ok","timestamp":1590280748779,"user_tz":180,"elapsed":4572,"user":{"displayName":"Dimitre Oliveira","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWI4QgYDUyC8ZcW5KHpuBwcY-MZWVqbPWcZSzxIg=s64","userId":"06256612867315483887"}},"colab":{"base_uri":"https://localhost:8080/","height":403}},"source":["lr_min = 1e-6\n","lr_max = config['LEARNING_RATE']\n","train_size = len(k_fold[k_fold['fold_1'] == 'train'])\n","step_size = train_size // config['BATCH_SIZE']\n","total_steps = config['EPOCHS'] * step_size\n","decay = .9985\n","\n","rng = [i for i in range(0, total_steps, config['BATCH_SIZE'])]\n","y = [exponential_schedule_with_warmup(tf.cast(x, tf.float32), warmup_steps=1, lr_start=lr_max, lr_max=lr_max, lr_min=lr_min, decay=decay) for x in rng]\n","\n","sns.set(style=\"whitegrid\")\n","fig, ax = plt.subplots(figsize=(20, 6))\n","plt.plot(rng, y)\n","print(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Learning rate schedule: 3e-05 to 3e-05 to 1e-06\n"],"name":"stdout"},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABIUAAAFzCAYAAACgv7kOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3SUZcL+8WtmMum99wAhIUASSgIB6aCCqxgs7LoWdi24CoruWmAV6+LuCyruKiqIrqvvuvZVpEiTHpqEGkoICAmpEEJCDyGZ3x/RvPgDTZCEZybz/ZzjOZNnnjznmuN9ZLi8i8lms9kEAAAAAAAAp2I2OgAAAAAAAAAuP0ohAAAAAAAAJ0QpBAAAAAAA4IQohQAAAAAAAJwQpRAAAAAAAIATohQCAAAAAABwQi5GBzjX5MmTtWDBAhUVFWn27NlKTExslufecccdKi4ulre3tyRp1KhRuummm5rl2QAAAAAAAI7IrkqhIUOGaNSoUbrtttua/dkTJ07UoEGDmv25AAAAAAAAjsiuSqH09PQLXt+yZYteeuklnThxQpI0btw4DRw48DImAwAAAAAAaF3sqhS6kKNHj+qZZ57RW2+9pdDQUB08eFA333yz5syZI19f3yY/Z8qUKZo6dao6dOigxx57TGFhYS2YGgAAAAAAwL7ZfSm0adMmFRYWavTo0Q3XTCaT8vPzlZKSooyMjAv+XmhoqGbPni2pvhCKiIhQbW2tZsyYoYcfflgffvjhZckPAAAAAABgj+y+FLLZbOrQoYM++OCDC76/bt26Rp8REREhSbJYLBo1apSmTZumuro6mc0cvgYAAAAAAJyT3bci3bp1U35+vtauXdtwbevWrbLZbE36/bNnz6q8vLzh57lz5yoxMZFCCAAAAAAAODWTrantymUwadIkLVy4UOXl5QoICJC/v7/mzp2rrVu36sUXX1RVVZVqamoUExOj6dOnN6nYOXnypG6//XbV1NRIql9W9uSTT6pdu3Yt/XEAAAAAAADsll2VQgAAAAAAALg8WEMFAAAAAADghOxio+m6ujqdOHFCVqtVJpPJ6DgAAAAAAAAOz2azqaamRl5eXhfcgscuSqETJ05o9+7dRscAAAAAAABodRITE+Xj43PedbsohaxWq6T6kK6urganuXQ5OTlKTk42OgbQohjncBaMdTgDxjmcBWMdzoBxjnOdOXNGu3fvbuhd/n92UQr9sGTM1dVVbm5uBqdpHq3lcwA/h3EOZ8FYhzNgnMNZMNbhDBjn+P/91FY9bDQNAAAAAADghCiFAAAAAAAAnBClEAAAAAAAgBOiFAIAAAAAAHBClEIAAAAAAABOiFIIAAAAAADACVEKAQAAAAAAOCFKIQAAAAAAACfk0pSbxowZo8LCQpnNZnl6euqpp55Sx44df3RPbW2tJk2apJUrV8pkMunee+/VyJEjWyQ0AAAAAAAALk2TSqHJkyfLx8dHkrR48WI98cQT+uKLL350z+zZs1VQUKCFCxeqsrJSI0aMUO/evRUdHd38qQEAAAAAAHBJmrR87IdCSJKOHz8uk8l03j3z5s3TyJEjZTabFRgYqCuvvFLz589vvqQAAAAAAABoNk2aKSRJTz75pLKysmSz2fT222+f935JSYkiIyMbfo6IiFBpaWnzpHQg/5qzXaYzJ5WWZnQSAAAAAACAn9bkUuiFF16QJH355ZeaMmWKZs6c2exhcnJymv2Zl9uuvYe148ApebhlKT7c3eg4QIvKzs42OgJwWTDW4QwY53AWjHU4A8Y5mqrJpdAPRowYoaefflpHjhxRQEBAw/WIiAgVFxcrNTVV0vkzh5oiOTlZbm5uFxvJrnTsXKMHX1yk/66u1JQH+yk23NfoSECLyM7OVhpT4uAEGOtwBoxzOAvGOpwB4xznqq6u/tkJOI3uKXTixAmVlJQ0/LxkyRL5+fnJ39//R/cNGzZMn376qerq6lRRUaHFixdr6NChlxDdMXm6W3XrgCC5Wi167p11OnLstNGRAAAAAAAAztPoTKFTp07poYce0qlTp2Q2m+Xn56fp06fLZDJp9OjRGjdunFJSUpSZmaktW7bo6quvliSNHTtWMTExLf4B7JG/l4ueujtDE17P0gv/XK8XxvSRm9VidCwAAAAAAIAGjZZCwcHB+uSTTy743rn7ClksFj333HPNl8zBJcQE6NHb0vS399brlQ836vHb02U2n39qGwAAAAAAgBGadCQ9fpneKRG687rOytpSrP/9eqfRcQAAAAAAABpc9EbTuDgjBsSruPyEPluSp8hgL12VEWd0JAAAAAAAAEqhlmYymfSHG1JUdviEXv9si0IDPNUlMcToWAAAAAAAwMmxfOwycLGYNX5UD0WFeutv763XgbJjRkcCAAAAAABOjlLoMvHysOqZu3vJarXoubfXqvJYtdGRAAAAAACAE6MUuoxCAz311F0ZOnKsWi+8u05namqNjgQAAAAAAJwUpdBllhgboD/d2l278o/o7x9tUl2dzehIAAAAAADACVEKGaBPaqR+f20nrdxcpA8W7DI6DgAAAAAAcEKcPmaQGwe1V3H5CX2yeLcigrx0Zc9YoyMBAAAAAAAnQilkEJPJpPtvStXBIyc17dPNCvZ3V9fEUKNjAQAAAAAAJ8HyMQO5WMyaMKqHYsJ89Lf3vlV+yVGjIwEAAAAAACdBKWQwLw+rnr67l9xdLXr27bU6XHXK6EgAAAAAAMAJUArZgZAADz19dy+dOHVGz7+zTqeqzxodCQAAAAAAtHKUQnYiPtpfj9/RQ/tLjmrK/25QbW2d0ZEAAAAAAEArRilkR9I7hum+G1O1YWeZZnyxTTabzehIAAAAAACgleL0MTtzTe82Kjt8Qp8v3aPwIE/dOCjB6EgAAAAAAKAVohSyQ6N+1UkHj5zSu3N2KCTAU/26RhkdCQAAAAAAtDKUQnbIbDbp4Vu6qbzylF75cKOC/NzVqW2Q0bEAAAAAAEArwp5CdsrVatHEuzIU4u+hSf9cr+JDx42OBAAAAAAAWhFKITvm6+WqZ0b3kskkPTtzraqOVxsdCQAAAAAAtBKUQnYuMthbT92VofKqU5r0z3Wqrqk1OhIAAAAAAGgFKIUcQFKbQD1ya5pyC47olf9sVF0dR9UDAAAAAIBLQynkIPp0idSd13VW1tZivTM7RzYbxRAAAAAAAPjlOH3MgYwYEK9Dlaf01Yrv5O/tppFDEo2OBAAAAAAAHBSlkAMxmUy65/pkVR2v1vvzdsrP201XZ8QZHQsAAAAAADggSiEHYzab9PAt3XX8ZI1e/3SzfDxd1TslwuhYAAAAAADAwbCnkAOyupg14Xc91D7GXy/+e4Ny9pYbHQkAAAAAADgYSiEH5eHmoqfv7qXQAE9N+uc67SuuMjoSAAAAAABwIJRCDszP203P/6G33N1c9Mxba1R6+ITRkQAAAAAAgIOgFHJwoQGeev7e3qo5W6en31qjymPVRkcCAAAAAAAOgFKoFYgN99Uz9/TS4arTevbtNTp5usboSAAAAAAAwM5RCrUSSW0C9eff9dC+4qN64d31qjlba3QkAAAAAABgxyiFWpH0jmF66DfdtHVPuV7+YKNq62xGRwIAAAAAAHaKUqiVGZweo7uv76ysrcWa8d+tstkohgAAAAAAwPlcjA6A5jdiQHtVHqvW50v3yN/HTbcOTTI6EgAAAAAAsDOUQq3U767tpKrjZ/Thwlx5e1p1fb94oyMBAAAAAAA7QinUSplMJj0wsouOnzqjmV/myN3VRVdnxBkdCwAAAAAA2An2FGrFLBazHr8jXd2TQjXt081atrHQ6EgAAAAAAMBOUAq1clYXi/78ux7q3C5Ir3y4UWu2FRsdCQAAAAAA2AFKISfg7uqip+7KUEKMv6b87wZl7yozOhIAAAAAADAYpZCT8HS36tnRvRUb5qu/vrte2/aUGx0JAAAAAAAYiFLIiXh7WPX8H3orLMhLz7+zVrvyK4yOBAAAAAAADNJoKXTkyBGNHj1aQ4cO1fDhw/XAAw+oouL8MmHChAnq37+/MjMzlZmZqTfffLNFAuPS+Hm7adJ9VyjAx13PvrVGewsrjY4EAAAAAAAM0GgpZDKZdM8992jBggWaPXu2YmJi9NJLL13w3nvvvVezZs3SrFmzdP/99zd7WDSPQF93TbrvCnm4W/X0W2tUUHrU6EgAAAAAAOAya7QU8vf3V0ZGRsPPXbt2VXExJ1g5utBAT71w/xWymE16asZqFZcfNzoSAAAAAAC4jC5qT6G6ujp9+OGHGjx48AXff/fddzV8+HCNGTNGe/fubZaAaDmRwd76y31XqOasTROnr9bBipNGRwIAAAAAAJeJyWaz2Zp683PPPaeysjJNmzZNZvOP+6SysjKFhITIbDbryy+/1D/+8Q8tXrxYFoul0edWV1crJyfn4tOjWZRUnNG/vjkkLzeL7rwqRD4ejf87AwAAAAAAjiE5OVlubm7nXW9yKTR58mTl5uZq+vTpcnV1bfT+jIwM/fe//1VUVFSj9/5QCv1USEeTnZ2ttLQ0o2NclF35FXp6xmoF+3vqb2P6yM/b8f89oGU54jgHfgnGOpwB4xzOgrEOZ8A4x7ka61uatHxs6tSpysnJ0euvv/6ThVBZWVnD65UrV8psNissLOwXxsbllhQXqKfu7qWyipN68s0sVR6rNjoSAAAAAABoQY2WQnl5eZoxY4YOHjyoW265RZmZmRo7dqwkKTMzs6EMGj9+vIYPH67rr79eb775pt588025uLi0bHo0q5T4YD19d4ZKDp/Uk9MphgAAAAAAaM0abW0SEhKUm5t7wfdmzZrV8Ppf//pXs4WCcbokhOjpuzP0/Dvr9OT0LL1wXx/5+7CUDAAAAACA1uaiTh+Dc+iSEKJn7slQ6eGTeuLNLB05dtroSAAAAAAAoJlRCuGCUtuH6Nl7fthjaDXFEAAAAAAArQylEH5SSvtgPXtPLx08QjEEAAAAAEBrQymEn5XSPljPNBRDWTpylGIIAAAAAIDWgFIIjUqJ/6EYOlW/xxDFEAAAAAAADo9SCE2SEl+/lKy8sr4YqqAYAgAAAADAoVEKocmS44P17Oje9cXQGxRDAAAAAAA4MkohXJTO7YL07OjeOlxFMQQAAAAAgCOjFMJF+6EYqjh6Sk+8sUqHq04ZHQkAAAAAAFwkSiH8Iv9XDFVr/LRVKj18wuhIAAAAAADgIlAK4Rfr1DZIk+67QidP12j8tFU6UHbM6EgAAAAAAKCJKIVwSRJjA/TXMX1VZ7NpwuurtLew0uhIAAAAAACgCSiFcMnaRPhq8ti+crVa9OSbWdq1v8LoSAAAAAAAoBGUQmgWkSHemvxAX/l6u+mpGau1Je+Q0ZEAAAAAAMDPoBRCswkN8NT/jO2rsEBPPff2Wq3fUWp0JAAAAAAA8BMohdCsAn3d9dcxfRUX4au/vrteKzcVGR0JAAAAAABcAKUQmp2vl6teuO8KJbUJ1EsfbNCidflGRwIAAAAAAP8fSiG0CE93q54d3UtdEkL06ieb9dXKvUZHAgAAAAAA56AUQotxd3XRU3dnqHdKhGZ+maOPF+fKZrMZHQsAAAAAAIhSCC3M6mLR+DvSNTAtWv/+epfem7uDYggAAAAAADvgYnQAtH4Wi1l/vKW7PFxd9PnSPTp2skZjbkqVxUInCQAAAACAUSiFcFmYzSbdf1OqfLxc9cni3ao6Xq3H7kiXm9VidDQAAAAAAJwSUzVw2ZhMJt1xTUf94YYUrd9RqqdnrNbxk2eMjgUAAAAAgFOiFMJld13fdnrs9nTtLqjU+NdXqbzylNGRAAAAAABwOpRCMES/rlF6dnQvHTpySo+9tlIHyo4ZHQkAAAAAAKdCKQTDdEkI0d/G9NHZ2jqNn7ZSu/ZXGB0JAAAAAACnQSkEQ8VH++vFB/vJ28NVT05frW93lBodCQAAAAAAp0ApBMOFB3lp8oN9FRPmrUnvrtfi9QVGRwIAAAAAoNWjFIJdCPBx11/v76OU+CD94+NN+mxJnmw2m9GxAAAAAABotSiFYDc83a165p5e6tc1Su/N3aG3Z+Woro5iCAAAAACAluBidADgXFYXix69LU3+Pm76auV3qjxerYdv6Sari8XoaAAAAAAAtCqUQrA7ZrNJozOTFeDjpvfn7dThqtN68s6e8vF0NToaAAAAAACtBsvHYJdMJpNGDknUI7elKTf/iB57dYVKyk8YHQsAAAAAgFaDUgh2bWD3aE267wodPXFGj766Qjv3VRgdCQAAAACAVoFSCHavc7sgvTiuv7w8rHpyepZWbi4yOhIAAAAAAA6PUggOISrEWy8+2E/to/015X83cGQ9AAAAAACXiFIIDsPP202T7rtC/b8/sv71z7bobG2d0bEAAAAAAHBInD4Gh+JqteiR29IUFuSpT7/J08GKkxo/qoe8PKxGRwMAAAAAwKEwUwgOx2w2adSvOunBX3fV1j3lGj9tpQ4eOWl0LAAAAAAAHAqlEBzW1RlxenZ0Lx2qPKVH/7FCew5UGh0JAAAAAACHQSkEh9Y1MVRTHuwnFxezJryxSutySoyOBAAAAACAQ6AUgsOLC/fVy+P6KybMRy/8a70+52QyAAAAAAAa1WgpdOTIEY0ePVpDhw7V8OHD9cADD6iiouK8+06dOqWHH35YV111lYYNG6alS5e2SGDgQgJ83fW3+/voitRI/WvuDr3y4Uadqak1OhYAAAAAAHar0VLIZDLpnnvu0YIFCzR79mzFxMTopZdeOu++d955R97e3lq0aJGmT5+uiRMn6sSJEy0SGrgQdzcXjb8jXbcNS9LS7EL9+Y1VOlx1yuhYAAAAAADYpUZLIX9/f2VkZDT83LVrVxUXF59339dff63f/OY3kqQ2bdooOTlZK1asaMaoQONMJpNuuaqDnvh9DxWUHtOf/r5CuwuOGB0LAAAAAAC7c1F7CtXV1enDDz/U4MGDz3uvuLhYUVFRDT9HRESotLT00hMCv0DvlMj6DagtJv359VVatrHQ6EgAAAAAANgVl4u5+S9/+Ys8PT11++23t0iYnJycFnmuEbKzs42OAEm/G+SvT1Yd1ssfZGvdpt0a3MVXZpPJ6FitBuMczoKxDmfAOIezYKzDGTDO0VRNLoUmT56s/Px8TZ8+XWbz+ROMIiMjVVRUpMDAQElSSUnJj5adNUVycrLc3Nwu6nfsUXZ2ttLS0oyOge/1zqjTjC+2asHafJ2Rlx65rbs83a1Gx3J4jHM4C8Y6nAHjHM6CsQ5nwDjHuaqrq392Ak6Tlo9NnTpVOTk5ev311+Xq6nrBe4YNG6aPP/5YkrR//35t27ZN/fr1+wWRgeZldTFr7M1ddN8NKdqwq0yPvbZSJeVsgg4AAAAAcG6NlkJ5eXmaMWOGDh48qFtuuUWZmZkaO3asJCkzM1NlZWWSpLvvvltHjx7VVVddpT/84Q96/vnn5e3t3bLpgSYymUy6tm87PT+6tyqqTuuRfyzXlrxDRscCAAAAAMAwjS4fS0hIUG5u7gXfmzVrVsNrT09Pvfrqq82XDGgBXRJDNPXhAfrLP9fp6bfW6N7MZP2qT1uZ2GcIAAAAAOBkLur0MaA1iAj20kvj+iktKVTTv9imv3+0SdU1tUbHAgAAAADgsqIUglPydLdq4p0Z+u3VHbRkwwE9/upKlR5mnyEAAAAAgPOgFILTMptNunVokp6+O0NlR07qj68s14adZUbHAgAAAADgsqAUgtPr0Slcrzw8QCEBHnr+nbX6cGGu6upsRscCAAAAAKBFUQoBqt9naMqD/TSge7T+s2CX/vLPdTp+8ozRsQAAAAAAaDGUQsD33F1d9Kffdtd9N6RoU+5B/envK7SvuMroWAAAAAAAtAhKIeAcJpNJ1/Ztp7+N6avqmlo9+upKLc0+YHQsAAAAAACaHaUQcAEd2wbq738coIQYf039z0bN+O9W1ZytMzoWAAAAAADNhlII+AkBvu6adN8VGjEgXnOy9unJN7N0uOqU0bEAAAAAAGgWlELAz3CxmHX39cl6/PZ07Suu0sNTl2vz7oNGxwIAAAAA4JJRCgFN0K9blF56qL98vFz19Ftr9O+vd6q2luVkAAAAAADHRSkENFFcuK+mPtRfg9Nj9PHi3Xpy+mqWkwEAAAAAHBalEHAR3N1c9PAt3fXH33bTnsJKPTR1mTbuYjkZAAAAAMDxUAoBv8Dg9Fi98vAA+Xu76ZmZa/T+vB0sJwMAAAAAOBRKIeAXignz0UsP9dfVGXH69Js8PfFmlsorWU4GAAAAAHAMlELAJXB3ddGDv+6qR25L077iKo17eZk27CwzOhYAAAAAAI2iFAKawcDu0XrljwMV7O+u595eq3dnb9dZlpMBAAAAAOwYpRDQTKJCvPXiuP66pncb/XfZHv359VU6eOSk0bEAAAAAALggSiGgGblZLRpzcxc9fnu68kuP6aGXlylrS7HRsQAAAAAAOA+lENAC+nWL0t//NECRIV76n/e/1T8+2qSTp2uMjgUAAAAAQANKIaCFRAZ7a/ID/fSbKxO1ZEOBHp66XLn5FUbHAgAAAABAEqUQ0KJcLGbdfk1H/XVMX52tq9Pj01bp40W5qq2zGR0NAAAAAODkKIWAy6BzuyC9+sgg9e0SqX/P36Un3lilgxVsQg0AAAAAMA6lEHCZeHtY9djt6frTrd21r/ioxr28VMs3FhodCwAAAADgpCiFgMtsUFqMXn1koGLDffXSB9l6+T/ZOnGKTagBAAAAAJcXpRBggPAgL/1tTB/dOjRJKzYVadzUZdq5j02oAQAAAACXD6UQYBCLxazfXt1Bk8f2lUnShNdX6t/zd6rmbJ3R0QAAAAAAToBSCDBYUptAvfrIQA1Mi9HHi3br0VdXaH/JUaNjAQAAAABaOUohwA54ulv1x9921xO/76mKqtP64yvL9dmSPI6uBwAAAAC0GEohwI70TonQtMcGqWfnML03d4fGT1upokPHjY4FAAAAAGiFKIUAO+Pn7aYJo3ro0dvSVHTwuMa9vExfrdirOmYNAQAAAACaEaUQYIdMJpMGdI/W648PVmr7YM2claOJ01er9PAJo6MBAAAAAFoJSiHAjgX6uuvpuzP00G+6ak9hpca9vFTz1+yXzcasIQAAAADApaEUAuycyWTSlT3jNO2xQUqMDdDrn23RszPXqrzylNHRAAAAAAAOjFIIcBChAZ56/t4rdN+Nqdq+77AeeHGJvvm2gFlDAAAAAIBfhFIIcCBms0nX9mmrVx8ZqLgIX/39o016duZaHaw4aXQ0AAAAAICDoRQCHFBksLf+Nqav/nBDinbuP6yxLy7R7JXfqZYTygAAAAAATUQpBDgos9mk6/q207RHB6tTuyC99eU2TZi2UgWlR42OBgAAAABwAJRCgIMLDfTUs/f00p9u7a6iQ8f10NTl+mhRrmrO1hkdDQAAAABgxyiFgFbAZDJpUFqM3nh8iK5IidAH83fpT39frt0FR4yOBgAAAACwU5RCQCvi7+Omx+5I11N3ZejYyTN67NUVeuerHJ0+c9boaAAAAAAAO+NidAAAza9n53B1bhekf83doS+X79XanBI9MLKruiSEGB0NAAAAAGAnmjRTaPLkyRo8eLA6dOig3bt3X/Ce1157Tb1791ZmZqYyMzP13HPPNWtQABfHy8OqsTd30V/H9JHJZNLE6av1j482qep4tdHRAAAAAAB2oEkzhYYMGaJRo0bptttu+9n7RowYofHjxzdLMADNIyU+WK89OkgfLtilL5fv1brtJbrzus4a0iNWZrPJ6HgAAAAAAIM0aaZQenq6IiIiWjoLgBbiZrXo99d11j/+NFAxYT569ZPN+vMbq5RfwvH1AAAAAOCsmnWj6blz52r48OG66667tGnTpuZ8NIBmEBfhq7+N6atxv+6qA2XH9dDUZfrXnO06Xc1G1AAAAADgbEw2m83W1JsHDx6s6dOnKzEx8bz3Dh06JH9/f1mtVmVlZenRRx/VvHnzFBAQ0Ohzq6urlZOTc3HJAVySE6drtXhzlTZ9d1J+nhZdk+6vpGgPo2MBAAAAAJpZcnKy3NzczrvebKePhYT836lGffr0UUREhPLy8tSzZ88mP+OnQjqa7OxspaWlGR0DaFT/PtL27w7rjc+36KMVh5XROVz33pCi0ADPRn+XcQ5nwViHM2Ccw1kw1uEMGOc4V2OTcJpt+VhZWVnD6507d6qoqEht27ZtrscDaCGd2wXpH38aqN9f20mb8w5pzJQl+u/SPJ2trTM6GgAAAACgBTVpptCkSZO0cOFClZeX684775S/v7/mzp2r0aNHa9y4cUpJSdHUqVO1fft2mc1mWa1WTZky5UezhwDYLxeLWTcNTlC/rlF668ttenfODi3ZcED339RFndsFGR0PAAAAANACmlQKTZw4URMnTjzv+syZMxteT548uflSATBEaKCnJt6VobU5JZrxxTZNeH2VBnSL1p3DOynIj/2GAAAAAKA1abY9hQC0Hr2SI9Q1IUSfLcnTf5ft0brtJfr1lYkaMSBeVheL0fEAAAAAAM2gWY+kB9B6uLu56PZrOuqNxwera2KI3p+3U2NfXKpvd5QaHQ0AAAAA0AyYKQTgZ4UHeenJOzO0Mfeg3vpim55/Z53SO4apV7zRyQAAAAAAl4JSCECTdO8QqtceHaS5Wd/pPwtytTH3rEpObNevr0yUp7vV6HgAAAAAgIvE8jEATWZ1MWvEgPaaMWGIUtt46vOle3T/5G+0LPuAbDab0fEAAAAAABeBUgjARQvwddeIXoF6cVw/Bfp56OX/bNT4aau050Cl0dEAAAAAAE1EKQTgF0uKC9TL4/pr3K+7qrj8uP749+Wa+p9sHTpyyuhoAAAAAIBGsKcQgEtiNpt0VUacrkiN1GdL8jRrxV5lbSnWDQPb68ZB7dlvCAAAAADsFDOFADQLLw+rfndtJ705foh6pUTo48W79Yf/+UYL1u5XbR37DQEAAACAvaEUAtCswgI99djt6XppXD9FBHlp2qdb9NDLS7Ux96DR0QAAAAAA56AUAtAiOsQFavIDfTVhVA9V19TqmbfW6JmZa5RfetToaAAAAAAAUQoBaEEmk0l9ukTqjaBZxrcAACAASURBVMcH667hnZWbf0TjXlqqaZ9u1pFjp42OBwAAAABOjY2mAbQ4q4tFNwxsryE9YvXRolzNy9qnFZsKdeOgBGX2j5eHG/8pAgAAAIDLjZlCAC4bXy9X3TsiRa8/PlhdE0P1wfxduvevizVn1XeqOVtndDwAAAAAcCqUQgAuu6gQbz3x+556cVw/RYd5a8YX2zRmyjdatrFQdZxUBgAAAACXBaUQAMMkxQXqr/f30bOje8nDzUUvf5Cth19Zpg07y2SzUQ4BAAAAQEtiIw8AhjKZTEpLClO3xFCt3Fykf8/fqefeXqvO7YL0+2s7KalNoNERAQAAAKBVohQCYBfMZpMGdI/WFamRWrguXx8tytVjr61URudwjfpVR8WG+xodEQAAAABaFZaPAbArVhezru3TVm/9+UrdPixJ2/aW68GXlurvH21UWcVJo+MBAAAAQKvBTCEAdsnDzUW/uaqDhvVuo8+W5Glu1j4t31ioq3rGaeSQRIUEeBgdEQAAAAAcGqUQALvm5+2mu69PVmb/eH3yzW4tWpevResLNKxXnG4ekqAgP8ohAAAAAPglKIUAOIRgfw+NuamLbh6UoE++2a2v1+zXwnX5GnZFG908OEEBPu5GRwQAAAAAh0IpBMChhAZ66oGRXXXz4AR9vGi35qzap/lr8nVdn7a6cVB7+Xm7GR0RAAAAABwCpRAAhxQe5KWHbummkUMS9OGiXH25fI/mrd6n4f3aacSA9vL1cjU6IgAAAADYNU4fA+DQIkO89citaZr22GD17BSuz5bk6Z4XFunf83fq+MkzRscDAAAAALtFKQSgVYgJ89Fjd6TrtUcGqXuHUH28aLfumrRI783docpj1UbHAwAAAAC7w/IxAK1KXISvJvyuh/YVV+nTb/L0+dI8fbXyOw3rHacbB7bntDIAAAAA+B6lEIBWqW2knx6/I123Du2gT7/J05xV+zQva7+u6hmrmwYnKCzQ0+iIAAAAAGAoSiEArVp0qI/++Nvu+u3VHfT50j1atL5AC9fla2BatEYOSVRUiLfREQEAAADAEJRCAJxCeJCXxt7cRb+5MlFfLNuj+Wv2a+mGA+rbJUojr0xUmwhfoyMCAAAAwGVFKQTAqQT7e2j0iBTdPCRBs5bv1bzV+7Ric5F6JYfr5sEJ6hAXaHREAAAAALgsKIUAOKUAH3f9/rrOumlwgmav/E5frfxOa3NKlRwfpJsGJSgtKVQmk8nomAAAAADQYiiFADg1H09X3To0SSMGxGvhugLNWr5Hz729VnHhPrpxUIL6d4uSi8VsdEwAAAAAaHb8TQcAJHm6WzViQLxmPnmV/vjb7pKkVz7cqNF/Xawvl+/VydM1BicEAAAAgObFTCEAOIeLxazB6TEalBat7F0H9fnSPL3zVY4+WpSra/u01XV92yrAx93omAAAAABwySiFAOACTCaT0juGKb1jmHLzK/T50j369Jvd+mLZHg3pEasbBsQrkuPsAQAAADgwSiEAaESHuEA98fueKjp0XF8s26PF6wu0YO1+ZXQO1/X945XcLohNqQEAAAA4HEohAGiiqBBvPTCyq24bmqQ5Wfv09er9WptTqnZRfsrs3079ukbL6sJWbQAAAAAcA397AYCLFODrrjuu6ah/PnWVHhjZRTVn6/TKh5t096SF+nhRrqqOVxsdEQAAAAAaxUwhAPiF3F1dNLRXG12dEadNuYc0a+Ve/Xv+Ln2yeLcGpsXo+n7tFBfha3RMAAAAALggSiEAuEQmk0ndk0LVPSlUBaVHNXvVPi3ZcEAL1+Wra2KIMvvHq3uHUJnN7DsEAAAAwH5QCgFAM4oN99XYm7vojms6av6a/Zqb9Z2ee3utokK8dV3fthqcHiNPd6vRMQEAAACAUggAWoKvl6t+fWWibhjYXllbivTVyu8044tten/eDg1Ki9G1fdoqNpylZQAAAACM02gpNHnyZC1YsEBFRUWaPXu2EhMTz7untrZWkyZN0sqVK2UymXTvvfdq5MiRLRIYAByJ1cWsgWkxGpgWo90FRzQ3a58WrS/QvNX7ldo+WL/q01a9OofLYmHffwAAAACXV6Ol0JAhQzRq1CjddtttP3nP7NmzVVBQoIULF6qyslIjRoxQ7969FR0d3axhAcCRJcYGKDE2QHcN76xF6wv09ep9+p/3vlWQn7uu6d1GV/eKU4CPu9ExAQAAADiJRkuh9PT0Rh8yb948jRw5UmazWYGBgbryyis1f/583XPPPc0SEgBaEz9vN908OEE3DGyvDTtKNTdrn/49f5c+WpSrPqlRurZPWyW1CZDJxMbUAAAAAFpOs+wpVFJSosjIyIafIyIiVFpaetHPycnJaY44diE7O9voCECLY5xfOhdJmelu6pMYpm/zTmhtTpGWbypUeIBV6e29lNzGU+5WlpYZjbEOZ8A4h7NgrMMZMM7RVHa10XRycrLc3NyMjnHJsrOzlZaWZnQMoEUxzpvf0EHSqeqzWraxUF+v3qc531Zq8ZZj6t8tWsN6x6l9tD+zhwzAWIczYJzDWTDW4QwY5zhXdXX1z07AaZZSKCIiQsXFxUpNTZV0/swhAEDTeLi56JrebTSsV5zyDlRq/pr9Wr6pUAvX5atdlJ+G9W6jAd2iONYeAAAAwCVrljUJw4YN06effqq6ujpVVFRo8eLFGjp0aHM8GgCckslkUmJsgMb9ppvee3qo7r8pVTabTW98tkW/e26BXvtks/IOHJHNZjM6KgAAAAAH1ehMoUmTJmnhwoUqLy/XnXfeKX9/f82dO1ejR4/WuHHjlJKSoszMTG3ZskVXX321JGns2LGKiYlp8fAA4Ay8PKz61RVtdU3vNsweAgAAANBsGi2FJk6cqIkTJ553febMmQ2vLRaLnnvuueZNBgD4kR9mDyXGBuju65O1fFOh5q/Zrzc+26J3vspRn9RIXdUzVp3bBbH3EAAAAIBG2dVG0wCApjl39tDugiNatL5AKzYVacmGA4oI8tKQnjEakh6rYH8Po6MCAAAAsFOUQgDgwEwmkzrEBapDXKDuuT5Zq7eVaPH6Av376136YP4udUsM1ZU9Y5XROVyuVovRcQEAAADYEUohAGgl3N1cNDg9RoPTY1R6+IQWf1ugb749oCn/u0HeHlYN7B6tIT1jFR/lx/IyAAAAAJRCANAahQd56fZhHfXbq5O0Ne+QFn9boAXr8jUna5/aRvrqyh6x6t8tWv4+bkZHBQAAAGAQSiEAaMUsZpO6dQhVtw6hOn7yjJZvKtLi9fmaOStH78zeru4dQjUoLVoZyRFyY3kZAAAA4FQohQDASXh7uuraPm11bZ+2yi89qmXZhVqWfUAv7iyTh5uL+qRGalB6tJLbBctsZnkZAAAA0NpRCgGAE4oL99Xvru2kO67pqJzvyrVkwwFlbS3S4m8LFOzvoYHdozUoLVqx4b5GRwUAAADQQiiFAMCJmc0mpbYPUWr7EN13Y6rW5ZRqafYB/XfZHn22JE/x0X4alBaj/t2iFODjbnRcAAAAAM2IUggAIElyd3XRgO7RGtA9WkeOndbKTUVamn1Ab8/K0T+/ylFq+xD17xal3qmR8vawGh0XAAAAwCWiFAIAnCfAx13X94/X9f3jVVB6VMs3FWnFpkK9+slmvfH5VqUlhap/tyj17BQudzf+KAEAAAAcEd/kAQA/KzbcV3dc46vbhyUp70ClVmwq0srNRVq3vVRurhZldApX/25R6p4UKqsLJ5gBAAAAjoJSCADQJCaTSYmxAUqMDdBdwztr+77DWrGpSFlbirVic5G8PKy6IiVC/btFKSU+WBaL2ejIAAAAAH4GpRAA4KKZzSalxAcrJT5Yf7ghRZt3H9KKTYVataVYi9YXyM/bVb1TItUnNYKCCAAAALBTlEIAgEviYjErvWOY0juGqbqmVht2lmnV5vpNquev2S8fT1f1TolQny6RSm0fLBcKIgAAAMAuUAoBAJqNm9WiPqmR6pMaqdNnzmrjroPK2lqslZsLtXBdvnw8reqVHKErUiPVJSFEVhcKIgAAAMAolEIAgBbh7uqiK1IjdUVqpM7U1Gpjbn1BlLW1fomZl4dVGZ3D1adLpLolhrBJNQAAAHCZUQoBAFqcq9WiXskR6pUcoZqztdq0+5CythRrXU6Jlmw4IA83F6V3DFPv5AildQyVp7vV6MgAAABAq0cpBAC4rKwuFvXsFK6encJVc7ZOW/IOac22Eq3bXqKVm4vkYjGrS0KweiVHKCM5XAE+7kZHBgAAAFolSiEAgGGsLv+3SfWYui7atb9Ca3NKtGZbiV7/bIve+HyLkuIC62cZpYQrMtjb6MgAAABAq0EpBACwCxazSZ3bBalzuyDdNbyz9pcc1dqcUq3dVqJ352zXu3O2Ky7cp2EGUXyUv8xmk9GxAQAAAIdFKQQAsDsmk0ltI/3UNtJPv726g0oPn9C67aVas61En36zWx8v3q1AXzf1+H4ZWmpCsNxd+SMNAAAAuBh8gwYA2L3wIC9l9o9XZv94VR2v1oadZfp2R5lWbCrSgrX5cnUxKzUhRD07h6tnpzAF+XkYHRkAAACwe5RCAACH4uftpiE9YjWkR6xqztYpZ2+5vt1ZpvXbS7VhZ5nekBQf7aeencLVo1MYy8wAAACAn0ApBABwWFYXs7p1CFW3DqEanZmsgrJj+nZHfUH08aJcfbgwV4G+bkpLqt/MumtiCMfdAwAAAN+jFAIAtAomk0lx4b6KC/fVzYMTVHW8Wtm7Dmr9jlKt3lqsResLZDGb1LFtoNKSwpSWFKo2Eb4ymZhFBAAAAOdEKQQAaJX8vN00OD1Gg9NjdLa2Trn5R5S9q0zZOw/qvbk79N7cHQryc1f3DqFK6ximrgkh8vJgFhEAAACcB6UQAKDVc7GYG467H/WrTjpcdUobdx1U9q6DP5pFlNQmUGlJoUpLClObCF/2IgIAAECrRikEAHA6QX4euiojTldlxDXMItqws0zZu8r0/ryden/eTvl7u6lrYoj8XU+oTftTnGgGAACAVodSCADg1M6dRfS7a+tnEW3JO6RNuYe0efchVR6v1pdrFyou3EddE0PVrUOIOrcLkrsrf4QCAADAsfGNFgCAcwT5eWhweqwGp8eqrs6meUvW6owlWJtyD2ne6n2atWKvXCxmdWobWH/yWWKI2kb6sdQMAAAADodSCACAn2A2mxQR4Kq0tATdOChBp8+c1Y59FdqUe1Cbdx/6fsNqycfTVantg9UlIVipCSGKDPbiVDMAAADYPUohAACayN3VRd07hKp7h1BJUsXR09q8+5C25B3S1rxDytpaLEkK9nNXakJIfUnUPkTB/uxHBAAAAPtDKQQAwC8U6OvecOy9zWZTSfkJbdlTri15h7RhZ5mWbDggSYoM9lKXhBClJgQrJT5Yft5uBicHAAAAKIUAAGgWJpNJkSHeigzx1jW926iuzqb80qPakleurXsOadnGQn29Zr8kqU2Er5Ljg5QcH6zkdkGURAAAADAEpRAAAC3AbDapbaSf2kb6acSAeJ2trdOewkptyTuknL2HtWh9geas2idJignzUcoPJVF8kAJ83A1ODwAAAGdAKQQAwGXgYjErKS5QSXGB+s2VUs3ZOu0trNS2veXK+e6wlmYf0LzV+yVJ0aHeDbOIkuODFOTHnkQAAABofpRCAAAYwOpiVlKbQCW1CdTIIVJtbZ32FlUpZ2+5tu09rBWbCjV/zX5JUkSQlzq2DVTndkHq1DZQUSHenG4GAACAS0YpBACAHbBYzEqMDVBibIBuHJSg2jqb9hVVKee7cu3YV/Gjjav9vF3VqW19QdSpbZDaRfnJxWI2+BMAAADA0VAKAQBghyxmk9rH+Kt9jL9GDJBsNpuKDh3X9u8qtGPfYe3cV6E120okSW6uFnWIDWiYSZQYGyBPd6vBnwAAAAD2jlIIAAAHYDKZFB3qo+hQHw3tFSdJOlx1Sjv3V2j7d4e1Y1+FPl6UqzqbZDZJcRG+9XsYtQlUUpsARQR5seQMAAAAP0IpBACAgwry81DfLlHq2yVKknTydI127T+iXfkV2rW/Qss3FerrNfsl1S85ayiJ4gLUPsZf7q58DQAAAHBmfBsEAKCV8HS3qntSqLonhUqSautsKiw7pp37KxqKonXbSyXVL09rG+WnpLgAdYgLVGKsP7OJAAAAnEyTSqF9+/ZpwoQJqqyslL+/vyZPnqw2bdr86J7XXntN//nPfxQaWv9FtHv37nrmmWeaPTAAAGgai9mkuAhfxUX4aljvNpKkquPVyi04ol37K7Rr/xEtWl+gOav2SZJ8PF2VGOuvDrEBSoyr3/Tax9PVwE8AAACAltSkUuiZZ57RrbfeqszMTM2aNUtPP/203n///fPuGzFihMaPH9/sIQEAQPPw83ZTz07h6tkpXJJUW1ungrJjys0/ot0FR5RbcEQbcw/KZqu/PyLYq74kig1Qh7gAtY30ldXFYuAnAAAAQHNptBQ6fPiwduzYoXfffVeSdN111+kvf/mLKioqFBgY2OIBAQBAy7FYzGob6ae2kX4Ns4lOnq7RnsLKhqJo655DWraxUJLkYjGrTaSvEmL8lRBdfzpabJiPLBazgZ8CAAAAv0SjpVBJSYnCwsJksdT/X0GLxaLQ0FCVlJScVwrNnTtXq1atUkhIiB588EF169atZVIDAIAW4+luVWr7EKW2D5Ek2Ww2lVee1u6C+pJoT2Gllm8s1Ner90uSXK0WxUf5qX2Mv9pH+yshxl+RId6ymNmfCAAAwJ6ZbLYfJohfWE5OjsaPH6+5c+c2XPvVr36lF198UZ07d264dujQIfn7+8tqtSorK0uPPvqo5s2bp4CAgEZDVFdXKycn5xI+BgAAuJzqbDZVHDur4ooaFR8+o+KKMyqpqFFNbf3XClcXkyICrYoMdFVEoKsiAqwK8nGRmaIIAADgsktOTpabm9t51xudKRQREaGysjLV1tbKYrGotrZWBw8eVERExI/uCwkJaXjdp08fRUREKC8vTz179rzkkI4mOztbaWlpRscAWhTjHM6Csd50tXU2FR48pryCSu0prNSeA5XasKdKNWePS5LcXC1qF+mn+Cg/xUf7KT7aXzFhPnJh6ZnhGOdwFox1OAPGOc7V2CScRkuhoKAgdezYUXPmzFFmZqbmzJmjjh07nrd0rKysTGFhYZKknTt3qqioSG3btr3E+AAAwFFYzCbFhfsqLtxXV/aMlSSdra1T4cHj2ltYqb1FVdpbWKlvNhRoTlatpO/3KIrwUXy0v+Kj/NQuyk9xEb5yd23SWRgAAAC4BE36xvXss89qwoQJeuONN+Tr66vJkydLkkaPHq1x48YpJSVFU6dO1fbt22U2m2W1WjVlypQfzR4CAADOp7708VWbCF8N6VF/ra7OppLDJ+qLosIqfVdUpdVbi7Vgbb4kyWySIoK91TbSV+2i/L7fCNtXgb7uMplYfgYAANBcmlQKxcfH69NPPz3v+syZMxte/1AUAQAA/Byz2aSoEG9FhXirf7doSfWbWR86ckp7i6q0v7hK3xVXKe9ApVZtKW74PV8vV7WL9FObc8qi6FBvlp8BAAD8QszNBgAAhjOZTAoN9FRooKd6p/zfvoUnTtVof8lRfVdUpX3F9f/MzdqnmrN1kupnIsWEeSsuwldtwn0VF+HLrCIAAIAmohQCAAB2y8vDqs7tgtS5XVDDtdraOhUeOq59xUe1v7hK+0uOKmdPuZZlFzbc4+1hrS+KzvknNtxHnu5WIz4GAACAXaIUAgAADsViMTdsaK3u0Q3Xj588o/0lR5VfclT7S49pf3GVlmw4oFPVZxvuCQ30VGyYj+LCfRQb7qPYMF9Fh3mzsTUAAHBKfAMCAACtgrenq5Ljg5UcH9xwzWaz6eCRU/VF0feFUUHZMW3efUhna+uXoJlMUligp2LD6mcT1RdGvooO9Zar1WLUxwEAAGhxlEIAAKDVMplMCgv0VFigp3p2Dm+4Xltbp+LyEyooO6aC0mMqKD2q/NJjyt5Vpto6m6T6U9DCgrwUE+qjmDBvxYT5KCbMR9Gh3ixDAwAArQKlEAAAcDoWi7mh5OmT+n/Xa87Wqbj8+PdF0TEdKDumAwePaWNumc7W2hruC/Jzry+Lwn0UE+qt6DAfxYT6yM/blQ2uAQCAw6AUAgAA+J7V5Zz9is5RW1unksMndKDsuAoP/lAWHdeidfk6faa24T5vD6uiQ70VFeqtqBDv+tch3ooI9pbVxXy5Pw4AAMDPohQCAABohMViVnSoj6JDfSRFNFy32WwqrzytA98XRUUHj6vw4HFtyj2ob7490HDfD0vRzi2KokK9FR3iLX8fN2YXAQAAQ1AKAQAA/EImk0khAR4KCfBQ9w6hP3rv5OkaFR063lAUFX7/emveIZ05W9dwn4ebRf+vvXv9jau61zj+zOzZs+fuuTh2xhSaggpYSqv2BKlSpVZqFOpUcppXKFVE3qDyCqgqxIsUVdDbCyykqhJK1T/goFKhclCbUkCIcig6p1JIDoUcHxKg4RJm4tvYnvt9nxd7fIvjsePYmbHn+5G29p619kzWRD+tRI/WXpPsD2mo3wmNhvYENdQf0tCekCJB783+SgAAoIcQCgEAAGyDgM/Ul2+N6cu3xla0N5u2pudKujyZV2o6r8+n8kpNF/TR5Xn917spNZe2LlLIb64IiZL9QQ31B5XsDyocIDACAAA3hlAIAADgJnK7XRqIBzQQD+jftHJ1Ua3e1ESmoNR0QampglJTTnD0v5dm9J//c1n2ssAo6DedkCgR1N7+oJIJJyxK9gcV45E0AACwAYRCAAAAXcL0LN+7aKVKraGJmYLS0wWlF87TBX3w2Zzeejel5rIlRpbXUDIR1N5EQHsTQe2NBzTYej0QC8hrGjfzawEAgC5FKAQAALADWKah2/ZGdNtVv4wmSfVGU5OzRV2ZLio9nVd6pqj0tLPi6NyFKVVrjRX3J/p82psIajDuhEalbEH++IwG4wHFwj653awyAgCgFxAKAQAA7HAew+3sO9Qfkq56JM22bc3lKroyU9SVTME5zxQ0kSnq3Q+m9Prbzq+k/cd/vyXJWa00EPNrIOasLhqI+TUYD2iw9chbNMSjaQAA7BaEQgAAALuYy+VSLOJTLOLT8Jfiq/qrtYb+9tYZJfZ+SROZoiYzRU1kipqYLeqjd1PKFqor7veahgbjTmg0EAtoT2zpeiDuZ6URAAA7CKEQAABAD/Oahvojpg4MD16zv1iuaWq2pInZoiZmipqcbYVGmaIufjqrXLG24n6P4VKiz39VYOSc+2N+9Uf9stjTCACArkAoBAAAgDUFfKa+mDT1xeTqvYwkqVSpa2q2qMnZ0uJ5craoqdmS3v1gSjPZ8opfTZOkSNCr/qhfe1pHf9SvPa3AqD/qVyLik2G4b8K3AwCgtxEKAQAAYNP8lmfNDbAlZxPs6bmSpmZLmporOdet80SmqPMfTatQrq94j9slxSM+JyCK+tXf51d/1KdEn3OdiPoUj/jkITgCAOCGEAoBAABg23gMt/YmgtqbCK55T7FcWxEWTbVCpMx8WZ+kszr7fxMqV1f+gprLJUVDVis08qm/z694nxMcJSK+1rVPAZ+53V8RAIAdi1AIAAAAHRXwmbptr7nmaiPbtlUo1zUzV9L0fEnTc2XNzDsB0sx8Wanpgt77cPWKI0nyW4biEb8Sfa2gaDEwcsKjWMSneMSS6WGfIwBA7yEUAgAAQFdzuVwK+U2F/GvvbSRJ5UpdmWxZM/NOaLR0XVYmW9b4v2aUyZZVb9ir3hsOmK2AyDliYUvxvoVrZ9VRNGzJ5+W/zwCA3YN/1QAAALAr+CyPhvaENLQntOY9zaatXLG6GBTNZp3zwjGbrejdyWnN5a4dHgV8HsXClqJhJziKtQKkWNinWKR1DluKhCwZbtd2fl0AAG4YoRAAAAB6htvtUl/IUl/I0u239K1530J4tBQWlTWbqzhH6/pfn89r9v1JlSqrH1tzu6RI0FI0vOwIWa1AyVI05Kw8ioUtRYJefm0NANARhEIAAADAVZaHR18aWjs8kpzH1ubyFSc8ylU0t3DOVzSXc47UdEFz2bKq9eaq97tcUiToVV/ICY6cP9e77LrVHnba/JZHLherkAAAN45QCAAAALgBPsujvZan7S+sSc6G2aVKXXOtFUfLQ6O5fEXzreNfn89pLl9VoVS75ud4PW5FWsFRX9BSpHXuC3kVWTw7IVNf0Kug3yREAgBcE6EQAAAAcBO4XC4FfKYCPrPtvkcLavWGsoWq5nIVzeerK4Ij57qq+XxFl6fyyuYrKlcb1/wcw+1SJOhtHdaya++a7ZbXIEgCgB5AKAQAAAB0IdNjKNHnV6LPv6H7K7WG5vMVZfNVzRec0ChbWAqPcsWqsoWqPrmSVbZQVb5YVXP1XtqtP9utcMAJiMIBr8JBU5GgpXDAXNbmVaR1DgecFUlsrg0AOwuhEAAAALALWKahgVhAA7HAhu5vNm0VyjVlC1VlWwFStlBdPBZCpFyxqs8mcsoWZpQr1tRcK0mSFPSbigS8CgVMha9xDgdMhQJehf1OW8hvKhQwZXqMrfprAABcB0IhAAAAoAe53a5WUOPVLXs29h7btlUo15W7KjTKFavKFWrKF6vKFWvKlZyVSOnpgnLFqgrlmuy1syRZXkNhvxMYBf2mEx4tBEet66C/FSL5zaVrAiUAuCGEQgAAAAA2xOVyLQYzSbXfWHu5RtNWsVxrhUdV5Us15Yo1FYrOtfO6qnzRub4yU1SuOKd8qabKGnslLfB63AoFFoIiJzwK+kwF/Z7F8Cjgc/pT6bLCn84q4FvqI1QC0MsIhQAAAABsK2PZqiT1X997a/WG8qWaCq3wqFCqKV+sqVBunZe3l6qazZX1+WTeaSuvftzt3//25orXpsetoM9UwOdRwG8q6PM4IZLPVMDvafW152FNLAAACjBJREFU2v2mApYTKAVa9wUsDxtzA9ixCIUAAAAAdC3TYygWNhQL+677vbZtq1JtOAFSqaZz75zXrV+8YzFEKpaXAqZiue6cSzVlshUVy05/qdJ+pZLkPIoXsDxLQdHVZ8sjv8/jnC2nzd+63+9b2cZm3QBuJkIhAAAAALuSy+WSz/LIZ3mU6PNreo+lA8OD1/UZjaatUrmmQrmuYrkVJlXqKpbrKi0Lk0rleqvdaZvLlZWacu4rVuqq1tYPlyRnfyV/KzxafiwES1e3+6ylsMlnGfJ5nbDJ53X6CJkAtEMoBAAAAABrMNwuhQJehQLeG/qceqOpcmUpJHJCJCdAKi0PmSrO64WQqVSpK5Mt6/OF9kp93X2WlvOahvyWEzT5vK3wyGvIZy27bgVI/laotNjv9cha3tY687gcsHsQCgEAAADANvMY7i0JlyRn9VKluhQmlVvX5UpjMThy2hqt9rpKVSdoWnhPJltWqdpQudVfrTevawwLIZHlNZYFS4Ys0yOv6ZbXNGSZhrymIa/pXnZtyGr1e01DpsctlwiYttKHqbJcwclOD2PH64/6dNveSKeHse0IhQAAAABgBzHcrtaeRaYSfVvzmY1GU5VaK0SqLoVJ5WpD5aoTOFWqdSdIqjqrlZb6nPsq1YayhaKqtYYqtaaqtcbicdV+39hub0x3egQ7nulx64WxI50exrYjFAIAAACAHmcYbgUMtwI+c8s/27Zt1Rv2YkBUqTVUqzshVP06Vyhhfe+//77uvvvuTg9jx4tHrn9z+52IUAgAAAAAsG1cLpdMj0umx62gf+tDJ6xUmLF09754p4eBHcLd6QEAAAAAAADg5iMUAgAAAAAA6EGEQgAAAAAAAD2IUAgAAAAAAKAHbSgUunTpko4dO6aRkREdO3ZMH3/88ap7Go2Gfv7zn+vQoUO699579fzzz2/1WAEAAAAAALBFNhQKPfnkkzp+/LheeeUVHT9+XE888cSqe/785z/r008/1auvvqo//OEPeuaZZ3T58uUtHzAAAAAAAABu3Lqh0MzMjMbHxzU6OipJGh0d1fj4uDKZzIr7XnrpJd13331yu92Kx+M6dOiQXn755e0ZNQAAAAAAAG7IuqFQOp3W4OCgDMOQJBmGoYGBAaXT6VX3DQ0NLb5OJpO6cuXKFg8XAAAAAAAAW8HT6QEsd/78+U4PYcucPXu200MAth11jl5BraMXUOfoFdQ6egF1jo1aNxRKJpOamJhQo9GQYRhqNBqanJxUMplcdV8qldJXv/pVSatXDm3E/v37ZVnWdb2nG509e1YHDhzo9DCAbUWdo1dQ6+gF1Dl6BbWOXkCdY7lKpdJ2Ac66j48lEgkNDw/r9OnTkqTTp09reHhY8Xh8xX2HDx/W888/r2azqUwmo9dee00jIyM3OHwAAAAAAABshw09Pvazn/1MJ0+e1G9/+1tFIhGNjY1Jkh588EH96Ec/0le+8hUdPXpU//znP/Xd735XkvTQQw/p1ltv3dAgbNuWJFWr1c18h65UqVQ6PQRg21Hn6BXUOnoBdY5eQa2jF1DnWLCQsyzkLldz2Wv13ES5XE4XL17s9DAAAAAAAAB2nTvvvFPhcHhVe1eEQs1mU4VCQaZpyuVydXo4AAAAAAAAO55t26rVagoGg3K7V+8g1BWhEAAAAAAAAG6udTeaBgAAAAAAwO5DKAQAAAAAANCDCIUAAAAAAAB6EKEQAAAAAABADyIUAgAAAAAA6EGEQgAAAAAAAD2IUAgAAAAAAKAHEQptoUuXLunYsWMaGRnRsWPH9PHHH3d6SMCmHTx4UIcPH9bRo0d19OhR/f3vf5ckvfPOO/r+97+vkZERPfDAA5qZmVl8T7s+oBuMjY3p4MGDuuuuu3Tx4sXF9nbz92b7gE5aq9bXmtsl5nfsPLOzs3rwwQc1MjKiI0eO6OGHH1Ymk5G0+Xqm1tFt2tX5XXfdpSNHjizO6RcuXFh83+uvv67Dhw/r3nvv1Y9//GOVSqUN9aEH2dgyJ06csF988UXbtm37xRdftE+cONHhEQGb953vfMe+cOHCirZGo2EfOnTIPnPmjG3btn3q1Cn75MmT6/YB3eLMmTN2KpVaVd/t5u/N9gGdtFatX2tut23md+xMs7Oz9j/+8Y/F10899ZT9k5/8ZNP1TK2jG61V57Zt23feeaedz+dXvSefz9vf/OY37UuXLtm2bduPP/64/cwzz6zbh97ESqEtMjMzo/HxcY2OjkqSRkdHNT4+vpjiArvB+fPnZVmW7rnnHknSD37wA7388svr9gHd4p577lEymVzR1m7+3mwf0GnXqvV2mN+xE0WjUX3jG99YfP21r31NqVRq0/VMraMbrVXn7bz55pvav3+/9u3bJ8mp5b/+9a/r9qE3eTo9gN0inU5rcHBQhmFIkgzD0MDAgNLptOLxeIdHB2zOY489Jtu2deDAAT366KNKp9MaGhpa7I/H42o2m5qbm2vbF41GOzF8YEPazd+2bW+qj3kf3ezquT0SiTC/Y8drNpv6/e9/r4MHD266nql1dLvldb7gxIkTajQa+va3v61HHnlEXq93VS0PDQ0pnU5LUts+9CZWCgG4pmeffVZ/+tOf9Mc//lG2besXv/hFp4cEALhBzO3YrX75y18qEAjo/vvv7/RQgG1zdZ2/8cYbeuGFF/Tss8/qww8/1KlTpzo8QuxEhEJbJJlMamJiQo1GQ5LUaDQ0OTl5XUu3gW6yULter1fHjx/XuXPnlEwmVyxXzWQycrvdikajbfuAbtZu/t5sH9CtrjW3L7Qzv2OnGhsb0yeffKLf/OY3crvdm65nah3d7Oo6l5bm9FAopPvuu2/NOT2VSi3e264PvYlQaIskEgkNDw/r9OnTkqTTp09reHiYRwiwIxWLReVyOUmSbdt66aWXNDw8rP3796tcLuvtt9+WJD333HM6fPiwJLXtA7pZu/l7s31AN1prbpfaz+HM7+hmv/71r3X+/HmdOnVKXq9X0ubrmVpHt7pWnc/Pz6tcLkuS6vW6XnnllcU5/Vvf+pbee++9xV9Ffe655/S9731v3T70Jpdt23anB7FbfPTRRzp58qSy2awikYjGxsZ0++23d3pYwHX77LPP9Mgjj6jRaKjZbOqOO+7QT3/6Uw0MDOjcuXN68sknValUdMstt+jpp59Wf3+/JLXtA7rBr371K7366quanp5WLBZTNBrVX/7yl7bz92b7gE66Vq3/7ne/W3Nul9rP4czv6EYffPCBRkdHtW/fPvl8PknSF77wBZ06dWrT9Uyto9usVec//OEP9cQTT8jlcqler+vrX/+6Hn/8cQWDQUnSa6+9pqefflrNZlPDw8N66qmnFAgE1u1D7yEUAgAAAAAA6EE8PgYAAAAAANCDCIUAAAAAAAB6EKEQAAAAAABADyIUAgAAAAAA6EGEQgAAAAAAAD2IUAgAAAAAAKAHEQoBAAAAAAD0IEIhAAAAAACAHvT/3zACWNszAoIAAAAASUVORK5CYII=\n","text/plain":["<Figure size 1440x432 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"3aHwZDF9nvrR","colab_type":"text"},"source":["# Model"]},{"cell_type":"code","metadata":{"id":"7gIO5wkQnvrS","colab_type":"code","colab":{}},"source":["module_config = RobertaConfig.from_pretrained(config['config_path'], output_hidden_states=False)\n","\n","def model_fn(MAX_LEN):\n","    input_ids = layers.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_ids')\n","    attention_mask = layers.Input(shape=(MAX_LEN,), dtype=tf.int32, name='attention_mask')\n","    \n","    base_model = TFRobertaModel.from_pretrained(config['base_model_path'], config=module_config, name=\"base_model\")\n","    last_hidden_state, _ = base_model({'input_ids': input_ids, 'attention_mask': attention_mask})\n","    \n","    x = layers.Bidirectional(layers.LSTM(128, return_sequences=True))(last_hidden_state)\n","    x = layers.Dropout(.1)(x)\n","    \n","    x_start = layers.TimeDistributed(layers.Dense(1))(x)\n","    x_start = layers.Flatten()(x_start)\n","    y_start = layers.Activation('softmax', name='y_start')(x_start)\n","    \n","    x_end = layers.TimeDistributed(layers.Dense(1))(x)\n","    x_end = layers.Flatten()(x_end)\n","    y_end = layers.Activation('softmax', name='y_end')(x_end)\n","\n","    model = Model(inputs=[input_ids, attention_mask], outputs=[y_start, y_end])\n","    \n","    return model"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"jy24FlCgnvrj","colab_type":"text"},"source":["# Train"]},{"cell_type":"code","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"id":"RU2sa648nvrj","colab_type":"code","outputId":"7c2731ce-12e0-4e58-d5aa-db701626f9bd","executionInfo":{"status":"error","timestamp":1590280555631,"user_tz":180,"elapsed":5966901,"user":{"displayName":"Dimitre Oliveira","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWI4QgYDUyC8ZcW5KHpuBwcY-MZWVqbPWcZSzxIg=s64","userId":"06256612867315483887"}},"colab":{"base_uri":"https://localhost:8080/","height":1000}},"source":["AUTO = tf.data.experimental.AUTOTUNE\n","strategy = tf.distribute.get_strategy()\n","k_fold_best = k_fold.copy()\n","history_list = []\n","\n","for n_fold in range(config['N_FOLDS']):\n","    n_fold +=1\n","    print('\\nFOLD: %d' % (n_fold))\n","\n","    # Load data\n","    base_data_path = 'fold_%d/' % (n_fold)\n","    x_train = np.load(base_data_path + 'x_train.npy')\n","    y_train = np.load(base_data_path + 'y_train.npy')\n","    x_valid = np.load(base_data_path + 'x_valid.npy')\n","    y_valid = np.load(base_data_path + 'y_valid.npy')\n","\n","    step_size = x_train.shape[1] // config['BATCH_SIZE']\n","    valid_step_size = x_valid.shape[1] // config['BATCH_SIZE']\n","\n","    # Train model\n","    model_path = 'model_fold_%d.h5' % (n_fold)\n","    model = model_fn(config['MAX_LEN'])\n","    optimizer = optimizers.Adam(learning_rate=lambda: exponential_schedule_with_warmup(tf.cast(optimizer.iterations, tf.float32), warmup_steps=1, \n","                                                                                       lr_start=lr_max, lr_max=lr_max, lr_min=lr_min, decay=decay))\n","    model.compile(optimizer, loss={'y_start': losses.CategoricalCrossentropy(),\n","                                   'y_end': losses.CategoricalCrossentropy()})\n","    es = EarlyStopping(monitor='val_loss', mode='min', patience=config['ES_PATIENCE'], \n","                       restore_best_weights=False, verbose=1)\n","    checkpoint = ModelCheckpoint((MODEL_BASE_PATH + model_path), monitor='val_loss', mode='min', \n","                                 save_best_only=True, save_weights_only=True)\n","    \n","    history = model.fit(get_training_dataset(x_train, y_train, config['BATCH_SIZE'], AUTO, seed=SEED),\n","                        validation_data=(get_validation_dataset(x_valid, y_valid, config['BATCH_SIZE'], AUTO, repeated=True, seed=SEED)),\n","                        epochs=config['EPOCHS'], \n","                        steps_per_epoch=step_size,\n","                        validation_steps=valid_step_size,\n","                        callbacks=[checkpoint, es],\n","                        verbose=2).history\n","    \n","    history_list.append(history)\n","    model.save_weights(MODEL_BASE_PATH +'last_' + model_path)\n","    \n","    # Make predictions (last model)\n","    predict_eval_df(k_fold, model, x_train, x_valid, get_test_dataset, decode, n_fold, tokenizer, config, config['question_size'])\n","\n","    # Make predictions (best model)\n","    model.load_weights(MODEL_BASE_PATH + model_path)\n","    predict_eval_df(k_fold_best, model, x_train, x_valid, get_test_dataset, decode, n_fold, tokenizer, config, config['question_size'])"],"execution_count":0,"outputs":[{"output_type":"stream","text":["\n","FOLD: 1\n","Epoch 1/4\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","687/687 - 283s - loss: 2.1109 - y_start_loss: 1.0675 - y_end_loss: 1.0434 - val_loss: 1.6076 - val_y_start_loss: 0.8441 - val_y_end_loss: 0.7635\n","Epoch 2/4\n","687/687 - 283s - loss: 1.5797 - y_start_loss: 0.8196 - y_end_loss: 0.7602 - val_loss: 1.5692 - val_y_start_loss: 0.8250 - val_y_end_loss: 0.7443\n","Epoch 3/4\n","687/687 - 283s - loss: 1.4802 - y_start_loss: 0.7694 - y_end_loss: 0.7108 - val_loss: 1.5624 - val_y_start_loss: 0.8198 - val_y_end_loss: 0.7426\n","Epoch 4/4\n","687/687 - 278s - loss: 1.4404 - y_start_loss: 0.7556 - y_end_loss: 0.6848 - val_loss: 1.5683 - val_y_start_loss: 0.8217 - val_y_end_loss: 0.7466\n","\n","FOLD: 2\n","Epoch 1/4\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","687/687 - 283s - loss: 2.1609 - y_start_loss: 1.0679 - y_end_loss: 1.0930 - val_loss: 1.7389 - val_y_start_loss: 0.8773 - val_y_end_loss: 0.8616\n","Epoch 2/4\n","687/687 - 284s - loss: 1.6270 - y_start_loss: 0.8331 - y_end_loss: 0.7940 - val_loss: 1.6806 - val_y_start_loss: 0.8595 - val_y_end_loss: 0.8211\n","Epoch 3/4\n","687/687 - 278s - loss: 1.5400 - y_start_loss: 0.7930 - y_end_loss: 0.7470 - val_loss: 1.6807 - val_y_start_loss: 0.8560 - val_y_end_loss: 0.8247\n","Epoch 4/4\n","687/687 - 283s - loss: 1.4981 - y_start_loss: 0.7757 - y_end_loss: 0.7224 - val_loss: 1.6579 - val_y_start_loss: 0.8485 - val_y_end_loss: 0.8095\n","\n","FOLD: 3\n","Epoch 1/4\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","687/687 - 284s - loss: 2.1352 - y_start_loss: 1.0478 - y_end_loss: 1.0874 - val_loss: 1.6824 - val_y_start_loss: 0.8415 - val_y_end_loss: 0.8409\n","Epoch 2/4\n","687/687 - 284s - loss: 1.6055 - y_start_loss: 0.8263 - y_end_loss: 0.7793 - val_loss: 1.5814 - val_y_start_loss: 0.8105 - val_y_end_loss: 0.7708\n","Epoch 3/4\n","687/687 - 284s - loss: 1.4927 - y_start_loss: 0.7763 - y_end_loss: 0.7164 - val_loss: 1.5796 - val_y_start_loss: 0.8137 - val_y_end_loss: 0.7658\n","Epoch 4/4\n","687/687 - 283s - loss: 1.4671 - y_start_loss: 0.7629 - y_end_loss: 0.7042 - val_loss: 1.5633 - val_y_start_loss: 0.7962 - val_y_end_loss: 0.7671\n","\n","FOLD: 4\n","Epoch 1/4\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","687/687 - 283s - loss: 2.1702 - y_start_loss: 1.0494 - y_end_loss: 1.1208 - val_loss: 1.6894 - val_y_start_loss: 0.8685 - val_y_end_loss: 0.8208\n","Epoch 2/4\n","687/687 - 286s - loss: 1.6120 - y_start_loss: 0.8180 - y_end_loss: 0.7941 - val_loss: 1.6057 - val_y_start_loss: 0.8383 - val_y_end_loss: 0.7673\n","Epoch 3/4\n","687/687 - 285s - loss: 1.5313 - y_start_loss: 0.7888 - y_end_loss: 0.7425 - val_loss: 1.5942 - val_y_start_loss: 0.8341 - val_y_end_loss: 0.7601\n","Epoch 4/4\n","687/687 - 287s - loss: 1.4936 - y_start_loss: 0.7686 - y_end_loss: 0.7250 - val_loss: 1.5879 - val_y_start_loss: 0.8314 - val_y_end_loss: 0.7565\n","\n","FOLD: 5\n","Epoch 1/4\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n","WARNING:tensorflow:Gradients do not exist for variables ['base_model/roberta/pooler/dense/kernel:0', 'base_model/roberta/pooler/dense/bias:0'] when minimizing the loss.\n"],"name":"stdout"},{"output_type":"error","ename":"InternalError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mInternalError\u001b[0m                             Traceback (most recent call last)","\u001b[0;32m<ipython-input-26-9ba093b6591b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     36\u001b[0m                         \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalid_step_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m                         \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcheckpoint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mes\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m                         verbose=2).history\n\u001b[0m\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m     \u001b[0mhistory_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     64\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m     \u001b[0;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m    846\u001b[0m                 batch_size=batch_size):\n\u001b[1;32m    847\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 848\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    849\u001b[0m               \u001b[0;31m# Catch OutOfRangeError for Datasets of unknown size.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    850\u001b[0m               \u001b[0;31m# This blocks until the batch has finished executing.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    578\u001b[0m         \u001b[0mxla_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    579\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 580\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    581\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    582\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    642\u001b[0m         \u001b[0;31m# Lifting succeeded, so variables are initialized and we can run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    643\u001b[0m         \u001b[0;31m# stateless function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 644\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    645\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    646\u001b[0m       \u001b[0mcanon_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcanon_kwds\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2418\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2419\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2420\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2421\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2422\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m   1663\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[1;32m   1664\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[0;32m-> 1665\u001b[0;31m         self.captured_inputs)\n\u001b[0m\u001b[1;32m   1666\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1667\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1744\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1745\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1746\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1747\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1748\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    596\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    597\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 598\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    599\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    600\u001b[0m           outputs = execute.execute_with_cancellation(\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mInternalError\u001b[0m:  [_Derived_]  Failed to call ThenRnnForward with model config: [rnn_mode, rnn_input_mode, rnn_direction_mode]: 2, 0, 0 , [num_layers, input_size, num_units, dir_count, max_seq_length, batch_size, cell_num_units]: [1, 768, 128, 1, 96, 32, 128] \n\t [[{{node CudnnRNN}}]]\n\t [[model_15/bidirectional_15/backward_lstm_12/StatefulPartitionedCall]]\n\t [[gradient_tape/model_15/base_model/roberta/embeddings/token_type_embeddings/embedding_lookup/Reshape/_50]] [Op:__inference_train_function_458558]\n\nFunction call stack:\ntrain_function -> train_function -> train_function\n"]}]},{"cell_type":"markdown","metadata":{"id":"_59x5WnGnvro","colab_type":"text"},"source":["# Model loss graph"]},{"cell_type":"code","metadata":{"_kg_hide-input":true,"id":"Tlq60Zvenvro","colab_type":"code","colab":{}},"source":["for n_fold in range(config['N_FOLDS']):\n","    print('Fold: %d' % (n_fold+1))\n","    plot_metrics(history_list[n_fold])"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"xxbg-_R_p2dE","colab_type":"text"},"source":["# Model evaluation (best model)"]},{"cell_type":"code","metadata":{"id":"KgsapEFup3zY","colab_type":"code","outputId":"e0d11bff-fd45-4d2e-de09-7ce22efd0585","executionInfo":{"status":"ok","timestamp":1590284478919,"user_tz":180,"elapsed":6846,"user":{"displayName":"Dimitre Oliveira","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWI4QgYDUyC8ZcW5KHpuBwcY-MZWVqbPWcZSzxIg=s64","userId":"06256612867315483887"}},"colab":{"base_uri":"https://localhost:8080/","height":588}},"source":["display(evaluate_model_kfold(k_fold_best, config['N_FOLDS']).style.applymap(color_map))"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"text/html":["<style  type=\"text/css\" >\n","    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col0 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col1 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col2 {\n","            color:  red;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col3 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col4 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col5 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col6 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col7 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col8 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col9 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col10 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col11 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col12 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col0 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col1 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col2 {\n","            color:  red;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col3 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col4 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col5 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col6 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col7 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col8 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col9 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col10 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col11 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col12 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col0 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col1 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col2 {\n","            color:  red;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col3 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col4 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col5 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col6 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col7 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col8 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col9 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col10 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col11 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col12 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col0 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col1 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col2 {\n","            color:  red;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col3 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col4 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col5 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col6 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col7 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col8 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col9 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col10 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col11 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col12 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col0 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col1 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col2 {\n","            color:  red;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col3 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col4 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col5 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col6 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col7 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col8 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col9 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col10 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col11 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col12 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col0 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col1 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col2 {\n","            color:  red;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col3 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col4 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col5 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col6 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col7 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col8 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col9 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col10 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col11 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col12 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col0 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col1 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col2 {\n","            color:  red;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col3 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col4 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col5 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col6 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col7 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col8 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col9 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col10 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col11 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col12 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col0 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col1 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col2 {\n","            color:  red;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col3 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col4 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col5 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col6 {\n","            color:  orange;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col7 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col8 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col9 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col10 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col11 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col12 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col0 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col1 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col2 {\n","            color:  red;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col3 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col4 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col5 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col6 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col7 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col8 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col9 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col10 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col11 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col12 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col0 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col1 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col2 {\n","            color:  red;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col3 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col4 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col5 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col6 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col7 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col8 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col9 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col10 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col11 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col12 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col0 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col1 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col2 {\n","            color:  red;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col3 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col4 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col5 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col6 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col7 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col8 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col9 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col10 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col11 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col12 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col0 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col1 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col2 {\n","            color:  red;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col3 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col4 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col5 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col6 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col7 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col8 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col9 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col10 {\n","            color:  green;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col11 {\n","            color:  black;\n","        }    #T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col12 {\n","            color:  green;\n","        }</style><table id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Train</th>        <th class=\"col_heading level0 col1\" >Valid</th>        <th class=\"col_heading level0 col2\" >Var</th>        <th class=\"col_heading level0 col3\" >Train_fold_1</th>        <th class=\"col_heading level0 col4\" >Valid_fold_1</th>        <th class=\"col_heading level0 col5\" >Train_fold_2</th>        <th class=\"col_heading level0 col6\" >Valid_fold_2</th>        <th class=\"col_heading level0 col7\" >Train_fold_3</th>        <th class=\"col_heading level0 col8\" >Valid_fold_3</th>        <th class=\"col_heading level0 col9\" >Train_fold_4</th>        <th class=\"col_heading level0 col10\" >Valid_fold_4</th>        <th class=\"col_heading level0 col11\" >Train_fold_5</th>        <th class=\"col_heading level0 col12\" >Valid_fold_5</th>    </tr>    <tr>        <th class=\"index_name level0\" >Metric/Jaccard</th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n","                <tr>\n","                        <th id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002level0_row0\" class=\"row_heading level0 row0\" >Overall</th>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col0\" class=\"data row0 col0\" >0.729426</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col1\" class=\"data row0 col1\" >0.704603</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col2\" class=\"data row0 col2\" >0.024823</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col3\" class=\"data row0 col3\" >0.735017</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col4\" class=\"data row0 col4\" >0.707167</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col5\" class=\"data row0 col5\" >0.731888</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col6\" class=\"data row0 col6\" >0.699560</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col7\" class=\"data row0 col7\" >0.730648</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col8\" class=\"data row0 col8\" >0.711082</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col9\" class=\"data row0 col9\" >0.727479</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col10\" class=\"data row0 col10\" >0.698394</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col11\" class=\"data row0 col11\" >0.722097</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row0_col12\" class=\"data row0 col12\" >0.706810</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002level0_row1\" class=\"row_heading level0 row1\" >Sentiment negative</th>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col0\" class=\"data row1 col0\" >0.568546</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col1\" class=\"data row1 col1\" >0.525464</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col2\" class=\"data row1 col2\" >0.043082</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col3\" class=\"data row1 col3\" >0.579559</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col4\" class=\"data row1 col4\" >0.521942</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col5\" class=\"data row1 col5\" >0.572275</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col6\" class=\"data row1 col6\" >0.518279</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col7\" class=\"data row1 col7\" >0.569780</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col8\" class=\"data row1 col8\" >0.535268</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col9\" class=\"data row1 col9\" >0.565591</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col10\" class=\"data row1 col10\" >0.520696</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col11\" class=\"data row1 col11\" >0.555525</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row1_col12\" class=\"data row1 col12\" >0.531133</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002level0_row2\" class=\"row_heading level0 row2\" >Sentiment neutral</th>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col0\" class=\"data row2 col0\" >0.974008</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col1\" class=\"data row2 col1\" >0.973218</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col2\" class=\"data row2 col2\" >0.000791</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col3\" class=\"data row2 col3\" >0.973659</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col4\" class=\"data row2 col4\" >0.973781</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col5\" class=\"data row2 col5\" >0.974307</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col6\" class=\"data row2 col6\" >0.973153</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col7\" class=\"data row2 col7\" >0.973771</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col8\" class=\"data row2 col8\" >0.973792</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col9\" class=\"data row2 col9\" >0.974649</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col10\" class=\"data row2 col10\" >0.971276</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col11\" class=\"data row2 col11\" >0.973656</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row2_col12\" class=\"data row2 col12\" >0.974087</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002level0_row3\" class=\"row_heading level0 row3\" >Sentiment positive</th>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col0\" class=\"data row3 col0\" >0.558461</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col1\" class=\"data row3 col1\" >0.519062</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col2\" class=\"data row3 col2\" >0.039399</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col3\" class=\"data row3 col3\" >0.566868</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col4\" class=\"data row3 col4\" >0.529579</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col5\" class=\"data row3 col5\" >0.562612</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col6\" class=\"data row3 col6\" >0.509352</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col7\" class=\"data row3 col7\" >0.561542</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col8\" class=\"data row3 col8\" >0.530278</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col9\" class=\"data row3 col9\" >0.554051</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col10\" class=\"data row3 col10\" >0.506128</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col11\" class=\"data row3 col11\" >0.547231</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row3_col12\" class=\"data row3 col12\" >0.519970</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002level0_row4\" class=\"row_heading level0 row4\" >0 <= text wordCnt < 3</th>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col0\" class=\"data row4 col0\" >0.863293</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col1\" class=\"data row4 col1\" >0.855205</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col2\" class=\"data row4 col2\" >0.008089</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col3\" class=\"data row4 col3\" >0.870921</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col4\" class=\"data row4 col4\" >0.817164</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col5\" class=\"data row4 col5\" >0.868168</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col6\" class=\"data row4 col6\" >0.856512</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col7\" class=\"data row4 col7\" >0.858089</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col8\" class=\"data row4 col8\" >0.876640</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col9\" class=\"data row4 col9\" >0.864242</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col10\" class=\"data row4 col10\" >0.834402</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col11\" class=\"data row4 col11\" >0.855047</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row4_col12\" class=\"data row4 col12\" >0.891304</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002level0_row5\" class=\"row_heading level0 row5\" >3 <= text wordCnt < 10</th>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col0\" class=\"data row5 col0\" >0.783675</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col1\" class=\"data row5 col1\" >0.766771</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col2\" class=\"data row5 col2\" >0.016904</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col3\" class=\"data row5 col3\" >0.789611</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col4\" class=\"data row5 col4\" >0.763377</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col5\" class=\"data row5 col5\" >0.784522</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col6\" class=\"data row5 col6\" >0.766646</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col7\" class=\"data row5 col7\" >0.783830</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col8\" class=\"data row5 col8\" >0.772263</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col9\" class=\"data row5 col9\" >0.783403</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col10\" class=\"data row5 col10\" >0.760885</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col11\" class=\"data row5 col11\" >0.777008</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row5_col12\" class=\"data row5 col12\" >0.770683</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002level0_row6\" class=\"row_heading level0 row6\" >10 <= text wordCnt < 30</th>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col0\" class=\"data row6 col0\" >0.695738</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col1\" class=\"data row6 col1\" >0.666221</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col2\" class=\"data row6 col2\" >0.029517</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col3\" class=\"data row6 col3\" >0.701199</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col4\" class=\"data row6 col4\" >0.672518</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col5\" class=\"data row6 col5\" >0.698463</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col6\" class=\"data row6 col6\" >0.661066</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col7\" class=\"data row6 col7\" >0.697291</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col8\" class=\"data row6 col8\" >0.674285</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col9\" class=\"data row6 col9\" >0.693353</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col10\" class=\"data row6 col10\" >0.659125</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col11\" class=\"data row6 col11\" >0.688386</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row6_col12\" class=\"data row6 col12\" >0.664112</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002level0_row7\" class=\"row_heading level0 row7\" >30 <= text wordCnt < 100</th>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col0\" class=\"data row7 col0\" >0.665865</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col1\" class=\"data row7 col1\" >0.626600</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col2\" class=\"data row7 col2\" >0.039266</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col3\" class=\"data row7 col3\" >0.674887</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col4\" class=\"data row7 col4\" >0.649958</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col5\" class=\"data row7 col5\" >0.701328</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col6\" class=\"data row7 col6\" >0.475297</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col7\" class=\"data row7 col7\" >0.682908</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col8\" class=\"data row7 col8\" >0.607397</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col9\" class=\"data row7 col9\" >0.642529</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col10\" class=\"data row7 col10\" >0.590621</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col11\" class=\"data row7 col11\" >0.627675</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row7_col12\" class=\"data row7 col12\" >0.809725</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002level0_row8\" class=\"row_heading level0 row8\" >0 <= selected text wordCnt < 3</th>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col0\" class=\"data row8 col0\" >0.657689</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col1\" class=\"data row8 col1\" >0.614018</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col2\" class=\"data row8 col2\" >0.043671</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col3\" class=\"data row8 col3\" >0.663833</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col4\" class=\"data row8 col4\" >0.606491</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col5\" class=\"data row8 col5\" >0.671125</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col6\" class=\"data row8 col6\" >0.613840</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col7\" class=\"data row8 col7\" >0.642598</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col8\" class=\"data row8 col8\" >0.599918</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col9\" class=\"data row8 col9\" >0.665653</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col10\" class=\"data row8 col10\" >0.624493</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col11\" class=\"data row8 col11\" >0.645237</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row8_col12\" class=\"data row8 col12\" >0.625349</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002level0_row9\" class=\"row_heading level0 row9\" >3 <= selected text wordCnt < 10</th>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col0\" class=\"data row9 col0\" >0.694192</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col1\" class=\"data row9 col1\" >0.673793</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col2\" class=\"data row9 col2\" >0.020399</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col3\" class=\"data row9 col3\" >0.704406</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col4\" class=\"data row9 col4\" >0.683794</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col5\" class=\"data row9 col5\" >0.687500</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col6\" class=\"data row9 col6\" >0.663402</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col7\" class=\"data row9 col7\" >0.708348</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col8\" class=\"data row9 col8\" >0.693999</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col9\" class=\"data row9 col9\" >0.682165</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col10\" class=\"data row9 col10\" >0.654892</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col11\" class=\"data row9 col11\" >0.688540</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row9_col12\" class=\"data row9 col12\" >0.672877</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002level0_row10\" class=\"row_heading level0 row10\" >10 <= selected text wordCnt < 30</th>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col0\" class=\"data row10 col0\" >0.860623</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col1\" class=\"data row10 col1\" >0.853315</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col2\" class=\"data row10 col2\" >0.007308</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col3\" class=\"data row10 col3\" >0.860202</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col4\" class=\"data row10 col4\" >0.856620</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col5\" class=\"data row10 col5\" >0.861532</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col6\" class=\"data row10 col6\" >0.848900</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col7\" class=\"data row10 col7\" >0.864998</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col8\" class=\"data row10 col8\" >0.866923</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col9\" class=\"data row10 col9\" >0.858581</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col10\" class=\"data row10 col10\" >0.847665</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col11\" class=\"data row10 col11\" >0.857803</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row10_col12\" class=\"data row10 col12\" >0.846466</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002level0_row11\" class=\"row_heading level0 row11\" >30 <= selected text wordCnt < 100</th>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col0\" class=\"data row11 col0\" >0.892353</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col1\" class=\"data row11 col1\" >0.884627</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col2\" class=\"data row11 col2\" >0.007726</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col3\" class=\"data row11 col3\" >0.907516</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col4\" class=\"data row11 col4\" >0.829738</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col5\" class=\"data row11 col5\" >0.911903</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col6\" class=\"data row11 col6\" >0.789814</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col7\" class=\"data row11 col7\" >0.903482</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col8\" class=\"data row11 col8\" >0.912576</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col9\" class=\"data row11 col9\" >0.884699</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col10\" class=\"data row11 col10\" >0.908811</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col11\" class=\"data row11 col11\" >0.854166</td>\n","                        <td id=\"T_a9a864a6_9d5f_11ea_afe9_0242ac1c0002row11_col12\" class=\"data row11 col12\" >0.982194</td>\n","            </tr>\n","    </tbody></table>"],"text/plain":["<pandas.io.formats.style.Styler at 0x7f2a0648fe48>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"MvzO_AjPnvrt","colab_type":"text"},"source":["# Model evaluation (last model)"]},{"cell_type":"code","metadata":{"_kg_hide-input":true,"id":"ELe26ujlnvrt","colab_type":"code","outputId":"525e57ee-f2e4-41a2-c9f5-71fc6cea13ae","executionInfo":{"status":"ok","timestamp":1590284486273,"user_tz":180,"elapsed":7325,"user":{"displayName":"Dimitre Oliveira","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWI4QgYDUyC8ZcW5KHpuBwcY-MZWVqbPWcZSzxIg=s64","userId":"06256612867315483887"}},"colab":{"base_uri":"https://localhost:8080/","height":588}},"source":["display(evaluate_model_kfold(k_fold, config['N_FOLDS']).style.applymap(color_map))"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"text/html":["<style  type=\"text/css\" >\n","    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col0 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col1 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col2 {\n","            color:  red;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col3 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col4 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col5 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col6 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col7 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col8 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col9 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col10 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col11 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col12 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col0 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col1 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col2 {\n","            color:  red;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col3 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col4 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col5 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col6 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col7 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col8 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col9 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col10 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col11 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col12 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col0 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col1 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col2 {\n","            color:  red;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col3 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col4 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col5 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col6 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col7 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col8 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col9 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col10 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col11 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col12 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col0 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col1 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col2 {\n","            color:  red;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col3 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col4 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col5 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col6 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col7 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col8 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col9 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col10 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col11 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col12 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col0 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col1 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col2 {\n","            color:  red;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col3 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col4 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col5 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col6 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col7 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col8 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col9 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col10 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col11 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col12 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col0 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col1 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col2 {\n","            color:  red;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col3 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col4 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col5 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col6 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col7 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col8 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col9 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col10 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col11 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col12 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col0 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col1 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col2 {\n","            color:  red;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col3 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col4 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col5 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col6 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col7 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col8 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col9 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col10 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col11 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col12 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col0 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col1 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col2 {\n","            color:  red;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col3 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col4 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col5 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col6 {\n","            color:  orange;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col7 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col8 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col9 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col10 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col11 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col12 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col0 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col1 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col2 {\n","            color:  red;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col3 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col4 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col5 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col6 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col7 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col8 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col9 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col10 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col11 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col12 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col0 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col1 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col2 {\n","            color:  red;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col3 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col4 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col5 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col6 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col7 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col8 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col9 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col10 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col11 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col12 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col0 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col1 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col2 {\n","            color:  red;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col3 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col4 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col5 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col6 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col7 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col8 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col9 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col10 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col11 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col12 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col0 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col1 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col2 {\n","            color:  red;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col3 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col4 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col5 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col6 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col7 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col8 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col9 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col10 {\n","            color:  green;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col11 {\n","            color:  black;\n","        }    #T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col12 {\n","            color:  green;\n","        }</style><table id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Train</th>        <th class=\"col_heading level0 col1\" >Valid</th>        <th class=\"col_heading level0 col2\" >Var</th>        <th class=\"col_heading level0 col3\" >Train_fold_1</th>        <th class=\"col_heading level0 col4\" >Valid_fold_1</th>        <th class=\"col_heading level0 col5\" >Train_fold_2</th>        <th class=\"col_heading level0 col6\" >Valid_fold_2</th>        <th class=\"col_heading level0 col7\" >Train_fold_3</th>        <th class=\"col_heading level0 col8\" >Valid_fold_3</th>        <th class=\"col_heading level0 col9\" >Train_fold_4</th>        <th class=\"col_heading level0 col10\" >Valid_fold_4</th>        <th class=\"col_heading level0 col11\" >Train_fold_5</th>        <th class=\"col_heading level0 col12\" >Valid_fold_5</th>    </tr>    <tr>        <th class=\"index_name level0\" >Metric/Jaccard</th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n","                <tr>\n","                        <th id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002level0_row0\" class=\"row_heading level0 row0\" >Overall</th>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col0\" class=\"data row0 col0\" >0.731070</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col1\" class=\"data row0 col1\" >0.704824</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col2\" class=\"data row0 col2\" >0.026247</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col3\" class=\"data row0 col3\" >0.738094</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col4\" class=\"data row0 col4\" >0.708793</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col5\" class=\"data row0 col5\" >0.731888</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col6\" class=\"data row0 col6\" >0.699560</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col7\" class=\"data row0 col7\" >0.730648</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col8\" class=\"data row0 col8\" >0.711082</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col9\" class=\"data row0 col9\" >0.727479</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col10\" class=\"data row0 col10\" >0.698394</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col11\" class=\"data row0 col11\" >0.727243</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row0_col12\" class=\"data row0 col12\" >0.706290</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002level0_row1\" class=\"row_heading level0 row1\" >Sentiment negative</th>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col0\" class=\"data row1 col0\" >0.571364</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col1\" class=\"data row1 col1\" >0.524456</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col2\" class=\"data row1 col2\" >0.046908</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col3\" class=\"data row1 col3\" >0.583877</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col4\" class=\"data row1 col4\" >0.521847</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col5\" class=\"data row1 col5\" >0.572275</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col6\" class=\"data row1 col6\" >0.518279</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col7\" class=\"data row1 col7\" >0.569780</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col8\" class=\"data row1 col8\" >0.535268</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col9\" class=\"data row1 col9\" >0.565591</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col10\" class=\"data row1 col10\" >0.520696</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col11\" class=\"data row1 col11\" >0.565296</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row1_col12\" class=\"data row1 col12\" >0.526188</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002level0_row2\" class=\"row_heading level0 row2\" >Sentiment neutral</th>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col0\" class=\"data row2 col0\" >0.974111</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col1\" class=\"data row2 col1\" >0.973143</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col2\" class=\"data row2 col2\" >0.000968</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col3\" class=\"data row2 col3\" >0.973728</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col4\" class=\"data row2 col4\" >0.973735</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col5\" class=\"data row2 col5\" >0.974307</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col6\" class=\"data row2 col6\" >0.973153</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col7\" class=\"data row2 col7\" >0.973771</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col8\" class=\"data row2 col8\" >0.973792</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col9\" class=\"data row2 col9\" >0.974649</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col10\" class=\"data row2 col10\" >0.971276</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col11\" class=\"data row2 col11\" >0.974100</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row2_col12\" class=\"data row2 col12\" >0.973758</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002level0_row3\" class=\"row_heading level0 row3\" >Sentiment positive</th>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col0\" class=\"data row3 col0\" >0.561039</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col1\" class=\"data row3 col1\" >0.520781</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col2\" class=\"data row3 col2\" >0.040258</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col3\" class=\"data row3 col3\" >0.572715</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col4\" class=\"data row3 col4\" >0.534935</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col5\" class=\"data row3 col5\" >0.562612</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col6\" class=\"data row3 col6\" >0.509352</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col7\" class=\"data row3 col7\" >0.561542</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col8\" class=\"data row3 col8\" >0.530278</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col9\" class=\"data row3 col9\" >0.554051</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col10\" class=\"data row3 col10\" >0.506128</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col11\" class=\"data row3 col11\" >0.554276</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row3_col12\" class=\"data row3 col12\" >0.523213</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002level0_row4\" class=\"row_heading level0 row4\" >0 <= text wordCnt < 3</th>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col0\" class=\"data row4 col0\" >0.862299</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col1\" class=\"data row4 col1\" >0.852284</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col2\" class=\"data row4 col2\" >0.010015</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col3\" class=\"data row4 col3\" >0.868298</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col4\" class=\"data row4 col4\" >0.813433</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col5\" class=\"data row4 col5\" >0.868168</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col6\" class=\"data row4 col6\" >0.856512</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col7\" class=\"data row4 col7\" >0.858089</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col8\" class=\"data row4 col8\" >0.876640</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col9\" class=\"data row4 col9\" >0.864242</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col10\" class=\"data row4 col10\" >0.834402</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col11\" class=\"data row4 col11\" >0.852700</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row4_col12\" class=\"data row4 col12\" >0.880435</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002level0_row5\" class=\"row_heading level0 row5\" >3 <= text wordCnt < 10</th>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col0\" class=\"data row5 col0\" >0.784743</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col1\" class=\"data row5 col1\" >0.766319</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col2\" class=\"data row5 col2\" >0.018424</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col3\" class=\"data row5 col3\" >0.792563</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col4\" class=\"data row5 col4\" >0.762008</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col5\" class=\"data row5 col5\" >0.784522</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col6\" class=\"data row5 col6\" >0.766646</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col7\" class=\"data row5 col7\" >0.783830</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col8\" class=\"data row5 col8\" >0.772263</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col9\" class=\"data row5 col9\" >0.783403</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col10\" class=\"data row5 col10\" >0.760885</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col11\" class=\"data row5 col11\" >0.779393</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row5_col12\" class=\"data row5 col12\" >0.769793</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002level0_row6\" class=\"row_heading level0 row6\" >10 <= text wordCnt < 30</th>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col0\" class=\"data row6 col0\" >0.697804</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col1\" class=\"data row6 col1\" >0.666861</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col2\" class=\"data row6 col2\" >0.030943</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col3\" class=\"data row6 col3\" >0.704538</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col4\" class=\"data row6 col4\" >0.675635</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col5\" class=\"data row6 col5\" >0.698463</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col6\" class=\"data row6 col6\" >0.661066</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col7\" class=\"data row6 col7\" >0.697291</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col8\" class=\"data row6 col8\" >0.674285</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col9\" class=\"data row6 col9\" >0.693353</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col10\" class=\"data row6 col10\" >0.659125</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col11\" class=\"data row6 col11\" >0.695373</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row6_col12\" class=\"data row6 col12\" >0.664195</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002level0_row7\" class=\"row_heading level0 row7\" >30 <= text wordCnt < 100</th>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col0\" class=\"data row7 col0\" >0.667016</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col1\" class=\"data row7 col1\" >0.633617</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col2\" class=\"data row7 col2\" >0.033399</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col3\" class=\"data row7 col3\" >0.681624</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col4\" class=\"data row7 col4\" >0.685045</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col5\" class=\"data row7 col5\" >0.701328</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col6\" class=\"data row7 col6\" >0.475297</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col7\" class=\"data row7 col7\" >0.682908</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col8\" class=\"data row7 col8\" >0.607397</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col9\" class=\"data row7 col9\" >0.642529</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col10\" class=\"data row7 col10\" >0.590621</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col11\" class=\"data row7 col11\" >0.626691</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row7_col12\" class=\"data row7 col12\" >0.809725</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002level0_row8\" class=\"row_heading level0 row8\" >0 <= selected text wordCnt < 3</th>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col0\" class=\"data row8 col0\" >0.664917</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col1\" class=\"data row8 col1\" >0.618621</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col2\" class=\"data row8 col2\" >0.046295</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col3\" class=\"data row8 col3\" >0.676083</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col4\" class=\"data row8 col4\" >0.617894</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col5\" class=\"data row8 col5\" >0.671125</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col6\" class=\"data row8 col6\" >0.613840</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col7\" class=\"data row8 col7\" >0.642598</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col8\" class=\"data row8 col8\" >0.599918</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col9\" class=\"data row8 col9\" >0.665653</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col10\" class=\"data row8 col10\" >0.624493</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col11\" class=\"data row8 col11\" >0.669124</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row8_col12\" class=\"data row8 col12\" >0.636963</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002level0_row9\" class=\"row_heading level0 row9\" >3 <= selected text wordCnt < 10</th>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col0\" class=\"data row9 col0\" >0.692250</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col1\" class=\"data row9 col1\" >0.671018</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col2\" class=\"data row9 col2\" >0.021232</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col3\" class=\"data row9 col3\" >0.701606</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col4\" class=\"data row9 col4\" >0.677602</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col5\" class=\"data row9 col5\" >0.687500</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col6\" class=\"data row9 col6\" >0.663402</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col7\" class=\"data row9 col7\" >0.708348</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col8\" class=\"data row9 col8\" >0.693999</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col9\" class=\"data row9 col9\" >0.682165</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col10\" class=\"data row9 col10\" >0.654892</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col11\" class=\"data row9 col11\" >0.681629</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row9_col12\" class=\"data row9 col12\" >0.665194</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002level0_row10\" class=\"row_heading level0 row10\" >10 <= selected text wordCnt < 30</th>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col0\" class=\"data row10 col0\" >0.860120</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col1\" class=\"data row10 col1\" >0.852011</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col2\" class=\"data row10 col2\" >0.008109</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col3\" class=\"data row10 col3\" >0.859788</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col4\" class=\"data row10 col4\" >0.856060</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col5\" class=\"data row10 col5\" >0.861532</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col6\" class=\"data row10 col6\" >0.848900</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col7\" class=\"data row10 col7\" >0.864998</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col8\" class=\"data row10 col8\" >0.866923</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col9\" class=\"data row10 col9\" >0.858581</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col10\" class=\"data row10 col10\" >0.847665</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col11\" class=\"data row10 col11\" >0.855703</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row10_col12\" class=\"data row10 col12\" >0.840509</td>\n","            </tr>\n","            <tr>\n","                        <th id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002level0_row11\" class=\"row_heading level0 row11\" >30 <= selected text wordCnt < 100</th>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col0\" class=\"data row11 col0\" >0.892872</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col1\" class=\"data row11 col1\" >0.884627</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col2\" class=\"data row11 col2\" >0.008245</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col3\" class=\"data row11 col3\" >0.909224</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col4\" class=\"data row11 col4\" >0.829738</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col5\" class=\"data row11 col5\" >0.911903</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col6\" class=\"data row11 col6\" >0.789814</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col7\" class=\"data row11 col7\" >0.903482</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col8\" class=\"data row11 col8\" >0.912576</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col9\" class=\"data row11 col9\" >0.884699</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col10\" class=\"data row11 col10\" >0.908811</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col11\" class=\"data row11 col11\" >0.855053</td>\n","                        <td id=\"T_ae04c814_9d5f_11ea_afe9_0242ac1c0002row11_col12\" class=\"data row11 col12\" >0.982194</td>\n","            </tr>\n","    </tbody></table>"],"text/plain":["<pandas.io.formats.style.Styler at 0x7f2d4b8e0c18>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"yP3xG-Cenvry","colab_type":"text"},"source":["# Visualize predictions"]},{"cell_type":"code","metadata":{"_kg_hide-input":true,"id":"cGxcMVRMnvrz","colab_type":"code","outputId":"38948ed9-f8a7-4c9d-9d8d-b0913d30303e","executionInfo":{"status":"ok","timestamp":1590284486277,"user_tz":180,"elapsed":7318,"user":{"displayName":"Dimitre Oliveira","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjWI4QgYDUyC8ZcW5KHpuBwcY-MZWVqbPWcZSzxIg=s64","userId":"06256612867315483887"}},"colab":{"base_uri":"https://localhost:8080/","height":738}},"source":["display(k_fold[[c for c in k_fold.columns if not (c.startswith('textID') or \n","                                                  c.startswith('text_len') or \n","                                                  c.startswith('selected_text_len') or \n","                                                  c.startswith('text_wordCnt') or \n","                                                  c.startswith('selected_text_wordCnt') or \n","                                                  c.startswith('fold_') or \n","                                                  c.startswith('start_fold_') or \n","                                                  c.startswith('end_fold_'))]].head(15))"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>text</th>\n","      <th>selected_text</th>\n","      <th>sentiment</th>\n","      <th>jaccard</th>\n","      <th>text_tokenCnt</th>\n","      <th>selected_text_tokenCnt</th>\n","      <th>prediction_fold_5</th>\n","      <th>jaccard_fold_5</th>\n","      <th>prediction_fold_1</th>\n","      <th>jaccard_fold_1</th>\n","      <th>prediction_fold_2</th>\n","      <th>jaccard_fold_2</th>\n","      <th>prediction_fold_3</th>\n","      <th>jaccard_fold_3</th>\n","      <th>prediction_fold_4</th>\n","      <th>jaccard_fold_4</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>i`d have responded, if i were going</td>\n","      <td>i`d have responded, if i were going</td>\n","      <td>neutral</td>\n","      <td>1.000000</td>\n","      <td>10</td>\n","      <td>10</td>\n","      <td>i`d have responded, if i were going</td>\n","      <td>1.000000</td>\n","      <td>i`d have responded, if i were going</td>\n","      <td>1.000000</td>\n","      <td>i`d have responded, if i were going</td>\n","      <td>1.000000</td>\n","      <td>i`d have responded, if i were going</td>\n","      <td>1.000000</td>\n","      <td>i`d have responded, if i were going</td>\n","      <td>1.000000</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>sooo sad i will miss you here in san diego!!!</td>\n","      <td>sooo sad</td>\n","      <td>negative</td>\n","      <td>0.200000</td>\n","      <td>13</td>\n","      <td>3</td>\n","      <td>sad</td>\n","      <td>0.500000</td>\n","      <td>sad</td>\n","      <td>0.500000</td>\n","      <td>sad</td>\n","      <td>0.500000</td>\n","      <td>sad</td>\n","      <td>0.500000</td>\n","      <td>sad</td>\n","      <td>0.500000</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>my boss is bullying me...</td>\n","      <td>bullying me</td>\n","      <td>negative</td>\n","      <td>0.166667</td>\n","      <td>6</td>\n","      <td>2</td>\n","      <td>bullying me...</td>\n","      <td>0.333333</td>\n","      <td>bullying me...</td>\n","      <td>0.333333</td>\n","      <td>bullying me...</td>\n","      <td>0.333333</td>\n","      <td>bullying me...</td>\n","      <td>0.333333</td>\n","      <td>bullying me...</td>\n","      <td>0.333333</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>what interview! leave me alone</td>\n","      <td>leave me alone</td>\n","      <td>negative</td>\n","      <td>0.600000</td>\n","      <td>6</td>\n","      <td>3</td>\n","      <td>leave me alone</td>\n","      <td>1.000000</td>\n","      <td>leave me alone</td>\n","      <td>1.000000</td>\n","      <td>leave me alone</td>\n","      <td>1.000000</td>\n","      <td>leave me alone</td>\n","      <td>1.000000</td>\n","      <td>leave me alone</td>\n","      <td>1.000000</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>sons of ****, why couldn`t they put them on th...</td>\n","      <td>sons of ****,</td>\n","      <td>negative</td>\n","      <td>0.214286</td>\n","      <td>17</td>\n","      <td>4</td>\n","      <td>sons of ****,</td>\n","      <td>1.000000</td>\n","      <td>****,</td>\n","      <td>0.333333</td>\n","      <td>****,</td>\n","      <td>0.333333</td>\n","      <td>****,</td>\n","      <td>0.333333</td>\n","      <td>****,</td>\n","      <td>0.333333</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>http://www.dothebouncy.com/smf - some shameles...</td>\n","      <td>http://www.dothebouncy.com/smf - some shameles...</td>\n","      <td>neutral</td>\n","      <td>1.000000</td>\n","      <td>27</td>\n","      <td>27</td>\n","      <td>some shameless plugging for the best rangers f...</td>\n","      <td>0.833333</td>\n","      <td>http://www.dothebouncy.com/smf - some shameles...</td>\n","      <td>1.000000</td>\n","      <td>http://www.dothebouncy.com/smf - some shameles...</td>\n","      <td>1.000000</td>\n","      <td>http://www.dothebouncy.com/smf - some shameles...</td>\n","      <td>1.000000</td>\n","      <td>some shameless plugging for the best rangers f...</td>\n","      <td>0.833333</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>2am feedings for the baby are fun when he is a...</td>\n","      <td>fun</td>\n","      <td>positive</td>\n","      <td>0.071429</td>\n","      <td>17</td>\n","      <td>1</td>\n","      <td>fun</td>\n","      <td>1.000000</td>\n","      <td>fun</td>\n","      <td>1.000000</td>\n","      <td>fun</td>\n","      <td>1.000000</td>\n","      <td>fun</td>\n","      <td>1.000000</td>\n","      <td>fun</td>\n","      <td>1.000000</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>soooo high</td>\n","      <td>soooo high</td>\n","      <td>neutral</td>\n","      <td>1.000000</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td>soooo high</td>\n","      <td>1.000000</td>\n","      <td>soooo high</td>\n","      <td>1.000000</td>\n","      <td>soooo high</td>\n","      <td>1.000000</td>\n","      <td>soooo high</td>\n","      <td>1.000000</td>\n","      <td>soooo high</td>\n","      <td>1.000000</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>both of you</td>\n","      <td>both of you</td>\n","      <td>neutral</td>\n","      <td>1.000000</td>\n","      <td>3</td>\n","      <td>3</td>\n","      <td>both of you</td>\n","      <td>1.000000</td>\n","      <td>both of you</td>\n","      <td>1.000000</td>\n","      <td>both of you</td>\n","      <td>1.000000</td>\n","      <td>both of you</td>\n","      <td>1.000000</td>\n","      <td>both of you</td>\n","      <td>1.000000</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>journey!? wow... u just became cooler.  hehe.....</td>\n","      <td>wow... u just became cooler.</td>\n","      <td>positive</td>\n","      <td>0.500000</td>\n","      <td>19</td>\n","      <td>7</td>\n","      <td>wow... u just became cooler.</td>\n","      <td>1.000000</td>\n","      <td>wow... u just became cooler.</td>\n","      <td>1.000000</td>\n","      <td>wow...</td>\n","      <td>0.200000</td>\n","      <td>wow... u just became cooler.</td>\n","      <td>1.000000</td>\n","      <td>wow... u just became cooler.</td>\n","      <td>1.000000</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>as much as i love to be hopeful, i reckon the ...</td>\n","      <td>as much as i love to be hopeful, i reckon the ...</td>\n","      <td>neutral</td>\n","      <td>1.000000</td>\n","      <td>27</td>\n","      <td>27</td>\n","      <td>as much as i love to be hopeful, i reckon the ...</td>\n","      <td>1.000000</td>\n","      <td>as much as i love to be hopeful, i reckon the ...</td>\n","      <td>1.000000</td>\n","      <td>as much as i love to be hopeful, i reckon the ...</td>\n","      <td>1.000000</td>\n","      <td>as much as i love to be hopeful, i reckon the ...</td>\n","      <td>1.000000</td>\n","      <td>as much as i love to be hopeful, i reckon the ...</td>\n","      <td>1.000000</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>i really really like the song love story by ta...</td>\n","      <td>like</td>\n","      <td>positive</td>\n","      <td>0.100000</td>\n","      <td>12</td>\n","      <td>1</td>\n","      <td>like</td>\n","      <td>1.000000</td>\n","      <td>like</td>\n","      <td>1.000000</td>\n","      <td>like</td>\n","      <td>1.000000</td>\n","      <td>i really really like</td>\n","      <td>0.333333</td>\n","      <td>like</td>\n","      <td>1.000000</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>my sharpie is running dangerously low on ink</td>\n","      <td>dangerously</td>\n","      <td>negative</td>\n","      <td>0.125000</td>\n","      <td>9</td>\n","      <td>1</td>\n","      <td>dangerously low</td>\n","      <td>0.500000</td>\n","      <td>dangerously</td>\n","      <td>1.000000</td>\n","      <td>dangerously low</td>\n","      <td>0.500000</td>\n","      <td>dangerously low on ink</td>\n","      <td>0.250000</td>\n","      <td>dangerously low</td>\n","      <td>0.500000</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>i want to go to music tonight but i lost my vo...</td>\n","      <td>lost</td>\n","      <td>negative</td>\n","      <td>0.100000</td>\n","      <td>13</td>\n","      <td>1</td>\n","      <td>lost my voice.</td>\n","      <td>0.333333</td>\n","      <td>lost</td>\n","      <td>1.000000</td>\n","      <td>lost my voice.</td>\n","      <td>0.333333</td>\n","      <td>lost my voice.</td>\n","      <td>0.333333</td>\n","      <td>lost my voice.</td>\n","      <td>0.333333</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>test test from the lg env2</td>\n","      <td>test test from the lg env2</td>\n","      <td>neutral</td>\n","      <td>1.000000</td>\n","      <td>8</td>\n","      <td>8</td>\n","      <td>test test from the lg env2</td>\n","      <td>1.000000</td>\n","      <td>test test from the lg env2</td>\n","      <td>1.000000</td>\n","      <td>test test from the lg env2</td>\n","      <td>1.000000</td>\n","      <td>test test from the lg env2</td>\n","      <td>1.000000</td>\n","      <td>test test from the lg env2</td>\n","      <td>1.000000</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                                                 text  ... jaccard_fold_4\n","0                 i`d have responded, if i were going  ...       1.000000\n","1       sooo sad i will miss you here in san diego!!!  ...       0.500000\n","2                           my boss is bullying me...  ...       0.333333\n","3                      what interview! leave me alone  ...       1.000000\n","4   sons of ****, why couldn`t they put them on th...  ...       0.333333\n","5   http://www.dothebouncy.com/smf - some shameles...  ...       0.833333\n","6   2am feedings for the baby are fun when he is a...  ...       1.000000\n","7                                          soooo high  ...       1.000000\n","8                                         both of you  ...       1.000000\n","9   journey!? wow... u just became cooler.  hehe.....  ...       1.000000\n","10  as much as i love to be hopeful, i reckon the ...  ...       1.000000\n","11  i really really like the song love story by ta...  ...       1.000000\n","12       my sharpie is running dangerously low on ink  ...       0.500000\n","13  i want to go to music tonight but i lost my vo...  ...       0.333333\n","14                         test test from the lg env2  ...       1.000000\n","\n","[15 rows x 16 columns]"]},"metadata":{"tags":[]}}]}]}