Emotion_classifier_Lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "# Import phase if every import is here it is to allow you to run this cell and know if you have any issue and not after a few hours of training\n",
    "\n",
    "import tensorflow\n",
    "from tensorflow import keras\n",
    "import os\n",
    "import numpy as np\n",
    "import librosa\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.utils import shuffle\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "from keras_preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.callbacks import Callback\n",
    "from tensorflow.keras.layers import Dense, Dropout, Conv1D, Activation, Flatten, MaxPooling1D, BatchNormalization, LSTM, GaussianNoise\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.regularizers import l2\n",
    "\n",
    "\n",
    "from src.VoiceModule import VoiceModule\n",
    "from src.classifiers.KerasClassifier import KerasClassifier\n",
    "from prepare_data_utils import load_wav, preprare_wav\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fearful\n",
      "savee\n",
      "calm\n",
      "happy\n",
      "bdes\n",
      "surprised\n",
      "angry\n",
      "sad\n",
      "keywords\n"
     ]
    }
   ],
   "source": [
    "# print environment\n",
    "folder_path = \"../../data\"\n",
    "for l in os.listdir(folder_path):\n",
    "    print(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform data method\n",
    "\n",
    "def mfcc(buffer, samplerate, dim):\n",
    "    a = librosa.feature.mfcc(y=buffer, sr=samplerate, n_mfcc=dim)\n",
    "    a = np.transpose(a)\n",
    "    return a\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create callback to later draw some graphs\n",
    "\n",
    "class PerformanceLogger(Callback):\n",
    "    def __init__(self):\n",
    "        # List of tuple (loss, accuracy)\n",
    "        self.training = []\n",
    "        self.testing = []\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        self.training.append((logs[\"loss\"], logs[\"acc\"]))\n",
    "        self.testing.append((logs[\"val_loss\"], logs[\"val_acc\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This function prints metrics about a model\n",
    "def print_metrics(voice_module, X, Y, label_name_list):\n",
    "    Y_pred = voice_module.model._model.predict(X)\n",
    "    Y_pred = np.argmax(Y_pred, axis=1)\n",
    "    Y_max = np.argmax(Y, axis=1)\n",
    "    print(confusion_matrix(Y_max, Y_pred))\n",
    "    print(classification_report(Y_max, Y_pred, target_names=label_name_list))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create class\n",
    "class EmotionClassifierLstm(KerasClassifier):\n",
    "    def __init__(self, file_path=None):\n",
    "        super().__init__(file_path)\n",
    "\n",
    "    def predict(self, x):\n",
    "        x = np.expand_dims(np.array([x]), axis=2)\n",
    "        return self._model.predict(x)\n",
    "\n",
    "    def build(self):\n",
    "        model = Sequential()\n",
    "        # This first layer add noises to the input data and serve as a data augmentation technique\n",
    "        # Used to prevent overfitting of the LSTM layer and try to extract more significant feature\n",
    "        model.add(GaussianNoise(0.4))\n",
    "        # This layer normalise the data to speed up the training and prevent the gradient of the LSTM to explode\n",
    "        # and reach exponential weight value\n",
    "        model.add(BatchNormalization())\n",
    "        # This is THE feature extraction layer\n",
    "        model.add(LSTM(128, input_shape=(128, 13), kernel_regularizer=l2(0.01)))\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(Flatten())\n",
    "        model.add(BatchNormalization())\n",
    "        # This is the second part of the network, this one will be fine tuned later\n",
    "        model.add(GaussianNoise(0.2))\n",
    "        # The two last layers will be fine-tuned at the end of this notebook\n",
    "        model.add(Dense(64))\n",
    "        model.add(Dense(5))\n",
    "        model.add(Activation('softmax'))\n",
    "        model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "        self._model = model\n",
    "        \n",
    "    def train(self, X, Y, batch_size=32, epoch=720, validation_data=None, callbacks=[]):\n",
    "        self._model.fit(X, Y, batch_size=batch_size, epochs=epoch, validation_data=validation_data, callbacks=callbacks)\n",
    "\n",
    "    def transform(self, x, samplerate):\n",
    "        to_process = mfcc(x, samplerate, 13)\n",
    "        to_process = pad_sequences([to_process], maxlen=128, padding='post')[0]\n",
    "        return to_process\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/arc/fdp5/.local/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    }
   ],
   "source": [
    "#Instanciate model\n",
    "emotion_list = [\"calm\", \"happy\", \"angry\", \"fearful\", \"sad\"]\n",
    "dimension_mfcc = 13\n",
    "samplerate = 16000\n",
    "nb_break=None\n",
    "sample_duration=3\n",
    "step=2\n",
    "\n",
    "logger = PerformanceLogger()\n",
    "\n",
    "cls = EmotionClassifierLstm()\n",
    "vm = VoiceModule(\"emotion\", emotion_list, cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Doing calm\n",
      "Doing happy\n",
      "Doing angry\n",
      "Doing fearful\n",
      "Doing sad\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "# Prepare data for training\n",
    "data = {f: load_wav(f\"{folder_path}/{f}\", nb_break) for f in emotion_list}\n",
    "X, Y = preprare_wav(data, vm, sample_duration, step)\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_file = f\"x_{'_'.join(emotion_list)}-{vm._name}.npy\"\n",
    "Y_file = f\"y_{'_'.join(emotion_list)}-{vm._name}.npy\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving the prepared input data \n",
    "np.save(X_file, X)\n",
    "np.save(Y_file, Y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data for training\n",
    "X, Y = np.load(X_file), np.load(Y_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "769 193\n",
      "(128, 13)\n",
      "float32\n"
     ]
    }
   ],
   "source": [
    "\"\"\"train_size = 80 * len(X) // 100\n",
    "test_size = 20 * len(X) // 100\n",
    "#X = np.expand_dims(X, axis=3)\n",
    "X_train, X_test, = X[:train_size], X[train_size:]\n",
    "Y_train, Y_test = Y[:train_size], Y[train_size:]\"\"\"\n",
    "X = X.astype('float32')\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)\n",
    "print(len(X_train), len(X_test))\n",
    "\n",
    "print(X[0].shape)\n",
    "print( X.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 769 samples, validate on 193 samples\n",
      "WARNING:tensorflow:From /home/arc/fdp5/.local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Epoch 1/100\n",
      "769/769 [==============================] - 8s 10ms/sample - loss: 1.8828 - acc: 0.1990 - val_loss: 1.8023 - val_acc: 0.1606\n",
      "Epoch 2/100\n",
      "769/769 [==============================] - 5s 6ms/sample - loss: 1.7344 - acc: 0.3121 - val_loss: 1.7458 - val_acc: 0.2124\n",
      "Epoch 3/100\n",
      "769/769 [==============================] - 5s 6ms/sample - loss: 1.5916 - acc: 0.3524 - val_loss: 1.6971 - val_acc: 0.2746\n",
      "Epoch 4/100\n",
      "769/769 [==============================] - 5s 7ms/sample - loss: 1.4726 - acc: 0.3810 - val_loss: 1.6685 - val_acc: 0.3212\n",
      "Epoch 5/100\n",
      "769/769 [==============================] - 5s 6ms/sample - loss: 1.4066 - acc: 0.4187 - val_loss: 1.6329 - val_acc: 0.2746\n",
      "Epoch 6/100\n",
      "769/769 [==============================] - 5s 6ms/sample - loss: 1.2844 - acc: 0.4603 - val_loss: 1.5940 - val_acc: 0.2642\n",
      "Epoch 7/100\n",
      "769/769 [==============================] - 4s 5ms/sample - loss: 1.2921 - acc: 0.4733 - val_loss: 1.6020 - val_acc: 0.3472\n",
      "Epoch 8/100\n",
      "769/769 [==============================] - 4s 5ms/sample - loss: 1.2035 - acc: 0.5254 - val_loss: 1.7158 - val_acc: 0.1762\n",
      "Epoch 9/100\n",
      "769/769 [==============================] - 4s 5ms/sample - loss: 1.1849 - acc: 0.5111 - val_loss: 1.5236 - val_acc: 0.3627\n",
      "Epoch 10/100\n",
      "769/769 [==============================] - 4s 5ms/sample - loss: 1.1181 - acc: 0.5449 - val_loss: 1.4999 - val_acc: 0.3316\n",
      "Epoch 11/100\n",
      "769/769 [==============================] - 4s 5ms/sample - loss: 1.0426 - acc: 0.5709 - val_loss: 1.4673 - val_acc: 0.4560\n",
      "Epoch 12/100\n",
      "769/769 [==============================] - 3s 4ms/sample - loss: 0.9989 - acc: 0.6151 - val_loss: 1.4752 - val_acc: 0.3834\n",
      "Epoch 13/100\n",
      "769/769 [==============================] - 4s 5ms/sample - loss: 0.9885 - acc: 0.6008 - val_loss: 1.5533 - val_acc: 0.3368\n",
      "Epoch 14/100\n",
      "769/769 [==============================] - 4s 5ms/sample - loss: 0.9751 - acc: 0.6034 - val_loss: 1.3815 - val_acc: 0.4767\n",
      "Epoch 15/100\n",
      "769/769 [==============================] - 4s 5ms/sample - loss: 0.8721 - acc: 0.6515 - val_loss: 1.5768 - val_acc: 0.3472\n",
      "Epoch 16/100\n",
      "769/769 [==============================] - 3s 5ms/sample - loss: 0.8998 - acc: 0.6567 - val_loss: 1.5441 - val_acc: 0.3472\n",
      "Epoch 17/100\n",
      "769/769 [==============================] - 4s 5ms/sample - loss: 0.9002 - acc: 0.6424 - val_loss: 1.5247 - val_acc: 0.4197\n",
      "Epoch 18/100\n",
      "769/769 [==============================] - 4s 5ms/sample - loss: 0.8436 - acc: 0.6892 - val_loss: 1.3011 - val_acc: 0.4767\n",
      "Epoch 19/100\n",
      "769/769 [==============================] - 4s 5ms/sample - loss: 0.8628 - acc: 0.6762 - val_loss: 1.3318 - val_acc: 0.4301\n",
      "Epoch 20/100\n",
      "769/769 [==============================] - 4s 5ms/sample - loss: 0.8138 - acc: 0.6801 - val_loss: 1.4299 - val_acc: 0.4404\n",
      "Epoch 21/100\n",
      "769/769 [==============================] - 3s 4ms/sample - loss: 0.8088 - acc: 0.6853 - val_loss: 1.1932 - val_acc: 0.5233\n",
      "Epoch 22/100\n",
      "769/769 [==============================] - 3s 5ms/sample - loss: 0.7562 - acc: 0.7178 - val_loss: 1.3651 - val_acc: 0.3886\n",
      "Epoch 23/100\n",
      "769/769 [==============================] - 3s 4ms/sample - loss: 0.7616 - acc: 0.7048 - val_loss: 1.1315 - val_acc: 0.5855\n",
      "Epoch 24/100\n",
      "769/769 [==============================] - 3s 4ms/sample - loss: 0.7677 - acc: 0.7113 - val_loss: 1.5529 - val_acc: 0.3679\n",
      "Epoch 25/100\n",
      "769/769 [==============================] - 3s 4ms/sample - loss: 0.7009 - acc: 0.7217 - val_loss: 1.1794 - val_acc: 0.5544\n",
      "Epoch 26/100\n",
      "769/769 [==============================] - 3s 4ms/sample - loss: 0.6129 - acc: 0.7737 - val_loss: 1.1114 - val_acc: 0.5959\n",
      "Epoch 27/100\n",
      "769/769 [==============================] - 3s 4ms/sample - loss: 0.6101 - acc: 0.7867 - val_loss: 1.3306 - val_acc: 0.5285\n",
      "Epoch 28/100\n",
      "769/769 [==============================] - 3s 4ms/sample - loss: 0.6097 - acc: 0.8023 - val_loss: 1.2122 - val_acc: 0.5078\n",
      "Epoch 29/100\n",
      "769/769 [==============================] - 3s 4ms/sample - loss: 0.5663 - acc: 0.7854 - val_loss: 0.9630 - val_acc: 0.6425\n",
      "Epoch 30/100\n",
      " 64/769 [=>............................] - ETA: 2s - loss: 0.4455 - acc: 0.8750"
     ]
    }
   ],
   "source": [
    "vm.model.train(X_train, Y_train, batch_size=64, epoch=100, validation_data=(X_test, Y_test), callbacks=[logger])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzsnXd4VeX9wD9v9k7IIIwMCCRhz7CRqYC4wb1wUlvR1tXa2lpr258dVluVuncddVVQUWQKyJAgM0AGkEDI3oPs+/7+eO/JvUlukptxM8j7eZ48555z3nPOeyF5v+e7hZQSjUaj0WgAnLp7AhqNRqPpOWihoNFoNJp6tFDQaDQaTT1aKGg0Go2mHi0UNBqNRlOPFgoajUajqUcLBY1Go9HUo4WCRqPRaOrRQkGj0Wg09bh09wTaSnBwsBwyZEh3T0Oj0Wh6Ffv27cuTUoa0Nq7XCYUhQ4YQHx/f3dPQaDSaXoUQIs2ecdp8pNFoNJp6tFDQaDQaTT1aKGg0Go2mHi0UNBqNRlOPw4SCEOINIUSOEOJIM+eFEOI5IUSKEOKQEGKSo+ai0Wg0GvtwpKbwFrCkhfMXA9Hmn5XAiw6ci0aj0WjswGFCQUq5DShoYcgVwDtSsRsIEEIMdNR8NBqNRtM63elTGAycsdpPNx/TaDSaXo/JJNmWlMvbO1NJzSvv7unYTXcmrwkbx2w2jBZCrESZmIiIiHDknDQajabdlFXVklVcybakXN7dncYpK2EQG+rLotGhLB49gNGD/BDC1hLY/XSnUEgHwq32w4AMWwOllK8ArwDExcXZFBwajUbTlaw7nMnGo9lklVSSVVJJdnEl5dV19ecnRQTw8+smMD48gC3Hc1ifkMXqLSk8vzmFwQGeXDpuIA9cFIOHq3M3foumdKdQWAusEkJ8CEwDiqWUmd04H41G08VU1tRx9zvxeLo68+CiGEYM8Ks/dyqvHH9PVwK93bpxhrZ5ZdsJ/m/dcUJ83Qnv58mIAb7MiQ5hgL8HA/w8iB3gy8iBlu8ydPZQ7pg9lPyyKjYdz+HbhCxe2X6SQ+nFvLoiDh/3nlNxSEjpmBdvIcQHwDwgGMgGfg+4AkgpXxJKd3oBFaF0DrhdStlqUaO4uDipax9pNN1LbmkVL393ggtHhTJtaGC7TCFSSh7++BCf/piOr7sLZdW1XD5+EIMCPPk2IYsTueX4ebjw9h1TmRjRr1PmXVpZQ3xaIdW1JgDcXJyYNSwYNxf73KtSSp7dkMRzm1O4ZNxAnr12gt3XNubz/Wd56OODjB3sz1u3TyHAy7HCTwixT0oZ1+o4RwkFR6GFgkbT/fzhiwTe/D4VgJhQH26dMYRr48LbtEC+syuVx9ck8POF0dwxaygvbzvBm9+nUlNnYnpUEPNH9OedXankllbx2oo4Zg4LbtdcpZR8si+ddYcz+T4ln+o6U4PzF0QH8/Itk/Fya/1t/a/fHOfFrSe4Ni6Mp5aNw9mpY36BbxOyWPX+fiKDvFg+OYzxYQGMDfN3iOaghYJGo3EIpZU1zHhqM3NigpkXqxbuI2dLiAn14f+uGkvckECb1+1MyeNcdR0D/D3ILavi7rfjmRsTwqu3xuFkXlxLKmuQEvw9XQHIKank5tf3kJp/jkcWxRLgpY6PHuTPqEF+Np/TmE/3pfPQxwcJD/Rk8agBLBjRH3/zfX48XcTv1xxhYkQ/3rhtSv1zbXHkbDGXvbCDayaH8Zdl4+rn3FF2JOfxuzVH6p3STgKmDg1k0agBLBodSlg/r055jhYKGo2mCa9tP4mfpyvXxoW3PrgZ3vr+FE98cZTP753FhPAApJRsPp7D42sSOFtUwY3TInh4UWy9L6C2zsQfvjjKu7sbVm4eGuzNmlWz8PNofiEGKCyv5rY3f+BgenH9MTdnJzY9NJfwwNYXzGtf3kVuaRWbH5pr08z19eFM7v9wP9H9ffnPXdNs+jCklNz46h6OZ5Ww9ZH5LQqP9lJQXs2h9CL2phaw4Wg2SdllCAEv3jSZJWMGdPj+WihoNJoGvLHjFE9+eRR3Fye2/2o+/X092nwPk0my8JnvCPBy5X8/m9XgXHlVLc9uSOKN70/h5ebCXRcM5dq4cH716SG2J+fxkzlRLB07kKySSvLLqlkwoj8D/O2bQ51JklFUAUBxRQ3XvLSLBSP7s/rGlqvjpOaVM+/prTyyOJZ75w9vdtx3SbmsfCeeWcODeX1FXBPhseFoNne/E8+TV4zm1hlD7JpzRzmVV8697/1IQXk1mx6ai3cHTUr2CgVdEE+j6YHUmSS5pVV2j5dSsvZgBik5pTbPf5uQxR+/OsrMYUHUmiQvbT3ZrnltTcrhVF45t88a2uSct7sLv710FOt/MYfZw4P558ZkZv5lM7tP5vO3q8fx66UjGR8ewOLRA7hxWoTdAgHA2UkQHuhFeKAXYwb7s3JOFF8dymRfWmGL132yLx0nAcsnhbU4bm5MCI9ePILNx3P4OD69wbmaOhNPrTtGVIg3N0ztujypocHe/PHK0WSVVPLClpQue64WChpNDyG/rIqn1h3j2pd2Meb365ny5418cdBm6k4DKmvq+PmHB7j/g/1c/K/t/OPbRCprVLy8lJK9qQXc/+F+xoUF8PqKKSybOJj39qSRU1LZ5jm++X0qoX7uXNyCOSM61JeXbpnMmntncfXkMP5z57QOmats8ZO5UfT3dedPXx2lOWtHnUk5mOfGhNglgFbMGML0qECe/PIo6YXn6o+/tzuNk3nlPLZ0JK7OXbtkTo4MZPmkMF7bfpKTuWVd8kwtFDSaHsC56lpuf2svr+84RY3JxHVTwhkfHsAvPznE8aySZq/LKa3k+ld2s/ZgBr+4MJrLxg3i+c0pXPyv7dz19l6m/HkT17y0i2Afd167NQ5PN2dWLRiutIXv2qYtJGeXsj05j1umR9q1OI4PD+Dpa8YzLSqoTc+xBy83Fx5eHMv+00V8ech2etP25FyySiq5xk6B5OQk+PvV45FS8sjHh9ienKtMRl8qDWvBiP6d+RXs5tGLR+Dh4swTXzQvADsTLRQ0mm6mziS5/4MDHDlbzEs3T+Z/P5vFE5eP5tVbJuPr4cJP3t1H8bmaJtcVn6vhqtU7Scwq5aWbJ/GLC2N45roJ/OfOabi7OHEqr5w5McH84fLRfPbTmYT4ugMQGeTdLm3ho/gzuDqLLjWhtMTySWGMHOjHX785Tk2jMFOAj/el08/LlYUj7V/MwwO9+N2lo9h1Mp9bXv+BfWmF3DN3GC/cOKnbylKE+LrzwEUxbEvK5duj2Q5/Xs9Jo9No+ih/+uooG49l84fLR3PhqND64/39PHjx5slc/8oufv7f/by+YkqDuPhnNiSSWVzBx/fMYHKkJQx0dnQw3/xiTovPvG9BNJ/tP8u/t57gictHtzrHOpPyWcyN6U+Qj3s7vmXn4+wkeOiiGO56J55vjmRx2fhB9eeKzlWzISGbG6dF4O7StjIS100Jp7iihhBfd5aOHdgjylDcOiOSA2eK6gW7I9GagkbTyUgpScwq5flNyby3J83mmIyiCt7emcoNr+zmze9TuXP2UFbMHNJk3OTIfvz+stFsTczlL18fqz9+NKOEd3encfP0yAYCwV4igry4ZnIY7+85TVp+wwqeb35/ihtf3V2f9Quw51Q+2SVVXDFhUONbdSsLRvQnMsiLt3amNjj+zq40qutM7fJlCCH4ydxhLJsU1iMEAoCLsxPP3TCRSZ2U2d3isxz+BI2mD/HVoUz+tv44afkWR2Wor0cDDeCZbxN5brOKJhne34eHLorhZy2ES940LYLk7FJe3X6K8EAvbpkeye/XHiHAy42HLopt91wfuCiGNQcy+Os3x/n3TZMB5Tf4v3XHqKmTfPZjOtebTUVrD2Tg7ebMhSNDW7pll+PkJFgxYwhPfnmUQ+lFjAsLIKekkpe+O8GS0QPsTnDTWNCagkbTSUgpefLLBJydBH+6cgzbfzmf0YP8eOSTg2QWqxj79/ec5rnNKVw1cTCbHprLxgfnct/C6BbLJQghePyy0Vw4sj9PrE3g0U8Psze1kF8tia3PzG0PoX4e3DN3GOsOZxGfWoDJJPn1Z4fxdndh5EA/nt+cQnWtiaraOtYdzmTx6AF4uvWMN2drrokLw8fdpb7sxjMbkqipM/HoxSO6d2K9FC0UNJpOIiGjhOySKn42bzg3T48kPNCL52+YSFWtiV98eIAtx3P43ZojzIsN4e9Xj2NYiI/d93Z2Evzr+omMGuTHf+PPMCE8gGsmdzzM8+45Qwn1c+ePXx3jvR9OE59WyG8vGcUvl8RytqiCT39MZ2tiLiWVtVzew0xHBr4erlw9OYwvD2XwXVIu/40/w60zhjAk2Lu7p9Yr0UJBo+kkNh3LQQiYFxtSfywqxIcnrxjDnlMF3PH2XmJDfXnhxkm4tCPe3dvdhTdWTOGKCYP46/LOqb3j5ebCw4tiOXimiCfWJjBreBDLJw1mXkwIE8IDeGFzCp/sSyfI243Zw9tXkK4rWDFzCLUmycp34vHzcOW+Bc2b4zQto4WCRtNJbD6ezYTwAIIbRecsnzSYG6aGExHoxRu3TelQBcz+fh786/qJxA7w7eh0reYXxuhBfrg4Cf585ViEEAgh+MWF0ZwtqmDD0WwuHTewXYKsqxga7M382P5U1Zr4+cJoh5ehPp/RjmaNphPIKa3kYHoxDy+KaXJOCMFTy8ZRZ5IdLrXsCJycBG/fMZXc0qoGJpe5Zm3hwJkiLp/Q89unP7I4lsEBntw8PbK7p9Kr0UJBo+kEth7PBWBhC9E5PVEgGAT7uDfRcIRQDvOvDmcyKSKgm2ZmPyMH+vHHK8d09zR6PQ7VB4UQS4QQiUKIFCHEozbORwohNgkhDgkhtgohWq5apdH0UDYey2aQvwcjOtGs0xMYM9ifXy0Z0WObzGs6H4cJBSGEM7AauBgYBdwghBjVaNjTwDtSynHAk8BTjpqPRtMeTuSW1ReXa47Kmjp2pOSxYGR/vXhqej2O1BSmAilSypNSymrgQ+CKRmNGAZvMn7fYOK/RdBvbknK56JnveH3HqRbH7TlVwLnquhZNRxpNb8GRQmEwcMZqP918zJqDwHLz56sAXyFE55dU1GjayOn8c9z3wX5MUgmHlth8LBtPV2dmOKAaqEbT1ThSKNjSoxvXfX0YmCuE2A/MBc4CtU1uJMRKIUS8ECI+N7flP1CNxpqyqlq2JOZQa6OKZnOcq65l5bvxSClZOnYA+08XUVFt24RUZ5J8ezSbWcODe0ydHI2mIzgy+igdsE65DAMadAyRUmYAywCEED7AcillMY2QUr4CvAKqHaejJqw5v8gvq2LFmz9w5GwJUcHePLgohqVjBjZJ+krNK+eZDUm4OAlC/T04lllCYnYpb942BQmsO5zFvrRCZkc3Td7afTKfzOJKHrtkZBd9K43GsThSKOwFooUQQ1EawPXAjdYDhBDBQIGU0gT8GnjDgfPR9CGyiiu5+fU9nCk4xyOLY1lz4Cyr3t/PqIEneGRxLPNiQxBCsPtkPvf8Zx91dRI/T1dySiupNUl+tWQE82L7U1ZVi4uTYOeJPJtC4dMf0/H1cOlxheI0mvbiMKEgpawVQqwC1gPOwBtSygQhxJNAvJRyLTAPeEoIIYFtwL2Omo/m/OZMwTm+OpyJlCCRfPDDaQrLa3j7jqlMjwrinrnD+OJgBs9sSOL2t/YyZUg/LogO4fnNyfWZxpFB3phMkoqauvom6T7uLowPD2DnifwmzzxXXcs3R7K4YsIgbTrSnDc4NHlNSrkOWNfo2ONWnz8BPnHkHDR9g7+tT2zQzzjYx5337prG+HCVdOXsJLhy4mCWjh3IR/FneG5TMntTk7ggOpgXbpyEv6eqNurkJOoFgsHMYUH8e+sJSitr8PWwVCVdn5DFueo6lrXSFF6j6U3ojGZNr6e2zsS2pFyWTRzM/y0bC4Crs5PNDGI3Fydunh7J8klh/JBawKxhQa3W9JkxLIjnN6ewN7WABSMsZqLPfjxLeKAncZGOb3yi0XQVPbfClUZjJwfOFFFcUcPCkaF4uDrj4ercakkJTzdn5saE2FXkbVJEP9xcnNiZYjEhZRVXsiMlj6smhumENc15hRYKml7P1sRcnJ2ETUdwZ+Dh6kxcZL8GfoXPD5xFSlg2secXitNo2oIWCppexdbEHNYcONvg2JbEHCZH9qv3CziCmcOCOJpZQn5ZFV8dyuT1HaeYHNlPN3LRnHdon4KmV/H0t4kkZZUxObIfYf28yC6pJCGjhF8tcWzrxRnDgoEklj63neySKmJCfXj80salvDSa3o/WFDS9hsqaOo5nllJdZ+KfG5MB+C5RZbhbdztzBOPC/AnxdcfNxYlnrxvP1z+fUx/ZpNGcT2hNQdMjMJkku0/lE97Pi/BAL5tjjmWWUGuSxIT68NmP6fxkThRbEnMY4Of4ktWuzk5sfmguHq7OuPbgDmQaTUfRv92abkVKydbEHC5fvYMbX93Dwme+4/lNyVTXNq1VdChdVUB59roJeLm58Jevj7MjOY/5I0K6JALI18NVCwTNeY/+Ddd0GyaT5K6347ntzb0UV9Tw1LKxXDQylH9sSOKS57ZzLLOkwfiD6UWE+LozaqAfd18QxabjOZRW1TIvtn83fQON5vxDCwVNt5GQUcKm4zncM3cYmx6cxw1TI1h90yReXxFHQXk1T6xNaDD+UHox48P8EUJw5wVDCfJ2w9VZMGu4Y0JRNZq+iBYKmm5j0/FshIC7LxiKm4vlV3HhyFBumhbB3tQC8suqACitrOFEbhnjwpRz18fdhb9dPY7fLB2Jj7t2jWk0nYUWCppuY/PxHCaGBxDUqGE8wKLRAzBJ2HQsB4DDZ4uREsaG+dePWTgylNtnDe2y+Wo0fQEtFDRdwoncMqS0tMLIKankUHpxsy0sRw/yI6yfJ98kZAEWJ/P4MB0GqtE4Ei0UNA5FSsnf1x9n4T++47Xtll7Hm48rDWDhSNtOYiEEi0cPYEdyHmVVtRxKLyKsnyeB3m5dMm+Npq+ihYLGYZhMkifWJrB6ywn8PFx4YUsKxRU1AGw6nsPgAE9iQ5vPL1g8egDVdSa2JuZw8Eyx1hI0mi5ACwWNQzCZJL/89BBv70rjrtlDef/u6RRX1PDqtpNU1tSxIzmPBSP6t5hfMDmyH8E+bry/5zRniyoYZ+VP0Gg0jsGhQkEIsUQIkSiESBFCPGrjfIQQYosQYr8Q4pAQYqkj56PpOr5LzuWTfenct2A4j10ykjGD/bls/CBe33GKLw9lUlFTx4JmTEcGzk6Ci0aF1lcnHac1BY3G4ThMKAghnIHVwMXAKOAGIUTjCmK/BT6SUk5E9XD+t6Pmo+lathzPwdPVmVULhtdrAw9eFEN1nYnffn4YT1dnZkQFtXqfRaMHACBEw8gjjUbjGBypKUwFUqSUJ6WU1cCHwBWNxkjAz/zZH8hA0+uRUrL5eA6zhgfj7mLpXTw02JvrpoRTWWNidnSwXX2NZw4LwtfdhWEhPjofQdM11FbDK/PgwPvdPZNuwZFCYTBwxmo/3XzMmieAm4UQ6ahezvc5cD6aLuJEbjnphRXMH9G0cunPF0YT7OPGVXY2p3F3cebXS0dy34LhnT1NjcY2iV9Bxn449N/unkm34MhXL1seRNlo/wbgLSnlP4QQM4B3hRBjpJQNqqEJIVYCKwEiIiIcMllN57E1UYWb2qpJFOrnwd7HLmxTAbsbp+n/c00Xsu8ttU3bBTUV4Oppe1xuIghnCD6/XlgcqSmkA+FW+2E0NQ/dCXwEIKXcBXgATQrZSClfkVLGSSnjQkIcWzdf03G2JOYQE+rD4ADbf0y6p7GmATUVUHTG9k9tddfOpeAknNwK4dOhrgrO7Gl+7Me3wX9v7qqZdRmO1BT2AtFCiKHAWZQj+cZGY04DC4G3hBAjUUIh14Fz0jiYsqpafjhVwB26/ITGHk7vVgtreTN/9lHz4dbPu24+P76j3v6vWA3/nqYERNS8puNKMiHnqPqcfRRCz58ufA4TClLKWiHEKmA94Ay8IaVMEEI8CcRLKdcCDwGvCiEeQJmWbpPWtRA0vY7vU/KoqZO6nLWmdQ58AF/cD/7hsOC3ajG25sQmOLoGzhWAV6Dj51NbDfv/AzFLlEkobKoSCrY49Z3lc8JntoWClLDzeUjZCBNugtFXgos7FKbC3tchLxmueQtcPSzXlGbD5/fA5S+Av31+t87GoeEcUsp1KAey9bHHrT4fBWY5cg6armVrYg4+7i7EDenX3VPR9FRKs2HHs7DnRRhyAVz7ju1FP3QUJPwPkjfA+Os6fx51tfDDyzBwAkTOhKSvlcYy+TZ1PmoebH3KtlA6uRU8A2HAGDjyKcx/TMVNG9RWwRc/h4MfqHH/WwnrfwOho+HUNurdq9kJEDbZcl3qdjixGQ5/BLMf6PzvbAc6o1nTIapq69h4NJtTeeWYTJItx3O5IDpYdyjTNOXsj/DJnfDsaCUQ4u6EW/7XvBYwcCL4hKrFuqMUnW567IdX1EL91lJ4cRZseQr8wmD4QnU+ah4g1UJtjZRms9JcGHO18kNkHrScL8uFty9TAmHeb+CRFLj5MwifCkVpMOdhuHWNGpuX2PDeueb9pPUd/87tRAd+azrEe7tP8+SXyrbq6+5CaVUt87XpSNOYkgx4YzG4eMKUu9RPa1E7Tk4QvUiZkOpqwNm1fc9O+Bw+XgGLn4IZP1PHSrOVFjBsAYy+SgmIrARlxnIym7EGTwI3XyUARlmlWOUlQWmmEhojL4OvHlTawqAJymn+3nLITVKmodFXqWuGL7QIG1BaipOrRQjU39u8f2ZP15nNGqFf5zQdYsPRbKKCvfnLsrFcOn4gC0b0Z9Fo2+WwNa1w5gd4YYpaDM439v8H6qph5Ra4+C/2h3HGLIGqEkjb2f5n731Nbb/9LZwyv/VveBxqK2Hp0zDpVvjJdlgVD7OsTDbOrjBkdlO/grEfNU8t2sMWKDOXyQRfPqC0hmvetAgEWzi7QNAwJWCsyU1S2oo0KbNZN6CFgqbdFFfUsDe1gMVjBnD91AieWjaON26bQoCXLm/dLk5sUYtE+t7unknnYqpTUT1R89RC2BaGzQdn9/abU/JSlPln9oMQGKXCSA9/Aoc+hJn3WeYjBARHq8Xamqh5yjxUmGY5dnIr9BsK/Yao/THLofgMrPmZxWQUe3HrcwuOaagp1NVCfgqMuarzzGbtQAsFTbvZnpxLrUmycIQ2F3UKhukg40DXP7v4bEO7eFsoz1MmmvI82+dPbFaLpuHAbQtu3jB0jlogbQUmnitQGlZz/PgWOLnAtHvg+veVA/jTO9Xb+AUPtf78qHlqe3KL2tbVQuqOhmGqsUuV4Dr4gfo85xG7vhrBMSoSqVa1nKUoDUw1EDJCmc1SNimzWRejhYKm3Ww+lkOAlysTI3SkUT11te1PuMo1mxIyu0EobHxCOUftXYSkhPR4+Own8MxIZbN/ZpTaT9/XcOy+t8A7BGIvad/cYhart/X8lKbndj4Pb11ie961Vap+UexS8A2FkBhY9jK4esPSvymB0xohsUrD2PC40hAy9itzVtQ8yxgPPxh3DYSOhateUr4QewiJBVmnvhtYtIbg2M4xm7UTLRQ07aLOJNmSmMP82P44O+kM5Xo+XgHvXmX7rbYlTHWQn6w+d4emkHscKovh9K6Wx9VUKP/AK/PgtYVw/CuYtAJu+VzZ5o9/Ca8tgHW/VAKyJBMSv1Zx+i7tNCvGLFHbRBvmlLwk5as4l9/03PEv1XFrDWXEJfBomtragxAqQsp3ELy7TPklEEp7seay5+An28CjDZV8g2PU1hAGhqYYEqOETkfMZh1ACwVNuzhwppDCczXNttPsk9RUKOdg2g5zLHozpGxUb9TWgqPotHJ8hoyA0gwoy3H8fA2khPwT6nPiN7bHFJxSC+IzI2HNvRYn7UPH4JKnle3/kqfhoeMw/Wcq/v/9a2D3v9Xb8KRb2z+/gHAIHWN7gSwwt3i19e+17y0IiFBZ0da0NYqp3xC481sVPXRmNwwc3zQqyMnZfg3BIDhabQ1nc24S+AxQgsXdB4Ze0LzZzIFooaCxi/KqWpKyS+v3Nx7LwcVJcEG0rkVVz5k9ql6OcIbt/2h+nOHoLLJyXhoLw7hr1bY92sLB/yobdVspzYSackBAUiOhUJgK710Dz02EXf9WyWYrvoSf7Yapd4N7o3aq7r6w5Cn15nxqG+x8DobObbuDuTGRM5XPw3qBlNJieilvJBTyT6jnT7q17Yu1LTz84IYPYdGfYOHvOn4/UOYr/wjL/31ektISDGKWqO+XsrFLBYMWCppWqayp45bX97Do2W08te4Y1bUmNh/LYcqQQPw92xk7fj5ycqtyas79pSqDkB5ve1zOMbW1Pm8sDGOvUdu2+hVKMlTW7K529KkybPUjLoGCE6r8gsHXv1J27bm/hAeOwHXvqjfY1ooaTjablIJjOyczNzgGqkuVADMozYLaCvW5sZPbSDgbvazjzzZwclYRS8Mv7Lx7Bkcr85GU6ncg2EoojLxM+WLeuxpevgD2vQ3V5zrv2c2ghYKGz/ef5aP4MzbPmUySBz86wP4zRVw4sj8vbzvJFau/JzG7VJuOGnNyq6qXM2MVePazrS2Y6iw2ZGuhkJuoFoCACAga3nZN4aS5Fk/2kbbP2xAKM1apraEtZB1Wn2f9Aub/BvwGte2+Qy+AVT8o01JHCYlVW+sQTkNLgKbmo9IstfUPp0cTEquEcGmmciwHx1rO+Q6A+w/Apc+qHIgv7rfkXDgQLRT6OFJK/rzuGL/+7DBHM0qanP/rN8dZdziLx5aO5LUVU/j3TZNIL1RvKwt0KKqFcwVqIY+ap+zB034KietUbRtrClMtb7fW+Qh5SZYFYeCEtmsKRkJV1pG2mxryT6hM4/Bp0H+0xa+w/RmV0Tv1rrbdzxEYb9DWyV7WQqGx+ag0C7yC2u/c7iqCY9TvQ8omtW9tPgL1uxR3B/z0e7j9a5jo+FLdWij0cRIySsgtraLOJPn1Z4eoM1kWlDd2nOLlbSe5dUYkd85WpbCXjh3I1z+/gNdXxBEV4tNd0+55pG4HpCVUcerd4OajCr8S/eaWAAAgAElEQVRZk3tcbSNnQ9YhFTYppXoDNhyPgyZAyVlVQ8cejFo8zm5QVdy0zs+el+HIZ81fn5esbP5OThC7REUgpe9TWbpT7lRaT3fjEwru/k2FgpML+A5saj4qzVLHezqGBnTsC7W11hSsEUL5Vbqg7IUWCn0co0va7y8bxcH0Yt7emQrA6i0pPPnlURaNCuXxS0c1aIwT1s+LhSN1KYsGnNyq3qoHT1L7XoHKP5D4tQrNNDBq8E+8WYVSZh1WlTkriywLxMAJamtLW5BSvcnXVFqO5SZCWRaMM1cStTYh1dWoHITP7ladxGyRn6JMVqCcm7IOPrpVlXmecW9b/hUchxDqLbqx+SggUi3+jc1HZVlKkPR0DA3o5Bb1++M7oHvngxYKfZ4tibmMD/PntplDmBcbwtPfJvLY/w7z9/WJXDFhEKtvmoSLrnjaOie3qjo51uGOQ2ZDdZnSCAxyjpnDJOep/fS9VklL5gVi4Di1teVXSPoGPrgOtv294bPBvIALZUIyyDwINefUG/VHtyqHtDV1NcqkZQiFwZPBKxhK0lXkjk8PMhEGxzbVFAKjlC/GlvmoN2gK3sGqtHZdtRJ6PaArof5r78MUllez/3Qhc2P7I4TgT1eOAeC9Pae5cVoEz1w7QZfAtofCNLVARc1reDxyptpaJ4TlHIP+o8BvIPgNVs5mY6EzNAUPfwgc1lRTkBK2Pa0+//AqVJp9QCe3qsWx/0gIHArZhy3XGBmxN36khMN/b7GUVQAlEGSdRSg4OSttwclFRdr0JEJioCwbKorM4ain1Pf2CWloajPVKc3BtxdoCmD5f2/OdNTFOPQvXgixRAiRKIRIEUI8auP8s0KIA+afJCFEkSPno2nItuRcTBLmx6pcg7B+Xvzr+ok8fuko/nzlGJ2pbC9GF66oeQ2P+w1S5g1jYa6rUfb7/iPVflic0hTyklTpBT+rTluDJjbVFE5tg7Px6g2+qlhFotTVNKzFM2BsQ03h9C4lYKLmwpUvqus3/sFy3og8CrKqWnrRH+COb5VG05OwdjaX56kQ1cAo8O4P5/JUhA6oc7Kud2gKYPlejZ3M3YTDhIIQwhlYDVwMjAJuEEI06FknpXxASjlBSjkBeB5owRum6Wy2JuYS6O3GuLCA+mMXjQrljtlDG/gQNK1wcqvKRA2x8aYXOVP1ITayhk01EGIWCoPjVAJb2vfKyWz9bz5ogjLhlFjF5W//h7KTX/x3Va5512olEKpLLUIhdCwUnoKqUrVInt4FkTPUuVGXK7/D/nctPol6oWCVXOYd3LAbWE/BWigYkUeG+chUq/wyoPwJ0Dt8CtCnNIWpQIqU8qSUshr4ELiihfE3AB84cD59huTsUqpq61ocU2eSfJeUy9yYEK0RdJS0Xcp/YEuQRsxQb7F5yRYnc72mMEVtsw43FSjDFigTzjuXK2GSHq80khmrVE/fCx5W9/3yAUCoTGNQ7SFBNZPPPQ4VhRAx03LfcdeqePgT5hDI/BQVutkNzVzaTL8hqh5QbmJDoWD4PQxnc2m22vYWTSFqnhIIYXHdPRPAsUJhMGCdEZVuPtYEIUQkMBTY3Mz5lUKIeCFEfG6unWF6fZTc0iqWPredf25MbnHcofQiCsqrmRery1R0iLIcVato0ETb5yPNLcjTvleLtHCyciiPVws/NMxkBdXL99Y1yhTy2kJY9zB4BKiYdVAaSPh0pRUMmmBZ1EMNoXAYTpvNVoamAKrkhFeQ6hQGqt9AkJ0Nb7obJ2c1V0NTEE7KxOVt/h0uN68NRtZzD4jksYvQ0SrJr4c49R0pFGy9fjaXVXM98ImU0ubrrZTyFSllnJQyLiREL2It8X1KHjV1kg9/OE1lTcN/ziNni9l0LJsjZ4v5+kgWTgLmnI+1i46uhVcXWmzMjsSw+w+aYPt80DC1aJ3epTSFwGHqTR/AzUstCGDb9DRkNty9SdnMM/bD9J+qZCZQWsmch9XnoXMt1/iHKUd11hGlwfgOVA1hDJxdYeTlKlS2urxhOGpvwAhLLTipspVd3KyEgllTKDNrCr3FfNTDcGSP5nTAOsc8DMhoZuz1QA8JiO7dbEvOxdlJUHiuhnWHM1k2KQyAzOIKrn5pJ5U1loVycmQ/+nn38IzP9pD8rXKoVhY1NIuU5cKOZ+DCP3RepqsRITRgnO3zQigTUtou9UzDdGQQNkWFjTbWFAwCo+CuDXDoI5hwY8Nzwy+Ey5+H6MUNnxc6VpmkSjLUsxubtcYsh31vKm2hLKvjxeq6kuAY1bPZ1Uv924CV+chKU+gN2cw9FEcKhb1AtBBiKHAWtfDf2HiQECIW6Ae0Ushd0xpSSnYk57FkzACOZZbw7u60eqHw9/WJmEzw5u1TqKqpI6u4kunDgrp5xg7CCPE8l99QKCR/q0o5j74Kwqd2zrMyDqg3bQ+/5sdEzoRja9XnMcsbnht/o7L7B7awMHv4qwzpxghhuyT1gDEqZFXWWcJiG8/HJ1SVsYDepSkEx6j+xTkJEDFdHfMMVKakevNRtnL8a9qFw4SClLJWCLEKWA84A29IKROEEE8C8VJK818JNwAfStnFRcPPQ5Kyy8gprWJudAiTI/rx5JdHOXK2GCnhsx/P8pO5UcyP7Rl2S4dhlIwAc+OVaMs5oxFLcXpToVBTAa6ebX9e5gH1Nt4S1uebaAqT4eo32v7clhgwVgmExs82cHJWgnHPS2o/KLrpmJ6KtZnN0BScnFTCnWE+Ks3sPf6EHohdPgUhxKdCiEuEEG3yQUgp10kpY6SUw6SUfzYfe9xKICClfEJK2SSHQdN2tierN6XZ0cEsnxyGh6sT7+5K409fHSXQ24175/eiN8L2YpSMAFWkzppz5vo4JWcbHi9Mg6fCVZ+DtlCWq+7VnD/BYMBYVcIAVOKaozGczR7+zT+vvqS0UAlvvYWg4dS7Kw2hAMqEZJiPyrK1UOgA9i7yL6JMP8lCiL8IIUY4cE6adrI9OY9hId4MCvDE39OVKycM5uN9Z9hzqoAHLorBz6MP9D6wLoPQuEWjtaZgTXaCyh9Y/xtLlrA9GP6Ega0IBSdniJgGTq4NFzJHETJCNfqJmNF8g5mwKap5vX94+zSk7sLVE/pFqs/W/5beIeqFwGTSQqGD2CUUpJQbpZQ3AZOAVGCDEGKnEOJ2IUQfWGl6PpU1dew5ld+gE9rN0yMxSRje34cbpvTwuvL2cnqPpdaPLawLpjURCmbNobFQMKqKlmXD1r/YPxcj8mhgM05may54CBb/ue2tINuDq4d61qxfND/GyUk1r1/wmOPn09kExwBC5S0YGPWPzuWrRDbtU2g3dvsUhBBBwM3ALcB+4D1gNrACmOeIyWnsZ19aIZU1JubEBNcfGzPYn98sHcH0qKDeUdSuJEPF4rt52T4vJXz+U1Ui+t7dtscYJSNMtU2FQnkz5qOi06qfwLhrlZ194s0QOsrSDavfUNuRLJkHlIPYnmbtkTNtO30dxfSftj7G3ub1PY3YpSq/wwjtBYv5qLflKPRA7PUpfAZsB7yAy6SUl0sp/yulvA/QRfUdiJSSlJxSGvvhj5wt5sJnvmP1lhSqa01sS87F1VkwbWjDiKKVc4Y1KGPRY/nxHfjnWNj0ZPNjsg6pdpGNK31aY/Ql8AqCisY+BcN81FgopKkkqAufUFFE6x6G+DfgxVmweirsfdX2szIOtO5P0HQ+cbfDDY2KH3iHqGY1BSfUvhYK7cbe18cXpJSjpJRPSSkzrU9IKXtGbvZ5ykfxZ7jwmW2s3pJSf6ywvJp7/rOPs4UV/H19Ipc+v52vD2cxKaIf3u6OjDJ2AKY6WP8YrL3P3CxmS/NjjSzcqmKoKrM9Ji9ZRah4BTbvaC7PaVgptOi0slN7BcLCx1X28ZcPqDBHjwCVONaY8jxVm6g1f4KmazAS2LLMFWK1UGg39gqFkUKI+tdNIUQ/IcTPHDQnjRWf/ngWJwFPf5vE6ztOUWeS3P/hfnJKq/hw5XReuzWOsspaThec44Lo4NZv2FOQUhWK+89y2PUCTF0J8x5VpSDK822PP/I/ZTqChg3cDarK1EIdHGMWClb3qauBymLwN1f+tNY2DE0BYNIKWPo03LEe7tmuQldzjjV9VmuZzJquxUhgyzT3rtDZzO3GXqFwt5Syvqy1lLIQsJFNo+lMzhZV8MOpAu5bEM3FYwbwxy+PctNru9menMefrhjD+PAALhwVyoYH5/LUsrGsmDmku6dsH0c+hZcugDcWw9kf4ZJnYOnfYegcdf60jTzG9HgoPq3s/tDULwAN+xJ4BTUUChWFams4hY3rK4qUsDCEgpOzShSLmK6Sw/qPVPe17p4GkGnWHgaOb9t31zgGa03BM1B1jdO0C3uFgpOwqqVsLoutc8gdzBcH1dvsskmD+df1E5kXG8LukwXcNC2Ca62iibzdXbhhagS+vSHktCwHPrlTdZq67F/w0DHVBxhUUTlnd9tCIeEzpSVMMb+L2PIrGEIhOMYsFKzMR4aT2VjEDb9CsblmY3O9A/qPUnO1bhIPSlMIjLLPyaxxPIZQKMvSpqMOYq8Bej3wkRDiJVRRu3uAbxw2Kw0Aaw5kMDEigMggbwBeunkym4/ncGFv7o9cmApIWPRHiFnc8JyLuyofbDSlMTCZVBP54RepGHywrSnkJqqolMAo9bZYUah8Fk7OFq3BqFFUYg5LNcJRAyJtz9d4Xs7Rhk1QMvZD+DR7vrGmK/C2KuyohUKHsFdT+BWqrPVPUYXrNgG/dNSkNJCUXcqxzBKuGD+o/piHqzNLxw7EzaUXhJc2h/Fm7t9M3kTEDFUgztqRfHqX8iGMWabCEL2CmtcUAqNULoBXECCVeQgsQsE/TAkMI1ehME1tmxUKsYBo6FcoOq2EkhYKPQcXNxUUADpHoYPYm7xmklK+KKW8Wkq5XEr5cnNlrjWdw5oDZ3F2ElwyblDrg3sTRYZQCLN9PnKGqtuT/oPl2JFPVVXM2IvVvt+g5oWCUW3UyxyaawgDI/LIOxj8B1vMR0WnVV5Dc01mXD2VoDEa5ICqeApdm3egaR1DW9CaQoewN08hWgjxiRDiqBDipPHj6Mn1VaSUrDmQwazhwYT4nmcOs+Izyg7fXFXR8GkqFNRYeAvT4MB7qgeAmzKj4Te4qVCoq1F2/3qhYF7kjVwFw7/gGajKO5RYCYWACNtd0wz6j1RRUQand4K7n6UXgqZnYEQgaaHQIey1Q7yJqn9UC8wH3gHeddSk+jo7UvJIL6xoYDo6byhOt4SF2sLdV9n9Db/C+t8oIbHwd5YxvgObCoWCkyqL2aiiaQgFQ1MozwN3f2Vm8B9sMR8ZOQot0X+kaolp9DVO26WEl5Nz699X03VoTaFTsFcoeEopNwFCSpkmpXwCWOC4afVNpJS8uyuVO9+OZ6C/B4vHnIe/3EVnIKCVOkyRM1WTnMSv4fiXqsOYtbnJb7AyBxmLNFhqHjVrPrLqreA3WFVSrS63aAot0X+kMmnlJyvhkpfYsMWlpmdgCAXtU+gQ9gqFSnPZ7GQhxCohxFXAeV6Yv2sprqjhp//5kd+tSWDmsCC+uG82Pr0tO/lcAXz9qMV5a4viM837EwwiZkBtJXy2UpVKnrGq4Xk/swZlncBmHY4KzQgF8zHj+dlHVXZ0q0LBXH4657glXNbovazpOWjzUadg76rzC1Tdo/uBP6JMSCscNam+RmllDbe+voeEjBIeWzqSO2cPxcmpBRt3T+X4V7DnRTj8MVz/nqUzlkFlMVSVNB95ZGA0hqkqgWveapqIZAiFkgxLL4DsI2pxN3oYu3qBi4fFl3AuT2kIYNmmfa+2rQmFwGGq7HXOUVUew9ld5VRoehbRi1SZE+P/V9MuWtUUzIlq10opy6SU6VLK280RSM2UqWxw7RIhRKIQIkUIYbORjhDiWrMDO0EI8X47vkOvpqK6jjvfiicho4SXbp7M3XOieqdAAPW27uymHMlvXwYHGhUtay3yyMAnBAZPVq0rhy9set74o7f2K2QcaFiHSAjlVD5n5Wiu1xTM1xtv/c2Foxq4uCmNJeeYcjKHxemM2Z7IoAmw/FVw7mUadg+jVaFgDj2dbJ3RbA9mYbIauBgYBdwghBjVaEw08GtglpRyNEoj6TNU1dax8t144tMKePa6CVw4qhcnpYESCkHRcNdGpSV8fg+c2Ws5bzh3W3szB7jjW1jWTHVSv4FqW2oWChVFUHiqaR0io9SFlMoXYAgF30GAsBIKdsyn/0jI+FHV1mmt/aZG04ux16ewH1gjhLhFCLHM+GnlmqlAipTypJSyGvgQuKLRmLuB1eZaSkgpc9oy+Z5Mda2JmjpTs+ellDzy8SG2J+fxl+XjuOx8iDSqL1sdCNe8rY6l7bCcby1xzRpnl+aje9x9VUiooSlkHlTbxhVLjaJ41eVQV2URCi5uqmBaZbFqk+nZr/X59B+lmvDIOu1k1pzX2CsUAoF8VMTRZeafS1u5ZjBwxmo/3XzMmhggRgjxvRBitxBiia0bCSFWCiHihRDxubm5dk65e7nz7b0sf3En5VW1Ns+/vuMUaw9m8PCiGK6NOw+6otVUqmqj1iGhARGWaqKgIn2c3RqWJGgvfoMsuQZGW8zGdn6jp4LhbPa2qiJrmJBay1Ew6D9SbYWTzmTWnNfYZXyTUt7ejnvb+kuTjfZdgGhU57YwYLsQYox1RVbz818BXgGIi4trfI8eh5SSA6eLKK2q5b4P9vPKLZMbdD7beSKPp74+zuLRodw7f3g3zrQTyU8BabJE/4B6c8+0EgrF6cqf0Fzf4LZgndWcsV/lPjTOSjY0BSOb2cuqAZHfYDi7zz7TEViEwoBxSlPRaM5T7BIKQog3abqgI6W8o4XL0gHrV+AwoHFtgnRgt5SyBjglhEhECYm99GKyS6ooraplcmQ/Nh/P4Q9fHOXJK1T264ncMla9v5+hwd7849oJtNFV03OxLlttMGgCHFurCtN59jOHo3aSVuQ3yFKPKOMADLJRwtorSPkbynIs+waGs7u1xDWDfkOU43rY/HZPWaPpDdjrpv/S6rMHcBVNF/jG7AWihRBDgbPA9cCNjcZ8DtwAvCWECEaZk3p9+YzknFIAHlkcy6Zj2by6/RSJWaWk5peTU1qFr7sLL98yufflIbREXhIgVJSOgWHjzzwIUfOUpjDMRjRRe/AbrGz85fnKyTzx5qZjjKJ4+Ses9q2uB/s1BSdn+OlO+/wPGk0vxl7z0afW+0KID4CNrVxTK4RYhSq77Qy8IaVMEEI8CcRLKdeazy0SQhwF6oBHpJQ22m71LpKzVYXP6P4+TB0SyLnqOvacKmD28GDGhfkzf0T/+nLY5w25iWqBdfW0HDNs/BkHIGImlGa1ns1sL36DlLkq+Vvzs2x0QDOEgKHFNNAU2igUwBL1pNGcx7T3VTUaaPWvSUq5DljX6NjjVp8l8KD557whOaeMQG83gnxULPufrxrbzTPqAvKSGpqOQNn0/SOUX6EkHZCt5yjYi685WivxK7UdaCOZzHirz0tWfRasG+JEzobYpUpYaTSaeuz1KZTS0KeQheqxoLFBSk4pw/v7dPc0ug5TnVp4o+Y1PTdovNIUjByFzvQpAKRsVvf0Dmo6pl5TSFSfrf03PiFwwwdNr9Fo+jj29lPwlVL6Wf3ENDYpaRRSSpKyy4juS0KhKE3lATTWFED5FQpPQdYRtd+Z5iOAmvLm+yRb1z/yCrY9RqPRNMDefgpXCSH8rfYDhBBXOm5avZfcsiqKK2rOP6GQvAHevQoOftiwOikoLQEg2IZQMGz9x81mns6qS+PZD1w8Gz6jMdY+hOaa6Gg0mgbYGzD+eyllsbFjziP4vWOm1LtJMZzMoedZLPuB9+HEZvjfT+DZUfDd31T5CLAqWx3d9DrD1n96pypp3Fk1g4SwaAu2/AkAbuaieNBQQGg0mmaxVyjYGncexVN2Hsk5lsij84rMAzDiUrjlcxgcB1v+DIc/UefyElWWsq23ce8gZfOXps5zMhsYQqE5TQEswsBbm480GnuwVyjECyGeEUIME0JECSGeBfY5cmK9leScUvw8XM6vNpqVxaqz2aAJKnnrhg9UuOm3v4XKEshNsm06MjBs/p3lTzAIjlEF+Fpa8A1BpTUFjcYu7BUK9wHVwH+Bj4AK4F5HTao3k5xdRnSo7/mTqQyWgnNG3oGTMyz9h0oe++6vSlMIiWn+euNNvrMijwwW/RHu+KblMYYw0I5mjcYu7E1eKwds9kPQNCQlp4yLensJ7MYYRe2sbfdhk2HSLbD73+aaRy1pCubrOlsouHmrn5bwNDQF7WjWaOzB3uijDUKIAKv9fkKI9Y6bVu8kv6yK/PLqnpWjcOYH1UugI2QesJ0LsPAJVcIaWtYUIqYrf4SthjmORvsUNJo2Ya/5KNi6cqm5/4Hu0dyIlJweFnlUWQxvLoUdz3bsPhkHbOcCeAfBoj+p1pcDxjV/vbuPas8ZNKxj82gP9eYj7VPQaOzBXqFgEkLUl7UQQgzBRtXUvk6PizxK/R5MNZaQ0fZQWQwFJ5qP8Jl0C/wqree+ifsNBIRqqqPRaFrF3rDSx4AdQojvzPtzgJWOmVLvJSWnDG83Zwb6e3T3VBQnt6ptfkr775F5SG2bywUA1cmspzLuOuXv8NGKrUZjD/aWufgGiAMSURFID6EikDRWJGWXMrwnRR4ZQqEoDWqr2neP+q5mLeQC9GRcPXX7TI2mDdjraL4L2IQSBg8B7wJPOG5avYvKmjqeWJvAzhP5TI7oIfX2SzJUqGjoWBUdVJjavvtk7Ae/sJ5rHtJoNJ2KvT6FnwNTgDQp5XxgItA7miV3Et8cyeRMwbkmx0/klrHs3zt5a2cqd84eyqMXj+iG2dngpNnSN/UutW2vCSnjQO/VEjQaTZuxVyhUSikrAYQQ7lLK40ALgekKIcQSIUSiECJFCNEkz0EIcZsQIlcIccD8c1fbpt81HMss4Z7//MhFz37HK9tOUFtnIq+siie/OMrF/9xOZnEFr6+I43eXjsLNpRP6D3cGJ7eqhK1RV6j99ggFw8k8UAsFjaavYK+jOd2cp/A5sEEIUUgr7TiFEM7AauAiVC/mvUKItVLKo42G/ldKuaqN8+5SvktSStHUoUH837rjfBSfTmZRBRU1dVw9OYwHL4plQE9xLoMqVHdyK0TNVdVEvUMslUzbguFk1pqCRtNnsDej+SrzxyeEEFsAf6CV+gJMBVKklCcBhBAfAlcAjYVCj2dbUi4jBvjy9u1TWJ+QxT83JjNvRH8evCiGYSE9JPzUmtxEKMuyNL0JGm7pU9wWDCez1hQ0mj5DmyudSim/a30UAIOBM1b76cA0G+OWCyHmAEnAA1LKMzbGdBvlVbXsTS3gjllDEUKwZMxAlozp4b16jaijqHlqGzQMkr5t+31Sd6h2mj4hnTQxjUbT03GkAdxWXGbjhLcvgCFSynHARuBtmzcSYqUQIl4IEZ+b27X+7V0n8qmpk8yJ6UUL48mtEBhlaUofNBzKc5SPwF4qCiFlE4y63CFT1Gg0PRNHCoV0wLoCWhiN/BBSynwppRFA/yow2daNpJSvSCnjpJRxISFduzh/l5SLp6szcUN6SKhpa9TVqjf8qHmWY0Hm5jdtcTYf/0plQ49e1pmz02g0PRxHCoW9QLQQYqgQwg24HlhrPUAIYW2HuRw45sD5tIttybnMHBaEu4tzd0/FPrIOQnUpDJltORY0XG3b4lc48hkERMLgSZ07P41G06NxmFCQUtYCq4D1qMX+IyllghDiSSGEYZO4XwiRIIQ4CNwP3Oao+bSH1Lxy0vLPMTe2F5mO0napbcRMy7HAoSCc7NcUyvOUCWrMMtX2UqPR9Bkc2lJTSrkOWNfo2ONWn38N/NqRc+gIRijqnOheJBRO74J+Q82F4My4uCv/gr1hqcfWgqyDMcsdM0eNRtNj6SGZVj2TbUm5RAZ5MSS4lUYuPQUpIW0nRM5sei5ouP2awpHPlB8idEznzk+j0fR4tFBohqraOnaeyGdub4o6yk2EigKIsFEAzshVkOYAsB9ehUQbqSalWcpRPWa5Nh1pNH0Qh5qPejPv7T5NRU0di0YN6O6p2M/pnWrbnKZQU64W/ZSNsO5hZVKKWdxw8U/4HJDKn6DRaPocWlOwQU5pJc9uSGJOTAizhveijl1pu1QzmcCopueMCKRDH8JXD6rSF0Wn4ey+huMOfagqq4a0WtpKo9Gch2ihYIO/fp1IZW0dT1w2quf0RrCH07uU6cjWnA2hsPEJ8B0Ad24AZzflPzDIOKBKZU+6tUumq9Foeh5aKDRiX1oBn/6Yzl0XRBHVE+saNUfRaSg+Y9t0BOA3GFw8wMUTrntPhakOvxAS/gcmkxrz49tqzLhru27eGo2mR6GFghV1JsnvPk9goL8H9y0Y3t3TaRv1+QnNdBlzcoILn4Dr3oWB49SxMcuhNAPO7IaqMjj0scpg9gzoihlrNJoeiHY0W7HxWDZHM0v41/UT8HLrZf80p3eCuz+Ejm5+zPSfNtyPWaI0hyOfqnDV6lKYfJtDp6nRaHo2vWzlcyxrD2QQ5O3GJWN7eBVUW6Ttgohp4NSGchzuPhCzCI6uUQ7nkJEQPtVxc9RoND0ebT4yU1pZw8Zj2Vw6biAuzr3sn+VcgerH3JzpqCXGLIfyXOVgnnybzk3QaPo4WlMwsz4hm6paE5dPGNzdU2k72UfUtj0d0qIXgZsPmGq1g1mj0WihYLDmwFnCAz2ZFOFgJ2thmkogi7DVb6id5JiLy/Yf1fZrXT1h7i9BmsArsPPmpNFoeiW9zE7iGHJKK/k+JY8rxg92fF7C5j/CG4tg+z8sJSc6Ss4x1YvZJ7R918/6Ocx+oHPmotFoejVaKABfHcrEJOGKCYMc/7DSLFXGetOT8L97oLaq9WtaI+eY0hK0P0Cj0XQQLRSANQcyGDXQj+hQX8c/rCwHRjkp4j4AABKoSURBVFwC8x9TJSX+e3PH7ielEgohIzpnfhqNpk/T54VCWn45B84UdY2WAKpXsk+osuNPu0f1QTYyittDSQZUFUP/kZ03R41G02dxqFAQQiwRQiQKIVKEEI+2MO5qIYQUQsQ5cj62MBrpXDymC3IT6mqgolAVowPoN0Q1s6ksav4akwmyjzYvOHI74GTWaDSaRjhMKAghnIHVwMXAKOAGIUSTlUsI4YtqxbnHUXNpib2phQzw8yA80NPxDyvPU1tDKBhb47gtEtfBizPg+Ynw/XMqJ8Ga+sgjrSloNJqO40hNYSqQIqU8KaWsBj4ErrAx7o/A34BKB86lWfalFjB5SL+uqYZanqO2Pv3V1jvYfDy3+WuyDinHtO8g2PA7+OdYyLPqoJZzTJmjdDipRqPpBBwpFAYDZ6z2083H6hFCTATCpZRfOnAezXK2qIKM4kqmRPbrmgcai7+hIXjZIRTyU1QznDu+hpXfQU0FHHjPcj7nmNYSNBpNp+FIoWDr1bs+MF8I4QQ8CzzU6o2EWCmEiBdCxOfmtrCAtpH4VGWKiRvSRW/ZZY2EQr35qBWhYPRCGDQBouZCwmcq6shkgtzj2p+g0Wg6DUcKhXQg3Go/DMiw2vcFxgBbhRCpwHRgrS1ns5TyFSllnJQyLiSk83omx6cW4u3mzIgBXRCKCk3NR17mrm7N+RSkVH2Vg6zKeI9eBoWpkPEjFKVBzTkdjqrRaDoNRwqFvUC0EGKoEMINuB5Ya5yUUhZLKYOllEOklEOA3cDlUsp4B86pAfFphUyM6Nd1BfDKc1UTGzdz8x5nF/AMbF5TKM2C6rKGQmHkpeDkqjqmdaS8hUaj0djAYauhlLIWWAWsB44BH0kpE4QQTwohLnfUc+2lpLKG41klxA3pIn8CKPORd/+GmcfeIc0LhXyzQ9laKHj2g+ELVce0nAR1TPdT1mg0nYRDC+JJKdcB6xode7yZsfMcOZfG7D9dhJQQF9mFUTvlOeDTyPzlHQLn8m2PtyUUQJW7TvoG9r8H/hHg4df5c9VoNH2SPpvRHJ9agLOTYIKjq6JaU55rcS4beAe3rCm4eKr+ytbEXqzMUIWnoL/2J2g0ms6jDwuFQkYO9MXHvQurh5e1QygEDVP9la1x91V9EECHo2o0mk6lTwqFmjoT+88Udq3pyGRSi78ReWTgHaJKX9TVNL3GEAq2GLNMbbWTWaPRdCJ9UigczSihssbUtU7myiJV58iWpgBN/Qp1NSr0tLE/wWDEZXDJP2Bkt/vsNRrNeUSfFAoJGSUAjA/rQn9CmTlHoYlQaCaBrei0apHZnFBwdoEpd4GbV+fOU6PR9Gn6pFDIK1ONbUL9PLruoY0T1wyaEwp5yWobFO3YeWk0Go0VfVYo+Hu64ubShV+/cd0jg+YqpdaHozbjU9BoNBoH0CeFQn5ZNcE+bl370Pq6R401BaMong2h4Bmoq59qNJoupU8KhdyyKoJ83Lv2oeU5IJxVRrI1HgHg5NLUfGRdCE+j0Wi6iD4pFPLLqgjpcqGQq7SCxjkHQqgS2raEQrD2J2g0mq6lTwqFvLJqgrrDfNTYdGTgHdLQfFRVBqWZ2p+g0Wi6nD4nFKprTRRX1BDcHeYjw3/QmMZZzQUn1FabjzQaTRfT54RCQXk1QDcIBRvZzAaNK6XWh6NqoaDRaLqWPicUjByFDpmPTm2DcwX2j5fSdt0jg8bmo4z94OymhYJGo+ly+qxQaLemUFUG71wJ//uJWuztoboMaitaEArBUFMO1efU/uldMDgOXLpYm9FoNH2ePigUDPNROzWFwlOqhlHyt5D4tX3XGKahZs1HRv2jPKguh8yDEDmjffPTaDSaDuBQoSCEWCKESBRCpAghHrVx/h4hxGEhxAEhxA4hhMNLfnZYUyg4qbYeAfDNr6CmovVrmktcM7AudZG+V9U8ipjZvvlpNBpNB3CYUBBCOAOrgYuBUcANNhb996WUY6WUE4C/Ac84aj4G+WVVeLo6493ePgqGULjyRVW0bsc/W7/GqHvUbPSRVamLtF0gnCB8avvmp9FoNB3AkZrCVCBFSnlSSlkNfAhcYT1ASllitesN2Gmkbz8dzlEoOKkW8RFLVVvMHc9C/omWr7HXfFSeC6d3QugY3WJTo9F0C44UCoOBM1b76eZjDRBC3CuEOIHSFO534HwAZT7qUDhqwSkIjFKfF/1JRQn9ewZ8/jM4+6PtawzzkVcrmkJJBpzZC5Gz2j8/jUaj6QCOFArCxrEmmoCUcrWUchjwK+C3Nm8kxEohRLwQIj43t5nWlXaS17gYXnYCPDcRSrPtu0HBSYtQ8BsEK7fAxJsg4XN4dT5sfKLpNeU5ygfh0oyG4uYNrl6QslFFKWkns0aj6SYcKRTSgXCr/TAgo4XxHwJX2johpXxFShknpYwLCWkmrNNOmmgKp7arhT7zQOsX11RAyVmLUABVn+jSZ+GhYzDqSti1GkqzGl5Xlt286cjAOxjO/KA+R2ihoNFougdHCoW9QLQQYqgQwg24HlhrPUAIYV3x7RIg2YHzwWSSFJQ38ikYfQsKU1u/gTHGWigYePjDhb9XkUO7XrAcLzgJSd9C2JSW7+0VDEiVsNaaANFoNBoH4TChIKWsBVYB64FjwEdSygQhxJNCCKOx8CohRIIQ4gDwILDCUfMBKKqooc4kG2oKbREKRuRR4FDb5wOjYMzVsPcNS8bz14+Csyss+F3L9zb8CpE6FFWj0XQf7YzLtA8p5Trg/9u79yAvqzqO4+8P4CKwIAshIZgLaiqmgJBK4mXUmSwd0SbHLhZTVlNjk5pN2mWstKabVjY5aWMXG53KyEqNtELHrBkpEErFTPO6arJcRFfkInz745zfj99uewP3tz94ns9rZmf3efbZ33POnN3fZ5/zPOecRV32XVrz9fn1PH9Xa7obo1B5cmiHQqGbK4WKeRfC/TfBkmthn5nwyB3phvSYSb2/diUUPD7BzBqorqGwq2nvOu/RlldgfX5Aqr+hMKLl/xfKqTVxOhx0Kiy5JnUpTTgYjvpo369deSzVN5nNrIFKFQpr8hQX1QV21j4GBDS/PoVCRFr0pie1Tx715tiL4OHfwcYXYMGtqfuoL4efDcObYex+fR9rZlYnpQqF7TOk5lCo3E844GRYcUPv01tDCoV9j+r7RFNmw6xzYPgYmHpc/wo3cXr6MDNroFJNiLe6YxNDh4ixI/J/7tVQOCl97q0L6dVNsL6tf1cKAPOvhlO+utNlNTNrhFKFwpqOzYwf1cSQIbmLaM1/YPSkNK0EdA6Fjna4akYaxwBpnqPY1v9QMDPbDZUqFFZ3bNredQRphbPxB8DYN6Tt2lB44p60fefl6V5Df548MjPbzZUnFNr/zVHtCztPcbHmURi/P+yxJ4zep3MotC1Nn59eAk/+1aFgZqVQnlB45A4+3HENBzfluZM2rIVX1sL4PKi6pbVzKDyzFCbNTGsg3HNlCoXhY2Dk+MEuuZnZoClNKMT0NK3SMRv/nHZUBq1V1kGuDYVXN8OzK6B1Hsw9D/5zZ1plbdzU3h9ZNTPbzZUmFDaMmMTSbW/ksBcWpx1r8jRLtaHw4rOwZSM8fz9s3QRT5sCcD6ZBaOufdteRmRVeaUJhdccmbt06l/EvPwqr/pXuJwwZBi15sFhLKxDpzb9tWdo35c1psZvKiGSHgpkVXIlCYTOLth5JIHjw5hQKLa3bRxu3tKbP655I6ySPngRj8ppAR30UJh4GU49vQMnNzAZPaUY0r+7YRDstvDxpLs0P/AqGDt/edQT/HwqTZ2+/fzByHHzsL4NdZDOzQVeiK4U0xcXW6Wemq4RVKzuHQvPeMGwEPLMM1j3e9/oHZmYFVJpQ2LhlG03DhjBixpmgoaQFbfbffoCUrhYeui1tOxTMrIRKEwrnzpvKw5efQtOYCTDthLRz/IGdD2pphc0vpdDYZ+Ygl9DMrPHqGgqSTpH0sKRHJV3Szfc/KWmlpH9KWiyprvNGq3KPYPYCaGqGiYd2PqByX2HidGgaVc+imJntkuoWCpKGAlcDbwOmA++W1HVu6OXAnIg4HFgIfKNe5elk+ny4+Ml0A7lWJRTcdWRmJVXPK4UjgUcj4rGI2Az8HJhfe0BE3BURG/LmvcCUOpans6HdPHhVCYXJcwatGGZmu5J6hsJk4Oma7ba8ryfnAr/v7huSPiJpqaSl7e3tA1jELqYeC3M/DoecVr9zmJntwuoZCt1NEhTdHiidA8wBvtnd9yPiBxExJyLmTJgwYQCL2EXTKHjrV9K0FmZmJVTPwWttwL4121OAZ7seJOlk4HPA8RGxqY7lMTOzPtTzSuHvwIGSpkpqAt4F3FJ7gKRZwLXA6RGxqo5lMTOzfqhbKETEq8DHgTuAh4CbIuJBSZdJOj0f9k2gGfilpBWSbunh5czMbBDUde6jiFgELOqy79Kar0+u5/nNzGzHlGZEs5mZ9c2hYGZmVQ4FMzOrciiYmVmVIrodT7bLktQOPLmTP/46YPUAFmd3UcZ6l7HOUM56l7HOsOP13i8i+hz9u9uFwmshaWlElG5iozLWu4x1hnLWu4x1hvrV291HZmZW5VAwM7OqsoXCDxpdgAYpY73LWGcoZ73LWGeoU71LdU/BzMx6V7YrBTMz60VpQqGv9aKLQNK+ku6S9JCkByWdn/ePk/RHSY/kzy2NLutAkzRU0nJJt+XtqZKW5Dr/Is/UWyiSxkpaKOlfuc3nlqStL8y/3w9I+pmkPYvW3pJ+JGmVpAdq9nXbtkq+m9/b/inpiNdy7lKEQj/Xiy6CV4GLIuIQ4GjgvFzPS4DFEXEgsDhvF835pNl4K74OfDvXeR1pZb+iuQq4PSIOBmaQ6l/otpY0GfgEaW33NwFDSdPyF629fwKc0mVfT237NuDA/PER4Puv5cSlCAX6sV50EUTEcxFxX/76JdKbxGRSXa/Ph10PnNGYEtaHpCnAqcB1eVvAicDCfEgR6zwGOA74IUBEbI6IFyh4W2fDgBGShgEjgecoWHtHxJ+BtV1299S284GfRnIvMFbSpJ09d1lCYUfXi97tSWoFZgFLgIkR8Ryk4AD2blzJ6uI7wKeBbXl7PPBCXtMDitne04B24Me52+w6SaMoeFtHxDPAFcBTpDBYDyyj+O0NPbftgL6/lSUU+r1edBFIagZ+BVwQES82ujz1JOk0YFVELKvd3c2hRWvvYcARwPcjYhbwMgXrKupO7kefD0wF9gFGkbpPuipae/dmQH/fyxIK/Vovuggk7UEKhBsj4ua8+/nK5WT+XKSlT48BTpf0BKlb8ETSlcPY3L0AxWzvNqAtIpbk7YWkkChyWwOcDDweEe0RsQW4GXgLxW9v6LltB/T9rSyh0Od60UWQ+9J/CDwUEd+q+dYtwIL89QLgt4NdtnqJiM9ExJSIaCW1650R8V7gLuCd+bBC1RkgIv4LPC3poLzrJGAlBW7r7CngaEkj8+97pd6Fbu+sp7a9BXh/fgrpaGB9pZtpZ5Rm8Jqkt5P+gxwK/CgivtLgIg04SfOAe4D72d6//lnSfYWbgDeQ/qjOioiuN7F2e5JOAD4VEadJmka6chgHLAfOiYhNjSzfQJM0k3RzvQl4DPgA6R+9Qre1pC8BZ5OetlsOfIjUh16Y9pb0M+AE0kyozwNfAH5DN22bw/F7pKeVNgAfiIilO33usoSCmZn1rSzdR2Zm1g8OBTMzq3IomJlZlUPBzMyqHApmZlblUDCrM0knVGZvNdvVORTMzKzKoWCWSTpH0t8krZB0bV6joUPSlZLuk7RY0oR87ExJ9+b5639dM7f9AZL+JOkf+Wf2zy/fXLP2wY15wBGSviZpZX6dKxpUdbMqh4IZIOkQ0ijZYyJiJrAVeC9pwrX7IuII4G7SyFKAnwIXR8ThpBHklf03AldHxAzSnDyV6QZmAReQ1vOYBhwjaRxwJnBofp0v17eWZn1zKJglJwGzgb9LWpG3p5GmC/lFPuYGYJ6kvYCxEXF33n89cJyk0cDkiPg1QERsjIgN+Zi/RURbRGwDVgCtwIvARuA6Se8gTVFg1lAOBbNEwPURMTN/HBQRX+zmuN7mheluCuOK2nl4tgLD8vz/R5JmtT0DuH0Hy2w24BwKZsli4J2S9obqerj7kf5GKrNvvgf4S0SsB9ZJOjbvfx9wd167ok3SGfk1hksa2dMJ87oXe0XEIlLX0sx6VMxsRwzr+xCz4ouIlZI+D/xB0hBgC3AeafGaQyUtI63ydXb+kQXANflNvzJDKaSAuFbSZfk1zurltKOB30rak3SVceEAV8tsh3mWVLNeSOqIiOZGl8NssLj7yMzMqnylYGZmVb5SMDOzKoeCmZlVORTMzKzKoWBmZlUOBTMzq3IomJlZ1f8AFvktEqyWtsIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fcfe0041ac8>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[20  5  0  0  6]\n",
      " [ 0 38  0  3  4]\n",
      " [ 1 13 25  0  2]\n",
      " [ 0  6  1 29  3]\n",
      " [ 1  4  0  2 30]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        calm       0.91      0.65      0.75        31\n",
      "       happy       0.58      0.84      0.68        45\n",
      "       angry       0.96      0.61      0.75        41\n",
      "     fearful       0.85      0.74      0.79        39\n",
      "         sad       0.67      0.81      0.73        37\n",
      "\n",
      "    accuracy                           0.74       193\n",
      "   macro avg       0.79      0.73      0.74       193\n",
      "weighted avg       0.78      0.74      0.74       193\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "plt.plot(list(range(0, len(logger.training))), list(map(lambda x:x[1], logger.training)), label=\"training data\")\n",
    "plt.plot(list(range(0, len(logger.testing))), list(map(lambda x:x[1], logger.testing)), label=\"testing data\")\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('accuracy')\n",
    "plt.show()\n",
    "print_metrics(vm, X_test, Y_test, emotion_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this graph we can see that the model largely overfit around the 20st epoch\n",
    "We can also see that the testing data's accuracy continues to grow."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we save the model so we don't have to retrain it each time\n",
    "vm.model.save(\"emotion_lstm.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we save the model so we don't have to retrain it each time\n",
    "vm.model.load(\"emotion_lstm.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Doing calm\n",
      "Doing happy\n",
      "Doing angry\n",
      "Doing fearful\n",
      "Doing sad\n"
     ]
    }
   ],
   "source": [
    "# Now we load the new dataset (savee)\n",
    "\n",
    "# First we change the folder path\n",
    "folder_path = \"../../data/savee\"\n",
    "\n",
    "# Now we load the data\n",
    "data = {f: load_wav(f\"{folder_path}/{f}\", nb_break) for f in emotion_list}\n",
    "X_savee, Y_savee = preprare_wav(data, vm, sample_duration, step)\n",
    "X_savee = X_savee.astype('float32')\n",
    "X_savee_train, X_savee_test, Y_savee_train, Y_savee_test = train_test_split(X_savee, Y_savee, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(f\"X_savee_{emotion_list}-{vm._name}\", X_savee)\n",
    "np.save(f\"Y_savee_{emotion_list}-{vm._name}\", Y_savee)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "360/360 [==============================] - 1s 2ms/sample - loss: 6.2077 - acc: 0.2222\n",
      "[[ 9 47  5  0  9]\n",
      " [ 9 48 11  0  4]\n",
      " [ 5 48 12  1  2]\n",
      " [ 6 40 14  2  3]\n",
      " [11 59  6  0  9]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        calm       0.23      0.13      0.16        70\n",
      "       happy       0.20      0.67      0.31        72\n",
      "       angry       0.25      0.18      0.21        68\n",
      "     fearful       0.67      0.03      0.06        65\n",
      "         sad       0.33      0.11      0.16        85\n",
      "\n",
      "    accuracy                           0.22       360\n",
      "   macro avg       0.33      0.22      0.18       360\n",
      "weighted avg       0.33      0.22      0.18       360\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vm.model._model.evaluate(X_savee, Y_savee)\n",
    "print_metrics(vm, X_savee, Y_savee, emotion_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/arc/fdp5/.local/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /home/arc/fdp5/.local/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:97: calling Ones.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /home/arc/fdp5/.local/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /home/arc/fdp5/.local/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:97: calling Orthogonal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "gaussian_noise_2\n",
      "batch_normalization_2\n",
      "lstm_1\n",
      "activation_2\n",
      "flatten_1\n",
      "batch_normalization_3\n",
      "gaussian_noise_3\n"
     ]
    }
   ],
   "source": [
    "# Now we freeze every layer used for extracting features from data\n",
    "savee_model = keras.models.clone_model(vm.model._model)\n",
    "for layer in savee_model.layers[:-3]:\n",
    "    print(layer.name)\n",
    "    layer.trainable = False\n",
    "savee_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "savee_cls = EmotionClassifierLstm()\n",
    "savee_cls._model = savee_model\n",
    "savee_vm = VoiceModule(\"emotion\", emotion_list, savee_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 288 samples, validate on 72 samples\n",
      "Epoch 1/100\n",
      "288/288 [==============================] - 3s 11ms/sample - loss: 1.9156 - acc: 0.2014 - val_loss: 1.8587 - val_acc: 0.1111\n",
      "Epoch 2/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.9065 - acc: 0.1840 - val_loss: 1.8603 - val_acc: 0.2639\n",
      "Epoch 3/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8898 - acc: 0.2153 - val_loss: 1.8613 - val_acc: 0.2639\n",
      "Epoch 4/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.9251 - acc: 0.1910 - val_loss: 1.8615 - val_acc: 0.2639\n",
      "Epoch 5/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.9054 - acc: 0.1806 - val_loss: 1.8635 - val_acc: 0.2639\n",
      "Epoch 6/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8757 - acc: 0.2431 - val_loss: 1.8599 - val_acc: 0.2639\n",
      "Epoch 7/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8984 - acc: 0.1562 - val_loss: 1.8597 - val_acc: 0.2639\n",
      "Epoch 8/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8925 - acc: 0.2222 - val_loss: 1.8630 - val_acc: 0.2639\n",
      "Epoch 9/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8789 - acc: 0.1944 - val_loss: 1.8590 - val_acc: 0.2639\n",
      "Epoch 10/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8917 - acc: 0.2083 - val_loss: 1.8617 - val_acc: 0.2639\n",
      "Epoch 11/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8846 - acc: 0.2292 - val_loss: 1.8603 - val_acc: 0.2639\n",
      "Epoch 12/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8920 - acc: 0.2292 - val_loss: 1.8592 - val_acc: 0.2639\n",
      "Epoch 13/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8654 - acc: 0.2083 - val_loss: 1.8607 - val_acc: 0.2639\n",
      "Epoch 14/100\n",
      "288/288 [==============================] - 2s 7ms/sample - loss: 1.8748 - acc: 0.2014 - val_loss: 1.8617 - val_acc: 0.2639\n",
      "Epoch 15/100\n",
      "288/288 [==============================] - 2s 7ms/sample - loss: 1.8911 - acc: 0.1667 - val_loss: 1.8633 - val_acc: 0.2639\n",
      "Epoch 16/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.9032 - acc: 0.2049 - val_loss: 1.8620 - val_acc: 0.2639\n",
      "Epoch 17/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8762 - acc: 0.2118 - val_loss: 1.8641 - val_acc: 0.2639\n",
      "Epoch 18/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.9044 - acc: 0.1771 - val_loss: 1.8640 - val_acc: 0.2639\n",
      "Epoch 19/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8759 - acc: 0.2083 - val_loss: 1.8602 - val_acc: 0.2639\n",
      "Epoch 20/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8791 - acc: 0.1910 - val_loss: 1.8616 - val_acc: 0.2639\n",
      "Epoch 21/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8881 - acc: 0.1771 - val_loss: 1.8615 - val_acc: 0.2639\n",
      "Epoch 22/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8919 - acc: 0.1875 - val_loss: 1.8614 - val_acc: 0.2639\n",
      "Epoch 23/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8785 - acc: 0.2292 - val_loss: 1.8627 - val_acc: 0.2639\n",
      "Epoch 24/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8814 - acc: 0.2153 - val_loss: 1.8610 - val_acc: 0.2639\n",
      "Epoch 25/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8869 - acc: 0.2083 - val_loss: 1.8617 - val_acc: 0.2639\n",
      "Epoch 26/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8940 - acc: 0.2083 - val_loss: 1.8587 - val_acc: 0.2639\n",
      "Epoch 27/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8783 - acc: 0.1840 - val_loss: 1.8604 - val_acc: 0.2639\n",
      "Epoch 28/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8719 - acc: 0.1979 - val_loss: 1.8613 - val_acc: 0.2639\n",
      "Epoch 29/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8704 - acc: 0.2083 - val_loss: 1.8596 - val_acc: 0.2639\n",
      "Epoch 30/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8802 - acc: 0.1910 - val_loss: 1.8604 - val_acc: 0.2639\n",
      "Epoch 31/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8716 - acc: 0.2014 - val_loss: 1.8614 - val_acc: 0.2639\n",
      "Epoch 32/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8795 - acc: 0.1979 - val_loss: 1.8631 - val_acc: 0.2639\n",
      "Epoch 33/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8912 - acc: 0.2083 - val_loss: 1.8609 - val_acc: 0.2639\n",
      "Epoch 34/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8674 - acc: 0.2049 - val_loss: 1.8600 - val_acc: 0.2639\n",
      "Epoch 35/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8913 - acc: 0.1701 - val_loss: 1.8588 - val_acc: 0.2639\n",
      "Epoch 36/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8700 - acc: 0.2083 - val_loss: 1.8607 - val_acc: 0.2639\n",
      "Epoch 37/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8874 - acc: 0.2049 - val_loss: 1.8602 - val_acc: 0.2639\n",
      "Epoch 38/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8814 - acc: 0.1840 - val_loss: 1.8588 - val_acc: 0.2639\n",
      "Epoch 39/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8856 - acc: 0.1875 - val_loss: 1.8624 - val_acc: 0.2639\n",
      "Epoch 40/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8708 - acc: 0.1979 - val_loss: 1.8610 - val_acc: 0.2639\n",
      "Epoch 41/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8767 - acc: 0.2083 - val_loss: 1.8607 - val_acc: 0.2639\n",
      "Epoch 42/100\n",
      "288/288 [==============================] - 2s 7ms/sample - loss: 1.8662 - acc: 0.2326 - val_loss: 1.8606 - val_acc: 0.2639\n",
      "Epoch 43/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8750 - acc: 0.1944 - val_loss: 1.8615 - val_acc: 0.2639\n",
      "Epoch 44/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8713 - acc: 0.2083 - val_loss: 1.8589 - val_acc: 0.2639\n",
      "Epoch 45/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8832 - acc: 0.1979 - val_loss: 1.8619 - val_acc: 0.2639\n",
      "Epoch 46/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8822 - acc: 0.1736 - val_loss: 1.8609 - val_acc: 0.2639\n",
      "Epoch 47/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8706 - acc: 0.2361 - val_loss: 1.8606 - val_acc: 0.2639\n",
      "Epoch 48/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8751 - acc: 0.1875 - val_loss: 1.8597 - val_acc: 0.2639\n",
      "Epoch 49/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8627 - acc: 0.2465 - val_loss: 1.8607 - val_acc: 0.2639\n",
      "Epoch 50/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8698 - acc: 0.2083 - val_loss: 1.8619 - val_acc: 0.2639\n",
      "Epoch 51/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.9012 - acc: 0.1632 - val_loss: 1.8613 - val_acc: 0.2639\n",
      "Epoch 52/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8694 - acc: 0.2257 - val_loss: 1.8600 - val_acc: 0.2639\n",
      "Epoch 53/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8738 - acc: 0.2292 - val_loss: 1.8610 - val_acc: 0.2639\n",
      "Epoch 54/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8805 - acc: 0.1736 - val_loss: 1.8606 - val_acc: 0.2639\n",
      "Epoch 55/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8762 - acc: 0.2188 - val_loss: 1.8598 - val_acc: 0.2639\n",
      "Epoch 56/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8651 - acc: 0.2188 - val_loss: 1.8623 - val_acc: 0.2639\n",
      "Epoch 57/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8638 - acc: 0.2188 - val_loss: 1.8613 - val_acc: 0.2639\n",
      "Epoch 58/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8796 - acc: 0.2153 - val_loss: 1.8633 - val_acc: 0.2639\n",
      "Epoch 59/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8696 - acc: 0.2153 - val_loss: 1.8611 - val_acc: 0.2639\n",
      "Epoch 60/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8752 - acc: 0.2257 - val_loss: 1.8603 - val_acc: 0.2639\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 61/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8771 - acc: 0.1875 - val_loss: 1.8628 - val_acc: 0.2639\n",
      "Epoch 62/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8723 - acc: 0.2083 - val_loss: 1.8612 - val_acc: 0.2639\n",
      "Epoch 63/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8803 - acc: 0.2049 - val_loss: 1.8622 - val_acc: 0.2639\n",
      "Epoch 64/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8736 - acc: 0.1979 - val_loss: 1.8624 - val_acc: 0.2639\n",
      "Epoch 65/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8543 - acc: 0.2882 - val_loss: 1.8630 - val_acc: 0.2639\n",
      "Epoch 66/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8761 - acc: 0.2326 - val_loss: 1.8614 - val_acc: 0.2639\n",
      "Epoch 67/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8695 - acc: 0.2188 - val_loss: 1.8623 - val_acc: 0.2639\n",
      "Epoch 68/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8738 - acc: 0.2118 - val_loss: 1.8613 - val_acc: 0.2639\n",
      "Epoch 69/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8647 - acc: 0.2465 - val_loss: 1.8624 - val_acc: 0.2639\n",
      "Epoch 70/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8789 - acc: 0.1840 - val_loss: 1.8609 - val_acc: 0.2639\n",
      "Epoch 71/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8743 - acc: 0.2153 - val_loss: 1.8607 - val_acc: 0.2639\n",
      "Epoch 72/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8750 - acc: 0.1875 - val_loss: 1.8603 - val_acc: 0.2639\n",
      "Epoch 73/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8655 - acc: 0.2222 - val_loss: 1.8601 - val_acc: 0.2639\n",
      "Epoch 74/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8770 - acc: 0.2083 - val_loss: 1.8594 - val_acc: 0.2639\n",
      "Epoch 75/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8649 - acc: 0.2326 - val_loss: 1.8603 - val_acc: 0.2639\n",
      "Epoch 76/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8779 - acc: 0.1840 - val_loss: 1.8606 - val_acc: 0.2639\n",
      "Epoch 77/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8765 - acc: 0.2118 - val_loss: 1.8597 - val_acc: 0.2639\n",
      "Epoch 78/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8640 - acc: 0.2396 - val_loss: 1.8592 - val_acc: 0.2639\n",
      "Epoch 79/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8962 - acc: 0.1840 - val_loss: 1.8617 - val_acc: 0.2639\n",
      "Epoch 80/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8821 - acc: 0.1806 - val_loss: 1.8600 - val_acc: 0.2639\n",
      "Epoch 81/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8631 - acc: 0.2222 - val_loss: 1.8604 - val_acc: 0.2639\n",
      "Epoch 82/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8739 - acc: 0.2014 - val_loss: 1.8604 - val_acc: 0.2639\n",
      "Epoch 83/100\n",
      "288/288 [==============================] - 1s 4ms/sample - loss: 1.8711 - acc: 0.2153 - val_loss: 1.8604 - val_acc: 0.2639\n",
      "Epoch 84/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8716 - acc: 0.1840 - val_loss: 1.8606 - val_acc: 0.2639\n",
      "Epoch 85/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8620 - acc: 0.2083 - val_loss: 1.8610 - val_acc: 0.2639\n",
      "Epoch 86/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8694 - acc: 0.2257 - val_loss: 1.8605 - val_acc: 0.2639\n",
      "Epoch 87/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8749 - acc: 0.2118 - val_loss: 1.8595 - val_acc: 0.2639\n",
      "Epoch 88/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8624 - acc: 0.2222 - val_loss: 1.8619 - val_acc: 0.2639\n",
      "Epoch 89/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8482 - acc: 0.2604 - val_loss: 1.8610 - val_acc: 0.2639\n",
      "Epoch 90/100\n",
      "288/288 [==============================] - 2s 7ms/sample - loss: 1.8757 - acc: 0.2153 - val_loss: 1.8606 - val_acc: 0.2639\n",
      "Epoch 91/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8598 - acc: 0.2431 - val_loss: 1.8603 - val_acc: 0.2639\n",
      "Epoch 92/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8621 - acc: 0.2431 - val_loss: 1.8606 - val_acc: 0.2639\n",
      "Epoch 93/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8585 - acc: 0.2222 - val_loss: 1.8614 - val_acc: 0.2639\n",
      "Epoch 94/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8758 - acc: 0.1875 - val_loss: 1.8618 - val_acc: 0.2639\n",
      "Epoch 95/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8787 - acc: 0.2083 - val_loss: 1.8603 - val_acc: 0.2639\n",
      "Epoch 96/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8664 - acc: 0.2292 - val_loss: 1.8606 - val_acc: 0.2639\n",
      "Epoch 97/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8764 - acc: 0.2118 - val_loss: 1.8612 - val_acc: 0.2639\n",
      "Epoch 98/100\n",
      "288/288 [==============================] - 2s 5ms/sample - loss: 1.8622 - acc: 0.2188 - val_loss: 1.8607 - val_acc: 0.2639\n",
      "Epoch 99/100\n",
      "288/288 [==============================] - 2s 6ms/sample - loss: 1.8714 - acc: 0.1840 - val_loss: 1.8611 - val_acc: 0.2639\n",
      "Epoch 100/100\n",
      "288/288 [==============================] - 1s 5ms/sample - loss: 1.8767 - acc: 0.1979 - val_loss: 1.8594 - val_acc: 0.2639\n"
     ]
    }
   ],
   "source": [
    "# And we train the two last layers\n",
    "# This is classic fine tuning\n",
    "savee_vm.model.train(X_savee_train, Y_savee_train, batch_size=10, validation_data=(X_savee_test, Y_savee_test), epoch=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0  0  0  0 16]\n",
      " [ 0  0  0  0 18]\n",
      " [ 0  0  0  0  9]\n",
      " [ 0  0  0  0 10]\n",
      " [ 0  0  0  0 19]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        calm       0.00      0.00      0.00        16\n",
      "       happy       0.00      0.00      0.00        18\n",
      "       angry       0.00      0.00      0.00         9\n",
      "     fearful       0.00      0.00      0.00        10\n",
      "         sad       0.26      1.00      0.42        19\n",
      "\n",
      "    accuracy                           0.26        72\n",
      "   macro avg       0.05      0.20      0.08        72\n",
      "weighted avg       0.07      0.26      0.11        72\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "print_metrics(savee_vm, X_savee_test, Y_savee_test, emotion_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Doing calm\n",
      "Doing happy\n",
      "Doing angry\n",
      "Doing fearful\n",
      "Doing sad\n"
     ]
    }
   ],
   "source": [
    "# Now we load the new dataset (savee)\n",
    "\n",
    "# First we change the folder path\n",
    "folder_path = \"../../data/bdes\"\n",
    "\n",
    "# Now we load the data\n",
    "data = {f: load_wav(f\"{folder_path}/{f}\", nb_break) for f in emotion_list}\n",
    "X_bdes, Y_bdes = preprare_wav(data, vm, sample_duration, step)\n",
    "X_bdes = X_bdes.astype('float32')\n",
    "X_bdes_train, X_bdes_test, Y_bdes_train, Y_bdes_test = train_test_split(X_bdes, Y_bdes, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(f\"X_bdes_{vm._name}\", X_bdes)\n",
    "np.save(f\"Y_bdes_{vm._name}\", Y_bdes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "388/388 [==============================] - 1s 3ms/sample - loss: 5.8306 - acc: 0.2861\n",
      "[[ 0 47 25  7  0]\n",
      " [ 1 33 34  3  0]\n",
      " [ 0 19 65  4  0]\n",
      " [ 0 21 40  8  0]\n",
      " [14 37 10 15  5]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        calm       0.00      0.00      0.00        79\n",
      "       happy       0.21      0.46      0.29        71\n",
      "       angry       0.37      0.74      0.50        88\n",
      "     fearful       0.22      0.12      0.15        69\n",
      "         sad       1.00      0.06      0.12        81\n",
      "\n",
      "    accuracy                           0.29       388\n",
      "   macro avg       0.36      0.28      0.21       388\n",
      "weighted avg       0.37      0.29      0.22       388\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vm.model._model.evaluate(X_bdes, Y_bdes)\n",
    "print_metrics(vm, X_bdes, Y_bdes, emotion_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gaussian_noise_2\n",
      "batch_normalization_2\n",
      "lstm_1\n",
      "activation_2\n",
      "flatten_1\n",
      "batch_normalization_3\n",
      "gaussian_noise_3\n"
     ]
    }
   ],
   "source": [
    "# Now we freeze every layer used for extracting features from data\n",
    "bdes_model = keras.models.clone_model(vm.model._model)\n",
    "for layer in bdes_model.layers[:-3]:\n",
    "    print(layer.name)\n",
    "    layer.trainable = False\n",
    "bdes_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "bdes_cls = EmotionClassifierLstm()\n",
    "bdes_cls._model = bdes_model\n",
    "bdes_vm = VoiceModule(\"emotion\", emotion_list, bdes_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 310 samples, validate on 78 samples\n",
      "Epoch 1/100\n",
      "310/310 [==============================] - 5s 16ms/sample - loss: 1.8981 - acc: 0.1613 - val_loss: 1.8674 - val_acc: 0.1923\n",
      "Epoch 2/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.9033 - acc: 0.2194 - val_loss: 1.8679 - val_acc: 0.1923\n",
      "Epoch 3/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.9133 - acc: 0.1710 - val_loss: 1.8677 - val_acc: 0.1923\n",
      "Epoch 4/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8736 - acc: 0.2258 - val_loss: 1.8699 - val_acc: 0.1923\n",
      "Epoch 5/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8972 - acc: 0.1839 - val_loss: 1.8706 - val_acc: 0.1923\n",
      "Epoch 6/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8684 - acc: 0.2387 - val_loss: 1.8723 - val_acc: 0.1923\n",
      "Epoch 7/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8908 - acc: 0.1871 - val_loss: 1.8741 - val_acc: 0.1923\n",
      "Epoch 8/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8971 - acc: 0.1935 - val_loss: 1.8742 - val_acc: 0.1923\n",
      "Epoch 9/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8949 - acc: 0.2032 - val_loss: 1.8734 - val_acc: 0.1923\n",
      "Epoch 10/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8645 - acc: 0.2419 - val_loss: 1.8731 - val_acc: 0.1923\n",
      "Epoch 11/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8817 - acc: 0.2226 - val_loss: 1.8749 - val_acc: 0.1923\n",
      "Epoch 12/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8850 - acc: 0.2194 - val_loss: 1.8724 - val_acc: 0.1923\n",
      "Epoch 13/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8693 - acc: 0.2065 - val_loss: 1.8756 - val_acc: 0.1923\n",
      "Epoch 14/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8741 - acc: 0.2290 - val_loss: 1.8752 - val_acc: 0.1923\n",
      "Epoch 15/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8762 - acc: 0.1903 - val_loss: 1.8764 - val_acc: 0.1923\n",
      "Epoch 16/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8859 - acc: 0.2194 - val_loss: 1.8761 - val_acc: 0.1923\n",
      "Epoch 17/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8634 - acc: 0.2484 - val_loss: 1.8760 - val_acc: 0.1923\n",
      "Epoch 18/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8900 - acc: 0.1903 - val_loss: 1.8777 - val_acc: 0.1923\n",
      "Epoch 19/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8904 - acc: 0.1903 - val_loss: 1.8758 - val_acc: 0.1923\n",
      "Epoch 20/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8760 - acc: 0.2161 - val_loss: 1.8757 - val_acc: 0.1923\n",
      "Epoch 21/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8745 - acc: 0.2097 - val_loss: 1.8751 - val_acc: 0.1923\n",
      "Epoch 22/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8879 - acc: 0.1903 - val_loss: 1.8766 - val_acc: 0.1923\n",
      "Epoch 23/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8791 - acc: 0.2226 - val_loss: 1.8761 - val_acc: 0.1923\n",
      "Epoch 24/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8767 - acc: 0.2161 - val_loss: 1.8783 - val_acc: 0.1923\n",
      "Epoch 25/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8644 - acc: 0.2419 - val_loss: 1.8744 - val_acc: 0.1923\n",
      "Epoch 26/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8842 - acc: 0.2226 - val_loss: 1.8756 - val_acc: 0.1923\n",
      "Epoch 27/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8735 - acc: 0.2484 - val_loss: 1.8748 - val_acc: 0.1923\n",
      "Epoch 28/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8677 - acc: 0.2548 - val_loss: 1.8768 - val_acc: 0.1923\n",
      "Epoch 29/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8792 - acc: 0.1903 - val_loss: 1.8776 - val_acc: 0.1923\n",
      "Epoch 30/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8701 - acc: 0.2548 - val_loss: 1.8762 - val_acc: 0.1923\n",
      "Epoch 31/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8718 - acc: 0.1903 - val_loss: 1.8744 - val_acc: 0.1923\n",
      "Epoch 32/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8873 - acc: 0.2097 - val_loss: 1.8774 - val_acc: 0.1923\n",
      "Epoch 33/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8658 - acc: 0.2323 - val_loss: 1.8777 - val_acc: 0.1923\n",
      "Epoch 34/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8756 - acc: 0.1968 - val_loss: 1.8766 - val_acc: 0.1923\n",
      "Epoch 35/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8847 - acc: 0.2194 - val_loss: 1.8769 - val_acc: 0.1923\n",
      "Epoch 36/100\n",
      "310/310 [==============================] - 3s 8ms/sample - loss: 1.8680 - acc: 0.2323 - val_loss: 1.8774 - val_acc: 0.1923\n",
      "Epoch 37/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8750 - acc: 0.2129 - val_loss: 1.8756 - val_acc: 0.1923\n",
      "Epoch 38/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8712 - acc: 0.2097 - val_loss: 1.8747 - val_acc: 0.1923\n",
      "Epoch 39/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8786 - acc: 0.2387 - val_loss: 1.8749 - val_acc: 0.1923\n",
      "Epoch 40/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8859 - acc: 0.2194 - val_loss: 1.8733 - val_acc: 0.1923\n",
      "Epoch 41/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8860 - acc: 0.2355 - val_loss: 1.8752 - val_acc: 0.1923\n",
      "Epoch 42/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8594 - acc: 0.2194 - val_loss: 1.8735 - val_acc: 0.1923\n",
      "Epoch 43/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8698 - acc: 0.2226 - val_loss: 1.8739 - val_acc: 0.1923\n",
      "Epoch 44/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8768 - acc: 0.2161 - val_loss: 1.8751 - val_acc: 0.1923\n",
      "Epoch 45/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8840 - acc: 0.1839 - val_loss: 1.8758 - val_acc: 0.1923\n",
      "Epoch 46/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8744 - acc: 0.2161 - val_loss: 1.8753 - val_acc: 0.1923\n",
      "Epoch 47/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8750 - acc: 0.2097 - val_loss: 1.8749 - val_acc: 0.1923\n",
      "Epoch 48/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8808 - acc: 0.1903 - val_loss: 1.8743 - val_acc: 0.1923\n",
      "Epoch 49/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8786 - acc: 0.2065 - val_loss: 1.8756 - val_acc: 0.1923\n",
      "Epoch 50/100\n",
      "310/310 [==============================] - 4s 12ms/sample - loss: 1.8702 - acc: 0.2097 - val_loss: 1.8755 - val_acc: 0.1923\n",
      "Epoch 51/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8709 - acc: 0.2161 - val_loss: 1.8743 - val_acc: 0.1923\n",
      "Epoch 52/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8537 - acc: 0.2258 - val_loss: 1.8760 - val_acc: 0.1923\n",
      "Epoch 53/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8516 - acc: 0.2323 - val_loss: 1.8763 - val_acc: 0.1923\n",
      "Epoch 54/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8740 - acc: 0.2065 - val_loss: 1.8761 - val_acc: 0.1923\n",
      "Epoch 55/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8627 - acc: 0.2677 - val_loss: 1.8754 - val_acc: 0.1923\n",
      "Epoch 56/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8549 - acc: 0.2452 - val_loss: 1.8770 - val_acc: 0.1923\n",
      "Epoch 57/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8641 - acc: 0.2097 - val_loss: 1.8759 - val_acc: 0.1923\n",
      "Epoch 58/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8799 - acc: 0.2484 - val_loss: 1.8783 - val_acc: 0.1923\n",
      "Epoch 59/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8733 - acc: 0.2000 - val_loss: 1.8752 - val_acc: 0.1923\n",
      "Epoch 60/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8718 - acc: 0.2000 - val_loss: 1.8759 - val_acc: 0.1923\n",
      "Epoch 61/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8650 - acc: 0.2452 - val_loss: 1.8753 - val_acc: 0.1923\n",
      "Epoch 62/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8722 - acc: 0.2290 - val_loss: 1.8751 - val_acc: 0.1923\n",
      "Epoch 63/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8640 - acc: 0.2355 - val_loss: 1.8762 - val_acc: 0.1923\n",
      "Epoch 64/100\n",
      "310/310 [==============================] - 4s 12ms/sample - loss: 1.8727 - acc: 0.2129 - val_loss: 1.8757 - val_acc: 0.1923\n",
      "Epoch 65/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8769 - acc: 0.2129 - val_loss: 1.8756 - val_acc: 0.1923\n",
      "Epoch 66/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8678 - acc: 0.2258 - val_loss: 1.8759 - val_acc: 0.1923\n",
      "Epoch 67/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8750 - acc: 0.2161 - val_loss: 1.8749 - val_acc: 0.1923\n",
      "Epoch 68/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8666 - acc: 0.2323 - val_loss: 1.8772 - val_acc: 0.1923\n",
      "Epoch 69/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8850 - acc: 0.2129 - val_loss: 1.8766 - val_acc: 0.1923\n",
      "Epoch 70/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8732 - acc: 0.2129 - val_loss: 1.8751 - val_acc: 0.1923\n",
      "Epoch 71/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8796 - acc: 0.2323 - val_loss: 1.8762 - val_acc: 0.1923\n",
      "Epoch 72/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8718 - acc: 0.1839 - val_loss: 1.8761 - val_acc: 0.1923\n",
      "Epoch 73/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8684 - acc: 0.2452 - val_loss: 1.8759 - val_acc: 0.1923\n",
      "Epoch 74/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8669 - acc: 0.2161 - val_loss: 1.8755 - val_acc: 0.1923\n",
      "Epoch 75/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8650 - acc: 0.2000 - val_loss: 1.8766 - val_acc: 0.1923\n",
      "Epoch 76/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8712 - acc: 0.1871 - val_loss: 1.8769 - val_acc: 0.1923\n",
      "Epoch 77/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8679 - acc: 0.1935 - val_loss: 1.8766 - val_acc: 0.1923\n",
      "Epoch 78/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8687 - acc: 0.2161 - val_loss: 1.8762 - val_acc: 0.1923\n",
      "Epoch 79/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8599 - acc: 0.2097 - val_loss: 1.8773 - val_acc: 0.1923\n",
      "Epoch 80/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8781 - acc: 0.2161 - val_loss: 1.8744 - val_acc: 0.1923\n",
      "Epoch 81/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8682 - acc: 0.2548 - val_loss: 1.8761 - val_acc: 0.1923\n",
      "Epoch 82/100\n",
      "310/310 [==============================] - 4s 12ms/sample - loss: 1.8767 - acc: 0.2097 - val_loss: 1.8745 - val_acc: 0.1923\n",
      "Epoch 83/100\n",
      "310/310 [==============================] - 3s 11ms/sample - loss: 1.8748 - acc: 0.2097 - val_loss: 1.8754 - val_acc: 0.1923\n",
      "Epoch 84/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8615 - acc: 0.2452 - val_loss: 1.8756 - val_acc: 0.1923\n",
      "Epoch 85/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8750 - acc: 0.2387 - val_loss: 1.8765 - val_acc: 0.1923\n",
      "Epoch 86/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8573 - acc: 0.2548 - val_loss: 1.8761 - val_acc: 0.1923\n",
      "Epoch 87/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8737 - acc: 0.2065 - val_loss: 1.8748 - val_acc: 0.1923\n",
      "Epoch 88/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8761 - acc: 0.2581 - val_loss: 1.8764 - val_acc: 0.1923\n",
      "Epoch 89/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8632 - acc: 0.2355 - val_loss: 1.8772 - val_acc: 0.1923\n",
      "Epoch 90/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8697 - acc: 0.2258 - val_loss: 1.8755 - val_acc: 0.1923\n",
      "Epoch 91/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8675 - acc: 0.2000 - val_loss: 1.8758 - val_acc: 0.1923\n",
      "Epoch 92/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8620 - acc: 0.2226 - val_loss: 1.8758 - val_acc: 0.1923\n",
      "Epoch 93/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8664 - acc: 0.2194 - val_loss: 1.8752 - val_acc: 0.1923\n",
      "Epoch 94/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8677 - acc: 0.2290 - val_loss: 1.8767 - val_acc: 0.1923\n",
      "Epoch 95/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8720 - acc: 0.2097 - val_loss: 1.8767 - val_acc: 0.1923\n",
      "Epoch 96/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8647 - acc: 0.2258 - val_loss: 1.8760 - val_acc: 0.1923\n",
      "Epoch 97/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8640 - acc: 0.2290 - val_loss: 1.8771 - val_acc: 0.1923\n",
      "Epoch 98/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8704 - acc: 0.2194 - val_loss: 1.8759 - val_acc: 0.1923\n",
      "Epoch 99/100\n",
      "310/310 [==============================] - 3s 10ms/sample - loss: 1.8647 - acc: 0.2419 - val_loss: 1.8768 - val_acc: 0.1923\n",
      "Epoch 100/100\n",
      "310/310 [==============================] - 3s 9ms/sample - loss: 1.8766 - acc: 0.1968 - val_loss: 1.8757 - val_acc: 0.1923\n"
     ]
    }
   ],
   "source": [
    "# And we train the two last layers\n",
    "# This is classic fine tuning\n",
    "bdes_vm.model.train(X_bdes_train, Y_bdes_train, batch_size=10, validation_data=(X_bdes_test, Y_bdes_test), epoch=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0  0 23  0  0]\n",
      " [ 0  0 15  0  0]\n",
      " [ 0  0 15  0  0]\n",
      " [ 0  0 10  0  0]\n",
      " [ 0  0 15  0  0]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        calm       0.00      0.00      0.00        23\n",
      "       happy       0.00      0.00      0.00        15\n",
      "       angry       0.19      1.00      0.32        15\n",
      "     fearful       0.00      0.00      0.00        10\n",
      "         sad       0.00      0.00      0.00        15\n",
      "\n",
      "    accuracy                           0.19        78\n",
      "   macro avg       0.04      0.20      0.06        78\n",
      "weighted avg       0.04      0.19      0.06        78\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arc/fdp5/.local/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "print_metrics(bdes_vm, X_bdes_test, Y_bdes_test, emotion_list)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}