Notebooks/4. RNN_seqtovec_seqtoseq.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "w1kHW6x-_b4d"
   },
   "source": [
    "# What is a Recurrent Neural Network?\n",
    "\n",
    "A recurrent layer is just a memory cell that computes. In diagrams, you may see it represented as having many cells, but it is just one cell used to calculate an output over a set amount of time steps. The cell calculates Y_0 at X_0 then moves on to calculating Y_1 at X_1 and so on. However, from X_0 to X_1 the memory cell produces a state vector. This state factor is used in the next time step as an additional input factor. This state factor is why it is called a recurrent neural network. Just as x +=1 is a recurrent function to add 1 every time to X, every time step receives an state vector from the last time step.\n",
    "\n",
    "This architecture enables you to use any sequence length as long as the parameters remain constant. An RNN can be though of as word context of reading a sentence. As you read the a sentence you may only be focusing on one word, however, if you encounter a word that is a homphone such as lead you will understand the context. You can immediate notice if lead is the verb for \"guiding\" or the soft metal which is a noun."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vidayERjaO5q"
   },
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "gqWabzlJ63nL",
    "outputId": "579a6636-417a-4bee-fc59-228021c05566"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
      "  import pandas.util.testing as tm\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "keras = tf.keras\n",
    "\n",
    "# set style of charts\n",
    "sns.set(style=\"darkgrid\")\n",
    "plt.rcParams['figure.figsize'] = [10, 10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cg1hfKCPldZG"
   },
   "outputs": [],
   "source": [
    "def plot_series(time, series, format=\"-\", start=0, end=None, label=None):\n",
    "    plt.plot(time[start:end], series[start:end], format, label=label)\n",
    "    plt.xlabel(\"Time\")\n",
    "    plt.ylabel(\"Value\")\n",
    "    if label:\n",
    "        plt.legend(fontsize=14)\n",
    "    plt.grid(True)\n",
    "  \n",
    "  \n",
    "def window_dataset(series, window_size, batch_size=128,\n",
    "                   shuffle_buffer=1000):\n",
    "    dataset = tf.data.Dataset.from_tensor_slices(series)\n",
    "    dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)\n",
    "    dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))\n",
    "    dataset = dataset.shuffle(shuffle_buffer)\n",
    "    dataset = dataset.map(lambda window: (window[:-1], window[-1]))\n",
    "    dataset = dataset.batch(batch_size).prefetch(1)\n",
    "    return dataset\n",
    "  \n",
    "def model_forecast(model, series, window_size):\n",
    "    ds = tf.data.Dataset.from_tensor_slices(series)\n",
    "    ds = ds.window(window_size, shift=1, drop_remainder=True)\n",
    "    ds = ds.flat_map(lambda w: w.batch(window_size))\n",
    "    ds = ds.batch(32).prefetch(1)\n",
    "    forecast = model.predict(ds)\n",
    "    return forecast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iL2DDjV3lel6"
   },
   "outputs": [],
   "source": [
    "# Read in data\n",
    "spy = pd.read_csv('SPY.csv')\n",
    "\n",
    "# Convert series into datetime type\n",
    "spy['Date'] = pd.to_datetime(spy['Date'])\n",
    "\n",
    "# Save target series\n",
    "series = spy['Close']\n",
    "\n",
    "# Create train data set\n",
    "train_split_date = '2014-12-31'\n",
    "train_split_index = np.where(spy.Date == train_split_date)[0][0]\n",
    "x_train = spy.loc[spy['Date'] <= train_split_date]['Close']\n",
    "\n",
    "# Create test data set\n",
    "test_split_date = '2019-01-02'\n",
    "test_split_index = np.where(spy.Date == test_split_date)[0][0]\n",
    "x_test = spy.loc[spy['Date'] >= test_split_date]['Close']\n",
    "\n",
    "# Create valid data set\n",
    "valid_split_index = (train_split_index.max(),test_split_index.min())\n",
    "x_valid = spy.loc[(spy['Date'] < test_split_date) & (spy['Date'] > train_split_date)]['Close']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 285
    },
    "colab_type": "code",
    "id": "Zmp1JXKxk9Vb",
    "outputId": "6130497e-14f5-4804-bc5b-1fc1e8fb55df"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5521 5522 6527 6528 6949\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD7CAYAAABpJS8eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUVfrA8e+UNEhCCaFKLwekV+kC9gJrL7tgQQV7d22r6+q6Yl11RUVWfiK6rquuwIqiuyoCgiJIEZVDDR0SQktC2szc3x8zmcwkM8nMZGryfp6Hh7nn3DvzzmV458y5555jMgwDIYQQ9Z851gEIIYSIDkn4QgjRQEjCF0KIBkISvhBCNBCS8IUQooGwxjoAP1KAocB+wB7jWIQQIlFYgDbAD0Bp1cp4TfhDgWWxDkIIIRLUGGB51cJ4Tfj7AY4cKcLhCP4+gaysdPLzC8MeVKRIvJGTSLGCxBtp9T1es9lEs2aNwZVDq4rXhG8HcDiMkBJ+xbGJROKNnESKFSTeSGsg8frsCpeLtkII0UBIwhdCiAYiXrt0fDIMgyNH8igrKwH8/8zJzTXjcDiiF1gdRT9eE8nJqTRrlo3JZIri6wohYimhEn5h4TFMJhOtWp2EyeT/x4nVasZmS5yEH+14DcPB0aOHKCw8RkZG06i9rhAithKqS6e4uJCMjKY1JntRO5PJTEZGM4qLE2e0ghCi7hIqczocdiyWhPpRErcsFisOh9zTJkRDklAJH5A+5zCR8yhE/Fmf9zMzVr2Iw4hMF680l0N0ww1XU15ejs1Wzu7du+jcuSsAPXooHnroj7UeP3/+h5SWlnL55b+LdKhCiASxr/AAuwv3YRgGRKBNFlDCV0rNBzoDDqAQuE1rvU4plQOUuP4A3K+1/tx1zHBgFpAG5ACTtda54Qw+lmbPngvA/v37uP76Kbz11j+86m02G1ar/9N7wQWXRDQ+IUTicRjOblZzhK5TBtrCv1prfQxAKfUbYA4wyFV3idZ6o+fOSikz8A5wjdZ6uVLqD8AMYGp4wo5Pl1wykdNOO5Mff/yBLl26MW3azTz22MMUFRVRVlbGyJGjuPnmOwB4881ZFBcXc+utd/LJJwv5/PPPyMjIZPv2bWRkpPPnPz9DVlaLGL8jIUQ02Q0HZpM5Yl2uASX8imTv0gRnS78mg4ESrXXF5D2v42zlhzXhf/vTfpZvqD5lhMkEdV2qd3S/Nozq2ybo44qKipg9+20ASktLefrpv9KoUSNsNht3330r3323guHDR1Y77tdff2Hu3Pdo1ao1Tz/9Zz788H2mT7+lbm9CCJFQfj28OWL99xBEH75S6u/AmTh7ls72qHpXKWXCOTPbQ1rro0AHYGfFDlrrQ0ops1Kqudb6cKCvmZWV7rWdm2vGaq38qWOxmPD3RVjXL0iLxeT1Wv73MwOV+55//vnux+Xl8MorL/PTT+sxDIPDh/PZvn0Lo0ePxmw2YTZXHte/f3/atWsLQN++/Vi16ruAXr8uzGYz2dkZIR0b6nGxkEixgsQbafEc766CPYB3jOGMN+CEr7W+HkApNQV4FjgXGKO13q2USgFeBF4BJocruPz8Qq+JgxwOh9cNSsNPbs3wk1tXOy5cNzIF8hx2uwMw3PsmJ6e6H7/77jyOHz/GrFlvkZKSwtNPP0lxcQk2m8M9MVzFvklJyR6vZ8Jms0X8ZiyHw0FeXkHQx2VnZ4R0XCwkUqwg8UZavMfbMaM9Owt2u2MMNl6z2VStoexVH2xAWut5wHilVJbWererrBR4FRjl2m0X0LHiGKVUC8ARTOu+PigoKCArqwUpKSnk5eWyfPk3sQ5JCBHHmqRk0i49+K7kQNWa8JVS6Uqp9h7bE4HDQIlSqomrzARcAaxz7bYGSFNKjXZt3wh8EM7AE8Gll17BTz+tZ8qUy3jqqScYPHhorEMSQsSxckc5VnPkRsubjFqubiqlWgELgMY451g+DNwLHAU+wrmklgX4Bbhda73fddxInMMyU6kclnkwwLg6ATuqdukcOLCT1q07+j2ogsylE5hAz2dV8f6z2FMixQoSb6TFe7wvrHkNs8nEnYNuBOrUpdMZZ971UutXiStJD/dTPbCG41YAfQOKUgghBOWOMjKSI3dROeGmVhBCiPpqV8Fe8osjd6lTEr4QQsSBMnsZAAdORG5CAkn4QggRZSW2EnYe3+1VtrfQ57rjYSUJXwghouyepY/yzOq/Uepq1QN8t381AJnShy+EEPVPub3c/TjNmgZAx8z2/navM0n4QggRI4bH2twVN1xd0PWciL2eJPwQ3XPP7cyf/6FXmWEYXHrpb1i7do3PY5588jE++uh9wDkf/vvvv+tzv08//Q9/+MPva41h6dIl/PLLxlr3E0LEj2Olx92PPSdKK7E7Z5lPszaK2GtLwg/ReedN4tNPP/EqW7t2DWaziQEDBvk5qtIFF1xS58VPli1bwq+//lyn5xBCRNff1s12P/ZM+OtynY23FEtSxF47oVe8Kt/8LeV6abVyk8lEbXcQ1yZJjSWpxyi/9WPGnMrzzz9FTs4OOnXqDMCiRQs566xzueWWGygpKaasrIxJky7ksst+W+14z/nwy8vLefbZGfz442qaNGlK9+7Kvd+2bVt5/vkZ1Z7v++9Xsnz5UlavXsV//rOAyy//Leeccz6fffYJ//73B9jtdtLT07n33gfo0KFTnc6FECJ8mqY0YX+Rc9IBzy6dTUe2AJBsSY7Yayd0wo+lpKQkzjjjHD79dCE333wHJ04UsWzZN8yb9z6TJ19DcnIyJ06cYNq0qxk2bIT7S8GXjz/+iP379/HOOx9gs9m45ZYbaNPG2Z/Xpk0bXnzx1WrPd8opIxg9eiw9e/bi4osvB2D9+rV89dV/mTlzNsnJyaxc+S1PPfU4r702JyrnRAhRu18Pb3Y/dvhomEZqtStI8ISf1GOUz1Z4tOamOe+8Sdx7721Mn34rX375X/r27U9SUhIzZjzB1q2bMZnMHDqUx9atm2tM+GvW/MA555yP1WrFarVy1lnnsGGDcx66kpISXnllRkDP9+23S9m6dQvTpl0DOK8pFBQcr7afECI+OAwHhmHwzd4VUXm9hE74sda9ew+ysrL57rsVfPrpQi699LfMmjWT5s2zmDPnXaxWK3fddQtlZWW1P5kfwTyfYTi/hK6//saQX08IET0GBluP7uCDzQui8npy0baOzjtvEnPmvMHu3bsYM+ZUCgsLaNmyFVarle3bt7J+/bpan2PIkKEsXvwpNpuN0tIS/vvfxe66mp6vcePGFBYWurdHjRrD4sWLyM119g/a7XY2bfo1jO9WCFEXJbZSr23DcHhduG2a0iSiry8t/Do644yzmTnzJSZNupCkpCSuvvo6nnjiURYtWkD79h0YMMDvhKJuF1xwMZs3b2Hy5Etp0qQpPXv25siRfIAan++ss87lySf/xNdff+m+aDtt2s088MDd2O0ObLZyxo8/nZ49e0Xs/QshAldQVui17TAMLGaLe/u09mMi+vq1zocfI52Q+fAjTubDjz8Sb2TFOt4dx3by3JqZDG7ZnzW563lw6J2U2st44cdXAejetIt7LnwI/3z40qUjhBBRUtHCz27UAgAH3l06jZIid9MVSMIXQoioKSwvAionSDMMw2ss/jmdTovo60sfvhBCRMm7m5zTsaS7WvIltlJ2Fexx16cnNY7o60sLXwghosxicl6ofXPjO15dOmaTxd8hYSEJXwghosBzhI7dleSLbCdoltrUXW41RzbhB9Slo5Saj/OqrwMoBG7TWq9TSvUA5gJZQD5wldZ6i+sYv3VCCNHQbD+W435sMpncj20Om/uxJYLTKkDgffhXa62PASilfgPMAQYBrwMztdbvKKUmA7OACa5jaqpLeDfccDXl5eXYbOXs3r2Lzp27AtCjh+Khh/4Y0HP8+ONqbDYbI0eOjGSoQog4UFhW5H5swk/CN0f2smpAz16R7F2aAA6lVEucSf8MV/l7wCtKqWzA5K9Oa50XlshjbPbsuQDs37+P66+fwltv/SPo51i7dg3FxcWS8IVoAD7L+RKAl8b9hY2HKu+Aj8cWPkqpvwNn4kzmZwPtgb1aazuA1tqulNrnKjfVUFcvEr4vK1cu5+2351BaWkZSUhK33XY3ffr0ZdeuHJ588k+UlJTgcNg555yJnHLKCBYs+DcOh4M1a1YxYcKZTJlyTazfghAiwqxmK3h16djdjyM5UyYEkfC11tcDKKWmAM8Cj0QqqAquO8bccnPNWK2VJ2TlvtWs2LsqIq89st0wRrQdUut+FosZMHHgwF7mzn2Tl16aSePG6Wzfvo277rqNBQs+Zf78jxg79lSuvnoqAMePHyczM5MLL7yY4uJibr/9roi8h9qYzWays0NbMDnU42IhkWIFiTfSYhXvkdKj7tfPLE11l6c0qrxQ6yu2cMYbdIeR1nqeUuoNYA/QTillcbXgLUBbYDfOFr6/uoBVnVrB4XB4TUHgsBv4mhnCZMJneTAcdiOg6Q7sdgdgsGLFCvbs2cP06de762w2G7m5efTrN4BXX32ZEyeKGTRoCIMGDcFmc+BwGO73F4upFRwOR0i3mcf69vRgJFKsIPFGWrTjPVF+Apthd99olWS2kpdXwPFjxe59jhecAOCFU/9cLbY6TK3gU60JXymVDjTTWu92bU8EDgO5wDrgSuAd199rK/rolVJ+68LllDaDOaXN4GrlsZibxjAMTjllBI888ni1unHjTqNPn36sWvUd77zzFosWLeTRR5+IanxCiOi7b9ljXtv9s/sA4NkeXbzzKwBSIrjSVYVAOowaAx8opX5yJfG7gIlaawO4EbhNKbUZuM21XaGmunpn2LDhfP/9SrZv3+Yuq1hvds+e3TRvnsW5507k2mtv4JdfnOWNGzemqKjQ5/MJIRLbvsID1cpWH6x9uvRIqrWFr7U+CAz3U7cJOCXYuvqoffsOPProE8yY8QSlpaXYbOX07dufXr1689VX/+WLLxaTlGTFZDJxxx33ADB27Hgeeug+pky5Qi7aClHPVEyj4Oni7hMB6NbUuWJdVmpz8ksORy0mmUunjtq0acuiRc7hVsOGDWfYsOrfjVddNZWrrpparbxt23a89dY/Em46ZyFE7XxNPd8psz0AGcnpJJuToprsQaZWEEKIiNhfVL1LJ9VSOTrH827baJEWvhBChNHugn3M+OFFn3XJHhdmPe+2jRZp4QshRBj5S/bgPTlaLFr4CZfw43RJxoQj51GI6EuRFn7grNZkioqOS7KqI8MwKCo6jtUa+XG/QtRXBW9cQ8GbN3iV1ZSbzu98FmnWNPe29OHXolmzbI4cyaOw8GiN+5nNZhyOxBn1Eot4rdZkmjXLjuprClFflG/+1vnAXu5VXmIv8XvMOZ29ly+MRQs/oRK+xWKlRYs2te4nt3sLISKpZMlsn+WHio8E/ByeCf++IbfWOaZAJFSXjhBCxBvPbpyqF2zP6DAOgNsGeHf9gHeXTseM9pEJroqEauELIUSsGbayKgV2MPlOpb2a9+CCbuf6rPNs4UerP19a+EII4YPhcFC6Zj5Gifd8VyXL5nrvaLfhj6WGNWplWKYQQsQJ+671lK2ZT8l3//Qqt2351mu7/NdvANh+bGe15zAM/4Mx5KKtEELECaPcNWe9w38LHqD0u/cwZWbz/PZ/Vqsrr+HYihZ+pFe58iQtfCGE8MGoGHJpTqosK61ciNzauXJFvJIvXvZK3L2a93Duj/9x+RUt/FPbRW9Na0n4Qgjhg+PwHueD0kKM8hIMh43CubcAkHTyaWBJ8tp/eOvKL4BTT3Im8Xbp/oeRV8yUGc2+fEn4Qgjhg233T86/d66l6F8P4ji8111n7dDPayFygBX7netr39L/Ovq2OJmZE56haUqTWl9HEr4QQsSYteMA92Oj6IhXdw5JqX4Xzj45SwX1OuYopmFJ+EII4UP5z//z2i5e9Kz7sSkplaQeo8LyOtLCF0KIWKs2vt7wemxp17tyV9ffE9qPCfplNh/ZVvtOYSIJXwghfDC37OK3ztKik1fLfFnTRgCU2sv8HeJXzvFdwQcXIkn4Qgjhg3Es12d5o4sfr1a2uEU6AN/u+z6iMdVVrTdeKaWygHlAV6AM2AJM11rnKaUM4Ceg4nayKVrrn1zHTQSedb3GGuBarfWJ8L8FIYQIr3K9DKO0sFp58sCJWLI6+D2uc6b/ungQSAvfAJ7RWiutdV9gGzDDo36k1nqA609Fsk8HZgMTtdbdgALg3jDHLoQQEWHb96vPcqPce7775KEXAzDusHMEzx0Dp0c2sDqqNeFrrQ9rrZd4FH0HdKzlsHOA1VrrLa7t14HLQ4pQCCGizLZlhe+KKtMgpAycSMqIK9nayLl6nNUc37PVBBWdUsoM3AQs9CheopSyAp8Bj2mtS4EOgOdMQruA6Ez4LIQQEZLc5/RqZabkRuxJdd51G8oQy4yk9DrHFahgv47+BhQCr7i2O2itdyulMnH28z8C/CFcwWVlhX4isrMzwhVGVEi8kZNIsYLEG2mBxOtv/blWXaqP3Ck40JjMnXaOWy0hnQuLxVzjceE8vwEnfKXUc0B3nP3yDgCt9W7X38eVUn8H7nbtvgsY73F4B2B3sMHl5xficAS/YHmiLRko8UZOIsUKEm+kBRKvUeacJdPaZRi27avc5ZYO/X0eW15UTla5newmHUI7F4bJ73HBnl+z2VRjQzmgYZlKqb8Ag4ELXF02KKWaKaXSXI+twCXAOtchi4GhSqnuru0bgX8FHLUQQsRI4Vs3OR9YvSdHSx17re8DTGZKTSZSq0ymFqgmKZkhHReKQIZl9gYeBDYDK5RSADuAZ4BZrqGZScAKnF06aK0LlFLTgE+UUhZgLXBHRN6BEEKEiefyhZasjqRMvhRMZsxp/pOyyWxlX2oSRUUHQ3rN3/W8JKTjQlFrwtda/wx+l2bpV8NxC4AFIcYlhBBRV7b+U/fjJDUGU3Ja7Qe5ljE8Zi8O6TUbWQN4jTCRO22FEMKlbI2zjdr4imcCS/bgXMS8DqI5eVp8DxoVQogosrTpgaPwMObMloEfFPy4Ei/RXNtWEr4QQgBGSSH2/Tr4A10t9MZGaB0m0sIXQogoMgwH5a4hmOam/pcl9Mlk4sEdh0hre3JQh5lNZhyGQ1r4QggRTYWzp7ofp46fFuTRJprYHViC7Nqxmq2U2cuimvDloq0QokGzH9jitW3KaBHcE1R0yfhZ8tDvYdUeRJ4kfCFEg2WUFXNi4ZNeZebUIKcycE+oFmzCd2Z6s7TwhRAi8kq+nee1bW52UvBPEmILv7JpL334QggRcbbtP7gfp515B5aO/YN+DnOWcyLg5D5nBnVcFAfnuEnCF0I0KIZhYNu+ClNKY7CXA5A26SGsrXuE9Hzm1Awypr0VwpEVGb+OA/mDIAlfCNGg2Pf+TMmXr3mVhZrs6yL66V768IUQDYxt14ZYhwBApybO9W8tpuilYWnhCyEaFPven722086NzXLb1/WezMETuaRaU6P2mtLCF0I0LFVa1NaT+sQkjFRrCh0zo7vyq7TwhRANitl1Y1XKyN85L9w2IJLwhRANhr24ENvOtQBY2/aMcTTRJ106QogGI3fBS7EOIaakhS+EaBCKv5qFbduPAKSMmhLjaGJDWvhCiAbBtnWl+3FSr/ExjCR2JOELIRoEa+chACT1GofJ3DBTX8N810KIBse2YzWYLaSOuSbWocRMrX34SqksYB7QFSgDtgDTtdZ5SqnhwCwgDcgBJmutc13H+a0TQohoMmxlzgeOui04nugCaeEbwDNaa6W17gtsA2YopczAO8AtWusewFJgBkBNdUIIEQ2Gw0HBG9dQ+I97sO3ZGOtw4kKtCV9rfVhrvcSj6DugIzAYKNFaL3eVvw5c5npcU50QQkRc4d+dyxYahfmUfPEyAK0uuT+WIcVcUH34rpb7TcBCoAOws6JOa30IMCulmtdSJ4QQEWWUFfssT23fK8qRxJdgx+H/DSgEXgEuDH843rKy0kM+Njs7yGXKYkzijZxEihUk3nAoyz1CoY9yc1pjshsl1liVcJ7fgBO+Uuo5oDswUWvtUErtwtm1U1HfAnBorQ/XVBdMcPn5hTgcwc8WnZ2dQV5eQdDHxYrEGzmJFCtIvOFSttG5klXygPMoW7fIXW4ymeMyXn+CPb9ms6nGhnJAX3VKqb/g7Je/QGtd6ipeA6QppUa7tm8EPgigTgghIsa252dKV/4DAGvnoaRf82qMI4ofgQzL7A08CGwGViilAHZorS9USk0BZimlUnENvQRw/QLwWSeEEMEq37EaAON4HtYuQzBnZOMoPg4OO6ZGTTC5pjwufO9ejIJDHkcamJIbYe040D1pWkNWa8LXWv+Mn2XVtdYrgL7B1gkhRCCM0iKMkgJK/vuKu6z0+/dJnzqLonm3u8sypr2FYRhVkj2Ys5yrSqWeeVt0Ao5zMnmaECIuFX/5GrZt3/usO/Hxn7y2yzcvB7Ol2n4mV5kpissIxjNJ+EKIuLNxRz4d/SR7AFOK94XJkiV/j3RI9YJ87Qkh4srC5Tt44f31lJj9r0Zlad2jSonPXmdRhSR8IUTcOFFiY/7yHbzU/G1SHUXu8rRz78Xcsot7235gs/eBliT3w4xpb0U6zIQlXTpCiLix95Cv26WcC41b2vbCcewAJz54uHrCt5d5bTa64BEcBfmRCjNhSQtfCBE3ftrufW+mKa0J6de85nxstmBp1s6rPvX0WwAwZ7X3Kre07EpS12ERjDQxSQtfCBFzJ0rKmffFZg4cPuFVbs7uhCk5ze9x1s5DMDc/CUeh84si9dTrIhpnopOEL4SIuVtfXOazPLnXuBqPM5lMOI7sB8M5z7398B6SajyiYZMuHSFEzGzfd5ypM76qVn7f4SsxX/Is1o4Da38So3JRE2u7hj0bZm2khS+EiJk/v726WtnIPq0Z2ac1jZsHP5u6pX3/cIRVb0kLXwgRE0cLS32Wt2+ZzsmdQls6w2SS8fg1kYQvhIiJRSt2em03aZwMQEpy9SkSRHhIwhdCxERSkjP9ZDZyXmZtk9XIWW4JLC2lnXM34BypIwIjffhCiKhyGAZb9xxj8fe7AHhs6jA2bMtn+Yb9AFgsNXfLpJ5+C+aMFliyOzu3x91A4Y7q1wJEdZLwhRBRYxgGD85aSd7REndZ0/QUxvZvyzfr9gFQXu6o8TmSugz12jYlpZDU72yZETMAkvCFEFGzJ6/IK9l7uu/KAXyxajcj+rQO+nlTh19R19AaBEn4QoiIMwyD657+ulr5nAcmuB+nJluZNLpzNMNqcOQ3kBAi4nYd9D0pmoguSfhCiIhb9N3OamXnj+wYg0gaNunSEUJE1Na9x1i9Kde9bTLBm/dPqOEIESnSwhdCRNRf5q3x2jaMGAUiAmvhK6WeAy4GOgF9tdYbXeU5QInrD8D9WuvPXXXDgVlAGpADTNZa5yKEaDAMj+zeoWU6u3ILSbJKOzNWAj3z84GxQPWOOLhEaz3A9aci2ZuBd4BbtNY9gKXAjHAELIRIHPnHK4dg3n3FAACG9WoZq3AavIBa+Frr5QBKqUCfdzBQUnEc8DrOVv7UIOMTQiSocpudmR9vBOCy8d3IbJTMn6YOc0+hIKIvHBdt31VKmYDlwENa66NABzx+DWitDymlzEqp5lrrw/6eqKqsrPSQg8rOzgj52FiQeCMnkWKF+hHvtj1H+deXm9l5oACAgb1akZ2dERfvLR5iCEY4461rwh+jtd6tlEoBXgReASbXPSyn/PxCHI7gr/BkZ2eQl1cQrjAiTuKNnESKFepHvA7D4M6/fuNVZi+zxcX7qg/ntyZms6nGhnKdrp5orXe7/i4FXgVGuap2Ae5BtkqpFoAjmNa9ECLxlJTZ2O3jJqvW0o0TF0Ju4SulGgNWrfUxV5fOFcA6V/UaIE0pNdrVj38j8EGdoxVCxLWbX1jqs9wsC5PEhUCHZb4MXAS0Bv6nlMoHJgIfKaUsgAX4BbgZQGvtUEpNAWYppVJxDcsMf/hCiHhhs/ue5fKJ64ZFORLhT6CjdG4HbvdR5XeFYa31CqBviHEJIRLM+q351cpOyk6nXXbogy9EeMnUCkKIsCgutXlte86EKeKD3PImhKizwuJy5nz6a6zDELWQhC+EqLPfv7bCa7tzm8Qa695QSJeOEKJOHA6DkjK7e3v278dhQkblxCNJ+EKIOln+03734wHdWmAxS8dBvJKEL4QImWEYvPXZJgDu/+1AVIdmMY5I1ES+ioUQIXvlg/Xux80yU2MYiQiEJHwhRMg857tvlp4Sw0hEIKRLRwgRsHKbndn/+QXDgOm/6c1/V+0CZMx9opCEL4QI2OpNeazWeQA8+bZz6cLkJOkoSBTyLyWECNjnP+xyP9550Dltb+9OzWMVjgiSJHwhImDDtnzWuFrCiep4URlzF29yr0lxosTGLh9TH1stkkYShfxLCREmh44Vc8tfv+FESTkvfrCemR//xLa9x2IdVsju/Ntyvlm3j+uf+ZoTJTaWb9jnc7/fnt49ypGJUEnCFyIM5n2h+f1rKykutfPGf35xlz85b01CJ/0K67ce4p9fbQXgsWuHusufv2MsTWR0TsKQhC9EHW3dc4yvf9zr3t6wzXua4CfnraHcZq96WFyruJmqwuxPKr/EOrTKYGz/tgCkN0qKalyibiThC1FHf3lnTa37bN0TP6384lIbO/Yfx2EY1aY0rrB0ve/umwq/O6M7D00ZTNsWMtd9IpFhmULUgcPjxqOa2ByV+5WU2UhOskR92T+Hw8BhGDz73lpyDlQujH3TBX0Y2rOle7vgRJn78QO/G8SMd390b199tgIgyWqhW7smUYhahJO08IUIgd3hYOqMr7jz5eUATBjUjjfvH+93/4qW9MYd+dz8wlIWf7/L776R8sx7a5n27BKvZA/w2vyNXtt3uN7T6L5t6NG+qVfdqQPaRTZIEVGS8IUI0vETZdzwzBLAufAHQO/OzTF5tNhP7uQ9idjrC34G4IX3nXPPfLhkm9e0BNGweffRGuvLbXb3EEyAy0/r5lX/4ORBEYlLRI8kfCGC4DAMd6veU/9uLQCYNKoTANec3ZNHrhuE8xsAABW+SURBVB7itY9n1whAWbnvRb8jYdHKHL91mY2TAZj+3Ddc/8zX7vLGqc4LsvdcMYCMRkmcJGvTJjxJ+MLtWGEpRwpKvcqi3QqNdzd4JERPFf3xk0Z15pkbR9CiaRqd22R6zTFTtYX9c87hyAVaxUffbPfa7tsly/24caq1xn/n3p2a89LtY0hLkUt+ia7Wf0Gl1HPAxUAnoK/WeqOrvAcwF8gC8oGrtNZbaqsT8euuV74F4M37x1NcamfRyhw++34XndtkVmutNkSrN+VSkRevOacnX/24h10HC72Sp9lsokXTtICe79PvdjKoR3YkQvWyN6/63bG3XtSX6c8tAcBuN6r16//ujB4Rj0tEXyBf2fOBl4BlVcpfB2Zqrd9RSk0GZgETAqgTcW7J2r3M+2Kze7tiCF+0R5XEm1c9Lm6O6deGU3q14tedRxjQvUVQz3P12Yq5izXb9x1n6oyveGracFo1bxTucN0efXMV4BxxYzaZaJxmJclqZs4DE5jz6a8s37CfJ+au9jpGVblYK+qHWrt0tNbLtda7PcuUUi2BQcB7rqL3gEFKqeya6sIXtggXh8Pg7c+1192gnsm+Qomf8doNxfGiyqGKs38/DpPJREqyJaBkb7V4f1FWHeny4BvfhSdIH95evImKzpoe7ZvS7aQmtMlq7K739+9a0a8v6pdQO+XaA3u11nYArbVdKbXPVW6qoS6o2aSyskK/SJSdnRHysbEQq3jf/69mydq9LFm7t8b9bn1xGX+6YQSDXOO1E+n8BhtrwYkypj/1JQUnynjr0TNpmpHK1BkLAejZsRmtWwU3/vz/HjmLT1fs4L0vtN94PMuCjdcwDCbd64zvP8//xqtuyTrnDVRN01N8Pu9qHxO8dW6bSddOWdXK/UmkzwI07Hjj+ipMfn6h1zCxQGVnZ5CXV1D7jnEimvF6ds1s3XuMdxZvquWISn+cvZI5D0xIqPMbTKx78grZk1fIGwsrpxH41xeaIwUl7u17Lu8f0ns/bUBb1ulcfndGD5/Hr9qwl85tMkM6t3s8+uj3Hzjmnr3yqx/3uMuPFpYG/LzNM1IC3jeRPgtQ/+M1m001NpRDHaWzG2inlLIAuP5u6yqvqU7E0LtfbOb6p7/muOtOyr/Mq3lKgMvGd+P+3w70Ktux/zg//HKg3o3e2Z9fxKNvrvJK9uC8sLry54MAXHdeLyzm0P7LmM0m7rtyIG1bOLtTKv6u8HUtv7BqUtFHD3hNX/yOR9fcn6YOC+i5br6gD9ee0yvkWER8C+nTq7XOBdYBV7qKrgTWaq3zaqqra7AidDa7gy9dLb6XP9zAMY8+aX/OPqUDqoP3DURPzF3N429+z4+b688/54Zt+Tw8+3uvsgvHdqm238g+rcP2muMHevfjL9+wP+jnWLp+H5+v8r5j919fOQfD5R+r/FXy5v3jad8ysO7RIT1b0ig1rn/4izoIZFjmy8BFQGvgf0qpfK11b+BGYK5S6lHgCHCVx2E11YkoOlZY6h5uWWH7vuPc9bfqNw95qu2uyk9W7GSwalnjPvFs6oyvABg3oC26yvj42y/ux4DuLfh4aeXY9VMHtPW6k7auJgxqR8+OzXj5w/XkHS2p/YAqDMOoNqMlwOY9x3h9wUZW/ZrrLqsp7gcnD+Kpd370Wy/ql1oTvtb6duB2H+WbgFP8HOO3TkTX3MW6xvqszBQevWaoe/6Ux64dSpusxiRZK3/8JVvNlNm87wrdebAAwzB4Yu5qRvRuzRlD24c/+DApLbez6peDjOrXBrPJxFyP6xYVFzUBnrj+FNpmNXInyD9eM5Q/vfUDt1zYJ+xfbiaTiXYtGjOsVysWrdwZ1LHlNjvTn/vGq6xru0y27T0O4JXs/3rrqBqfq/tJTZnzwARWb8qlgY+6bRDkt1s9ZrM7WLf1UI37TDlLkdEomcevG8ahoyV0aFV9RMDLd4zhaGEpD8yqHD5oMZsoLrWTc6CAnAMFnD7kpLC2gMNh4fIdzF++w72de7TYb3JNTbbQrkq/esfWGV53ykbCb0Z3DirhHy8q404fv85uvbBvtV9yQMCLkwzpmbi/1kTgZGqFeqq0zM76WpI9wO5c50W+k7LT/Y4pT06y0LKZ941BdofBrS8udW8XuCYRixc/bMr1SvZAtcR656X93I9bBnh3bLhZzMF9ST46Z5XPcll1SgRCEn49dPxEGTe98A0zP668MzQlyUKbrEbceWk/r9v5Tzm5VcDPe+el/ejVsZnvupeXsz+/KPSgw6zqlL+eOrRKZ+ZdY+nXtQU3X9AHgIevGhyt0LxU/Crq2aH2O1s3bs/3ugHsuZtHAtCqWWy+rETikS6demhNlQFRl43vxtmndHBv9+vagsf+bxW7DhbSokngyaJf1xb069qCz37YzQdfVp8a6eHZ30e8CwScQ0MrpgK4+YI+9O2SxScrcxg/sB3NM1NZ9etB9763XjqAnL1HMZtg4bc5ANx2UT/3RGBDeraMSsw1OSm7cUATk1WdAK15Zip3XtqfTm2c3XCv3DnW/atr2qST6dAysW4wEpEnCb8emve594Xas4ZVv6D62LXDQh5Lf/7oLu6En5WZQv7xyhk2v1m3l1MHtGPlxgP8nHOY80Z05OHZ3/PwVYPp2jY8KyR5zvviOb9N1S6bS8Z15azhHcnLa45hGCz8NoezhrUnq0lqWOIIF6vFjL2WGwwLTpSx82DlDTiTz3RObtava+UdsY1Srdx2UV+27j3G8JPDN4RU1B+S8BOcYRgUl9rdY6dzDhz3qr/23J5+L6aGepG1UZXW6E0X9HF3ocxdrFn580H3VMArNh4A4Mm314SlJb3zQOB3HTbPrOzXNplMMW/J+1NabmfXwZrf1x0ec/A/f8sommX47rMf2CObgVGYgVMkJunDT3AffrONW19c6r7R5vG3Klu/cx6YwJh+bcP+mqkeCX/S6M4M7dmSl24f7S7zt7LSwioXUUNRMYf8OcM71LIn9Org+3pDvNmff4KjhWUBnR8T+E32QtRGEn6C++w7552Wa7fE5s7Xin7ijEa1z644f/kOlqwLfQqBX3MO8+GSbQBccmpXbru4L1NcXRuPe0wdMOveU5nzwISEG7lSdVRRhZKyyhktZ903LkrRiPpIunQS0K6DBTz2fz94lf3jf1uYMPgk93agt9LXlec0ukNUtnv2xWYZKWSkJbEr13vxjbcXa8YFuRB2WbmdJ+etcQ8hBWcXzcDuzq6L8YOc7/u2i/uyO7eQJKslpPcSjyruCAa44fyT3ROjCREK+fQkoKrJvsIWj66UP14zNKIxvHHfOJ67eaRX94Jnou3SJpPfuyZemz6pt9exdkdwa7ne+Pw3Xsn+lTvH+txvYPdsJo3qHNRzx5uJ9yzgf6ud8wxW/TXUNF3mqBd1Iy38BFNYww1OT/9jLQBXnaUwB3lDT7CsFjPNM71Hu5SW292PLzq1C41Sk9wXSnOPnODjZc4uixU/HWBM/5qvLWzZc9TnHC+PXjOk3k/u9fGy7Zw+pD1vV5kWo3FaUowiEvWFtPATzBsLfwbgwjGdaennhps+XZpHMyS3Pa5W+HXn9fJaVQlg4qjOtMt2lgWyeLevZH/npf3p1DozDJHGN7vdOUSz6t2/soi4qCv5BCWYjTucyXLcwHZMdHVf7M8v8preN5ibqSKhasu/ws0X9OHh2d+z6tdcfjO6qNqXQk3idUhlJJTZHLz7xWZyjxYDzl81Ow8UkB2j6R9E/SEt/ATiOQe956iYRqnx8VO/d2fnLwvP8e+ePBPWw7O/Z8O2fAqLy/nXV1s5dKzY3SX05iLnIiRWi5npk3pz9+X9Ixx5bN19WfX396XHalWdWmdWWwdXiFBICz/O/WdFDna7wz0tAMCzN4302ifYCbgi5YrTujOiT2taVZlorULVESY/bs7jxQ/WA7B41S4yGyfz4m2j+fYn581aJ3dqFtRcP4mqhbTcRZRIwo9TKzbup7Tc4bUIR4WqUwMkeSTSM2M4L32S1Uy3djVPn2Axm9zTCCxdv8+r7nhRGY95zAZZdXRPfdW6eSNaNksj90hxtbpnbhoRg4hEfSUJPw5t2JbP3z/5NeD9U5ItPHbtUFo1a0RKcnyPQT9jSHsWV1mWz1PFuP3MxskN6iJlx1YZ5B4p5uLx3fjo663u8iw/10OECIX04cehim4OX6ae63uB6Q6tMuI+2QNeK2lVuPjU6uvHPjVteDTCiRsVE9l1a+89TXK8LSojEpsk/DjzTx/TDoOzK2TOAxMY3a9NlCMKL1/567wRnUhJqvyyatEktUG17gF3V1hrj5FLtXWPCREsSfhxpmJOnA4t07nvyoFMm3gyAA/Usqh4ojD7abG+fEfl5GsTBp3kc5/67Iyh7Xlq+nC6ndSU011TZPRoX/uiKEIEo87NKKVUDlDi+gNwv9b6c6XUcGAWkAbkAJO11rm+nkM4fbIih7yjztP4x2uHYjKZMAyDXp2a06Rx/bitvqur1XrHJf146cMN7vIkq4WZd411LmQyqOENQTSZTO7RTU1cUyh43rksRDiE63fzJVpr90oUSikz8A5wjdZ6uVLqD8AMYGqYXq/ecBgGE+9Z4FXWu3Nzd9+tyWSqN8kenO/tr7eNpknjZM4d3tHrbuG0FCuXjusWw+jiQ8U9FkkyUZoIs0h1lA4GSrTWFas2vI6zlS8Jv4o3P/mlWtk9lw+IQSTRU/EFdsm4rjGOJD6N6deG5CQzQ1TLWIci6plwJfx3lVImYDnwENABcK83p7U+pJQyK6Waa61rn0ilAcg/VsLLH23wmgVSCHD+qpMlCkUkhCPhj9Fa71ZKpQAvAq8AH4fhecnKCn1O9+zs+F7A+fG5q30m+wHds+M+doj/8+spkWIFiTfSGnK8dU74Wuvdrr9LlVKvAguBl4COFfsopVoAjmBb9/n5hThqWdzZl+zsDPLyAl/7NNqWbdhHzv7KtWf/9ZfzWLZ6F68t2MgN5/eK69gh/s+vp0SKFSTeSKvv8ZrNphobynW6KqSUaqyUauJ6bAKuANYBa4A0pVTFWLsbgQ/q8lqJrNzmcN9Ys0bn8X+fbgJgUI9snrj+FNJSrAzskc0b941vcOPPhRDRU9fs0gr4SCllASzAL8DNWmuHUmoKMEsplYprWGYdXythGIbBLzlH6HZSE/KOFPOox/wwFdJSLNxyYR+5k1IIETV1Svha6+3AQD91K4C+dXn+RPXDplxeX/BzjfvMvOvUKEUjhBBO0n8QRsWlNm7569Jq5VVnQnz6RpkBUQgRfZLww6CwuJxfcg5Xa9XfdVl/zCYTvTs3J+9oMfOX7eC683v5nV5ACCEiSRJ+HZSU2Vi/NZ9ZC70TfVqKhcennuI1b3120zRucM2LI4QQsSAJP0Trth7iZY+5YAA6ts7g4SmDq63sJIQQ8UASfpAMw+C1BT+zelPlPHAXje3CuSM6SleNECKuScIPwsYd+bzwfuXiJN3aNeGhKYNjGJEQQgSuwSZ8h8PgvS+3kHukmFsv6uu1ElNxqY05i34lMz2ZTTuPMG1ib178cD3HCsvc+7x4+2gyG9WfWSyFEPVfvUz4ZeV2ikttpKVYOV5Uhs3uYOeBAv7275987j/9uSU1Pt+f3vrB/fiK07rHdKFwIYQIVb1L+F+s2sU/v9pa+461GN67Fd/9fNC93aJJKjNuHCH99EKIhFXvEv5Xa/f6LO/SNpMpZypaNkurNl/Nhm2H+CXnCKcOaMu+QycY1KMFJpOJaRN7RyNkIYSIinqX8GdMH0FyWjJlxWXYHQ4s5tqHSPbr2oJ+XVsA0MZjEWkhhKhP6uWA8SbpKQABJXshhGgoJCMKIUQDIQlfCCEaCEn4QgjRQEjCF0KIBkISvhBCNBCS8IUQooGI13H4FnCuwB6quhwbCxJv5CRSrCDxRlp9jtdjX4uvepNhGGEIKexGA8tiHYQQQiSoMcDyqoXxmvBTgKHAfsAe41iEECJRWIA2wA9AadXKeE34Qgghwkwu2gohRAMhCV8IIRoISfhCCNFASMIXQogGQhK+EEI0EJLwhRCigZCEL4QQDUS8Tq0QEqVUD2AukAXkA1dprbdEOYbngIuBTkBfrfXG2mILtS4MsWYB84CuQBmwBZiutc5TSg0HZgFpQA4wWWud6zoupLowxTwf6Aw4gELgNq31ung8v1Xi/iPwGK7PRByf3xygxPUH4H6t9efxGK9SKhX4K3C6K96VWutp8fhZUEp1AuZ7FDUFMrXWzaMZb31r4b8OzNRa9wBm4vygRdt8YCyws0p5TbGFWldXBvCM1lpprfsC24AZSikz8A5wi+t1lwIzAEKtC6Ortdb9tdYDgeeAOa7yeDy/ACilBgHDcX0m4vz8AlyitR7g+vN5HMf7DM5E38P1+X3EVR53nwWtdY7HOR2AM0/8I9rx1puEr5RqCQwC3nMVvQcMUkplRzMOrfVyrfXuQGMLtS5MsR7WWi/xKPoO6AgMBkq01hVzcbwOXOZ6HGpdWGitj3lsNgEc8Xp+AZRSKTj/M97kURy359ePuItXKZUOXAU8orU2ALTWB+P5s+ARezLwO2BOtOOtNwkfaA/s1VrbAVx/73OVx1pNsYVaF1au1thNwEKgAx6/ULTWhwCzUqp5HerCGevflVK7gCeBq4nv8/s48I7WOsejLK7PL/CuUmqDUupVpVTTOI23K85ujD8qpVYrpZYopUYT35+FCpNcr/VjtOOtTwlf1M3fcPaJvxLrQGqjtb5ea90BeAh4Ntbx+KOUGgEMAV6NdSxBGKO17o9z8kIT8ft5sABdgLVa6yHA/cC/gfSYRhWYqVR2RUZVfUr4u4F2SikLgOvvtq7yWKsptlDrwsZ1obk7cLnW2gHswtm1U1HfAnBorQ/XoS7stNbzgPHAHuLz/J4K9AJ2uC6GngR8DnQjTs9vRXek1roU5xfVqDrEFMl4dwE2XF0aWuvvgUNAMfH5WcD1vO1wfi7edRVFNTfUm4TvuvK/DrjSVXQlzm//vNhF5VRTbKHWhSs2pdRfcPa1XuD6Tw6wBkhz/UQGuBH4oI514Yg1XSnV3mN7InAYiMvzq7WeobVuq7XupLXuhPOL6Sycv0ri8fw2Vko1cT02AVfgPD9x93lwdQ99DZzhircH0BLYTBx+FjxcDSzSWue73kdUP7v1anpkpVRPnMOUmgFHcA5T0lGO4WXgIqA1zhZHvta6d02xhVoXhlh7Axtx/icpdhXv0FpfqJQaifOqfyqVw+kOuo4LqS4M8bYCFgCNca6TcBi4V2v9YzyeXx/x5wDna+ewzHg8v12Aj3B2l1iAX4Dbtdb74zjeOTiHJZYDD2utP4vnz4JSajPOc7rYoyxq8darhC+EEMK/etOlI4QQomaS8IUQooGQhC+EEA2EJHwhhGggJOELIUQDIQlfCCEaCEn4QgjRQEjCF0KIBuL/Af94erlV74ofAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot all lines on one chart to see where one segment starts and another ends\n",
    "plt.plot(x_train, label = 'Train')\n",
    "plt.plot(x_valid, label = 'Validate')\n",
    "plt.plot(x_test, label = 'Test')\n",
    "plt.legend()\n",
    "print(x_train.index.max(),x_valid.index.min(),x_valid.index.max(),x_test.index.min(),x_test.index.max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D-8aiQRzmG7O"
   },
   "outputs": [],
   "source": [
    "# Reshape values\n",
    "x_train_values = x_train.values.reshape(-1, 1)\n",
    "x_valid_values = x_valid.values.reshape(-1, 1)\n",
    "x_test_values = x_test.values.reshape(-1, 1)\n",
    "\n",
    "#  Create Scaler Object\n",
    "x_train_scaler = MinMaxScaler(feature_range=(0, 1))\n",
    "\n",
    "# Fit x_train values\n",
    "normalized_x_train = x_train_scaler.fit_transform(x_train_values)\n",
    "\n",
    "# Fit x_valid values\n",
    "normalized_x_valid = x_train_scaler.transform(x_valid_values)\n",
    "\n",
    "# Fit x_test values\n",
    "normalized_x_test = x_train_scaler.transform(x_test_values)\n",
    "\n",
    "# All values normalized to training data\n",
    "spy_normalized_to_traindata = x_train_scaler.transform(series.values.reshape(-1, 1))\n",
    "\n",
    "# Example of how to iverse\n",
    "# inversed = scaler.inverse_transform(normalized_x_train).flatten()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vDs_w3kZ8OIw"
   },
   "source": [
    "## Simple RNN Forecasting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-rKEOyuFHN0N"
   },
   "source": [
    "The input for the memory cell at each time step is the batch size by our feature dimensionality (1). The output is thiese same two dimensions times the number of units in the memory cell. Our memory cell is comprised of 100 units in both layers. So the output of our RNN layer is batch(128), window_size(30), and number of units (100) which is obviously 3 dimensional. The output Y_0 is the state vector which is used when calculating Y_1 at the next time step. \n",
    "\n",
    "In this instance we are doing sequence to vector which means we ignore all outputs except for the one at the very last time step. This is the default behavior of all reccurent layers in Keras unless return_sequences = True is selected. This sequence to vector takes in a batch (128) of time windows and outputs the next time step of window of values. This one at a time output proves to be very slow when training.\n",
    "\n",
    "For a faster training convergence, we use a sequence to sequence RNN. Compared to the sequence to vector which adjusted the gradient of the loss from the very end of the model all the at layer 2 unit 100, the sequence to sequence RNN calculates the loss at each time step and backpropagates the loss from there. This provides much more gradients and speeds up training. It is important to note we still ignore all outputs besides the last one. We just calculate all intermediate values to update the gradient more quickly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "vbmXjeXSZr96",
    "outputId": "4f39e4e3-1cd6-4bc2-d3f0-19be3cb632b9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 1,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "f9L64bJOiZhK"
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YU4xRp9G8OIx"
   },
   "outputs": [],
   "source": [
    "# Clear any back end stored data due to multiple iterations\n",
    "keras.backend.clear_session()\n",
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "# Set window size\n",
    "window_size = 30\n",
    "\n",
    "# Create 2D batches of batch size and features (1 feature = 1 time step in window)\n",
    "train_set = window_dataset(normalized_x_train, window_size, batch_size=128)\n",
    "\n",
    "# Establish Model\n",
    "model = keras.models.Sequential([\n",
    "  keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=-1), # Add a 3rd dimension (feature dimensionality which in this case in univariate)\n",
    "                      input_shape=[None]), # 1st dimension is batch size, input shape = None allows windows of any size to be used\n",
    "  keras.layers.SimpleRNN(100, return_sequences=True), # takes in sequence (batch size, time, dimensionality per time step (univariate))\n",
    "  keras.layers.SimpleRNN(100), # produces a single vector\n",
    "  keras.layers.Dense(1), # produces 1 output \n",
    "])\n",
    "\n",
    "# create standard learning rate scheduler\n",
    "lr_schedule = keras.callbacks.LearningRateScheduler(\n",
    "    lambda epoch: 1e-5 * 10**(epoch / 20))\n",
    "\n",
    "# establish optimizer\n",
    "optimizer = keras.optimizers.Nadam(lr=1e-7)\n",
    "\n",
    "# Put model all together\n",
    "model.compile(loss=keras.losses.Huber(),\n",
    "              optimizer=optimizer,\n",
    "              metrics=[\"mae\"])\n",
    "history = model.fit(train_set, epochs=100, callbacks=[lr_schedule])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YJTlFAXF8OIy"
   },
   "outputs": [],
   "source": [
    "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n",
    "plt.axis([1e-5, 1, 0, .1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "T3yNjxWE8OIz"
   },
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "window_size = 30\n",
    "train_set = window_dataset(normalized_x_train, window_size, batch_size=128)\n",
    "valid_set = window_dataset(normalized_x_valid, window_size, batch_size=128)\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "  keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=-1),\n",
    "                      input_shape=[None]),\n",
    "  keras.layers.SimpleRNN(100, return_sequences=True),\n",
    "  keras.layers.SimpleRNN(100),\n",
    "  keras.layers.Dense(1),\n",
    "])\n",
    "optimizer = keras.optimizers.Nadam(lr=5e-5)\n",
    "\n",
    "model.compile(loss=keras.losses.Huber(),\n",
    "              optimizer=optimizer,\n",
    "              metrics=[\"mae\"])\n",
    "\n",
    "# Set early stopping to prevent over fitting\n",
    "early_stopping = keras.callbacks.EarlyStopping(patience=50)\n",
    "\n",
    "# save best model to load later\n",
    "model_checkpoint = keras.callbacks.ModelCheckpoint(\n",
    "    \"my_checkpoint\", save_best_only=True)\n",
    "\n",
    "# compile model\n",
    "model.fit(train_set, epochs=500,\n",
    "          validation_data=valid_set,\n",
    "          callbacks=[early_stopping, model_checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4KuPtKFe8OI0"
   },
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\"my_checkpoint\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cxq09qOg8OI1"
   },
   "outputs": [],
   "source": [
    "rnn_forecast = model_forecast(\n",
    "    model,\n",
    "    spy_normalized_to_traindata[x_test.index.min() - window_size:-1],\n",
    "    window_size)[:, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "p-ddUraXnXJo",
    "outputId": "110e2be8-b894-4b8e-ea29-bedde6f1a657"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(422,)"
      ]
     },
     "execution_count": 12,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnn_unscaled_forecast = x_train_scaler.inverse_transform(rnn_forecast.reshape(-1,1)).flatten()\n",
    "rnn_unscaled_forecast.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PkC_JssS8OI2"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10,6))\n",
    "\n",
    "plt.title('SeqtoVec RNN Forecast')\n",
    "plt.ylabel('Dollars $')\n",
    "plt.xlabel('Timestep in Days')\n",
    "plot_series(x_test.index, x_test)\n",
    "plot_series(x_test.index, rnn_unscaled_forecast)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1mwfgEK08OI3"
   },
   "outputs": [],
   "source": [
    "keras.metrics.mean_absolute_error(x_test, rnn_unscaled_forecast).numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KNG7s8jt8OI4"
   },
   "source": [
    "## Sequence-to-Sequence Forecasting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bsKGxfiE8OI4"
   },
   "outputs": [],
   "source": [
    "def seq2seq_window_dataset(series, window_size, batch_size=128,\n",
    "                           shuffle_buffer=1000):\n",
    "    series = tf.expand_dims(series, axis=-1)\n",
    "    ds = tf.data.Dataset.from_tensor_slices(series)\n",
    "    ds = ds.window(window_size + 1, shift=1, drop_remainder=True)\n",
    "    ds = ds.flat_map(lambda w: w.batch(window_size + 1))\n",
    "    ds = ds.shuffle(shuffle_buffer)\n",
    "    ds = ds.map(lambda w: (w[:-1], w[1:]))\n",
    "    return ds.batch(batch_size).prefetch(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EmbEO5kkcUnS"
   },
   "source": [
    "The cell below illustrates exactly what the function above is doing. the cell above creates batches and laebls for those batches. The point Y[0] is the label for X[0] to try to calculate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 212
    },
    "colab_type": "code",
    "id": "5Nk2C7WP8OI5",
    "outputId": "bd1382ca-6f4a-4dda-8378-72c8e3f711d2"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-bf466f274a9a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m for X_batch, Y_batch in seq2seq_window_dataset(tf.range(10), 3,\n\u001b[0m\u001b[1;32m      2\u001b[0m                                                batch_size=1):\n\u001b[1;32m      3\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"X:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_batch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Y:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_batch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'seq2seq_window_dataset' is not defined"
     ]
    }
   ],
   "source": [
    "# Show example of what seq-seq looks like\n",
    "for X_batch, Y_batch in seq2seq_window_dataset(tf.range(10), 3, batch_size=1):\n",
    "    print(\"X:\", X_batch.numpy())\n",
    "    print(\"Y:\", Y_batch.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4JSc-Btk8OI7"
   },
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "window_size = 30\n",
    "\n",
    "# in the seq2seq_window we removed the need for the lambda layer to expand dimensions as it is already 3D \n",
    "train_set = seq2seq_window_dataset(normalized_x_train.flatten(), window_size,\n",
    "                                   batch_size=128)\n",
    "\n",
    "# Create model\n",
    "model = keras.models.Sequential([\n",
    "  keras.layers.SimpleRNN(100, return_sequences=True,\n",
    "                         input_shape=[None, 1]),\n",
    "  keras.layers.SimpleRNN(100, return_sequences=True),\n",
    "  keras.layers.Dense(1), # now dense layer is applied at every time step\n",
    "])\n",
    "\n",
    "lr_schedule = keras.callbacks.LearningRateScheduler(\n",
    "    lambda epoch: 1e-5 * 10**(epoch / 30))\n",
    "\n",
    "# choose optimizer\n",
    "optimizer = keras.optimizers.Nadam(lr=1e-5)\n",
    "\n",
    "# compile model\n",
    "model.compile(loss=keras.losses.Huber(),\n",
    "              optimizer=optimizer,\n",
    "              metrics=[\"mae\"])\n",
    "\n",
    "# create history callback from fit\n",
    "history = model.fit(train_set, epochs=100, callbacks=[lr_schedule])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YGNsWceq8OI8"
   },
   "outputs": [],
   "source": [
    "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n",
    "plt.axis([1e-5, 1, 0, .1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "G9lDnb0X8OI9"
   },
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "# set window size and create input batch sequences\n",
    "window_size = 20\n",
    "train_set = seq2seq_window_dataset(normalized_x_train.flatten(), window_size,\n",
    "                                   batch_size=128)\n",
    "valid_set = seq2seq_window_dataset(normalized_x_valid.flatten(), window_size,\n",
    "                                   batch_size=128)\n",
    "\n",
    "# Create model for seq:seq RNN\n",
    "model = keras.models.Sequential([\n",
    "  keras.layers.SimpleRNN(100, return_sequences=True,\n",
    "                         input_shape=[None, 1]),\n",
    "  keras.layers.SimpleRNN(100, return_sequences=True),\n",
    "  keras.layers.Dense(1),\n",
    "])\n",
    "\n",
    "# choose optimizer and LR\n",
    "optimizer = keras.optimizers.Nadam(lr=1e-3)\n",
    "\n",
    "# set model params\n",
    "model.compile(loss=keras.losses.Huber(),\n",
    "              optimizer=optimizer,\n",
    "              metrics=[\"mae\"])\n",
    "\n",
    "# set early stopping\n",
    "early_stopping = keras.callbacks.EarlyStopping(patience=20)\n",
    "\n",
    "# set model checkpoint to save best model\n",
    "model_checkpoint = keras.callbacks.ModelCheckpoint(\n",
    "    \"my_checkpoint\", save_best_only=True)\n",
    "\n",
    "# fit model\n",
    "model.fit(train_set, epochs=500,\n",
    "          validation_data=valid_set,\n",
    "          callbacks=[early_stopping, model_checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iGlOWOtPz3fs"
   },
   "outputs": [],
   "source": [
    "# recall best model\n",
    "model = keras.models.load_model(\"my_checkpoint\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4mglBRex8OI_"
   },
   "outputs": [],
   "source": [
    "# create forecast and clip to only show test values\n",
    "rnn_forecast = model_forecast(model, spy_normalized_to_traindata.flatten()[..., np.newaxis], window_size)\n",
    "rnn_forecast = rnn_forecast[x_test.index.min() - window_size:-1, -1, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "W3QO5IfmxmSD",
    "outputId": "b40ee48f-2c98-45f3-c4e6-791b8af628a3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(422,)"
      ]
     },
     "execution_count": 29,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get data back to normal scale\n",
    "rnn_unscaled_forecast = x_train_scaler.inverse_transform(rnn_forecast.reshape(-1,1)).flatten()\n",
    "rnn_unscaled_forecast.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Zl_FkcdI8OJA"
   },
   "outputs": [],
   "source": [
    "# Plot results\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.title('SeqtoSeq RNN')\n",
    "plt.ylabel('Dollars $')\n",
    "plt.xlabel('Timestep in Days')\n",
    "plot_series(x_test.index, x_test)\n",
    "plot_series(x_test.index, rnn_unscaled_forecast)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cznEtSVK8OJB"
   },
   "outputs": [],
   "source": [
    "keras.metrics.mean_absolute_error(x_test, rnn_unscaled_forecast).numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Bnb6Ngxcx8t-"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "vidayERjaO5q"
   ],
   "name": "RNN_seqtovec_seqtoseq.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}