S+P_Week_4_Lesson_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zX4Kg8DUTKWO"
   },
   "outputs": [],
   "source": [
    "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "# https://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/lmoroney/dlaicourse/blob/master/TensorFlow%20In%20Practice/Course%204%20-%20S%2BP/S%2BP%20Week%204%20Lesson%201.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "id": "D1J15Vh_1Jih"
   },
   "outputs": [],
   "source": [
    "try:\n",
    "  # %tensorflow_version only exists in Colab.\n",
    "  %tensorflow_version 2.x\n",
    "except Exception:\n",
    "  pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BOjujz601HcS"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Zswl7jRtGzkk"
   },
   "outputs": [],
   "source": [
    "def plot_series(time, series, format=\"-\", start=0, end=None):\n",
    "    plt.plot(time[start:end], series[start:end], format)\n",
    "    plt.xlabel(\"Time\")\n",
    "    plt.ylabel(\"Value\")\n",
    "    plt.grid(True)\n",
    "\n",
    "def trend(time, slope=0):\n",
    "    return slope * time\n",
    "\n",
    "def seasonal_pattern(season_time):\n",
    "    \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n",
    "    return np.where(season_time < 0.4,\n",
    "                    np.cos(season_time * 2 * np.pi),\n",
    "                    1 / np.exp(3 * season_time))\n",
    "\n",
    "def seasonality(time, period, amplitude=1, phase=0):\n",
    "    \"\"\"Repeats the same pattern at each period\"\"\"\n",
    "    season_time = ((time + phase) % period) / period\n",
    "    return amplitude * seasonal_pattern(season_time)\n",
    "\n",
    "def noise(time, noise_level=1, seed=None):\n",
    "    rnd = np.random.RandomState(seed)\n",
    "    return rnd.randn(len(time)) * noise_level\n",
    "\n",
    "time = np.arange(4 * 365 + 1, dtype=\"float32\")\n",
    "baseline = 10\n",
    "series = trend(time, 0.1)  \n",
    "baseline = 10\n",
    "amplitude = 40\n",
    "slope = 0.05\n",
    "noise_level = 5\n",
    "\n",
    "# Create the series\n",
    "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n",
    "# Update with noise\n",
    "series += noise(time, noise_level, seed=42)\n",
    "\n",
    "split_time = 1000\n",
    "time_train = time[:split_time]\n",
    "x_train = series[:split_time]\n",
    "time_valid = time[split_time:]\n",
    "x_valid = series[split_time:]\n",
    "\n",
    "window_size = 20\n",
    "batch_size = 32\n",
    "shuffle_buffer_size = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4sTTIOCbyShY"
   },
   "outputs": [],
   "source": [
    "def windowed_dataset(series, window_size, batch_size, shuffle_buffer):\n",
    "    series = tf.expand_dims(series, axis=-1)\n",
    "    ds = tf.data.Dataset.from_tensor_slices(series)\n",
    "    ds = ds.window(window_size + 1, shift=1, drop_remainder=True)\n",
    "    ds = ds.flat_map(lambda w: w.batch(window_size + 1))\n",
    "    ds = ds.shuffle(shuffle_buffer)\n",
    "    ds = ds.map(lambda w: (w[:-1], w[1:]))\n",
    "    return ds.batch(batch_size).prefetch(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_eaAX9g_jS5W"
   },
   "outputs": [],
   "source": [
    "def model_forecast(model, series, window_size):\n",
    "    ds = tf.data.Dataset.from_tensor_slices(series)\n",
    "    ds = ds.window(window_size, shift=1, drop_remainder=True)\n",
    "    ds = ds.flat_map(lambda w: w.batch(window_size))\n",
    "    ds = ds.batch(32).prefetch(1)\n",
    "    forecast = model.predict(ds)\n",
    "    return forecast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Yqc2GTsps0qf"
   },
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "tf.random.set_seed(51)\n",
    "np.random.seed(51)\n",
    "\n",
    "window_size = 30\n",
    "train_set = windowed_dataset(x_train, window_size, batch_size=128, shuffle_buffer=shuffle_buffer_size)\n",
    "\n",
    "model = tf.keras.models.Sequential([\n",
    "  tf.keras.layers.Conv1D(filters=32, kernel_size=5,\n",
    "                      strides=1, padding=\"causal\",\n",
    "                      activation=\"relu\",\n",
    "                      input_shape=[None, 1]),\n",
    "  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32, return_sequences=True)),\n",
    "  tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32, return_sequences=True)),\n",
    "  tf.keras.layers.Dense(1),\n",
    "  tf.keras.layers.Lambda(lambda x: x * 200)\n",
    "])\n",
    "lr_schedule = tf.keras.callbacks.LearningRateScheduler(\n",
    "    lambda epoch: 1e-8 * 10**(epoch / 20))\n",
    "optimizer = tf.keras.optimizers.SGD(lr=1e-8, momentum=0.9)\n",
    "model.compile(loss=tf.keras.losses.Huber(),\n",
    "              optimizer=optimizer,\n",
    "              metrics=[\"mae\"])\n",
    "history = model.fit(train_set, epochs=100, callbacks=[lr_schedule])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MjTvASUns0qh"
   },
   "outputs": [],
   "source": [
    "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n",
    "plt.axis([1e-8, 1e-4, 0, 30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4uh-97bpLZCA"
   },
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "tf.random.set_seed(51)\n",
    "np.random.seed(51)\n",
    "#batch_size = 16\n",
    "dataset = windowed_dataset(x_train, window_size, batch_size, shuffle_buffer_size)\n",
    "\n",
    "model = tf.keras.models.Sequential([\n",
    "  tf.keras.layers.Conv1D(filters=32, kernel_size=3,\n",
    "                      strides=1, padding=\"causal\",\n",
    "                      activation=\"relu\",\n",
    "                      input_shape=[None, 1]),\n",
    "  tf.keras.layers.LSTM(32, return_sequences=True),\n",
    "  tf.keras.layers.LSTM(32, return_sequences=True),\n",
    "  tf.keras.layers.Dense(1),\n",
    "  tf.keras.layers.Lambda(lambda x: x * 200)\n",
    "])\n",
    "\n",
    "optimizer = tf.keras.optimizers.SGD(lr=1e-5, momentum=0.9)\n",
    "model.compile(loss=tf.keras.losses.Huber(),\n",
    "              optimizer=optimizer,\n",
    "              metrics=[\"mae\"])\n",
    "history = model.fit(dataset,epochs=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MKkic-mLdkRZ"
   },
   "outputs": [],
   "source": [
    "rnn_forecast = model_forecast(model, series[..., np.newaxis], window_size)\n",
    "rnn_forecast = rnn_forecast[split_time - window_size:-1, -1, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4N1toSetdnQq"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plot_series(time_valid, x_valid)\n",
    "plot_series(time_valid, rnn_forecast)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ULKO3JINdqkp"
   },
   "outputs": [],
   "source": [
    "tf.keras.metrics.mean_absolute_error(x_valid, rnn_forecast).numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ok8LjNbbkig4"
   },
   "outputs": [],
   "source": [
    "import matplotlib.image  as mpimg\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#-----------------------------------------------------------\n",
    "# Retrieve a list of list results on training and test data\n",
    "# sets for each training epoch\n",
    "#-----------------------------------------------------------\n",
    "mae=history.history['mae']\n",
    "loss=history.history['loss']\n",
    "\n",
    "epochs=range(len(loss)) # Get number of epochs\n",
    "\n",
    "#------------------------------------------------\n",
    "# Plot MAE and Loss\n",
    "#------------------------------------------------\n",
    "plt.plot(epochs, mae, 'r')\n",
    "plt.plot(epochs, loss, 'b')\n",
    "plt.title('MAE and Loss')\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.legend([\"MAE\", \"Loss\"])\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "epochs_zoom = epochs[200:]\n",
    "mae_zoom = mae[200:]\n",
    "loss_zoom = loss[200:]\n",
    "\n",
    "#------------------------------------------------\n",
    "# Plot Zoomed MAE and Loss\n",
    "#------------------------------------------------\n",
    "plt.plot(epochs_zoom, mae_zoom, 'r')\n",
    "plt.plot(epochs_zoom, loss_zoom, 'b')\n",
    "plt.title('MAE and Loss')\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.legend([\"MAE\", \"Loss\"])\n",
    "\n",
    "plt.figure()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "S+P Week 4 Lesson 1.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}