SSD_car_detection_keras2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ===== Car detecion using SSD start =====#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "import keras\n",
    "from keras.applications.imagenet_utils import preprocess_input\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "from keras.models import Model\n",
    "from keras.preprocessing import image\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from scipy.misc import imread\n",
    "import tensorflow as tf\n",
    "\n",
    "# ===== import SSD for keras 2 =====\n",
    "from ssd_v2 import SSD300v2\n",
    "from ssd_utils import BBoxUtility\n",
    "# ===== import SSD for keras 2 =====\n",
    "\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (8, 8)\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "\n",
    "np.set_printoptions(suppress=True)\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.95#0.45\n",
    "set_session(tf.Session(config=config))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "voc_classes = ['Aeroplane', 'Bicycle', 'Bird', 'Boat', 'Bottle',\n",
    "               'Bus', 'Car', 'Cat', 'Chair', 'Cow', 'Diningtable',\n",
    "               'Dog', 'Horse','Motorbike', 'Person', 'Pottedplant',\n",
    "               'Sheep', 'Sofa', 'Train', 'Tvmonitor']\n",
    "NUM_CLASSES = len(voc_classes) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_shape=(300, 300, 3)\n",
    "model = SSD300(input_shape, num_classes=NUM_CLASSES)\n",
    "model.load_weights('weights_SSD300.hdf5', by_name=True)\n",
    "bbox_util = BBoxUtility(NUM_CLASSES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_bbox(xmin, ymin, xmax, ymax):\n",
    "    return [xmin, ymin, xmax, ymax]\n",
    "\n",
    "def center_is_near(prev_bbox, bbox):\n",
    "    IS_NEAR_THRESHOLD = 30\n",
    "    \n",
    "    prev_center_x = (prev_bbox[0] + prev_bbox[2])/2.\n",
    "    prev_center_y = (prev_bbox[1] + prev_bbox[3])/2.\n",
    "    center_x = (bbox[0] + bbox[2])/2.\n",
    "    center_y = (bbox[1] + bbox[3])/2.\n",
    "    \n",
    "    dist = np.sqrt((prev_center_x - center_x)**2 + (prev_center_y - center_y)**2)\n",
    "    \n",
    "    if dist <= IS_NEAR_THRESHOLD:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "def draw_boxes(img, preds, results):\n",
    "    global first_frame_has_car, prev_bboxes, prev_bboxes_len, bbox_disappear_frame_count\n",
    "    \n",
    "    # Parse the outputs.\n",
    "    det_label = results[0][:, 0]\n",
    "    det_conf = results[0][:, 1]\n",
    "    det_xmin = results[0][:, 2]\n",
    "    det_ymin = results[0][:, 3]\n",
    "    det_xmax = results[0][:, 4]\n",
    "    det_ymax = results[0][:, 5]\n",
    "\n",
    "    # Get detections with confidence higher than 0.6.\n",
    "    top_indices = [i for i, conf in enumerate(det_conf) if conf >= 0.6]\n",
    "\n",
    "    top_conf = det_conf[top_indices]\n",
    "    top_label_indices = det_label[top_indices].tolist()\n",
    "    top_xmin = det_xmin[top_indices]\n",
    "    top_ymin = det_ymin[top_indices]\n",
    "    top_xmax = det_xmax[top_indices]\n",
    "    top_ymax = det_ymax[top_indices]\n",
    "    \n",
    "    bboxes_len = 0\n",
    "    bboxes = []   \n",
    "    for i in range(top_conf.shape[0]):\n",
    "        \n",
    "        label = int(top_label_indices[i])        \n",
    "        label_name = voc_classes[label - 1]\n",
    "        \n",
    "        if label_name == 'Car':\n",
    "            bboxes_len += 1\n",
    "            \n",
    "            xmin = int(round(top_xmin[i] * img.shape[1]))\n",
    "            ymin = int(round(top_ymin[i] * img.shape[0]))\n",
    "            xmax = int(round(top_xmax[i] * img.shape[1]))\n",
    "            ymax = int(round(top_ymax[i] * img.shape[0]))\n",
    "            \n",
    "            if first_frame_has_car or len(prev_bboxes) == 0:\n",
    "                prev_bboxes.append(get_bbox(xmin, ymin, xmax, ymax))\n",
    "                first_frame_has_car = False\n",
    "                prev_bboxes_len = 0\n",
    "            else:\n",
    "                has_near_in_prev_bboxes = False\n",
    "                for i_prev_bbox in range(len(prev_bboxes)):\n",
    "                    if center_is_near(prev_bboxes[i_prev_bbox], [xmin, ymin, xmax, ymax]):\n",
    "                        ratiox = 0.5\n",
    "                        ratioy = 0.65\n",
    "                        xmin = int((1-ratiox)*xmin + ratiox*prev_bboxes[i_prev_bbox][0])\n",
    "                        ymin = int((1-ratioy)*ymin + ratioy*prev_bboxes[i_prev_bbox][1])\n",
    "                        xmax = int((1-ratiox)*xmax + ratiox*prev_bboxes[i_prev_bbox][2])\n",
    "                        ymax = int((1-ratioy)*ymax + ratioy*prev_bboxes[i_prev_bbox][3])\n",
    "                        prev_bboxes[i_prev_bbox][0] = xmin \n",
    "                        prev_bboxes[i_prev_bbox][1] = ymin\n",
    "                        prev_bboxes[i_prev_bbox][2] = xmax\n",
    "                        prev_bboxes[i_prev_bbox][3] = ymax\n",
    "                        has_near_in_prev_bboxes = True\n",
    "                        \n",
    "                if not has_near_in_prev_bboxes:\n",
    "                    prev_bboxes.append(get_bbox(xmin, ymin, xmax, ymax))\n",
    "                    \n",
    "            bboxes.append(get_bbox(xmin, ymin, xmax, ymax))  \n",
    "           \n",
    "    if prev_bboxes_len > bboxes_len and bbox_disappear_frame_count < 5:\n",
    "        for i_prev_bbox in range(len(prev_bboxes)):\n",
    "            for i_bbox in range(len(bboxes)):\n",
    "                if not center_is_near(prev_bboxes[i_prev_bbox], bboxes[i_bbox]):\n",
    "                    cv2.rectangle(img, \n",
    "                                  (prev_bboxes[i_prev_bbox][0],prev_bboxes[i_prev_bbox][1]), \n",
    "                                  (prev_bboxes[i_prev_bbox][2],prev_bboxes[i_prev_bbox][3]), (0,255,0), 5)\n",
    "            if len(bboxes) == 0:\n",
    "                cv2.rectangle(img, \n",
    "                              (prev_bboxes[i_prev_bbox][0],prev_bboxes[i_prev_bbox][1]), \n",
    "                              (prev_bboxes[i_prev_bbox][2],prev_bboxes[i_prev_bbox][3]), (0,255,0), 5)\n",
    "        bbox_disappear_frame_count += 1\n",
    "    else:\n",
    "        bbox_disappear_frame_count = 0\n",
    "        prev_bboxes_len = len(bboxes)\n",
    "        prev_bboxes = bboxes\n",
    "    for i_bbox in range(len(bboxes)):\n",
    "        cv2.rectangle(img, (bboxes[i_bbox][0],bboxes[i_bbox][1]), (bboxes[i_bbox][2],bboxes[i_bbox][3]), (0,255,0), 5)  \n",
    "            \n",
    "    if len(prev_bboxes) > 10:\n",
    "        prev_bboxes = []\n",
    "        bbox_disappear_frame_count = 10\n",
    "        \n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_video(input_img):\n",
    "    \n",
    "    inputs = []\n",
    "    #input_img_cropped = input_img[120:720,680:1280,:]\n",
    "    #img = cv2.resize(input_img_cropped, (300, 300))\n",
    "    img = cv2.resize(input_img, (300, 300))\n",
    "    img = image.img_to_array(img)\n",
    "    inputs.append(img.copy())\n",
    "    inputs = preprocess_input(np.array(inputs))\n",
    "    inputs = np.expand_dims(inputs[0], axis=0)\n",
    "    \n",
    "    preds = model.predict(inputs, batch_size=1, verbose=0)\n",
    "    results = bbox_util.detection_out(preds)\n",
    "    \n",
    "    final_img = draw_boxes(input_img, preds, results)\n",
    "    \n",
    "    return final_img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[MoviePy] >>>> Building video project_video_video_SSD_smooth_disappear.mp4\n",
      "[MoviePy] Writing video project_video_video_SSD_smooth_disappear.mp4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 1260/1261 [01:42<00:00, 12.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[MoviePy] Done.\n",
      "[MoviePy] >>>> Video ready: project_video_video_SSD_smooth_disappear.mp4 \n",
      "\n",
      "CPU times: user 37.1 s, sys: 3.25 s, total: 40.4 s\n",
      "Wall time: 1min 44s\n"
     ]
    }
   ],
   "source": [
    "from moviepy.editor import VideoFileClip\n",
    "from IPython.display import HTML\n",
    "\n",
    "first_frame_has_car = True\n",
    "prev_bboxes = []\n",
    "bbox_disappear_frame_count = 0\n",
    "prev_bboxes_len = 0\n",
    "\n",
    "output = 'project_video_video_SSD_smooth_disappear.mp4'\n",
    "clip1 = VideoFileClip(\"project_video.mp4\")\n",
    "clip = clip1.fl_image(process_video) #NOTE: this function expects color images!!\n",
    "%time clip.write_videofile(output, audio=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ===== Car detection using SSD end =====#\n",
    "\n",
    "# Advanced Lane Detection#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Import\n",
    "import numpy as np\n",
    "import cv2\n",
    "import glob\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "import pickle\n",
    "import time\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import skimage \n",
    "from skimage import io\n",
    "from skimage.color import rgb2lab\n",
    "import numpy as np\n",
    "import scipy.ndimage as ndi\n",
    "from scipy import signal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Load Camera Calibration Pickle Data\n",
    "dist_data = pickle.load( open( \"dist_pickle.p\", \"rb\" ) )\n",
    "camera_mtx = dist_data[\"mtx\"]\n",
    "camera_dist = dist_data[\"dist\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Perspective Transform\n",
    "\n",
    "def undistort_img(image):\n",
    "    # Undistort test image\n",
    "    undist_image = cv2.undistort(image, camera_mtx, camera_dist, None, camera_mtx)\n",
    "\n",
    "    img_size = (undist_image.shape[1], undist_image.shape[0])\n",
    "    \n",
    "    # Lane line vertices\n",
    "    # Upper and low are based on visual locations, not grid locations\n",
    "    center_x = img_size[0]//2\n",
    "    upper_y = img_size[1]//1.58\n",
    "    low_y = img_size[1]\n",
    "    upper_left_x = center_x//1.18\n",
    "    upper_right_x = center_x//0.83\n",
    "    low_left_x = 0\n",
    "    low_right_x = 2*center_x\n",
    "    \n",
    "    # Calculate source points based on fractions of imade dimensions\n",
    "    src_corners = np.float32([[low_left_x, low_y], \n",
    "                              [upper_left_x, upper_y], \n",
    "                              [upper_right_x, upper_y],\n",
    "                              [low_right_x, low_y]])\n",
    "   \n",
    "    \n",
    "    # Calculate destination points based on entire image's dimensions.\n",
    "    dst_corners = np.float32([[0, img_size[1]],\n",
    "                              [0, 0],\n",
    "                              [img_size[0],0],\n",
    "                              [img_size[0], img_size[1]]])\n",
    "    \n",
    "    return undist_image, src_corners, dst_corners\n",
    "\n",
    "def perspective_transform(image):\n",
    "    # Calculate perspective transform\n",
    "    \n",
    "    undist_image, src_corners, dst_corners = undistort_img(image)    \n",
    "    \n",
    "    img_size = (undist_image.shape[1], undist_image.shape[0])\n",
    "    \n",
    "    M = cv2.getPerspectiveTransform(src_corners, dst_corners)\n",
    "\n",
    "    warped = cv2.warpPerspective(undist_image, M, img_size)\n",
    "    \n",
    "    M_inv = cv2.getPerspectiveTransform(dst_corners, src_corners)\n",
    "    \n",
    "    \"\"\"\n",
    "    # Draw points and lines to mark region for transform\n",
    "    for i in range(4):\n",
    "        cv2.circle(undist_image, (src_corners[i,0], src_corners[i,1]), 6, (255, 0, 0), 6)\n",
    "    for i in range(4):\n",
    "        cv2.line(undist_image, \n",
    "                 (src_corners[i-1,0], src_corners[i-1,1]), \n",
    "                 (src_corners[i,0], src_corners[i,1]),  \n",
    "                 (0,255,0), 2)\n",
    "    \"\"\"\n",
    "        \n",
    "    return warped, M_inv\n",
    "def color_threshold(image):\n",
    "    # Debug rule of thumb: check cv2.cvtColor(blurred_warped, cv2.COLOR_RGB2LAB)\n",
    "\n",
    "    # color_threshold: lab\n",
    "    #undist_lab = rgb2lab(skimage.img_as_float(image)).astype(np.uint8)#cv2.cvtColor(img, cv2.COLOR_RGB2Lab)\n",
    "    undist_lab = cv2.cvtColor(image, cv2.COLOR_RGB2LAB)\n",
    "    undist_B = undist_lab[:,:,2]\n",
    "    _, binary_lab_B = cv2.threshold(undist_B, 150, 255, cv2.THRESH_BINARY)\n",
    "    \"\"\"\n",
    "    # color_threshold: luv\n",
    "    undist_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)\n",
    "    undist_L = undist_luv[:,:,0]\n",
    "    \"\"\"\n",
    "    undist_L = undist_lab[:,:,0]\n",
    "    thresh_L = (210, 255)\n",
    "    binary_luv_L = np.zeros_like(undist_L)\n",
    "    _, binary_luv_L = cv2.threshold(undist_L, thresh_L[0], thresh_L[1], cv2.THRESH_BINARY)\n",
    "    # color_threshold: hsv\n",
    "    undist_hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)\n",
    "    undist_S = undist_hsv[:,:,2]\n",
    "    thresh_S = (230, 255)\n",
    "    binary_hsv_S = np.zeros_like(undist_S)\n",
    "    _, binary_hsv_S = cv2.threshold(undist_S, thresh_S[0], thresh_S[1], cv2.THRESH_BINARY)\n",
    "    # sobelx threshold\n",
    "    kernel = np.ones((3,3),np.uint8)\n",
    "    undist_gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n",
    "    binary_gray = cv2.Sobel(undist_gray,cv2.CV_32F,1,0,ksize=5) # threshold doesn't work on cv2.CV_64F\n",
    "    binary_gray = np.sqrt(binary_gray**2)/binary_gray.max()*255.\n",
    "    _, binary_gray = cv2.threshold(binary_gray, 90, 255, cv2.THRESH_BINARY)    \n",
    "    binary_gray = cv2.erode(binary_gray,kernel,iterations = 1)\n",
    "    thresh_S2 = (150, 255)\n",
    "    _, binary_hsv_S2 = cv2.threshold(undist_S, thresh_S2[0], thresh_S2[1], cv2.THRESH_BINARY)\n",
    "    binary_hsv_S2 = cv2.dilate(binary_hsv_S2, kernel, iterations = 1)\n",
    "    binary_gray2 = np.zeros_like(undist_S)\n",
    "    binary_gray2[(binary_gray == 255) & (binary_hsv_S2 == 255)] = 255    \n",
    "\n",
    "    combined_color_binary = np.zeros_like(binary_lab_B)\n",
    "    combined_color_binary[(binary_lab_B == 255) | (binary_luv_L == 255) | (binary_gray2 == 255) | (binary_hsv_S == 255)] = 255\n",
    "    # !\n",
    "    kernel_comb = np.ones((3,3),np.uint8)\n",
    "    combined_color_binary = cv2.morphologyEx(combined_color_binary, cv2.MORPH_OPEN, kernel_comb)\n",
    "    kernel_comb = np.ones((8,8),np.uint8)\n",
    "    combined_color_binary = cv2.morphologyEx(combined_color_binary, cv2.MORPH_CLOSE, kernel_comb)\n",
    "    \n",
    "    return combined_color_binary\n",
    "\n",
    "def find_boxes(image, img_warped):\n",
    "    img_draw = np.array(img_warped)\n",
    "\n",
    "    steps = 16\n",
    "    win_size = image.shape[0]//steps\n",
    "    half_frame = image.shape[1] // 2\n",
    "    medianfilt_kernel_size = 45\n",
    "\n",
    "    left_peaks_xy = []\n",
    "    right_peaks_xy = [] \n",
    "    \n",
    "    for i in range(steps):\n",
    "        histogram = np.sum(image[(i)*win_size:(i+1)*win_size,:], axis=0)\n",
    "        histogram_smooth = signal.medfilt(histogram, medianfilt_kernel_size)\n",
    "\n",
    "        # Find left/right peak(s)\n",
    "        left_peaks = np.array(signal.find_peaks_cwt(histogram_smooth[:half_frame], np.arange(10, 50)))\n",
    "        right_peaks = np.array(signal.find_peaks_cwt(histogram_smooth[half_frame:], np.arange(5, 20))) + half_frame\n",
    "\n",
    "        # Draw boxes that contains a hist. peak\n",
    "        if left_peaks.any():\n",
    "            ind = np.argmax(left_peaks) # if there are multiple peaks, choose that on the right\n",
    "            cv2.rectangle(img_draw,(left_peaks[ind]-win_size//2,i*win_size), (left_peaks[ind]+win_size//2,(i+1)*win_size), (0,255,0), 3)\n",
    "            # append (x,y) coords. \n",
    "            # (x,y): center of the boxes int pixel coord. (width, height)\n",
    "            left_peaks_xy.append(np.array([left_peaks[ind], (2*i+1)*win_size//2]))\n",
    "        if right_peaks.any():\n",
    "            ind = np.argmax(right_peaks)\n",
    "            cv2.rectangle(img_draw,(right_peaks[ind]-win_size//2,i*win_size), (right_peaks[ind]+win_size//2,(i+1)*win_size), (0,255,0), 3)\n",
    "            # append (x,y) coords.\n",
    "            # (x,y): center of the boxes int pixel coord. (width, height)\n",
    "            right_peaks_xy.append(np.array([right_peaks[ind], (2*i+1)*win_size//2]))    \n",
    "            \n",
    "    return left_peaks_xy, right_peaks_xy, img_draw\n",
    "\n",
    "\n",
    "def param_polyfit(img_height, left_peaks_xy, right_peaks_xy):\n",
    "    # polyfit\n",
    "    global prev_left_fit, prev_right_fit\n",
    "    global frame_count\n",
    "    curve_limit = 5.3e-4\n",
    "    \n",
    "    steps = 16#30\n",
    "    y_steps = np.arange(0,img_height,img_height//steps)\n",
    "    y_steps = np.append(y_steps, img_height)\n",
    "    damp_ratio = 0.85\n",
    "    \n",
    "    x = [xy[0] for xy in left_peaks_xy]\n",
    "    y = [xy[1] for xy in left_peaks_xy]\n",
    "    if frame_count == 0:         \n",
    "        left_fit = np.polyfit(y , x, 2)\n",
    "        prev_left_fit = left_fit\n",
    "    elif len(left_peaks_xy) == 0:\n",
    "        left_fit = prev_left_fit\n",
    "    else: \n",
    "        left_fit = np.polyfit(y , x, 2)\n",
    "        #left_fit[0] = np.clip(left_fit[0], a_min=-1*curve_limit, a_max=curve_limit)# limit max min curvature\n",
    "        left_fit = damp_ratio*prev_left_fit + (1-damp_ratio)*left_fit\n",
    "        prev_left_fit = left_fit\n",
    "    left_fit_x = left_fit[0]*y_steps**2 + left_fit[1]*y_steps  + left_fit[2]\n",
    "    \n",
    "    x = [xy[0] for xy in right_peaks_xy]\n",
    "    y = [xy[1] for xy in right_peaks_xy]      \n",
    "    if frame_count == 0: \n",
    "        right_fit = np.polyfit(y, x, 2)  \n",
    "        prev_right_fit = right_fit\n",
    "    elif len(right_peaks_xy) == 0:\n",
    "        right_fit = prev_right_fit\n",
    "    else: \n",
    "        right_fit = np.polyfit(y, x, 2)\n",
    "        #right_fit[0] = np.clip(right_fit[0], a_min=-1*curve_limit, a_max=curve_limit)# limit max min curvature\n",
    "        right_fit = damp_ratio*prev_right_fit + (1-damp_ratio)*right_fit\n",
    "        prev_right_fit = right_fit\n",
    "    right_fit_x = right_fit[0]*y_steps**2 + right_fit[1]*y_steps + right_fit[2]        \n",
    "    \n",
    "    frame_count += 1\n",
    "    if frame_count >= 1e6:\n",
    "        frame_count = 1\n",
    "    \n",
    "    return left_fit_x, right_fit_x, y_steps        \n",
    "        \n",
    "def draw_line_and_poly(image, left_fit_x, right_fit_x, y_steps):\n",
    "    img_draw2 = np.zeros_like(image)\n",
    "\n",
    "    for i in range(len(left_fit_x)-1):\n",
    "        #cv2.circle(img_draw2, (int(left_fit_x[i]), int(y_steps[i])), 20, (0,255,0),-1)\n",
    "        #cv2.circle(img_draw2, (int(right_fit_x[i]), int(y_steps[i])), 20, (0,255,0),-1)\n",
    "        poly_pnts = [(int(left_fit_x[i]), int(y_steps[i])), (int(right_fit_x[i]), int(y_steps[i])),\n",
    "                    (int(right_fit_x[i+1]), int(y_steps[i+1])), (int(left_fit_x[i+1]), int(y_steps[i+1]))]\n",
    "        cv2.fillConvexPoly(img_draw2, np.array(poly_pnts), (50,255,50))\n",
    "        line_pnt1 = (int(left_fit_x[i]), int(y_steps[i]))\n",
    "        line_pnt2 = (int(left_fit_x[i+1]), int(y_steps[i+1]))\n",
    "        cv2.line(img_draw2, line_pnt1, line_pnt2, (0,0,255), 30)\n",
    "        line_pnt1 = (int(right_fit_x[i]), int(y_steps[i]))\n",
    "        line_pnt2 = (int(right_fit_x[i+1]), int(y_steps[i+1]))\n",
    "        cv2.line(img_draw2, line_pnt1, line_pnt2, (0,0,255), 30)\n",
    "    \n",
    "    return img_draw2\n",
    "\n",
    "def draw_on_original_img(image, img_line_poly, M_inv):\n",
    "    new_warp = cv2.warpPerspective(img_line_poly, M_inv, (image.shape[1], image.shape[0]))\n",
    "    new_img = cv2.addWeighted(image, 1, new_warp, 0.5, 0)\n",
    "    \n",
    "    return new_img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Video pipeline\n",
    "def process_video(input_img):\n",
    "    img_test = input_img\n",
    "    \n",
    "    img_width = img_test.shape[1]\n",
    "    img_height = img_test.shape[0]\n",
    "\n",
    "    warped, M_inv = perspective_transform(img_test)\n",
    "    blurred_warped = cv2.GaussianBlur(warped,(5,5),0)\n",
    "\n",
    "    img_binary_lines = color_threshold(blurred_warped)\n",
    "\n",
    "    left_peaks_xy, right_peaks_xy, _ = find_boxes(img_binary_lines, warped)\n",
    "\n",
    "    left_fit_x, right_fit_x, y_steps = param_polyfit(img_height, left_peaks_xy, right_peaks_xy)\n",
    "\n",
    "    img_draw = draw_line_and_poly(img_test, left_fit_x, right_fit_x, y_steps)\n",
    "\n",
    "    final_img = draw_on_original_img(img_test, img_draw, M_inv)\n",
    "    \n",
    "    inputs = []\n",
    "    img = cv2.resize(input_img, (300, 300))\n",
    "    img = image.img_to_array(img)\n",
    "    inputs.append(img.copy())\n",
    "    inputs = preprocess_input(np.array(inputs))\n",
    "    inputs = np.expand_dims(inputs[0], axis=0)\n",
    "    \n",
    "    preds = model.predict(inputs, batch_size=1, verbose=0)\n",
    "    results = bbox_util.detection_out(preds)\n",
    "    \n",
    "    final_img = draw_boxes(final_img, preds, results)\n",
    "    \n",
    "    return final_img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore', np.RankWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from joblib import Parallel, delayed\n",
    "import multiprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[MoviePy] >>>> Building video project_video_comb.mp4\n",
      "[MoviePy] Writing video project_video_comb.mp4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 1260/1261 [15:53<00:00,  1.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[MoviePy] Done.\n",
      "[MoviePy] >>>> Video ready: project_video_comb.mp4 \n",
      "\n",
      "CPU times: user 15min 38s, sys: 8.07 s, total: 15min 46s\n",
      "Wall time: 15min 55s\n"
     ]
    }
   ],
   "source": [
    "### Prrocess video\n",
    "# Import everything needed to edit/save/watch video clips\n",
    "from moviepy.editor import VideoFileClip\n",
    "from IPython.display import HTML\n",
    "\n",
    "right_fit_count = 0\n",
    "frame_count = 0\n",
    "\n",
    "output = 'project_video_comb.mp4'\n",
    "clip1 = VideoFileClip(\"project_video.mp4\")\n",
    "clip = clip1.fl_image(process_video) #NOTE: this function expects color images!!\n",
    "%time clip.write_videofile(output, audio=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_appaned(x):\n",
    "    x.append(10)\n",
    "xx = []\n",
    "test_appaned(xx)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}