Added mask overlay and adapted image size for better performance

imageElaboration.py

@@ -0,0 +1,47 @@
+import numpy as np
+import cv2
+from utilities import *
+
+
+# This function applies all elaboration steps to the image
+def elaborateImage(newFrame):
+    # Adjusting brightness and contrast
+    newFrameAdjusted = apply_brightness_contrast(newFrame, 90, 90)
+
+    # Threshold so that only yellow and white are kept. Result is greyscale
+    newFrameThreshold = thresholdWhiteAndYellow(newFrameAdjusted)
+
+    # Apply Gaussian blur to reduce noise
+    newFrameBlurred = cv2.GaussianBlur(newFrameThreshold, (5, 5), 0)
+
+    # Applying canny edge detection
+    newFrameEdges = cv2.Canny(newFrameBlurred, 100, 200)
+
+    # Cutting a region of interest
+    height, width = newFrameEdges.shape
+    # Creating white polygonal shape on black image
+    bottomLeft = [10, height - 130]
+    topLeft = [width / 3 + 60, height / 2]
+    topRight = [width * 2 / 3 - 60, height / 2]
+    bottomRight = [width - 10, height - 130]
+    pts = np.array([bottomLeft, topLeft, topRight, bottomRight], np.int32)
+    pts = pts.reshape((-1, 1, 2))
+    blackImage = np.zeros((height, width, 1), np.uint8)
+    polygonalShape = cv2.fillPoly(blackImage, [pts], (255, 255, 255))
+    # Doing AND operation with newFrameEdges
+    newFrameROI = cv2.bitwise_and(newFrameEdges, newFrameEdges, mask=polygonalShape)
+
+    # Hough transform to detect straight lines. Returns an array of r and theta values
+    lines = cv2.HoughLinesP(newFrameROI, 1, np.pi / 180, 15)
+    blackImage = np.zeros((height, width, 1), np.uint8)
+    newFrameHough = drawHoughTransformLines(blackImage, lines)
+
+    # Drawing road from original frame
+    newFrameGrey = cv2.cvtColor(newFrameAdjusted, cv2.COLOR_BGR2GRAY)
+    coloredMaskedRoad = cv2.bitwise_and(newFrameGrey, newFrameGrey, mask=polygonalShape)
+    newFrameMaskAndRoad = cv2.add(coloredMaskedRoad, newFrameROI)   # Adding canny edge overlay to highlight the lane markers
+
+    # Cutting image basing on mask size
+    result = cutTopAndBottom(coloredMaskedRoad, int(height / 2), int(height - 130))
+
+    return result

main.py

@@ -9,6 +9,7 @@
 from keras import backend as K
 import tensorflow as tf
 import cv2
+from imageElaboration import *
 
 
 # TODO ignore lateral movements
@@ -17,79 +18,7 @@
 # TODO Calculate average in test to soften changes in speed
 
 
-def drawHoughTransformLines(img, lines):
-    if lines is None:
-        return img
-    a, b, c = lines.shape
-    for i in range(a):
-        cv2.line(img, (lines[i][0][0], lines[i][0][1]), (lines[i][0][2], lines[i][0][3]), (255, 255, 255), 3, cv2.LINE_AA)
-
-    return img
-
-
-def apply_brightness_contrast(input_img, brightness = 0, contrast = 0):
-
-    if brightness != 0:
-        if brightness > 0:
-            shadow = brightness
-            highlight = 255
-        else:
-            shadow = 0
-            highlight = 255 + brightness
-        alpha_b = (highlight - shadow)/255
-        gamma_b = shadow
-
-        buf = cv2.addWeighted(input_img, alpha_b, input_img, 0, gamma_b)
-    else:
-        buf = input_img.copy()
-
-    if contrast != 0:
-        f = 131*(contrast + 127)/(127*(131-contrast))
-        alpha_c = f
-        gamma_c = 127*(1-f)
-
-        buf = cv2.addWeighted(buf, alpha_c, buf, 0, gamma_c)
-
-    return buf
-
-
-def thresholdWhiteAndYellow(image):
-    gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
-    img_hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
-    lower_yellow = np.array([20, 100, 100], dtype="uint8")
-    upper_yellow = np.array([30, 255, 255], dtype="uint8")
-    mask_yellow = cv2.inRange(img_hsv, lower_yellow, upper_yellow)
-    mask_white = cv2.inRange(gray_image, 200, 255)
-    mask_yw = cv2.bitwise_or(mask_white, mask_yellow)
-    mask_yw_image = cv2.bitwise_and(gray_image, mask_yw)
-
-    return mask_yw_image
-
-
-
-
-
-# This method draws the optical flow onto img with a given step (distance between one arrow origin and the other)
-def draw_flow(img, flow, step=16):
-    h, w = img.shape[:2]
-    y, x = np.mgrid[step/2:h:step, step/2:w:step].reshape(2,-1).astype(int)
-    fx, fy = flow[y,x].T
-    lines = np.vstack([x, y, x+fx, y+fy]).T.reshape(-1, 2, 2)
-    lines = np.int32(lines + 0.5)
-    vis = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
-    cv2.polylines(vis, lines, 0, (0, 255, 0))
-    for (x1, y1), (x2, y2) in lines:
-        cv2.circle(vis, (x1, y1), 1, (0, 255, 0), -1)
-    return vis
-
-# This method cuts top and bottom portions of the frame (which are only black areas of the car's dashboard or sky)
-def cutTopAndBottom(img):
-    height, width = img.shape
-    heightBeginning = 20
-    heightEnd = height - 30
-    crop_img = img[heightBeginning : heightEnd, 0 : width]
-    return crop_img
-
+# Setting up a Keras model of: Norm + 4 Conv and Pool + Flat + 5 Dense
 def setupNvidiaModel(inputShape):
     model = Sequential()
 
@@ -246,43 +175,13 @@ def setupTestModel(inputShape):
 frameCounter = 0
 batchFrames = []
 batchSpeeds = []
-while(coupleCounter < videoLengthInFrames-50):
+while(coupleCounter < videoLengthInFrames-20):
 
     # Read a couple of new frames from the video feed
     ret2, newFrame = videoFeed.read()
 
-    # Adjusting brightness and contrast
-    newFrame = apply_brightness_contrast(newFrame, 100, 100)
-
-    # Threshold so that only yellow and white are kept. Result is greyscale
-    newFrameThreshold = thresholdWhiteAndYellow(newFrame)
-
-    # Apply Gaussian blur to reduce noise
-    newFrameBlurred = cv2.GaussianBlur(newFrameThreshold, (5,5),0)
-
-    # Applying canny edge detection
-    newFrameEdges = cv2.Canny(newFrameBlurred, 100, 200)
-
-    # Cutting a region of interest
-    height, width = newFrameEdges.shape
-    # Creating white polygonal shape on black image
-    bottomLeft = [10, height-110]
-    topLeft = [width/3+60, height/2]
-    topRight = [width*2/3-60, height/2]
-    bottomRight = [width-10, height-110]
-    pts = np.array([bottomLeft, topLeft, topRight, bottomRight], np.int32)
-    pts = pts.reshape((-1, 1, 2))
-    blackImage = np.zeros((height, width, 1), np.uint8)
-    polygonalShape = cv2.fillPoly(blackImage, [pts], (255, 255, 255))
-    # Doing AND operation with newFrameEdges
-    newFrameROI = cv2.bitwise_and(newFrameEdges, newFrameEdges, mask=polygonalShape)
-
-    # Hough transform to detect straight lines. Returns an array of r and theta values
-    lines = cv2.HoughLinesP(newFrameROI, 1, np.pi / 180, 15)
-    blackImage = np.zeros((height, width, 1), np.uint8)
-    linesDrawn = drawHoughTransformLines(blackImage, lines)
-
-
+    # Elaborating image
+    newFrameROI = elaborateImage(newFrame)
 
     # Calculating the optical flow
     if coupleCounter == 0:
@@ -319,8 +218,8 @@ def setupTestModel(inputShape):
         model.fit(x=X,
                   y=Y,
                   verbose=1,
-                  epochs=5,
-                  batch_size=20
+                  epochs=50,
+                  batch_size=50
                   )
         # Resetting counter and x and y arrays
         frameCounter = 0

testing.py

@@ -2,15 +2,7 @@
 import numpy as np
 import keras
 import cv2
-
-
-# This method cuts top and bottom portions of the frame (which are only black areas of the car's dashboard or sky)
-def cutTopAndBottom(img):
-    height, width = img.shape
-    heightBeginning = 20
-    heightEnd = height - 30
-    crop_img = img[heightBeginning : heightEnd, 0 : width]
-    return crop_img
+from imageElaboration import *
 
 
 # Reading all the speed ground truths
@@ -32,37 +24,29 @@ def cutTopAndBottom(img):
 videoLengthInFrames = int(videoFeed.get(cv2.CAP_PROP_FRAME_COUNT))
 print(videoLengthInFrames)
 
-# Reading the first frame
-coupleCounter = 0
-frameCoupleArray = []
-ret1, oldFrame = videoFeed.read()
-oldFrameGrey = cv2.cvtColor(oldFrame, cv2.COLOR_BGR2GRAY)
-
-# Saving the size of the flow
-oldFrameGrey = cutTopAndBottom(oldFrameGrey)
-oldFrameGrey = cv2.equalizeHist(oldFrameGrey)
-dummyFlow = cv2.calcOpticalFlowFarneback(oldFrameGrey , oldFrameGrey, 0.5, 0.5, 5, 20, 3, 5, 1.2, 0)
-flowShape = dummyFlow.shape  # Original non cropped size is (480, 640, 2)
-
 # Iterating through all couples of frames of the video
+coupleCounter = 0
 frameToPredict = [ 0 ]
 while(coupleCounter < videoLengthInFrames-20):
 
     # Read a couple of new frames from the video feed
     ret2, newFrame = videoFeed.read()
 
-    # Convert to greyscale
-    newFrameGrey = cv2.cvtColor(newFrame, cv2.COLOR_BGR2GRAY)
-
-    # Cut top and bottom portions of the image
-    newFrameGrey = cutTopAndBottom(newFrameGrey)
+    # Elaborating image
+    newFrameROI = elaborateImage(newFrame)
 
-    # Apply Histogram Equalization to increase contrast
-    newFrameGrey = cv2.equalizeHist(newFrameGrey)
+    # Calculating the optical flow
+    if coupleCounter == 0:
+        # If this is the first frame...
+        oldFrameROI = newFrameROI
+        flow = cv2.calcOpticalFlowFarneback(oldFrameROI, newFrameROI, 0.5, 0.5, 5, 20, 3, 5, 1.2, 0)
+        # Also, set up the CNN model
+        flowShape = flow.shape
+    else:
+        flow = cv2.calcOpticalFlowFarneback(oldFrameROI, newFrameROI, 0.5, 0.5, 5, 20, 3, 5, 1.2, 0)
 
-    # Calculate flow for this couple
-    flow = cv2.calcOpticalFlowFarneback(oldFrameGrey , newFrameGrey, 0.5, 0.5, 5, 20, 3, 5, 1.2, 0)
-    frameToPredict[0] = flow    # This format is required by how the model was trained through np arrays
+    # This format is required by how the model was trained through np arrays
+    frameToPredict[0] = flow
 
     # Making speed prediction
     X = np.array(frameToPredict)
@@ -77,7 +61,7 @@ def cutTopAndBottom(img):
 
     # Incrementing couples counter and swapping frames
     coupleCounter = coupleCounter + 1
-    oldFrameGrey = newFrameGrey
+    oldFrameROI = newFrameROI
     #print(str(coupleCounter))
     cv2.imshow('frame',newFrame)
     cv2.waitKey(1)

utilities.py

@@ -0,0 +1,81 @@
+import numpy as np
+import cv2
+
+
+# Given the lines coming from the Hough transform result, this function draws them over a given image
+def drawHoughTransformLines(img, lines):
+    if lines is None:
+        return img
+    a, b, c = lines.shape
+    for i in range(a):
+        cv2.line(img, (lines[i][0][0], lines[i][0][1]), (lines[i][0][2], lines[i][0][3]), (255, 255, 255), 3,
+                 cv2.LINE_AA)
+
+    return img
+
+
+# This function applies a given brightness and contrast to an input image
+def apply_brightness_contrast(input_img, brightness = 0, contrast = 0):
+
+    if brightness != 0:
+        if brightness > 0:
+            shadow = brightness
+            highlight = 255
+        else:
+            shadow = 0
+            highlight = 255 + brightness
+        alpha_b = (highlight - shadow)/255
+        gamma_b = shadow
+
+        buf = cv2.addWeighted(input_img, alpha_b, input_img, 0, gamma_b)
+    else:
+        buf = input_img.copy()
+
+    if contrast != 0:
+        f = 131*(contrast + 127)/(127*(131-contrast))
+        alpha_c = f
+        gamma_c = 127*(1-f)
+
+        buf = cv2.addWeighted(buf, alpha_c, buf, 0, gamma_c)
+
+    return buf
+
+
+# This function
+def thresholdWhiteAndYellow(image):
+    gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    img_hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
+    # Get yellow from HSV version of our image
+    lower_yellow = np.array([20, 100, 100], dtype="uint8")
+    upper_yellow = np.array([30, 255, 255], dtype="uint8")
+    mask_yellow = cv2.inRange(img_hsv, lower_yellow, upper_yellow)
+    # Get white from greyscale version of our image
+    mask_white = cv2.inRange(gray_image, 200, 255)
+    # Combine and apply filters
+    mask_yw = cv2.bitwise_or(mask_white, mask_yellow)
+    mask_yw_image = cv2.bitwise_and(gray_image, mask_yw)
+
+    return mask_yw_image
+
+
+# This method draws the optical flow onto img with a given step (distance between one arrow origin and the other)
+def draw_flow(img, flow, step=16):
+    h, w = img.shape[:2]
+    y, x = np.mgrid[step/2:h:step, step/2:w:step].reshape(2,-1).astype(int)
+    fx, fy = flow[y,x].T
+    lines = np.vstack([x, y, x+fx, y+fy]).T.reshape(-1, 2, 2)
+    lines = np.int32(lines + 0.5)
+    vis = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
+    cv2.polylines(vis, lines, 0, (0, 255, 0))
+    for (x1, y1), (x2, y2) in lines:
+        cv2.circle(vis, (x1, y1), 1, (0, 255, 0), -1)
+    return vis
+
+
+# This method cuts top and bottom portions of the frame (which are only black areas of the car's dashboard or sky)
+def cutTopAndBottom(img, top, bottom):
+    height, width = img.shape
+    heightBeginning = 20
+    heightEnd = height - 30
+    crop_img = img[top : bottom, 0 : width]
+    return crop_img