From 5e2be641c8dfafdd618550bdf553b8855455b12e Mon Sep 17 00:00:00 2001 From: arturo Date: Fri, 15 Feb 2019 11:18:19 -0600 Subject: [PATCH] Added mask overlay and adapted image size for better performance --- imageElaboration.py | 47 ++++++++++++++++++ main.py | 115 +++----------------------------------------- testing.py | 48 ++++++------------ utilities.py | 81 +++++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 140 deletions(-) create mode 100644 imageElaboration.py create mode 100644 utilities.py diff --git a/imageElaboration.py b/imageElaboration.py new file mode 100644 index 0000000..c411388 --- /dev/null +++ b/imageElaboration.py @@ -0,0 +1,47 @@ +import numpy as np +import cv2 +from utilities import * + + +# This function applies all elaboration steps to the image +def elaborateImage(newFrame): + # Adjusting brightness and contrast + newFrameAdjusted = apply_brightness_contrast(newFrame, 90, 90) + + # Threshold so that only yellow and white are kept. Result is greyscale + newFrameThreshold = thresholdWhiteAndYellow(newFrameAdjusted) + + # Apply Gaussian blur to reduce noise + newFrameBlurred = cv2.GaussianBlur(newFrameThreshold, (5, 5), 0) + + # Applying canny edge detection + newFrameEdges = cv2.Canny(newFrameBlurred, 100, 200) + + # Cutting a region of interest + height, width = newFrameEdges.shape + # Creating white polygonal shape on black image + bottomLeft = [10, height - 130] + topLeft = [width / 3 + 60, height / 2] + topRight = [width * 2 / 3 - 60, height / 2] + bottomRight = [width - 10, height - 130] + pts = np.array([bottomLeft, topLeft, topRight, bottomRight], np.int32) + pts = pts.reshape((-1, 1, 2)) + blackImage = np.zeros((height, width, 1), np.uint8) + polygonalShape = cv2.fillPoly(blackImage, [pts], (255, 255, 255)) + # Doing AND operation with newFrameEdges + newFrameROI = cv2.bitwise_and(newFrameEdges, newFrameEdges, mask=polygonalShape) + + # Hough transform to detect straight lines. Returns an array of r and theta values + lines = cv2.HoughLinesP(newFrameROI, 1, np.pi / 180, 15) + blackImage = np.zeros((height, width, 1), np.uint8) + newFrameHough = drawHoughTransformLines(blackImage, lines) + + # Drawing road from original frame + newFrameGrey = cv2.cvtColor(newFrameAdjusted, cv2.COLOR_BGR2GRAY) + coloredMaskedRoad = cv2.bitwise_and(newFrameGrey, newFrameGrey, mask=polygonalShape) + newFrameMaskAndRoad = cv2.add(coloredMaskedRoad, newFrameROI) # Adding canny edge overlay to highlight the lane markers + + # Cutting image basing on mask size + result = cutTopAndBottom(coloredMaskedRoad, int(height / 2), int(height - 130)) + + return result \ No newline at end of file diff --git a/main.py b/main.py index eae1ff1..3d55d93 100644 --- a/main.py +++ b/main.py @@ -9,6 +9,7 @@ from keras import backend as K import tensorflow as tf import cv2 +from imageElaboration import * # TODO ignore lateral movements @@ -17,79 +18,7 @@ # TODO Calculate average in test to soften changes in speed -def drawHoughTransformLines(img, lines): - if lines is None: - return img - a, b, c = lines.shape - for i in range(a): - cv2.line(img, (lines[i][0][0], lines[i][0][1]), (lines[i][0][2], lines[i][0][3]), (255, 255, 255), 3, cv2.LINE_AA) - - return img - - -def apply_brightness_contrast(input_img, brightness = 0, contrast = 0): - - if brightness != 0: - if brightness > 0: - shadow = brightness - highlight = 255 - else: - shadow = 0 - highlight = 255 + brightness - alpha_b = (highlight - shadow)/255 - gamma_b = shadow - - buf = cv2.addWeighted(input_img, alpha_b, input_img, 0, gamma_b) - else: - buf = input_img.copy() - - if contrast != 0: - f = 131*(contrast + 127)/(127*(131-contrast)) - alpha_c = f - gamma_c = 127*(1-f) - - buf = cv2.addWeighted(buf, alpha_c, buf, 0, gamma_c) - - return buf - - -def thresholdWhiteAndYellow(image): - gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - img_hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) - lower_yellow = np.array([20, 100, 100], dtype="uint8") - upper_yellow = np.array([30, 255, 255], dtype="uint8") - mask_yellow = cv2.inRange(img_hsv, lower_yellow, upper_yellow) - mask_white = cv2.inRange(gray_image, 200, 255) - mask_yw = cv2.bitwise_or(mask_white, mask_yellow) - mask_yw_image = cv2.bitwise_and(gray_image, mask_yw) - - return mask_yw_image - - - - - -# This method draws the optical flow onto img with a given step (distance between one arrow origin and the other) -def draw_flow(img, flow, step=16): - h, w = img.shape[:2] - y, x = np.mgrid[step/2:h:step, step/2:w:step].reshape(2,-1).astype(int) - fx, fy = flow[y,x].T - lines = np.vstack([x, y, x+fx, y+fy]).T.reshape(-1, 2, 2) - lines = np.int32(lines + 0.5) - vis = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) - cv2.polylines(vis, lines, 0, (0, 255, 0)) - for (x1, y1), (x2, y2) in lines: - cv2.circle(vis, (x1, y1), 1, (0, 255, 0), -1) - return vis - -# This method cuts top and bottom portions of the frame (which are only black areas of the car's dashboard or sky) -def cutTopAndBottom(img): - height, width = img.shape - heightBeginning = 20 - heightEnd = height - 30 - crop_img = img[heightBeginning : heightEnd, 0 : width] - return crop_img - +# Setting up a Keras model of: Norm + 4 Conv and Pool + Flat + 5 Dense def setupNvidiaModel(inputShape): model = Sequential() @@ -246,43 +175,13 @@ def setupTestModel(inputShape): frameCounter = 0 batchFrames = [] batchSpeeds = [] -while(coupleCounter < videoLengthInFrames-50): +while(coupleCounter < videoLengthInFrames-20): # Read a couple of new frames from the video feed ret2, newFrame = videoFeed.read() - # Adjusting brightness and contrast - newFrame = apply_brightness_contrast(newFrame, 100, 100) - - # Threshold so that only yellow and white are kept. Result is greyscale - newFrameThreshold = thresholdWhiteAndYellow(newFrame) - - # Apply Gaussian blur to reduce noise - newFrameBlurred = cv2.GaussianBlur(newFrameThreshold, (5,5),0) - - # Applying canny edge detection - newFrameEdges = cv2.Canny(newFrameBlurred, 100, 200) - - # Cutting a region of interest - height, width = newFrameEdges.shape - # Creating white polygonal shape on black image - bottomLeft = [10, height-110] - topLeft = [width/3+60, height/2] - topRight = [width*2/3-60, height/2] - bottomRight = [width-10, height-110] - pts = np.array([bottomLeft, topLeft, topRight, bottomRight], np.int32) - pts = pts.reshape((-1, 1, 2)) - blackImage = np.zeros((height, width, 1), np.uint8) - polygonalShape = cv2.fillPoly(blackImage, [pts], (255, 255, 255)) - # Doing AND operation with newFrameEdges - newFrameROI = cv2.bitwise_and(newFrameEdges, newFrameEdges, mask=polygonalShape) - - # Hough transform to detect straight lines. Returns an array of r and theta values - lines = cv2.HoughLinesP(newFrameROI, 1, np.pi / 180, 15) - blackImage = np.zeros((height, width, 1), np.uint8) - linesDrawn = drawHoughTransformLines(blackImage, lines) - - + # Elaborating image + newFrameROI = elaborateImage(newFrame) # Calculating the optical flow if coupleCounter == 0: @@ -319,8 +218,8 @@ def setupTestModel(inputShape): model.fit(x=X, y=Y, verbose=1, - epochs=5, - batch_size=20 + epochs=50, + batch_size=50 ) # Resetting counter and x and y arrays frameCounter = 0 diff --git a/testing.py b/testing.py index def0dcd..d80def6 100644 --- a/testing.py +++ b/testing.py @@ -2,15 +2,7 @@ import numpy as np import keras import cv2 - - -# This method cuts top and bottom portions of the frame (which are only black areas of the car's dashboard or sky) -def cutTopAndBottom(img): - height, width = img.shape - heightBeginning = 20 - heightEnd = height - 30 - crop_img = img[heightBeginning : heightEnd, 0 : width] - return crop_img +from imageElaboration import * # Reading all the speed ground truths @@ -32,37 +24,29 @@ def cutTopAndBottom(img): videoLengthInFrames = int(videoFeed.get(cv2.CAP_PROP_FRAME_COUNT)) print(videoLengthInFrames) -# Reading the first frame -coupleCounter = 0 -frameCoupleArray = [] -ret1, oldFrame = videoFeed.read() -oldFrameGrey = cv2.cvtColor(oldFrame, cv2.COLOR_BGR2GRAY) - -# Saving the size of the flow -oldFrameGrey = cutTopAndBottom(oldFrameGrey) -oldFrameGrey = cv2.equalizeHist(oldFrameGrey) -dummyFlow = cv2.calcOpticalFlowFarneback(oldFrameGrey , oldFrameGrey, 0.5, 0.5, 5, 20, 3, 5, 1.2, 0) -flowShape = dummyFlow.shape # Original non cropped size is (480, 640, 2) - # Iterating through all couples of frames of the video +coupleCounter = 0 frameToPredict = [ 0 ] while(coupleCounter < videoLengthInFrames-20): # Read a couple of new frames from the video feed ret2, newFrame = videoFeed.read() - # Convert to greyscale - newFrameGrey = cv2.cvtColor(newFrame, cv2.COLOR_BGR2GRAY) - - # Cut top and bottom portions of the image - newFrameGrey = cutTopAndBottom(newFrameGrey) + # Elaborating image + newFrameROI = elaborateImage(newFrame) - # Apply Histogram Equalization to increase contrast - newFrameGrey = cv2.equalizeHist(newFrameGrey) + # Calculating the optical flow + if coupleCounter == 0: + # If this is the first frame... + oldFrameROI = newFrameROI + flow = cv2.calcOpticalFlowFarneback(oldFrameROI, newFrameROI, 0.5, 0.5, 5, 20, 3, 5, 1.2, 0) + # Also, set up the CNN model + flowShape = flow.shape + else: + flow = cv2.calcOpticalFlowFarneback(oldFrameROI, newFrameROI, 0.5, 0.5, 5, 20, 3, 5, 1.2, 0) - # Calculate flow for this couple - flow = cv2.calcOpticalFlowFarneback(oldFrameGrey , newFrameGrey, 0.5, 0.5, 5, 20, 3, 5, 1.2, 0) - frameToPredict[0] = flow # This format is required by how the model was trained through np arrays + # This format is required by how the model was trained through np arrays + frameToPredict[0] = flow # Making speed prediction X = np.array(frameToPredict) @@ -77,7 +61,7 @@ def cutTopAndBottom(img): # Incrementing couples counter and swapping frames coupleCounter = coupleCounter + 1 - oldFrameGrey = newFrameGrey + oldFrameROI = newFrameROI #print(str(coupleCounter)) cv2.imshow('frame',newFrame) cv2.waitKey(1) diff --git a/utilities.py b/utilities.py new file mode 100644 index 0000000..90988be --- /dev/null +++ b/utilities.py @@ -0,0 +1,81 @@ +import numpy as np +import cv2 + + +# Given the lines coming from the Hough transform result, this function draws them over a given image +def drawHoughTransformLines(img, lines): + if lines is None: + return img + a, b, c = lines.shape + for i in range(a): + cv2.line(img, (lines[i][0][0], lines[i][0][1]), (lines[i][0][2], lines[i][0][3]), (255, 255, 255), 3, + cv2.LINE_AA) + + return img + + +# This function applies a given brightness and contrast to an input image +def apply_brightness_contrast(input_img, brightness = 0, contrast = 0): + + if brightness != 0: + if brightness > 0: + shadow = brightness + highlight = 255 + else: + shadow = 0 + highlight = 255 + brightness + alpha_b = (highlight - shadow)/255 + gamma_b = shadow + + buf = cv2.addWeighted(input_img, alpha_b, input_img, 0, gamma_b) + else: + buf = input_img.copy() + + if contrast != 0: + f = 131*(contrast + 127)/(127*(131-contrast)) + alpha_c = f + gamma_c = 127*(1-f) + + buf = cv2.addWeighted(buf, alpha_c, buf, 0, gamma_c) + + return buf + + +# This function +def thresholdWhiteAndYellow(image): + gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + img_hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) + # Get yellow from HSV version of our image + lower_yellow = np.array([20, 100, 100], dtype="uint8") + upper_yellow = np.array([30, 255, 255], dtype="uint8") + mask_yellow = cv2.inRange(img_hsv, lower_yellow, upper_yellow) + # Get white from greyscale version of our image + mask_white = cv2.inRange(gray_image, 200, 255) + # Combine and apply filters + mask_yw = cv2.bitwise_or(mask_white, mask_yellow) + mask_yw_image = cv2.bitwise_and(gray_image, mask_yw) + + return mask_yw_image + + +# This method draws the optical flow onto img with a given step (distance between one arrow origin and the other) +def draw_flow(img, flow, step=16): + h, w = img.shape[:2] + y, x = np.mgrid[step/2:h:step, step/2:w:step].reshape(2,-1).astype(int) + fx, fy = flow[y,x].T + lines = np.vstack([x, y, x+fx, y+fy]).T.reshape(-1, 2, 2) + lines = np.int32(lines + 0.5) + vis = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + cv2.polylines(vis, lines, 0, (0, 255, 0)) + for (x1, y1), (x2, y2) in lines: + cv2.circle(vis, (x1, y1), 1, (0, 255, 0), -1) + return vis + + +# This method cuts top and bottom portions of the frame (which are only black areas of the car's dashboard or sky) +def cutTopAndBottom(img, top, bottom): + height, width = img.shape + heightBeginning = 20 + heightEnd = height - 30 + crop_img = img[top : bottom, 0 : width] + return crop_img \ No newline at end of file