progtest-tester

#!/bin/bash

# set -o errexit -o noclobber -o nounset
set -o noclobber -o nounset

read_args(){
    p= v= t= a= i= h= s= l=

    POSITIONAL=()
    while [[ $# -gt 0 ]]; do
        key="$1"
        case $key in
            -i|--interactive)
                i=1
                shift
                ;;
            -t|--test)
                t="$2"
                shift
                shift
                ;;
	    -s|--sort)
		s=1
		shift
		;;
	    -l|--linesort)
		l=1
		shift
		;;
            -a|--address)
                a=1
                shift
                ;;
            -v|--valgrind)
                v=1
                shift
                ;;
	    -h|--help)
		h=1
		shift
		;;
            *)
                POSITIONAL+=("$1")
                shift
                ;;
        esac
    done

    set -- "${POSITIONAL[@]}"

    if [[ ! -z "$h" ]]; then
	print_help
	exit
    fi

    if [[ ! $# -eq 1 ]]; then
        echo "You must use exactly one positional argument"
        exit 1
    fi

    p=$1

    if [[ ! -z "$a" ]] && [[ ! -z "$v" ]]; then
        echo "You cannot use --address and --valgrind at the same time"
        exit 1
    fi
}

compile(){
    dir_tests="$p/tests"
    executable="$p/a.out"
    main=`echo "$p"/main.c*`
    compile_cmd="g++ -g -Wall -D__PROGTEST__ -pedantic $main -o $executable"

    if [[ ! -z "$a" ]]; then
        compile_cmd="$compile_cmd -fsanitize=address"
    fi

    echo "Compiling with \"${compile_cmd}\"..."

    ${compile_cmd}

    if [[ ! -z "$v" ]]; then
	    executable="valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all ${executable}"
    fi

    if [[ $? != 0 ]]; then
        echo "Compilation failed"

    fi
}

tests_checks(){
    if [[ ! -d "$dir_tests" ]]; then
        echo 'Error: invalid test directory path' >&2
        die
    fi

    if [[ -z "`ls ${dir_tests}/*_in[^_]* 2>/dev/null`" ]]; then
        echo 'Error: no test inputs found in specified folder' >&2
        die
    fi

    if [[ -z "`ls ${dir_tests}/*_out[^_]*` 2>/dev/null" ]]; then
        echo 'Error: no test outputs found in specified folder' >&2
        die
    fi
}

diff_run(){
	
	PIPES=

	if [[ ! -z "$s" ]]; then
		PIPES="${PIPES} | sort "
	fi

	if [[ ! -z "$l" ]]; then
		PIPES="${PIPES} | grep -o . | sort | tr -d \"\\n\""
	fi
	
	EXPECTED=`cat ${dir_tests}/${t}_out.txt`
	APP=`${executable} < ${dir_tests}/${t}_in.txt`
	
	diff <( eval "echo \"$EXPECTED\" $PIPES" ) <(eval "echo \"$APP\" $PIPES") >/dev/null 2>&1
	
	if [[ $? != 0 ]]; then
		diff <(eval "echo \"$APP\"") <( eval "echo \"$EXPECTED\"" )
	fi
	# DIS WORKS diff <( eval "${executable} < ${dir_tests}/${t}_in.txt $PIPES" ) <( eval "cat ${dir_tests}/${t}_out.txt $PIPES" )

}

test_singular(){
	
	# Test input
	echo -e "\e[1m--------Input--------\e[0m"
	cat "${dir_tests}/${t}_in.txt"
	
	# Expected output
	echo -e "\e[1m-Expected Output-----\e[0m"
	cat "${dir_tests}/${t}_out.txt"
	
	# Your output
	echo -e "\e[1m----Your Output------\e[0m"
	echo "`${executable} < "$dir_tests/${t}_in.txt"`"
	
	# Diff
	echo -e "\e[1m-----Diff run--------\e[0m"
	DIFF_OUT=`diff_run`

	echo "$DIFF_OUT"
	echo ---------------------
	
	if [[ -n "$DIFF_OUT" ]]; then
		echo -e "\e[1;31mDIFF ALERT\e[0m"
	fi
}

test_broad(){
    for input in "$dir_tests"/*_in*; do
        TEST_IN=`basename $input`
        TEST_NUM=${TEST_IN%_in.txt}
        TEST_OUT="${TEST_NUM}_out.txt"

        printf "\e[1;33mtesting\e[0m: %s..." "$TEST_NUM"
        #DIFF_OUT="`${executable} < "$dir_tests/$TEST_IN" | diff - "$dir_tests/$TEST_OUT"`"
	
	t=$TEST_NUM
	DIFF_OUT=`diff_run`

        if [[ -n "$DIFF_OUT" ]]; then
            echo -e " \e[1;31mFAIL\e[0m"
            #echo -e "\n\e[1;31mALERT\e[0m: diff produced output:\n${DIFF_OUT}\n"
            #echo "Previous input: " "`cat $input`"
        else
             echo -e " \e[1;32mok\e[0m"
        fi

    done
}

run_tests(){

    echo "Executable: " ${executable}

    if [[ ! -z "$i" ]]; then
        ${executable}
        exit
    fi

    echo "---Diff legend----"
    echo "< got"
    echo "> expected"
    echo "------------------"

    if [[ ! -z "$t" ]]; then
        echo Performing singular test...
        test_singular
        exit
    fi

    test_broad
}

die(){

    exit
    if [[ ! -z "$executable" ]]; then
    	executable=${executable#"valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all"}
        rm ${executable}
    fi
}

print_help(){
    echo "Usage:"
    echo $' progtest-tester <project directory> [-v|--valgrind] [-a|--address] [-h|--help] [-i|--interactive] [-t|--test <testnum>] [-s|--sort]'
    echo ""
    echo "Arguments:"
    echo $' -v \tRun program with valgrind'
    echo $' -a \tCompile program with -fsanitize=address'
    echo $' -i \tInteractive mode - ignore tests and just run the program'
    echo $' -h \tPrint this message'
    echo $' -t <testnum> \tRun only a specific test and show details'
    echo $' -s \tPerform sort on output lines and expected lines - use to compare outputs that may have scrambled lines'
    echo $' -l \tPerform sort on each line and expected each output line - use to compare outputs that may have scrambled values'
    echo ""
    echo "Environment"
    echo " <project directory> must contain file \"main.c*\" and folder \"tests\""
}


read_args $@


echo "Running with settings: project: $p, valgrind: $v, test: $t, addresscheck: $a"

compile
tests_checks
run_tests
die