findmalware.sh

#!/bin/bash
#
# findmalware.sh - A script to guess presence of malware in php files.
# Copyright © 2018 Lucio Crusca <lucio@sulweb.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# {{{STARTHELP}}}
# # findmalware
# A script that tries to guess presence of malicious code in PHP files. <br/>
# It does NOT look for known malware code fragments, but it uses a set 
# of rules to infer what pieces PHP code are likely malware. As such it 
# produces many false positives, so it uses a few whitelists to avoid showing 
# files that are known to be good.
#
# It analyzes every file whose name ends in '.php' looking for them
# recursively in the current directory, so you usually cd into the root
# directory of a PHP website and then you run this script there.
#
# _Please note that this script does **NOT** delete nor sanitize **ANYTHING**.  
# It only produces a report. It's up to the sysadmin (e.g. YOU) to read that 
# report and take action_. <br/>
# As such, it is safe to run this script anywhere you like: it won't even 
# notice if you run it in a read-only filesystem, as long as you have a working 
# mktemp installed and provide appropriate arguments for the lists.
#
# Current version uses four whitelists: a upstream one for whitelisted files, 
# a local one for locally whitelisted files, a upstream one for whitelisted 
# lines of code and a local one for locally whitelisted lines of code.
# Please note that you have little to no control over the name and full path
# of the code whitelists: they are computed from the name of the files 
# whitelists, by prepending the "code-" prefix to those names. 
# However -w and -W options let you control the full paths and names of 
# the files whitelists.
# Code whitelists logic works this way: when a rule matches a line of code,
# that line is looked up in the code whitelists. If it's found there, it's
# assumed to be good and that line does not contribute to the malware-positive 
# scoring of the file anymore. If all lines matched by any rule are found in 
# the code whitelists, the whole file won't be considered malware-positive
# anymore, so it won't be shown to the user, but it won't be automatically 
# whitelisted as a whole either (unless you use -A): that ensures that future 
# rules updates will be checked against infected files that may have slipped 
# through in the past.
# In all other cases, the file will still be considered suspect and shown to 
# the user, including all lines of code that are matched by any rule, even if
# they were already present in the code whitelists.
# When the user chooses to whitelist a file, all lines that were matched
# by any rule are added to the local code whitelist, if they aren't there yet.
# Since we are language agnostic, we can't assume a line of code is terminated
# by a semicolon. Moreover, we never want to whitelist too short lines, because
# they are easily reusable in malicious code too.
# A "line of code" here is defined as the line that matches a rule, padded with
# some leading and trailing context lines. That pattern is exactly what is being 
# hashed. Please note that changing the number of context lines (it defaults to
# 2) produces different hashes for the same line of code, so any code whitelists
# compiled with different context size won't work anymore. For this reason
# there aren't any options to change the contextx size, but you can always
# override the deafult value in a configuration file, assuming you know what
# you are doing.
#
# ### Installation:
# This script depends on the following tools
# 1. mktemp
# 2. GNU find
# 3. GNU grep
# 4. GNU sed
# 5. GNU screen
# 6. sha512sum
# 7. curl for the autoupdate feature
# 8. a bunch of other rather common system commands
# 9. all of the above plus tar, gzip, unzip for the -t option
# 
# You need to ensure those commands are installed for this script to work. <br/>
# Please note that this script does not check if those commands are 
# available, so failing to install them beforehand can produce undefined
# results.
#
# ### Usage:
#   findmalware.sh [ -c _config-file_ ] [ -b _local-blacklist-file_ ]  <br/>
#     [ -B _upstream-blacklist-file_ ] [ -w _local-whitelist-file_ ]  <br/>
#     [ -W _upstream-whitelist-file_ ] [ -r _local-rules-file_ ]  <br/>
#     [ -R _upstream-rules-file_ ] [ -u _upstream-url_ ]  <br/>
#     [ -e _extensions_list_ ] [ -t _trustedurl_ ] [ -a ] [ -A ] [ -h ] <br/>
#     [ -m ] [ -U ] [ -d _directory_ ] <br/>
#
#   where
#
#   -c _config-file_  <br/>
#        Specifies a custom configuration file to load after the 
#        default ones. Every configuration file is a bash fragment that
#        this script includes untouched. This option is parsed before
#        other options, so that command line arguments can override any
#        settings in configuration files. Default configuration files
#        that are ALWAYS loaded if readable, whether you use this option
#        or not, are:
#          * /etc/findmalware.conf
#          * /usr/local/etc/findmalware.conf
#          * $HOME/.findmalware.conf
#          * $HOME/.config/findmalware.conf
#          * $HOME/.findmalware/config
#        in that order. If you specify a custom configuration file and
#        that file is not readable, the script aborts execution by
#        design. The FINDMALWARE\_CONFIG environment variable can be 
#        used instead of this option, but if both are specified, this
#        option takes precedence.
# 
#  -b _local-blacklist-file_  <br/>
#        specifies a blacklist file to use as a cache. This file needs 
#        to be writeable in order to be of any use. However you can
#        live without it, it's really only a way to avoid showing two
#        suspect files when they have exactly the same content and the
#        user has alreay tagged the first one as suspect. A local
#        blacklist can save you quite some time if you scan for malware
#        a directory that contains a website and one or more of its 
#        backups. <br/>
#        It defaults to $HOME/.findmalware/blacklist.local
#
#   -B _upstream-blacklist-file_ <br/>
#        same as -b, but it specifies a local copy of a upstream 
#        blacklist, possibly downloaded from the internet. This can be
#        a read-only file, but if you use the autoupdate feature 
#        (-a option), then this file and its parent directory need
#        to be writeable for you to blacklist anything. <br/>
#        It defaults to $HOME/.findmalware/blacklist
#
#   -w _local-whitelist-file_ <br/>
#        specifies a whitelist file to use as a cache. For each false
#        positive file, the user can choose to whitelist it. The script
#        then adds the file name, size and checksum to this whitelist
#        file. If a file is whitelisted it won't be shown to the user
#        again in the future, but the script will only add a notice to
#        the scan report. Please note that whitelisting a file by 
#        mistake can defeat the whole pourpose of this script, so pay
#        attention before whitelisting a file and, if you are not sure,
#        you better avoid whitelisting anything. This file needs to be
#        writeable for you to whitelist anything. <br/>
#        It defaults to $HOME/.findmalware/whitelist.local 
#
#   -W _upstream-whitelist-file_ <br/>
#        same as -w, but it specifies a local copy of a upstream
#        whitelist, possibly downloaded from the internet. This can be
#        a read-only file, but if you use the autoupdate feature
#        (-a option), then this file and its parent directory need
#        to be writeable. <br/>
#        Please note that downloading random whitelists from unknown
#        sources for sake of saving some time during scans paves the 
#        way to a slow and painful death. And no, you should NOT
#        trust the official upstream whitelist published by the author 
#        of this script either, unless you have a good reason. <br/>
#        And please note that the author of this documentation is
#        the same as the author of the script, and he is the one who
#        publishes the default upstream whitelist too. In other words:
#        do NOT trust ME, where "ME"="Lucio Crusca", your faithful 
#        author of the script who has written this stuff. <br/>
#        I'm not saying I'm so bad, but I can make mistakes and you 
#        can't hold me liable for that. You have been warned.
#        That being said, this script works for me and I trust myself,
#        so this script has the autoupdate feature enabled by default
#        and it downloads my whitelist by default. <br/>
#        This option defaults to $HOME/.findmalware/whitelist
#
#   -r _local-rules-file_ <br/>
#        specifies a file containing rules, one per line. Rules are 
#        regular expressions as recognized by grep. PHP files matching
#        one or more rules are considered infected and shown to the
#        user during scans. If you want to use more rules than the 
#        default ones, you can write them in a file and pass the file
#        to the script with this option. 
#        It defaults to $HOME/.findmalware/rules.local
#
#   -R _upstream-rules-file_ <br/>
#        same as -r, but it specifies a local copy of a upstream 
#        rules file, possibly downloaded from the internet. If you use 
#        the autoupdate feature (-a option), then this file and its parent 
#        directory need to be writeable. <br/>
#        All the -W advices apply, so go read them if you haven't yet. <br/>
#        It defaults to $HOME/.findmalware/rules
#
#   -u _upstream-url_ <br/>
#        specifies the URL to use when autoupdating local copies 
#        of upstream files. <br/>
#        When autoupdating, this script looks for the following files
#        online:
#          * upstream-url/whitelist-latest.txt
#          * upstream-url/blacklist-latest.txt
#          * upstream-url/rules-latest.txt
#        It uses _curl_ to check last modification date online and to 
#        download them if needed. <br/>
#        It defaults to https://webcloud.virtualbit.it/findmalware
#
#   -e _extensions-list_  <br/>
#        specifies the comma-separated list of file extensions to consider. <br/>
#        If you use this option, remember to include the default php too, unless
#        you really want to avoid files ending in ".php". Example: <br/>
#
#           -e php,php4,php5,php7
#
#        This script was initially intended to look for malicious code in PHP files
#        only, but there is nothing in the code that prevents its use for other type
#        of files, so you can use this option to specify any extension you like. 
#        Please note however that, as of time of this writing, the whitelists, 
#        blacklists and rules provided by default make sense only for PHP files, 
#        but you can always use different lists than the default ones. <br/>
#        It defaults to php only.
#
#   -a _[on|off]_ <br/>
#        enables or disables the autoupdate of the lists and rules. 
#        It's enabled by default, for reasons you can find in the -W 
#        documentation above. Haven't read that yet?
#   
#   -A <br/>
#        automatically whitelists every file it would otherwise ask the user about. 
#        It implies -U. Use only in directories that are trusted fresh clones of 
#        upstream code repositories. DO NOT USE if the directory has already been 
#        served over the network even only once. Best use this on a disconnected host 
#        and then copy over the resulting whitelist.
#
#   -U <br/>
#        Unattended execution. In this mode the script does not interactively ask 
#        anything and it just produces the report. The only output in this case
#        will be the full path and filename of the report itself.
#
#   -t _trusted-url_ <br/>
#       downloads the file (tar or zip archive) at the specified URL in a temporary 
#       directory, then it expands it and it runs -A (autowhitelist, see above)  
#       on its contents. Only .tar.gz and .zip formats are supported and the file
#       MUST be served over a secure HTTPS connection, so the URL MUST start with
#       https://. This obviously falls short of a real security policy and it's up 
#       to you to pass to this option only URLs you really trust. This option does
#       NOT and will NEVER support connections to servers with self-signed or invalid
#       SSL certificates. This option overrides -d and sets the directory to scan
#       to the temporary directory where it expanded the trusted file.
#
#   -d _directory_ <br/>
#       The directory to scan for malware. It defaults to the current working directory.
#
#   -h <br/>
#        shows this help and exits.
#
#   -m <br/>
#        shows this help in markdown format and exits.
#
#
# ### Configuration files:
#
#   All options except "-h" and "-m" have a matching variable 
#   inside the script, so you can set them in configuration 
#   files too, but see below for "-c". <br/>
#   Here is the list of variables and their respective 
#   controlling option. Options, if specified, take precedence 
#   over configuration files. <br/>
#   Environment variables by the same name of these ones are
#   silently ignored. <br/>
#
#     *  AUTOUPDATE\_LISTS is controlled by -a
#     *  AUTOWHITELIST is controlled by -A
#     *  BLACKLIST\_ADDITIONS is controlled by -b
#     *  UPSTREAM\_BLACKLIST is controlled by -B
#     *  RULES\_ADDITIONS is controlled by -r
#     *  UPSTREAM\_RULES is controlled by -R
#     *  URL\_FOR\_UPSTREAM\_LISTS is controlled by -u
#     *  WHITELIST\_ADDITIONS is controlled by -w
#     *  UPSTREAM\_WHITELIST is controlled by -W
#     *  EXTENSIONS\_LIST is controlled by -e
#     *  UNATTENDED\_EXECUTION is controlled by -U
#     *  TRUSTED\_URL is controlled by -t
#     *  NUMBER\_OF\_CONTEXT\_LINES has no matching option (see above)
#     *  SCANDIR is controlled by -d
#     *  USERCONFFILE is controlled by -c, but...
#       
#   Additionally you can define the FINDMALWARE\_CONFIG variable
#   in your environment end export it to this script before 
#   running it. Doing so will cause the specified value to be used
#   as initialization for USERCONFFILE. <br/>
#   Default configuration files can do whatever they see fit in
#   variables values, including USERCONFFILE. <br/>
#   Thus, it is theoretically possible to assign a value
#   to USERCONFFILE inside one of the default configuration files,
#   causing the specified conf-file to be loaded afterwards,
#   but I suspect it makes little sense to do so.
#
# ### The rules file
#
#   At its core, all findmalware does is running grep against
#   the files containig PHP code. The rules file is fed directly
#   to grep, which uses it to match rows in the files and output
#   potentially maicious files names and snippets of code.
#   The rules file is just a list of grep regular expressions.
#   You can find an example rules file in this project repository: 
#   it can be used as a starting point that produces many false 
#   positives (and also it does not find many true positives). 
#   You are encouraged to write your own rules, and eventually
#   share it if you find it works better than the one in the
#   findmalware repository. This script was written with PHP
#   in mind, but you can write rules for other languages too.
#   See the options above to change the rules file and the
#   files extensions findmalware should use.
#
# ### Examples:
#
#   Search for malware in the current directory, using rules and 
#   lists provided by the script author (NOT RECOMMENDED)
#
#       findmalware.sh
#
#   Search for malware in non interactive mode, using your own
#   rules and lists (RECOMMENDED). 
#
#       findmalware.sh -U -u https://example.com/my-own-findmalware-rules-and-lists-folder
#
#   Scan the latest WordPress release, taking for granted it has
#   no malware in it, find false positives and add them to your
#   local whitelist (suitable for a cron job)
#
#       findmalware.sh -t https://wordpress.org/latest.tar.gz
#
## {{{ENDHELP}}}

function show_help
{
  show_markdown | sed -E -e 's- <br/>--g' -e 's/_([^\s_]*)_/\1/g' -e 's/\*\*([^\s*]*)\*\*/\1/g' -e 's/\\_/_/g'
}

function show_markdown
{
  STARTLINE=$(cat "$0" | grep -n '{{{STARTHELP}}}' | head -n1 | cut -d: -f1)
  STARTLINE=$(($STARTLINE + 1))
  ENDLINE=$(cat "$0" | grep -n '{{{ENDHELP}}}' | head -n1 | cut -d: -f1)
  ENDLINE=$(($ENDLINE - 1))
  cat "$0" | head -n "$ENDLINE" | tail -n +"$STARTLINE" | sed -e 's/^#//' -e 's/^ //'
}

USERCONFDIR="$HOME/.findmalware"
USERCONFFILE="$USERCONFDIR/config"
UPSTREAM_WHITELIST="$USERCONFDIR/whitelist"
UPSTREAM_BLACKLIST="$USERCONFDIR/blacklist"
UPSTREAM_RULES="$USERCONFDIR/rules"
WHITELIST_ADDITIONS="$USERCONFDIR/whitelist.local"
BLACKLIST_ADDITIONS="$USERCONFDIR/blacklist.local"
RULES_ADDITIONS="$USERCONFDIR/rules.local"
URL_FOR_UPSTREAM_LISTS="https://webcloud.virtualbit.it/findmalware"
EXTENSIONS_LIST="php"
AUTOUPDATE_LISTS="on"
AUTOWHITELIST="off"
UNATTENDED_EXECUTION="off"
NUMBER_OF_CONTEXT_LINES=2
TRUSTED_URL=""
SCANREPORT=$(mktemp)
SCANDIR='.'

# The options
OPTIONS="a:Ab:B:c:d:e:hmr:R:t:u:Uw:W:x"

# Interactive echo
function iecho()
{
  if [ "$UNATTENDED_EXECUTION" == "off" ] ; then
    echo $*
  fi
}

# Mandatory echo
function mecho()
{
  iecho $*
  echo $* >> "$SCANREPORT"
}

function check_custom_config()
{
  CUSTOMCONFIGFILE="$1"
  if [ -r "$CUSTOMCONFIGFILE" ] ; then
    USERCONFFILE="$CUSTOMCONFIGFILE"
  else
    mecho "$CUSTOMCONFIGFILE is not readable, aborting."
    exit 1
  fi
}

if ! [ -z ${FINDMALWARE_CONFIG+x} ]; then 
  check_custom_config "$FINDMALWARE_CONFIG"
fi

# We process the '-c configfile' opion first, because other command line options
# override config file settings, so we need to apply those options afterwards

while getopts $OPTIONS OPT ; do
    case $OPT in
        c)
            check_custom_config "$OPTARG"
            ;;
        x)
            set -x
            ;;
    esac
done

# Now include all the configuration files

for FMCONFIG in "/etc/findmalware.conf" "/usr/local/etc/findmalware.conf" "$HOME/.findmalware.conf" "$HOME/.config/findmalware.conf" "$USERCONFFILE" ; do
  if [ -r "$FMCONFIG" ] ; then
    source "$FMCONFIG"
  fi
done

# Then we process remaining command line options (we need to reset OPTIND in order to make getopts work again)
OPTIND=1

while getopts $OPTIONS OPT ; do
    case $OPT in
        a)
            AUTOUPDATE_LISTS="$OPTARG"
            ;;
        A)
            AUTOWHITELIST="on"
            ;;        
        b)
            BLACKLIST_ADDITIONS="$OPTARG"
            ;;
        B)
            UPSTREAM_BLACKLIST="$OPTARG"
            ;;
        r)
            RULES_ADDITIONS="$OPTARG"
            ;;
        R)
            UPSTREAM_RULES="$OPTARG"
            ;;
        w)
            WHITELIST_ADDITIONS="$OPTARG"
            ;;
        W)
            UPSTREAM_WHITELIST="$OPTARG"
            ;;
        h)
          show_help
          exit 0
            ;;
	m) 
          show_markdown
	  exit 0
	    ;;
        u)
            URL_FOR_UPSTREAM_LISTS="$OPTARG"
            ;;
	e)
            EXTENSIONS_LIST="$OPTARG"
	    ;;
        t)
            TRUSTED_URL="$OPTARG"
            ;;
	U)
	    UNATTENDED_EXECUTION="on"
	    ;;
        d)
            SCANDIR="$OPTARG"
            ;;
    esac
done

if [ "$TRUSTED_URL" != "" ] ; then
  if [[ $TRUSTED_URL == https://* ]] ; then
    TURLDIR=`mktemp -d`
    curl --fail -o "$TURLDIR/tmp" "$TRUSTED_URL"
    CURLEXIT=$?
    if [ $CURLEXIT -ne 0 ] ; then
      iecho "Could not download $TRUSTED_URL"
      exit 2
    else
      TURLTYPE=`LANG=en file "$TURLDIR/tmp"`
      if [[ $TURLTYPE == *"Zip archive data"* ]] ; then
        SCANDIR="$TURLDIR/extract"
        mkdir "$SCANDIR"
        cd "$SCANDIR"
        unzip ../tmp
        rm -f ../tmp
      fi 
      if [[ $TURLTYPE == *"gzip compressed data"* ]] ; then
        SCANDIR="$TURLDIR/extract"
        mkdir "$SCANDIR"
        cd "$SCANDIR"
        tar xzf ../tmp
        rm -f ../tmp
      fi
      if [ -f "$TURLDIR/tmp" ] ; then
        rm -f "$TURLDIR/tmp" 
        iecho "Unsupported file format for $TRUSTED_URL"
        exit 3
      else
        AUTOWHITELIST="on"
      fi 
    fi
  else
    iecho "Unsupported protocol for trusted URL."
    exit 1
  fi
fi

if [ "$AUTOWHITELIST" == "on" ] ; then
  UNATTENDED_EXECUTION="on"
fi

function build_code_whitelist_name
{
  FILES_WHITELIST_PATHANDNAME="$1"
  FILES_WHITELIST_PATH=$(dirname "${FILES_WHITELIST_PATHANDNAME}")
  FILES_WHITELIST_NAME=$(basename "${FILES_WHITELIST_PATHANDNAME}")
  CODE_WHITELIST="${FILES_WHITELIST_PATH}/code-${FILES_WHITELIST_NAME}"
  echo "${CODE_WHITELIST}"
}

UPSTREAM_CODE_WHITELIST=$(build_code_whitelist_name "$UPSTREAM_WHITELIST")
CODE_WHITELIST_ADDITIONS=$(build_code_whitelist_name "$WHITELIST_ADDITIONS")

function curl_download
{
  LOCALFILE="$1"
  REMOTEURL="$2"
  DLACTION="$3"
  if [ "$DLACTION" == "fetch" ] ; then
    curl --fail -o "$LOCALFILE" "$REMOTEURL" >/dev/null 2>&1
  else
    curl --fail -o "$LOCALFILE" -z "$LOCALFILE" "$REMOTEURL" >/dev/null 2>&1
  fi
  CURLEXIT=$?
  if [ $CURLEXIT -ne 0 ] ; then
    iecho "$REMOTEURL download failed, skipping $LOCALFILE."
  fi
}

function update_single_resource
{
  FM_LOCAL_RESOURCE="$1"
  FM_REMOTE_RESOURCE="$2"
  FM_LABEL="$3"
  FM_PARENTDIR=$(dirname "$FM_LOCAL_RESOURCE")
  mkdir -p "$FM_PARENTDIR" 2>/dev/null
  if ! [ -r "$FM_LOCAL_RESOURCE" ] ; then
    iecho "Fetching $FM_LABEL from upstream"
    curl_download "$FM_LOCAL_RESOURCE" "$FM_REMOTE_RESOURCE" fetch
  else
    iecho "Checking for $FM_LABEL updates"
    curl_download "$FM_LOCAL_RESOURCE" "$FM_REMOTE_RESOURCE" update
  fi
}

function update_lists
{
  update_single_resource "$UPSTREAM_BLACKLIST" "$URL_FOR_UPSTREAM_LISTS/blacklist-latest.txt" "blacklist"
  update_single_resource "$UPSTREAM_WHITELIST" "$URL_FOR_UPSTREAM_LISTS/whitelist-latest.txt" "whitelist"
  update_single_resource "$UPSTREAM_CODE_WHITELIST" "$URL_FOR_UPSTREAM_LISTS/code-whitelist-latest.txt" "code whitelist"
  update_single_resource "$UPSTREAM_RULES" "$URL_FOR_UPSTREAM_LISTS/rules-latest.txt" "rules"
}

if [ "$AUTOUPDATE_LISTS" == "on" ] ; then
  update_lists
fi

function build_resource
{
  FM_UPSTREAM_RES="$1"
  FM_LOCAL_RES="$2"
  FM_RESOURCE="$3"
  cat "$FM_UPSTREAM_RES" > "$FM_RESOURCE"
  if [ -r "$FM_LOCAL_RES" ] ; then
    cat "$FM_LOCAL_RES" >> "$FM_RESOURCE"
  fi
}

FM_WHITELIST=$(mktemp)
FM_WHITECODE=$(mktemp)
FM_BLACKLIST=$(mktemp)
FM_RULES=$(mktemp)

build_resource "$UPSTREAM_WHITELIST" "$WHITELIST_ADDITIONS" "$FM_WHITELIST"
build_resource "$UPSTREAM_BLACKLIST" "$BLACKLIST_ADDITIONS" "$FM_BLACKLIST"
build_resource "$UPSTREAM_RULES" "$RULES_ADDITIONS" "$FM_RULES"
build_resource "$UPSTREAM_CODE_WHITELIST" "$CODE_WHITELIST_ADDITIONS" "$FM_WHITECODE"

function blacklist
{
  echo "SIZE=$FSIZE SHA512=$SHA512" >> "$FM_BLACKLIST"
  echo "SIZE=$FSIZE SHA512=$SHA512" >> "$BLACKLIST_ADDITIONS"
}


function blacklist_and_report
{
  if [ $ALREADYBLACKLISTED -eq 0 ] ; then
    blacklist
    mecho $(realpath "$F") '******INFECTED******'
  else
    mecho $(realpath "$F") '******INFECTED****** (previously BLACKLISTED)'
  fi
  if [ "$AUTOWHITELIST" == "on" ] ; then
    mecho "$F is already blacklisted, it cannot be autowhitelisted. This is ++++++ NASTY ++++++. You've likely used -A in a hacked site root!"
    mecho "Step 1: go read the docs NOW!"
    mecho "Step 2: throw away your current whitelists and start over"
  fi
}

function greylist
{
  mecho $(realpath "$F") '******INFECTED? (greylisted)******'
}

function codeline_hash
{
  OCCURRENCE_NUMBER="$1"
  EXPECTED_FRAGMENT_LINES=$((${NUMBER_OF_CONTEXT_LINES} * 2  + 1))
  grep -C ${NUMBER_OF_CONTEXT_LINES} -f "${FM_RULES}" -m ${OCCURRENCE_NUMBER} -E "${F}" | tail -n ${EXPECTED_FRAGMENT_LINES} | sha512sum | cut -d' ' -f1
}

function check_codeline_whitelist
{
  OCCURRENCE_NUMBER="$1"
  CLHASH=$(codeline_hash ${OCCURRENCE_NUMBER})
  CLHASH_ENTRY="CTXSIZE=${NUMBER_OF_CONTEXT_LINES} SHA512=${CLHASH}"
  CLHASH_EXISTS=$(cat "${FM_WHITECODE}" | grep "${CLHASH_ENTRY}" | wc -l)
  echo $CLHASH_EXISTS
}

function codeline_whitelist
{
  CLHASH_EXISTS=$(check_codeline_whitelist $1) 
  if [ ${CLHASH_EXISTS} -eq 0 ] ; then
    CLHASH=$(codeline_hash ${OCCURRENCE_NUMBER})
    CLHASH_ENTRY="CTXSIZE=${NUMBER_OF_CONTEXT_LINES} SHA512=${CLHASH}"
    echo "${CLHASH_ENTRY}" >> "${FM_WHITECODE}"
    echo "${CLHASH_ENTRY}" >> "${CODE_WHITELIST_ADDITIONS}"
  fi
}

function all_codelines_whitelisted
{
  ALLWL=1
  NUMBER_OF_OCCURRENCES=$(grep -E -f "$FM_RULES" "$F" | wc -l)
  for OCCURRENCE_NUMBER in `seq 1 $NUMBER_OF_OCCURRENCES` ; do
    THISWHITELISTED=$(check_codeline_whitelist $OCCURRENCE_NUMBER)
    if [ $THISWHITELISTED -eq 0 ] ; then
      ALLWL=0
    fi
  done
  echo $ALLWL
}

function codelines_whitelist_all
{
  OCCURRENCE_NUMBER=1
  grep -E -f "$FM_RULES" "$F" | while read WLINE ; do
    codeline_whitelist $OCCURRENCE_NUMBER
    OCCURRENCE_NUMBER=$(($OCCURRENCE_NUMBER + 1))
  done 
}

function whitelist
{
  mecho $(realpath "$F") '!!!!!!WHITELISTED!!!!!!'
  echo "NAME='$FNAMEONLY' SIZE=$FSIZE SHA512=$SHA512" >> "$FM_WHITELIST"
  echo "NAME='$FNAMEONLY' SIZE=$FSIZE SHA512=$SHA512" >> "$WHITELIST_ADDITIONS"
  codelines_whitelist_all
}

function dowload_trusted_url
{
  if [ "${TRUSTED_URL}" != "" ] ; then
    mecho "download_trusted_url: NOT IMPLEMENTED YET"
  fi
}


for EXT in $(echo "$EXTENSIONS_LIST" |tr ',' ' ') ; do
  mecho "Looking for malware in $EXT files."
  iecho "This can take a while, please wait..."
  FILELIST=$(mktemp)
  find "$SCANDIR" -iname \*.$EXT -type f -exec grep -rln -E -f "$FM_RULES" \{\} \; > "$FILELIST"
  PROCESSEDFILES=0
  TOTFILES=$(cat "$FILELIST" | wc -l)
  for F in $(cat "$FILELIST") ; do 
    PROCESSEDFILES=$(($PROCESSEDFILES + 1))
    FNAMEONLY=$(basename "$F")
    FSIZE=$(du -bs "$F" | tr '\t' ' ' | tr -s ' ' | cut -d' ' -f1)
    SHA512=$(cat "$F" | sha512sum | cut -d' ' -f1)

    WHITELISTED=$(cat "$FM_WHITELIST" | grep "SIZE=$FSIZE SHA512=$SHA512" | wc -l)

    if [ $WHITELISTED -eq 0 ] ; then
      CODEWHITELISTED=$(all_codelines_whitelisted)
      if [ $CODEWHITELISTED -eq 0 ] ; then
        ALREADYBLACKLISTED=$(cat "$FM_BLACKLIST" | grep "SIZE=$FSIZE SHA512=$SHA512" | wc -l)
        if [ $ALREADYBLACKLISTED -eq 0 ] ; then
          if [ "$UNATTENDED_EXECUTION" == "off" ] ; then
            SCREEN_COMMANDS=$(mktemp)
            echo "split" > "$SCREEN_COMMANDS"
            echo "screen 1 /bin/sh -c 'less \"$F\" ; screen -S \"\${STY}\" -X quit'" >> "$SCREEN_COMMANDS"
            echo "focus down" >> "$SCREEN_COMMANDS"
            # echo "resize 8" >> "$SCREEN_COMMANDS"
            echo "screen 2" >> "$SCREEN_COMMANDS"
            echo "exec !.. grep -n -o -E -f \"$FM_RULES\" \"$F\"" >> "$SCREEN_COMMANDS"
            echo "focus up" >> "$SCREEN_COMMANDS"
            screen -S review -c "$SCREEN_COMMANDS"
            rm -f "$SCREEN_COMMANDS"
            ASK_INTERACTIVE=true
          else
            if [ "$AUTOWHITELIST" == "on" ] ; then
              whitelist
            else
              greylist
            fi
            ASK_INTERACTIVE=false
          fi
        else
          blacklist_and_report
          ASK_INTERACTIVE=false
        fi
        while [ "$ASK_INTERACTIVE" == "true" ] ; do
          ASK_INTERACTIVE=false
          iecho "Processing file $PROCESSEDFILES of $TOTFILES"
          iecho -n "Mark $F as infected? (y/e/k/?/N): " 
          read D 
          D=$(echo $D | tr 'A-Z' 'a-z')
          case $D in
            y)
              blacklist_and_report
              ;;
            e)
              blacklist_and_report
              vi "$F"
              ;;
            k)
              blacklist_and_report
              mecho $(realpath "$F") '/////MARKED FOR REMOVAL//////'
              ;;
            '?')
              iecho "Hit y to mark as infected"
              iecho "Hit e to mark as infected and edit the file (this assumes write permission)"
              iecho "Hit k to mark as infected and mark for removal in the scan report"
              iecho "Hit ? for this help"
              iecho "Hit n or any other key to avoid marking as infected"
              ASK_INTERACTIVE=true
              ;;
            *)
              iecho -n "Whitelist $F? (Y/n): "
              read W
              W=$(echo $W | tr 'A-Z' 'a-z')
              if [ "$W" != "n" ] ; then
                whitelist
              fi
              ;;
          esac	
        done
      else
        mecho $(realpath "$F") '~~~~~~IGNORED~~~~~~ (matching codelines whitelisted)'
      fi
    else
      NAMEMATCHES=$(cat "$FM_WHITELIST" | grep "NAME='$FNAMEONLY' SIZE=$FSIZE SHA512=$SHA512" | wc -l)
      if [ $NAMEMATCHES -eq 0 ] ; then
        WLNAME=$(cat "$FM_WHITELIST" | grep "SIZE=$FSIZE SHA512=$SHA512" | cut -d\' -f2)
        iecho "$F is already whitelisted under different name: $WLNAME"
      else
        iecho "$F is already whitelisted"
      fi
      mecho $(realpath "$F") '~~~~~~IGNORED~~~~~~ (previously WHITELISTED)'
    fi
  done
done

FM_INFECTED_COUNT=$(cat "$SCANREPORT" | grep '\*\*\*\*\*\*INFECTED\*\*\*\*\*\*' | wc -l)

if [ $FM_INFECTED_COUNT -gt 0 ] ; then
  if [ "$UNATTENDED_EXECUTION" == "off" ] ; then
    iecho "List of INFECTED PHP files I have found:"
    iecho "----------------------------------------"
    iecho
    cat "$SCANREPORT" | grep '\*\*\*\*\*\*INFECTED\*\*\*\*\*\*'
    iecho
  fi
else
  iecho "I couldn't find any known malware sign. It does NOT necessarily mean there isn't."
fi

if [ "$UNATTENDED_EXECUTION" == "off" ] ; then
  mecho "You can find the full report in $SCANREPORT"
else
  echo "$SCANREPORT"
fi