Skip to content

Commit

Permalink
Fix DLP event processing (#26)
Browse files Browse the repository at this point in the history 
* Update config.py

Fix pep8 issues in config.py

* Update name_mapping.py

Fix DLP rulename quoting, pep8 compliance.

* Update siem.py

pep8 compliance

* Update test_regression.py

pep8 compliance

* Create .gitignore

Add .gitignore file

* Update name_mapping.py

Remove erroneously added text.
  • Loading branch information
@keeely
keeely authored Mar 14, 2019
1 parent e6aa870 commit 8f28ea0
Show file tree
Hide file tree
Showing 5 changed files with 205 additions and 177 deletions.
6 changes: 6 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
*.pyc
*.zip
*~
/.idea/
/log/
/state/
23 changes: 12 additions & 11 deletions config.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env python

# Copyright 2017 Sophos Limited
# Copyright 2019 Sophos Limited
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
# compliance with the License.
Expand All @@ -12,6 +12,10 @@
# License.
#

import unittest
import shutil
import tempfile
import os
import re
try:
import ConfigParser
Expand All @@ -20,9 +24,9 @@


class Config:
"Class providing config values"
"""Class providing config values"""
def __init__(self, path):
"Open the config file"
"""Open the config file"""
self.config = ConfigParser.ConfigParser()
self.config.read(path)

Expand All @@ -32,8 +36,9 @@ def __getattr__(self, name):

class Token:
def __init__(self, token_txt):
"Initialize with the token text"
rex = re.compile(r"url\: (?P<url>https\://.+), x-api-key\: (?P<api_key>.+), Authorization\: (?P<authorization>.+)$")
"""Initialize with the token text"""
rex_txt = r"url\: (?P<url>https\://.+), x-api-key\: (?P<api_key>.+), Authorization\: (?P<authorization>.+)$"
rex = re.compile(rex_txt)
m = rex.search(token_txt)
self.url = m.group("url")
self.api_key = m.group("api_key")
Expand All @@ -46,11 +51,8 @@ def __init__(self, token_txt):
#


import unittest, copy, shutil, tempfile, os


class TestConfig(unittest.TestCase):
"Test Config file items are exposed as attributes on config object"
"""Test Config file items are exposed as attributes on config object"""

def setUp(self):
self.tmpdir = tempfile.mkdtemp(prefix="config_test", dir=".")
Expand All @@ -68,7 +70,7 @@ def testReadingWhenAttributeExists(self):


class TestToken(unittest.TestCase):
"Test the token gets parsed"
"""Test the token gets parsed"""
def testParse(self):
txt = " url: https://anywhere.com/api, x-api-key: random, Authorization: Basic KJNKLJNjklNLKHB= "
t = Token(txt)
Expand All @@ -79,4 +81,3 @@ def testParse(self):

if __name__ == '__main__':
unittest.main()

104 changes: 53 additions & 51 deletions name_mapping.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2017 Sophos Limited
# Copyright 2019 Sophos Limited
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
# compliance with the License.
Expand All @@ -11,42 +11,44 @@
#

import re
import unittest
import copy


threat_regex = re.compile("'(?P<detection_identity_name>.*?)'.+'(?P<filePath>.*?)'")

# What to do with the different types of event. None indicates drop the event, otherwise a regex extracts the
# various fields and inserts them into the event dictionary.
TYPE_HANDLERS = {
"Event::Endpoint::Threat::Detected" : threat_regex,
"Event::Endpoint::Threat::CleanedUp" : threat_regex,
"Event::Endpoint::Threat::HIPSDismissed" : threat_regex,
"Event::Endpoint::Threat::HIPSDetected" : threat_regex,
"Event::Endpoint::Threat::PuaDetected" : threat_regex,
"Event::Endpoint::Threat::PuaCleanupFailed" : threat_regex,
"Event::Endpoint::Threat::HIPSDetected" : threat_regex,
"Event::Endpoint::Threat::HIPSDismissed" : threat_regex,
"Event::Endpoint::Threat::CleanupFailed" : threat_regex,
"Event::Endpoint::Threat::CommandAndControlDismissed" : threat_regex,
"Event::Endpoint::Threat::HIPSCleanupFailed" : threat_regex,
"Event::Endpoint::DataLossPreventionUserAllowed" :
re.compile(u"An \u2033(?P<name>.+)\u2033.+ Username: (?P<user>.+?) Rule names: '(?P<rule>.+?)' "
"User action: (?P<user_action>.+?) Application Name: (?P<app_name>.+?) Data Control action: (?P<action>.+?) "
"File type: (?P<file_type>.+?) File size: (?P<file_size>\\d+?) Source path: (?P<file_path>.+)$"),

"Event::Endpoint::NonCompliant" : None, # None == ignore the event
"Event::Endpoint::Compliant" : None,
"Event::Endpoint::Device::AlertedOnly" : None,
"Event::Endpoint::UpdateFailure" : None,
"Event::Endpoint::SavScanComplete" : None,
"Event::Endpoint::Application::Allowed" : None,
"Event::Endpoint::UpdateSuccess" : None,
"Event::Endpoint::WebControlViolation" : None,
"Event::Endpoint::WebFilteringBlocked" : None,
"Event::Endpoint::Threat::Detected": threat_regex,
"Event::Endpoint::Threat::CleanedUp": threat_regex,
"Event::Endpoint::Threat::HIPSDismissed": threat_regex,
"Event::Endpoint::Threat::HIPSDetected": threat_regex,
"Event::Endpoint::Threat::PuaDetected": threat_regex,
"Event::Endpoint::Threat::PuaCleanupFailed": threat_regex,
"Event::Endpoint::Threat::CleanupFailed": threat_regex,
"Event::Endpoint::Threat::CommandAndControlDismissed": threat_regex,
"Event::Endpoint::Threat::HIPSCleanupFailed": threat_regex,
"Event::Endpoint::DataLossPreventionUserAllowed":
re.compile(u"An \u2033(?P<name>.+)\u2033.+ Username: (?P<user>.+?) {2}"
u"Rule names: \u2032(?P<rule>.+?)\u2032 {2}"
"User action: (?P<user_action>.+?) {2}Application Name: (?P<app_name>.+?) {2}"
"Data Control action: (?P<action>.+?) {2}"
"File type: (?P<file_type>.+?) {2}File size: (?P<file_size>\\d+?) {2}"
"Source path: (?P<file_path>.+)$"),

"Event::Endpoint::NonCompliant": None, # None == ignore the event
"Event::Endpoint::Compliant": None,
"Event::Endpoint::Device::AlertedOnly": None,
"Event::Endpoint::UpdateFailure": None,
"Event::Endpoint::SavScanComplete": None,
"Event::Endpoint::Application::Allowed": None,
"Event::Endpoint::UpdateSuccess": None,
"Event::Endpoint::WebControlViolation": None,
"Event::Endpoint::WebFilteringBlocked": None,
}



def update_fields(log, data):
"""
Split 'name' field into multiple fields based on regex and field names specified in TYPE_HANDLERS
Expand All @@ -62,7 +64,7 @@ def update_fields(log, data):
return
result = prog_regex.search(data[u'name'])
if not result:
log("Failed to split name field for event type %s" % data[u'type'])
log("Failed to split name field for event type %r" % data[u'type'])
return

# Make sure record has a name field corresponding to the first field (for the CEF format)
Expand All @@ -80,10 +82,12 @@ def update_fields(log, data):
#


import unittest, copy
def contains(dict_outer, dict_inner):
return all(item in dict_outer.items() for item in dict_inner.items())


class TestNameExtraction(unittest.TestCase):
"Test logging output"
"""Test logging output"""

def setUp(self):
self.output = []
Expand All @@ -94,16 +98,15 @@ def tearDown(self):
def log(self, s):
self.output.append(s)

def contains(self, dict_outer, dict_inner):
return all(item in dict_outer.items() for item in dict_inner.items())

def testUpdateNameDLPValid(self):
"DLP event with data that can be extracted"
"""DLP event with data that can be extracted"""
data = {
"type": "Event::Endpoint::DataLossPreventionUserAllowed",
"name": u"An \u2033allow transfer on acceptance by user\u2033 action was taken. Username: WIN10CLOUD2\\Sophos "
"Rule names: 'test' User action: File open Application Name: Google Chrome Data Control action: Allow "
"File type: Plain text (ASCII/UTF-8) File size: 36 Source path: C:\\Users\\Sophos\\Desktop\\test.txt"
"name": u"An \u2033allow transfer on acceptance by user\u2033 action was taken. "
u"Username: WIN10CLOUD2\\Sophos Rule names: \u2032test\u2032 User action: File open "
u"Application Name: Google Chrome Data Control action: Allow "
u"File type: Plain text (ASCII/UTF-8) File size: 36 "
u"Source path: C:\\Users\\Sophos\\Desktop\\test.txt"
}
expected = {
"type": "Event::Endpoint::DataLossPreventionUserAllowed",
Expand All @@ -118,55 +121,55 @@ def testUpdateNameDLPValid(self):
"file_path": "C:\\Users\\Sophos\\Desktop\\test.txt"
}
update_fields(self.log, data)
self.assertTrue(self.contains(data, expected))
self.assertTrue(all(item in data.items() for item in expected.items()))
self.assertEqual(len(self.output), 0)

def testUpdateNameThreatValid(self):
"Threat event with data that can be extracted"
data = { "type": "Event::Endpoint::Threat::CleanedUp", "name": u"Threat 'EICAR' in 'myfile.com' "}
"""Threat event with data that can be extracted"""
data = {"type": "Event::Endpoint::Threat::CleanedUp", "name": u"Threat 'EICAR' in 'myfile.com' "}
expected = {
"type": "Event::Endpoint::Threat::CleanedUp",
"name": u"EICAR",
"filePath": "myfile.com",
"detection_identity_name": "EICAR"
}
update_fields(self.log, data)
self.assertTrue(self.contains(data, expected)) # expected data present
self.assertTrue(contains(data, expected)) # expected data present
self.assertEqual(len(self.output), 0) # no error

def testUpdateNameInvalid(self):
"A known type, but information can't be extracted (regex mismatch)"
"""A known type, but information can't be extracted (regex mismatch)"""
data = {"type": "Event::Endpoint::DataLossPreventionUserAllowed", "name": u"XXXX Garbage data XXXX"}
before = copy.copy(data)
update_fields(self.log, data)
self.assertEqual(len(self.output), 1) # a line of error output, when the function bails.
self.assertEqual(data, before) # ... and data remains unchanged

def testUpdateNameFromDescription(self):
"Ensure the name gets updated from the description, if present"
"""Ensure the name gets updated from the description, if present"""
data = {"type": "", "description": "XXX"}
expected = copy.copy(data)
expected["name"] = "XXX"
update_fields(self.log, data)
self.assertEqual(data, expected)

def testInvalidType(self):
"Ensure that nothing gets changed when the type isn't recognised"
data = {"type": "<<Garbage>>", "name":"some name"}
"""Ensure that nothing gets changed when the type isn't recognised"""
data = {"type": "<<Garbage>>", "name": "some name"}
expected = copy.copy(data)
update_fields(self.log, data)
self.assertEqual(len(self.output), 0) # not considered an error
self.assertEqual(data, expected)

def testSkippedType(self):
"Ensure that entry is skipped if it's to be ignored."
"""Ensure that entry is skipped if it's to be ignored."""
# First find an event type that is set to 'None'
toskip = None
for k,v in TYPE_HANDLERS.items():
for k, v in TYPE_HANDLERS.items():
if not v:
toskip = k
break
data = {"type": toskip, "name":"some name"}
toskip = k
break
data = {"type": toskip, "name": "some name"}
expected = copy.copy(data)
update_fields(self.log, data)
self.assertEqual(len(self.output), 0) # not considered an error
Expand All @@ -175,4 +178,3 @@ def testSkippedType(self):

if __name__ == '__main__':
unittest.main()

27 changes: 19 additions & 8 deletions siem.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env python

# Copyright 2017 Sophos Limited
# Copyright 2019 Sophos Limited
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
# compliance with the License.
Expand Down Expand Up @@ -41,10 +41,17 @@
import config


def get_syslog_facilities():
"""Create a mapping between our names and the python syslog defines"""
out = {}
possible = "auth cron daemon kern lpr mail news syslog user uucp " \
"local0 local1 local2 local3 local4 local5 local6 local7".split()
for facility in possible:
out[facility] = getattr(logging.handlers.SysLogHandler, "LOG_%s" % facility.upper())
return out


SYSLOG_FACILITY = {}
for facility in ['auth','cron','daemon','kern','lpr','mail','news','syslog','user','uucp','local0','local1','local2','local3','local4','local5','local6','local7']:
SYSLOG_FACILITY[facility] = getattr(logging.handlers.SysLogHandler, "LOG_%s" % facility.upper())
SYSLOG_FACILITY = get_syslog_facilities()


SYSLOG_SOCKTYPE = {'udp': socket.SOCK_DGRAM,
Expand All @@ -68,7 +75,11 @@
'very_high': 10}


NOISY_EVENTTYPES = [k for k,v in name_mapping.TYPE_HANDLERS.items() if not v]
def get_noisy_event_types():
return [k for k, v in name_mapping.TYPE_HANDLERS.items() if not v]


NOISY_EVENTTYPES = get_noisy_event_types()

EVENTS_V1 = '/siem/v1/events'
ALERTS_V1 = '/siem/v1/alerts'
Expand All @@ -92,7 +103,7 @@
# CEF_header_prefix: JSON_key
"device_event_class_id": "type",
"name": "name",
"severity" :"severity",
"severity": "severity",

# json to CEF extension mapping
# Format
Expand Down Expand Up @@ -325,7 +336,6 @@ def call_endpoint(opener, endpoint, since, cursor, state_file_path, token):
if DEBUG:
log("RESPONSE: %s" % events_response)
events = json.loads(events_response)


# events looks like this
# {
Expand Down Expand Up @@ -469,5 +479,6 @@ def format_cef(data):
def remove_null_values(data):
return {k: v for k, v in data.items() if v is not None}


if __name__ == "__main__":
main()
main()
Loading

0 comments on commit 8f28ea0

Please sign in to comment.