From 754198a06675884ce5c810885f7656745c3c0fde Mon Sep 17 00:00:00 2001 From: Thomas Chopitea Date: Wed, 11 Oct 2023 15:50:33 +0000 Subject: [PATCH 1/3] Add investigation + timesketch feed --- core/schemas/entity.py | 10 + core/taskmanager.py | 1 + plugins/feeds/public/timesketch.py | 82 ++++++++ poetry.lock | 290 ++++++++++++++++++++++++++++- pyproject.toml | 1 + tests/feeds.py | 6 + tests/schemas/fixture.py | 7 +- yeti.conf.sample | 5 + 8 files changed, 400 insertions(+), 2 deletions(-) create mode 100644 plugins/feeds/public/timesketch.py diff --git a/core/schemas/entity.py b/core/schemas/entity.py index 0fe04c804..11d3d089e 100644 --- a/core/schemas/entity.py +++ b/core/schemas/entity.py @@ -21,6 +21,7 @@ class EntityType(str, Enum): company = "company" phone = "phone" note = "note" + investigation = "investigation" class Entity(BaseModel, database_arango.ArangoYetiConnector): @@ -115,6 +116,12 @@ class Identity(Entity): sectors: list[str] = [] contact_information: str = "" +class Investigation(Entity): + _type_filter: ClassVar[str] = EntityType.investigation + type: Literal["investigation"] = EntityType.investigation + + reference: str = "" + TYPE_MAPPING: dict[str, "EntityClasses"] = { "threat-actor": ThreatActor, @@ -129,6 +136,7 @@ class Identity(Entity): "phone": Phone, "note": Note, "identity": Identity, + "investigation": Investigation, } REGEXES_ENTITIES = { EntityType.exploit: re.compile( @@ -152,6 +160,7 @@ class Identity(Entity): | Company | Phone | Note + | Investigation ) EntityClasses = ( Type[ThreatActor] @@ -164,4 +173,5 @@ class Identity(Entity): | Type[Company] | Type[Phone] | Type[Note] + | Type[Investigation] ) diff --git a/core/taskmanager.py b/core/taskmanager.py index b5dd811f5..7d0163ce4 100644 --- a/core/taskmanager.py +++ b/core/taskmanager.py @@ -69,6 +69,7 @@ "plugins.feeds.public.sslblacklist_ip", "plugins.feeds.public.threatfox", "plugins.feeds.public.threatview_c2", + "plugins.feeds.public.timesketch", "plugins.feeds.public.tor_exit_nodes", "plugins.feeds.public.urlhaus", "plugins.feeds.public.viriback_tracker", diff --git a/plugins/feeds/public/timesketch.py b/plugins/feeds/public/timesketch.py new file mode 100644 index 000000000..3987bc00d --- /dev/null +++ b/plugins/feeds/public/timesketch.py @@ -0,0 +1,82 @@ +from datetime import timedelta, datetime +from time import sleep +import logging + +from core.schemas import observable +from core.schemas.observables import ipv4, hostname, url, sha1, md5, sha256, path +from core.schemas.entity import Investigation +from core.schemas import task +from core import taskmanager +from core.config.config import yeti_config + +from timesketch_api_client import client + + +TIMESKETCH_TYPE_MAPPING = { + 'ipv4': ipv4.IPv4, + 'hostname': hostname.Hostname, + 'hash_sha1': sha1.SHA1, + 'hash_md5': md5.MD5, + 'hash_sha256': sha256.SHA256, + 'url': url.Url, + 'fs_path': path.Path, +} + +class Timesketch(task.FeedTask): + + _defaults = { + "name": "Timesketch", + "frequency": timedelta(hours=1), + "type": "feed", + "description": "This feed creates Investigations from a Timesketch server.", + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._endpoint = yeti_config.get("timesketch", "endpoint") + if not self._endpoint: + logging.error("Timesketch cannot proceed without an endpoint.") + return + username = yeti_config.get("timesketch", "username") + password = yeti_config.get("timesketch", "password") + + self._ts_client = client.TimesketchApi(self._endpoint, username, password) + + + def run(self): + if not self._endpoint: + logging.error("Timesketch cannot proceed without an endpoint.") + return + + sketches = self._ts_client.list_sketches() + + for sketch in sketches: + description = "# Timelines\n\n" + for timeline in sketch.list_timelines(): + description += f"- {timeline.name}\n" + + created_at = datetime.strptime(sketch.resource_data['objects'][0]['created_at'], "%Y-%m-%dT%H:%M:%S.%f") + + investigation = Investigation( + name=sketch.name, + created=created_at, + reference=f'{self._endpoint}/sketch/{sketch.id}', + description=description, + ).save() + for intel in sketch.get_intelligence_attribute(): + observable_type = TIMESKETCH_TYPE_MAPPING.get(intel['type']) + if observable_type: + obs = observable_type(value=intel['ioc']).save() + else: + try: + obs = observable.Observable.add_text(intel['ioc']) + except ValueError as error: + logging.error('Error adding observable %s from Timesketch: %s', intel['ioc'], error) + continue + + obs.tag(intel['tags']) + obs.add_context("timesketch", {'sketch_link': intel['externalURI'], 'sketch_tags': intel['tags']}) + obs.link_to(investigation, "seen in", f"Observable seen in {investigation.name}") + + +taskmanager.TaskManager.register_task(Timesketch) diff --git a/poetry.lock b/poetry.lock index 3f089de70..0c28d4dbc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,29 @@ # This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +[[package]] +name = "altair" +version = "5.1.2" +description = "Vega-Altair: A declarative statistical visualization library for Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "altair-5.1.2-py3-none-any.whl", hash = "sha256:7219708ec33c152e53145485040f428954ed15fd09b2a2d89e543e6d111dae7f"}, + {file = "altair-5.1.2.tar.gz", hash = "sha256:e5f52a71853a607c61ce93ad4a414b3d486cd0d46ac597a24ae8bd1ac99dd460"}, +] + +[package.dependencies] +jinja2 = "*" +jsonschema = ">=3.0" +numpy = "*" +packaging = "*" +pandas = ">=0.25" +toolz = "*" +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["anywidget", "black (<24)", "hatch", "ipython", "m2r", "mypy", "pandas-stubs", "pyarrow (>=11)", "pytest", "pytest-cov", "ruff", "types-jsonschema", "types-setuptools", "vega-datasets", "vegafusion[embed] (>=1.4.0)", "vl-convert-python (>=0.14.0)"] +doc = ["docutils", "geopandas", "jinja2", "myst-parser", "numpydoc", "pillow (>=9,<10)", "pydata-sphinx-theme", "scipy", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinxext-altair"] + [[package]] name = "amqp" version = "5.1.1" @@ -128,6 +152,24 @@ files = [ tests = ["pytest (>=3.2.1,!=3.3.0)"] typecheck = ["mypy"] +[[package]] +name = "beautifulsoup4" +version = "4.12.2" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"}, + {file = "beautifulsoup4-4.12.2.tar.gz", hash = "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "billiard" version = "4.1.0" @@ -173,6 +215,17 @@ d = ["aiohttp (>=3.7.4)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "cachetools" +version = "5.3.1" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.1-py3-none-any.whl", hash = "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590"}, + {file = "cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"}, +] + [[package]] name = "celery" version = "5.3.4" @@ -620,6 +673,47 @@ docs = ["furo (>=2023.7.26)", "sphinx (>=7.1.2)", "sphinx-autodoc-typehints (>=1 testing = ["covdefaults (>=2.3)", "coverage (>=7.3)", "diff-cover (>=7.7)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-timeout (>=2.1)"] typing = ["typing-extensions (>=4.7.1)"] +[[package]] +name = "google-auth" +version = "2.23.3" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-auth-2.23.3.tar.gz", hash = "sha256:6864247895eea5d13b9c57c9e03abb49cb94ce2dc7c58e91cba3248c7477c9e3"}, + {file = "google_auth-2.23.3-py2.py3-none-any.whl", hash = "sha256:a8f4608e65c244ead9e0538f181a96c6e11199ec114d41f1d7b1bffa96937bda"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-auth-oauthlib" +version = "1.1.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "google-auth-oauthlib-1.1.0.tar.gz", hash = "sha256:83ea8c3b0881e453790baff4448e8a6112ac8778d1de9da0b68010b843937afb"}, + {file = "google_auth_oauthlib-1.1.0-py2.py3-none-any.whl", hash = "sha256:089c6e587d36f4803ac7e0720c045c6a8b1fd1790088b8424975b90d0ee61c12"}, +] + +[package.dependencies] +google-auth = ">=2.15.0" +requests-oauthlib = ">=0.7.0" + +[package.extras] +tool = ["click (>=6.0.0)"] + [[package]] name = "h11" version = "0.14.0" @@ -722,6 +816,23 @@ pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib" plugins = ["setuptools"] requirements-deprecated-finder = ["pip-api", "pipreqs"] +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + [[package]] name = "jsonschema" version = "4.19.1" @@ -834,6 +945,75 @@ files = [ {file = "lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, ] +[[package]] +name = "markupsafe" +version = "2.1.3" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, + {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, +] + [[package]] name = "mccabe" version = "0.7.0" @@ -902,6 +1082,24 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "networkx" +version = "3.1" +description = "Python package for creating and manipulating graphs and networks" +optional = false +python-versions = ">=3.8" +files = [ + {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, + {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, +] + +[package.extras] +default = ["matplotlib (>=3.4)", "numpy (>=1.20)", "pandas (>=1.3)", "scipy (>=1.8)"] +developer = ["mypy (>=1.1)", "pre-commit (>=3.2)"] +doc = ["nb2plots (>=0.6)", "numpydoc (>=1.5)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.13)", "sphinx (>=6.1)", "sphinx-gallery (>=0.12)", "texext (>=0.6.7)"] +extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.10)", "sympy (>=1.10)"] +test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] + [[package]] name = "numpy" version = "1.26.0" @@ -943,6 +1141,22 @@ files = [ {file = "numpy-1.26.0.tar.gz", hash = "sha256:f93fc78fe8bf15afe2b8d6b6499f1c73953169fad1e9a8dd086cdff3190e7fdf"}, ] +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +optional = false +python-versions = ">=3.6" +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + [[package]] name = "otxv2" version = "1.5.12" @@ -1123,6 +1337,20 @@ files = [ {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, ] +[[package]] +name = "pyasn1-modules" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.6.0" + [[package]] name = "pycparser" version = "2.21" @@ -1497,6 +1725,24 @@ files = [ requests = ">=1.0.0" six = "*" +[[package]] +name = "requests-oauthlib" +version = "1.3.1" +description = "OAuthlib authentication support for Requests." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, + {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, +] + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + [[package]] name = "requests-toolbelt" version = "1.0.0" @@ -1704,6 +1950,17 @@ files = [ {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, ] +[[package]] +name = "soupsieve" +version = "2.5" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + [[package]] name = "starlette" version = "0.27.0" @@ -1721,6 +1978,26 @@ anyio = ">=3.4.0,<5" [package.extras] full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"] +[[package]] +name = "timesketch-api-client" +version = "20230721" +description = "Timesketch API client" +optional = false +python-versions = "*" +files = [ + {file = "timesketch-api-client-20230721.tar.gz", hash = "sha256:72cf32b2d6c194686b0c5e8b2bfe225a49d7bb6b9116a379a81559d7c325be25"}, +] + +[package.dependencies] +altair = "*" +beautifulsoup4 = "*" +cryptography = "*" +google-auth = "*" +google_auth_oauthlib = "*" +networkx = "*" +pandas = "*" +requests = "*" + [[package]] name = "tldextract" version = "3.6.0" @@ -1760,6 +2037,17 @@ files = [ {file = "tomlkit-0.12.1.tar.gz", hash = "sha256:38e1ff8edb991273ec9f6181244a6a391ac30e9f5098e7535640ea6be97a7c86"}, ] +[[package]] +name = "toolz" +version = "0.12.0" +description = "List processing tools and functional utilities" +optional = false +python-versions = ">=3.5" +files = [ + {file = "toolz-0.12.0-py3-none-any.whl", hash = "sha256:2059bd4148deb1884bb0eb770a3cde70e7f954cfbbdc2285f1f2de01fd21eb6f"}, + {file = "toolz-0.12.0.tar.gz", hash = "sha256:88c570861c440ee3f2f6037c4654613228ff40c93a6c25e0eba70d17282c6194"}, +] + [[package]] name = "typing-extensions" version = "4.8.0" @@ -1975,4 +2263,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "ba1a200eb2c63b3b46cc070004d1f8fa9651e89864abadd65119df049bf0ce5b" +content-hash = "5e9d430887e4febe4322978966b00d13de1b6f880dd4ce3bbc5144127ef8de06" diff --git a/pyproject.toml b/pyproject.toml index 8c696da87..1fe99ba1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ httpx = "^0.23.3" pymisp = "^2.4.176" otxv2 = "^1.5.12" shodan = "^1.30.0" +timesketch-api-client = "^20230721" [build-system] requires = ["poetry-core"] diff --git a/tests/feeds.py b/tests/feeds.py index c3b661123..daedf8382 100644 --- a/tests/feeds.py +++ b/tests/feeds.py @@ -5,6 +5,7 @@ from plugins.feeds.public import feodo_tracker_ip_blocklist from plugins.feeds.public import openphish from plugins.feeds.public import lolbas +from plugins.feeds.public import timesketch class FeedTest(unittest.TestCase): @@ -29,3 +30,8 @@ def test_lolbas(self): defaults = lolbas.LoLBAS._defaults.copy() feed = lolbas.LoLBAS(**defaults) feed.run() + + def test_timesketch(self): + defaults = timesketch.Timesketch._defaults.copy() + feed = timesketch.Timesketch(**defaults) + feed.run() diff --git a/tests/schemas/fixture.py b/tests/schemas/fixture.py index 12aed5d63..e6618e881 100644 --- a/tests/schemas/fixture.py +++ b/tests/schemas/fixture.py @@ -5,7 +5,7 @@ from core import database_arango from core.schemas.observable import Observable from core.schemas.observables import ipv4, hostname, url -from core.schemas.entity import ThreatActor, Malware +from core.schemas.entity import ThreatActor, Malware, Investigation from core.schemas.tag import Tag from core.schemas.indicator import Regex, DiamondModel, Query, QueryType from core.schemas.template import Template @@ -59,6 +59,11 @@ def test_something(self): query_type=QueryType.opensearch, target_systems=['timesketch', 'plaso'], relevant_tags=['ssh', 'login']).save() + + i = Investigation( + name='coin mining case', + reference='http://timesketch-server/sketch/12345', + relevant_tags=['coin', 'mining']).save() template = Template(name="RandomTemplate", template="").save() export = ExportTask( name="RandomExport", diff --git a/yeti.conf.sample b/yeti.conf.sample index fc02dc5b3..9dd036584 100644 --- a/yeti.conf.sample +++ b/yeti.conf.sample @@ -143,3 +143,8 @@ api_key= [malshare] api_key= + +[timesketch] +endpoint = +username = +password = From 0b48236099a51b4ab4552921de2dbd0e520d0893 Mon Sep 17 00:00:00 2001 From: Thomas Chopitea Date: Wed, 11 Oct 2023 15:53:11 +0000 Subject: [PATCH 2/3] Add some regexes for parsing paths --- core/schemas/observable.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/core/schemas/observable.py b/core/schemas/observable.py index 2e7368fde..569ef4994 100644 --- a/core/schemas/observable.py +++ b/core/schemas/observable.py @@ -1,17 +1,18 @@ # TODO Observable value normalization import datetime +import re from enum import Enum -from typing import Literal, Optional, ClassVar +from typing import ClassVar, Literal, Optional -from pydantic import BaseModel, Field import validators +from pydantic import BaseModel, Field + from core import database_arango -from core.helpers import refang +from core.helpers import now, refang from core.schemas.entity import Entity -from core.schemas.tag import DEFAULT_EXPIRATION_DAYS, Tag from core.schemas.graph import TagRelationship -from core.helpers import now +from core.schemas.tag import DEFAULT_EXPIRATION_DAYS, Tag # Data Schema @@ -177,7 +178,13 @@ def delete_context( ObservableType.email: validators.email, } -REGEXES_OBSERVABLES = {} +REGEXES_OBSERVABLES = { + # Unix + ObservableType.path : [ + re.compile(r"^(\/[^\/\0]+)+$"), + re.compile(r"^(?:[a-zA-Z]\:|\\\\[\w\.]+\\[\w.$]+)\\(?:[\w]+\\)*\w([\w.])+") + ] +} def validate_observable(obs: Observable) -> bool: @@ -193,9 +200,10 @@ def find_type(value: str) -> ObservableType | None: for obs_type in TYPE_VALIDATOR_MAP: if TYPE_VALIDATOR_MAP[obs_type](value): return obs_type - for type_obs, regex in REGEXES_OBSERVABLES.items(): - if regex.match(value): - return type_obs + for obs_type, regexes in REGEXES_OBSERVABLES.items(): + for regex in regexes: + if regex.match(value): + return obs_type return None From 4ea3f260011f8b0a2f57ecf6859561b94fbf21b7 Mon Sep 17 00:00:00 2001 From: Thomas Chopitea Date: Wed, 11 Oct 2023 15:53:59 +0000 Subject: [PATCH 3/3] Add identity again? --- core/schemas/entity.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/schemas/entity.py b/core/schemas/entity.py index 11d3d089e..d4595fd7a 100644 --- a/core/schemas/entity.py +++ b/core/schemas/entity.py @@ -132,6 +132,7 @@ class Investigation(Entity): "campaign": Campaign, "entities": Entity, "entity": Entity, + "identity": Identity, "company": Company, "phone": Phone, "note": Note,