From 4159795316047ec3846b9da4d32241eef32ca888 Mon Sep 17 00:00:00 2001 From: yampelo Date: Sun, 2 Jun 2019 18:10:29 -0400 Subject: [PATCH 1/3] begins OSSEM Transformer --- beagle/transformers/__init__.py | 3 +- beagle/transformers/ossem_transformer.py | 60 ++++++++++++++++++++ tests/transformers/test_ossem_transformer.py | 28 +++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 beagle/transformers/ossem_transformer.py create mode 100644 tests/transformers/test_ossem_transformer.py diff --git a/beagle/transformers/__init__.py b/beagle/transformers/__init__.py index 7b9d87a3..da4755f7 100644 --- a/beagle/transformers/__init__.py +++ b/beagle/transformers/__init__.py @@ -8,7 +8,7 @@ from .procmon_transformer import ProcmonTransformer from .sysmon_transformer import SysmonTransformer from .darpa_tc_transformer import DRAPATCTransformer - +from .ossem_transformer import OSSEMTransformer __all__ = [ "Transformer", @@ -19,4 +19,5 @@ "ProcmonTransformer", "SysmonTransformer", "DRAPATCTransformer", + "OSSEMTransformer", ] diff --git a/beagle/transformers/ossem_transformer.py b/beagle/transformers/ossem_transformer.py new file mode 100644 index 00000000..adc80798 --- /dev/null +++ b/beagle/transformers/ossem_transformer.py @@ -0,0 +1,60 @@ +from typing import List, Optional, Tuple + +from beagle.nodes import Process, File + +from beagle.common import logger +from beagle.transformers.base_transformer import Transformer + + +class OSSEMTransformer(Transformer): + """Transformer based on the formats defined here: + https://github.com/Cyb3rWard0g/OSSEM/tree/master/common_information_model + + Parameters + ---------- + Transformer : [type] + [description] + """ + + name = "OSSEM" + + def __init__(self, *args, **kwargs) -> None: + + super().__init__(*args, **kwargs) + + logger.info("Created OSSEM Transformer.") + + def transform(self, event: dict) -> Optional[Tuple]: + + relationship = event["event_type"] + + if relationship == "created": + return self.created(event) + + return tuple() + + def created(self, event: dict) -> Tuple[Process, File, Process, File]: + + proc = Process( + process_id=event.get("process_id"), + process_image=event.get("process_name"), + process_image_path=event.get("process_path"), + command_line=event.get("process_command_line"), + ) + + parent = Process( + process_id=event.get("process_parent_id"), + process_image=event.get("process_parent_path"), + process_image_path=event.get("process_path"), + command_line=event.get("process_parent_command_line"), + ) + + parent_file = parent.get_file_node() + proc_file = proc.get_file_node() + + parent_file.file_of[parent] + proc_file.file_of[proc] + + parent.launched[proc].append(timestamp=event["time"]) + + return (proc, proc_file, parent, parent_file) diff --git a/tests/transformers/test_ossem_transformer.py b/tests/transformers/test_ossem_transformer.py new file mode 100644 index 00000000..10f2ee9f --- /dev/null +++ b/tests/transformers/test_ossem_transformer.py @@ -0,0 +1,28 @@ +@pytest.fixture +def transformer() -> DRAPATCTransformer: + return DRAPATCTransformer(datasource=None) # type: ignore + + +def test_make_process(transformer): + test_event = { + "event_type": "subject", + "uuid": "B80F3806-0000-0000-0000-000000000020", + "type": "SUBJECT_PROCESS", + "cid": 4024, + "parentSubject": { + "com.bbn.tc.schema.avro.cdm18.UUID": "52071700-0000-0000-0000-000000000020" + }, + "hostId": "0A00063C-5254-00F0-0D60-000000000070", + "localPrincipal": "EC000000-0000-0000-0000-000000000060", + "startTimestampNanos": 1522943310819901200, + "unitId": None, + "iteration": None, + "count": None, + "cmdLine": {"string": "/usr/bin/firefox"}, + "privilegeLevel": None, + "importedLibraries": None, + "exportedLibraries": None, + "properties": { + "map": {"tgid": "3934", "path": "/home/admin/Downloads/firefox/firefox", "ppid": "1874"} + }, + } From 68803ff6af64a4ae24f19cb78f53f16cc99dc05b Mon Sep 17 00:00:00 2001 From: yampelo Date: Sun, 2 Jun 2019 18:47:31 -0400 Subject: [PATCH 2/3] Adds Process Events to OSSEM Transformer --- beagle/transformers/ossem_transformer.py | 19 +++--- tests/transformers/test_ossem_transformer.py | 69 +++++++++++++------- 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/beagle/transformers/ossem_transformer.py b/beagle/transformers/ossem_transformer.py index adc80798..e0635528 100644 --- a/beagle/transformers/ossem_transformer.py +++ b/beagle/transformers/ossem_transformer.py @@ -1,8 +1,7 @@ from typing import List, Optional, Tuple -from beagle.nodes import Process, File - -from beagle.common import logger +from beagle.common import logger, split_path +from beagle.nodes import File, Process from beagle.transformers.base_transformer import Transformer @@ -35,17 +34,21 @@ def transform(self, event: dict) -> Optional[Tuple]: def created(self, event: dict) -> Tuple[Process, File, Process, File]: + proc_name, proc_path = split_path(event["process_path"]) + proc = Process( process_id=event.get("process_id"), - process_image=event.get("process_name"), - process_image_path=event.get("process_path"), + process_image=proc_name, + process_image_path=proc_path, command_line=event.get("process_command_line"), ) + proc_name, proc_path = split_path(event["process_parent_path"]) + parent = Process( process_id=event.get("process_parent_id"), - process_image=event.get("process_parent_path"), - process_image_path=event.get("process_path"), + process_image=proc_name, + process_image_path=proc_path, command_line=event.get("process_parent_command_line"), ) @@ -55,6 +58,6 @@ def created(self, event: dict) -> Tuple[Process, File, Process, File]: parent_file.file_of[parent] proc_file.file_of[proc] - parent.launched[proc].append(timestamp=event["time"]) + parent.launched[proc].append(timestamp=event["event_creation_time"]) return (proc, proc_file, parent, parent_file) diff --git a/tests/transformers/test_ossem_transformer.py b/tests/transformers/test_ossem_transformer.py index 10f2ee9f..b15a2b3f 100644 --- a/tests/transformers/test_ossem_transformer.py +++ b/tests/transformers/test_ossem_transformer.py @@ -1,28 +1,53 @@ +import pytest + +from beagle.nodes import Process, File +from beagle.transformers import OSSEMTransformer + + @pytest.fixture -def transformer() -> DRAPATCTransformer: - return DRAPATCTransformer(datasource=None) # type: ignore +def transformer() -> OSSEMTransformer: + return OSSEMTransformer(datasource=None) # type: ignore + + +def test_unknown_event(transformer): + assert transformer.transform({"event_type": "foooo"}) == () def test_make_process(transformer): test_event = { - "event_type": "subject", - "uuid": "B80F3806-0000-0000-0000-000000000020", - "type": "SUBJECT_PROCESS", - "cid": 4024, - "parentSubject": { - "com.bbn.tc.schema.avro.cdm18.UUID": "52071700-0000-0000-0000-000000000020" - }, - "hostId": "0A00063C-5254-00F0-0D60-000000000070", - "localPrincipal": "EC000000-0000-0000-0000-000000000060", - "startTimestampNanos": 1522943310819901200, - "unitId": None, - "iteration": None, - "count": None, - "cmdLine": {"string": "/usr/bin/firefox"}, - "privilegeLevel": None, - "importedLibraries": None, - "exportedLibraries": None, - "properties": { - "map": {"tgid": "3934", "path": "/home/admin/Downloads/firefox/firefox", "ppid": "1874"} - }, + "event_creation_time": "1", + "event_type": "created", + "process_id": "4756", + "process_name": "conhost.exe", + "process_path": "C:\\Windows\\System32\\conhost.exe", + "process_command_line": "??\\C:\\WINDOWS\\system32\\conhost.exe 0xffffffff -ForceV1", + "process_integrity_level": "Medium", + "process_parent_guid": "A98268C1-9C2E-5ACD-0000-00100266AB00}", + "process_parent_id": "240", + "process_parent_name": "cmd.exe", + "process_parent_path": "C:\\Windows\\System32\\cmd.exe", + "process_parent_command_line": "C:\\WINDOWS\\system32\\cmd.exe", } + + nodes = transformer.transform(test_event) + + assert len(nodes) == 4 + + proc: Process = nodes[0] + proc_file: File = nodes[1] + parent_proc: Process = nodes[2] + parent_proc_file: File = nodes[3] + + assert {"timestamp": "1"} in parent_proc.launched[proc] + assert proc in proc_file.file_of + assert parent_proc in parent_proc_file.file_of + + assert proc.process_id == "4756" + assert proc.process_image == "conhost.exe" + assert proc.process_image_path == "C:\\Windows\\System32" + assert proc.command_line == "??\\C:\\WINDOWS\\system32\\conhost.exe 0xffffffff -ForceV1" + + assert parent_proc.process_id == "240" + assert parent_proc.process_image == "cmd.exe" + assert parent_proc.process_image_path == "C:\\Windows\\System32" + assert parent_proc.command_line == "C:\\WINDOWS\\system32\\cmd.exe" From 183f68b6f065443637c6fc2ae0166e8fdbbe4810 Mon Sep 17 00:00:00 2001 From: yampelo Date: Sun, 2 Jun 2019 22:10:30 -0400 Subject: [PATCH 3/3] Adds file events --- beagle/transformers/ossem_transformer.py | 33 +++++++++++++++-- tests/transformers/test_ossem_transformer.py | 38 +++++++++++++++++++- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/beagle/transformers/ossem_transformer.py b/beagle/transformers/ossem_transformer.py index e0635528..5f77e97d 100644 --- a/beagle/transformers/ossem_transformer.py +++ b/beagle/transformers/ossem_transformer.py @@ -6,9 +6,12 @@ class OSSEMTransformer(Transformer): - """Transformer based on the formats defined here: + """Transformer based on the fields defined here: https://github.com/Cyb3rWard0g/OSSEM/tree/master/common_information_model + And the relationships defined here: + https://docs.google.com/spreadsheets/d/1ow7YRDEDJs67kcKMZZ66_5z1ipJry9QrsDQkjQvizJM/edit#gid=0 + Parameters ---------- Transformer : [type] @@ -27,8 +30,10 @@ def transform(self, event: dict) -> Optional[Tuple]: relationship = event["event_type"] - if relationship == "created": + if relationship == "launched": return self.created(event) + elif relationship in ["loaded", "created", "modified", "downloaded"]: + return self.file_ops(event) return tuple() @@ -61,3 +66,27 @@ def created(self, event: dict) -> Tuple[Process, File, Process, File]: parent.launched[proc].append(timestamp=event["event_creation_time"]) return (proc, proc_file, parent, parent_file) + + def file_ops(self, event: dict) -> Tuple[Process, File, File]: + proc_name, proc_path = split_path(event["process_path"]) + + proc = Process( + process_id=event.get("process_id"), + process_image=proc_name, + process_image_path=proc_path, + command_line=event.get("process_command_line"), + ) + proc_file = proc.get_file_node() + + proc_file.file_of[proc] + + name, path = split_path(event["file_path"]) + + dest_file = File(file_name=name, file_path=path, extension=event.get("file_extension")) + + if event["event_type"] == "loaded": + proc.loaded[dest_file].append(timestamp=event["event_creation_time"]) + elif event["event_type"] in ["created", "modified", "downloaded"]: + proc.wrote[dest_file].append(timestamp=event["event_creation_time"]) + + return (proc, proc_file, dest_file) diff --git a/tests/transformers/test_ossem_transformer.py b/tests/transformers/test_ossem_transformer.py index b15a2b3f..3fbf45aa 100644 --- a/tests/transformers/test_ossem_transformer.py +++ b/tests/transformers/test_ossem_transformer.py @@ -16,7 +16,7 @@ def test_unknown_event(transformer): def test_make_process(transformer): test_event = { "event_creation_time": "1", - "event_type": "created", + "event_type": "launched", "process_id": "4756", "process_name": "conhost.exe", "process_path": "C:\\Windows\\System32\\conhost.exe", @@ -51,3 +51,39 @@ def test_make_process(transformer): assert parent_proc.process_image == "cmd.exe" assert parent_proc.process_image_path == "C:\\Windows\\System32" assert parent_proc.command_line == "C:\\WINDOWS\\system32\\cmd.exe" + + +@pytest.mark.parametrize( + "eventtype,attribute", + [("loaded", "loaded"), ("created", "wrote"), ("modified", "wrote"), ("downloaded", "wrote")], +) +def test_file_operations(transformer, eventtype, attribute): + test_event = { + "event_creation_time": "1", + "event_type": eventtype, + "process_id": "4756", + "process_name": "conhost.exe", + "process_path": "C:\\Windows\\System32\\conhost.exe", + "process_command_line": "??\\C:\\WINDOWS\\system32\\conhost.exe 0xffffffff -ForceV1", + "file_name": "cmd.exe", + "file_path": "C:\\Windows\\System32\\cmd.exe", + } + nodes = transformer.transform(test_event) + + assert len(nodes) == 3 + + proc: Process = nodes[0] + proc_file: File = nodes[1] + dest_file: File = nodes[2] + + assert proc in proc_file.file_of + + assert proc.process_id == "4756" + assert proc.process_image == "conhost.exe" + assert proc.process_image_path == "C:\\Windows\\System32" + assert proc.command_line == "??\\C:\\WINDOWS\\system32\\conhost.exe 0xffffffff -ForceV1" + + assert dest_file.file_name == "cmd.exe" + assert dest_file.file_path == "C:\\Windows\\System32" + + assert {"timestamp": "1"} in getattr(proc, attribute)[dest_file]