From c96a9f357b057be272ea2f1ff52e4dd7a52430c3 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 19 Nov 2023 22:35:27 +0000 Subject: [PATCH 1/7] Fix for windows.strings revmap offsets --- .../framework/plugins/windows/strings.py | 76 +++++++++++++------ 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index 0eaa65884b..d3ed2b9c2e 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -18,7 +18,7 @@ class Strings(interfaces.plugins.PluginInterface): """Reads output from the strings command and indicates which process(es) each string belongs to.""" - _version = (1, 2, 0) + _version = (2, 0, 0) _required_framework_version = (2, 0, 0) strings_pattern = re.compile(rb"^(?:\W*)([0-9]+)(?:\W*)(\w[\w\W]+)\n?") @@ -47,7 +47,13 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] def run(self): return renderers.TreeGrid( - [("String", str), ("Physical Address", format_hints.Hex), ("Result", str)], + [ + ("String", str), + ("Region", str), + ("PID", int), + ("Physical Address", format_hints.Hex), + ("Virtual Address", format_hints.Hex), + ], self._generator(), ) @@ -81,22 +87,45 @@ def _generator(self) -> Generator[Tuple, None, None]: last_prog: float = 0 line_count: float = 0 num_strings = len(string_list) - for offset, string in string_list: + for phys_offset, string in string_list: line_count += 1 - try: - revmap_list = [ - name + ":" + hex(offset) for (name, offset) in revmap[offset >> 12] - ] - except (IndexError, KeyError): - revmap_list = ["FREE MEMORY"] - yield ( - 0, - ( - str(string, "latin-1"), - format_hints.Hex(offset), - ", ".join(revmap_list), - ), + + # We should really take care of this in the revmap generator + mapping_entry = revmap.get( + phys_offset & 0xFFFFFFFFFFFFF000, [("In Unallocated Space", 0)] ) + for item in mapping_entry: + region, offset = item + if "Process" in region: + location = "Process" + pid = region.split(" ")[1] + elif "kernel" in region: + location = "Kernel" + pid = -1 + elif "In Unallocated Space" in region: + location = "Unallocated Space" + pid = -1 + else: + location = "Unknown" + pid = -1 + + # Get the full virtual address not just the page start + if offset == 0: + virtual_address = 0 + else: + virtual_address = (offset & ~0xFFF) | (phys_offset & 0xFFF) + + yield ( + 0, + ( + str(string, "latin-1"), + location, + int(pid), + format_hints.Hex(phys_offset), + format_hints.Hex(virtual_address), + ), + ) + prog = line_count / num_strings * 100 if round(prog, 1) > last_prog: last_prog = round(prog, 1) @@ -149,9 +178,9 @@ def generate_mapping( for mapval in layer.mapping(0x0, layer.maximum_address, ignore_errors=True): offset, _, mapped_offset, mapped_size, maplayer = mapval for val in range(mapped_offset, mapped_offset + mapped_size, 0x1000): - cur_set = reverse_map.get(val >> 12, set()) - cur_set.add(("kernel", offset)) - reverse_map[val >> 12] = cur_set + cur_set = reverse_map.get(val, set()) + cur_set.add(("kernel", val)) + reverse_map[offset] = cur_set if progress_callback: progress_callback( (offset * 100) / layer.maximum_address, @@ -185,11 +214,14 @@ def generate_mapping( for val in range( mapped_offset, mapped_offset + mapped_size, 0x1000 ): - cur_set = reverse_map.get(mapped_offset >> 12, set()) + cur_set = reverse_map.get(mapped_offset, set()) cur_set.add( - (f"Process {process.UniqueProcessId}", offset) + ( + f"Process {process.UniqueProcessId}", + mapped_offset, + ) ) - reverse_map[mapped_offset >> 12] = cur_set + reverse_map[offset] = cur_set # FIXME: make the progress for all processes, rather than per-process if progress_callback: progress_callback( From cc420b14fec4d07c065eb573071d7fb8fa5158b7 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Sun, 19 Nov 2023 23:32:28 +0000 Subject: [PATCH 2/7] Clean up newlines from the output in strings --- volatility3/framework/plugins/windows/strings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index d3ed2b9c2e..35d2cd03f8 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -118,7 +118,7 @@ def _generator(self) -> Generator[Tuple, None, None]: yield ( 0, ( - str(string, "latin-1"), + str(string.strip(), "latin-1"), location, int(pid), format_hints.Hex(phys_offset), From 0eaffb59f0c1855206070f41beb269333ffba5e0 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Mon, 20 Nov 2023 00:24:37 +0000 Subject: [PATCH 3/7] Revert layer.mask index --- volatility3/framework/plugins/windows/strings.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index 35d2cd03f8..ce51d95869 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -92,7 +92,7 @@ def _generator(self) -> Generator[Tuple, None, None]: # We should really take care of this in the revmap generator mapping_entry = revmap.get( - phys_offset & 0xFFFFFFFFFFFFF000, [("In Unallocated Space", 0)] + phys_offset >> 12, [("In Unallocated Space", 0)] ) for item in mapping_entry: region, offset = item @@ -178,9 +178,9 @@ def generate_mapping( for mapval in layer.mapping(0x0, layer.maximum_address, ignore_errors=True): offset, _, mapped_offset, mapped_size, maplayer = mapval for val in range(mapped_offset, mapped_offset + mapped_size, 0x1000): - cur_set = reverse_map.get(val, set()) + cur_set = reverse_map.get(val >> 12, set()) cur_set.add(("kernel", val)) - reverse_map[offset] = cur_set + reverse_map[offset >> 12] = cur_set if progress_callback: progress_callback( (offset * 100) / layer.maximum_address, @@ -214,14 +214,14 @@ def generate_mapping( for val in range( mapped_offset, mapped_offset + mapped_size, 0x1000 ): - cur_set = reverse_map.get(mapped_offset, set()) + cur_set = reverse_map.get(mapped_offset >> 12, set()) cur_set.add( ( f"Process {process.UniqueProcessId}", mapped_offset, ) ) - reverse_map[offset] = cur_set + reverse_map[offset >> 12] = cur_set # FIXME: make the progress for all processes, rather than per-process if progress_callback: progress_callback( From f035eb8e0d17df4974f555a2eddd2fd36cdb23b4 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Mon, 20 Nov 2023 01:03:09 +0000 Subject: [PATCH 4/7] revmapper returns a dict instead of tupple --- .../framework/plugins/windows/strings.py | 45 +++++++------------ 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index ce51d95869..47c448df9d 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -90,28 +90,16 @@ def _generator(self) -> Generator[Tuple, None, None]: for phys_offset, string in string_list: line_count += 1 - # We should really take care of this in the revmap generator mapping_entry = revmap.get( - phys_offset >> 12, [("In Unallocated Space", 0)] + phys_offset >> 12, [{"region": "Unallocated", "pid": -1, "offset": 0x00}] ) for item in mapping_entry: - region, offset = item - if "Process" in region: - location = "Process" - pid = region.split(" ")[1] - elif "kernel" in region: - location = "Kernel" - pid = -1 - elif "In Unallocated Space" in region: - location = "Unallocated Space" - pid = -1 - else: - location = "Unknown" - pid = -1 # Get the full virtual address not just the page start - if offset == 0: - virtual_address = 0 + # If the string is in unalloacted memory, we set the offset to 0x00 + offset = item.get('offset', 0x00) + if offset == 0x00: + virtual_address = 0x00 else: virtual_address = (offset & ~0xFFF) | (phys_offset & 0xFFF) @@ -119,8 +107,8 @@ def _generator(self) -> Generator[Tuple, None, None]: 0, ( str(string.strip(), "latin-1"), - location, - int(pid), + item.get("region", "Unallocated"), + item.get("pid", -1), format_hints.Hex(phys_offset), format_hints.Hex(virtual_address), ), @@ -172,14 +160,17 @@ def generate_mapping( filter = pslist.PsList.create_pid_filter(pid_list) layer = context.layers[layer_name] - reverse_map: Dict[int, Set[Tuple[str, int]]] = dict() + #reverse_map: Dict[int, Set[Tuple[str, int]]] = dict() + reverse_map: Dict[int, List[Dict[str, Union[str, int]]]] = dict() if isinstance(layer, intel.Intel): # We don't care about errors, we just wanted chunks that map correctly for mapval in layer.mapping(0x0, layer.maximum_address, ignore_errors=True): offset, _, mapped_offset, mapped_size, maplayer = mapval for val in range(mapped_offset, mapped_offset + mapped_size, 0x1000): - cur_set = reverse_map.get(val >> 12, set()) - cur_set.add(("kernel", val)) + + cur_set = reverse_map.get(val >> 12, list()) + cur_set.append({"region": "kernel", "pid": -1, "offset": val}) + reverse_map[offset >> 12] = cur_set if progress_callback: progress_callback( @@ -214,13 +205,9 @@ def generate_mapping( for val in range( mapped_offset, mapped_offset + mapped_size, 0x1000 ): - cur_set = reverse_map.get(mapped_offset >> 12, set()) - cur_set.add( - ( - f"Process {process.UniqueProcessId}", - mapped_offset, - ) - ) + cur_set = reverse_map.get(mapped_offset >> 12, list()) + cur_set.append({"region": "process", "pid": process.UniqueProcessId, "offset": mapped_offset}) + reverse_map[offset >> 12] = cur_set # FIXME: make the progress for all processes, rather than per-process if progress_callback: From 044bd8eea31a86679ac282fb4a186526f854cb8d Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Mon, 20 Nov 2023 01:11:34 +0000 Subject: [PATCH 5/7] Fix formatting with black --- .../framework/plugins/windows/strings.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index 47c448df9d..982d35a25a 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -91,13 +91,13 @@ def _generator(self) -> Generator[Tuple, None, None]: line_count += 1 mapping_entry = revmap.get( - phys_offset >> 12, [{"region": "Unallocated", "pid": -1, "offset": 0x00}] + phys_offset >> 12, + [{"region": "Unallocated", "pid": -1, "offset": 0x00}], ) for item in mapping_entry: - # Get the full virtual address not just the page start # If the string is in unalloacted memory, we set the offset to 0x00 - offset = item.get('offset', 0x00) + offset = item.get("offset", 0x00) if offset == 0x00: virtual_address = 0x00 else: @@ -160,14 +160,13 @@ def generate_mapping( filter = pslist.PsList.create_pid_filter(pid_list) layer = context.layers[layer_name] - #reverse_map: Dict[int, Set[Tuple[str, int]]] = dict() + # reverse_map: Dict[int, Set[Tuple[str, int]]] = dict() reverse_map: Dict[int, List[Dict[str, Union[str, int]]]] = dict() if isinstance(layer, intel.Intel): # We don't care about errors, we just wanted chunks that map correctly for mapval in layer.mapping(0x0, layer.maximum_address, ignore_errors=True): offset, _, mapped_offset, mapped_size, maplayer = mapval for val in range(mapped_offset, mapped_offset + mapped_size, 0x1000): - cur_set = reverse_map.get(val >> 12, list()) cur_set.append({"region": "kernel", "pid": -1, "offset": val}) @@ -206,7 +205,13 @@ def generate_mapping( mapped_offset, mapped_offset + mapped_size, 0x1000 ): cur_set = reverse_map.get(mapped_offset >> 12, list()) - cur_set.append({"region": "process", "pid": process.UniqueProcessId, "offset": mapped_offset}) + cur_set.append( + { + "region": "process", + "pid": process.UniqueProcessId, + "offset": mapped_offset, + } + ) reverse_map[offset >> 12] = cur_set # FIXME: make the progress for all processes, rather than per-process From a1b0f5178d8a79252d629c944e7e0ae7c9de850a Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Mon, 18 Dec 2023 10:33:00 +0000 Subject: [PATCH 6/7] Update RevMap structure and calculations --- .../framework/plugins/windows/strings.py | 88 ++++++++++++++----- 1 file changed, 65 insertions(+), 23 deletions(-) diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index 982d35a25a..862b494cd3 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -90,18 +90,26 @@ def _generator(self) -> Generator[Tuple, None, None]: for phys_offset, string in string_list: line_count += 1 + # calculate the offset for this string within a 4096 page so + # that this offset can be added to mappings which are all + # page aligned. This ensures that a string located at phy + # add 0x1e64cd20 would carry the 0xd20 to the virtual offsets + # displayed in the plugin output. Without this it would show + # only the page that the string was found, rather than the + # actually addr. 0xFFF is 4095 e.g. all lower bits set. + offset_within_page = phys_offset & 0xFFF + + mapping_entry = revmap.get( phys_offset >> 12, [{"region": "Unallocated", "pid": -1, "offset": 0x00}], ) + for item in mapping_entry: # Get the full virtual address not just the page start # If the string is in unalloacted memory, we set the offset to 0x00 offset = item.get("offset", 0x00) - if offset == 0x00: - virtual_address = 0x00 - else: - virtual_address = (offset & ~0xFFF) | (phys_offset & 0xFFF) + virtual_address = offset + offset_within_page yield ( 0, @@ -114,6 +122,7 @@ def _generator(self) -> Generator[Tuple, None, None]: ), ) + prog = line_count / num_strings * 100 if round(prog, 1) > last_prog: last_prog = round(prog, 1) @@ -160,22 +169,46 @@ def generate_mapping( filter = pslist.PsList.create_pid_filter(pid_list) layer = context.layers[layer_name] - # reverse_map: Dict[int, Set[Tuple[str, int]]] = dict() - reverse_map: Dict[int, List[Dict[str, Union[str, int]]]] = dict() + reverse_map: Dict[int, Set[Tuple[str, int]]] = dict() if isinstance(layer, intel.Intel): # We don't care about errors, we just wanted chunks that map correctly for mapval in layer.mapping(0x0, layer.maximum_address, ignore_errors=True): - offset, _, mapped_offset, mapped_size, maplayer = mapval - for val in range(mapped_offset, mapped_offset + mapped_size, 0x1000): - cur_set = reverse_map.get(val >> 12, list()) - cur_set.append({"region": "kernel", "pid": -1, "offset": val}) + ( + virt_offset, + _virt_size, + phy_offset, + phy_mapping_size, + _phy_layer_name, + ) = mapval + + # for each page within the mapping we need to store the phy_offset and + # the matching virt_offset + for offset_to_page_within_mapping in range(0, phy_mapping_size, 0x1000): + # calculate the page number for this phy_offset, e.g. the ">> 12" + # drops the bits that would address an offset within the page. + # This means that all offsets within the same page get the same + # physical_page number. + physical_page = ( + phy_mapping_size + offset_to_page_within_mapping + ) >> 12 - reverse_map[offset >> 12] = cur_set + # get the existing mappings for this physical page from the + # reverse map set. + cur_set = reverse_map.get(physical_page, list()) + + # add a mapping for this virtual offset, taking care to add the + # offset_to_page_within_mapping to ensure that all pages match correctly. + # Without this the 2nd, 3rd etc pages would all incorrectly map to the same + # virtual offset. + cur_set.append({"region": "Kernel", "pid": -1, "offset": virt_offset + offset_to_page_within_mapping}) + + # store these results back in the reverse_map + reverse_map[physical_page] = cur_set if progress_callback: progress_callback( - (offset * 100) / layer.maximum_address, - "Creating reverse kernel map", - ) + (virt_offset * 100) / layer.maximum_address, + "Creating reverse kernel map", + ) # TODO: Include kernel modules @@ -197,27 +230,36 @@ def generate_mapping( proc_layer = context.layers[proc_layer_name] if isinstance(proc_layer, linear.LinearlyMappedLayer): + # this follows the same pattern as the kernel mappings above. for mapval in proc_layer.mapping( 0x0, proc_layer.maximum_address, ignore_errors=True ): - mapped_offset, _, offset, mapped_size, maplayer = mapval - for val in range( - mapped_offset, mapped_offset + mapped_size, 0x1000 + ( + virt_offset, + _virt_size, + phy_offset, + phy_mapping_size, + _phy_layer_name, + ) = mapval + for offset_to_page_within_mapping in range( + 0, phy_mapping_size, 0x1000 ): - cur_set = reverse_map.get(mapped_offset >> 12, list()) + physical_page = ( + phy_offset + offset_to_page_within_mapping + ) >> 12 + cur_set = reverse_map.get(physical_page, list()) cur_set.append( { - "region": "process", + "region": "Process", "pid": process.UniqueProcessId, - "offset": mapped_offset, + "offset": virt_offset + offset_to_page_within_mapping, } ) - - reverse_map[offset >> 12] = cur_set + reverse_map[physical_page] = cur_set # FIXME: make the progress for all processes, rather than per-process if progress_callback: progress_callback( - (offset * 100) / layer.maximum_address, + (virt_offset * 100) / proc_layer.maximum_address, f"Creating mapping for task {process.UniqueProcessId}", ) From d2992cf2bc7902e4c29be1b87f768438c35fe953 Mon Sep 17 00:00:00 2001 From: KevTheHermit Date: Mon, 18 Dec 2023 10:39:18 +0000 Subject: [PATCH 7/7] Apply Formatting with Black to Strings --- .../framework/plugins/windows/strings.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index 862b494cd3..122c9a44a9 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -99,7 +99,6 @@ def _generator(self) -> Generator[Tuple, None, None]: # actually addr. 0xFFF is 4095 e.g. all lower bits set. offset_within_page = phys_offset & 0xFFF - mapping_entry = revmap.get( phys_offset >> 12, [{"region": "Unallocated", "pid": -1, "offset": 0x00}], @@ -122,7 +121,6 @@ def _generator(self) -> Generator[Tuple, None, None]: ), ) - prog = line_count / num_strings * 100 if round(prog, 1) > last_prog: last_prog = round(prog, 1) @@ -200,15 +198,21 @@ def generate_mapping( # offset_to_page_within_mapping to ensure that all pages match correctly. # Without this the 2nd, 3rd etc pages would all incorrectly map to the same # virtual offset. - cur_set.append({"region": "Kernel", "pid": -1, "offset": virt_offset + offset_to_page_within_mapping}) + cur_set.append( + { + "region": "Kernel", + "pid": -1, + "offset": virt_offset + offset_to_page_within_mapping, + } + ) # store these results back in the reverse_map reverse_map[physical_page] = cur_set if progress_callback: progress_callback( (virt_offset * 100) / layer.maximum_address, - "Creating reverse kernel map", - ) + "Creating reverse kernel map", + ) # TODO: Include kernel modules @@ -252,7 +256,8 @@ def generate_mapping( { "region": "Process", "pid": process.UniqueProcessId, - "offset": virt_offset + offset_to_page_within_mapping, + "offset": virt_offset + + offset_to_page_within_mapping, } ) reverse_map[physical_page] = cur_set