From e5f40714780f83091a310848bd6d46e74c9877a0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 21 Jan 2024 00:06:50 +0100 Subject: [PATCH 01/86] arm layer integrating aarch64 support --- volatility3/framework/layers/arm.py | 461 ++++++++++++++++++++++++++++ 1 file changed, 461 insertions(+) create mode 100644 volatility3/framework/layers/arm.py diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py new file mode 100644 index 0000000000..e6c8300774 --- /dev/null +++ b/volatility3/framework/layers/arm.py @@ -0,0 +1,461 @@ +import logging +import functools +import collections +from typing import Optional, Dict, Any, List, Iterable, Tuple + +from volatility3 import classproperty +from volatility3.framework import interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import linear + +vollog = logging.getLogger(__name__) +AARCH64_TRANSLATION_DEBUGGING = False +AARCH64_DEBUGGING = False + +""" +Webography : + [1] Arm, "Arm Architecture Reference Manual for A-profile architecture, DDI 0487J.a (ID042523)", https://developer.arm.com/documentation/ddi0487/ja/?lang=en + [2] Linux, Linux Kernel source code, v6.7 + +Glossary : + TTB : Translation Table Base + TCR : Translation Control Register + EL : Exception Level (0:Application,1:Kernel,2:Hypervisor,3:Secure Monitor) + Granule : Translation granule (smallest block of memory that can be described) + """ + + +class AArch64Exception(exceptions.LayerException): + pass + + +class AArch64(linear.LinearlyMappedLayer): + """Translation Layer for the Arm AArch64 memory mapping.""" + + _direct_metadata = collections.ChainMap( + {"architecture": "AArch64"}, + {"mapped": True}, + interfaces.layers.TranslationLayerInterface._direct_metadata, + ) + + _bits_per_register = 64 + _register_size = _bits_per_register // 8 + # NOTE: _maxphyaddr is as defined in the AArch64 specs *NOT* the maximum physical address + # _maxvirtaddr actually depends on the context + # We need the full 64 bits masking on addresses to determine their TTB at bit 55 + _maxphyaddr = 64 + _maxvirtaddr = _maxphyaddr + + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + name: str, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: + super().__init__( + context=context, config_path=config_path, name=name, metadata=metadata + ) + self._base_layer = self.config["memory_layer"] + # self._swap_layers = [] # TODO + self._page_map_offset = self.config["page_map_offset"] + self._tcr_el1_tnsz = self.config["tcr_el1_tnsz"] + self._page_size = self.config["page_size"] + + # Context : TTB0 (user) or TTB1 (kernel) + self._virtual_addr_space = ( + self._page_map_offset == self.config["page_map_offset_kernel"] + ) + self._ttbs_tnsz = [self._tcr_el1_tnsz, self._tcr_el1_tnsz] + + # [1], see D8.1.9, page 5818 + self._ttbs_bitsizes = [64 - self._ttbs_tnsz[0], 64 - self._ttbs_tnsz[1]] + self._ttbs_granules = [self._page_size, self._page_size] + + self._is_52bits = [ + True if self._ttbs_tnsz[ttb] < 16 else False for ttb in range(2) + ] + + # [1], see D8.2.7 to D8.2.9, starting at page 5828 + self._granules_indexes = { + 4: [(51, 48), (47, 39), (38, 30), (29, 21), (20, 12)], + 16: [(51, 47), (46, 36), (35, 25), (24, 14)], + 64: [(51, 42), (41, 29), (28, 16)], + } + self._ttb_lookups_descriptors = self._determine_ttbs_lookup_descriptors() + + # [1], see D8.3, page 5852 + self._descriptors_bits = [ + ( + 49 + if self._ttbs_granules[ttb] in [4, 16] and self._is_52bits[ttb] + else 47, + self._ttb_lookups_descriptors[ttb][-1][1], + ) + for ttb in range(2) + ] + + self._virtual_addr_range = self._get_virtual_addr_ranges()[ + self._virtual_addr_space + ] + + self._context_maxvirtaddr = self._ttbs_bitsizes[self._virtual_addr_space] + self._canonical_prefix = self._mask( + (1 << self._bits_per_register) - 1, + self._bits_per_register, + self._context_maxvirtaddr, + ) + + if AARCH64_DEBUGGING: + vollog.debug(f"Base layer : {self._base_layer}") + vollog.debug( + f"Virtual address space : {'kernel' if self._virtual_addr_space else 'user'}" + ) + vollog.debug( + f"Virtual addresses spaces ranges : {[tuple([hex(y) for y in x]) for x in self._get_virtual_addr_ranges()]}" + ) + vollog.debug(f"Pages sizes : {self._ttbs_granules}") + vollog.debug(f"TnSZ values : {self._ttbs_bitsizes}") + vollog.debug(f"Page map offset : {hex(self._page_map_offset)}") + vollog.debug(f"Descriptors mappings : {self._ttb_lookups_descriptors}") + + def _determine_ttbs_lookup_descriptors(self) -> List[int]: + """Returns the bits to extract from a translation address (highs and lows)""" + ttb_lookups_descriptors = [] + + for ttb, ttb_granule in enumerate(self._ttbs_granules): + va_bit_size = self._ttbs_bitsizes[ttb] + indexes = [ + index + for index in self._granules_indexes[ttb_granule] + if va_bit_size > index[1] + ] + indexes[0] = (va_bit_size - 1, indexes[0][1]) + ttb_lookups_descriptors.append(indexes) + return ttb_lookups_descriptors + + def _translate(self, virtual_offset: int) -> Tuple[int, int, str]: + """Translates a specific offset based on paging tables. + + Returns the translated offset, the contiguous pagesize that the + translated address lives in and the layer_name that the address + lives in + """ + table_address, position, _ = self._translate_entry(virtual_offset) + offset_within_page = self._mask(virtual_offset, position - 1, 0) + physical_offset = table_address + offset_within_page + + return physical_offset, 2**position, self._base_layer + + def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: + """Translates a virtual offset to a physical one within this segment + Returns the translated address, the maximum offset within the block and the page + the translated address lives in + """ + base_layer = self.context.layers[self._base_layer] + + # [1], see D8.2.4, page 5824 + ttb_selector = self._mask(virtual_offset, 55, 55) + + # Check if requested address belongs to the context virtual memory space + if ttb_selector != self._virtual_addr_space: + raise exceptions.InvalidAddressException( + layer_name=self.name, + invalid_address=virtual_offset, + ) + + lookup_descriptor = self._ttb_lookups_descriptors[ttb_selector] + + table_address = self._page_map_offset + level = 0 + max_level = len(lookup_descriptor) + + for high_bit, low_bit in lookup_descriptor: + index = self._mask(virtual_offset, high_bit, low_bit) + + # TODO: Adapt endianness ? + descriptor = int.from_bytes( + base_layer.read( + table_address + (index * self._register_size), self._register_size + ), + byteorder="little", + ) + + # [1], see D8.3, page 5852 + descriptor_type = self._mask(descriptor, 1, 0) + # Table descriptor + if level < max_level and descriptor_type == 0b11: + table_address = ( + self._mask( + descriptor, + self._descriptors_bits[ttb_selector][0], + self._descriptors_bits[ttb_selector][1], + ) + << self._descriptors_bits[ttb_selector][1] + ) + # Block descriptor + elif level < max_level and descriptor_type == 0b01: + table_address = ( + self._mask( + descriptor, + self._descriptors_bits[ttb_selector][0], + low_bit, + ) + << low_bit + ) + break + # Page descriptor + elif level == max_level and descriptor_type == 0b11: + break + # Invalid descriptor || Reserved descriptor (level 3) + else: + raise exceptions.PagedInvalidAddressException( + layer_name=self.name, + invalid_address=virtual_offset, + invalid_bits=low_bit, + entry=descriptor, + ) + + # [1], see D8.3, page 5852 + if self._is_52bits[ttb_selector]: + if self._ttbs_granules[ttb_selector] in [4, 16]: + ta_51_x_bits = (9, 8) + elif self._ttbs_granules[ttb_selector] == 64: + ta_51_x_bits = (15, 12) + + ta_51_x = self._mask( + descriptor, + ta_51_x_bits[0], + ta_51_x_bits[1], + ) + ta_51_x = ta_51_x << (52 - ta_51_x.bit_length()) + table_address = ta_51_x | table_address + + level += 1 + + if AARCH64_TRANSLATION_DEBUGGING: + vollog.debug( + f"Virtual {hex(virtual_offset)} lives in page frame {hex(table_address)} at offset {hex(self._mask(virtual_offset, low_bit-1, 0))}", + ) + + return table_address, low_bit, descriptor + + def mapping( + self, offset: int, length: int, ignore_errors: bool = False + ) -> Iterable[Tuple[int, int, int, int, str]]: + """Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer) + mappings. + + This allows translation layers to provide maps of contiguous + regions in one layer + """ + stashed_offset = ( + stashed_mapped_offset + ) = stashed_size = stashed_mapped_size = stashed_map_layer = None + for offset, size, mapped_offset, mapped_size, map_layer in self._mapping( + offset, length, ignore_errors + ): + if ( + stashed_offset is None + or (stashed_offset + stashed_size != offset) + or (stashed_mapped_offset + stashed_mapped_size != mapped_offset) + or (stashed_map_layer != map_layer) + ): + # The block isn't contiguous + if stashed_offset is not None: + yield stashed_offset, stashed_size, stashed_mapped_offset, stashed_mapped_size, stashed_map_layer + # Update all the stashed values after output + stashed_offset = offset + stashed_mapped_offset = mapped_offset + stashed_size = size + stashed_mapped_size = mapped_size + stashed_map_layer = map_layer + else: + # Part of an existing block + stashed_size += size + stashed_mapped_size += mapped_size + # Yield whatever's left + if ( + stashed_offset is not None + and stashed_mapped_offset is not None + and stashed_size is not None + and stashed_mapped_size is not None + and stashed_map_layer is not None + ): + yield stashed_offset, stashed_size, stashed_mapped_offset, stashed_mapped_size, stashed_map_layer + + def _mapping( + self, offset: int, length: int, ignore_errors: bool = False + ) -> Iterable[Tuple[int, int, int, int, str]]: + """Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer) + mappings. + + This allows translation layers to provide maps of contiguous + regions in one layer + """ + if length == 0: + try: + mapped_offset, _, layer_name = self._translate(offset) + if not self._context.layers[layer_name].is_valid(mapped_offset): + raise exceptions.InvalidAddressException( + layer_name=layer_name, invalid_address=mapped_offset + ) + except exceptions.InvalidAddressException: + if not ignore_errors: + raise + return None + yield offset, length, mapped_offset, length, layer_name + return None + while length > 0: + """ + A bit of lexical definition : "page" means "virtual page" (i.e. a chunk of virtual address space) and "page frame" means "physical page" (i.e. a chunk of physical memory). + + What this is actually doing : + - translate a virtual offset to a physical offset + - determine the page size the virtual offset lives in (page_size) + - based on the position of the offset in the page, calculate how many bytes to add to get to the end of the page (chunk_size) + - add the chunk_size to the virtual offset, so that we can point to the start of the next page frame + + Example (assume page size is 4096): + -> 0xffff800000f92140 lives in page 0xfffffc0000170640 at offset 0x140, which maps to page frame 0x45c19000 at offset 0x140 + -> 4096 - 0x140 = 3776 + -> 0xffff800000f92140 + 3776 = 0xffff800000f93000 + -> we know the start of the next page is at virtual offset 0xffff800000f93000, so we can directly jump to it (no need to translate every byte in between) + -> 0xffff800000f93000 lives in page 0xfffffc0000087040 at offset 0x0, which maps to page frame 0x421c1000 at offset 0x0 + -> 4096 - 0x0 = 4096 + -> 0xffff800000f93000 + 4096 = 0xffff800000f94000 + etc. while "length" > 0 + """ + try: + chunk_offset, page_size, layer_name = self._translate(offset) + chunk_size = min(page_size - (chunk_offset % page_size), length) + if not self._context.layers[layer_name].is_valid( + chunk_offset, chunk_size + ): + raise exceptions.InvalidAddressException( + layer_name=layer_name, invalid_address=chunk_offset + ) + except ( + exceptions.PagedInvalidAddressException, + exceptions.InvalidAddressException, + ) as excp: + """ + Contiguous pages might not be mapped, but if we specifically ignore errors, we still want to read the n + 1 page as it might be mapped. + """ + if not ignore_errors: + raise + # We can jump more if we know where the page fault failed + if isinstance(excp, exceptions.PagedInvalidAddressException): + mask = (1 << excp.invalid_bits) - 1 + else: + mask = ( + 1 + << ( + self._ttbs_granules[self._virtual_addr_space].bit_length() + - 1 + ) + ) - 1 + length_diff = mask + 1 - (offset & mask) + length -= length_diff + offset += length_diff + else: + yield offset, chunk_size, chunk_offset, chunk_size, layer_name + length -= chunk_size + offset += chunk_size + + def _get_virtual_addr_ranges( + self, + ) -> Tuple[list[int, int, int], list[int, int, int]]: + """Returns the virtual address space ranges as [(LOW_START, LOW_END), (HIGH_START, HIGH_END)]""" + # [2], see source/arch/arm64/include/asm/memory.h#L62 + ttb0_start = 0 + ttb0_size = 1 << (self._ttbs_bitsizes[0] - 1) + ttb0_end = ttb0_start + (ttb0_size - 1) + ttb1_end = 2**64 - 1 + ttb1_size = 1 << (self._ttbs_bitsizes[1] - 1) + ttb1_start = ttb1_end - (ttb1_size - 1) + + return [(ttb0_start, ttb0_end), (ttb1_start, ttb1_end)] + + def is_valid(self, offset: int, length: int = 1) -> bool: + """Returns whether the address offset can be translated to a valid + address.""" + try: + # TODO: Consider reimplementing this, since calls to mapping can call is_valid + return all( + [ + self._context.layers[layer].is_valid(mapped_offset) + for _, _, mapped_offset, _, layer in self.mapping(offset, length) + ] + ) + except exceptions.InvalidAddressException: + return False + + def is_dirty(self, offset: int) -> bool: + """Returns whether the page at offset is marked dirty""" + return self._page_is_dirty(self._translate_entry(offset)[2]) + + @staticmethod + def _page_is_dirty(entry: int) -> bool: + """Returns whether a particular page is dirty based on its entry.""" + # [2], see arch/arm64/include/asm/pgtable-prot.h#L18 + return bool(entry & (1 << 55)) + + @classproperty + @functools.lru_cache() + def minimum_address(cls) -> int: + return 0 + + @classproperty + @functools.lru_cache() + def maximum_address(cls) -> int: + return (1 << cls._maxvirtaddr) - 1 + + def __canonicalize(self, addr: int) -> int: + """Canonicalizes an address by performing an appropiate sign extension on the higher addresses""" + if self._bits_per_register <= self._context_maxvirtaddr: + return addr & self.address_mask + elif addr < (1 << self._context_maxvirtaddr - 1): + return addr + return self._mask(addr, self._context_maxvirtaddr, 0) + self._canonical_prefix + + def __decanonicalize(self, addr: int) -> int: + """Removes canonicalization to ensure an adress fits within the correct range if it has been canonicalized + + This will produce an address outside the range if the canonicalization is incorrect + """ + if addr < (1 << self._context_maxvirtaddr - 1): + return addr + return addr ^ self._canonical_prefix + + @staticmethod + def _mask( + value: int, high_bit: int, low_bit: int, shift: Optional[int] = None + ) -> int: + """Returns the bits of a value between highbit and lowbit inclusive.""" + if shift is None: + shift = low_bit + high_mask = (1 << (high_bit + 1)) - 1 + low_mask = (1 << low_bit) - 1 + mask = high_mask ^ low_mask + return (value & mask) >> shift + + @property + def dependencies(self) -> List[str]: + """Returns a list of the lower layer names that this layer is dependent + upon.""" + return [self._base_layer] + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.TranslationLayerRequirement( + name="memory_layer", optional=False + ), + requirements.IntRequirement(name="page_map_offset", optional=False), + requirements.IntRequirement(name="page_map_offset_kernel", optional=False), + requirements.IntRequirement(name="tcr_el1_tnsz", optional=False), + requirements.IntRequirement(name="page_size", optional=False), + requirements.IntRequirement(name="kernel_virtual_offset", optional=True), + requirements.StringRequirement(name="kernel_banner", optional=True), + ] From 1fd3708c014483f79cd2f9fbc4e1d93e6873e29e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 21 Jan 2024 00:09:11 +0100 Subject: [PATCH 02/86] adapt linux_stacker for aarch64 support --- volatility3/framework/automagic/linux.py | 253 ++++++++++++++++++----- 1 file changed, 207 insertions(+), 46 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2eebcc2dcd..334b92d3ab 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -6,18 +6,19 @@ import os from typing import Optional, Tuple, Type -from volatility3.framework import constants, interfaces +from volatility3.framework import constants, interfaces, exceptions from volatility3.framework.automagic import symbol_cache, symbol_finder from volatility3.framework.configuration import requirements -from volatility3.framework.layers import intel, scanners +from volatility3.framework.layers import intel, scanners, arm from volatility3.framework.symbols import linux vollog = logging.getLogger(__name__) -class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): +class LinuxStacker(interfaces.automagic.StackerLayerInterface): stack_order = 35 exclusion_list = ["mac", "windows"] + join = interfaces.configuration.path_join @classmethod def stack( @@ -39,11 +40,10 @@ def stack( # Bail out by default unless we can stack properly layer = context.layers[layer_name] - join = interfaces.configuration.path_join - # Never stack on top of an intel layer + # Never stack on top of a linux layer # FIXME: Find a way to improve this check - if isinstance(layer, intel.Intel): + if isinstance(layer, intel.Intel) or isinstance(layer, arm.AArch64): return None identifiers_path = os.path.join( @@ -63,12 +63,11 @@ def stack( for _, banner in layer.scan( context=context, scanner=mss, progress_callback=progress_callback ): - dtb = None vollog.debug(f"Identified banner: {repr(banner)}") isf_path = linux_banners.get(banner, None) if isf_path: - table_name = context.symbol_space.free_table_name("LintelStacker") + table_name = context.symbol_space.free_table_name("LinuxStacker") table = linux.LinuxKernelIntermedSymbols( context, "temporary." + table_name, @@ -76,53 +75,212 @@ def stack( isf_url=isf_path, ) context.symbol_space.append(table) - kaslr_shift, aslr_shift = cls.find_aslr( - context, table_name, layer_name, progress_callback=progress_callback - ) - - layer_class: Type = intel.Intel - if "init_top_pgt" in table.symbols: - layer_class = intel.Intel32e - dtb_symbol_name = "init_top_pgt" - elif "init_level4_pgt" in table.symbols: - layer_class = intel.Intel32e - dtb_symbol_name = "init_level4_pgt" - else: - dtb_symbol_name = "swapper_pg_dir" - - dtb = cls.virtual_to_physical_address( - table.get_symbol(dtb_symbol_name).address + kaslr_shift - ) - - # Build the new layer - new_layer_name = context.layers.free_layer_name("IntelLayer") - config_path = join("IntelHelper", new_layer_name) - context.config[join(config_path, "memory_layer")] = layer_name - context.config[join(config_path, "page_map_offset")] = dtb + new_layer_name = context.layers.free_layer_name("LinuxLayer") + config_path = cls.join("LinuxHelper", new_layer_name) + context.config[cls.join(config_path, "memory_layer")] = layer_name context.config[ - join(config_path, LinuxSymbolFinder.banner_config_key) + cls.join(config_path, LinuxSymbolFinder.banner_config_key) ] = str(banner, "latin-1") - layer = layer_class( - context, - config_path=config_path, - name=new_layer_name, - metadata={"os": "Linux"}, - ) - layer.config["kernel_virtual_offset"] = aslr_shift + linux_arch_stackers = [cls.intel_stacker, cls.aarch64_stacker] + + for linux_arch_stacker in linux_arch_stackers: + try: + layer = linux_arch_stacker( + context=context, + layer_name=layer_name, + table=table, + table_name=table_name, + config_path=config_path, + new_layer_name=new_layer_name, + banner=banner, + progress_callback=progress_callback, + ) + if layer: + return layer + except Exception as e: + vollog.log( + constants.LOGLEVEL_VVVV, + f"{linux_arch_stacker.__name__} exception: {e}", + ) - if layer and dtb: - vollog.debug(f"DTB was found at: 0x{dtb:0x}") - return layer vollog.debug("No suitable linux banner could be matched") return None + @classmethod + def intel_stacker( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + table: linux.LinuxKernelIntermedSymbols, + table_name: str, + config_path: str, + new_layer_name: str, + banner: str, + progress_callback: constants.ProgressCallback = None, + ): + layer_class: Type = intel.Intel + if "init_top_pgt" in table.symbols: + layer_class = intel.Intel32e + dtb_symbol_name = "init_top_pgt" + elif "init_level4_pgt" in table.symbols: + layer_class = intel.Intel32e + dtb_symbol_name = "init_level4_pgt" + else: + dtb_symbol_name = "swapper_pg_dir" + + kaslr_shift, aslr_shift = cls.find_aslr( + context, + table_name, + layer_name, + layer_class, + progress_callback=progress_callback, + ) + + dtb = cls.virtual_to_physical_address( + table.get_symbol(dtb_symbol_name).address + kaslr_shift + ) + + # Build the new layer + context.config[cls.join(config_path, "page_map_offset")] = dtb + + layer = layer_class( + context, + config_path=config_path, + name=new_layer_name, + metadata={"os": "Linux"}, + ) + layer.config["kernel_virtual_offset"] = aslr_shift + linux_banner_address = table.get_symbol("linux_banner").address + aslr_shift + test_banner_equality = cls.verify_translation_by_banner( + context=context, + layer=layer, + layer_name=layer_name, + linux_banner_address=linux_banner_address, + target_banner=banner, + ) + + if layer and dtb and test_banner_equality: + vollog.debug(f"DTB was found at: 0x{dtb:0x}") + vollog.debug("Intel image found") + return layer + + @classmethod + def aarch64_stacker( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + table: linux.LinuxKernelIntermedSymbols, + table_name: str, + config_path: str, + new_layer_name: str, + banner: bytes, + progress_callback: constants.ProgressCallback = None, + ): + layer_class = arm.AArch64 + kaslr_shift, aslr_shift = cls.find_aslr( + context, + table_name, + layer_name, + layer_class, + progress_callback=progress_callback, + ) + dtb = table.get_symbol("swapper_pg_dir").address + kaslr_shift + + # CREDIT : https://github.com/crash-utility/crash/blob/28891d1127542dbb2d5ba16c575e14e741ed73ef/arm64.c#L941 + kernel_flags = 0 + if "_kernel_flags_le" in table.symbols: + kernel_flags = table.get_symbol("_kernel_flags_le").address + if "_kernel_flags_le_hi32" in table.symbols: + kernel_flags |= table.get_symbol("_kernel_flags_le_hi32").address << 32 + if "_kernel_flags_le_lo32" in table.symbols: + kernel_flags |= table.get_symbol("_kernel_flags_le_lo32").address + + # TODO: There is no simple way to determine TTB0 page size for now, so assume TTB0 and TTB1 have the same (which is most likely). + # https://www.kernel.org/doc/Documentation/arm64/booting.txt + page_size_bit = (kernel_flags >> 1) & 3 + linux_banner_address = table.get_symbol("linux_banner").address + aslr_shift + + # v6.7/source/arch/arm64/include/asm/memory.h#L186 - v5.7/source/arch/arm64/include/asm/memory.h#L160 + if "vabits_actual" in table.symbols: + vabits_actual_phys_addr = ( + table.get_symbol("vabits_actual").address + kaslr_shift + ) + va_bits = int.from_bytes( + context.layers[layer_name].read(vabits_actual_phys_addr, 8), + "little", + ) + else: + # TODO: If KASAN space is large enough, it *might* push kernel addresses higher and generate inaccurate results ? + # TODO: There is no simple way to determine T0SZ for now, so assume T1SZ and T0SZ are the same (which is most likely). + # We count the number of high bits equal to 1, which gives us the kernel space address mask and ultimately TCR_EL1.T1SZ. + va_bits = (linux_banner_address ^ (2**64 - 1)).bit_length() + 1 + + if page_size_bit != 0 and va_bits != None: + page_size_bit_map = {1: 4, 2: 16, 3: 64} + page_size = page_size_bit_map[page_size_bit] + tcr_el1_tnsz = 64 - va_bits + + context.config[cls.join(config_path, "page_map_offset")] = dtb + context.config[cls.join(config_path, "page_map_offset_kernel")] = dtb + context.config[cls.join(config_path, "tcr_el1_tnsz")] = tcr_el1_tnsz + context.config[cls.join(config_path, "page_size")] = page_size + + layer = layer_class( + context, + config_path=config_path, + name=new_layer_name, + metadata={"os": "Linux"}, + ) + layer.config["kernel_virtual_offset"] = aslr_shift + test_banner_equality = cls.verify_translation_by_banner( + context=context, + layer=layer, + layer_name=layer_name, + linux_banner_address=linux_banner_address, + target_banner=banner, + ) + if layer and dtb and test_banner_equality: + vollog.debug(f"Kernel DTB was found at: 0x{dtb:0x}") + vollog.debug("AArch64 image found") + return layer + + @classmethod + def verify_translation_by_banner( + cls, + context: interfaces.context.ContextInterface, + layer, + layer_name: str, + linux_banner_address: int, + target_banner: bytes, + ): + """Determine if a stacked layer is correct or a false positive, by callind the underlying + _translate method against the linux_banner symbol virtual address. Then, compare it with + the detected banner to verify the correct translation.""" + + test_banner_equality = True + try: + banner_phys_address = layer._translate(linux_banner_address)[0] + banner_value = context.layers[layer_name].read( + banner_phys_address, len(target_banner) + ) + except exceptions.InvalidAddressException: + raise Exception('Cannot translate "linux_banner" virtual address') + + if not banner_value == target_banner: + raise Exception( + f'Translated "linux_banner" virtual address mismatches detected banner : \n{banner_value}\n!=\n{target_banner}' + ) + + return test_banner_equality + @classmethod def find_aslr( cls, context: interfaces.context.ContextInterface, symbol_table: str, layer_name: str, + layer_class, progress_callback: constants.ProgressCallback = None, ) -> Tuple[int, int]: """Determines the offset of the actual DTB in physical space and its @@ -162,9 +320,12 @@ def find_aslr( init_task.files.cast("long unsigned int") - module.get_symbol("init_files").address ) - kaslr_shift = init_task_address - cls.virtual_to_physical_address( - init_task_json_address - ) + if layer_class == arm.AArch64: + kaslr_shift = init_task_address - init_task_json_address + else: + kaslr_shift = init_task_address - cls.virtual_to_physical_address( + init_task_json_address + ) if address_mask: aslr_shift = aslr_shift & address_mask @@ -196,5 +357,5 @@ class LinuxSymbolFinder(symbol_finder.SymbolFinder): banner_config_key = "kernel_banner" operating_system = "linux" symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" - find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] + find_aslr = lambda cls, *args: LinuxStacker.find_aslr(*args)[1] exclusion_list = ["mac", "windows"] From 490b593f8c13700f9749917aeeedf92da6f6694c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 21 Jan 2024 00:39:03 +0100 Subject: [PATCH 03/86] remove irrelevant va_bits comparison --- volatility3/framework/automagic/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 334b92d3ab..ec087adb9b 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -216,7 +216,7 @@ def aarch64_stacker( # We count the number of high bits equal to 1, which gives us the kernel space address mask and ultimately TCR_EL1.T1SZ. va_bits = (linux_banner_address ^ (2**64 - 1)).bit_length() + 1 - if page_size_bit != 0 and va_bits != None: + if page_size_bit != 0: page_size_bit_map = {1: 4, 2: 16, 3: 64} page_size = page_size_bit_map[page_size_bit] tcr_el1_tnsz = 64 - va_bits From c9e7a3bc4e18b843993f482128c0cf58c095b9ef Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 21 Jan 2024 13:14:26 +0100 Subject: [PATCH 04/86] add volatility license --- volatility3/framework/layers/arm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index e6c8300774..3a44c33e3f 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -1,3 +1,7 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + import logging import functools import collections From 584d5f1e60ca80135e4a4e54de85815673160c43 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 21 Jan 2024 13:14:48 +0100 Subject: [PATCH 05/86] correct return type hinting --- volatility3/framework/automagic/linux.py | 12 +++++++----- volatility3/framework/layers/arm.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index ec087adb9b..c900da4fe1 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -4,7 +4,7 @@ import logging import os -from typing import Optional, Tuple, Type +from typing import Optional, Tuple, Type, Union, Literal from volatility3.framework import constants, interfaces, exceptions from volatility3.framework.automagic import symbol_cache, symbol_finder @@ -118,7 +118,7 @@ def intel_stacker( new_layer_name: str, banner: str, progress_callback: constants.ProgressCallback = None, - ): + ) -> Union[intel.Intel, intel.Intel32e, None]: layer_class: Type = intel.Intel if "init_top_pgt" in table.symbols: layer_class = intel.Intel32e @@ -176,7 +176,7 @@ def aarch64_stacker( new_layer_name: str, banner: bytes, progress_callback: constants.ProgressCallback = None, - ): + ) -> Optional[arm.AArch64]: layer_class = arm.AArch64 kaslr_shift, aslr_shift = cls.find_aslr( context, @@ -253,10 +253,12 @@ def verify_translation_by_banner( layer_name: str, linux_banner_address: int, target_banner: bytes, - ): + ) -> Literal[True]: """Determine if a stacked layer is correct or a false positive, by callind the underlying _translate method against the linux_banner symbol virtual address. Then, compare it with - the detected banner to verify the correct translation.""" + the detected banner to verify the correct translation. + This will directly raise an exception, as any failed attempt indicates a wrong layer selection. + """ test_banner_equality = True try: diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 3a44c33e3f..e7d62f080c 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -369,7 +369,7 @@ def _mapping( def _get_virtual_addr_ranges( self, - ) -> Tuple[list[int, int, int], list[int, int, int]]: + ) -> List[Tuple[int]]: """Returns the virtual address space ranges as [(LOW_START, LOW_END), (HIGH_START, HIGH_END)]""" # [2], see source/arch/arm64/include/asm/memory.h#L62 ttb0_start = 0 From 156eb51822e6d896b565adbe90ed4aeab2c2849b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 21 Jan 2024 13:19:51 +0100 Subject: [PATCH 06/86] python3.7 compatibility on Literal type --- volatility3/framework/automagic/linux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index c900da4fe1..4902438ba8 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -4,7 +4,7 @@ import logging import os -from typing import Optional, Tuple, Type, Union, Literal +from typing import Optional, Tuple, Type, Union from volatility3.framework import constants, interfaces, exceptions from volatility3.framework.automagic import symbol_cache, symbol_finder @@ -253,7 +253,7 @@ def verify_translation_by_banner( layer_name: str, linux_banner_address: int, target_banner: bytes, - ) -> Literal[True]: + ) -> bool: """Determine if a stacked layer is correct or a false positive, by callind the underlying _translate method against the linux_banner symbol virtual address. Then, compare it with the detected banner to verify the correct translation. From f63a6f4bca156569c9b73b56029686a9b6e48ce1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 21 Jan 2024 13:27:29 +0100 Subject: [PATCH 07/86] add explicit return None --- volatility3/framework/automagic/linux.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 4902438ba8..c9e1372a25 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -165,6 +165,8 @@ def intel_stacker( vollog.debug("Intel image found") return layer + return None + @classmethod def aarch64_stacker( cls, @@ -245,6 +247,8 @@ def aarch64_stacker( vollog.debug("AArch64 image found") return layer + return None + @classmethod def verify_translation_by_banner( cls, From f54c2d13ec48971a44a0ad4ed36055e0de7f710e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 21 Jan 2024 14:07:40 +0100 Subject: [PATCH 08/86] correct descriptors choices and order --- volatility3/framework/layers/arm.py | 46 ++++++++++++++++++----------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index e7d62f080c..acf6c77d3d 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -136,6 +136,7 @@ def _determine_ttbs_lookup_descriptors(self) -> List[int]: ] indexes[0] = (va_bit_size - 1, indexes[0][1]) ttb_lookups_descriptors.append(indexes) + return ttb_lookups_descriptors def _translate(self, virtual_offset: int) -> Tuple[int, int, str]: @@ -169,10 +170,9 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: ) lookup_descriptor = self._ttb_lookups_descriptors[ttb_selector] - table_address = self._page_map_offset level = 0 - max_level = len(lookup_descriptor) + max_level = len(lookup_descriptor) - 1 for high_bit, low_bit in lookup_descriptor: index = self._mask(virtual_offset, high_bit, low_bit) @@ -184,6 +184,21 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: ), byteorder="little", ) + ta_51_x = None + + # [1], see D8.3, page 5852 + if self._is_52bits[ttb_selector]: + if self._ttbs_granules[ttb_selector] in [4, 16]: + ta_51_x_bits = (9, 8) + elif self._ttbs_granules[ttb_selector] == 64: + ta_51_x_bits = (15, 12) + + ta_51_x = self._mask( + descriptor, + ta_51_x_bits[0], + ta_51_x_bits[1], + ) + ta_51_x = ta_51_x << (52 - ta_51_x.bit_length()) # [1], see D8.3, page 5852 descriptor_type = self._mask(descriptor, 1, 0) @@ -197,6 +212,7 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: ) << self._descriptors_bits[ttb_selector][1] ) + table_address = ta_51_x | table_address if ta_51_x else table_address # Block descriptor elif level < max_level and descriptor_type == 0b01: table_address = ( @@ -207,9 +223,19 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: ) << low_bit ) + table_address = ta_51_x | table_address if ta_51_x else table_address break # Page descriptor elif level == max_level and descriptor_type == 0b11: + table_address = ( + self._mask( + descriptor, + self._descriptors_bits[ttb_selector][0], + self._descriptors_bits[ttb_selector][1], + ) + << self._descriptors_bits[ttb_selector][1] + ) + table_address = ta_51_x | table_address if ta_51_x else table_address break # Invalid descriptor || Reserved descriptor (level 3) else: @@ -219,22 +245,6 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: invalid_bits=low_bit, entry=descriptor, ) - - # [1], see D8.3, page 5852 - if self._is_52bits[ttb_selector]: - if self._ttbs_granules[ttb_selector] in [4, 16]: - ta_51_x_bits = (9, 8) - elif self._ttbs_granules[ttb_selector] == 64: - ta_51_x_bits = (15, 12) - - ta_51_x = self._mask( - descriptor, - ta_51_x_bits[0], - ta_51_x_bits[1], - ) - ta_51_x = ta_51_x << (52 - ta_51_x.bit_length()) - table_address = ta_51_x | table_address - level += 1 if AARCH64_TRANSLATION_DEBUGGING: From 1833dbd84b9c0147579f99424c04f141b893e3f8 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 22 Jan 2024 13:39:31 +0100 Subject: [PATCH 09/86] debug requirements and descriptions --- volatility3/framework/layers/arm.py | 56 +++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index acf6c77d3d..34ce7629ba 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -466,10 +466,54 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] requirements.TranslationLayerRequirement( name="memory_layer", optional=False ), - requirements.IntRequirement(name="page_map_offset", optional=False), - requirements.IntRequirement(name="page_map_offset_kernel", optional=False), - requirements.IntRequirement(name="tcr_el1_tnsz", optional=False), - requirements.IntRequirement(name="page_size", optional=False), - requirements.IntRequirement(name="kernel_virtual_offset", optional=True), - requirements.StringRequirement(name="kernel_banner", optional=True), + requirements.IntRequirement( + name="page_map_offset", + optional=False, + description='DTB of the target context (either "kernel space" or "user space process").', + ), + requirements.IntRequirement( + name="page_map_offset_kernel", + optional=False, + description="DTB of the kernel space, it is primarily used to determine the target context of the layer (page_map_offset == page_map_offset_kernel). Conveniently calculated by LinuxStacker.", + ), + requirements.IntRequirement( + name="tcr_el1_t0sz", + optional=False, + description="The size offset of the memory region addressed by TTBR0_EL1. Conveniently calculated by LinuxStacker.", + ), + requirements.IntRequirement( + name="tcr_el1_t1sz", + optional=False, + description="The size offset of the memory region addressed by TTBR1_EL1. Conveniently calculated by LinuxStacker.", + ), + requirements.IntRequirement( + name="page_size_user_space", + optional=False, + description="Page size used by the user address space. Conveniently calculated by LinuxStacker.", + ), + requirements.IntRequirement( + name="page_size_kernel_space", + optional=False, + description="Page size used by the kernel address space. Conveniently calculated by LinuxStacker.", + ), + requirements.BooleanRequirement( + name="layer_debug", + optional=True, + description="Specify if debugging informations about the layer should be printed to user.", + default=False, + ), + requirements.BooleanRequirement( + name="translation_debug", + optional=True, + description="Specify if translation debugging informations should be printed to user.", + default=False, + ), + requirements.IntRequirement( + name="kernel_virtual_offset", optional=True, description="ASLR offset" + ), + requirements.StringRequirement( + name="kernel_banner", + optional=True, + description="Linux banner (/proc/version)", + ), ] From fee620c46dfa6c9e26975be71a3b78450512e36d Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 22 Jan 2024 13:43:36 +0100 Subject: [PATCH 10/86] parameterize debug --- volatility3/framework/layers/arm.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 34ce7629ba..9d8b1f4670 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -13,8 +13,6 @@ from volatility3.framework.layers import linear vollog = logging.getLogger(__name__) -AARCH64_TRANSLATION_DEBUGGING = False -AARCH64_DEBUGGING = False """ Webography : @@ -60,6 +58,8 @@ def __init__( super().__init__( context=context, config_path=config_path, name=name, metadata=metadata ) + self._layer_debug = self.config.get("layer_debug", False) + self._translation_debug = self.config.get("translation_debug", False) self._base_layer = self.config["memory_layer"] # self._swap_layers = [] # TODO self._page_map_offset = self.config["page_map_offset"] @@ -109,6 +109,8 @@ def __init__( self._bits_per_register, self._context_maxvirtaddr, ) + if self._layer_debug: + self._print_layer_debug_informations() if AARCH64_DEBUGGING: vollog.debug(f"Base layer : {self._base_layer}") @@ -247,7 +249,7 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: ) level += 1 - if AARCH64_TRANSLATION_DEBUGGING: + if self._translation_debug: vollog.debug( f"Virtual {hex(virtual_offset)} lives in page frame {hex(table_address)} at offset {hex(self._mask(virtual_offset, low_bit-1, 0))}", ) From 720aa603942408d562363823fe1fb84b6811f9a1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 22 Jan 2024 13:44:54 +0100 Subject: [PATCH 11/86] layer docstring enhancement --- volatility3/framework/layers/arm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 9d8b1f4670..d05aa87e3d 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -32,7 +32,10 @@ class AArch64Exception(exceptions.LayerException): class AArch64(linear.LinearlyMappedLayer): - """Translation Layer for the Arm AArch64 memory mapping.""" + """Translation Layer for the Arm AArch64 memory mapping. + + This layer can be instantiated in two contexts : Low space (user land), High space (kernel land). + """ _direct_metadata = collections.ChainMap( {"architecture": "AArch64"}, From 001e4736229d3ab09c736b304ab0e87f4d2bd115 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 22 Jan 2024 13:45:52 +0100 Subject: [PATCH 12/86] set default mappings in cls context --- volatility3/framework/layers/arm.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index d05aa87e3d..5dbcbfe5c8 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -51,6 +51,13 @@ class AArch64(linear.LinearlyMappedLayer): _maxphyaddr = 64 _maxvirtaddr = _maxphyaddr + # [1], see D8.2.7 to D8.2.9, starting at page 5828 + _granules_indexes = { + 4: [(51, 48), (47, 39), (38, 30), (29, 21), (20, 12)], + 16: [(51, 47), (46, 36), (35, 25), (24, 14)], + 64: [(51, 42), (41, 29), (28, 16)], + } + def __init__( self, context: interfaces.context.ContextInterface, From 8e561983f2c1f4fbbbf58171ba004486df293e69 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 22 Jan 2024 13:47:05 +0100 Subject: [PATCH 13/86] more granularity for context dependent options --- volatility3/framework/layers/arm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 5dbcbfe5c8..16bf574c67 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -73,8 +73,12 @@ def __init__( self._base_layer = self.config["memory_layer"] # self._swap_layers = [] # TODO self._page_map_offset = self.config["page_map_offset"] - self._tcr_el1_tnsz = self.config["tcr_el1_tnsz"] - self._page_size = self.config["page_size"] + self._page_map_offset_kernel = self.config["page_map_offset_kernel"] + self._ttbs_tnsz = [self.config["tcr_el1_t0sz"], self.config["tcr_el1_t1sz"]] + self._ttbs_granules = [ + self.config["page_size_user_space"], + self.config["page_size_kernel_space"], + ] # Context : TTB0 (user) or TTB1 (kernel) self._virtual_addr_space = ( From f46d2a6067a205a61c6a017981c3abdc0f387c16 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 22 Jan 2024 13:49:00 +0100 Subject: [PATCH 14/86] keep only space context values for more clarity --- volatility3/framework/layers/arm.py | 100 ++++++++++++---------------- 1 file changed, 41 insertions(+), 59 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 16bf574c67..a2767b959d 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -81,47 +81,33 @@ def __init__( ] # Context : TTB0 (user) or TTB1 (kernel) - self._virtual_addr_space = ( + self._virtual_addr_space = int( self._page_map_offset == self.config["page_map_offset_kernel"] ) - self._ttbs_tnsz = [self._tcr_el1_tnsz, self._tcr_el1_tnsz] # [1], see D8.1.9, page 5818 - self._ttbs_bitsizes = [64 - self._ttbs_tnsz[0], 64 - self._ttbs_tnsz[1]] - self._ttbs_granules = [self._page_size, self._page_size] + self._ttb_bitsize = 64 - self._ttbs_tnsz[self._virtual_addr_space] + self._ttb_granule = self._ttbs_granules[self._virtual_addr_space] - self._is_52bits = [ - True if self._ttbs_tnsz[ttb] < 16 else False for ttb in range(2) - ] + self._is_52bits = ( + True if self._ttbs_tnsz[self._virtual_addr_space] < 16 else False + ) - # [1], see D8.2.7 to D8.2.9, starting at page 5828 - self._granules_indexes = { - 4: [(51, 48), (47, 39), (38, 30), (29, 21), (20, 12)], - 16: [(51, 47), (46, 36), (35, 25), (24, 14)], - 64: [(51, 42), (41, 29), (28, 16)], - } - self._ttb_lookups_descriptors = self._determine_ttbs_lookup_descriptors() - - # [1], see D8.3, page 5852 - self._descriptors_bits = [ - ( - 49 - if self._ttbs_granules[ttb] in [4, 16] and self._is_52bits[ttb] - else 47, - self._ttb_lookups_descriptors[ttb][-1][1], - ) - for ttb in range(2) - ] + self._ttb_lookup_indexes = self._determine_ttb_lookup_indexes( + self._ttb_granule, self._ttb_bitsize + ) + self._ttb_descriptor_bits = self._determine_ttb_descriptor_bits( + self._ttb_granule, self._ttb_lookup_indexes, self._is_52bits + ) - self._virtual_addr_range = self._get_virtual_addr_ranges()[ + self._virtual_addr_range = self._get_virtual_addr_range()[ self._virtual_addr_space ] - self._context_maxvirtaddr = self._ttbs_bitsizes[self._virtual_addr_space] self._canonical_prefix = self._mask( (1 << self._bits_per_register) - 1, self._bits_per_register, - self._context_maxvirtaddr, + self._ttb_bitsize, ) if self._layer_debug: self._print_layer_debug_informations() @@ -185,12 +171,11 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: invalid_address=virtual_offset, ) - lookup_descriptor = self._ttb_lookups_descriptors[ttb_selector] table_address = self._page_map_offset level = 0 - max_level = len(lookup_descriptor) - 1 + max_level = len(self._ttb_lookup_indexes) - 1 - for high_bit, low_bit in lookup_descriptor: + for high_bit, low_bit in self._ttb_lookup_indexes: index = self._mask(virtual_offset, high_bit, low_bit) # TODO: Adapt endianness ? @@ -203,10 +188,10 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: ta_51_x = None # [1], see D8.3, page 5852 - if self._is_52bits[ttb_selector]: - if self._ttbs_granules[ttb_selector] in [4, 16]: + if self._is_52bits: + if self._ttb_granule in [4, 16]: ta_51_x_bits = (9, 8) - elif self._ttbs_granules[ttb_selector] == 64: + elif self._ttb_granule == 64: ta_51_x_bits = (15, 12) ta_51_x = self._mask( @@ -223,10 +208,10 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: table_address = ( self._mask( descriptor, - self._descriptors_bits[ttb_selector][0], - self._descriptors_bits[ttb_selector][1], + self._ttb_descriptor_bits[0], + self._ttb_descriptor_bits[1], ) - << self._descriptors_bits[ttb_selector][1] + << self._ttb_descriptor_bits[1] ) table_address = ta_51_x | table_address if ta_51_x else table_address # Block descriptor @@ -234,7 +219,7 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: table_address = ( self._mask( descriptor, - self._descriptors_bits[ttb_selector][0], + self._ttb_descriptor_bits[0], low_bit, ) << low_bit @@ -246,10 +231,10 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: table_address = ( self._mask( descriptor, - self._descriptors_bits[ttb_selector][0], - self._descriptors_bits[ttb_selector][1], + self._ttb_descriptor_bits[0], + self._ttb_descriptor_bits[1], ) - << self._descriptors_bits[ttb_selector][1] + << self._ttb_descriptor_bits[1] ) table_address = ta_51_x | table_address if ta_51_x else table_address break @@ -378,13 +363,7 @@ def _mapping( if isinstance(excp, exceptions.PagedInvalidAddressException): mask = (1 << excp.invalid_bits) - 1 else: - mask = ( - 1 - << ( - self._ttbs_granules[self._virtual_addr_space].bit_length() - - 1 - ) - ) - 1 + mask = (1 << (self._ttb_granule.bit_length() - 1)) - 1 length_diff = mask + 1 - (offset & mask) length -= length_diff offset += length_diff @@ -393,19 +372,22 @@ def _mapping( length -= chunk_size offset += chunk_size - def _get_virtual_addr_ranges( + def _get_virtual_addr_range( self, - ) -> List[Tuple[int]]: - """Returns the virtual address space ranges as [(LOW_START, LOW_END), (HIGH_START, HIGH_END)]""" + ) -> Tuple[int]: + """Returns the virtual address space range for the current context (user or kernel space)""" + # [2], see source/arch/arm64/include/asm/memory.h#L62 - ttb0_start = 0 - ttb0_size = 1 << (self._ttbs_bitsizes[0] - 1) - ttb0_end = ttb0_start + (ttb0_size - 1) - ttb1_end = 2**64 - 1 - ttb1_size = 1 << (self._ttbs_bitsizes[1] - 1) - ttb1_start = ttb1_end - (ttb1_size - 1) - - return [(ttb0_start, ttb0_end), (ttb1_start, ttb1_end)] + if self._virtual_addr_space == 0: + ttb_start = 0 + ttb_size = 1 << (self._ttb_bitsize - 1) + ttb_end = ttb_start + (ttb_size - 1) + else: + ttb_end = 2**64 - 1 + ttb_size = 1 << (self._ttb_bitsize - 1) + ttb_start = ttb_end - (ttb_size - 1) + + return (ttb_start, ttb_end) def is_valid(self, offset: int, length: int = 1) -> bool: """Returns whether the address offset can be translated to a valid From 8c0c6f626dc914cc1c3f1e89b97ca9e3ed1e3a58 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 22 Jan 2024 13:50:10 +0100 Subject: [PATCH 15/86] wrap utilities into functions --- volatility3/framework/layers/arm.py | 81 +++++++++++++++++++---------- 1 file changed, 53 insertions(+), 28 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index a2767b959d..b708652b06 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -112,34 +112,59 @@ def __init__( if self._layer_debug: self._print_layer_debug_informations() - if AARCH64_DEBUGGING: - vollog.debug(f"Base layer : {self._base_layer}") - vollog.debug( - f"Virtual address space : {'kernel' if self._virtual_addr_space else 'user'}" - ) - vollog.debug( - f"Virtual addresses spaces ranges : {[tuple([hex(y) for y in x]) for x in self._get_virtual_addr_ranges()]}" - ) - vollog.debug(f"Pages sizes : {self._ttbs_granules}") - vollog.debug(f"TnSZ values : {self._ttbs_bitsizes}") - vollog.debug(f"Page map offset : {hex(self._page_map_offset)}") - vollog.debug(f"Descriptors mappings : {self._ttb_lookups_descriptors}") - - def _determine_ttbs_lookup_descriptors(self) -> List[int]: - """Returns the bits to extract from a translation address (highs and lows)""" - ttb_lookups_descriptors = [] - - for ttb, ttb_granule in enumerate(self._ttbs_granules): - va_bit_size = self._ttbs_bitsizes[ttb] - indexes = [ - index - for index in self._granules_indexes[ttb_granule] - if va_bit_size > index[1] - ] - indexes[0] = (va_bit_size - 1, indexes[0][1]) - ttb_lookups_descriptors.append(indexes) - - return ttb_lookups_descriptors + def _print_layer_debug_informations(self) -> None: + vollog.debug(f"Base layer : {self._base_layer}") + vollog.debug( + f"Virtual address space : {'kernel' if self._virtual_addr_space else 'user'}" + ) + vollog.debug( + f"Virtual addresses space range : {tuple([hex(x) for x in self._get_virtual_addr_range()])}" + ) + vollog.debug(f"Page size : {self._ttb_granule}") + vollog.debug( + f"T{self._virtual_addr_space}SZ : {self._ttbs_tnsz[self._virtual_addr_space]}" + ) + vollog.debug(f"Page map offset : {hex(self._page_map_offset)}") + vollog.debug(f"Translation mappings : {self._ttb_lookup_indexes}") + + return None + + @classmethod + def _determine_ttb_descriptor_bits( + cls, ttb_granule: int, ttb_lookup_indexes: int, is_52bits: bool + ) -> Tuple[int]: + """Returns the descriptor bits to extract from a descriptor (high and low) + + Example with granule = 4 kB without 52 bits : + (47,12) + Example with granule = 16 kB and 52 bits : + (49,14) + """ + + # [1], see D8.3, page 5852 + return ( + 49 if ttb_granule in [4, 16] and is_52bits else 47, + ttb_lookup_indexes[-1][1], + ) + + @classmethod + def _determine_ttb_lookup_indexes( + cls, ttb_granule: int, ttb_bitsize: int + ) -> List[Tuple[int]]: + """Returns the bits to extract from a translation address (highs and lows) + + Example with bitsize = 47 and granule = 4 kB : + indexes = [(51, 48), (47, 39), (38, 30), (29, 21), (20, 12)] + result = [(46, 39), (38, 30), (29, 21), (20, 12)] + """ + indexes = [ + index + for index in cls._granules_indexes[ttb_granule] + if ttb_bitsize > index[1] + ] + indexes[0] = (ttb_bitsize - 1, indexes[0][1]) + + return indexes def _translate(self, virtual_offset: int) -> Tuple[int, int, str]: """Translates a specific offset based on paging tables. From 19e65ea83870cc0d34bb7b6ee80bfd0f2243352d Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 22 Jan 2024 13:50:32 +0100 Subject: [PATCH 16/86] set correct visibility for (de)canonicalize --- volatility3/framework/layers/arm.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index b708652b06..ba8507927a 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -448,20 +448,20 @@ def minimum_address(cls) -> int: def maximum_address(cls) -> int: return (1 << cls._maxvirtaddr) - 1 - def __canonicalize(self, addr: int) -> int: + def canonicalize(self, addr: int) -> int: """Canonicalizes an address by performing an appropiate sign extension on the higher addresses""" - if self._bits_per_register <= self._context_maxvirtaddr: + if self._bits_per_register <= self._ttb_bitsize: return addr & self.address_mask - elif addr < (1 << self._context_maxvirtaddr - 1): + elif addr < (1 << self._ttb_bitsize - 1): return addr - return self._mask(addr, self._context_maxvirtaddr, 0) + self._canonical_prefix + return self._mask(addr, self._ttb_bitsize, 0) + self._canonical_prefix - def __decanonicalize(self, addr: int) -> int: + def decanonicalize(self, addr: int) -> int: """Removes canonicalization to ensure an adress fits within the correct range if it has been canonicalized This will produce an address outside the range if the canonicalization is incorrect """ - if addr < (1 << self._context_maxvirtaddr - 1): + if addr < (1 << self._ttb_bitsize - 1): return addr return addr ^ self._canonical_prefix From 254a3e7feb6436eba27e8da2cae72146cd67da1d Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 22 Jan 2024 13:52:19 +0100 Subject: [PATCH 17/86] simplify page size choice --- volatility3/framework/automagic/linux.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index c9e1372a25..2b7c1fe31f 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -198,9 +198,8 @@ def aarch64_stacker( if "_kernel_flags_le_lo32" in table.symbols: kernel_flags |= table.get_symbol("_kernel_flags_le_lo32").address - # TODO: There is no simple way to determine TTB0 page size for now, so assume TTB0 and TTB1 have the same (which is most likely). # https://www.kernel.org/doc/Documentation/arm64/booting.txt - page_size_bit = (kernel_flags >> 1) & 3 + page_size_kernel_space_bit = (kernel_flags >> 1) & 3 linux_banner_address = table.get_symbol("linux_banner").address + aslr_shift # v6.7/source/arch/arm64/include/asm/memory.h#L186 - v5.7/source/arch/arm64/include/asm/memory.h#L160 @@ -214,14 +213,13 @@ def aarch64_stacker( ) else: # TODO: If KASAN space is large enough, it *might* push kernel addresses higher and generate inaccurate results ? - # TODO: There is no simple way to determine T0SZ for now, so assume T1SZ and T0SZ are the same (which is most likely). # We count the number of high bits equal to 1, which gives us the kernel space address mask and ultimately TCR_EL1.T1SZ. va_bits = (linux_banner_address ^ (2**64 - 1)).bit_length() + 1 - if page_size_bit != 0: - page_size_bit_map = {1: 4, 2: 16, 3: 64} - page_size = page_size_bit_map[page_size_bit] - tcr_el1_tnsz = 64 - va_bits + if 1 <= page_size_kernel_space_bit <= 3: + # 4 || 16 || 64 + page_size_kernel_space = 4**page_size_kernel_space_bit + tcr_el1_t1sz = 64 - va_bits context.config[cls.join(config_path, "page_map_offset")] = dtb context.config[cls.join(config_path, "page_map_offset_kernel")] = dtb From bc1712f13c30e583a523de7d1c45553903c9b20f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 22 Jan 2024 13:52:54 +0100 Subject: [PATCH 18/86] be explicit about arm registers and page sizes --- volatility3/framework/automagic/linux.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2b7c1fe31f..ccf106a4fd 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -221,11 +221,20 @@ def aarch64_stacker( page_size_kernel_space = 4**page_size_kernel_space_bit tcr_el1_t1sz = 64 - va_bits + # Kernel space page size is considered equal to the user space page size + # T1SZ is considered equal to T0SZ context.config[cls.join(config_path, "page_map_offset")] = dtb context.config[cls.join(config_path, "page_map_offset_kernel")] = dtb - context.config[cls.join(config_path, "tcr_el1_tnsz")] = tcr_el1_tnsz - context.config[cls.join(config_path, "page_size")] = page_size - + context.config[cls.join(config_path, "tcr_el1_t1sz")] = tcr_el1_t1sz + context.config[cls.join(config_path, "tcr_el1_t0sz")] = tcr_el1_t1sz + context.config[ + cls.join(config_path, "page_size_kernel_space") + ] = page_size_kernel_space + context.config[ + cls.join(config_path, "page_size_user_space") + ] = page_size_kernel_space + + # Build layer layer = layer_class( context, config_path=config_path, From a2145beb6a95a85d58195cef1083e9704dcd9235 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 22 Jan 2024 19:31:24 +0100 Subject: [PATCH 19/86] typo --- volatility3/framework/automagic/linux.py | 2 +- volatility3/framework/layers/arm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index ccf106a4fd..6908713de3 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -265,7 +265,7 @@ def verify_translation_by_banner( linux_banner_address: int, target_banner: bytes, ) -> bool: - """Determine if a stacked layer is correct or a false positive, by callind the underlying + """Determine if a stacked layer is correct or a false positive, by calling the underlying _translate method against the linux_banner symbol virtual address. Then, compare it with the detected banner to verify the correct translation. This will directly raise an exception, as any failed attempt indicates a wrong layer selection. diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index ba8507927a..8e8118a33f 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -354,7 +354,7 @@ def _mapping( - translate a virtual offset to a physical offset - determine the page size the virtual offset lives in (page_size) - based on the position of the offset in the page, calculate how many bytes to add to get to the end of the page (chunk_size) - - add the chunk_size to the virtual offset, so that we can point to the start of the next page frame + - add the chunk_size to the virtual offset, so that we can point to the start of the next page Example (assume page size is 4096): -> 0xffff800000f92140 lives in page 0xfffffc0000170640 at offset 0x140, which maps to page frame 0x45c19000 at offset 0x140 From edee718f2434bcd43d8ef1754a70d7ece26fa8b9 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 23 Jan 2024 18:54:49 +0100 Subject: [PATCH 20/86] do a quick bruteforce on page size if necessary --- volatility3/framework/automagic/linux.py | 45 ++++++++++++++++-------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 6908713de3..2abb928228 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -188,6 +188,8 @@ def aarch64_stacker( progress_callback=progress_callback, ) dtb = table.get_symbol("swapper_pg_dir").address + kaslr_shift + context.config[cls.join(config_path, "page_map_offset")] = dtb + context.config[cls.join(config_path, "page_map_offset_kernel")] = dtb # CREDIT : https://github.com/crash-utility/crash/blob/28891d1127542dbb2d5ba16c575e14e741ed73ef/arm64.c#L941 kernel_flags = 0 @@ -216,17 +218,21 @@ def aarch64_stacker( # We count the number of high bits equal to 1, which gives us the kernel space address mask and ultimately TCR_EL1.T1SZ. va_bits = (linux_banner_address ^ (2**64 - 1)).bit_length() + 1 - if 1 <= page_size_kernel_space_bit <= 3: - # 4 || 16 || 64 - page_size_kernel_space = 4**page_size_kernel_space_bit - tcr_el1_t1sz = 64 - va_bits + tcr_el1_t1sz = 64 - va_bits + context.config[cls.join(config_path, "tcr_el1_t1sz")] = tcr_el1_t1sz + context.config[cls.join(config_path, "tcr_el1_t0sz")] = tcr_el1_t1sz + + # If "_kernel_flags_le*" aren't in the symbols, we can still do a quick bruteforce on [4,16,64] page sizes + # False positives cannot happen, as translation indexes will be off on a wrong page size + page_size_kernel_space_candidates = ( + [4**page_size_kernel_space_bit] + if 1 <= page_size_kernel_space_bit <= 3 + else [4, 16, 64] + ) + for i, page_size_kernel_space in enumerate(page_size_kernel_space_candidates): # Kernel space page size is considered equal to the user space page size # T1SZ is considered equal to T0SZ - context.config[cls.join(config_path, "page_map_offset")] = dtb - context.config[cls.join(config_path, "page_map_offset_kernel")] = dtb - context.config[cls.join(config_path, "tcr_el1_t1sz")] = tcr_el1_t1sz - context.config[cls.join(config_path, "tcr_el1_t0sz")] = tcr_el1_t1sz context.config[ cls.join(config_path, "page_size_kernel_space") ] = page_size_kernel_space @@ -242,13 +248,22 @@ def aarch64_stacker( metadata={"os": "Linux"}, ) layer.config["kernel_virtual_offset"] = aslr_shift - test_banner_equality = cls.verify_translation_by_banner( - context=context, - layer=layer, - layer_name=layer_name, - linux_banner_address=linux_banner_address, - target_banner=banner, - ) + + try: + test_banner_equality = cls.verify_translation_by_banner( + context=context, + layer=layer, + layer_name=layer_name, + linux_banner_address=linux_banner_address, + target_banner=banner, + ) + except Exception as e: + # Only raise the banner translation error if there are no more candidates + if i < len(page_size_kernel_space_candidates) - 1: + continue + else: + raise e + if layer and dtb and test_banner_equality: vollog.debug(f"Kernel DTB was found at: 0x{dtb:0x}") vollog.debug("AArch64 image found") From 6d60a08e94b01105fa22617d0f4deb755acd578c Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 29 Jan 2024 11:54:58 +0100 Subject: [PATCH 21/86] black 24.1.1 --- volatility3/framework/automagic/linux.py | 12 ++++++------ volatility3/framework/layers/arm.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2abb928228..04f04e37ab 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -233,12 +233,12 @@ def aarch64_stacker( for i, page_size_kernel_space in enumerate(page_size_kernel_space_candidates): # Kernel space page size is considered equal to the user space page size # T1SZ is considered equal to T0SZ - context.config[ - cls.join(config_path, "page_size_kernel_space") - ] = page_size_kernel_space - context.config[ - cls.join(config_path, "page_size_user_space") - ] = page_size_kernel_space + context.config[cls.join(config_path, "page_size_kernel_space")] = ( + page_size_kernel_space + ) + context.config[cls.join(config_path, "page_size_user_space")] = ( + page_size_kernel_space + ) # Build layer layer = layer_class( diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 8e8118a33f..d5cd5a254b 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -289,9 +289,9 @@ def mapping( This allows translation layers to provide maps of contiguous regions in one layer """ - stashed_offset = ( - stashed_mapped_offset - ) = stashed_size = stashed_mapped_size = stashed_map_layer = None + stashed_offset = stashed_mapped_offset = stashed_size = stashed_mapped_size = ( + stashed_map_layer + ) = None for offset, size, mapped_offset, mapped_size, map_layer in self._mapping( offset, length, ignore_errors ): From 0e23aee64460774224f139a2f4777e1ed59d4bae Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 23 Feb 2024 16:15:18 +0100 Subject: [PATCH 22/86] minor optimizations --- volatility3/framework/layers/arm.py | 43 +++++++++++++++-------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index d5cd5a254b..edb29bc522 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -92,18 +92,15 @@ def __init__( self._is_52bits = ( True if self._ttbs_tnsz[self._virtual_addr_space] < 16 else False ) - self._ttb_lookup_indexes = self._determine_ttb_lookup_indexes( self._ttb_granule, self._ttb_bitsize ) self._ttb_descriptor_bits = self._determine_ttb_descriptor_bits( self._ttb_granule, self._ttb_lookup_indexes, self._is_52bits ) - self._virtual_addr_range = self._get_virtual_addr_range()[ self._virtual_addr_space ] - self._canonical_prefix = self._mask( (1 << self._bits_per_register) - 1, self._bits_per_register, @@ -196,10 +193,16 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: invalid_address=virtual_offset, ) + # [1], see D8.3, page 5852 + if self._is_52bits: + if self._ttb_granule in [4, 16]: + ta_51_x_bits = (9, 8) + elif self._ttb_granule == 64: + ta_51_x_bits = (15, 12) + table_address = self._page_map_offset level = 0 max_level = len(self._ttb_lookup_indexes) - 1 - for high_bit, low_bit in self._ttb_lookup_indexes: index = self._mask(virtual_offset, high_bit, low_bit) @@ -210,27 +213,21 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: ), byteorder="little", ) - ta_51_x = None - - # [1], see D8.3, page 5852 + table_address = 0 + # Bits 51->x need to be extracted from the descriptor if self._is_52bits: - if self._ttb_granule in [4, 16]: - ta_51_x_bits = (9, 8) - elif self._ttb_granule == 64: - ta_51_x_bits = (15, 12) - ta_51_x = self._mask( descriptor, ta_51_x_bits[0], ta_51_x_bits[1], ) - ta_51_x = ta_51_x << (52 - ta_51_x.bit_length()) + table_address = ta_51_x << (52 - ta_51_x.bit_length()) # [1], see D8.3, page 5852 descriptor_type = self._mask(descriptor, 1, 0) # Table descriptor if level < max_level and descriptor_type == 0b11: - table_address = ( + table_address |= ( self._mask( descriptor, self._ttb_descriptor_bits[0], @@ -238,10 +235,9 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: ) << self._ttb_descriptor_bits[1] ) - table_address = ta_51_x | table_address if ta_51_x else table_address # Block descriptor elif level < max_level and descriptor_type == 0b01: - table_address = ( + table_address |= ( self._mask( descriptor, self._ttb_descriptor_bits[0], @@ -249,11 +245,10 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: ) << low_bit ) - table_address = ta_51_x | table_address if ta_51_x else table_address break # Page descriptor elif level == max_level and descriptor_type == 0b11: - table_address = ( + table_address |= ( self._mask( descriptor, self._ttb_descriptor_bits[0], @@ -261,7 +256,6 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: ) << self._ttb_descriptor_bits[1] ) - table_address = ta_51_x | table_address if ta_51_x else table_address break # Invalid descriptor || Reserved descriptor (level 3) else: @@ -275,7 +269,7 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: if self._translation_debug: vollog.debug( - f"Virtual {hex(virtual_offset)} lives in page frame {hex(table_address)} at offset {hex(self._mask(virtual_offset, low_bit-1, 0))}", + f"Virtual {hex(virtual_offset)} lives in page frame {hex(table_address)} at offset {hex(self._mask(virtual_offset, low_bit-1, 0))} with descriptor {hex(descriptor)}", ) return table_address, low_bit, descriptor @@ -434,7 +428,14 @@ def is_dirty(self, offset: int) -> bool: @staticmethod def _page_is_dirty(entry: int) -> bool: - """Returns whether a particular page is dirty based on its entry.""" + """Returns whether a particular page is dirty based on its entry. + The bit indicates that its associated block of memory + has been modified and has not been saved to storage yet + + Hardware management (only > Armv8.1-A) : https://developer.arm.com/documentation/102376/0200/Access-Flag/Dirty-state + + [1], see D8.4.6, page 5877 and [1], see D8-16, page 5857 + """ + # The following is based on Linux software implementation : # [2], see arch/arm64/include/asm/pgtable-prot.h#L18 return bool(entry & (1 << 55)) From c10ab69cac3404fc637162e9e6172766c99bc960 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 6 Mar 2024 00:44:17 +0100 Subject: [PATCH 23/86] pass endianness as config --- volatility3/framework/layers/arm.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index edb29bc522..378ca980d2 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -68,6 +68,7 @@ def __init__( super().__init__( context=context, config_path=config_path, name=name, metadata=metadata ) + self._kernel_endianness = self.config["kernel_endianness"] self._layer_debug = self.config.get("layer_debug", False) self._translation_debug = self.config.get("translation_debug", False) self._base_layer = self.config["memory_layer"] @@ -205,13 +206,11 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: max_level = len(self._ttb_lookup_indexes) - 1 for high_bit, low_bit in self._ttb_lookup_indexes: index = self._mask(virtual_offset, high_bit, low_bit) - - # TODO: Adapt endianness ? descriptor = int.from_bytes( base_layer.read( table_address + (index * self._register_size), self._register_size ), - byteorder="little", + byteorder=self._kernel_endianness, ) table_address = 0 # Bits 51->x need to be extracted from the descriptor @@ -520,6 +519,12 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] optional=False, description="Page size used by the kernel address space. Conveniently calculated by LinuxStacker.", ), + requirements.ChoiceRequirement( + choices=["little", "big"], + name="kernel_endianness", + optional=False, + description="Kernel endianness (little or big)", + ), requirements.BooleanRequirement( name="layer_debug", optional=True, From 6a821e71fdfca2638921a4524f3140a855f047a1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 6 Mar 2024 00:46:21 +0100 Subject: [PATCH 24/86] add seen banners check --- volatility3/framework/automagic/linux.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 04f04e37ab..37290bc9cd 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -59,10 +59,17 @@ def stack( ) return None + seen_banners = [] mss = scanners.MultiStringScanner([x for x in linux_banners if x is not None]) for _, banner in layer.scan( context=context, scanner=mss, progress_callback=progress_callback ): + # No need to try stackers on the same banner more than once + if banner in seen_banners: + continue + else: + seen_banners.append(banner) + vollog.debug(f"Identified banner: {repr(banner)}") isf_path = linux_banners.get(banner, None) From 37512f931fa3218f25354d9f708e9207f16f9206 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 6 Mar 2024 00:48:06 +0100 Subject: [PATCH 25/86] exception management is less restrictive --- volatility3/framework/automagic/linux.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 37290bc9cd..1b478eef35 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -90,7 +90,6 @@ def stack( ] = str(banner, "latin-1") linux_arch_stackers = [cls.intel_stacker, cls.aarch64_stacker] - for linux_arch_stacker in linux_arch_stackers: try: layer = linux_arch_stacker( @@ -106,10 +105,7 @@ def stack( if layer: return layer except Exception as e: - vollog.log( - constants.LOGLEVEL_VVVV, - f"{linux_arch_stacker.__name__} exception: {e}", - ) + vollog.exception(e) vollog.debug("No suitable linux banner could be matched") return None @@ -290,24 +286,27 @@ def verify_translation_by_banner( """Determine if a stacked layer is correct or a false positive, by calling the underlying _translate method against the linux_banner symbol virtual address. Then, compare it with the detected banner to verify the correct translation. - This will directly raise an exception, as any failed attempt indicates a wrong layer selection. """ - test_banner_equality = True try: banner_phys_address = layer._translate(linux_banner_address)[0] banner_value = context.layers[layer_name].read( banner_phys_address, len(target_banner) ) - except exceptions.InvalidAddressException: - raise Exception('Cannot translate "linux_banner" virtual address') + except exceptions.InvalidAddressException as e: + vollog.log( + constants.LOGLEVEL_VVVV, + 'Cannot translate "linux_banner" symbol virtual address.', + ) + return False if not banner_value == target_banner: - raise Exception( - f'Translated "linux_banner" virtual address mismatches detected banner : \n{banner_value}\n!=\n{target_banner}' + vollog.error( + f"Mismatch between scanned and virtually translated linux banner : {target_banner} != {banner_value}." ) + return False - return test_banner_equality + return True @classmethod def find_aslr( From 535512675f7d70b6ceffa68ebf422c386dd1824f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 6 Mar 2024 00:49:26 +0100 Subject: [PATCH 26/86] add kernel endianness automagic --- volatility3/framework/automagic/linux.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 1b478eef35..f3f0fed8ce 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -193,6 +193,8 @@ def aarch64_stacker( dtb = table.get_symbol("swapper_pg_dir").address + kaslr_shift context.config[cls.join(config_path, "page_map_offset")] = dtb context.config[cls.join(config_path, "page_map_offset_kernel")] = dtb + kernel_endianness = table.get_type("pointer").vol.data_format.byteorder + context.config[cls.join(config_path, "kernel_endianness")] = kernel_endianness # CREDIT : https://github.com/crash-utility/crash/blob/28891d1127542dbb2d5ba16c575e14e741ed73ef/arm64.c#L941 kernel_flags = 0 @@ -214,7 +216,7 @@ def aarch64_stacker( ) va_bits = int.from_bytes( context.layers[layer_name].read(vabits_actual_phys_addr, 8), - "little", + kernel_endianness, ) else: # TODO: If KASAN space is large enough, it *might* push kernel addresses higher and generate inaccurate results ? From eb872e6817f4e7e23178a42b962cc3e896d19974 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 6 Mar 2024 00:50:58 +0100 Subject: [PATCH 27/86] change linux source comment position --- volatility3/framework/automagic/linux.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f3f0fed8ce..de3acc1d43 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -207,9 +207,14 @@ def aarch64_stacker( # https://www.kernel.org/doc/Documentation/arm64/booting.txt page_size_kernel_space_bit = (kernel_flags >> 1) & 3 - linux_banner_address = table.get_symbol("linux_banner").address + aslr_shift + page_size_kernel_space_candidates = ( + [4**page_size_kernel_space_bit] + if 1 <= page_size_kernel_space_bit <= 3 + else [4, 16, 64] + ) - # v6.7/source/arch/arm64/include/asm/memory.h#L186 - v5.7/source/arch/arm64/include/asm/memory.h#L160 + linux_banner_address = table.get_symbol("linux_banner").address + aslr_shift + # Linux source : v6.7/source/arch/arm64/include/asm/memory.h#L186 - v5.7/source/arch/arm64/include/asm/memory.h#L160 if "vabits_actual" in table.symbols: vabits_actual_phys_addr = ( table.get_symbol("vabits_actual").address + kaslr_shift From f994d28c7039ecfeefbaea6b5f37116f6e53eb37 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 6 Mar 2024 00:51:26 +0100 Subject: [PATCH 28/86] remove initial va_bits calculation method --- volatility3/framework/automagic/linux.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index de3acc1d43..d34f224a5b 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -224,9 +224,7 @@ def aarch64_stacker( kernel_endianness, ) else: - # TODO: If KASAN space is large enough, it *might* push kernel addresses higher and generate inaccurate results ? - # We count the number of high bits equal to 1, which gives us the kernel space address mask and ultimately TCR_EL1.T1SZ. - va_bits = (linux_banner_address ^ (2**64 - 1)).bit_length() + 1 + va_bits = 0 tcr_el1_t1sz = 64 - va_bits context.config[cls.join(config_path, "tcr_el1_t1sz")] = tcr_el1_t1sz From f9f5c0fd45d33bff7cd0fb4aa87d018bc4dea970 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 6 Mar 2024 00:52:34 +0100 Subject: [PATCH 29/86] calculate va_bits by elimination if no info available --- volatility3/framework/automagic/linux.py | 80 ++++++++++++------------ 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index d34f224a5b..170ae527f3 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -226,38 +226,44 @@ def aarch64_stacker( else: va_bits = 0 - tcr_el1_t1sz = 64 - va_bits - context.config[cls.join(config_path, "tcr_el1_t1sz")] = tcr_el1_t1sz - context.config[cls.join(config_path, "tcr_el1_t0sz")] = tcr_el1_t1sz - - # If "_kernel_flags_le*" aren't in the symbols, we can still do a quick bruteforce on [4,16,64] page sizes - # False positives cannot happen, as translation indexes will be off on a wrong page size - page_size_kernel_space_candidates = ( - [4**page_size_kernel_space_bit] - if 1 <= page_size_kernel_space_bit <= 3 - else [4, 16, 64] + """ + Determining the number of bits available for virtual addresses (va_bits) in 64 bits addresses + is not straightforward, and not available in the kernel symbols. + Calculation by symbols addresses masking isn't accurate, as kernel addresses can be + pushed too high from the TTB1 region start, skewing results. + See https://www.kernel.org/doc/html/v5.5/arm64/memory.html. + Testing every possible value is quick, and starting from the highest possible one + blocks false positives. The va_bits value is eventually used to calculate + which bits to extract from a virtual address, for translation purposes. + """ + va_bits_candidates = ( + [va_bits] if va_bits != 0 else [x for x in range(52, 16, -1)] ) - - for i, page_size_kernel_space in enumerate(page_size_kernel_space_candidates): - # Kernel space page size is considered equal to the user space page size + for va_bits in va_bits_candidates: + tcr_el1_t1sz = 64 - va_bits # T1SZ is considered equal to T0SZ - context.config[cls.join(config_path, "page_size_kernel_space")] = ( - page_size_kernel_space - ) - context.config[cls.join(config_path, "page_size_user_space")] = ( - page_size_kernel_space - ) - - # Build layer - layer = layer_class( - context, - config_path=config_path, - name=new_layer_name, - metadata={"os": "Linux"}, - ) - layer.config["kernel_virtual_offset"] = aslr_shift + context.config[cls.join(config_path, "tcr_el1_t1sz")] = tcr_el1_t1sz + context.config[cls.join(config_path, "tcr_el1_t0sz")] = tcr_el1_t1sz + + # If "_kernel_flags_le*" aren't in the symbols, we can still do a quick bruteforce on [4,16,64] page sizes + # False positives cannot happen, as translation indexes will be off on a wrong page size + for page_size_kernel_space in page_size_kernel_space_candidates: + # Kernel space page size is considered equal to the user space page size + context.config[cls.join(config_path, "page_size_kernel_space")] = ( + page_size_kernel_space + ) + context.config[cls.join(config_path, "page_size_user_space")] = ( + page_size_kernel_space + ) + # Build layer + layer = layer_class( + context, + config_path=config_path, + name=new_layer_name, + metadata={"os": "Linux"}, + ) + layer.config["kernel_virtual_offset"] = aslr_shift - try: test_banner_equality = cls.verify_translation_by_banner( context=context, layer=layer, @@ -265,17 +271,11 @@ def aarch64_stacker( linux_banner_address=linux_banner_address, target_banner=banner, ) - except Exception as e: - # Only raise the banner translation error if there are no more candidates - if i < len(page_size_kernel_space_candidates) - 1: - continue - else: - raise e - - if layer and dtb and test_banner_equality: - vollog.debug(f"Kernel DTB was found at: 0x{dtb:0x}") - vollog.debug("AArch64 image found") - return layer + + if layer and dtb and test_banner_equality: + vollog.debug(f"Kernel DTB was found at: 0x{dtb:0x}") + vollog.debug("AArch64 image found") + return layer return None From 66e989bd228cf535059787bdfbf0bd849adc0020 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 7 Mar 2024 21:25:54 +0100 Subject: [PATCH 30/86] destroy invalid layers --- volatility3/framework/automagic/linux.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 170ae527f3..2d8ba1bf2c 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -162,12 +162,14 @@ def intel_stacker( linux_banner_address=linux_banner_address, target_banner=banner, ) - + if layer and dtb and test_banner_equality: vollog.debug(f"DTB was found at: 0x{dtb:0x}") vollog.debug("Intel image found") return layer - + else: + layer.destroy() + return None @classmethod @@ -276,6 +278,8 @@ def aarch64_stacker( vollog.debug(f"Kernel DTB was found at: 0x{dtb:0x}") vollog.debug("AArch64 image found") return layer + else: + layer.destroy() return None From 70406451aaebe891b6f25ea43205cbbac430a1e1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 7 Mar 2024 21:27:59 +0100 Subject: [PATCH 31/86] black formatting --- volatility3/framework/automagic/linux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 2d8ba1bf2c..c8ecab1ef6 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -162,14 +162,14 @@ def intel_stacker( linux_banner_address=linux_banner_address, target_banner=banner, ) - + if layer and dtb and test_banner_equality: vollog.debug(f"DTB was found at: 0x{dtb:0x}") vollog.debug("Intel image found") return layer else: layer.destroy() - + return None @classmethod From 65de635b1b3d5b4093bb6abbe3261e038738ff0e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 8 Mar 2024 20:18:01 +0100 Subject: [PATCH 32/86] explicit comments + revert and improve va_bits calculation --- volatility3/framework/automagic/linux.py | 28 +++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index c8ecab1ef6..ec81c71477 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -221,26 +221,34 @@ def aarch64_stacker( vabits_actual_phys_addr = ( table.get_symbol("vabits_actual").address + kaslr_shift ) + # Linux source : v6.7/source/arch/arm64/Kconfig#L1263, VA_BITS va_bits = int.from_bytes( context.layers[layer_name].read(vabits_actual_phys_addr, 8), kernel_endianness, ) else: - va_bits = 0 + """ + Count leftmost bits equal to 1, deduce number of used bits for virtual addressing. + Example : + linux_banner_address = 0xffffffd733aae820 = 0b1111111111111111111111111101011100110011101010101110100000100000 + va_bits = (linux_banner_address ^ (2**64 - 1)).bit_length() + 1 = 39 + """ + va_bits = (linux_banner_address ^ (2**64 - 1)).bit_length() + 1 """ - Determining the number of bits available for virtual addresses (va_bits) in 64 bits addresses + Determining the number of useful bits in virtual addresses (VA_BITS) is not straightforward, and not available in the kernel symbols. - Calculation by symbols addresses masking isn't accurate, as kernel addresses can be - pushed too high from the TTB1 region start, skewing results. + Calculation by masking works great, but not in every case, due to the AArch64 memory layout, + sometimes pushing kernel addresses "too far" from the TTB1 start. See https://www.kernel.org/doc/html/v5.5/arm64/memory.html. - Testing every possible value is quick, and starting from the highest possible one - blocks false positives. The va_bits value is eventually used to calculate - which bits to extract from a virtual address, for translation purposes. + Errors are by 1 or 2 bits, so we can try va_bits - {1,2,3}. + Example, assuming the good va_bits value is 39 : + # Case where calculation was correct : 1 iteration + va_bits_candidates = [**39**, 38, 37, 36] + # Case where calculation is off by 1 : 2 iterations + va_bits_candidates = [40, **39**, 38, 37] """ - va_bits_candidates = ( - [va_bits] if va_bits != 0 else [x for x in range(52, 16, -1)] - ) + va_bits_candidates = [va_bits] + [va_bits + i for i in range(-1, -4, -1)] for va_bits in va_bits_candidates: tcr_el1_t1sz = 64 - va_bits # T1SZ is considered equal to T0SZ From bc47e76a4a63a05fae2009779d2a5f3b6de662c6 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 4 Apr 2024 18:15:23 +0200 Subject: [PATCH 33/86] explain ttb_granule and page_size similarity --- volatility3/framework/layers/arm.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 378ca980d2..a1421dfb1a 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -89,7 +89,11 @@ def __init__( # [1], see D8.1.9, page 5818 self._ttb_bitsize = 64 - self._ttbs_tnsz[self._virtual_addr_space] self._ttb_granule = self._ttbs_granules[self._virtual_addr_space] - + """ + Translation Table Granule is in fact the page size, as it is the + smallest block of memory that can be described. + Possibles values are 4, 16 or 64 (kB). + """ self._is_52bits = ( True if self._ttbs_tnsz[self._virtual_addr_space] < 16 else False ) From 8978b380913cd7a08f1a898f0f894ac86fbfa7f1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 4 Apr 2024 18:15:56 +0200 Subject: [PATCH 34/86] add page related getters --- volatility3/framework/layers/arm.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index a1421dfb1a..cd69a39f32 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -114,6 +114,28 @@ def __init__( if self._layer_debug: self._print_layer_debug_informations() + @property + @functools.lru_cache() + def page_shift(self) -> int: + """Page shift for this layer, which is the page size bit length. + - Typical values : 12, 14, 16 + """ + return self.page_size.bit_length() - 1 + + @property + @functools.lru_cache() + def page_size(self) -> int: + """Page size of this layer, in bytes. + - Typical values : 4096, 16384, 65536 + """ + return self._ttb_granule * 1024 + + @property + @functools.lru_cache() + def page_mask(cls) -> int: + """Page mask for this layer.""" + return ~(cls.page_size - 1) + def _print_layer_debug_informations(self) -> None: vollog.debug(f"Base layer : {self._base_layer}") vollog.debug( From 6cafcc591db5b5c1b0299ce3bb70cc639bcc3c76 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 4 Apr 2024 18:16:48 +0200 Subject: [PATCH 35/86] tidy up incremental level variable --- volatility3/framework/layers/arm.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index cd69a39f32..caedd3be37 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -228,9 +228,8 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: ta_51_x_bits = (15, 12) table_address = self._page_map_offset - level = 0 max_level = len(self._ttb_lookup_indexes) - 1 - for high_bit, low_bit in self._ttb_lookup_indexes: + for level, (high_bit, low_bit) in enumerate(self._ttb_lookup_indexes): index = self._mask(virtual_offset, high_bit, low_bit) descriptor = int.from_bytes( base_layer.read( @@ -290,7 +289,6 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: invalid_bits=low_bit, entry=descriptor, ) - level += 1 if self._translation_debug: vollog.debug( From dc559fc0d2ce9acf627347848d2f17ec70955831 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 4 Apr 2024 18:17:10 +0200 Subject: [PATCH 36/86] more generic kernel_banner comment --- volatility3/framework/layers/arm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index caedd3be37..45d4c1a9fe 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -567,6 +567,6 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] requirements.StringRequirement( name="kernel_banner", optional=True, - description="Linux banner (/proc/version)", + description="Kernel unique identifier, including compiler name and version, kernel version, compile time.", ), ] From 25f94df19816952e9f5c7ebe4402b6eb887be229 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 16 Apr 2024 19:08:21 +0200 Subject: [PATCH 37/86] handle null va_bits in symbols --- volatility3/framework/automagic/linux.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index ec81c71477..defaba3677 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -217,6 +217,7 @@ def aarch64_stacker( linux_banner_address = table.get_symbol("linux_banner").address + aslr_shift # Linux source : v6.7/source/arch/arm64/include/asm/memory.h#L186 - v5.7/source/arch/arm64/include/asm/memory.h#L160 + va_bits = 0 if "vabits_actual" in table.symbols: vabits_actual_phys_addr = ( table.get_symbol("vabits_actual").address + kaslr_shift @@ -226,7 +227,7 @@ def aarch64_stacker( context.layers[layer_name].read(vabits_actual_phys_addr, 8), kernel_endianness, ) - else: + if not va_bits: """ Count leftmost bits equal to 1, deduce number of used bits for virtual addressing. Example : From 7aae9c80ba57ac0ddb64676a4f9aba172e84d75f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 16 Apr 2024 19:09:46 +0200 Subject: [PATCH 38/86] decrease banner error log level --- volatility3/framework/automagic/linux.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index defaba3677..4870918ce1 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -319,8 +319,9 @@ def verify_translation_by_banner( return False if not banner_value == target_banner: - vollog.error( - f"Mismatch between scanned and virtually translated linux banner : {target_banner} != {banner_value}." + vollog.log( + constants.LOGLEVEL_VV, + f"Mismatch between scanned and virtually translated linux banner : {target_banner} != {banner_value}.", ) return False From 680d0e414b06f4bf6ab350b96ceed3d67aa1c718 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 16 Apr 2024 19:10:44 +0200 Subject: [PATCH 39/86] bits_per_register getter --- volatility3/framework/layers/arm.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 45d4c1a9fe..d5fa766a70 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -136,6 +136,13 @@ def page_mask(cls) -> int: """Page mask for this layer.""" return ~(cls.page_size - 1) + @classproperty + @functools.lru_cache() + def bits_per_register(cls) -> int: + """Returns the bits_per_register to determine the range of an + AArch64TranslationLayer.""" + return cls._bits_per_register + def _print_layer_debug_informations(self) -> None: vollog.debug(f"Base layer : {self._base_layer}") vollog.debug( From ff0bfc3a7d752de5e86db65cd70453cd23a3ea88 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 16 Apr 2024 19:13:03 +0200 Subject: [PATCH 40/86] move generic getters to the the end --- volatility3/framework/layers/arm.py | 58 ++++++++++++++--------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index d5fa766a70..1143f5e23f 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -114,35 +114,6 @@ def __init__( if self._layer_debug: self._print_layer_debug_informations() - @property - @functools.lru_cache() - def page_shift(self) -> int: - """Page shift for this layer, which is the page size bit length. - - Typical values : 12, 14, 16 - """ - return self.page_size.bit_length() - 1 - - @property - @functools.lru_cache() - def page_size(self) -> int: - """Page size of this layer, in bytes. - - Typical values : 4096, 16384, 65536 - """ - return self._ttb_granule * 1024 - - @property - @functools.lru_cache() - def page_mask(cls) -> int: - """Page mask for this layer.""" - return ~(cls.page_size - 1) - - @classproperty - @functools.lru_cache() - def bits_per_register(cls) -> int: - """Returns the bits_per_register to determine the range of an - AArch64TranslationLayer.""" - return cls._bits_per_register - def _print_layer_debug_informations(self) -> None: vollog.debug(f"Base layer : {self._base_layer}") vollog.debug( @@ -469,6 +440,35 @@ def _page_is_dirty(entry: int) -> bool: # [2], see arch/arm64/include/asm/pgtable-prot.h#L18 return bool(entry & (1 << 55)) + @property + @functools.lru_cache() + def page_shift(self) -> int: + """Page shift for this layer, which is the page size bit length. + - Typical values : 12, 14, 16 + """ + return self.page_size.bit_length() - 1 + + @property + @functools.lru_cache() + def page_size(self) -> int: + """Page size of this layer, in bytes. + - Typical values : 4096, 16384, 65536 + """ + return self._ttb_granule * 1024 + + @property + @functools.lru_cache() + def page_mask(cls) -> int: + """Page mask for this layer.""" + return ~(cls.page_size - 1) + + @classproperty + @functools.lru_cache() + def bits_per_register(cls) -> int: + """Returns the bits_per_register to determine the range of an + AArch64TranslationLayer.""" + return cls._bits_per_register + @classproperty @functools.lru_cache() def minimum_address(cls) -> int: From f55a1cdfbce16a3a2e841ff7cb868edc843f1cdb Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 7 Aug 2024 18:12:47 +0200 Subject: [PATCH 41/86] add cpu_registers requirement --- volatility3/framework/layers/arm.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 1143f5e23f..30738c8faf 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -5,6 +5,7 @@ import logging import functools import collections +import json from typing import Optional, Dict, Any, List, Iterable, Tuple from volatility3 import classproperty @@ -81,6 +82,18 @@ def __init__( self.config["page_size_kernel_space"], ] + # Unserialize cpu_registers config attribute into a dict + self._cpu_regs = self.config.get("cpu_registers", "{}") + try: + self._cpu_regs: dict = json.loads(self._cpu_regs) + except json.JSONDecodeError: + vollog.error( + 'Could not JSON deserialize "cpu_registers" layer requirement.' + ) + + # Determine CPU features + self._cpu_features = {"feat_HAFDBS": self._get_cpu_feature_HAFDBS()} + # Context : TTB0 (user) or TTB1 (kernel) self._virtual_addr_space = int( self._page_map_offset == self.config["page_map_offset_kernel"] @@ -556,6 +569,12 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] optional=False, description="Kernel endianness (little or big)", ), + requirements.StringRequirement( + name="cpu_registers", + optional=True, + description="Serialized dict of cpu register keys bound to their corresponding value. Needed for specific (non-mandatory) uses (ex: dirty bit management).", + default="{}", + ), requirements.BooleanRequirement( name="layer_debug", optional=True, From 69ee43196aadca783e75221aaccbd4e77201c24a Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 7 Aug 2024 18:13:35 +0200 Subject: [PATCH 42/86] add cpu registers attributes mappings --- volatility3/framework/layers/arm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 30738c8faf..b21edefcf6 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -59,6 +59,10 @@ class AArch64(linear.LinearlyMappedLayer): 64: [(51, 42), (41, 29), (28, 16)], } + # (high_bit, low_bit) + # [1], see D19.2, page 6339 + _cpu_regs_mappings = {"aa64mmfr1_el1": {"HAFDBS": (3, 0)}} + def __init__( self, context: interfaces.context.ContextInterface, From cf51013bda593b93dc742d976efcfe21fb180456 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 7 Aug 2024 18:15:05 +0200 Subject: [PATCH 43/86] doc --- volatility3/framework/layers/arm.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index b21edefcf6..3d615a249d 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -19,13 +19,18 @@ Webography : [1] Arm, "Arm Architecture Reference Manual for A-profile architecture, DDI 0487J.a (ID042523)", https://developer.arm.com/documentation/ddi0487/ja/?lang=en [2] Linux, Linux Kernel source code, v6.7 + [3] Arm, "Programmer's Guide for ARMv8-A", https://cs140e.sergio.bz/docs/ARMv8-A-Programmer-Guide.pdf Glossary : TTB : Translation Table Base TCR : Translation Control Register EL : Exception Level (0:Application,1:Kernel,2:Hypervisor,3:Secure Monitor) Granule : Translation granule (smallest block of memory that can be described) - """ + +Definitions : + + The OS-controlled translation is called stage 1 translation, and the hypervisor-controlled translation is called stage 2 translation. +""" class AArch64Exception(exceptions.LayerException): @@ -205,10 +210,10 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: """ base_layer = self.context.layers[self._base_layer] - # [1], see D8.2.4, page 5824 + # [1], see D8.2.4, page 5825 ttb_selector = self._mask(virtual_offset, 55, 55) - # Check if requested address belongs to the context virtual memory space + # Check if requested address belongs to the virtual memory space context if ttb_selector != self._virtual_addr_space: raise exceptions.InvalidAddressException( layer_name=self.name, From 26d92c7c1826083cf7ad83a9b6d72f4a46c469e8 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 7 Aug 2024 18:16:05 +0200 Subject: [PATCH 44/86] calculate 52 bits mappings once --- volatility3/framework/layers/arm.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 3d615a249d..cf1ba8fc69 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -133,6 +133,14 @@ def __init__( self._bits_per_register, self._ttb_bitsize, ) + + # [1], see D8.3, page 5852 + if self._is_52bits: + if self._ttb_granule in [4, 16]: + self._ta_51_x_bits = (9, 8) + elif self._ttb_granule == 64: + self._ta_51_x_bits = (15, 12) + if self._layer_debug: self._print_layer_debug_informations() @@ -220,13 +228,6 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: invalid_address=virtual_offset, ) - # [1], see D8.3, page 5852 - if self._is_52bits: - if self._ttb_granule in [4, 16]: - ta_51_x_bits = (9, 8) - elif self._ttb_granule == 64: - ta_51_x_bits = (15, 12) - table_address = self._page_map_offset max_level = len(self._ttb_lookup_indexes) - 1 for level, (high_bit, low_bit) in enumerate(self._ttb_lookup_indexes): @@ -242,8 +243,8 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: if self._is_52bits: ta_51_x = self._mask( descriptor, - ta_51_x_bits[0], - ta_51_x_bits[1], + self._ta_51_x_bits[0], + self._ta_51_x_bits[1], ) table_address = ta_51_x << (52 - ta_51_x.bit_length()) From 5c63b13f64d90b07232a03106bdbfc86a8c2f282 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 7 Aug 2024 18:16:33 +0200 Subject: [PATCH 45/86] wrapper to read a CPU register field --- volatility3/framework/layers/arm.py | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index cf1ba8fc69..80291cd9f0 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -502,6 +502,41 @@ def minimum_address(cls) -> int: def maximum_address(cls) -> int: return (1 << cls._maxvirtaddr) - 1 + def _read_cpu_register_field(self, register: str, field: str) -> int: + # Prefer speeding up read operations, as keys are most likely to be valid + # Handle errors in a dedicated except block, if needed + try: + cpu_reg_mapping = self._cpu_regs_mappings[register][field] + return self._mask( + self._cpu_regs[register], cpu_reg_mapping[0], cpu_reg_mapping[1] + ) + except: + if not self._cpu_regs.get(register): + raise KeyError( + f"Access to CPU register {register} was requested, but the register value wasn't provided to this layer initially." + ) + + if not self._cpu_regs_mappings.get(register) or not self._cpu_regs_mappings[ + register + ].get(field): + raise KeyError( + f"Access of field {field} from CPU register {register} was requested, but no pre-defined mapping was found." + ) + + return None + + # CPU features + def _get_cpu_feature_HAFDBS(self): + """ + Hardware updates to Access flag and Dirty state in translation tables. + [1], see D19.2.65, page 6784 + """ + try: + reg_HAFDBS = self._read_cpu_register_field("aa64mmfr1_el1", "HAFDBS") + return reg_HAFDBS >= 0b10 + except KeyError: + return None + def canonicalize(self, addr: int) -> int: """Canonicalizes an address by performing an appropiate sign extension on the higher addresses""" if self._bits_per_register <= self._ttb_bitsize: From 3449de278370b53a3a8fae0f3a893425dca76b25 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 7 Aug 2024 18:17:09 +0200 Subject: [PATCH 46/86] hw dirty state management --- volatility3/framework/layers/arm.py | 38 +++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 80291cd9f0..09f1bdfd1a 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -450,18 +450,36 @@ def is_dirty(self, offset: int) -> bool: """Returns whether the page at offset is marked dirty""" return self._page_is_dirty(self._translate_entry(offset)[2]) - @staticmethod - def _page_is_dirty(entry: int) -> bool: - """Returns whether a particular page is dirty based on its entry. - The bit indicates that its associated block of memory - has been modified and has not been saved to storage yet + def _page_is_dirty(self, entry: int) -> bool: + """ + Hardware management of the dirty state (only > Armv8.1-A). + + General documentation : + https://developer.arm.com/documentation/102376/0200/Access-Flag/Dirty-state + + Technical documentation : + [1], see D8.4.6, page 5877 : "Hardware management of the dirty state" + [1], see D8-16 and page 5861 : "Stage 1 attribute fields in Block and Page descriptors" - Hardware management (only > Armv8.1-A) : https://developer.arm.com/documentation/102376/0200/Access-Flag/Dirty-state - + [1], see D8.4.6, page 5877 and [1], see D8-16, page 5857 + > For the purpose of FEAT_HAFDBS, a Block descriptor or Page descriptor can be described as having one of the following states: + • Non-writeable. + • Writeable-clean. + • Writeable-dirty. + + [1], see D8-41, page 5868 : + AP[2] | Access permission + -------|------------------ + 0 | Read/write + 1 | Read-only """ - # The following is based on Linux software implementation : - # [2], see arch/arm64/include/asm/pgtable-prot.h#L18 - return bool(entry & (1 << 55)) + if self._cpu_features.get("feat_HAFDBS", None): + # Dirty Bit Modifier and Access Permissions bits + # DBM == 1 and AP == 0 -> HW dirty state + return bool((entry & (1 << 51)) and not (entry & (1 << 7))) + else: + raise NotImplementedError( + "Hardware updates to Access flag and Dirty state in translation tables are not available in the target kernel. Please try using a software based implementation of dirty bit management." + ) @property @functools.lru_cache() From 62526b58835295af77441677f7102fb7bb6e79ff Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 7 Aug 2024 18:17:38 +0200 Subject: [PATCH 47/86] linuxaarch64 layer mixin --- volatility3/framework/layers/arm.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 09f1bdfd1a..8213149625 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -659,3 +659,26 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] description="Kernel unique identifier, including compiler name and version, kernel version, compile time.", ), ] + + +class LinuxAArch64Mixin(AArch64): + def _page_is_dirty(self, entry: int) -> bool: + """Returns whether a particular page is dirty based on its (page table) entry. + The bit indicates that its associated block of memory + has been modified and has not been saved to storage yet. + + The following is based on Linux software AArch64 dirty bit management. + [2], see arch/arm64/include/asm/pgtable-prot.h#L18 + [3], see page 12-25 + https://lkml.org/lkml/2023/7/7/77 -> Linux implementation detail + """ + sw_dirty = bool(entry & (1 << 55)) + try: + hw_dirty = super()._page_is_dirty(entry) + return sw_dirty or hw_dirty + except NotImplementedError: + return sw_dirty + + +class LinuxAArch64(LinuxAArch64Mixin, AArch64): + pass From 953eec0d6f933acc3f8985023d8fe0af3863732b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 7 Aug 2024 18:21:18 +0200 Subject: [PATCH 48/86] imports and minor cleanups --- volatility3/framework/automagic/linux.py | 35 +++++++++++++----------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 342595a027..59fc4d743f 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -4,13 +4,15 @@ import logging import os -from typing import Optional, Tuple, Type, Union +import json +from typing import Optional, Tuple, Union from volatility3.framework import constants, interfaces, exceptions from volatility3.framework.automagic import symbol_cache, symbol_finder from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners, arm from volatility3.framework.symbols import linux +from volatility3.framework.interfaces.configuration import path_join vollog = logging.getLogger(__name__) @@ -18,7 +20,6 @@ class LinuxStacker(interfaces.automagic.StackerLayerInterface): stack_order = 35 exclusion_list = ["mac", "windows"] - join = interfaces.configuration.path_join @classmethod def stack( @@ -83,10 +84,10 @@ def stack( ) context.symbol_space.append(table) new_layer_name = context.layers.free_layer_name("LinuxLayer") - config_path = cls.join("LinuxHelper", new_layer_name) - context.config[cls.join(config_path, "memory_layer")] = layer_name + config_path = path_join("LinuxHelper", new_layer_name) + context.config[path_join(config_path, "memory_layer")] = layer_name context.config[ - cls.join(config_path, LinuxSymbolFinder.banner_config_key) + path_join(config_path, LinuxSymbolFinder.banner_config_key) ] = str(banner, "latin-1") linux_arch_stackers = [cls.intel_stacker, cls.aarch64_stacker] @@ -107,7 +108,7 @@ def stack( except Exception as e: vollog.exception(e) - vollog.debug("No suitable linux banner could be matched") + vollog.debug("No suitable Linux banner could be matched") return None @classmethod @@ -122,7 +123,8 @@ def intel_stacker( banner: str, progress_callback: constants.ProgressCallback = None, ) -> Union[intel.Intel, intel.Intel32e, None]: - layer_class: Type = intel.Intel + layer_class = intel.Intel + if "init_top_pgt" in table.symbols: layer_class = intel.Intel32e dtb_symbol_name = "init_top_pgt" @@ -164,7 +166,7 @@ def intel_stacker( ) if layer and dtb and test_banner_equality: - vollog.debug(f"DTB was found at: 0x{dtb:0x}") + vollog.debug(f"DTB was found at: {hex(dtb)}") vollog.debug("Intel image found") return layer else: @@ -193,10 +195,10 @@ def aarch64_stacker( progress_callback=progress_callback, ) dtb = table.get_symbol("swapper_pg_dir").address + kaslr_shift - context.config[cls.join(config_path, "page_map_offset")] = dtb - context.config[cls.join(config_path, "page_map_offset_kernel")] = dtb + context.config[path_join(config_path, "page_map_offset")] = dtb + context.config[path_join(config_path, "page_map_offset_kernel")] = dtb kernel_endianness = table.get_type("pointer").vol.data_format.byteorder - context.config[cls.join(config_path, "kernel_endianness")] = kernel_endianness + context.config[path_join(config_path, "kernel_endianness")] = kernel_endianness # CREDIT : https://github.com/crash-utility/crash/blob/28891d1127542dbb2d5ba16c575e14e741ed73ef/arm64.c#L941 kernel_flags = 0 @@ -252,18 +254,18 @@ def aarch64_stacker( va_bits_candidates = [va_bits] + [va_bits + i for i in range(-1, -4, -1)] for va_bits in va_bits_candidates: tcr_el1_t1sz = 64 - va_bits - # T1SZ is considered equal to T0SZ - context.config[cls.join(config_path, "tcr_el1_t1sz")] = tcr_el1_t1sz - context.config[cls.join(config_path, "tcr_el1_t0sz")] = tcr_el1_t1sz + # T1SZ is considered to be equal to T0SZ + context.config[path_join(config_path, "tcr_el1_t1sz")] = tcr_el1_t1sz + context.config[path_join(config_path, "tcr_el1_t0sz")] = tcr_el1_t1sz # If "_kernel_flags_le*" aren't in the symbols, we can still do a quick bruteforce on [4,16,64] page sizes # False positives cannot happen, as translation indexes will be off on a wrong page size for page_size_kernel_space in page_size_kernel_space_candidates: # Kernel space page size is considered equal to the user space page size - context.config[cls.join(config_path, "page_size_kernel_space")] = ( + context.config[path_join(config_path, "page_size_kernel_space")] = ( page_size_kernel_space ) - context.config[cls.join(config_path, "page_size_user_space")] = ( + context.config[path_join(config_path, "page_size_user_space")] = ( page_size_kernel_space ) # Build layer @@ -412,6 +414,7 @@ class LinuxSymbolFinder(symbol_finder.SymbolFinder): banner_config_key = "kernel_banner" operating_system = "linux" + # TODO: Avoid hardcoded strings symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" find_aslr = lambda cls, *args: LinuxStacker.find_aslr(*args)[1] exclusion_list = ["mac", "windows"] From f9a54e4a826c0c3dea2292cb055a619538a63c90 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 7 Aug 2024 18:24:36 +0200 Subject: [PATCH 49/86] put arch substackers in classes, unify linux kaslr calculations, extract AArch64 CPU registers --- volatility3/framework/automagic/linux.py | 293 +++++++++++++---------- 1 file changed, 168 insertions(+), 125 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 59fc4d743f..5fb584e291 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -90,10 +90,11 @@ def stack( path_join(config_path, LinuxSymbolFinder.banner_config_key) ] = str(banner, "latin-1") - linux_arch_stackers = [cls.intel_stacker, cls.aarch64_stacker] + linux_arch_stackers = [LinuxIntelSubStacker, LinuxAArch64SubStacker] for linux_arch_stacker in linux_arch_stackers: try: - layer = linux_arch_stacker( + sub_stacker = linux_arch_stacker(cls) + layer = sub_stacker.stack( context=context, layer_name=layer_name, table=table, @@ -112,9 +113,117 @@ def stack( return None @classmethod - def intel_stacker( + def verify_translation_by_banner( cls, context: interfaces.context.ContextInterface, + layer, + layer_name: str, + linux_banner_address: int, + target_banner: bytes, + ) -> bool: + """Determine if a stacked layer is correct or a false positive, by calling the underlying + _translate method against the linux_banner symbol virtual address. Then, compare it with + the detected banner to verify the correct translation. + """ + + try: + banner_phys_address = layer._translate(linux_banner_address)[0] + banner_value = context.layers[layer_name].read( + banner_phys_address, len(target_banner) + ) + except exceptions.InvalidAddressException: + vollog.log( + constants.LOGLEVEL_VVVV, + 'Cannot translate "linux_banner" symbol virtual address.', + ) + return False + + if not banner_value == target_banner: + vollog.log( + constants.LOGLEVEL_VV, + f"Mismatch between scanned and virtually translated linux banner : {target_banner} != {banner_value}.", + ) + return False + + return True + + @classmethod + def find_aslr( + cls, + context: interfaces.context.ContextInterface, + symbol_table: str, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Tuple[int, int]: + """Determines the offset of the actual DTB in physical space and its + symbol offset.""" + + module = context.module(symbol_table, layer_name, 0) + swapper_signature = rb"swapper(\/0|\x00\x00)\x00\x00\x00\x00\x00\x00" + address_mask = context.symbol_space[symbol_table].config.get( + "symbol_mask", None + ) + init_task_symbol = symbol_table + constants.BANG + "init_task" + init_task_json_address = context.symbol_space.get_symbol( + init_task_symbol + ).address + task_symbol = module.get_type("task_struct") + comm_child_offset = task_symbol.relative_child_offset("comm") + + for offset in context.layers[layer_name].scan( + scanner=scanners.RegExScanner(swapper_signature), + context=context, + progress_callback=progress_callback, + ): + init_task_address = offset - comm_child_offset + init_task = module.object( + object_type="task_struct", offset=init_task_address, absolute=True + ) + if init_task.pid != 0: + continue + elif ( + init_task.has_member("state") + and init_task.state.cast("unsigned int") != 0 + ): + continue + + # ASLR calculation + aslr_shift = ( + int.from_bytes( + init_task.files.cast("bytes", length=init_task.files.vol.size), + byteorder=init_task.files.vol.data_format.byteorder, + ) + - module.get_symbol("init_files").address + ) + if address_mask: + aslr_shift = aslr_shift & address_mask + + # KASLR calculation (physical symbol address - virtual symbol address) + kaslr_shift = init_task_address - init_task_json_address + + # Check ASLR and KASLR candidates + if aslr_shift & 0xFFF != 0 or kaslr_shift & 0xFFF != 0: + continue + vollog.debug( + f"Linux addresses shift values determined: KASLR (physical) = {hex(kaslr_shift)}, ASLR (virtual) = {hex(aslr_shift)}" + ) + return kaslr_shift, aslr_shift + + # We don't throw an exception, because we may legitimately not have an ASLR shift, but we report it + vollog.debug("Scanners could not determine any ASLR shifts, using 0 for both") + return 0, 0 + + +class LinuxIntelSubStacker: + __START_KERNEL_map_x64 = 0xFFFFFFFF80000000 + __START_KERNEL_map_x86 = 0xC0000000 + + def __init__(self, parent_stacker: LinuxStacker) -> None: + self.parent_stacker = parent_stacker + + def stack( + self, + context: interfaces.context.ContextInterface, layer_name: str, table: linux.LinuxKernelIntermedSymbols, table_name: str, @@ -134,21 +243,17 @@ def intel_stacker( else: dtb_symbol_name = "swapper_pg_dir" - kaslr_shift, aslr_shift = cls.find_aslr( + kaslr_shift, aslr_shift = self.parent_stacker.find_aslr( context, table_name, layer_name, - layer_class, progress_callback=progress_callback, ) - dtb = cls.virtual_to_physical_address( - table.get_symbol(dtb_symbol_name).address + kaslr_shift - ) + dtb = table.get_symbol(dtb_symbol_name).address + kaslr_shift # Build the new layer - context.config[cls.join(config_path, "page_map_offset")] = dtb - + context.config[path_join(config_path, "page_map_offset")] = dtb layer = layer_class( context, config_path=config_path, @@ -156,8 +261,10 @@ def intel_stacker( metadata={"os": "Linux"}, ) layer.config["kernel_virtual_offset"] = aslr_shift + + # Verify layer by translating the "linux_banner" symbol virtual address linux_banner_address = table.get_symbol("linux_banner").address + aslr_shift - test_banner_equality = cls.verify_translation_by_banner( + test_banner_equality = self.parent_stacker.verify_translation_by_banner( context=context, layer=layer, layer_name=layer_name, @@ -175,8 +282,25 @@ def intel_stacker( return None @classmethod - def aarch64_stacker( - cls, + def virtual_to_physical_address(cls, addr: int) -> int: + """Converts a virtual Intel Linux address to a physical one (does not account + of ASLR)""" + # Detect x64/x86 address space + if addr > cls.__START_KERNEL_map_x64: + return addr - cls.__START_KERNEL_map_x64 + return addr - cls.__START_KERNEL_map_x86 + + +class LinuxAArch64SubStacker: + # https://developer.arm.com/documentation/ddi0601/latest/AArch64-Registers/ + # Lowercase CPU register, bound to its attribute name in the "cpuinfo_arm64" kernel struct + _required_cpu_registers = {"aa64mmfr1_el1": "reg_id_aa64mmfr1"} + + def __init__(self, parent_stacker: LinuxStacker) -> None: + self.parent_stacker = parent_stacker + + def stack( + self, context: interfaces.context.ContextInterface, layer_name: str, table: linux.LinuxKernelIntermedSymbols, @@ -185,13 +309,12 @@ def aarch64_stacker( new_layer_name: str, banner: bytes, progress_callback: constants.ProgressCallback = None, - ) -> Optional[arm.AArch64]: - layer_class = arm.AArch64 - kaslr_shift, aslr_shift = cls.find_aslr( + ) -> Optional[arm.LinuxAArch64]: + layer_class = arm.LinuxAArch64 + kaslr_shift, aslr_shift = self.parent_stacker.find_aslr( context, table_name, layer_name, - layer_class, progress_callback=progress_callback, ) dtb = table.get_symbol("swapper_pg_dir").address + kaslr_shift @@ -277,7 +400,7 @@ def aarch64_stacker( ) layer.config["kernel_virtual_offset"] = aslr_shift - test_banner_equality = cls.verify_translation_by_banner( + test_banner_equality = self.parent_stacker.verify_translation_by_banner( context=context, layer=layer, layer_name=layer_name, @@ -286,6 +409,16 @@ def aarch64_stacker( ) if layer and dtb and test_banner_equality: + try: + cpu_registers = self.construct_cpu_registers_dict( + context=context, + layer_name=layer_name, + table_name=table_name, + kaslr_shift=kaslr_shift, + ) + layer.config["cpu_registers"] = json.dumps(cpu_registers) + except exceptions.SymbolError as e: + vollog.log(constants.LOGLEVEL_VVV, e, exc_info=True) vollog.debug(f"Kernel DTB was found at: 0x{dtb:0x}") vollog.debug("AArch64 image found") return layer @@ -295,118 +428,28 @@ def aarch64_stacker( return None @classmethod - def verify_translation_by_banner( - cls, - context: interfaces.context.ContextInterface, - layer, - layer_name: str, - linux_banner_address: int, - target_banner: bytes, - ) -> bool: - """Determine if a stacked layer is correct or a false positive, by calling the underlying - _translate method against the linux_banner symbol virtual address. Then, compare it with - the detected banner to verify the correct translation. - """ - - try: - banner_phys_address = layer._translate(linux_banner_address)[0] - banner_value = context.layers[layer_name].read( - banner_phys_address, len(target_banner) - ) - except exceptions.InvalidAddressException as e: - vollog.log( - constants.LOGLEVEL_VVVV, - 'Cannot translate "linux_banner" symbol virtual address.', - ) - return False - - if not banner_value == target_banner: - vollog.log( - constants.LOGLEVEL_VV, - f"Mismatch between scanned and virtually translated linux banner : {target_banner} != {banner_value}.", - ) - return False - - return True - - @classmethod - def find_aslr( + def construct_cpu_registers_dict( cls, context: interfaces.context.ContextInterface, - symbol_table: str, layer_name: str, - layer_class, - progress_callback: constants.ProgressCallback = None, - ) -> Tuple[int, int]: - """Determines the offset of the actual DTB in physical space and its - symbol offset.""" - init_task_symbol = symbol_table + constants.BANG + "init_task" - init_task_json_address = context.symbol_space.get_symbol( - init_task_symbol - ).address - swapper_signature = rb"swapper(\/0|\x00\x00)\x00\x00\x00\x00\x00\x00" - module = context.module(symbol_table, layer_name, 0) - address_mask = context.symbol_space[symbol_table].config.get( - "symbol_mask", None - ) - - task_symbol = module.get_type("task_struct") - comm_child_offset = task_symbol.relative_child_offset("comm") - - for offset in context.layers[layer_name].scan( - scanner=scanners.RegExScanner(swapper_signature), - context=context, - progress_callback=progress_callback, - ): - init_task_address = offset - comm_child_offset - init_task = module.object( - object_type="task_struct", offset=init_task_address, absolute=True - ) - if init_task.pid != 0: - continue - elif ( - init_task.has_member("state") - and init_task.state.cast("unsigned int") != 0 - ): - continue - - # This we get for free - aslr_shift = ( - int.from_bytes( - init_task.files.cast("bytes", length=init_task.files.vol.size), - byteorder=init_task.files.vol.data_format.byteorder, - ) - - module.get_symbol("init_files").address - ) - if layer_class == arm.AArch64: - kaslr_shift = init_task_address - init_task_json_address - else: - kaslr_shift = init_task_address - cls.virtual_to_physical_address( - init_task_json_address - ) - if address_mask: - aslr_shift = aslr_shift & address_mask - - if aslr_shift & 0xFFF != 0 or kaslr_shift & 0xFFF != 0: - continue - vollog.debug( - "Linux ASLR shift values determined: physical {:0x} virtual {:0x}".format( - kaslr_shift, aslr_shift + table_name: str, + kaslr_shift: int, + ) -> dict[str, int]: + + tmp_kernel_module = context.module(table_name, layer_name, kaslr_shift) + boot_cpu_data_struct = tmp_kernel_module.object_from_symbol("boot_cpu_data") + cpu_registers = {} + for cpu_reg, cpu_reg_attribute_name in cls._required_cpu_registers.items(): + try: + cpu_reg_value = getattr(boot_cpu_data_struct, cpu_reg_attribute_name) + cpu_registers[cpu_reg] = cpu_reg_value + except AttributeError: + vollog.log( + constants.LOGLEVEL_VVV, + f"boot_cpu_data struct does not include the {cpu_reg_attribute_name} field.", ) - ) - return kaslr_shift, aslr_shift - # We don't throw an exception, because we may legitimately not have an ASLR shift, but we report it - vollog.debug("Scanners could not determine any ASLR shifts, using 0 for both") - return 0, 0 - - @classmethod - def virtual_to_physical_address(cls, addr: int) -> int: - """Converts a virtual linux address to a physical one (does not account - of ASLR)""" - if addr > 0xFFFFFFFF80000000: - return addr - 0xFFFFFFFF80000000 - return addr - 0xC0000000 + return cpu_registers class LinuxSymbolFinder(symbol_finder.SymbolFinder): From 2e86c11fb221f52370c15c9a18118197f77ccd8b Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 7 Aug 2024 18:26:47 +0200 Subject: [PATCH 50/86] unify dtb debug statement --- volatility3/framework/automagic/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 5fb584e291..f173298547 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -419,7 +419,7 @@ def stack( layer.config["cpu_registers"] = json.dumps(cpu_registers) except exceptions.SymbolError as e: vollog.log(constants.LOGLEVEL_VVV, e, exc_info=True) - vollog.debug(f"Kernel DTB was found at: 0x{dtb:0x}") + vollog.debug(f"DTB was found at: {hex(dtb)}") vollog.debug("AArch64 image found") return layer else: From e6b68e71311770f400be6f644a30fd38d9414c2d Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 7 Aug 2024 18:31:53 +0200 Subject: [PATCH 51/86] use Dict instead of dict for type hinting --- volatility3/framework/automagic/linux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index f173298547..94d5c76348 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -5,7 +5,7 @@ import logging import os import json -from typing import Optional, Tuple, Union +from typing import Optional, Tuple, Union, Dict from volatility3.framework import constants, interfaces, exceptions from volatility3.framework.automagic import symbol_cache, symbol_finder @@ -434,7 +434,7 @@ def construct_cpu_registers_dict( layer_name: str, table_name: str, kaslr_shift: int, - ) -> dict[str, int]: + ) -> Dict[str, int]: tmp_kernel_module = context.module(table_name, layer_name, kaslr_shift) boot_cpu_data_struct = tmp_kernel_module.object_from_symbol("boot_cpu_data") From 40f78d6fb092bcf55b8abafcebb1d4517534e8ce Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 7 Aug 2024 18:36:18 +0200 Subject: [PATCH 52/86] handle explicit keyerror exception --- volatility3/framework/layers/arm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 8213149625..bd0a755127 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -528,7 +528,7 @@ def _read_cpu_register_field(self, register: str, field: str) -> int: return self._mask( self._cpu_regs[register], cpu_reg_mapping[0], cpu_reg_mapping[1] ) - except: + except KeyError: if not self._cpu_regs.get(register): raise KeyError( f"Access to CPU register {register} was requested, but the register value wasn't provided to this layer initially." From 287f0296aeabf1db7099cad29c69b9d190dd6e5d Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 10 Aug 2024 03:03:01 +0200 Subject: [PATCH 53/86] remove hardcoded register attribute --- volatility3/framework/automagic/linux.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 94d5c76348..44703ada7f 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -293,8 +293,10 @@ def virtual_to_physical_address(cls, addr: int) -> int: class LinuxAArch64SubStacker: # https://developer.arm.com/documentation/ddi0601/latest/AArch64-Registers/ - # Lowercase CPU register, bound to its attribute name in the "cpuinfo_arm64" kernel struct - _required_cpu_registers = {"aa64mmfr1_el1": "reg_id_aa64mmfr1"} + # CPU register, bound to its attribute name in the "cpuinfo_arm64" kernel struct + _optional_cpu_registers = { + arm.AArch64RegMap.ID_AA64MMFR1_EL1.__name__: "reg_id_aa64mmfr1" + } def __init__(self, parent_stacker: LinuxStacker) -> None: self.parent_stacker = parent_stacker @@ -439,7 +441,7 @@ def construct_cpu_registers_dict( tmp_kernel_module = context.module(table_name, layer_name, kaslr_shift) boot_cpu_data_struct = tmp_kernel_module.object_from_symbol("boot_cpu_data") cpu_registers = {} - for cpu_reg, cpu_reg_attribute_name in cls._required_cpu_registers.items(): + for cpu_reg, cpu_reg_attribute_name in cls._optional_cpu_registers.items(): try: cpu_reg_value = getattr(boot_cpu_data_struct, cpu_reg_attribute_name) cpu_registers[cpu_reg] = cpu_reg_value From a8a641b0a36adf068ad48b7c9423d95b175e7422 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 10 Aug 2024 03:05:29 +0200 Subject: [PATCH 54/86] prefer the use of constants to reference technical cpu registers --- volatility3/framework/layers/arm.py | 120 ++++++++++++++++++---------- 1 file changed, 80 insertions(+), 40 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index bd0a755127..3c83575315 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -6,12 +6,15 @@ import functools import collections import json +import inspect from typing import Optional, Dict, Any, List, Iterable, Tuple +from enum import Enum from volatility3 import classproperty from volatility3.framework import interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.layers import linear +from volatility3.framework.interfaces.configuration import path_join vollog = logging.getLogger(__name__) @@ -64,10 +67,6 @@ class AArch64(linear.LinearlyMappedLayer): 64: [(51, 42), (41, 29), (28, 16)], } - # (high_bit, low_bit) - # [1], see D19.2, page 6339 - _cpu_regs_mappings = {"aa64mmfr1_el1": {"HAFDBS": (3, 0)}} - def __init__( self, context: interfaces.context.ContextInterface, @@ -96,12 +95,15 @@ def __init__( try: self._cpu_regs: dict = json.loads(self._cpu_regs) except json.JSONDecodeError: - vollog.error( - 'Could not JSON deserialize "cpu_registers" layer requirement.' + raise json.JSONDecodeError( + 'Could not JSON deserialize provided "cpu_registers" layer requirement.' ) + self._cpu_regs_mapped = self._map_reg_values(self._cpu_regs) # Determine CPU features - self._cpu_features = {"feat_HAFDBS": self._get_cpu_feature_HAFDBS()} + self._cpu_features = { + AArch64Features.FEAT_HAFDBS.name: self._get_feature_HAFDBS() + } # Context : TTB0 (user) or TTB1 (kernel) self._virtual_addr_space = int( @@ -116,9 +118,7 @@ def __init__( smallest block of memory that can be described. Possibles values are 4, 16 or 64 (kB). """ - self._is_52bits = ( - True if self._ttbs_tnsz[self._virtual_addr_space] < 16 else False - ) + self._is_52bits = True if self._ttb_bitsize < 16 else False self._ttb_lookup_indexes = self._determine_ttb_lookup_indexes( self._ttb_granule, self._ttb_bitsize ) @@ -153,9 +153,7 @@ def _print_layer_debug_informations(self) -> None: f"Virtual addresses space range : {tuple([hex(x) for x in self._get_virtual_addr_range()])}" ) vollog.debug(f"Page size : {self._ttb_granule}") - vollog.debug( - f"T{self._virtual_addr_space}SZ : {self._ttbs_tnsz[self._virtual_addr_space]}" - ) + vollog.debug(f"T{self._virtual_addr_space}SZ : {self._ttb_bitsize}") vollog.debug(f"Page map offset : {hex(self._page_map_offset)}") vollog.debug(f"Translation mappings : {self._ttb_lookup_indexes}") @@ -472,7 +470,7 @@ def _page_is_dirty(self, entry: int) -> bool: 0 | Read/write 1 | Read-only """ - if self._cpu_features.get("feat_HAFDBS", None): + if self._cpu_features.get(AArch64Features.FEAT_HAFDBS.name): # Dirty Bit Modifier and Access Permissions bits # DBM == 1 and AP == 0 -> HW dirty state return bool((entry & (1 << 51)) and not (entry & (1 << 7))) @@ -499,9 +497,9 @@ def page_size(self) -> int: @property @functools.lru_cache() - def page_mask(cls) -> int: + def page_mask(self) -> int: """Page mask for this layer.""" - return ~(cls.page_size - 1) + return ~(self.page_size - 1) @classproperty @functools.lru_cache() @@ -520,41 +518,49 @@ def minimum_address(cls) -> int: def maximum_address(cls) -> int: return (1 << cls._maxvirtaddr) - 1 - def _read_cpu_register_field(self, register: str, field: str) -> int: - # Prefer speeding up read operations, as keys are most likely to be valid - # Handle errors in a dedicated except block, if needed + def _read_register_field(self, register_field: Enum) -> int: + reg_field_path = str(register_field) try: - cpu_reg_mapping = self._cpu_regs_mappings[register][field] - return self._mask( - self._cpu_regs[register], cpu_reg_mapping[0], cpu_reg_mapping[1] - ) + return self._cpu_regs_mapped[reg_field_path] except KeyError: - if not self._cpu_regs.get(register): - raise KeyError( - f"Access to CPU register {register} was requested, but the register value wasn't provided to this layer initially." - ) - - if not self._cpu_regs_mappings.get(register) or not self._cpu_regs_mappings[ - register - ].get(field): - raise KeyError( - f"Access of field {field} from CPU register {register} was requested, but no pre-defined mapping was found." - ) - - return None + raise KeyError( + f"{reg_field_path} register field wasn't provided to this layer initially." + ) - # CPU features - def _get_cpu_feature_HAFDBS(self): + def _get_feature_HAFDBS(self) -> bool: """ Hardware updates to Access flag and Dirty state in translation tables. [1], see D19.2.65, page 6784 """ try: - reg_HAFDBS = self._read_cpu_register_field("aa64mmfr1_el1", "HAFDBS") - return reg_HAFDBS >= 0b10 + field_HAFDBS = self._read_register_field( + AArch64RegMap.ID_AA64MMFR1_EL1.HAFDBS + ) + return field_HAFDBS >= 0b10 except KeyError: return None + @classmethod + def _map_reg_values(cls, registers_values: dict) -> dict: + """Generates a dict of dot joined AArch64 CPU registers and fields. + Iterates over every mapped register in AArch64RegMap, + check if a register value was provided to this layer, + mask every field accordingly and store the result. + + Example return value : {'ID_AA64MMFR1_EL1.HAFDBS': 2} + """ + + masked_trees = {} + for mm_cls_name, mm_cls in inspect.getmembers(AArch64RegMap, inspect.isclass): + if issubclass(mm_cls, Enum) and mm_cls_name in registers_values.keys(): + reg_value = registers_values[mm_cls_name] + for field in mm_cls: + dot_joined = path_join(mm_cls_name, field.name) + high_bit, low_bit = field.value + masked_value = cls._mask(reg_value, high_bit, low_bit) + masked_trees[dot_joined] = masked_value + return masked_trees + def canonicalize(self, addr: int) -> int: """Canonicalizes an address by performing an appropiate sign extension on the higher addresses""" if self._bits_per_register <= self._ttb_bitsize: @@ -682,3 +688,37 @@ def _page_is_dirty(self, entry: int) -> bool: class LinuxAArch64(LinuxAArch64Mixin, AArch64): pass + + +"""Avoid cluttering the layer code with static mappings.""" + + +class AArch64RegMap: + """ + List of static Enum's, binding fields (high bit, low bit) of AArch64 CPU registers. + Prevents the use of hardcoded string values by unifying everything here. + Contains only essential mappings, needed by the framework. + """ + + class TCR_EL1(Enum): + """TCR_EL1, Translation Control Register (EL1). + The control register for stage 1 of the EL1&0 translation regime. + [1], see D19.2.139, page 7071 + """ + + TG1 = (31, 30) + T1SZ = (21, 16) + TG0 = (15, 14) + T0SZ = (5, 0) + + class ID_AA64MMFR1_EL1(Enum): + """ID_AA64MMFR1_EL1, AArch64 Memory Model Feature Register 1. + [1], see D19.2.65, page 6781""" + + HAFDBS = (3, 0) + + +class AArch64Features(Enum): + """AArch64 CPU features.""" + + FEAT_HAFDBS = 1 From 0defdd64b1a482a7454ad183f27a1ddced8bc082 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sat, 10 Aug 2024 03:16:15 +0200 Subject: [PATCH 55/86] jsonerror raise fix --- volatility3/framework/layers/arm.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 3c83575315..675e28ccf3 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -94,9 +94,11 @@ def __init__( self._cpu_regs = self.config.get("cpu_registers", "{}") try: self._cpu_regs: dict = json.loads(self._cpu_regs) - except json.JSONDecodeError: + except json.JSONDecodeError as e: raise json.JSONDecodeError( - 'Could not JSON deserialize provided "cpu_registers" layer requirement.' + 'Could not JSON deserialize provided "cpu_registers" layer requirement.', + e.doc, + e.pos, ) self._cpu_regs_mapped = self._map_reg_values(self._cpu_regs) @@ -450,7 +452,7 @@ def is_dirty(self, offset: int) -> bool: def _page_is_dirty(self, entry: int) -> bool: """ - Hardware management of the dirty state (only > Armv8.1-A). + Hardware management of the dirty state (only >= Armv8.1-A). General documentation : https://developer.arm.com/documentation/102376/0200/Access-Flag/Dirty-state From 9d05c2ffd29d031a0e4ad53a6ecf023a0fb5b08e Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 15 Aug 2024 00:15:01 +0200 Subject: [PATCH 56/86] fix misplaced variables --- volatility3/framework/layers/arm.py | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 675e28ccf3..ee2003c903 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -127,9 +127,7 @@ def __init__( self._ttb_descriptor_bits = self._determine_ttb_descriptor_bits( self._ttb_granule, self._ttb_lookup_indexes, self._is_52bits ) - self._virtual_addr_range = self._get_virtual_addr_range()[ - self._virtual_addr_space - ] + self._virtual_addr_range = self._get_virtual_addr_range() self._canonical_prefix = self._mask( (1 << self._bits_per_register) - 1, self._bits_per_register, @@ -346,26 +344,8 @@ def _mapping( self, offset: int, length: int, ignore_errors: bool = False ) -> Iterable[Tuple[int, int, int, int, str]]: """Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer) - mappings. + mappings.This allows translation layers to provide maps of contiguous regions in one layer. - This allows translation layers to provide maps of contiguous - regions in one layer - """ - if length == 0: - try: - mapped_offset, _, layer_name = self._translate(offset) - if not self._context.layers[layer_name].is_valid(mapped_offset): - raise exceptions.InvalidAddressException( - layer_name=layer_name, invalid_address=mapped_offset - ) - except exceptions.InvalidAddressException: - if not ignore_errors: - raise - return None - yield offset, length, mapped_offset, length, layer_name - return None - while length > 0: - """ A bit of lexical definition : "page" means "virtual page" (i.e. a chunk of virtual address space) and "page frame" means "physical page" (i.e. a chunk of physical memory). What this is actually doing : @@ -402,11 +382,11 @@ def _mapping( """ if not ignore_errors: raise - # We can jump more if we know where the page fault failed + # We can jump more if we know where the page fault occured if isinstance(excp, exceptions.PagedInvalidAddressException): mask = (1 << excp.invalid_bits) - 1 else: - mask = (1 << (self._ttb_granule.bit_length() - 1)) - 1 + mask = (1 << self.page_shift) - 1 length_diff = mask + 1 - (offset & mask) length -= length_diff offset += length_diff From 96404f718111be19c4292693c441a25d2c2a7947 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 15 Aug 2024 00:15:23 +0200 Subject: [PATCH 57/86] precalculate getters --- volatility3/framework/layers/arm.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index ee2003c903..5bd0f1bbe5 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -141,6 +141,9 @@ def __init__( elif self._ttb_granule == 64: self._ta_51_x_bits = (15, 12) + self._page_size = self._ttb_granule * 1024 + self._page_size_in_bits = self._page_size.bit_length() - 1 + if self._layer_debug: self._print_layer_debug_informations() @@ -467,7 +470,7 @@ def page_shift(self) -> int: """Page shift for this layer, which is the page size bit length. - Typical values : 12, 14, 16 """ - return self.page_size.bit_length() - 1 + return self._page_size_in_bits @property @functools.lru_cache() @@ -475,7 +478,7 @@ def page_size(self) -> int: """Page size of this layer, in bytes. - Typical values : 4096, 16384, 65536 """ - return self._ttb_granule * 1024 + return self._page_size @property @functools.lru_cache() @@ -490,15 +493,15 @@ def bits_per_register(cls) -> int: AArch64TranslationLayer.""" return cls._bits_per_register - @classproperty + @property @functools.lru_cache() - def minimum_address(cls) -> int: - return 0 + def minimum_address(self) -> int: + return self._virtual_addr_range[0] - @classproperty + @property @functools.lru_cache() - def maximum_address(cls) -> int: - return (1 << cls._maxvirtaddr) - 1 + def maximum_address(self) -> int: + return self._virtual_addr_range[1] def _read_register_field(self, register_field: Enum) -> int: reg_field_path = str(register_field) From 07c416cb3e9f531e6fd0efba65452ac3e5311291 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 15 Aug 2024 00:17:25 +0200 Subject: [PATCH 58/86] move _mapping comment up --- volatility3/framework/layers/arm.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 5bd0f1bbe5..8b143cc1c7 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -366,7 +366,22 @@ def _mapping( -> 4096 - 0x0 = 4096 -> 0xffff800000f93000 + 4096 = 0xffff800000f94000 etc. while "length" > 0 - """ + """ + + if length == 0: + try: + mapped_offset, _, layer_name = self._translate(offset) + if not self._context.layers[layer_name].is_valid(mapped_offset): + raise exceptions.InvalidAddressException( + layer_name=layer_name, invalid_address=mapped_offset + ) + except exceptions.InvalidAddressException: + if not ignore_errors: + raise + return None + yield offset, length, mapped_offset, length, layer_name + return None + while length > 0: try: chunk_offset, page_size, layer_name = self._translate(offset) chunk_size = min(page_size - (chunk_offset % page_size), length) From 06688a1317b2c93be6a0817c69305d32610a40ef Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 16 Aug 2024 19:39:09 +0200 Subject: [PATCH 59/86] switch to a single cpu_registers requirements --- volatility3/framework/layers/arm.py | 94 +++++++++++------------------ 1 file changed, 36 insertions(+), 58 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 8b143cc1c7..77f1f10e5a 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -77,18 +77,6 @@ def __init__( super().__init__( context=context, config_path=config_path, name=name, metadata=metadata ) - self._kernel_endianness = self.config["kernel_endianness"] - self._layer_debug = self.config.get("layer_debug", False) - self._translation_debug = self.config.get("translation_debug", False) - self._base_layer = self.config["memory_layer"] - # self._swap_layers = [] # TODO - self._page_map_offset = self.config["page_map_offset"] - self._page_map_offset_kernel = self.config["page_map_offset_kernel"] - self._ttbs_tnsz = [self.config["tcr_el1_t0sz"], self.config["tcr_el1_t1sz"]] - self._ttbs_granules = [ - self.config["page_size_user_space"], - self.config["page_size_kernel_space"], - ] # Unserialize cpu_registers config attribute into a dict self._cpu_regs = self.config.get("cpu_registers", "{}") @@ -102,14 +90,31 @@ def __init__( ) self._cpu_regs_mapped = self._map_reg_values(self._cpu_regs) - # Determine CPU features - self._cpu_features = { - AArch64Features.FEAT_HAFDBS.name: self._get_feature_HAFDBS() - } + self._kernel_endianness = self.config["kernel_endianness"] + self._layer_debug = self.config.get("layer_debug", False) + self._translation_debug = self.config.get("translation_debug", False) + self._base_layer = self.config["memory_layer"] + # self._swap_layers = [] # TODO + self._page_map_offset = self.config["page_map_offset"] + self._page_map_offset_kernel = self._read_register_field( + AArch64RegMap.TTBR1_EL1.BADDR + ) + self._ttbs_tnsz = [ + self._read_register_field(AArch64RegMap.TCR_EL1.T0SZ), + self._read_register_field(AArch64RegMap.TCR_EL1.T1SZ), + ] + self._ttbs_granules = [ + AArch64RegFieldValues._get_ttbr0_el1_granule_size( + self._read_register_field(AArch64RegMap.TCR_EL1.TG0) + ), + AArch64RegFieldValues._get_ttbr1_el1_granule_size( + self._read_register_field(AArch64RegMap.TCR_EL1.TG1) + ), + ] # Context : TTB0 (user) or TTB1 (kernel) self._virtual_addr_space = int( - self._page_map_offset == self.config["page_map_offset_kernel"] + self._page_map_offset == self._page_map_offset_kernel ) # [1], see D8.1.9, page 5818 @@ -144,6 +149,13 @@ def __init__( self._page_size = self._ttb_granule * 1024 self._page_size_in_bits = self._page_size.bit_length() - 1 + # CPU features + hafdbs = self._read_register_field(AArch64RegMap.ID_AA64MMFR1_EL1.HAFDBS, True) + if hafdbs: + self._feat_hafdbs = AArch64RegFieldValues._get_feature_HAFDBS(hafdbs) + else: + self._feat_hafdbs = None + if self._layer_debug: self._print_layer_debug_informations() @@ -470,7 +482,7 @@ def _page_is_dirty(self, entry: int) -> bool: 0 | Read/write 1 | Read-only """ - if self._cpu_features.get(AArch64Features.FEAT_HAFDBS.name): + if self._feat_hafdbs: # Dirty Bit Modifier and Access Permissions bits # DBM == 1 and AP == 0 -> HW dirty state return bool((entry & (1 << 51)) and not (entry & (1 << 7))) @@ -518,28 +530,19 @@ def minimum_address(self) -> int: def maximum_address(self) -> int: return self._virtual_addr_range[1] - def _read_register_field(self, register_field: Enum) -> int: + def _read_register_field( + self, register_field: Enum, ignore_errors: bool = False + ) -> int: reg_field_path = str(register_field) try: return self._cpu_regs_mapped[reg_field_path] except KeyError: + if ignore_errors: + return None raise KeyError( f"{reg_field_path} register field wasn't provided to this layer initially." ) - def _get_feature_HAFDBS(self) -> bool: - """ - Hardware updates to Access flag and Dirty state in translation tables. - [1], see D19.2.65, page 6784 - """ - try: - field_HAFDBS = self._read_register_field( - AArch64RegMap.ID_AA64MMFR1_EL1.HAFDBS - ) - return field_HAFDBS >= 0b10 - except KeyError: - return None - @classmethod def _map_reg_values(cls, registers_values: dict) -> dict: """Generates a dict of dot joined AArch64 CPU registers and fields. @@ -607,31 +610,6 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] optional=False, description='DTB of the target context (either "kernel space" or "user space process").', ), - requirements.IntRequirement( - name="page_map_offset_kernel", - optional=False, - description="DTB of the kernel space, it is primarily used to determine the target context of the layer (page_map_offset == page_map_offset_kernel). Conveniently calculated by LinuxStacker.", - ), - requirements.IntRequirement( - name="tcr_el1_t0sz", - optional=False, - description="The size offset of the memory region addressed by TTBR0_EL1. Conveniently calculated by LinuxStacker.", - ), - requirements.IntRequirement( - name="tcr_el1_t1sz", - optional=False, - description="The size offset of the memory region addressed by TTBR1_EL1. Conveniently calculated by LinuxStacker.", - ), - requirements.IntRequirement( - name="page_size_user_space", - optional=False, - description="Page size used by the user address space. Conveniently calculated by LinuxStacker.", - ), - requirements.IntRequirement( - name="page_size_kernel_space", - optional=False, - description="Page size used by the kernel address space. Conveniently calculated by LinuxStacker.", - ), requirements.ChoiceRequirement( choices=["little", "big"], name="kernel_endianness", @@ -640,7 +618,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] ), requirements.StringRequirement( name="cpu_registers", - optional=True, + optional=False, description="Serialized dict of cpu register keys bound to their corresponding value. Needed for specific (non-mandatory) uses (ex: dirty bit management).", default="{}", ), From ec5758a74c4ada0478300845f105af7c1b0a941d Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 16 Aug 2024 19:39:44 +0200 Subject: [PATCH 60/86] enhance registers mappings --- volatility3/framework/layers/arm.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 77f1f10e5a..e68afc17dd 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -685,9 +685,33 @@ class TCR_EL1(Enum): """ TG1 = (31, 30) + "Granule size for the TTBR1_EL1." T1SZ = (21, 16) + "The size offset of the memory region addressed by TTBR1_EL1. The region size is 2**(64-T1SZ) bytes." TG0 = (15, 14) + "Granule size for the TTBR0_EL1." T0SZ = (5, 0) + "The size offset of the memory region addressed by TTBR0_EL1. The region size is 2**(64-T0SZ) bytes." + + class TTBR0_EL1(Enum): + """TTBR0_EL1, Translation Table Base Register 0 (EL1) + Holds the base address of the translation table for the initial lookup for stage 1 of the translation of an address from the lower VA range in the EL1&0 translation regime, and other information for this translation regime. [1], see D19.2.155, page 7152 + [1], see D19.2.152, page 7139 + """ + + ASID = (63, 48) + BADDR = (47, 1) + CnP = (0, 0) + + class TTBR1_EL1(Enum): + """TTBR1_EL1, Translation Table Base Register 1 (EL1) + Holds the base address of the translation table for the initial lookup for stage 1 of the translation of an address from the higher VA range in the EL1&0 stage 1 translation regime, and other information for this translation regime. + [1], see D19.2.155, page 7152 + """ + + ASID = (63, 48) + BADDR = (47, 1) + CnP = (0, 0) class ID_AA64MMFR1_EL1(Enum): """ID_AA64MMFR1_EL1, AArch64 Memory Model Feature Register 1. From 927365b853367200ff672d9d73c4a0057f56c8f0 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 16 Aug 2024 19:39:55 +0200 Subject: [PATCH 61/86] add register manipulation function --- volatility3/framework/layers/arm.py | 91 ++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index e68afc17dd..bbbf78c48d 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -718,9 +718,94 @@ class ID_AA64MMFR1_EL1(Enum): [1], see D19.2.65, page 6781""" HAFDBS = (3, 0) + "Hardware updates to Access flag and Dirty state in translation tables." -class AArch64Features(Enum): - """AArch64 CPU features.""" +class AArch64RegFieldValues: + @classmethod + def _table_lookup( + cls, value: int, lookup_table: dict, reverse_lookup: bool = False + ): + if reverse_lookup: + lookup_table = {v: k for k, v in lookup_table.items()} + if lookup_table.get(value, None) != None: + return lookup_table[value] + else: + raise KeyError( + f"Value {value} could not be mapped inside lookup_table : {lookup_table}" + ) + + @classmethod + def _get_feature_HAFDBS(cls, value: int) -> bool: + """ + Hardware updates to Access flag and Dirty state in translation tables. + [1], see D19.2.65, page 6784 + """ + return value >= 0b10 + + @classmethod + def _get_ttbr0_el1_granule_size(cls, value: int, reverse_lookup: bool = False): + """ + Granule size for the TTBR0_EL1. + """ + lookup_table = { + 0b00: 4, # 4kB + 0b01: 64, # 64kB + 0b10: 16, # 16kB + } + return cls._table_lookup(value, lookup_table, reverse_lookup) + + @classmethod + def _get_ttbr1_el1_granule_size( + cls, value: int, reverse_lookup: bool = False + ) -> Optional[int]: + """ + Granule size for the TTBR1_EL1. + """ + lookup_table = { + 0b01: 16, # 16kB + 0b10: 4, # 4kB + 0b11: 64, # 64kB + } + return cls._table_lookup(value, lookup_table, reverse_lookup) + + +def set_reg_bits(value: int, reg_field: Enum, reg_value: int = 0) -> int: + """Sets the bits from high_bit to low_bit (inclusive) in current_value to the given value. + + Args: + value: The value to set in the specified bit range. + reg_field: The register field to update, inside the register. + reg_value: The register value to modify (default is 0). + + Returns: + The modified integer with the specified bits set. + + Raises: + ValueError: If the value is too large to fit in the specified bit range. + """ + high_bit = reg_field.value[1] + low_bit = reg_field.value[0] + + # Calculate the number of bits to set + num_bits = low_bit - high_bit + 1 + + # Calculate the maximum value that can fit in the specified number of bits + max_value = (1 << num_bits) - 1 + + # Check if the value can fit in the specified bit range + if value > max_value: + raise ValueError( + f"Value {value} is too large to fit in {num_bits} bits (max value is {max_value})." + ) + + # Create a mask for the bit range + mask = (1 << num_bits) - 1 + + # Clear the bits in the range in the current value + reg_value &= ~(mask << high_bit) + + # Set the bits with the new value + reg_value |= (value & mask) << high_bit - FEAT_HAFDBS = 1 + return reg_value From 20fcc68521fd19ff9799316fceeceb5451585ed3 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 16 Aug 2024 19:40:51 +0200 Subject: [PATCH 62/86] switch to a single cpu_registers requirements --- volatility3/framework/automagic/linux.py | 40 +++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 44703ada7f..a4edd2b431 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -320,8 +320,8 @@ def stack( progress_callback=progress_callback, ) dtb = table.get_symbol("swapper_pg_dir").address + kaslr_shift + ttbr1_el1 = arm.set_reg_bits(dtb, arm.AArch64RegMap.TTBR1_EL1.BADDR) context.config[path_join(config_path, "page_map_offset")] = dtb - context.config[path_join(config_path, "page_map_offset_kernel")] = dtb kernel_endianness = table.get_type("pointer").vol.data_format.byteorder context.config[path_join(config_path, "kernel_endianness")] = kernel_endianness @@ -378,20 +378,37 @@ def stack( """ va_bits_candidates = [va_bits] + [va_bits + i for i in range(-1, -4, -1)] for va_bits in va_bits_candidates: - tcr_el1_t1sz = 64 - va_bits - # T1SZ is considered to be equal to T0SZ - context.config[path_join(config_path, "tcr_el1_t1sz")] = tcr_el1_t1sz - context.config[path_join(config_path, "tcr_el1_t0sz")] = tcr_el1_t1sz + cpu_registers = {} + tcr_el1 = 0 + # T1SZ is considered to equal to T0SZ + tcr_el1 = arm.set_reg_bits( + 64 - va_bits, arm.AArch64RegMap.TCR_EL1.T1SZ, tcr_el1 + ) + tcr_el1 = arm.set_reg_bits( + 64 - va_bits, arm.AArch64RegMap.TCR_EL1.T0SZ, tcr_el1 + ) # If "_kernel_flags_le*" aren't in the symbols, we can still do a quick bruteforce on [4,16,64] page sizes # False positives cannot happen, as translation indexes will be off on a wrong page size for page_size_kernel_space in page_size_kernel_space_candidates: # Kernel space page size is considered equal to the user space page size - context.config[path_join(config_path, "page_size_kernel_space")] = ( - page_size_kernel_space + tcr_el1_tg1 = arm.AArch64RegFieldValues._get_ttbr1_el1_granule_size( + page_size_kernel_space, True + ) + tcr_el1_tg0 = arm.AArch64RegFieldValues._get_ttbr0_el1_granule_size( + page_size_kernel_space, True ) - context.config[path_join(config_path, "page_size_user_space")] = ( - page_size_kernel_space + tcr_el1 = arm.set_reg_bits( + tcr_el1_tg1, arm.AArch64RegMap.TCR_EL1.TG1, tcr_el1 + ) + tcr_el1 = arm.set_reg_bits( + tcr_el1_tg0, arm.AArch64RegMap.TCR_EL1.TG0, tcr_el1 + ) + + cpu_registers[arm.AArch64RegMap.TCR_EL1.__name__] = tcr_el1 + cpu_registers[arm.AArch64RegMap.TTBR1_EL1.__name__] = ttbr1_el1 + context.config[path_join(config_path, "cpu_registers")] = json.dumps( + cpu_registers ) # Build layer layer = layer_class( @@ -412,12 +429,13 @@ def stack( if layer and dtb and test_banner_equality: try: - cpu_registers = self.construct_cpu_registers_dict( + optional_cpu_registers = self.extract_cpu_registers( context=context, layer_name=layer_name, table_name=table_name, kaslr_shift=kaslr_shift, ) + cpu_registers.update(optional_cpu_registers) layer.config["cpu_registers"] = json.dumps(cpu_registers) except exceptions.SymbolError as e: vollog.log(constants.LOGLEVEL_VVV, e, exc_info=True) @@ -430,7 +448,7 @@ def stack( return None @classmethod - def construct_cpu_registers_dict( + def extract_cpu_registers( cls, context: interfaces.context.ContextInterface, layer_name: str, From 5798394264cb88db2a1be1f180914cb2161ebeb2 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 18 Aug 2024 02:07:15 +0200 Subject: [PATCH 63/86] arm comments, and typos --- volatility3/framework/layers/arm.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index bbbf78c48d..ea75bd25ee 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -33,6 +33,11 @@ Definitions : The OS-controlled translation is called stage 1 translation, and the hypervisor-controlled translation is called stage 2 translation. + +Notes : + If hardware management of the dirty state is enabled, the DBM bit is set to 1. ([1], D8.4.6) + If hardware management of the Access Flag bit is not enabled, software must implement it. ([1], D8.4.5) + Access Permissions bits can be updated by hardware in some situations, but is mostly managed by software. ([1], D8.4.3) """ @@ -502,7 +507,7 @@ def page_shift(self) -> int: @property @functools.lru_cache() def page_size(self) -> int: - """Page size of this layer, in bytes. + """Page size for this layer, in bytes. - Typical values : 4096, 16384, 65536 """ return self._page_size @@ -772,6 +777,7 @@ def _get_ttbr1_el1_granule_size( def set_reg_bits(value: int, reg_field: Enum, reg_value: int = 0) -> int: """Sets the bits from high_bit to low_bit (inclusive) in current_value to the given value. + Allows to manipulate the bits at arbitrary positions inside a register. Args: value: The value to set in the specified bit range. From 2bb84d6ad59f82cb27e38eb1c6f86c80fef4cf95 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 18 Aug 2024 02:07:41 +0200 Subject: [PATCH 64/86] wrong page_mask calc (negative int) --- volatility3/framework/layers/arm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index ea75bd25ee..f905ba1ab9 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -516,7 +516,7 @@ def page_size(self) -> int: @functools.lru_cache() def page_mask(self) -> int: """Page mask for this layer.""" - return ~(self.page_size - 1) + return self.page_size - 1 @classproperty @functools.lru_cache() From f7614e38bc756261b63e1ad3542d8e7545e8eb5a Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 18 Aug 2024 02:17:45 +0200 Subject: [PATCH 65/86] initial windowsaarch64mixin and windowsaarch64 --- volatility3/framework/layers/arm.py | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index f905ba1ab9..62338bd6c5 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -673,6 +673,49 @@ class LinuxAArch64(LinuxAArch64Mixin, AArch64): pass +class WindowsAArch64Mixin(AArch64): + def _page_is_dirty(self, entry: int) -> bool: + """Returns whether a particular page is dirty based on its (page table) entry. + The bit indicates that its associated block of memory + has been modified and has not been saved to storage yet. + + The following is based on the Windows kernel function MiMarkPteDirty(). + Windows software DBM bit is located at offset 56, and does not account + of hardware bit 51. + """ + return bool((entry & (1 << 56)) and not (entry & (1 << 7))) + + +class WindowsAArch64(LinuxAArch64Mixin, AArch64): + """Windows AArch64 page size is constant, and statically defined in + CmSiGetPageSize() kernel function. + + Takes advantage of the @classproperty, as @property is dynamic + and breaks static accesses in windows automagic. + """ + + @classproperty + @functools.lru_cache() + def page_shift(self) -> int: + """Page shift for this layer, which is the page size bit length.""" + return 12 + + @classproperty + @functools.lru_cache() + def page_size(self) -> int: + """Page size for this layer, in bytes. + Prefer returning the value directly, instead of adding an additional + "_page_size" constant that could cause confusion with the parent class. + """ + return 0x1000 + + @classproperty + @functools.lru_cache() + def page_mask(self) -> int: + """Page mask for this layer.""" + return self.page_size - 1 + + """Avoid cluttering the layer code with static mappings.""" From 47f8a91196ee77d34ede316a980379433ff6b1bd Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 18 Aug 2024 02:21:21 +0200 Subject: [PATCH 66/86] interrupt on missing cpu_registers config --- volatility3/framework/layers/arm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 62338bd6c5..8f25c8d12e 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -84,7 +84,7 @@ def __init__( ) # Unserialize cpu_registers config attribute into a dict - self._cpu_regs = self.config.get("cpu_registers", "{}") + self._cpu_regs = self.config["cpu_registers"] try: self._cpu_regs: dict = json.loads(self._cpu_regs) except json.JSONDecodeError as e: From 7dced24235f35fd13bae9868b7a59f63f978658f Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 20 Aug 2024 12:32:41 +0200 Subject: [PATCH 67/86] wrong windows parent class --- volatility3/framework/layers/arm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 8f25c8d12e..f3f615e252 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -686,7 +686,7 @@ def _page_is_dirty(self, entry: int) -> bool: return bool((entry & (1 << 56)) and not (entry & (1 << 7))) -class WindowsAArch64(LinuxAArch64Mixin, AArch64): +class WindowsAArch64(WindowsAArch64Mixin, AArch64): """Windows AArch64 page size is constant, and statically defined in CmSiGetPageSize() kernel function. From 9f609f13afdea72729ad146b0c5a2a8c459d8c55 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 21 Aug 2024 18:15:21 +0200 Subject: [PATCH 68/86] switch kernel_endianness req to struct format --- volatility3/framework/layers/arm.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index f3f615e252..33ecdd2544 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -7,6 +7,7 @@ import collections import json import inspect +import struct from typing import Optional, Dict, Any, List, Iterable, Tuple from enum import Enum @@ -95,7 +96,7 @@ def __init__( ) self._cpu_regs_mapped = self._map_reg_values(self._cpu_regs) - self._kernel_endianness = self.config["kernel_endianness"] + self._entry_format = self.config["entry_format"] self._layer_debug = self.config.get("layer_debug", False) self._translation_debug = self.config.get("translation_debug", False) self._base_layer = self.config["memory_layer"] @@ -153,6 +154,8 @@ def __init__( self._page_size = self._ttb_granule * 1024 self._page_size_in_bits = self._page_size.bit_length() - 1 + self._entry_size = struct.calcsize(self._entry_format) + self._entry_number = self._page_size // self._entry_size # CPU features hafdbs = self._read_register_field(AArch64RegMap.ID_AA64MMFR1_EL1.HAFDBS, True) @@ -250,11 +253,11 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: max_level = len(self._ttb_lookup_indexes) - 1 for level, (high_bit, low_bit) in enumerate(self._ttb_lookup_indexes): index = self._mask(virtual_offset, high_bit, low_bit) - descriptor = int.from_bytes( + (descriptor,) = struct.unpack( + self._entry_format, base_layer.read( table_address + (index * self._register_size), self._register_size ), - byteorder=self._kernel_endianness, ) table_address = 0 # Bits 51->x need to be extracted from the descriptor @@ -615,11 +618,10 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] optional=False, description='DTB of the target context (either "kernel space" or "user space process").', ), - requirements.ChoiceRequirement( - choices=["little", "big"], - name="kernel_endianness", + requirements.StringRequirement( + name="entry_format", optional=False, - description="Kernel endianness (little or big)", + description='Format and byte order of table descriptors, represented in the "struct" format.', ), requirements.StringRequirement( name="cpu_registers", From 2e8a467ffcb4c45331cca26ababc08136fc20575 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 21 Aug 2024 18:15:39 +0200 Subject: [PATCH 69/86] add bogus descriptor tables detection --- volatility3/framework/layers/arm.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 33ecdd2544..3a2e0ffd04 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -281,6 +281,13 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: ) << self._ttb_descriptor_bits[1] ) + if self._get_valid_table(table_address) == None: + raise exceptions.PagedInvalidAddressException( + layer_name=self.name, + invalid_address=virtual_offset, + invalid_bits=low_bit, + entry=descriptor, + ) # Block descriptor elif level < max_level and descriptor_type == 0b01: table_address |= ( @@ -319,6 +326,18 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: return table_address, low_bit, descriptor + @functools.lru_cache(1025) + def _get_valid_table(self, base_address: int) -> Optional[bytes]: + """Extracts the translation table, validates it and returns it if it's valid.""" + table = self._context.layers.read( + self._base_layer, base_address, self.page_size + ) + # If the table is entirely duplicates, then mark the whole table as bad + if table == table[: self._entry_size] * self._entry_number: + return None + + return table + def mapping( self, offset: int, length: int, ignore_errors: bool = False ) -> Iterable[Tuple[int, int, int, int, str]]: From b3279f1e6b66e2154ef3ab03c869daa8b47450f8 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 21 Aug 2024 18:16:18 +0200 Subject: [PATCH 70/86] switch kernel_endianness req to struct format --- volatility3/framework/automagic/linux.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index a4edd2b431..3c9188fd5f 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -5,6 +5,7 @@ import logging import os import json +import struct from typing import Optional, Tuple, Union, Dict from volatility3.framework import constants, interfaces, exceptions @@ -322,8 +323,15 @@ def stack( dtb = table.get_symbol("swapper_pg_dir").address + kaslr_shift ttbr1_el1 = arm.set_reg_bits(dtb, arm.AArch64RegMap.TTBR1_EL1.BADDR) context.config[path_join(config_path, "page_map_offset")] = dtb - kernel_endianness = table.get_type("pointer").vol.data_format.byteorder - context.config[path_join(config_path, "kernel_endianness")] = kernel_endianness + entry_format = ( + "<" + if table.get_type("pointer").vol.data_format.byteorder == "little" + else "big" + ) + entry_format += ( + "Q" if table.get_type("pointer").vol.data_format.length == 8 else "I" + ) + context.config[path_join(config_path, "entry_format")] = entry_format # CREDIT : https://github.com/crash-utility/crash/blob/28891d1127542dbb2d5ba16c575e14e741ed73ef/arm64.c#L941 kernel_flags = 0 @@ -350,9 +358,9 @@ def stack( table.get_symbol("vabits_actual").address + kaslr_shift ) # Linux source : v6.7/source/arch/arm64/Kconfig#L1263, VA_BITS - va_bits = int.from_bytes( + (va_bits,) = struct.unpack( + entry_format, context.layers[layer_name].read(vabits_actual_phys_addr, 8), - kernel_endianness, ) if not va_bits: """ From d706bd709dfbaae8dad9b382c3cdca049eb2424a Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 22 Aug 2024 10:41:44 +0200 Subject: [PATCH 71/86] use is instead of == None --- volatility3/framework/layers/arm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 3a2e0ffd04..9614d802f0 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -281,7 +281,7 @@ def _translate_entry(self, virtual_offset: int) -> Tuple[int, int, int]: ) << self._ttb_descriptor_bits[1] ) - if self._get_valid_table(table_address) == None: + if self._get_valid_table(table_address) is None: raise exceptions.PagedInvalidAddressException( layer_name=self.name, invalid_address=virtual_offset, From c5afe391df7764105d9d6ccc5d168105264e0200 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 30 Aug 2024 18:45:21 +0200 Subject: [PATCH 72/86] cache find_aslr() --- volatility3/framework/automagic/linux.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 3c9188fd5f..9ee77dc797 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -6,6 +6,7 @@ import os import json import struct +import functools from typing import Optional, Tuple, Union, Dict from volatility3.framework import constants, interfaces, exceptions @@ -149,6 +150,7 @@ def verify_translation_by_banner( return True @classmethod + @functools.lru_cache() def find_aslr( cls, context: interfaces.context.ContextInterface, From 9676330bd7bb931c8055247495a96c8e2cc8a318 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 30 Aug 2024 18:46:36 +0200 Subject: [PATCH 73/86] add subclass granularity to logging --- volatility3/framework/automagic/linux.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 9ee77dc797..b501a7f5dc 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -107,6 +107,7 @@ def stack( progress_callback=progress_callback, ) if layer: + vollog.debug(f"Detected {layer.__class__.__name__} layer") return layer except Exception as e: vollog.exception(e) @@ -122,6 +123,7 @@ def verify_translation_by_banner( layer_name: str, linux_banner_address: int, target_banner: bytes, + logger: logging.Logger = vollog, ) -> bool: """Determine if a stacked layer is correct or a false positive, by calling the underlying _translate method against the linux_banner symbol virtual address. Then, compare it with @@ -134,14 +136,14 @@ def verify_translation_by_banner( banner_phys_address, len(target_banner) ) except exceptions.InvalidAddressException: - vollog.log( + logger.log( constants.LOGLEVEL_VVVV, - 'Cannot translate "linux_banner" symbol virtual address.', + 'Unable to translate "linux_banner" symbol virtual address.', ) return False if not banner_value == target_banner: - vollog.log( + logger.log( constants.LOGLEVEL_VV, f"Mismatch between scanned and virtually translated linux banner : {target_banner} != {banner_value}.", ) @@ -218,6 +220,7 @@ def find_aslr( class LinuxIntelSubStacker: + _logger = logging.getLogger(f"{__module__}.{__qualname__}") __START_KERNEL_map_x64 = 0xFFFFFFFF80000000 __START_KERNEL_map_x86 = 0xC0000000 @@ -273,11 +276,11 @@ def stack( layer_name=layer_name, linux_banner_address=linux_banner_address, target_banner=banner, + logger=self._logger, ) if layer and dtb and test_banner_equality: - vollog.debug(f"DTB was found at: {hex(dtb)}") - vollog.debug("Intel image found") + self._logger.debug(f"DTB was found at: {hex(dtb)}") return layer else: layer.destroy() @@ -295,6 +298,7 @@ def virtual_to_physical_address(cls, addr: int) -> int: class LinuxAArch64SubStacker: + _logger = logging.getLogger(f"{__module__}.{__qualname__}") # https://developer.arm.com/documentation/ddi0601/latest/AArch64-Registers/ # CPU register, bound to its attribute name in the "cpuinfo_arm64" kernel struct _optional_cpu_registers = { @@ -435,6 +439,7 @@ def stack( layer_name=layer_name, linux_banner_address=linux_banner_address, target_banner=banner, + logger=self._logger, ) if layer and dtb and test_banner_equality: @@ -448,9 +453,8 @@ def stack( cpu_registers.update(optional_cpu_registers) layer.config["cpu_registers"] = json.dumps(cpu_registers) except exceptions.SymbolError as e: - vollog.log(constants.LOGLEVEL_VVV, e, exc_info=True) - vollog.debug(f"DTB was found at: {hex(dtb)}") - vollog.debug("AArch64 image found") + self._logger.log(constants.LOGLEVEL_VVV, e, exc_info=True) + self._logger.debug(f"DTB was found at: {hex(dtb)}") return layer else: layer.destroy() @@ -474,7 +478,7 @@ def extract_cpu_registers( cpu_reg_value = getattr(boot_cpu_data_struct, cpu_reg_attribute_name) cpu_registers[cpu_reg] = cpu_reg_value except AttributeError: - vollog.log( + cls._logger.log( constants.LOGLEVEL_VVV, f"boot_cpu_data struct does not include the {cpu_reg_attribute_name} field.", ) From c4766c2ba04b81efab31f415861a64f687d2daf1 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 30 Aug 2024 19:03:43 +0200 Subject: [PATCH 74/86] use TTB1 terminology instead of DTB for aarch64 --- volatility3/framework/automagic/linux.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index b501a7f5dc..81ca7c1ab4 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -326,9 +326,9 @@ def stack( layer_name, progress_callback=progress_callback, ) - dtb = table.get_symbol("swapper_pg_dir").address + kaslr_shift - ttbr1_el1 = arm.set_reg_bits(dtb, arm.AArch64RegMap.TTBR1_EL1.BADDR) - context.config[path_join(config_path, "page_map_offset")] = dtb + ttb1 = table.get_symbol("swapper_pg_dir").address + kaslr_shift + ttbr1_el1 = arm.set_reg_bits(ttb1, arm.AArch64RegMap.TTBR1_EL1.BADDR) + context.config[path_join(config_path, "page_map_offset")] = ttb1 entry_format = ( "<" if table.get_type("pointer").vol.data_format.byteorder == "little" @@ -442,7 +442,7 @@ def stack( logger=self._logger, ) - if layer and dtb and test_banner_equality: + if layer and ttb1 and test_banner_equality: try: optional_cpu_registers = self.extract_cpu_registers( context=context, @@ -454,7 +454,7 @@ def stack( layer.config["cpu_registers"] = json.dumps(cpu_registers) except exceptions.SymbolError as e: self._logger.log(constants.LOGLEVEL_VVV, e, exc_info=True) - self._logger.debug(f"DTB was found at: {hex(dtb)}") + self._logger.debug(f"TTB1 was found at: {hex(ttb1)}") return layer else: layer.destroy() From cb69983d1c7465061c434ea8513ad3ae2b8ada60 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 30 Aug 2024 23:43:47 +0200 Subject: [PATCH 75/86] fix cpu_registers description --- volatility3/framework/layers/arm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 9614d802f0..045e59e9b0 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -645,7 +645,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] requirements.StringRequirement( name="cpu_registers", optional=False, - description="Serialized dict of cpu register keys bound to their corresponding value. Needed for specific (non-mandatory) uses (ex: dirty bit management).", + description="Serialized dict of cpu register keys bound to their corresponding value.", default="{}", ), requirements.BooleanRequirement( From 537b404215ed5b2b95815d095649a7dbf21546eb Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Fri, 30 Aug 2024 23:49:10 +0200 Subject: [PATCH 76/86] add ttbr comments --- volatility3/framework/layers/arm.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 045e59e9b0..00f572a767 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -769,8 +769,11 @@ class TTBR0_EL1(Enum): """ ASID = (63, 48) + "An ASID for the translation table base address." BADDR = (47, 1) + "Translation table base address." CnP = (0, 0) + "Common not Private." class TTBR1_EL1(Enum): """TTBR1_EL1, Translation Table Base Register 1 (EL1) @@ -779,8 +782,11 @@ class TTBR1_EL1(Enum): """ ASID = (63, 48) + "An ASID for the translation table base address." BADDR = (47, 1) + "Translation table base address." CnP = (0, 0) + "Common not Private." class ID_AA64MMFR1_EL1(Enum): """ID_AA64MMFR1_EL1, AArch64 Memory Model Feature Register 1. @@ -840,7 +846,7 @@ def _get_ttbr1_el1_granule_size( def set_reg_bits(value: int, reg_field: Enum, reg_value: int = 0) -> int: - """Sets the bits from high_bit to low_bit (inclusive) in current_value to the given value. + """Sets the bits from high_bit to low_bit (inclusive) in "reg_value" to the given value. Allows to manipulate the bits at arbitrary positions inside a register. Args: From 9f17302c9bf09849fd01415692e27c3fed965ba4 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 5 Sep 2024 19:35:26 +0200 Subject: [PATCH 77/86] use explicit cpu requirements, to prevent confusion --- volatility3/framework/layers/arm.py | 73 ++++++++++++++++++----------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 00f572a767..717583cc49 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -83,19 +83,19 @@ def __init__( super().__init__( context=context, config_path=config_path, name=name, metadata=metadata ) + self._cpu_regs = {} + for register in [ + AArch64RegMap.TCR_EL1.__name__, + AArch64RegMap.TTBR1_EL1.__name__, + AArch64RegMap.ID_AA64MMFR1_EL1.__name__, + ]: + # Sanity check for optional registers. + # Missing required CPU registers will have + # previously raised a layer requirement exception. + if self.config.get(register): + self._cpu_regs[register] = self.config[register] - # Unserialize cpu_registers config attribute into a dict - self._cpu_regs = self.config["cpu_registers"] - try: - self._cpu_regs: dict = json.loads(self._cpu_regs) - except json.JSONDecodeError as e: - raise json.JSONDecodeError( - 'Could not JSON deserialize provided "cpu_registers" layer requirement.', - e.doc, - e.pos, - ) self._cpu_regs_mapped = self._map_reg_values(self._cpu_regs) - self._entry_format = self.config["entry_format"] self._layer_debug = self.config.get("layer_debug", False) self._translation_debug = self.config.get("translation_debug", False) @@ -118,7 +118,7 @@ def __init__( ), ] - # Context : TTB0 (user) or TTB1 (kernel) + # Context : TTB0 (user space) or TTB1 (kernel space) self._virtual_addr_space = int( self._page_map_offset == self._page_map_offset_kernel ) @@ -126,18 +126,31 @@ def __init__( # [1], see D8.1.9, page 5818 self._ttb_bitsize = 64 - self._ttbs_tnsz[self._virtual_addr_space] self._ttb_granule = self._ttbs_granules[self._virtual_addr_space] + self._page_size = self._ttb_granule * 1024 + self._page_size_in_bits = self._page_size.bit_length() - 1 """ Translation Table Granule is in fact the page size, as it is the smallest block of memory that can be described. Possibles values are 4, 16 or 64 (kB). """ + + # 52 bits VA detection self._is_52bits = True if self._ttb_bitsize < 16 else False + # [1], see D8.3, page 5852 + if self._is_52bits: + if self._ttb_granule in [4, 16]: + self._ta_51_x_bits = (9, 8) + elif self._ttb_granule == 64: + self._ta_51_x_bits = (15, 12) + + # Translation indexes calculations self._ttb_lookup_indexes = self._determine_ttb_lookup_indexes( self._ttb_granule, self._ttb_bitsize ) self._ttb_descriptor_bits = self._determine_ttb_descriptor_bits( self._ttb_granule, self._ttb_lookup_indexes, self._is_52bits ) + self._virtual_addr_range = self._get_virtual_addr_range() self._canonical_prefix = self._mask( (1 << self._bits_per_register) - 1, @@ -145,15 +158,6 @@ def __init__( self._ttb_bitsize, ) - # [1], see D8.3, page 5852 - if self._is_52bits: - if self._ttb_granule in [4, 16]: - self._ta_51_x_bits = (9, 8) - elif self._ttb_granule == 64: - self._ta_51_x_bits = (15, 12) - - self._page_size = self._ttb_granule * 1024 - self._page_size_in_bits = self._page_size.bit_length() - 1 self._entry_size = struct.calcsize(self._entry_format) self._entry_number = self._page_size // self._entry_size @@ -577,7 +581,10 @@ def _map_reg_values(cls, registers_values: dict) -> dict: check if a register value was provided to this layer, mask every field accordingly and store the result. - Example return value : {'ID_AA64MMFR1_EL1.HAFDBS': 2} + Example return value : + {'TCR_EL1.TG1': 3, 'TCR_EL1.T1SZ': 12, 'TCR_EL1.TG0': 1, + 'TCR_EL1.T0SZ': 12, 'TTBR1_EL1.ASID': 0, 'TTBR1_EL1.BADDR': 1092419584, + 'TTBR1_EL1.CnP': 0} """ masked_trees = {} @@ -642,12 +649,6 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] optional=False, description='Format and byte order of table descriptors, represented in the "struct" format.', ), - requirements.StringRequirement( - name="cpu_registers", - optional=False, - description="Serialized dict of cpu register keys bound to their corresponding value.", - default="{}", - ), requirements.BooleanRequirement( name="layer_debug", optional=True, @@ -668,6 +669,22 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] optional=True, description="Kernel unique identifier, including compiler name and version, kernel version, compile time.", ), + requirements.IntRequirement( + name=AArch64RegMap.TCR_EL1.__name__, + optional=False, + description="TCR_EL1 register", + ), + requirements.IntRequirement( + name=AArch64RegMap.TTBR1_EL1.__name__, + optional=False, + description="TTBR1_EL1 register", + ), + requirements.IntRequirement( + name=AArch64RegMap.ID_AA64MMFR1_EL1.__name__, + optional=True, + description="ID_AA64MMFR1_EL1 register", + default=None, + ), ] From d18496f0f90e9aae89d84cf8f25551d1906bacc9 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 5 Sep 2024 19:40:14 +0200 Subject: [PATCH 78/86] switch to explicit cpu requirements --- volatility3/framework/automagic/linux.py | 42 +++++++++++++----------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 81ca7c1ab4..ea32bac6c7 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -7,7 +7,7 @@ import json import struct import functools -from typing import Optional, Tuple, Union, Dict +from typing import Optional, Tuple, Union, Dict, List from volatility3.framework import constants, interfaces, exceptions from volatility3.framework.automagic import symbol_cache, symbol_finder @@ -327,7 +327,9 @@ def stack( progress_callback=progress_callback, ) ttb1 = table.get_symbol("swapper_pg_dir").address + kaslr_shift - ttbr1_el1 = arm.set_reg_bits(ttb1, arm.AArch64RegMap.TTBR1_EL1.BADDR) + context.config[path_join(config_path, arm.AArch64RegMap.TTBR1_EL1.__name__)] = ( + arm.set_reg_bits(ttb1, arm.AArch64RegMap.TTBR1_EL1.BADDR) + ) context.config[path_join(config_path, "page_map_offset")] = ttb1 entry_format = ( "<" @@ -392,9 +394,8 @@ def stack( """ va_bits_candidates = [va_bits] + [va_bits + i for i in range(-1, -4, -1)] for va_bits in va_bits_candidates: - cpu_registers = {} + # T1SZ is considered equal to T0SZ tcr_el1 = 0 - # T1SZ is considered to equal to T0SZ tcr_el1 = arm.set_reg_bits( 64 - va_bits, arm.AArch64RegMap.TCR_EL1.T1SZ, tcr_el1 ) @@ -418,12 +419,10 @@ def stack( tcr_el1 = arm.set_reg_bits( tcr_el1_tg0, arm.AArch64RegMap.TCR_EL1.TG0, tcr_el1 ) + context.config[ + path_join(config_path, arm.AArch64RegMap.TCR_EL1.__name__) + ] = tcr_el1 - cpu_registers[arm.AArch64RegMap.TCR_EL1.__name__] = tcr_el1 - cpu_registers[arm.AArch64RegMap.TTBR1_EL1.__name__] = ttbr1_el1 - context.config[path_join(config_path, "cpu_registers")] = json.dumps( - cpu_registers - ) # Build layer layer = layer_class( context, @@ -444,14 +443,18 @@ def stack( if layer and ttb1 and test_banner_equality: try: - optional_cpu_registers = self.extract_cpu_registers( + optional_cpu_registers = self.get_registers_from_cpuinfo_arm64( context=context, layer_name=layer_name, table_name=table_name, + registers=self._optional_cpu_registers, kaslr_shift=kaslr_shift, ) - cpu_registers.update(optional_cpu_registers) - layer.config["cpu_registers"] = json.dumps(cpu_registers) + layer_req = [req.name for req in layer.get_requirements()] + for reg, reg_val in optional_cpu_registers.items(): + # Verify register's presence in the requirements + if reg in layer_req: + layer.config[reg] = reg_val except exceptions.SymbolError as e: self._logger.log(constants.LOGLEVEL_VVV, e, exc_info=True) self._logger.debug(f"TTB1 was found at: {hex(ttb1)}") @@ -462,28 +465,29 @@ def stack( return None @classmethod - def extract_cpu_registers( + def get_registers_from_cpuinfo_arm64( cls, context: interfaces.context.ContextInterface, layer_name: str, table_name: str, + registers: Dict[str, str], kaslr_shift: int, ) -> Dict[str, int]: tmp_kernel_module = context.module(table_name, layer_name, kaslr_shift) boot_cpu_data_struct = tmp_kernel_module.object_from_symbol("boot_cpu_data") - cpu_registers = {} - for cpu_reg, cpu_reg_attribute_name in cls._optional_cpu_registers.items(): + results = {} + for reg, reg_attribute_name in registers.items(): try: - cpu_reg_value = getattr(boot_cpu_data_struct, cpu_reg_attribute_name) - cpu_registers[cpu_reg] = cpu_reg_value + cpu_reg_value = getattr(boot_cpu_data_struct, reg_attribute_name) + results[reg] = cpu_reg_value except AttributeError: cls._logger.log( constants.LOGLEVEL_VVV, - f"boot_cpu_data struct does not include the {cpu_reg_attribute_name} field.", + f"boot_cpu_data struct does not include the {reg_attribute_name} field.", ) - return cpu_registers + return results class LinuxSymbolFinder(symbol_finder.SymbolFinder): From e1ffb7e8997ae003550013d1b5a98ddcd01a53f3 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 5 Sep 2024 19:41:12 +0200 Subject: [PATCH 79/86] remove json import --- volatility3/framework/layers/arm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 717583cc49..198648e75d 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -5,7 +5,6 @@ import logging import functools import collections -import json import inspect import struct from typing import Optional, Dict, Any, List, Iterable, Tuple From 74b30cc8da0bafacdfa621c5270aac3132e35dec Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Mon, 9 Sep 2024 18:30:39 +0200 Subject: [PATCH 80/86] remove unused imports --- volatility3/framework/automagic/linux.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index ea32bac6c7..bb8494d2a4 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -4,10 +4,9 @@ import logging import os -import json import struct import functools -from typing import Optional, Tuple, Union, Dict, List +from typing import Optional, Tuple, Union, Dict from volatility3.framework import constants, interfaces, exceptions from volatility3.framework.automagic import symbol_cache, symbol_finder From b682f49abb0eda95f0c24bc4e9bf12968fcfe886 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 10 Sep 2024 14:12:49 +0200 Subject: [PATCH 81/86] remove aslr_shift from va calculations --- volatility3/framework/automagic/linux.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index bb8494d2a4..e5e7b8461d 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -134,10 +134,10 @@ def verify_translation_by_banner( banner_value = context.layers[layer_name].read( banner_phys_address, len(target_banner) ) - except exceptions.InvalidAddressException: + except exceptions.InvalidAddressException as e: logger.log( constants.LOGLEVEL_VVVV, - 'Unable to translate "linux_banner" symbol virtual address.', + f'Unable to translate "linux_banner" symbol virtual address : {e}', ) return False @@ -268,12 +268,11 @@ def stack( layer.config["kernel_virtual_offset"] = aslr_shift # Verify layer by translating the "linux_banner" symbol virtual address - linux_banner_address = table.get_symbol("linux_banner").address + aslr_shift test_banner_equality = self.parent_stacker.verify_translation_by_banner( context=context, layer=layer, layer_name=layer_name, - linux_banner_address=linux_banner_address, + linux_banner_address=table.get_symbol("linux_banner").address + aslr_shift, target_banner=banner, logger=self._logger, ) @@ -357,7 +356,6 @@ def stack( else [4, 16, 64] ) - linux_banner_address = table.get_symbol("linux_banner").address + aslr_shift # Linux source : v6.7/source/arch/arm64/include/asm/memory.h#L186 - v5.7/source/arch/arm64/include/asm/memory.h#L160 va_bits = 0 if "vabits_actual" in table.symbols: @@ -373,10 +371,12 @@ def stack( """ Count leftmost bits equal to 1, deduce number of used bits for virtual addressing. Example : - linux_banner_address = 0xffffffd733aae820 = 0b1111111111111111111111111101011100110011101010101110100000100000 - va_bits = (linux_banner_address ^ (2**64 - 1)).bit_length() + 1 = 39 + linux_banner_address = 0xffff800081822f08 = 0b1111111111111111100000000000000010000001100000100010111100001000 + va_bits = (linux_banner_address ^ (2**64 - 1)).bit_length() + 1 = 48 """ - va_bits = (linux_banner_address ^ (2**64 - 1)).bit_length() + 1 + va_bits = ( + table.get_symbol("linux_banner").address ^ (2**64 - 1) + ).bit_length() + 1 """ Determining the number of useful bits in virtual addresses (VA_BITS) @@ -384,7 +384,7 @@ def stack( Calculation by masking works great, but not in every case, due to the AArch64 memory layout, sometimes pushing kernel addresses "too far" from the TTB1 start. See https://www.kernel.org/doc/html/v5.5/arm64/memory.html. - Errors are by 1 or 2 bits, so we can try va_bits - {1,2,3}. + Errors are by 1 or 2 bits, so we can safely try va_bits - {1,2,3}. Example, assuming the good va_bits value is 39 : # Case where calculation was correct : 1 iteration va_bits_candidates = [**39**, 38, 37, 36] @@ -403,7 +403,7 @@ def stack( ) # If "_kernel_flags_le*" aren't in the symbols, we can still do a quick bruteforce on [4,16,64] page sizes - # False positives cannot happen, as translation indexes will be off on a wrong page size + # False positives cannot happen, as translation indexes will be off on wrong page and va sizes for page_size_kernel_space in page_size_kernel_space_candidates: # Kernel space page size is considered equal to the user space page size tcr_el1_tg1 = arm.AArch64RegFieldValues._get_ttbr1_el1_granule_size( @@ -435,7 +435,8 @@ def stack( context=context, layer=layer, layer_name=layer_name, - linux_banner_address=linux_banner_address, + linux_banner_address=table.get_symbol("linux_banner").address + + aslr_shift, target_banner=banner, logger=self._logger, ) From 59b58a526e94db716128b6c2cd7081ec7b8261aa Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Thu, 12 Sep 2024 16:36:11 +0200 Subject: [PATCH 82/86] handle no kaslr_shift --- volatility3/framework/automagic/linux.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index e5e7b8461d..e179d63e06 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -254,8 +254,12 @@ def stack( layer_name, progress_callback=progress_callback, ) - - dtb = table.get_symbol(dtb_symbol_name).address + kaslr_shift + dtb = table.get_symbol(dtb_symbol_name).address + dtb = ( + dtb + kaslr_shift + if kaslr_shift != 0 + else self.virtual_to_physical_address(dtb) + ) # Build the new layer context.config[path_join(config_path, "page_map_offset")] = dtb From 046c806a7ca1ca0f917a6fb0e553d8bd21257757 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 25 Sep 2024 11:13:42 +0200 Subject: [PATCH 83/86] introduce an explicit class constant for win page size --- volatility3/framework/layers/arm.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 198648e75d..86395386b3 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -724,33 +724,31 @@ def _page_is_dirty(self, entry: int) -> bool: class WindowsAArch64(WindowsAArch64Mixin, AArch64): - """Windows AArch64 page size is constant, and statically defined in - CmSiGetPageSize() kernel function. - - Takes advantage of the @classproperty, as @property is dynamic - and breaks static accesses in windows automagic. + """As Windows (AArch64) page size is constant, + we take advantage of the @classproperty decorator, + because @property is dynamic and breaks static accesses + in windows automagic. """ + _windows_fixed_page_size = 0x1000 + @classproperty @functools.lru_cache() - def page_shift(self) -> int: + def page_shift(cls) -> int: """Page shift for this layer, which is the page size bit length.""" - return 12 + return cls.page_size.bit_length() - 1 @classproperty @functools.lru_cache() - def page_size(self) -> int: - """Page size for this layer, in bytes. - Prefer returning the value directly, instead of adding an additional - "_page_size" constant that could cause confusion with the parent class. - """ - return 0x1000 + def page_size(cls) -> int: + """Page size for this layer, in bytes.""" + return cls._windows_fixed_page_size @classproperty @functools.lru_cache() - def page_mask(self) -> int: + def page_mask(cls) -> int: """Page mask for this layer.""" - return self.page_size - 1 + return cls.page_size - 1 """Avoid cluttering the layer code with static mappings.""" From d1e26c9646c319457727c957605797692c3fa8d9 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Wed, 25 Sep 2024 12:40:31 +0200 Subject: [PATCH 84/86] correct page_mask calculation --- volatility3/framework/layers/arm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 86395386b3..87509a4852 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -541,7 +541,7 @@ def page_size(self) -> int: @functools.lru_cache() def page_mask(self) -> int: """Page mask for this layer.""" - return self.page_size - 1 + return ~(self.page_size - 1) @classproperty @functools.lru_cache() @@ -748,7 +748,7 @@ def page_size(cls) -> int: @functools.lru_cache() def page_mask(cls) -> int: """Page mask for this layer.""" - return cls.page_size - 1 + return ~(cls.page_size - 1) """Avoid cluttering the layer code with static mappings.""" From b302885fa0d01bd4cc751d33e3767d9a530f0547 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Tue, 8 Oct 2024 23:10:09 +0200 Subject: [PATCH 85/86] enhance dirty state management references --- volatility3/framework/layers/arm.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 87509a4852..3fbcee0607 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -511,6 +511,9 @@ def _page_is_dirty(self, entry: int) -> bool: -------|------------------ 0 | Read/write 1 | Read-only + + AF=0. Region not accessed. + AF=1. Region accessed. """ if self._feat_hafdbs: # Dirty Bit Modifier and Access Permissions bits @@ -518,7 +521,7 @@ def _page_is_dirty(self, entry: int) -> bool: return bool((entry & (1 << 51)) and not (entry & (1 << 7))) else: raise NotImplementedError( - "Hardware updates to Access flag and Dirty state in translation tables are not available in the target kernel. Please try using a software based implementation of dirty bit management." + "Hardware updates to Access flag and Dirty state in translation tables are not available. Please try using a software based implementation of dirty state management." ) @property @@ -695,6 +698,7 @@ def _page_is_dirty(self, entry: int) -> bool: The following is based on Linux software AArch64 dirty bit management. [2], see arch/arm64/include/asm/pgtable-prot.h#L18 + [2], see pte_wrprotect() [3], see page 12-25 https://lkml.org/lkml/2023/7/7/77 -> Linux implementation detail """ @@ -717,10 +721,19 @@ def _page_is_dirty(self, entry: int) -> bool: has been modified and has not been saved to storage yet. The following is based on the Windows kernel function MiMarkPteDirty(). - Windows software DBM bit is located at offset 56, and does not account - of hardware bit 51. + Capabilities are initialized in MiInitializeSystemDefaults(). + When marking a PTE dirty, MiMarkPteDirty() will: + - set AF to 1 (ACCESSED) manually if hardware does not support it; + - set AP to 0 (RW) manually if hardware does not support it. + In the end, we need to detect if the DBM is software (56) or hardware (51), + and if in any scenario the AP bit is set to 0. """ - return bool((entry & (1 << 56)) and not (entry & (1 << 7))) + sw_dirty = bool((entry & (1 << 56)) and not (entry & (1 << 7))) + try: + hw_dirty = super()._page_is_dirty(entry) + return sw_dirty or hw_dirty + except NotImplementedError: + return sw_dirty class WindowsAArch64(WindowsAArch64Mixin, AArch64): From 5ee05ce7b440c0d6a0d45f7f9a343b702c7b38e4 Mon Sep 17 00:00:00 2001 From: Abyss Watcher Date: Sun, 20 Oct 2024 21:37:37 +0200 Subject: [PATCH 86/86] fix canonicalization --- volatility3/framework/layers/arm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/volatility3/framework/layers/arm.py b/volatility3/framework/layers/arm.py index 3fbcee0607..36d4190b79 100644 --- a/volatility3/framework/layers/arm.py +++ b/volatility3/framework/layers/arm.py @@ -155,6 +155,7 @@ def __init__( (1 << self._bits_per_register) - 1, self._bits_per_register, self._ttb_bitsize, + 0, ) self._entry_size = struct.calcsize(self._entry_format) @@ -606,7 +607,7 @@ def canonicalize(self, addr: int) -> int: return addr & self.address_mask elif addr < (1 << self._ttb_bitsize - 1): return addr - return self._mask(addr, self._ttb_bitsize, 0) + self._canonical_prefix + return self._mask(addr, self._ttb_bitsize, 0) | self._canonical_prefix def decanonicalize(self, addr: int) -> int: """Removes canonicalization to ensure an adress fits within the correct range if it has been canonicalized