From 0ad8054f125c360afadcd25db70c1f67d8265e57 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 7 Nov 2024 20:26:26 +1100 Subject: [PATCH 1/8] intel layer: minor improve lru_cache argument --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 2b0df53728..33d5432cc2 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -252,7 +252,7 @@ def _translate_entry(self, offset: int) -> Tuple[int, int]: return entry, position - @functools.lru_cache(1025) + @functools.lru_cache(maxsize=1025) def _get_valid_table(self, base_address: int) -> Optional[bytes]: """Extracts the table, validates it and returns it if it's valid.""" table = self._context.layers.read( From 9ae7c2bb109681bbd8b6cffc5a1f5b7f1dfa71cf Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 7 Nov 2024 20:28:21 +1100 Subject: [PATCH 2/8] Data layer interface: Convert address_mask to a cached property --- volatility3/framework/interfaces/layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index e2a68780a7..78687d8d5d 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -136,7 +136,7 @@ def maximum_address(self) -> int: def minimum_address(self) -> int: """Returns the minimum valid address of the space.""" - @property + @functools.cached_property def address_mask(self) -> int: """Returns a mask which encapsulates all the active bits of an address for this layer.""" From 73d4f2fc88e15ac038f962cb5f5b11acf7855091 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Thu, 7 Nov 2024 20:31:20 +1100 Subject: [PATCH 3/8] Linux: Add support for PROT_NONE, Intel Side Channel Vulnerability L1TF changes and fix _maxphyaddr in x86-64 --- volatility3/framework/automagic/linux.py | 6 +- .../framework/constants/linux/__init__.py | 8 ++ volatility3/framework/layers/intel.py | 100 ++++++++++++++++-- 3 files changed, 105 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 52a73f45a8..27e07e564d 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -80,14 +80,14 @@ def stack( context, table_name, layer_name, progress_callback=progress_callback ) - layer_class: Type = intel.Intel if "init_top_pgt" in table.symbols: - layer_class = intel.Intel32e + layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_top_pgt" elif "init_level4_pgt" in table.symbols: - layer_class = intel.Intel32e + layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_level4_pgt" else: + layer_class = intel.LinuxIntel dtb_symbol_name = "swapper_pg_dir" dtb = cls.virtual_to_physical_address( diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 7c485d3c39..303c534ed7 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -11,6 +11,14 @@ """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" +# Translation Layer constants +PAGE_BIT_PRESENT = 0 +PAGE_BIT_PSE = 7 # Page Size Extension: 4 MB (or 2MB) page +PAGE_BIT_PROTNONE = 8 +PAGE_BIT_PAT_LARGE = 12 # 2MB or 1GB pages +PAGE_PRESENT = 1 << PAGE_BIT_PRESENT +PAGE_PROTNONE = 1 << PAGE_BIT_PROTNONE + # include/linux/sched.h PF_KTHREAD = 0x00200000 # I'm a kernel thread diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 33d5432cc2..e6d20d9922 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -13,6 +13,7 @@ from volatility3.framework import exceptions, interfaces, constants from volatility3.framework.configuration import requirements from volatility3.framework.layers import linear +from volatility3.framework.constants import linux as linux_constants vollog = logging.getLogger(__name__) @@ -163,12 +164,17 @@ def _translate(self, offset: int) -> Tuple[int, int, str]: entry, f"Page Fault at entry {hex(entry)} in page entry", ) - page = self._mask(entry, self._maxphyaddr - 1, position + 1) | self._mask( - offset, position, 0 - ) + + pfn = self.pte_pfn(entry) + page_offset = self._mask(offset, position, 0) + page = pfn << self.page_shift | page_offset return page, 1 << (position + 1), self._base_layer + def pte_pfn(self, entry: int) -> int: + """Extracts the page frame number (PFN) from the page table entry (PTE) entry""" + return entry >> self.page_shift + def _translate_entry(self, offset: int) -> Tuple[int, int]: """Translates a specific offset based on paging tables. @@ -203,10 +209,10 @@ def _translate_entry(self, offset: int) -> Tuple[int, int]: "Page Fault at entry " + hex(entry) + " in table " + name, ) # Check if we're a large page - if large_page and (entry & (1 << 7)): + if large_page and (entry & (1 << linux_constants.PAGE_BIT_PSE)): # Mask off the PAT bit - if entry & (1 << 12): - entry -= 1 << 12 + if entry & (1 << linux_constants.PAGE_BIT_PAT_LARGE): + entry -= 1 << linux_constants.PAGE_BIT_PAT_LARGE # We're a large page, the rest is finished below # If we want to implement PSE-36, it would need to be done here break @@ -501,3 +507,85 @@ class WindowsIntel32e(WindowsMixin, Intel32e): def _translate(self, offset: int) -> Tuple[int, int, str]: return self._translate_swap(self, offset, self._bits_per_register // 2) + + +class LinuxMixin(Intel): + @functools.cached_property + def register_mask(self) -> int: + return (1 << self._bits_per_register) - 1 + + @functools.cached_property + def physical_mask(self) -> int: + # From kernels 4.18 the physical mask is dynamic: See AMD SME, Intel Multi-Key Total + # Memory Encryption and CONFIG_DYNAMIC_PHYSICAL_MASK: 94d49eb30e854c84d1319095b5dd0405a7da9362 + physical_mask = (1 << self._maxphyaddr) - 1 + # TODO: Come back once SME support is available in the framework + return physical_mask + + @functools.cached_property + def page_mask(self) -> int: + # Note that within the Intel class it's a class method. However, since it uses + # complement operations and we are working in Python, it would be more careful to + # limit it to the architecture's pointer size. + return ~(self.page_size - 1) & self.register_mask + + @functools.cached_property + def physical_page_mask(self) -> int: + return self.page_mask & self.physical_mask + + @functools.cached_property + def pte_pfn_mask(self) -> int: + return self.physical_page_mask + + @functools.cached_property + def pte_flags_mask(self) -> int: + return ~self.pte_pfn_mask & self.register_mask + + def pte_flags(self, pte) -> int: + return pte & self.pte_flags_mask + + def is_pte_present(self, entry: int) -> bool: + return ( + self.pte_flags(entry) + & (linux_constants.PAGE_PRESENT | linux_constants.PAGE_PROTNONE) + ) != 0 + + def _page_is_valid(self, entry: int) -> bool: + # Overrides the Intel static method with the Linux-specific implementation + return self.is_pte_present(entry) + + def pte_needs_invert(self, entry) -> bool: + # Entries that were set to PROT_NONE (PAGE_PRESENT/PAGE_GLOBAL) are inverted + return not (entry & linux_constants.PAGE_PRESENT) + + def protnone_mask(self, entry: int) -> int: + """Gets a mask to XOR with the page table entry to get the correct PFN""" + return ~0 & self.register_mask if self.pte_needs_invert(entry) else 0 + + def pte_pfn(self, entry: int) -> int: + """Extracts the page frame number from the page table entry""" + pfn = entry ^ self.protnone_mask(entry) + return (pfn & self.pte_pfn_mask) >> self.page_shift + + +class LinuxIntel(LinuxMixin, Intel): + pass + + +class LinuxIntelPAE(LinuxMixin, IntelPAE): + pass + + +class LinuxIntel32e(LinuxMixin, Intel32e): + # In the Linux kernel, the __PHYSICAL_MASK_SHIFT is a mask used to extract the + # physical address from a PTE. In Volatility3, this is referred to as _maxphyaddr. + # + # Until kernel version 4.17, Linux x86-64 used a 46-bit mask. With commit + # b83ce5ee91471d19c403ff91227204fb37c95fb2, this was extended to 52 bits, + # applying to both 4 and 5-level page tables. + # + # We initially used 52 bits for all Intel 64-bit systems, but this produced incorrect + # results for PROT_NONE pages. Since the mask value is defined by a preprocessor macro, + # it's difficult to detect the exact bit shift used in the current kernel. + # Using 46 bits has proven reliable for our use case, as seen in tools like crashtool. + _maxphyaddr = 46 From 05e8f8ff1075adbb398076a7d91701ba256b8d8a Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 13 Nov 2024 12:20:03 +1100 Subject: [PATCH 4/8] intel layer: Move constants to the Intel class --- .../framework/constants/linux/__init__.py | 8 ------- volatility3/framework/layers/intel.py | 24 ++++++++++++------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index 303c534ed7..7c485d3c39 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -11,14 +11,6 @@ """The value hard coded from the Linux Kernel (hence not extracted from the layer itself)""" -# Translation Layer constants -PAGE_BIT_PRESENT = 0 -PAGE_BIT_PSE = 7 # Page Size Extension: 4 MB (or 2MB) page -PAGE_BIT_PROTNONE = 8 -PAGE_BIT_PAT_LARGE = 12 # 2MB or 1GB pages -PAGE_PRESENT = 1 << PAGE_BIT_PRESENT -PAGE_PROTNONE = 1 << PAGE_BIT_PROTNONE - # include/linux/sched.h PF_KTHREAD = 0x00200000 # I'm a kernel thread diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index e6d20d9922..25a98fc21e 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -13,7 +13,6 @@ from volatility3.framework import exceptions, interfaces, constants from volatility3.framework.configuration import requirements from volatility3.framework.layers import linear -from volatility3.framework.constants import linux as linux_constants vollog = logging.getLogger(__name__) @@ -23,6 +22,16 @@ class Intel(linear.LinearlyMappedLayer): """Translation Layer for the Intel IA32 memory mapping.""" + _PAGE_BIT_PRESENT = 0 + _PAGE_BIT_PSE = 7 # Page Size Extension: 4 MB (or 2MB) page + _PAGE_BIT_PROTNONE = 8 + _PAGE_BIT_PAT_LARGE = 12 # 2MB or 1GB pages + + _PAGE_PRESENT = 1 << _PAGE_BIT_PRESENT + _PAGE_PSE = 1 << _PAGE_BIT_PSE + _PAGE_PROTNONE = 1 << _PAGE_BIT_PROTNONE + _PAGE_PAT_LARGE = 1 << _PAGE_BIT_PAT_LARGE + _entry_format = " Tuple[int, int]: "Page Fault at entry " + hex(entry) + " in table " + name, ) # Check if we're a large page - if large_page and (entry & (1 << linux_constants.PAGE_BIT_PSE)): + if large_page and (entry & self._PAGE_PSE): # Mask off the PAT bit - if entry & (1 << linux_constants.PAGE_BIT_PAT_LARGE): - entry -= 1 << linux_constants.PAGE_BIT_PAT_LARGE + if entry & self._PAGE_PAT_LARGE: + entry -= self._PAGE_PAT_LARGE # We're a large page, the rest is finished below # If we want to implement PSE-36, it would need to be done here break @@ -545,10 +554,7 @@ def pte_flags(self, pte) -> int: return pte & self.pte_flags_mask def is_pte_present(self, entry: int) -> bool: - return ( - self.pte_flags(entry) - & (linux_constants.PAGE_PRESENT | linux_constants.PAGE_PROTNONE) - ) != 0 + return (self.pte_flags(entry) & (self._PAGE_PRESENT | self._PAGE_PROTNONE)) != 0 def _page_is_valid(self, entry: int) -> bool: # Overrides the Intel static method with the Linux-specific implementation @@ -556,7 +562,7 @@ def _page_is_valid(self, entry: int) -> bool: def pte_needs_invert(self, entry) -> bool: # Entries that were set to PROT_NONE (PAGE_PRESENT/PAGE_GLOBAL) are inverted - return not (entry & linux_constants.PAGE_PRESENT) + return not (entry & self._PAGE_PRESENT) def protnone_mask(self, entry: int) -> int: """Gets a mask to XOR with the page table entry to get the correct PFN""" From 36c19aa6d91b561ab57db34eb6c6a2ba7bbe2316 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 13 Nov 2024 12:21:03 +1100 Subject: [PATCH 5/8] core: Bump framework minor version --- volatility3/framework/constants/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index ce803a6877..55ef19e4b1 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,6 +1,6 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 11 # Number of changes that only add to the interface +VERSION_MINOR = 12 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" From 173e35105a6fc614e0ae329331e8e21e63f52f14 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 13 Nov 2024 12:30:11 +1100 Subject: [PATCH 6/8] LinuxMixin: Make new methods internal --- volatility3/framework/layers/intel.py | 46 ++++++++++++++------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 25a98fc21e..dc88c20bf3 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -174,13 +174,13 @@ def _translate(self, offset: int) -> Tuple[int, int, str]: f"Page Fault at entry {hex(entry)} in page entry", ) - pfn = self.pte_pfn(entry) + pfn = self._pte_pfn(entry) page_offset = self._mask(offset, position, 0) page = pfn << self.page_shift | page_offset return page, 1 << (position + 1), self._base_layer - def pte_pfn(self, entry: int) -> int: + def _pte_pfn(self, entry: int) -> int: """Extracts the page frame number (PFN) from the page table entry (PTE) entry""" return entry >> self.page_shift @@ -520,11 +520,11 @@ def _translate(self, offset: int) -> Tuple[int, int, str]: class LinuxMixin(Intel): @functools.cached_property - def register_mask(self) -> int: + def _register_mask(self) -> int: return (1 << self._bits_per_register) - 1 @functools.cached_property - def physical_mask(self) -> int: + def _physical_mask(self) -> int: # From kernels 4.18 the physical mask is dynamic: See AMD SME, Intel Multi-Key Total # Memory Encryption and CONFIG_DYNAMIC_PHYSICAL_MASK: 94d49eb30e854c84d1319095b5dd0405a7da9362 physical_mask = (1 << self._maxphyaddr) - 1 @@ -536,42 +536,44 @@ def page_mask(self) -> int: # Note that within the Intel class it's a class method. However, since it uses # complement operations and we are working in Python, it would be more careful to # limit it to the architecture's pointer size. - return ~(self.page_size - 1) & self.register_mask + return ~(self.page_size - 1) & self._register_mask @functools.cached_property - def physical_page_mask(self) -> int: - return self.page_mask & self.physical_mask + def _physical_page_mask(self) -> int: + return self.page_mask & self._physical_mask @functools.cached_property - def pte_pfn_mask(self) -> int: - return self.physical_page_mask + def _pte_pfn_mask(self) -> int: + return self._physical_page_mask @functools.cached_property - def pte_flags_mask(self) -> int: - return ~self.pte_pfn_mask & self.register_mask + def _pte_flags_mask(self) -> int: + return ~self._pte_pfn_mask & self._register_mask - def pte_flags(self, pte) -> int: - return pte & self.pte_flags_mask + def _pte_flags(self, pte) -> int: + return pte & self._pte_flags_mask - def is_pte_present(self, entry: int) -> bool: - return (self.pte_flags(entry) & (self._PAGE_PRESENT | self._PAGE_PROTNONE)) != 0 + def _is_pte_present(self, entry: int) -> bool: + return ( + self._pte_flags(entry) & (self._PAGE_PRESENT | self._PAGE_PROTNONE) + ) != 0 def _page_is_valid(self, entry: int) -> bool: # Overrides the Intel static method with the Linux-specific implementation - return self.is_pte_present(entry) + return self._is_pte_present(entry) - def pte_needs_invert(self, entry) -> bool: + def _pte_needs_invert(self, entry) -> bool: # Entries that were set to PROT_NONE (PAGE_PRESENT/PAGE_GLOBAL) are inverted return not (entry & self._PAGE_PRESENT) - def protnone_mask(self, entry: int) -> int: + def _protnone_mask(self, entry: int) -> int: """Gets a mask to XOR with the page table entry to get the correct PFN""" - return ~0 & self.register_mask if self.pte_needs_invert(entry) else 0 + return ~0 & self._register_mask if self._pte_needs_invert(entry) else 0 - def pte_pfn(self, entry: int) -> int: + def _pte_pfn(self, entry: int) -> int: """Extracts the page frame number from the page table entry""" - pfn = entry ^ self.protnone_mask(entry) - return (pfn & self.pte_pfn_mask) >> self.page_shift + pfn = entry ^ self._protnone_mask(entry) + return (pfn & self._pte_pfn_mask) >> self.page_shift class LinuxIntel(LinuxMixin, Intel): From 641e008caf34b93b3c12360115066e1e291500ca Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 27 Nov 2024 15:06:33 +1100 Subject: [PATCH 7/8] Linux: intel: The non-present mapping shouldn't be inverted --- volatility3/framework/layers/intel.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index dc88c20bf3..6d0aa67465 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -563,8 +563,9 @@ def _page_is_valid(self, entry: int) -> bool: return self._is_pte_present(entry) def _pte_needs_invert(self, entry) -> bool: - # Entries that were set to PROT_NONE (PAGE_PRESENT/PAGE_GLOBAL) are inverted - return not (entry & self._PAGE_PRESENT) + # Entries that were set to PROT_NONE (PAGE_PRESENT) are inverted + # A clear PTE shouldn't be inverted. See f19f5c4 + return entry and not (entry & self._PAGE_PRESENT) def _protnone_mask(self, entry: int) -> int: """Gets a mask to XOR with the page table entry to get the correct PFN""" From eff2a529c8e2e264553aca0550a5cd9ce3c9c298 Mon Sep 17 00:00:00 2001 From: Gustavo Moreira Date: Wed, 27 Nov 2024 18:10:14 +1100 Subject: [PATCH 8/8] Linux: intel: Remove the bitwise zero complement to simplify the protnone mask calculation --- volatility3/framework/layers/intel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 6d0aa67465..d762b41a8b 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -569,7 +569,7 @@ def _pte_needs_invert(self, entry) -> bool: def _protnone_mask(self, entry: int) -> int: """Gets a mask to XOR with the page table entry to get the correct PFN""" - return ~0 & self._register_mask if self._pte_needs_invert(entry) else 0 + return self._register_mask if self._pte_needs_invert(entry) else 0 def _pte_pfn(self, entry: int) -> int: """Extracts the page frame number from the page table entry"""