From b0cbd3c61cae8fce0a295021418ce1f798fb3ab9 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Thu, 26 Oct 2023 02:49:07 +0200 Subject: [PATCH 01/22] Adding hibernation support --- .../framework/layers/codecs/__init__.py | 161 +++++++++++++++- volatility3/framework/layers/hib.py | 176 ++++++++++++++++++ 2 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 volatility3/framework/layers/hib.py diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index e019bcbcd9..c7013111ce 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -3,6 +3,165 @@ # """Codecs used for encoding or decoding data should live here +""" +import io, struct +def encoded_bit_length(data, symbol): + if (symbol % 2) == 0: + return int(data[symbol//2] & 0x0f) + else: + return int(data[symbol//2] >> 4) + +def Read16Bits(input, current_position): + return (input[current_position] << 8) | input[current_position + 1] + +def ReadByte(input, current_position): + stream = io.BytesIO(input) + stream.seek(current_position) + return int.from_bytes(stream.read(1),"little") + +def lz77_huffman_decompress(compressed_input): + decoding_table = [0]*pow(2,15) + current_table_entry = 0 + + for bit_length in range(1, 16): + for symbol in range(0, 512): + if encoded_bit_length(compressed_input[0:256],symbol) == bit_length: + entry_count = 1 << (15 - bit_length) + for _ in range(entry_count): + if current_table_entry >= pow(2,15): + raise ValueError("The compressed data is not valid.") + decoding_table[current_table_entry] = symbol + current_table_entry += 1 + if current_table_entry != pow(2,15): + raise ValueError("The compressed data is not valid.") + + output_buffer = [] + current_position = 256 + next_bits = Read16Bits(compressed_input,current_position) + current_position += 2 + next_bits <<= 16 + next_bits |= Read16Bits(compressed_input,current_position) + current_position += 2 + extra_bit_count = 16 + block_end = len(output_buffer) + 65536 + + while True: + # Start of new block + if current_position + 512 > len(compressed_input): + print("EOF reached. Terminating decoding.") + return output_buffer + if len(output_buffer) >= block_end: + break + next_15bits = next_bits >> (32 - 15) + huffman_symbol = decoding_table[next_15bits] + huffman_symbol_bit_length = encoded_bit_length(compressed_input[0:256],huffman_symbol) + + next_bits <<= huffman_symbol_bit_length + extra_bit_count -= huffman_symbol_bit_length + + if extra_bit_count < 0: + next_bits |= Read16Bits(compressed_input,current_position) << (-extra_bit_count) + extra_bit_count += 16 + current_position += 2 + + if huffman_symbol < 256: + output_buffer.append(huffman_symbol) + + elif huffman_symbol == 256 and current_position == len(compressed_input): + return output_buffer + + else: + huffman_symbol -= 256 + match_length = huffman_symbol % 16 + match_offset_bit_length = huffman_symbol // 16 + if match_length == 15: + match_length = ReadByte(compressed_input,current_position) + current_position += 1 + if match_length == 255: + match_length = Read16Bits(compressed_input,current_position) + current_position += 2 + if match_length < 15: + raise ValueError("The compressed data is invalid.") + match_length -= 15 + match_length += 15 + match_length += 3 + match_offset = next_bits >> (32 - match_offset_bit_length) + match_offset += pow(2, match_offset_bit_length) + next_bits <<= match_offset_bit_length + extra_bit_count -= match_offset_bit_length + if extra_bit_count < 0: + next_bits |= Read16Bits(compressed_input,current_position) << (-extra_bit_count) + extra_bit_count += 16 + current_position += 2 + for i in range(match_length): + output_buffer.append(output_buffer[len(output_buffer) - match_offset + i]) + +def lz77_plain_decompress(in_buf): + out_idx = 0 + in_idx = 0 + nibble_idx = 0 + + flags = 0 + flag_count = 0 + + out_buf = [] + + while in_idx < len(in_buf): + if flag_count == 0: + flags = struct.unpack(' out_idx: + raise ValueError('CorruptedData') + out_buf.append(out_buf[out_idx - offset]) + out_idx += 1 + + return bytes(out_buf) -""" diff --git a/volatility3/framework/layers/hib.py b/volatility3/framework/layers/hib.py new file mode 100644 index 0000000000..ea52985245 --- /dev/null +++ b/volatility3/framework/layers/hib.py @@ -0,0 +1,176 @@ +from typing import Optional, Tuple +import logging, struct +from volatility3.framework import automagic, interfaces, constants, exceptions +from volatility3.framework.interfaces import context +from volatility3.framework.automagic import stacker +from volatility3.framework.layers import intel, physical, segmented +from volatility3.framework.renderers import conversion +from volatility3.framework.layers.codecs import lz77_plain_decompress, lz77_huffman_decompress + + +vollog = logging.getLogger(__name__) + + +def uncompress(data: bytes, huffman: bool): + if huffman ==1: + return lz77_huffman_decompress(data) + else: + return lz77_plain_decompress(data) + + +def readBytes(data, position, num): + end = position + num + byte_vals = data[position:end] + return int.from_bytes(byte_vals, byteorder='big') + +class HibernationFileException(exceptions.LayerException): + """Thrown when an error occurs with the underlying Hibernation file format.""" + +class HibernationLayer(segmented.NonLinearlySegmentedLayer): + """ + A TranslationLayer that maps physical memory against a Microsoft Windows hibernation file (x64 only for now). + """ + PAGE_SIZE = 4096 + HEADER_SIZE = 4 + PAGE_DESC_SIZE = 8 + def __init__(self, context: interfaces.context.ContextInterface, config_path: str, name: str, **kwargs): + """ + Initializes the Hibernation file layer. + """ + # Call the superclass constructor. + self._compressed = {} + self._mapping = {} + super().__init__(context, config_path, name, **kwargs) + + @classmethod + def _check_header( + cls, base_layer: interfaces.layers.DataLayerInterface, name: str = "" + ): + header = base_layer.read(0, 4) + if header != b'HIBR': + raise exceptions.LayerException(name, "No Hibernation magic bytes") + else: + vollog.info("Detecting an hibernation file") + + def _load_segments(self): + base_layer = self.context.layers[self._base_layer] + system_time = int.from_bytes(base_layer.read(0x020, 8), "little") + systemTime = conversion.wintime_to_datetime(system_time) + NumPagesForLoader = int.from_bytes(base_layer.read(0x058, 8), "little") + FirstBootRestorePage = int.from_bytes(base_layer.read(0x068, 8), "little") + FirstKernelRestorePage = int.from_bytes(base_layer.read(0x070, 8), "little") + KernelPagesProcessed = int.from_bytes(base_layer.read(0x230, 8), "little") + + # vollog.info(f""" + # SystemTime : {systemTime} \n + # NumPagesForLoader : {NumPagesForLoader} \n + # FirstBootRestorePage : {hex(FirstBootRestorePage)} \n + # KernelPageProcessed : {KernelPagesProcessed} \n + # FirstKernelRestorePage : {FirstKernelRestorePage} \n + # """) + # TODO : If the FirstKernelRestorePage member of the header is non-zero, + # its value gives the page number of the start of the KernelRestore- Pages. + # We need to check if this value is zero and not process the KernelRestorePages if so. + + offset = FirstBootRestorePage * self.PAGE_SIZE + total_pages = NumPagesForLoader + treated = 0 + + while total_pages > treated: + page_read, next_cs = self._read_compression_set(offset) + offset += next_cs + treated += page_read + + offset = FirstKernelRestorePage * self.PAGE_SIZE + total_pages = KernelPagesProcessed + + treated = 0 + while total_pages > treated: + page_read, next_cs = self._read_compression_set(offset) + offset += next_cs + treated += page_read + + self._segments = sorted(self._segments, key=lambda x: x[0]) + + def _read_compression_set(self, offset): + """ + Desc : Read one compression set an extract the address of the compressed data + Params : + - offset : the location of the compression set to read. + - stream : the hibernation file stream. + Return : The offset of the compressed data and the size. + """ + base_layer = self.context.layers[self._base_layer] + + header = base_layer.read(offset, self.HEADER_SIZE) + data = struct.unpack('> 8) & 0x3fffff + huffman_compressed = (data >> 30) & 0x1 + mapped_address = offset+self.HEADER_SIZE+number_of_descs*self.PAGE_DESC_SIZE + total_page_count = 0 + position = 0 + for i in range(number_of_descs): + location = offset+self.HEADER_SIZE+i*self.PAGE_DESC_SIZE + page_descriptor = base_layer.read(location, self.PAGE_DESC_SIZE) + data = struct.unpack('> 4 # shift right 4 bits to get the upper 60 bits + page_count = (1+ Numpages) + total_page_count += page_count + self._segments.append( + (PageNum*self.PAGE_SIZE, + mapped_address, + self.PAGE_SIZE*page_count, + size_of_compressed_data + ) + ) + for j in range(page_count): + self._mapping[(PageNum+j)*self.PAGE_SIZE] = position*self.PAGE_SIZE + position += 1 + + total_page_size = total_page_count*self.PAGE_SIZE + if total_page_size != size_of_compressed_data: + self._compressed[mapped_address] = huffman_compressed + return total_page_count, (4 + size_of_compressed_data + number_of_descs * self.PAGE_DESC_SIZE) #Number of pages in the set, Size of the entire compression set + + + def _decode_data( + self, data: bytes, mapped_offset: int, offset: int, output_length: int + ) -> bytes: + start_offset, _, _, _ = self._find_segment(offset) + if mapped_offset in self._compressed: + try: + decoded_data = uncompress(data, self._compressed[mapped_offset]) + except: + return bytearray(output_length) + else: + decoded_data = data + page_offset = self._mapping[start_offset] + decoded_data = decoded_data[page_offset + (offset - start_offset):] + decoded_data = decoded_data[:output_length] + if len(decoded_data) == output_length: + return decoded_data + else: + return bytearray(output_length) + +class HibernationFileStacker(interfaces.automagic.StackerLayerInterface): + stack_order = 10 + @classmethod + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: + try: + HibernationLayer._check_header(context.layers[layer_name]) + except exceptions.LayerException: + return None + new_name = context.layers.free_layer_name("HibernationLayer") + context.config[ + interfaces.configuration.path_join(new_name, "base_layer") + ] = layer_name + layer = HibernationLayer(context, new_name, new_name) + cls.stacker_slow_warning() + return layer \ No newline at end of file From c91e417a6337c1a19d23c90355897eda92c8b509 Mon Sep 17 00:00:00 2001 From: ForensicXlab Date: Fri, 27 Oct 2023 16:11:27 +0200 Subject: [PATCH 02/22] Fixing Huffman Hibernation layerwriter is now working from Win1809 to Win22h2 hibernation. --- .../framework/layers/codecs/__init__.py | 124 ++++++++++-------- volatility3/framework/layers/hib.py | 23 ++-- 2 files changed, 82 insertions(+), 65 deletions(-) diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index c7013111ce..e992abcb10 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -4,7 +4,7 @@ """Codecs used for encoding or decoding data should live here """ -import io, struct +import io, struct, numpy def encoded_bit_length(data, symbol): if (symbol % 2) == 0: @@ -13,89 +13,109 @@ def encoded_bit_length(data, symbol): return int(data[symbol//2] >> 4) def Read16Bits(input, current_position): - return (input[current_position] << 8) | input[current_position + 1] + if current_position > len(input): + print("Incomplete Prefetch") + exit(1) + stream = io.BytesIO(input) + stream.seek(current_position) + byte_value = bytearray(stream.read(2)) + val = numpy.uint16(0) + j = 0 + for i in byte_value: + val = val | (numpy.uint16(i) << numpy.uint(j*8)) + j = j+1 + return val def ReadByte(input, current_position): stream = io.BytesIO(input) stream.seek(current_position) return int.from_bytes(stream.read(1),"little") -def lz77_huffman_decompress(compressed_input): - decoding_table = [0]*pow(2,15) +def lz77_huffman_decompress(in_buf): + """ + Description : Decompress the prefetch using LZ77+Huffman Decompression Algorithm + Params : + @data : The compressed prefetch data extracted from memory + @result : The uncompressed prefetch file ready to be forensically analysed + Possible errors : + Invalid compressed data. + """ + if len(in_buf) < 256: + print("Error : The prefetch must use a 256-byte Huffman table. -> Invalid data") + + #First, we construct our Huffman decoding table + decoding_table = [0] * (2**15) current_table_entry = 0 - - for bit_length in range(1, 16): + encoded_data = in_buf[0:256] + for bit_length in range(1,16): for symbol in range(0, 512): - if encoded_bit_length(compressed_input[0:256],symbol) == bit_length: - entry_count = 1 << (15 - bit_length) - for _ in range(entry_count): - if current_table_entry >= pow(2,15): - raise ValueError("The compressed data is not valid.") - decoding_table[current_table_entry] = symbol + if encoded_bit_length(encoded_data, symbol) == bit_length: # If the encoded bit length of symbol equals bit_length + entry_count = (1 << (15 - bit_length)) + for i in range(0, entry_count): + if current_table_entry >= 2**15: #Huffman table length + raise ValueError('CorruptedData') + decoding_table[current_table_entry] = numpy.uint16(symbol) current_table_entry += 1 - if current_table_entry != pow(2,15): - raise ValueError("The compressed data is not valid.") + if current_table_entry != 2**15: + raise ValueError('CorruptedData') + - output_buffer = [] - current_position = 256 - next_bits = Read16Bits(compressed_input,current_position) + #Then, it's time to decompress the data + """ + The compression stream is designed to be read in (mostly) 16-bit chunks, with a 32-bit register + maintaining at least the next 16 bits of input. This strategy allows the code to seamlessly handle the + bytes for long match lengths, which would otherwise be awkward. + """ + out_buf = [] + input_buffer = in_buf + current_position = 256 # start at the end of the Huffman table + next_bits = Read16Bits(input_buffer, current_position) current_position += 2 - next_bits <<= 16 - next_bits |= Read16Bits(compressed_input,current_position) + next_bits = numpy.uint32(next_bits) << numpy.int64(16) + next_bits = next_bits | numpy.uint32(Read16Bits(input_buffer, current_position)) current_position += 2 extra_bit_count = 16 - block_end = len(output_buffer) + 65536 - + # Loop until a block terminating condition while True: - # Start of new block - if current_position + 512 > len(compressed_input): - print("EOF reached. Terminating decoding.") - return output_buffer - if len(output_buffer) >= block_end: - break - next_15bits = next_bits >> (32 - 15) - huffman_symbol = decoding_table[next_15bits] - huffman_symbol_bit_length = encoded_bit_length(compressed_input[0:256],huffman_symbol) - - next_bits <<= huffman_symbol_bit_length + next_15_bits = numpy.uint32(next_bits) >> numpy.uint32((32 - 15)) + huffman_symbol = decoding_table[next_15_bits] + huffman_symbol_bit_length = encoded_bit_length(encoded_data, huffman_symbol) + next_bits = numpy.int32(next_bits << huffman_symbol_bit_length) extra_bit_count -= huffman_symbol_bit_length - if extra_bit_count < 0: - next_bits |= Read16Bits(compressed_input,current_position) << (-extra_bit_count) - extra_bit_count += 16 + next_bits = next_bits | (numpy.uint32(Read16Bits(input_buffer, current_position)) << (-extra_bit_count)) current_position += 2 - + extra_bit_count += 16 if huffman_symbol < 256: - output_buffer.append(huffman_symbol) - - elif huffman_symbol == 256 and current_position == len(compressed_input): - return output_buffer - + out_buf.append(huffman_symbol) + elif huffman_symbol == 256 and (len(input_buffer) - current_position) == 0: + return bytes(out_buf) else: - huffman_symbol -= 256 + huffman_symbol = huffman_symbol - 256 match_length = huffman_symbol % 16 match_offset_bit_length = huffman_symbol // 16 if match_length == 15: - match_length = ReadByte(compressed_input,current_position) - current_position += 1 + match_length = numpy.uint16(ReadByte(input_buffer, current_position)) + current_position+=1 if match_length == 255: - match_length = Read16Bits(compressed_input,current_position) + match_length = Read16Bits(input_buffer, current_position) current_position += 2 if match_length < 15: - raise ValueError("The compressed data is invalid.") + raise ValueError('CorruptedData') match_length -= 15 match_length += 15 match_length += 3 match_offset = next_bits >> (32 - match_offset_bit_length) - match_offset += pow(2, match_offset_bit_length) - next_bits <<= match_offset_bit_length + match_offset += (1 << match_offset_bit_length) + next_bits = next_bits << match_offset_bit_length extra_bit_count -= match_offset_bit_length if extra_bit_count < 0: - next_bits |= Read16Bits(compressed_input,current_position) << (-extra_bit_count) - extra_bit_count += 16 + next_bits = next_bits | (numpy.uint32(Read16Bits(input_buffer, current_position)) << (-extra_bit_count)) current_position += 2 - for i in range(match_length): - output_buffer.append(output_buffer[len(output_buffer) - match_offset + i]) + extra_bit_count += 16 + for _ in range(0, match_length): + to_write = out_buf[(len(out_buf)-1) - int(match_offset)] + out_buf.append(to_write) def lz77_plain_decompress(in_buf): out_idx = 0 diff --git a/volatility3/framework/layers/hib.py b/volatility3/framework/layers/hib.py index ea52985245..2bf60ec931 100644 --- a/volatility3/framework/layers/hib.py +++ b/volatility3/framework/layers/hib.py @@ -11,11 +11,13 @@ vollog = logging.getLogger(__name__) -def uncompress(data: bytes, huffman: bool): - if huffman ==1: - return lz77_huffman_decompress(data) - else: +def uncompress(data: bytes, huffman): + if huffman == 0: return lz77_plain_decompress(data) + elif huffman == 2 or huffman == 3: + return lz77_huffman_decompress(data) + else: + raise ValueError('Cannot decompress the data.') def readBytes(data, position, num): @@ -106,7 +108,7 @@ def _read_compression_set(self, offset): data = struct.unpack('> 8) & 0x3fffff - huffman_compressed = (data >> 30) & 0x1 + huffman_compressed = (data >> 30) & 0x3 mapped_address = offset+self.HEADER_SIZE+number_of_descs*self.PAGE_DESC_SIZE total_page_count = 0 position = 0 @@ -140,19 +142,14 @@ def _decode_data( ) -> bytes: start_offset, _, _, _ = self._find_segment(offset) if mapped_offset in self._compressed: - try: - decoded_data = uncompress(data, self._compressed[mapped_offset]) - except: - return bytearray(output_length) + decoded_data = uncompress(data, self._compressed[mapped_offset]) else: decoded_data = data page_offset = self._mapping[start_offset] decoded_data = decoded_data[page_offset + (offset - start_offset):] decoded_data = decoded_data[:output_length] - if len(decoded_data) == output_length: - return decoded_data - else: - return bytearray(output_length) + return decoded_data + class HibernationFileStacker(interfaces.automagic.StackerLayerInterface): stack_order = 10 From 5c4446776a70648b4e68d5de15d559604ec7b8b7 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 27 Oct 2023 19:29:54 +0200 Subject: [PATCH 03/22] Adapting comments to the codec instead of the volatility3 prefetch plugin --- volatility3/framework/layers/codecs/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index e992abcb10..c6f1eb978b 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -14,8 +14,7 @@ def encoded_bit_length(data, symbol): def Read16Bits(input, current_position): if current_position > len(input): - print("Incomplete Prefetch") - exit(1) + raise ValueError("Incomplete Data") stream = io.BytesIO(input) stream.seek(current_position) byte_value = bytearray(stream.read(2)) @@ -33,15 +32,15 @@ def ReadByte(input, current_position): def lz77_huffman_decompress(in_buf): """ - Description : Decompress the prefetch using LZ77+Huffman Decompression Algorithm + Description : Decompress the data using LZ77+Huffman Decompression Algorithm Params : - @data : The compressed prefetch data extracted from memory - @result : The uncompressed prefetch file ready to be forensically analysed + @data : The compressed data extracted from memory + @result : The uncompressed data Possible errors : Invalid compressed data. """ if len(in_buf) < 256: - print("Error : The prefetch must use a 256-byte Huffman table. -> Invalid data") + raise ValueError("Error : The data must use a 256-byte Huffman table. -> Invalid data") #First, we construct our Huffman decoding table decoding_table = [0] * (2**15) From a13e105d34814c5d1ca70f6f7f478ad96ffc69e4 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Mon, 30 Oct 2023 04:16:51 +0100 Subject: [PATCH 04/22] Update the Xpress LZ77+Huffman decoder --- .../framework/layers/codecs/__init__.py | 330 ++++++++++++------ volatility3/framework/layers/hib.py | 14 +- 2 files changed, 221 insertions(+), 123 deletions(-) diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index c6f1eb978b..e237a556e3 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -1,120 +1,220 @@ -# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -"""Codecs used for encoding or decoding data should live here -""" -import io, struct, numpy - -def encoded_bit_length(data, symbol): - if (symbol % 2) == 0: - return int(data[symbol//2] & 0x0f) - else: - return int(data[symbol//2] >> 4) - -def Read16Bits(input, current_position): - if current_position > len(input): - raise ValueError("Incomplete Data") - stream = io.BytesIO(input) - stream.seek(current_position) - byte_value = bytearray(stream.read(2)) - val = numpy.uint16(0) - j = 0 - for i in byte_value: - val = val | (numpy.uint16(i) << numpy.uint(j*8)) - j = j+1 - return val - -def ReadByte(input, current_position): - stream = io.BytesIO(input) - stream.seek(current_position) - return int.from_bytes(stream.read(1),"little") - -def lz77_huffman_decompress(in_buf): - """ - Description : Decompress the data using LZ77+Huffman Decompression Algorithm - Params : - @data : The compressed data extracted from memory - @result : The uncompressed data - Possible errors : - Invalid compressed data. - """ - if len(in_buf) < 256: - raise ValueError("Error : The data must use a 256-byte Huffman table. -> Invalid data") - - #First, we construct our Huffman decoding table - decoding_table = [0] * (2**15) - current_table_entry = 0 - encoded_data = in_buf[0:256] - for bit_length in range(1,16): - for symbol in range(0, 512): - if encoded_bit_length(encoded_data, symbol) == bit_length: # If the encoded bit length of symbol equals bit_length - entry_count = (1 << (15 - bit_length)) - for i in range(0, entry_count): - if current_table_entry >= 2**15: #Huffman table length - raise ValueError('CorruptedData') - decoding_table[current_table_entry] = numpy.uint16(symbol) - current_table_entry += 1 - if current_table_entry != 2**15: - raise ValueError('CorruptedData') - - - #Then, it's time to decompress the data - """ - The compression stream is designed to be read in (mostly) 16-bit chunks, with a 32-bit register - maintaining at least the next 16 bits of input. This strategy allows the code to seamlessly handle the - bytes for long match lengths, which would otherwise be awkward. - """ - out_buf = [] - input_buffer = in_buf - current_position = 256 # start at the end of the Huffman table - next_bits = Read16Bits(input_buffer, current_position) - current_position += 2 - next_bits = numpy.uint32(next_bits) << numpy.int64(16) - next_bits = next_bits | numpy.uint32(Read16Bits(input_buffer, current_position)) - current_position += 2 - extra_bit_count = 16 - # Loop until a block terminating condition +import struct +from typing import Tuple, List, Union + +class BitStream: + def __init__(self, source: bytes, in_pos: int): + self.source = source + self.index = in_pos + 4 + # read UInt16 little endian + mask = struct.unpack_from(' int: + if n == 0: + return 0 + return self.mask >> (32 - n) + + def skip(self, n: int) -> Union[None, Exception]: + self.mask = ((self.mask << n) & 0xFFFFFFFF) + self.bits -= n + if self.bits < 16: + if self.index + 2 > len(self.source): + return Exception("EOF Error") + # read UInt16 little endian + self.mask += ((struct.unpack_from(' int: + node = treeNodes[0] + i = leafIndex + 1 + childIndex = None + + while bits > 1: + bits -= 1 + childIndex = (mask >> bits) & 1 + if node.child[childIndex] == None: + node.child[childIndex] = treeNodes[i] + treeNodes[i].leaf = False + i += 1 + node = node.child[childIndex] + + node.child[mask&1] = treeNodes[leafIndex] + + return i + +def prefix_code_tree_rebuild(input: bytes) -> PREFIX_CODE_NODE: + treeNodes = [PREFIX_CODE_NODE() for _ in range(1024)] + symbolInfo = [PREFIX_CODE_SYMBOL() for _ in range(512)] + + for i in range(256): + value = input[i] + + symbolInfo[2*i].id = 2 * i + symbolInfo[2*i].symbol = 2 * i + symbolInfo[2*i].length = value & 0xf + + value >>= 4 + + symbolInfo[2*i+1].id = 2*i + 1 + symbolInfo[2*i+1].symbol = 2*i + 1 + symbolInfo[2*i+1].length = value & 0xf + + symbolInfo = sorted(symbolInfo, key=lambda x: (x.length, x.symbol)) + + i = 0 + while i < 512 and symbolInfo[i].length == 0: + i += 1 + + mask = 0 + bits = 1 + + root = treeNodes[0] + root.leaf = False + + j = 1 + while i < 512: + treeNodes[j].id = j + treeNodes[j].symbol = symbolInfo[i].symbol + treeNodes[j].leaf = True + mask = mask << (symbolInfo[i].length - bits) + bits = symbolInfo[i].length + j = prefix_code_tree_add_leaf(treeNodes, j, mask, bits) + mask += 1 + i += 1 + + return root + +def prefix_code_tree_decode_symbol(bstr: BitStream, root: PREFIX_CODE_NODE) -> Tuple[int, Union[None, Exception]]: + node = root + i = 0 while True: - next_15_bits = numpy.uint32(next_bits) >> numpy.uint32((32 - 15)) - huffman_symbol = decoding_table[next_15_bits] - huffman_symbol_bit_length = encoded_bit_length(encoded_data, huffman_symbol) - next_bits = numpy.int32(next_bits << huffman_symbol_bit_length) - extra_bit_count -= huffman_symbol_bit_length - if extra_bit_count < 0: - next_bits = next_bits | (numpy.uint32(Read16Bits(input_buffer, current_position)) << (-extra_bit_count)) - current_position += 2 - extra_bit_count += 16 - if huffman_symbol < 256: - out_buf.append(huffman_symbol) - elif huffman_symbol == 256 and (len(input_buffer) - current_position) == 0: - return bytes(out_buf) + bit = bstr.lookup(1) + err = bstr.skip(1) + if err is not None: + return 0, err + + node = node.child[bit] + if node == None: + return 0, Exception("Corruption detected") + + if node.leaf: + break + return node.symbol, None + +def lz77_huffman_decompress_chunck(in_idx: int, + input: bytes, + out_idx: int, + output: bytearray, + chunk_size: int) -> Tuple[int, int, Union[None, Exception]]: + + # Ensure there are at least 256 bytes available to read + if in_idx + 256 > len(input): + return 0, 0, Exception("EOF Error") + + root = prefix_code_tree_rebuild(input[in_idx:]) + #print_tree(root) + bstr = BitStream(input, in_idx+256) + + i = out_idx + + while i < out_idx + chunk_size: + symbol, err = prefix_code_tree_decode_symbol(bstr, root) + + if err is not None: + return int(bstr.index), i, err + + if symbol < 256: + output[i] = symbol + i += 1 else: - huffman_symbol = huffman_symbol - 256 - match_length = huffman_symbol % 16 - match_offset_bit_length = huffman_symbol // 16 - if match_length == 15: - match_length = numpy.uint16(ReadByte(input_buffer, current_position)) - current_position+=1 - if match_length == 255: - match_length = Read16Bits(input_buffer, current_position) - current_position += 2 - if match_length < 15: - raise ValueError('CorruptedData') - match_length -= 15 - match_length += 15 - match_length += 3 - match_offset = next_bits >> (32 - match_offset_bit_length) - match_offset += (1 << match_offset_bit_length) - next_bits = next_bits << match_offset_bit_length - extra_bit_count -= match_offset_bit_length - if extra_bit_count < 0: - next_bits = next_bits | (numpy.uint32(Read16Bits(input_buffer, current_position)) << (-extra_bit_count)) - current_position += 2 - extra_bit_count += 16 - for _ in range(0, match_length): - to_write = out_buf[(len(out_buf)-1) - int(match_offset)] - out_buf.append(to_write) + symbol -= 256 + length = symbol & 15 + symbol >>= 4 + + offset = 0 + if symbol != 0: + offset = int(bstr.lookup(symbol)) + + offset |= 1 << symbol + offset = -offset + + if length == 15: + length = bstr.source[bstr.index] + 15 + bstr.index += 1 + + if length == 270: + length = struct.unpack_from(' 0: + if i + offset < 0: + print(i + offset) + return int(bstr.index), i, Exception("Decompression Error") + + output[i] = output[i + offset] + i += 1 + length -= 1 + if length==0: + break + return int(bstr.index), i, None + + +def lz77_huffman_decompress(input: bytes, output_size: int) -> Tuple[bytes, Union[None, Exception]]: + output = bytearray(output_size) + err = None + + # Index into the input buffer. + in_idx = 0 + + # Index into the output buffer. + out_idx = 0 + + while True: + # How much data belongs in the current chunk. Chunks + # are split into maximum 65536 bytes. + chunk_size = output_size - out_idx + if chunk_size > 65536: + chunk_size = 65536 + + in_idx, out_idx, err = lz77_huffman_decompress_chunck( + in_idx, input, out_idx, output, chunk_size) + if err is not None: + return output, err + if out_idx >= len(output) or in_idx >= len(input): + break + return output, None def lz77_plain_decompress(in_buf): out_idx = 0 diff --git a/volatility3/framework/layers/hib.py b/volatility3/framework/layers/hib.py index 2bf60ec931..3f97b635d6 100644 --- a/volatility3/framework/layers/hib.py +++ b/volatility3/framework/layers/hib.py @@ -1,9 +1,7 @@ -from typing import Optional, Tuple +from typing import Optional import logging, struct -from volatility3.framework import automagic, interfaces, constants, exceptions -from volatility3.framework.interfaces import context -from volatility3.framework.automagic import stacker -from volatility3.framework.layers import intel, physical, segmented +from volatility3.framework import interfaces, constants, exceptions +from volatility3.framework.layers import segmented from volatility3.framework.renderers import conversion from volatility3.framework.layers.codecs import lz77_plain_decompress, lz77_huffman_decompress @@ -11,11 +9,11 @@ vollog = logging.getLogger(__name__) -def uncompress(data: bytes, huffman): +def uncompress(data: bytes, huffman, out_size): if huffman == 0: return lz77_plain_decompress(data) elif huffman == 2 or huffman == 3: - return lz77_huffman_decompress(data) + return lz77_huffman_decompress(data,out_size)[0] else: raise ValueError('Cannot decompress the data.') @@ -142,7 +140,7 @@ def _decode_data( ) -> bytes: start_offset, _, _, _ = self._find_segment(offset) if mapped_offset in self._compressed: - decoded_data = uncompress(data, self._compressed[mapped_offset]) + decoded_data = uncompress(data, self._compressed[mapped_offset],65536) else: decoded_data = data page_offset = self._mapping[start_offset] From 3568d3f3544fc7d400ccf5e221bc8752f00c8d31 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Mon, 30 Oct 2023 15:27:17 +0100 Subject: [PATCH 05/22] Code comments and cleaning. --- volatility3/framework/layers/codecs/__init__.py | 3 +++ volatility3/framework/layers/hib.py | 16 +++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index e237a556e3..0ca8fab484 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -1,3 +1,6 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# import struct from typing import Tuple, List, Union diff --git a/volatility3/framework/layers/hib.py b/volatility3/framework/layers/hib.py index 3f97b635d6..fe88304303 100644 --- a/volatility3/framework/layers/hib.py +++ b/volatility3/framework/layers/hib.py @@ -1,3 +1,6 @@ +# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# from typing import Optional import logging, struct from volatility3.framework import interfaces, constants, exceptions @@ -18,11 +21,6 @@ def uncompress(data: bytes, huffman, out_size): raise ValueError('Cannot decompress the data.') -def readBytes(data, position, num): - end = position + num - byte_vals = data[position:end] - return int.from_bytes(byte_vals, byteorder='big') - class HibernationFileException(exceptions.LayerException): """Thrown when an error occurs with the underlying Hibernation file format.""" @@ -30,7 +28,7 @@ class HibernationLayer(segmented.NonLinearlySegmentedLayer): """ A TranslationLayer that maps physical memory against a Microsoft Windows hibernation file (x64 only for now). """ - PAGE_SIZE = 4096 + PAGE_SIZE = 4096 #For x64. HEADER_SIZE = 4 PAGE_DESC_SIZE = 8 def __init__(self, context: interfaces.context.ContextInterface, config_path: str, name: str, **kwargs): @@ -68,9 +66,9 @@ def _load_segments(self): # KernelPageProcessed : {KernelPagesProcessed} \n # FirstKernelRestorePage : {FirstKernelRestorePage} \n # """) - # TODO : If the FirstKernelRestorePage member of the header is non-zero, - # its value gives the page number of the start of the KernelRestore- Pages. - # We need to check if this value is zero and not process the KernelRestorePages if so. + + # TODO : The offset of the FirstKernelRestorePage vary for some Windows version. Need to check the other location if == 0. + offset = FirstBootRestorePage * self.PAGE_SIZE total_pages = NumPagesForLoader From 7e87e2d7f0a3676e627a159a0664c388f45238d8 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 3 Nov 2023 22:10:40 +0100 Subject: [PATCH 06/22] Plugins added : hibernation.Info and hibernation.Dump. Support for added from Win8 to Win11. Testing in progress on hibernation files from the different OS versions. --- .../framework/layers/codecs/__init__.py | 9 +- volatility3/framework/layers/hib.py | 58 +++++---- .../framework/plugins/windows/hibernation.py | 115 ++++++++++++++++++ 3 files changed, 157 insertions(+), 25 deletions(-) create mode 100644 volatility3/framework/plugins/windows/hibernation.py diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index 0ca8fab484..02f4f7e721 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -1,8 +1,10 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import struct +import struct, logging from typing import Tuple, List, Union +vollog = logging.getLogger(__name__) + class BitStream: def __init__(self, source: bytes, in_pos: int): @@ -121,10 +123,12 @@ def prefix_code_tree_decode_symbol(bstr: BitStream, root: PREFIX_CODE_NODE) -> T bit = bstr.lookup(1) err = bstr.skip(1) if err is not None: + vollog.warning("Some data could not be decompressed.") return 0, err node = node.child[bit] if node == None: + vollog.warning("Corruption detected when decompressing the data.") return 0, Exception("Corruption detected") if node.leaf: @@ -142,7 +146,6 @@ def lz77_huffman_decompress_chunck(in_idx: int, return 0, 0, Exception("EOF Error") root = prefix_code_tree_rebuild(input[in_idx:]) - #print_tree(root) bstr = BitStream(input, in_idx+256) i = out_idx @@ -183,7 +186,7 @@ def lz77_huffman_decompress_chunck(in_idx: int, length += 3 while length > 0: if i + offset < 0: - print(i + offset) + vollog.warning("Some data could not be decompressed.") return int(bstr.index), i, Exception("Decompression Error") output[i] = output[i + offset] diff --git a/volatility3/framework/layers/hib.py b/volatility3/framework/layers/hib.py index fe88304303..cfb08e1f5d 100644 --- a/volatility3/framework/layers/hib.py +++ b/volatility3/framework/layers/hib.py @@ -13,11 +13,12 @@ def uncompress(data: bytes, huffman, out_size): - if huffman == 0: + if huffman == 0 or huffman == 1: return lz77_plain_decompress(data) elif huffman == 2 or huffman == 3: return lz77_huffman_decompress(data,out_size)[0] else: + vollog.warning(f"A compression set could not be decompressed : Compression algorithm : {huffman}") raise ValueError('Cannot decompress the data.') @@ -28,7 +29,7 @@ class HibernationLayer(segmented.NonLinearlySegmentedLayer): """ A TranslationLayer that maps physical memory against a Microsoft Windows hibernation file (x64 only for now). """ - PAGE_SIZE = 4096 #For x64. + PAGE_SIZE = 4096 #For x64 by default HEADER_SIZE = 4 PAGE_DESC_SIZE = 8 def __init__(self, context: interfaces.context.ContextInterface, config_path: str, name: str, **kwargs): @@ -38,6 +39,32 @@ def __init__(self, context: interfaces.context.ContextInterface, config_path: st # Call the superclass constructor. self._compressed = {} self._mapping = {} + self.version = 0 #By default we want to analyze modern windows hiberfiles (Windows 10 2016 1703 to Windows 11 23H2) + if "plugins.Dump.version" in context.config: + # The user is using the hibernation.Dump plugin + self.version = context.config["plugins.Dump.version"] + self.NPFL_OFFSET = 0x058 + self.FBRP_OFFSET = 0x068 + + """ + | Windows Versions | FirstKernelRestorePage | KernelPagesProcessed | + | ------------------------------------------|:----------------------:|:----------------------:| + | Windows 8/8.1 | 0x68 | 0x1C8 | + | Windows 10 2016 1507-1511 | 0x70 | 0x218 | + | Windows 10 2016 1607 | 0x70 | 0x220 | + | Windows 10 2016 1703 - Windows 11 23H2 | 0x70 | 0x230 | + """ + if self.version == 0: + self.FKRP_OFFSET = 0x070 + self.KPP_OFFSET = 0x230 + if self.version == 1: + self.FKRP_OFFSET = 0x68 + self.KPP_OFFSET = 0x1C8 + if self.version == 2: + self.FKRP_OFFSET = 0x70 + self.KPP_OFFSET = 0x218 + + super().__init__(context, config_path, name, **kwargs) @classmethod @@ -52,24 +79,11 @@ def _check_header( def _load_segments(self): base_layer = self.context.layers[self._base_layer] - system_time = int.from_bytes(base_layer.read(0x020, 8), "little") - systemTime = conversion.wintime_to_datetime(system_time) - NumPagesForLoader = int.from_bytes(base_layer.read(0x058, 8), "little") - FirstBootRestorePage = int.from_bytes(base_layer.read(0x068, 8), "little") - FirstKernelRestorePage = int.from_bytes(base_layer.read(0x070, 8), "little") - KernelPagesProcessed = int.from_bytes(base_layer.read(0x230, 8), "little") - - # vollog.info(f""" - # SystemTime : {systemTime} \n - # NumPagesForLoader : {NumPagesForLoader} \n - # FirstBootRestorePage : {hex(FirstBootRestorePage)} \n - # KernelPageProcessed : {KernelPagesProcessed} \n - # FirstKernelRestorePage : {FirstKernelRestorePage} \n - # """) - - # TODO : The offset of the FirstKernelRestorePage vary for some Windows version. Need to check the other location if == 0. - - + NumPagesForLoader = int.from_bytes(base_layer.read(self.NPFL_OFFSET, 8), "little") + FirstBootRestorePage = int.from_bytes(base_layer.read(self.FBRP_OFFSET, 8), "little") + FirstKernelRestorePage = int.from_bytes(base_layer.read(self.FKRP_OFFSET, 8), "little") + KernelPagesProcessed = int.from_bytes(base_layer.read(self.KPP_OFFSET, 8), "little") + offset = FirstBootRestorePage * self.PAGE_SIZE total_pages = NumPagesForLoader treated = 0 @@ -86,7 +100,7 @@ def _load_segments(self): while total_pages > treated: page_read, next_cs = self._read_compression_set(offset) offset += next_cs - treated += page_read + treated += page_read self._segments = sorted(self._segments, key=lambda x: x[0]) @@ -143,7 +157,7 @@ def _decode_data( decoded_data = data page_offset = self._mapping[start_offset] decoded_data = decoded_data[page_offset + (offset - start_offset):] - decoded_data = decoded_data[:output_length] + decoded_data = decoded_data[:output_length] return decoded_data diff --git a/volatility3/framework/plugins/windows/hibernation.py b/volatility3/framework/plugins/windows/hibernation.py new file mode 100644 index 0000000000..5e58512976 --- /dev/null +++ b/volatility3/framework/plugins/windows/hibernation.py @@ -0,0 +1,115 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from typing import List +import logging +from volatility3.framework.renderers import conversion +from volatility3.framework.configuration import requirements +from volatility3.framework import interfaces, renderers +from volatility3.framework.interfaces import plugins +from volatility3.plugins import layerwriter + + +vollog = logging.getLogger(__name__) + +class Info(plugins.PluginInterface): + """Plugin to parse an hiberfil.sys to make sure it is safe to be converted to a raw file and not corrupted""" + + _required_framework_version = (2, 0, 0) + _version = (2, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.TranslationLayerRequirement(name="base_layer", optional=False), + ] + + def _generator(self): + base_layer = self.context.layers["base_layer"] + header = base_layer.read(0, 4) + yield (0, ("Signature", str(header))) + if header == b'HIBR': + # The hibernation file seems exploitable. Next step is to extract important information for the examiner + PageSize = int.from_bytes(base_layer.read(0x18, 4), "little") + yield (0, ("PageSize", str(PageSize))) + if PageSize == 4096: + yield (0, ("Comment", "The hibernation file header signature is correct.")) + system_time = int.from_bytes(base_layer.read(0x020, 8), "little") + systemTime = conversion.wintime_to_datetime(system_time) + yield (0, ("System Time", str(systemTime))) + FirstBootRestorePage = int.from_bytes(base_layer.read(0x068, 8), "little") + yield (0, ("FirstBootRestorePage", str(hex(FirstBootRestorePage)))) + NumPagesForLoader = int.from_bytes(base_layer.read(0x058, 8), "little") + yield (0, ("NumPagesForLoader", str(NumPagesForLoader))) + elif PageSize == 2048: + yield (0, ("Comment", "The hibernation file header signature is correct but x32 compatibility is not available yet.")) + else: + yield (0, ("Comment : ", "The file is corrupted.")) + elif header == b'RSTR': + # The hibernation file was extracted when Windows was in a resuming state which makes it not exploitable + yield (0, ("Comment : ", "The hibernation file header signature is 'RSTR', the file cannot be exploited.")) + else: + yield (0, ("Comment : ", "The file is not an hibernation file or is corrupted.")) + def run(self): + return renderers.TreeGrid([("Variable", str), ("Value", str)], self._generator()) + + +class Dump(plugins.PluginInterface): + """Plugin to parse an hiberfil.sys to make sure it is safe to be converted to a raw file and not corrupted""" + _required_framework_version = (2, 0, 0) + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.PluginRequirement( + name="hibernation", plugin=Info, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="layerwriter", plugin=layerwriter.LayerWriter, version=(2, 0, 0) + ), + requirements.IntRequirement( + name="version", + description="The Windows version of the hibernation file : 0=>[Windows 10 1703 to Windows 11 23H2] 1=>[Windows 8/8.1] 2=>[Windows 10 1507 to 1511] 3=>[Windows 10 1607]", + optional=False, + ), + requirements.TranslationLayerRequirement(name="memory_layer", optional=False) + ] + + def _generator(self): + base_layer = self.context.layers["base_layer"] + header = base_layer.read(0, 4) + if header == b'HIBR': + # Choose the most recently added layer that isn't virtual + self.config["layers"] = [] + for name in self.context.layers: + if not self.context.layers[name].metadata.get("mapped", False): + self.config["layers"] = [name] + for name in self.config["layers"]: + # Check the layer exists and validate the output file + if name not in self.context.layers: + yield 0, (f"Layer Name {name} does not exist",) + else: + output_name = self.config.get("output", ".".join([name, "raw"])) + try: + file_handle = layerwriter.LayerWriter.write_layer( + self.context, + name, + output_name, + self.open, + self.config.get("block_size", layerwriter.LayerWriter.default_block_size), + progress_callback=self._progress_callback, + ) + file_handle.close() + except IOError as excp: + yield 0, ( + f"Layer cannot be written to {self.config['output_name']}: {excp}", + ) + + yield 0, (f"The hibernation file was converted to {output_name}",) + elif header == b'RSTR': + yield (0, ("The hibernation file header signature is 'RSTR', the file cannot be exploited.")) + else: + yield (0, ("The file is not an hibernation file or is corrupted.")) + + def run(self): + return renderers.TreeGrid([("Status", str)], self._generator()) \ No newline at end of file From 142baa6f7462aadbccf487ac2ae8cc0285c4cbe0 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Fri, 3 Nov 2023 23:03:51 +0100 Subject: [PATCH 07/22] Adding support for Windows 10 2016 1607 --- volatility3/framework/layers/hib.py | 4 +++- volatility3/framework/plugins/windows/hibernation.py | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/layers/hib.py b/volatility3/framework/layers/hib.py index cfb08e1f5d..c09ab027b4 100644 --- a/volatility3/framework/layers/hib.py +++ b/volatility3/framework/layers/hib.py @@ -63,7 +63,9 @@ def __init__(self, context: interfaces.context.ContextInterface, config_path: st if self.version == 2: self.FKRP_OFFSET = 0x70 self.KPP_OFFSET = 0x218 - + if self.version == 3: + self.FKRP_OFFSET = 0x70 + self.KPP_OFFSET = 0x220 super().__init__(context, config_path, name, **kwargs) diff --git a/volatility3/framework/plugins/windows/hibernation.py b/volatility3/framework/plugins/windows/hibernation.py index 5e58512976..23f1e04aea 100644 --- a/volatility3/framework/plugins/windows/hibernation.py +++ b/volatility3/framework/plugins/windows/hibernation.py @@ -61,9 +61,6 @@ class Dump(plugins.PluginInterface): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.PluginRequirement( - name="hibernation", plugin=Info, version=(2, 0, 0) - ), requirements.PluginRequirement( name="layerwriter", plugin=layerwriter.LayerWriter, version=(2, 0, 0) ), From cfab14fdc2ddedbf66792afc4b87d4b795ea3279 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sat, 4 Nov 2023 00:26:06 +0100 Subject: [PATCH 08/22] Only parse the kernel section if the user is using the 'windows.hibernation.Dump' plugin (provide the version of Windows) --- volatility3/framework/layers/hib.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/volatility3/framework/layers/hib.py b/volatility3/framework/layers/hib.py index c09ab027b4..72235cd332 100644 --- a/volatility3/framework/layers/hib.py +++ b/volatility3/framework/layers/hib.py @@ -39,7 +39,6 @@ def __init__(self, context: interfaces.context.ContextInterface, config_path: st # Call the superclass constructor. self._compressed = {} self._mapping = {} - self.version = 0 #By default we want to analyze modern windows hiberfiles (Windows 10 2016 1703 to Windows 11 23H2) if "plugins.Dump.version" in context.config: # The user is using the hibernation.Dump plugin self.version = context.config["plugins.Dump.version"] @@ -83,8 +82,6 @@ def _load_segments(self): base_layer = self.context.layers[self._base_layer] NumPagesForLoader = int.from_bytes(base_layer.read(self.NPFL_OFFSET, 8), "little") FirstBootRestorePage = int.from_bytes(base_layer.read(self.FBRP_OFFSET, 8), "little") - FirstKernelRestorePage = int.from_bytes(base_layer.read(self.FKRP_OFFSET, 8), "little") - KernelPagesProcessed = int.from_bytes(base_layer.read(self.KPP_OFFSET, 8), "little") offset = FirstBootRestorePage * self.PAGE_SIZE total_pages = NumPagesForLoader @@ -95,14 +92,18 @@ def _load_segments(self): offset += next_cs treated += page_read - offset = FirstKernelRestorePage * self.PAGE_SIZE - total_pages = KernelPagesProcessed - - treated = 0 - while total_pages > treated: - page_read, next_cs = self._read_compression_set(offset) - offset += next_cs - treated += page_read + if "plugins.Dump.version" in self.context.config: + FirstKernelRestorePage = int.from_bytes(base_layer.read(self.FKRP_OFFSET, 8), "little") + KernelPagesProcessed = int.from_bytes(base_layer.read(self.KPP_OFFSET, 8), "little") + # The user is using the hibernation.Dump plugin + offset = FirstKernelRestorePage * self.PAGE_SIZE + total_pages = KernelPagesProcessed + + treated = 0 + while total_pages > treated: + page_read, next_cs = self._read_compression_set(offset) + offset += next_cs + treated += page_read self._segments = sorted(self._segments, key=lambda x: x[0]) From f6b960a4f06b1e9675339e3878a0f3df9b2e7c7d Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 4 Feb 2024 17:52:14 +0100 Subject: [PATCH 09/22] Quick code review and comments to make it more readable. Enhanced plugins (Info + Dump) --- .../framework/layers/codecs/__init__.py | 25 +- volatility3/framework/layers/hib.py | 247 ++++++++++++------ .../framework/plugins/windows/hibernation.py | 119 +++++---- 3 files changed, 258 insertions(+), 133 deletions(-) diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index 02f4f7e721..5f29d53b6e 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -1,6 +1,13 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# + + +# Temporary comment: +# Both decompression algorithm are implemented differently. +# The LZ77+Huffman decompression algorithm is largely inspired from the velocidex implementation. +# Need to know if we choose to raise an exeception inside the decompression algorithm (like plain LZ77) or along with the result (like the Huffman+LZ77) +# Both methods are presented here but this will need some discussions in the PR to choose a convention. + import struct, logging from typing import Tuple, List, Union vollog = logging.getLogger(__name__) @@ -198,6 +205,11 @@ def lz77_huffman_decompress_chunck(in_idx: int, def lz77_huffman_decompress(input: bytes, output_size: int) -> Tuple[bytes, Union[None, Exception]]: + """ + Refs : + - https://learn.microsoft.com/en-us/windows/win32/cmpapi/using-the-compression-api + - https://raw.githubusercontent.com/Velocidex/go-prefetch/master/lzxpress.go + """ output = bytearray(output_size) err = None @@ -208,8 +220,8 @@ def lz77_huffman_decompress(input: bytes, output_size: int) -> Tuple[bytes, Unio out_idx = 0 while True: - # How much data belongs in the current chunk. Chunks - # are split into maximum 65536 bytes. + # How much data belongs in the current chunk. + # Chunks are split into maximum 65536 bytes. chunk_size = output_size - out_idx if chunk_size > 65536: chunk_size = 65536 @@ -223,6 +235,10 @@ def lz77_huffman_decompress(input: bytes, output_size: int) -> Tuple[bytes, Unio return output, None def lz77_plain_decompress(in_buf): + """ + Refs : + - https://learn.microsoft.com/en-us/windows/win32/cmpapi/using-the-compression-api + """ out_idx = 0 in_idx = 0 nibble_idx = 0 @@ -287,6 +303,5 @@ def lz77_plain_decompress(in_buf): raise ValueError('CorruptedData') out_buf.append(out_buf[out_idx - offset]) out_idx += 1 - return bytes(out_buf) diff --git a/volatility3/framework/layers/hib.py b/volatility3/framework/layers/hib.py index 72235cd332..c7c5073e07 100644 --- a/volatility3/framework/layers/hib.py +++ b/volatility3/framework/layers/hib.py @@ -1,71 +1,111 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +# References: +# - https://www.forensicxlab.com/posts/hibernation/ : Vulgarized description of the hibernation file structure and the implementation of this layer. +# - https://www.cct.lsu.edu/~golden/Papers/sylvehiber.pdf : Scientific paper. +# - https://www.vergiliusproject.com/kernels/x64/ : Windows kernel structures used to track the evolution of the hibernation file structure in time. + from typing import Optional import logging, struct from volatility3.framework import interfaces, constants, exceptions from volatility3.framework.layers import segmented -from volatility3.framework.renderers import conversion -from volatility3.framework.layers.codecs import lz77_plain_decompress, lz77_huffman_decompress - +from volatility3.framework.layers.codecs import ( + lz77_plain_decompress, + lz77_huffman_decompress, +) vollog = logging.getLogger(__name__) -def uncompress(data: bytes, huffman, out_size): - if huffman == 0 or huffman == 1: - return lz77_plain_decompress(data) - elif huffman == 2 or huffman == 3: - return lz77_huffman_decompress(data,out_size)[0] - else: - vollog.warning(f"A compression set could not be decompressed : Compression algorithm : {huffman}") - raise ValueError('Cannot decompress the data.') - +def uncompress(data: bytes, flag): + """ + Desc: + Params: + - data: the compressed data from a compression set + - flag: what is the decompression algorithm to use. + - out_size: Size of the decompressed data + Return: The decompressed data (consecutive pages). + """ + if flag == 0 or flag == 1: + return lz77_plain_decompress(data) # See layers.codecs + elif flag == 2 or flag == 3: + return lz77_huffman_decompress(data, 65536)[0] # See layers.codecs + else: + vollog.warning( + f"A compression set could not be decompressed: Compression algorithm : {flag}" + ) + raise ValueError("Cannot decompress the data.") -class HibernationFileException(exceptions.LayerException): - """Thrown when an error occurs with the underlying Hibernation file format.""" class HibernationLayer(segmented.NonLinearlySegmentedLayer): """ - A TranslationLayer that maps physical memory against a Microsoft Windows hibernation file (x64 only for now). + A TranslationLayer that maps physical memory against a x64 Microsoft Windows hibernation file. + This Translation Layer is meant to be used in conjunction with the Hibernation.Info and Hibernation.Dump plugins. """ - PAGE_SIZE = 4096 #For x64 by default + + WINDOWS_10_2016_1703_TO_23H2 = 0 + WINDOW_8 = 1 + WINDOWS_10_2016_1507_1511 = 2 + WINDOWS_10_2016_1607 = 3 + + # TODO: Make me compatible with x86 by adding options to the Hib plugins. + PAGE_SIZE = 4096 # x64 page size. HEADER_SIZE = 4 PAGE_DESC_SIZE = 8 - def __init__(self, context: interfaces.context.ContextInterface, config_path: str, name: str, **kwargs): + + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + name: str, + **kwargs, + ): """ Initializes the Hibernation file layer. """ - # Call the superclass constructor. - self._compressed = {} - self._mapping = {} + self._compressed = ( + {} + ) # Keep track of which compression algorithm by each mapped compressed data. + self._mapping = ( + {} + ) # This will hold the mapping between the PageNumber in the decompressed data vs the physical page number. + if "plugins.Dump.version" in context.config: - # The user is using the hibernation.Dump plugin + # The user is using the hibernation.Dump plugin, so the version must be known. + # See possible version in the table below. self.version = context.config["plugins.Dump.version"] - self.NPFL_OFFSET = 0x058 - self.FBRP_OFFSET = 0x068 + else: + self.version = -1 + + self.NPFL_OFFSET = 0x058 # (NumPagesForLoader) + self.FBRP_OFFSET = 0x068 # (FirstBootRestorePage) """ - | Windows Versions | FirstKernelRestorePage | KernelPagesProcessed | - | ------------------------------------------|:----------------------:|:----------------------:| - | Windows 8/8.1 | 0x68 | 0x1C8 | - | Windows 10 2016 1507-1511 | 0x70 | 0x218 | - | Windows 10 2016 1607 | 0x70 | 0x220 | - | Windows 10 2016 1703 - Windows 11 23H2 | 0x70 | 0x230 | + Mapping for each 'group' of Windows version sharing the same offsets + --------------------------------------------------------------------------------------------------------- + | Windows Versions | FirstKernelRestorePage (FKRP) | KernelPagesProcessed (KPP)| + | ------------------------------------------|:-----------------------------:|:-------------------------:| + | Windows 8/8.1 | 0x68 | 0x1C8 | + | Windows 10 2016 1507-1511 | 0x70 | 0x218 | + | Windows 10 2016 1607 | 0x70 | 0x220 | + | Windows 10 2016 1703 - Windows 11 23H2 | 0x70 | 0x230 | + --------------------------------------------------------------------------------------------------------- """ - if self.version == 0: + if self.version == self.WINDOWS_10_2016_1703_TO_23H2: self.FKRP_OFFSET = 0x070 self.KPP_OFFSET = 0x230 - if self.version == 1: + elif self.version == self.WINDOW_8: self.FKRP_OFFSET = 0x68 self.KPP_OFFSET = 0x1C8 - if self.version == 2: + elif self.version == self.WINDOWS_10_2016_1507_1511: self.FKRP_OFFSET = 0x70 self.KPP_OFFSET = 0x218 - if self.version == 3: + elif self.version == self.WINDOWS_10_2016_1607: self.FKRP_OFFSET = 0x70 self.KPP_OFFSET = 0x220 - + else: + raise exceptions.LayerException(name, "The version provided is not valid") super().__init__(context, config_path, name, **kwargs) @classmethod @@ -73,99 +113,142 @@ def _check_header( cls, base_layer: interfaces.layers.DataLayerInterface, name: str = "" ): header = base_layer.read(0, 4) - if header != b'HIBR': + if header != b"HIBR": raise exceptions.LayerException(name, "No Hibernation magic bytes") - else: + else: vollog.info("Detecting an hibernation file") def _load_segments(self): + """ + Loading segments is a 2 STEP operation: + - Step 1: extracting the pages from the BootSection if any. + - Step 2: extracting the pages from the KernelSection if any. + """ base_layer = self.context.layers[self._base_layer] - NumPagesForLoader = int.from_bytes(base_layer.read(self.NPFL_OFFSET, 8), "little") - FirstBootRestorePage = int.from_bytes(base_layer.read(self.FBRP_OFFSET, 8), "little") - + NumPagesForLoader = int.from_bytes( + base_layer.read(self.NPFL_OFFSET, 8), "little" + ) + FirstBootRestorePage = int.from_bytes( + base_layer.read(self.FBRP_OFFSET, 8), "little" + ) + offset = FirstBootRestorePage * self.PAGE_SIZE total_pages = NumPagesForLoader treated = 0 while total_pages > treated: - page_read, next_cs = self._read_compression_set(offset) - offset += next_cs + page_read, next_compression_set = self._read_compression_set(offset) + offset += next_compression_set treated += page_read if "plugins.Dump.version" in self.context.config: - FirstKernelRestorePage = int.from_bytes(base_layer.read(self.FKRP_OFFSET, 8), "little") - KernelPagesProcessed = int.from_bytes(base_layer.read(self.KPP_OFFSET, 8), "little") - # The user is using the hibernation.Dump plugin + # The user is using the hibernation.Dump plugin so we can parse the KernelSection + FirstKernelRestorePage = int.from_bytes( + base_layer.read(self.FKRP_OFFSET, 8), "little" + ) + KernelPagesProcessed = int.from_bytes( + base_layer.read(self.KPP_OFFSET, 8), "little" + ) offset = FirstKernelRestorePage * self.PAGE_SIZE total_pages = KernelPagesProcessed - + treated = 0 while total_pages > treated: - page_read, next_cs = self._read_compression_set(offset) - offset += next_cs - treated += page_read - + page_read, next_compression_set = self._read_compression_set(offset) + offset += next_compression_set + treated += page_read self._segments = sorted(self._segments, key=lambda x: x[0]) def _read_compression_set(self, offset): """ - Desc : Read one compression set an extract the address of the compressed data - Params : - - offset : the location of the compression set to read. - - stream : the hibernation file stream. - Return : The offset of the compressed data and the size. + Desc: Read one compression set an extract the address of the compressed data + Params: + - offset : the location of the compression set to read. + - stream : the hibernation file stream. + Return: The offset of the compressed data and the size. """ - base_layer = self.context.layers[self._base_layer] + base_layer = self.context.layers[self._base_layer] header = base_layer.read(offset, self.HEADER_SIZE) - data = struct.unpack('> 8) & 0x3fffff - huffman_compressed = (data >> 30) & 0x3 - mapped_address = offset+self.HEADER_SIZE+number_of_descs*self.PAGE_DESC_SIZE + data = struct.unpack(" 16: + # See references + raise exceptions.LayerException( + self.name, "The hibernation file is corrupted." + ) + + size_of_compressed_data = ( + data >> 8 + ) & 0x3FFFFF # Next 22 least significant bytes. + huffman_compressed = (data >> 30) & 0x3 # Most significant bit. + + # Now we know where is the start of the page descriptors in the hibernation file. + mapped_address = ( + offset + self.HEADER_SIZE + number_of_descs * self.PAGE_DESC_SIZE + ) total_page_count = 0 position = 0 for i in range(number_of_descs): - location = offset+self.HEADER_SIZE+i*self.PAGE_DESC_SIZE + # Go fetch and parse each page descriptor. + location = offset + self.HEADER_SIZE + i * self.PAGE_DESC_SIZE page_descriptor = base_layer.read(location, self.PAGE_DESC_SIZE) - data = struct.unpack('> 4 # shift right 4 bits to get the upper 60 bits - page_count = (1+ Numpages) + data = struct.unpack("> 4 # Shift right 4 bits to get the upper 60 bits + page_count = 1 + Numpages total_page_count += page_count self._segments.append( - (PageNum*self.PAGE_SIZE, - mapped_address, - self.PAGE_SIZE*page_count, - size_of_compressed_data + ( + PageNum * self.PAGE_SIZE, + mapped_address, + self.PAGE_SIZE * page_count, + size_of_compressed_data, ) ) for j in range(page_count): - self._mapping[(PageNum+j)*self.PAGE_SIZE] = position*self.PAGE_SIZE + # Track the physical page number vs the page number in the compression set + self._mapping[(PageNum + j) * self.PAGE_SIZE] = ( + position * self.PAGE_SIZE + ) position += 1 - total_page_size = total_page_count*self.PAGE_SIZE + total_page_size = total_page_count * self.PAGE_SIZE + if total_page_size != size_of_compressed_data: + # This means compression so we track wich compression sets we actually need to decompress self._compressed[mapped_address] = huffman_compressed - return total_page_count, (4 + size_of_compressed_data + number_of_descs * self.PAGE_DESC_SIZE) #Number of pages in the set, Size of the entire compression set - + return total_page_count, ( + 4 + size_of_compressed_data + number_of_descs * self.PAGE_DESC_SIZE + ) # Number of pages in the set, Size of the entire compression set def _decode_data( self, data: bytes, mapped_offset: int, offset: int, output_length: int ) -> bytes: - start_offset, _, _, _ = self._find_segment(offset) + """ + Desc: decode the compressed data of one compression set + Params: + - data : the compressed data + - mapped_offset : starting location of the compressed data in the hib file + - offset: The offset inside the resulting raw file + - output_length: what is the size of the expected decompressed pages + Return: The decompressed data + """ + start_offset, _mapped_offset, _size, _mapped_size = self._find_segment(offset) if mapped_offset in self._compressed: - decoded_data = uncompress(data, self._compressed[mapped_offset],65536) + decoded_data = uncompress(data=data, flag=self._compressed[mapped_offset]) else: + # The data is not in our mapping so it's uncompressed. decoded_data = data page_offset = self._mapping[start_offset] - decoded_data = decoded_data[page_offset + (offset - start_offset):] - decoded_data = decoded_data[:output_length] + decoded_data = decoded_data[page_offset + (offset - start_offset) :] + decoded_data = decoded_data[:output_length] return decoded_data class HibernationFileStacker(interfaces.automagic.StackerLayerInterface): stack_order = 10 + @classmethod def stack( cls, @@ -178,9 +261,9 @@ def stack( except exceptions.LayerException: return None new_name = context.layers.free_layer_name("HibernationLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) layer = HibernationLayer(context, new_name, new_name) cls.stacker_slow_warning() - return layer \ No newline at end of file + return layer diff --git a/volatility3/framework/plugins/windows/hibernation.py b/volatility3/framework/plugins/windows/hibernation.py index 23f1e04aea..46d306d07a 100644 --- a/volatility3/framework/plugins/windows/hibernation.py +++ b/volatility3/framework/plugins/windows/hibernation.py @@ -1,4 +1,4 @@ -# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # @@ -13,6 +13,7 @@ vollog = logging.getLogger(__name__) + class Info(plugins.PluginInterface): """Plugin to parse an hiberfil.sys to make sure it is safe to be converted to a raw file and not corrupted""" @@ -29,35 +30,60 @@ def _generator(self): base_layer = self.context.layers["base_layer"] header = base_layer.read(0, 4) yield (0, ("Signature", str(header))) - if header == b'HIBR': + if header == b"HIBR": # The hibernation file seems exploitable. Next step is to extract important information for the examiner PageSize = int.from_bytes(base_layer.read(0x18, 4), "little") yield (0, ("PageSize", str(PageSize))) if PageSize == 4096: - yield (0, ("Comment", "The hibernation file header signature is correct.")) + yield ( + 0, + ("Comment", "The hibernation file header signature is correct."), + ) system_time = int.from_bytes(base_layer.read(0x020, 8), "little") systemTime = conversion.wintime_to_datetime(system_time) yield (0, ("System Time", str(systemTime))) - FirstBootRestorePage = int.from_bytes(base_layer.read(0x068, 8), "little") + FirstBootRestorePage = int.from_bytes( + base_layer.read(0x068, 8), "little" + ) yield (0, ("FirstBootRestorePage", str(hex(FirstBootRestorePage)))) NumPagesForLoader = int.from_bytes(base_layer.read(0x058, 8), "little") yield (0, ("NumPagesForLoader", str(NumPagesForLoader))) elif PageSize == 2048: - yield (0, ("Comment", "The hibernation file header signature is correct but x32 compatibility is not available yet.")) + yield ( + 0, + ( + "Comment", + "The hibernation file header signature is correct but x32 compatibility is not available yet.", + ), + ) else: yield (0, ("Comment : ", "The file is corrupted.")) - elif header == b'RSTR': + elif header == b"RSTR": # The hibernation file was extracted when Windows was in a resuming state which makes it not exploitable - yield (0, ("Comment : ", "The hibernation file header signature is 'RSTR', the file cannot be exploited.")) + yield ( + 0, + ( + "Comment : ", + "The hibernation file header signature is 'RSTR', the file cannot be exploited.", + ), + ) else: - yield (0, ("Comment : ", "The file is not an hibernation file or is corrupted.")) + yield ( + 0, + ("Comment : ", "The file is not an hibernation file or is corrupted."), + ) + def run(self): - return renderers.TreeGrid([("Variable", str), ("Value", str)], self._generator()) - + return renderers.TreeGrid( + [("Variable", str), ("Value", str)], self._generator() + ) + class Dump(plugins.PluginInterface): - """Plugin to parse an hiberfil.sys to make sure it is safe to be converted to a raw file and not corrupted""" + """Plugin to convert an hiberfil.sys to a raw file""" + _required_framework_version = (2, 0, 0) + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ @@ -69,44 +95,45 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] description="The Windows version of the hibernation file : 0=>[Windows 10 1703 to Windows 11 23H2] 1=>[Windows 8/8.1] 2=>[Windows 10 1507 to 1511] 3=>[Windows 10 1607]", optional=False, ), - requirements.TranslationLayerRequirement(name="memory_layer", optional=False) + requirements.TranslationLayerRequirement( + name="memory_layer", optional=False + ), ] def _generator(self): - base_layer = self.context.layers["base_layer"] - header = base_layer.read(0, 4) - if header == b'HIBR': - # Choose the most recently added layer that isn't virtual - self.config["layers"] = [] - for name in self.context.layers: - if not self.context.layers[name].metadata.get("mapped", False): - self.config["layers"] = [name] - for name in self.config["layers"]: - # Check the layer exists and validate the output file - if name not in self.context.layers: - yield 0, (f"Layer Name {name} does not exist",) - else: - output_name = self.config.get("output", ".".join([name, "raw"])) - try: - file_handle = layerwriter.LayerWriter.write_layer( - self.context, - name, - output_name, - self.open, - self.config.get("block_size", layerwriter.LayerWriter.default_block_size), - progress_callback=self._progress_callback, - ) - file_handle.close() - except IOError as excp: - yield 0, ( - f"Layer cannot be written to {self.config['output_name']}: {excp}", - ) - - yield 0, (f"The hibernation file was converted to {output_name}",) - elif header == b'RSTR': - yield (0, ("The hibernation file header signature is 'RSTR', the file cannot be exploited.")) + """ + Check if the memory_layer is indeed the HibernationLayer, then perform the conversion using layerwritter. + """ + if self.context.layers["memory_layer"].__class__.__name__ == "HibernationLayer": + output_name = self.config.get("output", ".".join(["memory_layer", "raw"])) + try: + file_handle = layerwriter.LayerWriter.write_layer( + self.context, + "memory_layer", + output_name, + self.open, + self.config.get( + "block_size", layerwriter.LayerWriter.default_block_size + ), + progress_callback=self._progress_callback, + ) + file_handle.close() + except IOError as excp: + yield 0, ( + f"Layer cannot be written to {self.config['output_name']}: {excp}", + ) + yield 0, (f"The hibernation file was converted to {output_name}",) else: - yield (0, ("The file is not an hibernation file or is corrupted.")) + yield ( + 0, + ( + """Your hibernation file could not be converted, this can be the case for multiple reasons: + - The hibernation file you are trying to dump is corrupted. + - The version you provided is not expected (see --help) + - The file you are trying to dump is not an hibernation file. + """, + ), + ) def run(self): - return renderers.TreeGrid([("Status", str)], self._generator()) \ No newline at end of file + return renderers.TreeGrid([("Status", str)], self._generator()) From 8db03c34e8052e88e18f6cd2d89260ca6686b81a Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 4 Feb 2024 17:53:59 +0100 Subject: [PATCH 10/22] using black on the codecs --- .../framework/layers/codecs/__init__.py | 99 +++++++++++-------- 1 file changed, 57 insertions(+), 42 deletions(-) diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index 5f29d53b6e..ee093e29e5 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -2,14 +2,15 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# Temporary comment: -# Both decompression algorithm are implemented differently. +# Temporary comment: +# Both decompression algorithm are implemented differently. # The LZ77+Huffman decompression algorithm is largely inspired from the velocidex implementation. # Need to know if we choose to raise an exeception inside the decompression algorithm (like plain LZ77) or along with the result (like the Huffman+LZ77) # Both methods are presented here but this will need some discussions in the PR to choose a convention. import struct, logging from typing import Tuple, List, Union + vollog = logging.getLogger(__name__) @@ -18,8 +19,8 @@ def __init__(self, source: bytes, in_pos: int): self.source = source self.index = in_pos + 4 # read UInt16 little endian - mask = struct.unpack_from(' int: if n == 0: return 0 return self.mask >> (32 - n) - + def skip(self, n: int) -> Union[None, Exception]: - self.mask = ((self.mask << n) & 0xFFFFFFFF) + self.mask = (self.mask << n) & 0xFFFFFFFF self.bits -= n if self.bits < 16: if self.index + 2 > len(self.source): return Exception("EOF Error") # read UInt16 little endian - self.mask += ((struct.unpack_from(' Union[None, Exception]: def __str__(self): return f"{self.id}: symbol {self.symbol} length {self.length}" + + class PREFIX_CODE_NODE: def __init__(self): self.id = 0 @@ -53,6 +59,7 @@ def __init__(self): def __str__(self): return f"Node {self.id}: symbol {self.symbol} leaf {self.leaf}" + class PREFIX_CODE_SYMBOL: def __init__(self): self.id = 0 @@ -63,7 +70,9 @@ def __str__(self): return f"Symbol {self.id}: symbol {self.symbol} length {self.length}" -def prefix_code_tree_add_leaf(treeNodes: List[PREFIX_CODE_NODE], leafIndex: int, mask: int, bits: int) -> int: +def prefix_code_tree_add_leaf( + treeNodes: List[PREFIX_CODE_NODE], leafIndex: int, mask: int, bits: int +) -> int: node = treeNodes[0] i = leafIndex + 1 childIndex = None @@ -77,10 +86,11 @@ def prefix_code_tree_add_leaf(treeNodes: List[PREFIX_CODE_NODE], leafIndex: int, i += 1 node = node.child[childIndex] - node.child[mask&1] = treeNodes[leafIndex] + node.child[mask & 1] = treeNodes[leafIndex] return i + def prefix_code_tree_rebuild(input: bytes) -> PREFIX_CODE_NODE: treeNodes = [PREFIX_CODE_NODE() for _ in range(1024)] symbolInfo = [PREFIX_CODE_SYMBOL() for _ in range(512)] @@ -88,15 +98,15 @@ def prefix_code_tree_rebuild(input: bytes) -> PREFIX_CODE_NODE: for i in range(256): value = input[i] - symbolInfo[2*i].id = 2 * i - symbolInfo[2*i].symbol = 2 * i - symbolInfo[2*i].length = value & 0xf + symbolInfo[2 * i].id = 2 * i + symbolInfo[2 * i].symbol = 2 * i + symbolInfo[2 * i].length = value & 0xF value >>= 4 - symbolInfo[2*i+1].id = 2*i + 1 - symbolInfo[2*i+1].symbol = 2*i + 1 - symbolInfo[2*i+1].length = value & 0xf + symbolInfo[2 * i + 1].id = 2 * i + 1 + symbolInfo[2 * i + 1].symbol = 2 * i + 1 + symbolInfo[2 * i + 1].length = value & 0xF symbolInfo = sorted(symbolInfo, key=lambda x: (x.length, x.symbol)) @@ -123,7 +133,10 @@ def prefix_code_tree_rebuild(input: bytes) -> PREFIX_CODE_NODE: return root -def prefix_code_tree_decode_symbol(bstr: BitStream, root: PREFIX_CODE_NODE) -> Tuple[int, Union[None, Exception]]: + +def prefix_code_tree_decode_symbol( + bstr: BitStream, root: PREFIX_CODE_NODE +) -> Tuple[int, Union[None, Exception]]: node = root i = 0 while True: @@ -142,27 +155,26 @@ def prefix_code_tree_decode_symbol(bstr: BitStream, root: PREFIX_CODE_NODE) -> T break return node.symbol, None -def lz77_huffman_decompress_chunck(in_idx: int, - input: bytes, - out_idx: int, - output: bytearray, - chunk_size: int) -> Tuple[int, int, Union[None, Exception]]: - + +def lz77_huffman_decompress_chunck( + in_idx: int, input: bytes, out_idx: int, output: bytearray, chunk_size: int +) -> Tuple[int, int, Union[None, Exception]]: + # Ensure there are at least 256 bytes available to read if in_idx + 256 > len(input): return 0, 0, Exception("EOF Error") root = prefix_code_tree_rebuild(input[in_idx:]) - bstr = BitStream(input, in_idx+256) + bstr = BitStream(input, in_idx + 256) i = out_idx while i < out_idx + chunk_size: symbol, err = prefix_code_tree_decode_symbol(bstr, root) - + if err is not None: return int(bstr.index), i, err - + if symbol < 256: output[i] = symbol i += 1 @@ -181,32 +193,34 @@ def lz77_huffman_decompress_chunck(in_idx: int, if length == 15: length = bstr.source[bstr.index] + 15 bstr.index += 1 - + if length == 270: - length = struct.unpack_from(' 0: if i + offset < 0: vollog.warning("Some data could not be decompressed.") return int(bstr.index), i, Exception("Decompression Error") - + output[i] = output[i + offset] i += 1 length -= 1 - if length==0: + if length == 0: break return int(bstr.index), i, None -def lz77_huffman_decompress(input: bytes, output_size: int) -> Tuple[bytes, Union[None, Exception]]: +def lz77_huffman_decompress( + input: bytes, output_size: int +) -> Tuple[bytes, Union[None, Exception]]: """ - Refs : + Refs : - https://learn.microsoft.com/en-us/windows/win32/cmpapi/using-the-compression-api - https://raw.githubusercontent.com/Velocidex/go-prefetch/master/lzxpress.go """ @@ -227,16 +241,18 @@ def lz77_huffman_decompress(input: bytes, output_size: int) -> Tuple[bytes, Unio chunk_size = 65536 in_idx, out_idx, err = lz77_huffman_decompress_chunck( - in_idx, input, out_idx, output, chunk_size) + in_idx, input, out_idx, output, chunk_size + ) if err is not None: return output, err if out_idx >= len(output) or in_idx >= len(input): break return output, None + def lz77_plain_decompress(in_buf): """ - Refs : + Refs : - https://learn.microsoft.com/en-us/windows/win32/cmpapi/using-the-compression-api """ out_idx = 0 @@ -250,7 +266,7 @@ def lz77_plain_decompress(in_buf): while in_idx < len(in_buf): if flag_count == 0: - flags = struct.unpack(' out_idx: - raise ValueError('CorruptedData') + raise ValueError("CorruptedData") out_buf.append(out_buf[out_idx - offset]) out_idx += 1 return bytes(out_buf) - From d152b485777f50fd9553007f5593e68797c4a875 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 4 Feb 2024 18:03:04 +0100 Subject: [PATCH 11/22] fixing mistakes in the lz77+huffman decompression algorithm --- volatility3/framework/layers/codecs/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index ee093e29e5..bc72bd6aef 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -80,7 +80,7 @@ def prefix_code_tree_add_leaf( while bits > 1: bits -= 1 childIndex = (mask >> bits) & 1 - if node.child[childIndex] == None: + if not node.child[childIndex]: node.child[childIndex] = treeNodes[i] treeNodes[i].leaf = False i += 1 @@ -138,7 +138,6 @@ def prefix_code_tree_decode_symbol( bstr: BitStream, root: PREFIX_CODE_NODE ) -> Tuple[int, Union[None, Exception]]: node = root - i = 0 while True: bit = bstr.lookup(1) err = bstr.skip(1) @@ -147,10 +146,9 @@ def prefix_code_tree_decode_symbol( return 0, err node = node.child[bit] - if node == None: + if not node: vollog.warning("Corruption detected when decompressing the data.") return 0, Exception("Corruption detected") - if node.leaf: break return node.symbol, None From 6213d45d9572163975bc4ea1e1870bd96cf1f833 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Mon, 15 Jul 2024 22:36:24 +0200 Subject: [PATCH 12/22] Adding codecs + black --- .../framework/layers/codecs/__init__.py | 322 +----------------- .../framework/layers/codecs/lz77huffman.py | 288 ++++++++++++++++ .../framework/layers/codecs/lz77plain.py | 127 +++++++ volatility3/framework/layers/hib.py | 29 +- 4 files changed, 436 insertions(+), 330 deletions(-) create mode 100644 volatility3/framework/layers/codecs/lz77huffman.py create mode 100644 volatility3/framework/layers/codecs/lz77plain.py diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index bc72bd6aef..b2d1eef6ce 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -1,320 +1,10 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +import codecs +from volatility3.framework.layers.codecs.lz77plain import find_lz77_plain +from volatility3.framework.layers.codecs.lz77huffman import find_lz77_huffman -# Temporary comment: -# Both decompression algorithm are implemented differently. -# The LZ77+Huffman decompression algorithm is largely inspired from the velocidex implementation. -# Need to know if we choose to raise an exeception inside the decompression algorithm (like plain LZ77) or along with the result (like the Huffman+LZ77) -# Both methods are presented here but this will need some discussions in the PR to choose a convention. - -import struct, logging -from typing import Tuple, List, Union - -vollog = logging.getLogger(__name__) - - -class BitStream: - def __init__(self, source: bytes, in_pos: int): - self.source = source - self.index = in_pos + 4 - # read UInt16 little endian - mask = struct.unpack_from(" int: - if n == 0: - return 0 - return self.mask >> (32 - n) - - def skip(self, n: int) -> Union[None, Exception]: - self.mask = (self.mask << n) & 0xFFFFFFFF - self.bits -= n - if self.bits < 16: - if self.index + 2 > len(self.source): - return Exception("EOF Error") - # read UInt16 little endian - self.mask += ( - (struct.unpack_from(" int: - node = treeNodes[0] - i = leafIndex + 1 - childIndex = None - - while bits > 1: - bits -= 1 - childIndex = (mask >> bits) & 1 - if not node.child[childIndex]: - node.child[childIndex] = treeNodes[i] - treeNodes[i].leaf = False - i += 1 - node = node.child[childIndex] - - node.child[mask & 1] = treeNodes[leafIndex] - - return i - - -def prefix_code_tree_rebuild(input: bytes) -> PREFIX_CODE_NODE: - treeNodes = [PREFIX_CODE_NODE() for _ in range(1024)] - symbolInfo = [PREFIX_CODE_SYMBOL() for _ in range(512)] - - for i in range(256): - value = input[i] - - symbolInfo[2 * i].id = 2 * i - symbolInfo[2 * i].symbol = 2 * i - symbolInfo[2 * i].length = value & 0xF - - value >>= 4 - - symbolInfo[2 * i + 1].id = 2 * i + 1 - symbolInfo[2 * i + 1].symbol = 2 * i + 1 - symbolInfo[2 * i + 1].length = value & 0xF - - symbolInfo = sorted(symbolInfo, key=lambda x: (x.length, x.symbol)) - - i = 0 - while i < 512 and symbolInfo[i].length == 0: - i += 1 - - mask = 0 - bits = 1 - - root = treeNodes[0] - root.leaf = False - - j = 1 - while i < 512: - treeNodes[j].id = j - treeNodes[j].symbol = symbolInfo[i].symbol - treeNodes[j].leaf = True - mask = mask << (symbolInfo[i].length - bits) - bits = symbolInfo[i].length - j = prefix_code_tree_add_leaf(treeNodes, j, mask, bits) - mask += 1 - i += 1 - - return root - - -def prefix_code_tree_decode_symbol( - bstr: BitStream, root: PREFIX_CODE_NODE -) -> Tuple[int, Union[None, Exception]]: - node = root - while True: - bit = bstr.lookup(1) - err = bstr.skip(1) - if err is not None: - vollog.warning("Some data could not be decompressed.") - return 0, err - - node = node.child[bit] - if not node: - vollog.warning("Corruption detected when decompressing the data.") - return 0, Exception("Corruption detected") - if node.leaf: - break - return node.symbol, None - - -def lz77_huffman_decompress_chunck( - in_idx: int, input: bytes, out_idx: int, output: bytearray, chunk_size: int -) -> Tuple[int, int, Union[None, Exception]]: - - # Ensure there are at least 256 bytes available to read - if in_idx + 256 > len(input): - return 0, 0, Exception("EOF Error") - - root = prefix_code_tree_rebuild(input[in_idx:]) - bstr = BitStream(input, in_idx + 256) - - i = out_idx - - while i < out_idx + chunk_size: - symbol, err = prefix_code_tree_decode_symbol(bstr, root) - - if err is not None: - return int(bstr.index), i, err - - if symbol < 256: - output[i] = symbol - i += 1 - else: - symbol -= 256 - length = symbol & 15 - symbol >>= 4 - - offset = 0 - if symbol != 0: - offset = int(bstr.lookup(symbol)) - - offset |= 1 << symbol - offset = -offset - - if length == 15: - length = bstr.source[bstr.index] + 15 - bstr.index += 1 - - if length == 270: - length = struct.unpack_from(" 0: - if i + offset < 0: - vollog.warning("Some data could not be decompressed.") - return int(bstr.index), i, Exception("Decompression Error") - - output[i] = output[i + offset] - i += 1 - length -= 1 - if length == 0: - break - return int(bstr.index), i, None - - -def lz77_huffman_decompress( - input: bytes, output_size: int -) -> Tuple[bytes, Union[None, Exception]]: - """ - Refs : - - https://learn.microsoft.com/en-us/windows/win32/cmpapi/using-the-compression-api - - https://raw.githubusercontent.com/Velocidex/go-prefetch/master/lzxpress.go - """ - output = bytearray(output_size) - err = None - - # Index into the input buffer. - in_idx = 0 - - # Index into the output buffer. - out_idx = 0 - - while True: - # How much data belongs in the current chunk. - # Chunks are split into maximum 65536 bytes. - chunk_size = output_size - out_idx - if chunk_size > 65536: - chunk_size = 65536 - - in_idx, out_idx, err = lz77_huffman_decompress_chunck( - in_idx, input, out_idx, output, chunk_size - ) - if err is not None: - return output, err - if out_idx >= len(output) or in_idx >= len(input): - break - return output, None - - -def lz77_plain_decompress(in_buf): - """ - Refs : - - https://learn.microsoft.com/en-us/windows/win32/cmpapi/using-the-compression-api - """ - out_idx = 0 - in_idx = 0 - nibble_idx = 0 - - flags = 0 - flag_count = 0 - - out_buf = [] - - while in_idx < len(in_buf): - if flag_count == 0: - flags = struct.unpack(" out_idx: - raise ValueError("CorruptedData") - out_buf.append(out_buf[out_idx - offset]) - out_idx += 1 - return bytes(out_buf) +codecs.register(find_lz77_plain) +codecs.register(find_lz77_huffman) +# Register other codecs here. diff --git a/volatility3/framework/layers/codecs/lz77huffman.py b/volatility3/framework/layers/codecs/lz77huffman.py new file mode 100644 index 0000000000..ba6fea11bd --- /dev/null +++ b/volatility3/framework/layers/codecs/lz77huffman.py @@ -0,0 +1,288 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 + +import struct, logging, codecs +from typing import Tuple, List, Union + +vollog = logging.getLogger(__name__) + + +class BitStream: + def __init__(self, source: bytes, in_pos: int): + self.source = source + self.index = in_pos + 4 + # read UInt16 little endian + mask = struct.unpack_from(" int: + if n == 0: + return 0 + return self.mask >> (32 - n) + + def skip(self, n: int) -> Union[None, Exception]: + self.mask = (self.mask << n) & 0xFFFFFFFF + self.bits -= n + if self.bits < 16: + if self.index + 2 > len(self.source): + return Exception("EOF Error") + # read UInt16 little endian + self.mask += ( + (struct.unpack_from(" int: + node = treeNodes[0] + i = leafIndex + 1 + childIndex = None + + while bits > 1: + bits -= 1 + childIndex = (mask >> bits) & 1 + if not node.child[childIndex]: + node.child[childIndex] = treeNodes[i] + treeNodes[i].leaf = False + i += 1 + node = node.child[childIndex] + + node.child[mask & 1] = treeNodes[leafIndex] + + return i + + +def prefix_code_tree_rebuild(input: bytes) -> PREFIX_CODE_NODE: + treeNodes = [PREFIX_CODE_NODE() for _ in range(1024)] + symbolInfo = [PREFIX_CODE_SYMBOL() for _ in range(512)] + + for i in range(256): + value = input[i] + + symbolInfo[2 * i].id = 2 * i + symbolInfo[2 * i].symbol = 2 * i + symbolInfo[2 * i].length = value & 0xF + + value >>= 4 + + symbolInfo[2 * i + 1].id = 2 * i + 1 + symbolInfo[2 * i + 1].symbol = 2 * i + 1 + symbolInfo[2 * i + 1].length = value & 0xF + + symbolInfo = sorted(symbolInfo, key=lambda x: (x.length, x.symbol)) + + i = 0 + while i < 512 and symbolInfo[i].length == 0: + i += 1 + + mask = 0 + bits = 1 + + root = treeNodes[0] + root.leaf = False + + j = 1 + while i < 512: + treeNodes[j].id = j + treeNodes[j].symbol = symbolInfo[i].symbol + treeNodes[j].leaf = True + mask = mask << (symbolInfo[i].length - bits) + bits = symbolInfo[i].length + j = prefix_code_tree_add_leaf(treeNodes, j, mask, bits) + mask += 1 + i += 1 + + return root + + +def prefix_code_tree_decode_symbol( + bstr: BitStream, root: PREFIX_CODE_NODE +) -> Tuple[int, Union[None, Exception]]: + node = root + while True: + bit = bstr.lookup(1) + err = bstr.skip(1) + if err is not None: + vollog.warning("Some data could not be decompressed.") + return 0, err + + node = node.child[bit] + if not node: + vollog.warning("Corruption detected when decompressing the data.") + return 0, Exception("Corruption detected") + if node.leaf: + break + return node.symbol, None + + +class Lz77HuffmanCodec: + def encode(self, input: bytes, errors: str = "strict") -> Tuple[bytes, int]: + raise NotImplementedError("Encoding not implemented for LZ77 Huffman.") + + def decode(self, input: bytes, errors: str = "strict") -> Tuple[bytes, int]: + output_size = 65536 + output, err = self.lz77_huffman_decompress(input, output_size) + if err is not None: + raise Exception(f"Decoding failed: {err}") + return output, len(input) + + @staticmethod + def lz77_huffman_decompress( + input: bytes, output_size: int + ) -> Tuple[bytes, Union[None, Exception]]: + output = bytearray(output_size) + err = None + + # Index into the input buffer. + in_idx = 0 + + # Index into the output buffer. + out_idx = 0 + + while True: + # How much data belongs in the current chunk. + # Chunks are split into maximum 65536 bytes. + chunk_size = output_size - out_idx + if chunk_size > 65536: + chunk_size = 65536 + + in_idx, out_idx, err = Lz77HuffmanCodec.lz77_huffman_decompress_chunk( + in_idx, input, out_idx, output, chunk_size + ) + if err is not None: + return bytes(output), err + if out_idx >= len(output) or in_idx >= len(input): + break + return bytes(output), None + + @staticmethod + def lz77_huffman_decompress_chunk( + in_idx: int, input: bytes, out_idx: int, output: bytearray, chunk_size: int + ) -> Tuple[int, int, Union[None, Exception]]: + if in_idx + 256 > len(input): + return 0, 0, Exception("EOF Error") + + root = prefix_code_tree_rebuild(input[in_idx:]) + bstr = BitStream(input, in_idx + 256) + + i = out_idx + + while i < out_idx + chunk_size: + symbol, err = prefix_code_tree_decode_symbol(bstr, root) + + if err is not None: + return int(bstr.index), i, err + + if symbol < 256: + output[i] = symbol + i += 1 + else: + symbol -= 256 + length = symbol & 15 + symbol >>= 4 + + offset = 0 + if symbol != 0: + offset = int(bstr.lookup(symbol)) + + offset |= 1 << symbol + offset = -offset + + if length == 15: + length = bstr.source[bstr.index] + 15 + bstr.index += 1 + + if length == 270: + length = struct.unpack_from(" 0: + if i + offset < 0: + vollog.warning("Some data could not be decompressed.") + return int(bstr.index), i, Exception("Decompression Error") + + output[i] = output[i + offset] + i += 1 + length -= 1 + if length == 0: + break + return int(bstr.index), i, None + + +class Lz77HuffmanIncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + raise NotImplementedError( + "Incremental encoding not implemented for LZ77 Huffman." + ) + + +class Lz77HuffmanIncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + codec = Lz77HuffmanCodec() + output, _ = codec.decode(input) + return output + + +class Lz77HuffmanStreamReader(codecs.StreamReader): + def decode(self, input, errors="strict"): + codec = Lz77HuffmanCodec() + output, _ = codec.decode(input, errors) + return output, len(input) + + +class Lz77HuffmanStreamWriter(codecs.StreamWriter): + def encode(self, input, errors="strict"): + codec = Lz77HuffmanCodec() + return codec.encode(input, errors) + + +# Codec search function +def find_lz77_huffman(name): + if name == "lz77_huffman": + return codecs.CodecInfo( + name="lz77_huffman", + encode=Lz77HuffmanCodec().encode, + decode=Lz77HuffmanCodec().decode, + incrementalencoder=Lz77HuffmanIncrementalEncoder, + incrementaldecoder=Lz77HuffmanIncrementalDecoder, + streamreader=Lz77HuffmanStreamReader, + streamwriter=Lz77HuffmanStreamWriter, + ) + return None diff --git a/volatility3/framework/layers/codecs/lz77plain.py b/volatility3/framework/layers/codecs/lz77plain.py new file mode 100644 index 0000000000..b505b86e56 --- /dev/null +++ b/volatility3/framework/layers/codecs/lz77plain.py @@ -0,0 +1,127 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +import struct, codecs + + +def lz77_plain_decompress(in_buf): + """ + Refs : + - https://learn.microsoft.com/en-us/windows/win32/cmpapi/using-the-compression-api + """ + out_idx = 0 + in_idx = 0 + nibble_idx = 0 + + flags = 0 + flag_count = 0 + + out_buf = [] + + while in_idx < len(in_buf): + if flag_count == 0: + flags = struct.unpack(" out_idx: + raise ValueError("CorruptedData") + out_buf.append(out_buf[out_idx - offset]) + out_idx += 1 + return bytes(out_buf) + + +class LZ77PlainCodec(codecs.Codec): + def encode(self, input, errors="strict"): + raise NotImplementedError("LZ77 plain compression is not implemented") + + def decode(self, input, errors="strict"): + return lz77_plain_decompress(input), len(input) + + def decode_bytes(self, input): + return lz77_plain_decompress(input) + + +class LZ77PlainIncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + raise NotImplementedError("LZ77 plain compression is not implemented") + + +class LZ77PlainIncrementalDecoder(codecs.IncrementalDecoder): + def decode(self, input, final=False): + return lz77_plain_decompress(input) + + +class LZ77PlainStreamReader(LZ77PlainCodec, codecs.StreamReader): + def __init__(self, *args, **kwargs): + codecs.StreamReader.__init__(self, *args, **kwargs) + + def read(self, size=-1, chars=-1, firstline=False): + data = self.stream.read(size) + return self.decode_bytes(data) + + +class LZ77PlainStreamWriter(LZ77PlainCodec, codecs.StreamWriter): + def write(self, obj): + raise NotImplementedError("LZ77 plain compression is not implemented") + + +def find_lz77_plain(name): + if name == "lz77_plain": + return codecs.CodecInfo( + name="lz77_plain", + encode=LZ77PlainCodec().encode, + decode=LZ77PlainCodec().decode, + incrementalencoder=LZ77PlainIncrementalEncoder, + incrementaldecoder=LZ77PlainIncrementalDecoder, + streamreader=LZ77PlainStreamReader, + streamwriter=LZ77PlainStreamWriter, + ) + return None + + +codecs.register(find_lz77_plain) diff --git a/volatility3/framework/layers/hib.py b/volatility3/framework/layers/hib.py index c7c5073e07..4e3f0ae2c9 100644 --- a/volatility3/framework/layers/hib.py +++ b/volatility3/framework/layers/hib.py @@ -7,13 +7,12 @@ # - https://www.vergiliusproject.com/kernels/x64/ : Windows kernel structures used to track the evolution of the hibernation file structure in time. from typing import Optional -import logging, struct +import logging, struct, codecs + +# import xpress_lz77 <- coming soon with pyo3 from volatility3.framework import interfaces, constants, exceptions from volatility3.framework.layers import segmented -from volatility3.framework.layers.codecs import ( - lz77_plain_decompress, - lz77_huffman_decompress, -) + vollog = logging.getLogger(__name__) @@ -28,9 +27,12 @@ def uncompress(data: bytes, flag): Return: The decompressed data (consecutive pages). """ if flag == 0 or flag == 1: - return lz77_plain_decompress(data) # See layers.codecs + # Comming soon with pyo3 + # return bytes(xpress_lz77.lz77_plain_decompress_py(data)) + return codecs.decode(data, "lz77_plain") # See layers.codecs + elif flag == 2 or flag == 3: - return lz77_huffman_decompress(data, 65536)[0] # See layers.codecs + return codecs.decode(data, "lz77_huffman") # See layers.codecs else: vollog.warning( f"A compression set could not be decompressed: Compression algorithm : {flag}" @@ -84,10 +86,10 @@ def __init__( """ Mapping for each 'group' of Windows version sharing the same offsets --------------------------------------------------------------------------------------------------------- - | Windows Versions | FirstKernelRestorePage (FKRP) | KernelPagesProcessed (KPP)| + | Windows Versions | FirstKernelRestorePage (FKRP) | KernelPagesProcessed (KPP)| | ------------------------------------------|:-----------------------------:|:-------------------------:| - | Windows 8/8.1 | 0x68 | 0x1C8 | - | Windows 10 2016 1507-1511 | 0x70 | 0x218 | + | Windows 8/8.1 | 0x68 | 0x1C8 | + | Windows 10 2016 1507-1511 | 0x70 | 0x218 | | Windows 10 2016 1607 | 0x70 | 0x220 | | Windows 10 2016 1703 - Windows 11 23H2 | 0x70 | 0x230 | --------------------------------------------------------------------------------------------------------- @@ -261,9 +263,8 @@ def stack( except exceptions.LayerException: return None new_name = context.layers.free_layer_name("HibernationLayer") - context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( - layer_name - ) + context.config[ + interfaces.configuration.path_join(new_name, "base_layer") + ] = layer_name layer = HibernationLayer(context, new_name, new_name) - cls.stacker_slow_warning() return layer From 2bebe9107f99d116cf6362f05d070e683da052d2 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Mon, 15 Jul 2024 22:41:13 +0200 Subject: [PATCH 13/22] Formatting using good black version --- volatility3/framework/layers/hib.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/volatility3/framework/layers/hib.py b/volatility3/framework/layers/hib.py index 4e3f0ae2c9..01b6b55831 100644 --- a/volatility3/framework/layers/hib.py +++ b/volatility3/framework/layers/hib.py @@ -263,8 +263,8 @@ def stack( except exceptions.LayerException: return None new_name = context.layers.free_layer_name("HibernationLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) layer = HibernationLayer(context, new_name, new_name) return layer From 9fd15671e4170c62fed25601421c066f9488cabe Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Tue, 23 Jul 2024 14:31:48 +0200 Subject: [PATCH 14/22] Deported the decompression algorithm in a dedicated python3 package --- requirements.txt | 3 + .../framework/layers/codecs/lz77huffman.py | 288 ------------------ .../framework/layers/codecs/lz77plain.py | 127 -------- volatility3/framework/layers/hib.py | 10 +- 4 files changed, 7 insertions(+), 421 deletions(-) delete mode 100644 volatility3/framework/layers/codecs/lz77huffman.py delete mode 100644 volatility3/framework/layers/codecs/lz77plain.py diff --git a/requirements.txt b/requirements.txt index 99e0786cc9..67c8fdb956 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,6 @@ pycryptodome # This is required for memory acquisition via leechcore/pcileech. leechcorepyc>=2.4.0 + +# This is required for hibernation dump +xpress_lz77 >=1.0.0 diff --git a/volatility3/framework/layers/codecs/lz77huffman.py b/volatility3/framework/layers/codecs/lz77huffman.py deleted file mode 100644 index ba6fea11bd..0000000000 --- a/volatility3/framework/layers/codecs/lz77huffman.py +++ /dev/null @@ -1,288 +0,0 @@ -# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 - -import struct, logging, codecs -from typing import Tuple, List, Union - -vollog = logging.getLogger(__name__) - - -class BitStream: - def __init__(self, source: bytes, in_pos: int): - self.source = source - self.index = in_pos + 4 - # read UInt16 little endian - mask = struct.unpack_from(" int: - if n == 0: - return 0 - return self.mask >> (32 - n) - - def skip(self, n: int) -> Union[None, Exception]: - self.mask = (self.mask << n) & 0xFFFFFFFF - self.bits -= n - if self.bits < 16: - if self.index + 2 > len(self.source): - return Exception("EOF Error") - # read UInt16 little endian - self.mask += ( - (struct.unpack_from(" int: - node = treeNodes[0] - i = leafIndex + 1 - childIndex = None - - while bits > 1: - bits -= 1 - childIndex = (mask >> bits) & 1 - if not node.child[childIndex]: - node.child[childIndex] = treeNodes[i] - treeNodes[i].leaf = False - i += 1 - node = node.child[childIndex] - - node.child[mask & 1] = treeNodes[leafIndex] - - return i - - -def prefix_code_tree_rebuild(input: bytes) -> PREFIX_CODE_NODE: - treeNodes = [PREFIX_CODE_NODE() for _ in range(1024)] - symbolInfo = [PREFIX_CODE_SYMBOL() for _ in range(512)] - - for i in range(256): - value = input[i] - - symbolInfo[2 * i].id = 2 * i - symbolInfo[2 * i].symbol = 2 * i - symbolInfo[2 * i].length = value & 0xF - - value >>= 4 - - symbolInfo[2 * i + 1].id = 2 * i + 1 - symbolInfo[2 * i + 1].symbol = 2 * i + 1 - symbolInfo[2 * i + 1].length = value & 0xF - - symbolInfo = sorted(symbolInfo, key=lambda x: (x.length, x.symbol)) - - i = 0 - while i < 512 and symbolInfo[i].length == 0: - i += 1 - - mask = 0 - bits = 1 - - root = treeNodes[0] - root.leaf = False - - j = 1 - while i < 512: - treeNodes[j].id = j - treeNodes[j].symbol = symbolInfo[i].symbol - treeNodes[j].leaf = True - mask = mask << (symbolInfo[i].length - bits) - bits = symbolInfo[i].length - j = prefix_code_tree_add_leaf(treeNodes, j, mask, bits) - mask += 1 - i += 1 - - return root - - -def prefix_code_tree_decode_symbol( - bstr: BitStream, root: PREFIX_CODE_NODE -) -> Tuple[int, Union[None, Exception]]: - node = root - while True: - bit = bstr.lookup(1) - err = bstr.skip(1) - if err is not None: - vollog.warning("Some data could not be decompressed.") - return 0, err - - node = node.child[bit] - if not node: - vollog.warning("Corruption detected when decompressing the data.") - return 0, Exception("Corruption detected") - if node.leaf: - break - return node.symbol, None - - -class Lz77HuffmanCodec: - def encode(self, input: bytes, errors: str = "strict") -> Tuple[bytes, int]: - raise NotImplementedError("Encoding not implemented for LZ77 Huffman.") - - def decode(self, input: bytes, errors: str = "strict") -> Tuple[bytes, int]: - output_size = 65536 - output, err = self.lz77_huffman_decompress(input, output_size) - if err is not None: - raise Exception(f"Decoding failed: {err}") - return output, len(input) - - @staticmethod - def lz77_huffman_decompress( - input: bytes, output_size: int - ) -> Tuple[bytes, Union[None, Exception]]: - output = bytearray(output_size) - err = None - - # Index into the input buffer. - in_idx = 0 - - # Index into the output buffer. - out_idx = 0 - - while True: - # How much data belongs in the current chunk. - # Chunks are split into maximum 65536 bytes. - chunk_size = output_size - out_idx - if chunk_size > 65536: - chunk_size = 65536 - - in_idx, out_idx, err = Lz77HuffmanCodec.lz77_huffman_decompress_chunk( - in_idx, input, out_idx, output, chunk_size - ) - if err is not None: - return bytes(output), err - if out_idx >= len(output) or in_idx >= len(input): - break - return bytes(output), None - - @staticmethod - def lz77_huffman_decompress_chunk( - in_idx: int, input: bytes, out_idx: int, output: bytearray, chunk_size: int - ) -> Tuple[int, int, Union[None, Exception]]: - if in_idx + 256 > len(input): - return 0, 0, Exception("EOF Error") - - root = prefix_code_tree_rebuild(input[in_idx:]) - bstr = BitStream(input, in_idx + 256) - - i = out_idx - - while i < out_idx + chunk_size: - symbol, err = prefix_code_tree_decode_symbol(bstr, root) - - if err is not None: - return int(bstr.index), i, err - - if symbol < 256: - output[i] = symbol - i += 1 - else: - symbol -= 256 - length = symbol & 15 - symbol >>= 4 - - offset = 0 - if symbol != 0: - offset = int(bstr.lookup(symbol)) - - offset |= 1 << symbol - offset = -offset - - if length == 15: - length = bstr.source[bstr.index] + 15 - bstr.index += 1 - - if length == 270: - length = struct.unpack_from(" 0: - if i + offset < 0: - vollog.warning("Some data could not be decompressed.") - return int(bstr.index), i, Exception("Decompression Error") - - output[i] = output[i + offset] - i += 1 - length -= 1 - if length == 0: - break - return int(bstr.index), i, None - - -class Lz77HuffmanIncrementalEncoder(codecs.IncrementalEncoder): - def encode(self, input, final=False): - raise NotImplementedError( - "Incremental encoding not implemented for LZ77 Huffman." - ) - - -class Lz77HuffmanIncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - codec = Lz77HuffmanCodec() - output, _ = codec.decode(input) - return output - - -class Lz77HuffmanStreamReader(codecs.StreamReader): - def decode(self, input, errors="strict"): - codec = Lz77HuffmanCodec() - output, _ = codec.decode(input, errors) - return output, len(input) - - -class Lz77HuffmanStreamWriter(codecs.StreamWriter): - def encode(self, input, errors="strict"): - codec = Lz77HuffmanCodec() - return codec.encode(input, errors) - - -# Codec search function -def find_lz77_huffman(name): - if name == "lz77_huffman": - return codecs.CodecInfo( - name="lz77_huffman", - encode=Lz77HuffmanCodec().encode, - decode=Lz77HuffmanCodec().decode, - incrementalencoder=Lz77HuffmanIncrementalEncoder, - incrementaldecoder=Lz77HuffmanIncrementalDecoder, - streamreader=Lz77HuffmanStreamReader, - streamwriter=Lz77HuffmanStreamWriter, - ) - return None diff --git a/volatility3/framework/layers/codecs/lz77plain.py b/volatility3/framework/layers/codecs/lz77plain.py deleted file mode 100644 index b505b86e56..0000000000 --- a/volatility3/framework/layers/codecs/lz77plain.py +++ /dev/null @@ -1,127 +0,0 @@ -# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -import struct, codecs - - -def lz77_plain_decompress(in_buf): - """ - Refs : - - https://learn.microsoft.com/en-us/windows/win32/cmpapi/using-the-compression-api - """ - out_idx = 0 - in_idx = 0 - nibble_idx = 0 - - flags = 0 - flag_count = 0 - - out_buf = [] - - while in_idx < len(in_buf): - if flag_count == 0: - flags = struct.unpack(" out_idx: - raise ValueError("CorruptedData") - out_buf.append(out_buf[out_idx - offset]) - out_idx += 1 - return bytes(out_buf) - - -class LZ77PlainCodec(codecs.Codec): - def encode(self, input, errors="strict"): - raise NotImplementedError("LZ77 plain compression is not implemented") - - def decode(self, input, errors="strict"): - return lz77_plain_decompress(input), len(input) - - def decode_bytes(self, input): - return lz77_plain_decompress(input) - - -class LZ77PlainIncrementalEncoder(codecs.IncrementalEncoder): - def encode(self, input, final=False): - raise NotImplementedError("LZ77 plain compression is not implemented") - - -class LZ77PlainIncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return lz77_plain_decompress(input) - - -class LZ77PlainStreamReader(LZ77PlainCodec, codecs.StreamReader): - def __init__(self, *args, **kwargs): - codecs.StreamReader.__init__(self, *args, **kwargs) - - def read(self, size=-1, chars=-1, firstline=False): - data = self.stream.read(size) - return self.decode_bytes(data) - - -class LZ77PlainStreamWriter(LZ77PlainCodec, codecs.StreamWriter): - def write(self, obj): - raise NotImplementedError("LZ77 plain compression is not implemented") - - -def find_lz77_plain(name): - if name == "lz77_plain": - return codecs.CodecInfo( - name="lz77_plain", - encode=LZ77PlainCodec().encode, - decode=LZ77PlainCodec().decode, - incrementalencoder=LZ77PlainIncrementalEncoder, - incrementaldecoder=LZ77PlainIncrementalDecoder, - streamreader=LZ77PlainStreamReader, - streamwriter=LZ77PlainStreamWriter, - ) - return None - - -codecs.register(find_lz77_plain) diff --git a/volatility3/framework/layers/hib.py b/volatility3/framework/layers/hib.py index 01b6b55831..21356df54a 100644 --- a/volatility3/framework/layers/hib.py +++ b/volatility3/framework/layers/hib.py @@ -5,11 +5,11 @@ # - https://www.forensicxlab.com/posts/hibernation/ : Vulgarized description of the hibernation file structure and the implementation of this layer. # - https://www.cct.lsu.edu/~golden/Papers/sylvehiber.pdf : Scientific paper. # - https://www.vergiliusproject.com/kernels/x64/ : Windows kernel structures used to track the evolution of the hibernation file structure in time. +# - https://pypi.org/project/xpress-lz77/: The decompression algorithm developped for the integration to volatility3 from typing import Optional -import logging, struct, codecs +import logging, struct, codecs, xpress_lz77 -# import xpress_lz77 <- coming soon with pyo3 from volatility3.framework import interfaces, constants, exceptions from volatility3.framework.layers import segmented @@ -27,12 +27,10 @@ def uncompress(data: bytes, flag): Return: The decompressed data (consecutive pages). """ if flag == 0 or flag == 1: - # Comming soon with pyo3 - # return bytes(xpress_lz77.lz77_plain_decompress_py(data)) - return codecs.decode(data, "lz77_plain") # See layers.codecs + return bytes(xpress_lz77.lz77_plain_decompress_py(data)) elif flag == 2 or flag == 3: - return codecs.decode(data, "lz77_huffman") # See layers.codecs + return bytes(xpress_lz77.lz77_huffman_decompress_py(data, 65536)) else: vollog.warning( f"A compression set could not be decompressed: Compression algorithm : {flag}" From 9b855f1c29d7e826c85f18e2567dccce65242b3d Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Tue, 23 Jul 2024 14:32:27 +0200 Subject: [PATCH 15/22] Deported the decompression algorithm in a dedicated python3 package --- volatility3/framework/layers/codecs/__init__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/volatility3/framework/layers/codecs/__init__.py b/volatility3/framework/layers/codecs/__init__.py index b2d1eef6ce..9aab926948 100644 --- a/volatility3/framework/layers/codecs/__init__.py +++ b/volatility3/framework/layers/codecs/__init__.py @@ -2,9 +2,4 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 import codecs -from volatility3.framework.layers.codecs.lz77plain import find_lz77_plain -from volatility3.framework.layers.codecs.lz77huffman import find_lz77_huffman - -codecs.register(find_lz77_plain) -codecs.register(find_lz77_huffman) -# Register other codecs here. +# Register codecs here. From 2ffa3a9c5d38544134adc4df66561e74d1da1805 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Tue, 23 Jul 2024 14:36:43 +0200 Subject: [PATCH 16/22] conflict --- requirements.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 67c8fdb956..fc66f7f724 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -# The following packages are required for core functionality. -pefile>=2017.8.1 +# Include the minimal requirements +-r requirements-minimal.txt # The following packages are optional. # If certain packages are not necessary, place a comment (#) at the start of the line. @@ -17,5 +17,9 @@ pycryptodome # This is required for memory acquisition via leechcore/pcileech. leechcorepyc>=2.4.0 +# This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage +gcsfs>=2023.1.0 +s3fs>=2023.1.0 + # This is required for hibernation dump xpress_lz77 >=1.0.0 From b17eab6e108fa20c6cb355325658fdb1e20c4a23 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Tue, 23 Jul 2024 16:47:11 +0200 Subject: [PATCH 17/22] requirements --- requirements.txt | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8078141c88..83126ed293 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,5 @@ -<<<<<<< HEAD # Include the minimal requirements -r requirements-minimal.txt -======= -# The following packages are required for core functionality. -pefile>=2023.2.7 ->>>>>>> develop # The following packages are optional. # If certain packages are not necessary, place a comment (#) at the start of the line. @@ -20,15 +15,8 @@ capstone>=3.0.5 pycryptodome # This is required for memory acquisition via leechcore/pcileech. -leechcorepyc>=2.4.0 +leechcorepyc>=2.4.0; sys_platform != 'darwin' # This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage gcsfs>=2023.1.0 -<<<<<<< HEAD s3fs>=2023.1.0 - -# This is required for hibernation dump -xpress_lz77 >=1.0.0 -======= -s3fs>=2023.1.0 ->>>>>>> develop From 3f428f7086eeea264706db1d1e954bd195214fc3 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Tue, 23 Jul 2024 17:05:07 +0200 Subject: [PATCH 18/22] Fixing requirements --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 83126ed293..1de1e99a7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,6 @@ leechcorepyc>=2.4.0; sys_platform != 'darwin' # This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage gcsfs>=2023.1.0 s3fs>=2023.1.0 + +# This is required for memory to work with the hibernation layer +xpress-lz77>=1.0.0 From ad31052d7983f96496aa742954cb0c04664f920d Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Tue, 23 Jul 2024 17:06:03 +0200 Subject: [PATCH 19/22] Fixing unused import --- volatility3/framework/layers/hib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/layers/hib.py b/volatility3/framework/layers/hib.py index 21356df54a..ca707768ae 100644 --- a/volatility3/framework/layers/hib.py +++ b/volatility3/framework/layers/hib.py @@ -8,7 +8,7 @@ # - https://pypi.org/project/xpress-lz77/: The decompression algorithm developped for the integration to volatility3 from typing import Optional -import logging, struct, codecs, xpress_lz77 +import logging, struct, xpress_lz77 from volatility3.framework import interfaces, constants, exceptions from volatility3.framework.layers import segmented From d779c0e75847de841760d4a276d15ed27b75088a Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 28 Jul 2024 11:58:31 +0200 Subject: [PATCH 20/22] Upgrading pipeline and python3 version minimum requirement --- volatility3/framework/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 74db773cf8..51310bfa29 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -7,7 +7,7 @@ import sys import zipfile -required_python_version = (3, 7, 3) +required_python_version = (3, 8, 0) if ( sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] From c9479999546ff471c877e7700b0c78836e7dd794 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Sun, 28 Jul 2024 11:59:01 +0200 Subject: [PATCH 21/22] Upgrading pipeline and python3 version minimum requirement --- .github/workflows/build-pypi.yml | 43 +++++++++-------- .github/workflows/install.yml | 33 +++++++------ .github/workflows/test.yaml | 79 ++++++++++++++++---------------- 3 files changed, 76 insertions(+), 79 deletions(-) diff --git a/.github/workflows/build-pypi.yml b/.github/workflows/build-pypi.yml index 346d0337e9..74afdf46e0 100644 --- a/.github/workflows/build-pypi.yml +++ b/.github/workflows/build-pypi.yml @@ -5,39 +5,38 @@ on: branches: - stable - develop - - 'release/**' + - "release/**" pull_request: branches: - stable - - 'release/**' + - "release/**" jobs: - build: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.7"] + python-version: ["3.8"] steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build - - name: Build PyPi packages - run: | - python -m build + - name: Build PyPi packages + run: | + python -m build - - name: Archive dist - uses: actions/upload-artifact@v4 - with: - name: volatility3-pypi - path: | - dist/ + - name: Archive dist + uses: actions/upload-artifact@v4 + with: + name: volatility3-pypi + path: | + dist/ diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index e13161085c..2ad03ead7c 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -1,31 +1,30 @@ name: Install Volatility3 test on: [push, pull_request] jobs: - install_test: runs-on: ${{ matrix.host }} strategy: fail-fast: false matrix: - host: [ ubuntu-latest, windows-latest ] - python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + host: [ubuntu-latest, windows-latest] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} - - name: Setup python-pip - run: python -m pip install --upgrade pip + - name: Setup python-pip + run: python -m pip install --upgrade pip - - name: Install dependencies - run: | - pip install -r requirements.txt + - name: Install dependencies + run: | + pip install -r requirements.txt - - name: Install volatility3 - run: pip install . + - name: Install volatility3 + run: pip install . - - name: Run volatility3 - run: vol --help \ No newline at end of file + - name: Run volatility3 + run: vol --help diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 55b2e4b602..212c22045b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,54 +1,53 @@ name: Test Volatility3 on: [push, pull_request] jobs: - build: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.7"] + python-version: ["3.8"] steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install Cmake - pip install build - pip install -r ./test/requirements-testing.txt + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install Cmake + pip install build + pip install -r ./test/requirements-testing.txt - - name: Build PyPi packages - run: | - python -m build + - name: Build PyPi packages + run: | + python -m build - - name: Download images - run: | - curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/linux-sample-1.bin.gz" - gunzip linux-sample-1.bin.gz - curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" - gunzip win-xp-laptop-2005-06-25.img.gz + - name: Download images + run: | + curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/linux-sample-1.bin.gz" + gunzip linux-sample-1.bin.gz + curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" + gunzip win-xp-laptop-2005-06-25.img.gz - - name: Download and Extract symbols - run: | - cd ./volatility3/symbols - curl -sLO https://downloads.volatilityfoundation.org/volatility3/symbols/linux.zip - unzip linux.zip - cd - + - name: Download and Extract symbols + run: | + cd ./volatility3/symbols + curl -sLO https://downloads.volatilityfoundation.org/volatility3/symbols/linux.zip + unzip linux.zip + cd - - - name: Testing... - run: | - py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v - py.test ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v + - name: Testing... + run: | + py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v + py.test ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v - - name: Clean up post-test - run: | - rm -rf *.lime - rm -rf *.img - cd volatility3/symbols - rm -rf linux - rm -rf linux.zip - cd - + - name: Clean up post-test + run: | + rm -rf *.lime + rm -rf *.img + cd volatility3/symbols + rm -rf linux + rm -rf linux.zip + cd - From 0d6c1ea90e0a3dc17c71087a4ce00956b848c258 Mon Sep 17 00:00:00 2001 From: k1nd0ne Date: Mon, 5 Aug 2024 22:11:24 +0200 Subject: [PATCH 22/22] Using symbol Tables (1/2) --- volatility3/framework/layers/hib.py | 450 +++++++++--------- .../framework/plugins/windows/hibernation.py | 85 ++-- .../symbols/windows/hibernation/header.json | 163 +++++++ 3 files changed, 444 insertions(+), 254 deletions(-) create mode 100644 volatility3/framework/symbols/windows/hibernation/header.json diff --git a/volatility3/framework/layers/hib.py b/volatility3/framework/layers/hib.py index ca707768ae..75ea5650ac 100644 --- a/volatility3/framework/layers/hib.py +++ b/volatility3/framework/layers/hib.py @@ -8,261 +8,273 @@ # - https://pypi.org/project/xpress-lz77/: The decompression algorithm developped for the integration to volatility3 from typing import Optional -import logging, struct, xpress_lz77 +import logging, struct from volatility3.framework import interfaces, constants, exceptions from volatility3.framework.layers import segmented +try: + import xpress_lz77 -vollog = logging.getLogger(__name__) - - -def uncompress(data: bytes, flag): - """ - Desc: - Params: - - data: the compressed data from a compression set - - flag: what is the decompression algorithm to use. - - out_size: Size of the decompressed data - Return: The decompressed data (consecutive pages). - """ - if flag == 0 or flag == 1: - return bytes(xpress_lz77.lz77_plain_decompress_py(data)) - - elif flag == 2 or flag == 3: - return bytes(xpress_lz77.lz77_huffman_decompress_py(data, 65536)) - else: - vollog.warning( - f"A compression set could not be decompressed: Compression algorithm : {flag}" - ) - raise ValueError("Cannot decompress the data.") + HAS_XPRESS = True +except ImportError: + HAS_XPRESS = False -class HibernationLayer(segmented.NonLinearlySegmentedLayer): - """ - A TranslationLayer that maps physical memory against a x64 Microsoft Windows hibernation file. - This Translation Layer is meant to be used in conjunction with the Hibernation.Info and Hibernation.Dump plugins. - """ - - WINDOWS_10_2016_1703_TO_23H2 = 0 - WINDOW_8 = 1 - WINDOWS_10_2016_1507_1511 = 2 - WINDOWS_10_2016_1607 = 3 +vollog = logging.getLogger(__name__) - # TODO: Make me compatible with x86 by adding options to the Hib plugins. - PAGE_SIZE = 4096 # x64 page size. - HEADER_SIZE = 4 - PAGE_DESC_SIZE = 8 +if HAS_XPRESS: - def __init__( - self, - context: interfaces.context.ContextInterface, - config_path: str, - name: str, - **kwargs, - ): + class HibernationLayer(segmented.NonLinearlySegmentedLayer): """ - Initializes the Hibernation file layer. + A TranslationLayer that maps physical memory against a x64 Microsoft Windows hibernation file. + This Translation Layer is meant to be used in conjunction with the Hibernation.Info and Hibernation.Dump plugins. """ - self._compressed = ( - {} - ) # Keep track of which compression algorithm by each mapped compressed data. - self._mapping = ( - {} - ) # This will hold the mapping between the PageNumber in the decompressed data vs the physical page number. - if "plugins.Dump.version" in context.config: - # The user is using the hibernation.Dump plugin, so the version must be known. - # See possible version in the table below. - self.version = context.config["plugins.Dump.version"] - else: - self.version = -1 + WINDOWS_10_2016_1703_TO_23H2 = 0 + WINDOW_8 = 1 + WINDOWS_10_2016_1507_1511 = 2 + WINDOWS_10_2016_1607 = 3 - self.NPFL_OFFSET = 0x058 # (NumPagesForLoader) - self.FBRP_OFFSET = 0x068 # (FirstBootRestorePage) + # TODO: Make me compatible with x86 by checking the page_size. + PAGE_SIZE = 4096 # x64 page size. + HEADER_SIZE = 4 + PAGE_DESC_SIZE = 8 - """ - Mapping for each 'group' of Windows version sharing the same offsets - --------------------------------------------------------------------------------------------------------- - | Windows Versions | FirstKernelRestorePage (FKRP) | KernelPagesProcessed (KPP)| - | ------------------------------------------|:-----------------------------:|:-------------------------:| - | Windows 8/8.1 | 0x68 | 0x1C8 | - | Windows 10 2016 1507-1511 | 0x70 | 0x218 | - | Windows 10 2016 1607 | 0x70 | 0x220 | - | Windows 10 2016 1703 - Windows 11 23H2 | 0x70 | 0x230 | - --------------------------------------------------------------------------------------------------------- - """ - if self.version == self.WINDOWS_10_2016_1703_TO_23H2: - self.FKRP_OFFSET = 0x070 - self.KPP_OFFSET = 0x230 - elif self.version == self.WINDOW_8: - self.FKRP_OFFSET = 0x68 - self.KPP_OFFSET = 0x1C8 - elif self.version == self.WINDOWS_10_2016_1507_1511: - self.FKRP_OFFSET = 0x70 - self.KPP_OFFSET = 0x218 - elif self.version == self.WINDOWS_10_2016_1607: - self.FKRP_OFFSET = 0x70 - self.KPP_OFFSET = 0x220 - else: - raise exceptions.LayerException(name, "The version provided is not valid") - super().__init__(context, config_path, name, **kwargs) + def _uncompress(self, data: bytes, flag): + """ + Desc: + Params: + - data: the compressed data from a compression set + - flag: what is the decompression algorithm to use. + - out_size: Size of the decompressed data + Return: The decompressed data (consecutive pages). + """ + if flag == 0 or flag == 1: + return bytes(xpress_lz77.lz77_plain_decompress_py(data)) - @classmethod - def _check_header( - cls, base_layer: interfaces.layers.DataLayerInterface, name: str = "" - ): - header = base_layer.read(0, 4) - if header != b"HIBR": - raise exceptions.LayerException(name, "No Hibernation magic bytes") - else: - vollog.info("Detecting an hibernation file") + elif flag == 2 or flag == 3: + return bytes(xpress_lz77.lz77_huffman_decompress_py(data, 65536)) + else: + vollog.warning( + f"A compression set could not be decompressed: Compression algorithm : {flag}" + ) + raise ValueError("Cannot decompress the data.") - def _load_segments(self): - """ - Loading segments is a 2 STEP operation: - - Step 1: extracting the pages from the BootSection if any. - - Step 2: extracting the pages from the KernelSection if any. - """ - base_layer = self.context.layers[self._base_layer] - NumPagesForLoader = int.from_bytes( - base_layer.read(self.NPFL_OFFSET, 8), "little" - ) - FirstBootRestorePage = int.from_bytes( - base_layer.read(self.FBRP_OFFSET, 8), "little" - ) + def __init__( + self, + context: interfaces.context.ContextInterface, + config_path: str, + name: str, + **kwargs, + ): + """ + Initializes the Hibernation file layer. + """ + self._compressed = ( + {} + ) # Keep track of which compression algorithm by each mapped compressed data. + self._mapping = ( + {} + ) # This will hold the mapping between the PageNumber in the decompressed data vs the physical page number. - offset = FirstBootRestorePage * self.PAGE_SIZE - total_pages = NumPagesForLoader - treated = 0 + if "plugins.Dump.version" in context.config: + # The user is using the hibernation.Dump plugin, so the version must be known. + # See possible version in the table below. + self.version = context.config["plugins.Dump.version"] + else: + self.version = -1 + + self.NPFL_OFFSET = 0x058 # (NumPagesForLoader) + self.FBRP_OFFSET = 0x068 # (FirstBootRestorePage) + + """ + Mapping for each 'group' of Windows version sharing the same offsets. + --------------------------------------------------------------------------------------------------------- + | Windows Versions | FirstKernelRestorePage (FKRP) | KernelPagesProcessed (KPP)| + | ------------------------------------------|:-----------------------------:|:-------------------------:| + | Windows 8/8.1 | 0x68 | 0x1C8 | + | Windows 10 2016 1507-1511 | 0x70 | 0x218 | + | Windows 10 2016 1607 | 0x70 | 0x220 | + | Windows 10 2016 1703 - Windows 11 23H2 | 0x70 | 0x230 | + --------------------------------------------------------------------------------------------------------- + """ + if self.version == self.WINDOWS_10_2016_1703_TO_23H2: + self.FKRP_OFFSET = 0x070 + self.KPP_OFFSET = 0x230 + elif self.version == self.WINDOW_8: + self.FKRP_OFFSET = 0x68 + self.KPP_OFFSET = 0x1C8 + elif self.version == self.WINDOWS_10_2016_1507_1511: + self.FKRP_OFFSET = 0x70 + self.KPP_OFFSET = 0x218 + elif self.version == self.WINDOWS_10_2016_1607: + self.FKRP_OFFSET = 0x70 + self.KPP_OFFSET = 0x220 + else: + raise exceptions.LayerException( + name, "The version provided is not valid" + ) + super().__init__(context, config_path, name, **kwargs) - while total_pages > treated: - page_read, next_compression_set = self._read_compression_set(offset) - offset += next_compression_set - treated += page_read + @classmethod + def _check_header( + cls, base_layer: interfaces.layers.DataLayerInterface, name: str = "" + ): + header = base_layer.read(0, 4) + if header != b"HIBR": + raise exceptions.LayerException(name, "No Hibernation magic bytes") + else: + vollog.info("Detecting an hibernation file") - if "plugins.Dump.version" in self.context.config: - # The user is using the hibernation.Dump plugin so we can parse the KernelSection - FirstKernelRestorePage = int.from_bytes( - base_layer.read(self.FKRP_OFFSET, 8), "little" + def _load_segments(self): + """ + Loading segments is a 2 STEP operation: + - Step 1: extracting the pages from the BootSection if any. + - Step 2: extracting the pages from the KernelSection if any. + """ + base_layer = self.context.layers[self._base_layer] + NumPagesForLoader = int.from_bytes( + base_layer.read(self.NPFL_OFFSET, 8), "little" ) - KernelPagesProcessed = int.from_bytes( - base_layer.read(self.KPP_OFFSET, 8), "little" + FirstBootRestorePage = int.from_bytes( + base_layer.read(self.FBRP_OFFSET, 8), "little" ) - offset = FirstKernelRestorePage * self.PAGE_SIZE - total_pages = KernelPagesProcessed + offset = FirstBootRestorePage * self.PAGE_SIZE + total_pages = NumPagesForLoader treated = 0 + while total_pages > treated: page_read, next_compression_set = self._read_compression_set(offset) offset += next_compression_set treated += page_read - self._segments = sorted(self._segments, key=lambda x: x[0]) - def _read_compression_set(self, offset): - """ - Desc: Read one compression set an extract the address of the compressed data - Params: - - offset : the location of the compression set to read. - - stream : the hibernation file stream. - Return: The offset of the compressed data and the size. - """ + if "plugins.Dump.version" in self.context.config: + # The user is using the hibernation.Dump plugin so we can parse the KernelSection + FirstKernelRestorePage = int.from_bytes( + base_layer.read(self.FKRP_OFFSET, 8), "little" + ) + KernelPagesProcessed = int.from_bytes( + base_layer.read(self.KPP_OFFSET, 8), "little" + ) + offset = FirstKernelRestorePage * self.PAGE_SIZE + total_pages = KernelPagesProcessed - base_layer = self.context.layers[self._base_layer] - header = base_layer.read(offset, self.HEADER_SIZE) - data = struct.unpack(" 16: - # See references - raise exceptions.LayerException( - self.name, "The hibernation file is corrupted." - ) + treated = 0 + while total_pages > treated: + page_read, next_compression_set = self._read_compression_set(offset) + offset += next_compression_set + treated += page_read + self._segments = sorted(self._segments, key=lambda x: x[0]) - size_of_compressed_data = ( - data >> 8 - ) & 0x3FFFFF # Next 22 least significant bytes. - huffman_compressed = (data >> 30) & 0x3 # Most significant bit. + def _read_compression_set(self, offset): + """ + Desc: Read one compression set an extract the address of the compressed data + Params: + - offset : the location of the compression set to read. + - stream : the hibernation file stream. + Return: The offset of the compressed data and the size. + """ - # Now we know where is the start of the page descriptors in the hibernation file. - mapped_address = ( - offset + self.HEADER_SIZE + number_of_descs * self.PAGE_DESC_SIZE - ) - total_page_count = 0 - position = 0 - for i in range(number_of_descs): - # Go fetch and parse each page descriptor. - location = offset + self.HEADER_SIZE + i * self.PAGE_DESC_SIZE - page_descriptor = base_layer.read(location, self.PAGE_DESC_SIZE) - data = struct.unpack("> 4 # Shift right 4 bits to get the upper 60 bits - page_count = 1 + Numpages - total_page_count += page_count - self._segments.append( - ( - PageNum * self.PAGE_SIZE, - mapped_address, - self.PAGE_SIZE * page_count, - size_of_compressed_data, + base_layer = self.context.layers[self._base_layer] + header = base_layer.read(offset, self.HEADER_SIZE) + data = struct.unpack(" 16: + # See references + raise exceptions.LayerException( + self.name, "The hibernation file is corrupted." ) + + size_of_compressed_data = ( + data >> 8 + ) & 0x3FFFFF # Next 22 least significant bytes. + huffman_compressed = (data >> 30) & 0x3 # Most significant bit. + + # Now we know where is the start of the page descriptors in the hibernation file. + mapped_address = ( + offset + self.HEADER_SIZE + number_of_descs * self.PAGE_DESC_SIZE ) - for j in range(page_count): - # Track the physical page number vs the page number in the compression set - self._mapping[(PageNum + j) * self.PAGE_SIZE] = ( - position * self.PAGE_SIZE + total_page_count = 0 + position = 0 + for i in range(number_of_descs): + # Go fetch and parse each page descriptor. + location = offset + self.HEADER_SIZE + i * self.PAGE_DESC_SIZE + page_descriptor = base_layer.read(location, self.PAGE_DESC_SIZE) + data = struct.unpack("> 4 # Shift right 4 bits to get the upper 60 bits + page_count = 1 + Numpages + total_page_count += page_count + self._segments.append( + ( + PageNum * self.PAGE_SIZE, + mapped_address, + self.PAGE_SIZE * page_count, + size_of_compressed_data, + ) ) - position += 1 - - total_page_size = total_page_count * self.PAGE_SIZE + for j in range(page_count): + # Track the physical page number vs the page number in the compression set + self._mapping[(PageNum + j) * self.PAGE_SIZE] = ( + position * self.PAGE_SIZE + ) + position += 1 - if total_page_size != size_of_compressed_data: - # This means compression so we track wich compression sets we actually need to decompress - self._compressed[mapped_address] = huffman_compressed - return total_page_count, ( - 4 + size_of_compressed_data + number_of_descs * self.PAGE_DESC_SIZE - ) # Number of pages in the set, Size of the entire compression set + total_page_size = total_page_count * self.PAGE_SIZE - def _decode_data( - self, data: bytes, mapped_offset: int, offset: int, output_length: int - ) -> bytes: - """ - Desc: decode the compressed data of one compression set - Params: - - data : the compressed data - - mapped_offset : starting location of the compressed data in the hib file - - offset: The offset inside the resulting raw file - - output_length: what is the size of the expected decompressed pages - Return: The decompressed data - """ - start_offset, _mapped_offset, _size, _mapped_size = self._find_segment(offset) - if mapped_offset in self._compressed: - decoded_data = uncompress(data=data, flag=self._compressed[mapped_offset]) - else: - # The data is not in our mapping so it's uncompressed. - decoded_data = data - page_offset = self._mapping[start_offset] - decoded_data = decoded_data[page_offset + (offset - start_offset) :] - decoded_data = decoded_data[:output_length] - return decoded_data + if total_page_size != size_of_compressed_data: + # This means compression so we track wich compression sets we actually need to decompress + self._compressed[mapped_address] = huffman_compressed + return total_page_count, ( + 4 + size_of_compressed_data + number_of_descs * self.PAGE_DESC_SIZE + ) # Number of pages in the set, Size of the entire compression set + def _decode_data( + self, data: bytes, mapped_offset: int, offset: int, output_length: int + ) -> bytes: + """ + Desc: decode the compressed data of one compression set + Params: + - data : the compressed data + - mapped_offset : starting location of the compressed data in the hib file + - offset: The offset inside the resulting raw file + - output_length: what is the size of the expected decompressed pages + Return: The decompressed data + """ + start_offset, _mapped_offset, _size, _mapped_size = self._find_segment( + offset + ) + if mapped_offset in self._compressed: + decoded_data = self._uncompress( + data=data, flag=self._compressed[mapped_offset] + ) + else: + # The data is not in our mapping so it's uncompressed. + decoded_data = data + page_offset = self._mapping[start_offset] + decoded_data = decoded_data[page_offset + (offset - start_offset) :] + decoded_data = decoded_data[:output_length] + return decoded_data -class HibernationFileStacker(interfaces.automagic.StackerLayerInterface): - stack_order = 10 + class HibernationFileStacker(interfaces.automagic.StackerLayerInterface): + stack_order = 10 - @classmethod - def stack( - cls, - context: interfaces.context.ContextInterface, - layer_name: str, - progress_callback: constants.ProgressCallback = None, - ) -> Optional[interfaces.layers.DataLayerInterface]: - try: - HibernationLayer._check_header(context.layers[layer_name]) - except exceptions.LayerException: - return None - new_name = context.layers.free_layer_name("HibernationLayer") - context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( - layer_name - ) - layer = HibernationLayer(context, new_name, new_name) - return layer + @classmethod + def stack( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None, + ) -> Optional[interfaces.layers.DataLayerInterface]: + try: + HibernationLayer._check_header(context.layers[layer_name]) + except exceptions.LayerException: + return None + new_name = context.layers.free_layer_name("HibernationLayer") + context.config[ + interfaces.configuration.path_join(new_name, "base_layer") + ] = layer_name + layer = HibernationLayer(context, new_name, new_name) + return layer diff --git a/volatility3/framework/plugins/windows/hibernation.py b/volatility3/framework/plugins/windows/hibernation.py index 46d306d07a..5bc68d6f38 100644 --- a/volatility3/framework/plugins/windows/hibernation.py +++ b/volatility3/framework/plugins/windows/hibernation.py @@ -3,9 +3,11 @@ # from typing import List -import logging +import logging, os +from volatility3.framework import constants from volatility3.framework.renderers import conversion from volatility3.framework.configuration import requirements +from volatility3.framework.symbols import intermed from volatility3.framework import interfaces, renderers from volatility3.framework.interfaces import plugins from volatility3.plugins import layerwriter @@ -26,45 +28,57 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] requirements.TranslationLayerRequirement(name="base_layer", optional=False), ] + @classmethod + def parse_hibernation_header(cls, hibernation_header): + if hibernation_header.PageSize == 4096: + system_time = hibernation_header.SystemTime + wintime = (system_time.High1Time << 32) | system_time.LowPart + system_time = conversion.wintime_to_datetime(wintime) + + comment = "The hibernation file header signature is correct." + return ( + ("Comment", comment), + ("PageSize", str(hibernation_header.PageSize)), + ("SystemTime", str(system_time)), + ("FirstBootRestorePage", str(hibernation_header.FirstBootRestorePage)), + ("NumPageForLoader", str(hibernation_header.NumPagesForLoader)), + ) + elif hibernation_header.PageSize == 2048: + comment = "The hibernation file is correct but x32 extraction isn't available yet." + else: + comment = "The hibernation file seems corrupted." + return ( + ("Comment", comment), + ("PageSize", str(hibernation_header.PageSize)), + ) + def _generator(self): base_layer = self.context.layers["base_layer"] - header = base_layer.read(0, 4) - yield (0, ("Signature", str(header))) - if header == b"HIBR": - # The hibernation file seems exploitable. Next step is to extract important information for the examiner - PageSize = int.from_bytes(base_layer.read(0x18, 4), "little") - yield (0, ("PageSize", str(PageSize))) - if PageSize == 4096: - yield ( - 0, - ("Comment", "The hibernation file header signature is correct."), - ) - system_time = int.from_bytes(base_layer.read(0x020, 8), "little") - systemTime = conversion.wintime_to_datetime(system_time) - yield (0, ("System Time", str(systemTime))) - FirstBootRestorePage = int.from_bytes( - base_layer.read(0x068, 8), "little" - ) - yield (0, ("FirstBootRestorePage", str(hex(FirstBootRestorePage)))) - NumPagesForLoader = int.from_bytes(base_layer.read(0x058, 8), "little") - yield (0, ("NumPagesForLoader", str(NumPagesForLoader))) - elif PageSize == 2048: - yield ( - 0, - ( - "Comment", - "The hibernation file header signature is correct but x32 compatibility is not available yet.", - ), - ) - else: - yield (0, ("Comment : ", "The file is corrupted.")) - elif header == b"RSTR": - # The hibernation file was extracted when Windows was in a resuming state which makes it not exploitable + symbol_table = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + os.path.join("windows", "hibernation"), + filename="header", + ) + hibernation_header_object = ( + symbol_table + constants.BANG + "PO_MEMORY_IMAGE_HEADER" + ) + hibernation_header = self.context.object( + hibernation_header_object, offset=0, layer_name=base_layer.name + ) + signature = hibernation_header.Signature.cast( + "string", max_length=4, encoding="latin-1" + ) + yield (0, ("Signature", signature)) + if signature == "HIBR": + for field in self.parse_hibernation_header(hibernation_header): + yield (0, field) + elif signature == "RSTR": yield ( 0, ( "Comment : ", - "The hibernation file header signature is 'RSTR', the file cannot be exploited.", + "RSTR : The hibernation file was extracted when Windows was in a resuming state which makes it not exploitable.", ), ) else: @@ -129,8 +143,9 @@ def _generator(self): ( """Your hibernation file could not be converted, this can be the case for multiple reasons: - The hibernation file you are trying to dump is corrupted. - - The version you provided is not expected (see --help) + - The version you provided is not expected (see --help). - The file you are trying to dump is not an hibernation file. + - Missing requirements: Make sure all the requirements are installed (requirements.txt). """, ), ) diff --git a/volatility3/framework/symbols/windows/hibernation/header.json b/volatility3/framework/symbols/windows/hibernation/header.json new file mode 100644 index 0000000000..d8cd881233 --- /dev/null +++ b/volatility3/framework/symbols/windows/hibernation/header.json @@ -0,0 +1,163 @@ +{ + "metadata": { + "producer": { + "version": "0.0.1", + "name": "k1nd0ne-by-hand", + "comment": "using structures found in the Windows Kernel public structures ", + "datetime": "2024-05-08T19:55:00" + }, + "format": "6.1.0" + }, + "base_types": { + "char": { + "endian": "little", + "kind": "char", + "signed": true, + "size": 1 + }, + "double": { + "endian": "little", + "kind": "float", + "signed": true, + "size": 8 + }, + "int": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 4 + }, + "long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 4 + }, + "long long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 8 + }, + "pointer": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "short": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 2 + }, + "unsigned char": { + "endian": "little", + "kind": "char", + "signed": false, + "size": 1 + }, + "unsigned int": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "unsigned long long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + }, + "unsigned short": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 2 + }, + "void": { + "endian": "little", + "kind": "void", + "signed": true, + "size": 0 + } + }, + "symbols": {}, + "enums": {}, + "user_types": { + "PO_MEMORY_IMAGE_HEADER": { + "fields": { + "PageSize": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "Signature": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "SystemTime": { + "offset": 32, + "type": { + "kind": "struct", + "name": "_KSYSTEM_TIME" + } + }, + "FirstBootRestorePage": { + "offset": 104, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "NumPagesForLoader": { + "offset": 88, + "type": { + "kind": "base", + "name": "unsigned long long" + } + } + }, + "kind": "struct", + "size": 168 + }, + "_KSYSTEM_TIME": { + "fields": { + "High1Time": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "High2Time": { + "offset": 8, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "kind": "struct", + "size": 12 + } + } +}