diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index e29ab6f290..d755d9402f 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,4 +1,4 @@ -name: Black python linter +name: Black python formatter on: [push, pull_request] diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 398ff8ae36..9b9cbed4df 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -20,12 +20,8 @@ jobs: - name: Setup python-pip run: python -m pip install --upgrade pip - - name: Install dependencies - run: | - pip install -r requirements.txt - - name: Install volatility3 run: pip install . - name: Run volatility3 - run: vol --help \ No newline at end of file + run: vol --help diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml new file mode 100644 index 0000000000..77e3aa864d --- /dev/null +++ b/.github/workflows/ruff.yaml @@ -0,0 +1,15 @@ +--- +name: Ruff + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/ruff-action@v1 + with: + args: check + src: "." diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6358dd45d3..ce27224575 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,10 +16,8 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install Cmake - pip install build - pip install -r ./test/requirements-testing.txt + python -m pip install --upgrade pip Cmake build + pip install .[test] - name: Build PyPi packages run: | @@ -27,10 +25,13 @@ jobs: - name: Download images run: | + mkdir test_images + cd test_images curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/linux-sample-1.bin.gz" gunzip linux-sample-1.bin.gz curl -sLO "https://downloads.volatilityfoundation.org/volatility3/images/win-xp-laptop-2005-06-25.img.gz" gunzip win-xp-laptop-2005-06-25.img.gz + cd - - name: Download and Extract symbols run: | @@ -41,13 +42,17 @@ jobs: - name: Testing... run: | - py.test ./test/test_volatility.py --volatility=vol.py --image win-xp-laptop-2005-06-25.img -k test_windows -v - py.test ./test/test_volatility.py --volatility=vol.py --image linux-sample-1.bin -k test_linux -v + # VolShell + pytest ./test/test_volatility.py --volatility=volshell.py --image-dir=./test_images -k test_windows_volshell -v + pytest ./test/test_volatility.py --volatility=volshell.py --image-dir=./test_images -k test_linux_volshell -v + + # Volatility + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k "test_windows and not test_windows_volshell" -v + pytest ./test/test_volatility.py --volatility=vol.py --image-dir=./test_images -k "test_linux and not test_linux_volshell" -v - name: Clean up post-test run: | - rm -rf *.bin - rm -rf *.img + rm -rf test_images cd volatility3/symbols rm -rf linux rm -rf linux.zip diff --git a/.readthedocs.yml b/.readthedocs.yml index e7c2b25d55..628e79ebf7 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -20,4 +20,7 @@ build: # Optionally set the version of Python and requirements required to build your docs python: install: - - requirements: doc/requirements.txt + - method: pip + path: . + extra_requirements: + - docs diff --git a/MANIFEST.in b/MANIFEST.in index 1cec729f69..8636213816 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ prune development include * .* -include doc/make.bat doc/Makefile doc/requirements.txt +include pyproject.toml doc/make.bat doc/Makefile recursive-include doc/source * recursive-include volatility3 *.json recursive-exclude doc/source volatility3.*.rst diff --git a/README.md b/README.md index 1463c2bde9..cc33d3cc43 100644 --- a/README.md +++ b/README.md @@ -18,62 +18,44 @@ the Volatility Software License (VSL). See the [LICENSE](https://www.volatilityfoundation.org/license/vsl-v1.0) file for more details. -## Requirements +## Installing -Volatility 3 requires Python 3.8.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: +Volatility 3 requires Python 3.8.0 or later and is published on the [PyPi registry](https://pypi.org/project/volatility3). ```shell -pip3 install -r requirements-minimal.txt +pip install volatility3 ``` -Alternately, the minimal packages will be installed automatically when Volatility 3 is installed using pip. However, as noted in the Quick Start section below, Volatility 3 does not *need* to be installed prior to using it. +If you want to use the latest development version of Volatility 3 we recommend you manually clone this repository and install an editable version of the project. +We recommend you use a virtual environment to keep installed dependencies separate from system packages. -```shell -pip3 install . -``` - -To enable the full range of Volatility 3 functionality, use a command like the one below. For partial functionality, comment out any unnecessary packages in [requirements.txt](requirements.txt) prior to running the command. - -```shell -pip3 install -r requirements.txt -``` - -## Downloading Volatility - -The latest stable version of Volatility will always be the stable branch of the GitHub repository. You can get the latest version of the code using the following command: +The latest stable version of Volatility will always be the `stable` branch of the GitHub repository. The default branch is `develop`. ```shell git clone https://github.com/volatilityfoundation/volatility3.git +cd volatility3/ +python3 -m venv venv && . venv/bin/activate +pip install -e .[dev] ``` ## Quick Start -1. Clone the latest version of Volatility from GitHub: - - ```shell - git clone https://github.com/volatilityfoundation/volatility3.git - ``` +1. Install Volatility 3 as documented in the Installing section of the readme. 2. See available options: ```shell - python3 vol.py -h + vol -h ``` -3. To get more information on a Windows memory sample and to make sure -Volatility supports that sample type, run -`python3 vol.py -f windows.info` - - Example: +3. To get more information on a Windows memory sample and to make sure Volatility supports that sample type, run `vol -f windows.info`: ```shell - python3 vol.py -f /home/user/samples/stuxnet.vmem windows.info + vol -f /home/user/samples/stuxnet.vmem windows.info ``` -4. Run some other plugins. The `-f` or `--single-location` is not strictly -required, but most plugins expect a single sample. Some also -require/accept other options. Run `python3 vol.py -h` -for more information on a particular command. +4. Run some other plugins. The `-f` or `--single-location` is not strictly required, but most plugins expect a single sample. +Some also require/accept other options. Run `vol -h` for more information on a particular command. ## Symbol Tables diff --git a/development/banner_server.py b/development/banner_server.py index 3aea41c825..b62477a269 100644 --- a/development/banner_server.py +++ b/development/banner_server.py @@ -28,10 +28,10 @@ def convert_url(self, url): def run(self): context = contexts.Context() - json_output = {'version': 1} + json_output = {"version": 1} path = self._path - filename = '*' + filename = "*" for banner_cache in [linux.LinuxBannerCache, mac.MacBannerCache]: sub_path = banner_cache.os @@ -39,37 +39,54 @@ def run(self): for extension in constants.ISF_EXTENSIONS: # Hopefully these will not be large lists, otherwise this might be slow try: - for found in pathlib.Path(path).joinpath(sub_path).resolve().rglob(filename + extension): + for found in ( + pathlib.Path(path) + .joinpath(sub_path) + .resolve() + .rglob(filename + extension) + ): potentials.append(found.as_uri()) except FileNotFoundError: # If there's no linux symbols, don't cry about it pass - new_banners = banner_cache.read_new_banners(context, 'BannerServer', potentials, banner_cache.symbol_name, - banner_cache.os, progress_callback = PrintedProgress()) + new_banners = banner_cache.read_new_banners( + context, + "BannerServer", + potentials, + banner_cache.symbol_name, + banner_cache.os, + progress_callback=PrintedProgress(), + ) result_banners = {} for new_banner in new_banners: # Only accept file schemes - value = [self.convert_url(url) for url in new_banners[new_banner] if - urllib.parse.urlparse(url).scheme == 'file'] + value = [ + self.convert_url(url) + for url in new_banners[new_banner] + if urllib.parse.urlparse(url).scheme == "file" + ] if value and new_banner: # Convert files into URLs - result_banners[str(base64.b64encode(new_banner), 'latin-1')] = value + result_banners[str(base64.b64encode(new_banner), "latin-1")] = value json_output[banner_cache.os] = result_banners - output_path = os.path.join(self._path, 'banners.json') - with open(output_path, 'w') as fp: + output_path = os.path.join(self._path, "banners.json") + with open(output_path, "w") as fp: vollog.warning(f"Banners file written to {output_path}") json.dump(json_output, fp) -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--path', default = os.path.dirname(__file__)) - parser.add_argument('--urlprefix', help = 'Web prefix that will eventually serve the ISF files', - default = 'http://localhost/symbols') + parser.add_argument("--path", default=os.path.dirname(__file__)) + parser.add_argument( + "--urlprefix", + help="Web prefix that will eventually serve the ISF files", + default="http://localhost/symbols", + ) args = parser.parse_args() diff --git a/development/compare-vol.py b/development/compare-vol.py index d0d8340389..a01d8e93c3 100644 --- a/development/compare-vol.py +++ b/development/compare-vol.py @@ -15,17 +15,17 @@ class VolatilityImage: filepath: str = "" vol2_profile: str = "" vol2_imageinfo_time: float = None - vol2_plugin_parameters: Dict[str, List[str]] = field(default_factory = dict) - vol3_plugin_parameters: Dict[str, List[str]] = field(default_factory = dict) - rekall_plugin_parameters: Dict[str, List[str]] = field(default_factory = dict) + vol2_plugin_parameters: Dict[str, List[str]] = field(default_factory=dict) + vol3_plugin_parameters: Dict[str, List[str]] = field(default_factory=dict) + rekall_plugin_parameters: Dict[str, List[str]] = field(default_factory=dict) @dataclass class VolatilityPlugin: name: str = "" - vol2_plugin_parameters: List[str] = field(default_factory = list) - vol3_plugin_parameters: List[str] = field(default_factory = list) - rekall_plugin_parameters: List[str] = field(default_factory = list) + vol2_plugin_parameters: List[str] = field(default_factory=list) + vol3_plugin_parameters: List[str] = field(default_factory=list) + rekall_plugin_parameters: List[str] = field(default_factory=list) class VolatilityTest: @@ -39,32 +39,50 @@ def __init__(self, path: str, output_directory: str) -> None: def result_titles(self) -> List[str]: return [self.long_name] - def create_prerequisites(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str) -> None: + def create_prerequisites( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str + ) -> None: pass - def create_results(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str) -> List[float]: + def create_results( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str + ) -> List[float]: self.create_prerequisites(plugin, image, image_hash) # Volatility 2 Test - print(f"[*] Testing {self.short_name} {plugin.name} with image {image.filepath}") + print( + f"[*] Testing {self.short_name} {plugin.name} with image {image.filepath}" + ) os.chdir(self.path) cmd = self.plugin_cmd(plugin, image) start_time = time.perf_counter() try: - completed = subprocess.run(cmd, cwd = self.path, capture_output = True, timeout = 420) + completed = subprocess.run( + cmd, cwd=self.path, capture_output=True, timeout=420 + ) except subprocess.TimeoutExpired as excp: completed = excp end_time = time.perf_counter() total_time = end_time - start_time - print(f" Tested {self.short_name} {plugin.name} with image {image.filepath}: {total_time}") + print( + f" Tested {self.short_name} {plugin.name} with image {image.filepath}: {total_time}" + ) with open( - os.path.join(self.output_directory, f'{self.short_name}_{plugin.name}_{image_hash}_stdout'), - "wb") as f: + os.path.join( + self.output_directory, + f"{self.short_name}_{plugin.name}_{image_hash}_stdout", + ), + "wb", + ) as f: f.write(completed.stdout) if completed.stderr: with open( - os.path.join(self.output_directory, f'{self.short_name}_{plugin.name}_{image_hash}_stderr'), - "wb") as f: + os.path.join( + self.output_directory, + f"{self.short_name}_{plugin.name}_{image_hash}_stderr", + ), + "wb", + ) as f: f.write(completed.stderr) return [total_time] @@ -77,31 +95,57 @@ class Volatility2Test(VolatilityTest): long_name = "Volatility 2" def plugin_cmd(self, plugin: VolatilityPlugin, image: VolatilityImage): - return ["python2", "-u", "vol.py", "-f", image.filepath, "--profile", image.vol2_profile - ] + plugin.vol2_plugin_parameters + image.vol2_plugin_parameters.get(plugin.name, []) + return ( + [ + "python2", + "-u", + "vol.py", + "-f", + image.filepath, + "--profile", + image.vol2_profile, + ] + + plugin.vol2_plugin_parameters + + image.vol2_plugin_parameters.get(plugin.name, []) + ) def result_titles(self): return [self.long_name, "Imageinfo", f"{self.long_name} + Imageinfo"] - def create_results(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash) -> List[float]: + def create_results( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash + ) -> List[float]: result = super().create_results(plugin, image, image_hash) result += [image.vol2_imageinfo_time, result[0] + image.vol2_imageinfo_time] return result - def create_prerequisites(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash): + def create_prerequisites( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash + ): # Volatility 2 image info if not image.vol2_profile: - print(f"[*] Testing {self.short_name} imageinfo with image {image.filepath}") + print( + f"[*] Testing {self.short_name} imageinfo with image {image.filepath}" + ) os.chdir(self.path) cmd = ["python2", "-u", "vol.py", "-f", image.filepath, "imageinfo"] start_time = time.perf_counter() - vol2_completed = subprocess.run(cmd, cwd = self.path, capture_output = True) + vol2_completed = subprocess.run(cmd, cwd=self.path, capture_output=True) end_time = time.perf_counter() image.vol2_imageinfo_time = end_time - start_time - print(f" Tested volatility2 imageinfo with image {image.filepath}: {end_time - start_time}") - with open(os.path.join(self.output_directory, f'vol2_imageinfo_{image_hash}_stdout'), "wb") as f: + print( + f" Tested volatility2 imageinfo with image {image.filepath}: {end_time - start_time}" + ) + with open( + os.path.join( + self.output_directory, f"vol2_imageinfo_{image_hash}_stdout" + ), + "wb", + ) as f: f.write(vol2_completed.stdout) - image.vol2_profile = re.search(b"Suggested Profile\(s\) : ([^,]+)", vol2_completed.stdout)[1] + image.vol2_profile = re.search( + rb"Suggested Profile\(s\) : ([^,]+)", vol2_completed.stdout + )[1] class RekallTest(VolatilityTest): @@ -113,11 +157,16 @@ def plugin_cmd(self, plugin: VolatilityPlugin, image: VolatilityImage) -> List[s plugin.rekall_plugin_parameters = plugin.vol2_plugin_parameters if not image.rekall_plugin_parameters: image.rekall_plugin_parameters = image.vol2_plugin_parameters - return ["rekall", "-f", image.filepath] + plugin.rekall_plugin_parameters + image.rekall_plugin_parameters.get( - plugin.name, []) + return ( + ["rekall", "-f", image.filepath] + + plugin.rekall_plugin_parameters + + image.rekall_plugin_parameters.get(plugin.name, []) + ) - def create_prerequisites(self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str) -> None: - shutil.rmtree('/home/mike/.rekall_cache/sessions') + def create_prerequisites( + self, plugin: VolatilityPlugin, image: VolatilityImage, image_hash: str + ) -> None: + shutil.rmtree("/home/mike/.rekall_cache/sessions") class Volatility3Test(VolatilityTest): @@ -125,14 +174,18 @@ class Volatility3Test(VolatilityTest): long_name = "Volatility 3" def plugin_cmd(self, plugin: VolatilityPlugin, image: VolatilityImage) -> List[str]: - return [ - "python", - "-u", - "vol.py", - "-q", - "-f", - image.filepath, - ] + plugin.vol3_plugin_parameters + image.vol3_plugin_parameters.get(plugin.name, []) + return ( + [ + "python", + "-u", + "vol.py", + "-q", + "-f", + image.filepath, + ] + + plugin.vol3_plugin_parameters + + image.vol3_plugin_parameters.get(plugin.name, []) + ) class Volatility3PyPyTest(VolatilityTest): @@ -140,26 +193,32 @@ class Volatility3PyPyTest(VolatilityTest): long_name = "Volatility 3 (PyPy)" def plugin_cmd(self, plugin: VolatilityPlugin, image: VolatilityImage) -> List[str]: - return [ - "pypy3", - "-u", - "vol.py", - "-q", - "-f", - image.filepath, - ] + plugin.vol3_plugin_parameters + image.vol3_plugin_parameters.get(plugin.name, []) + return ( + [ + "pypy3", + "-u", + "vol.py", + "-q", + "-f", + image.filepath, + ] + + plugin.vol3_plugin_parameters + + image.vol3_plugin_parameters.get(plugin.name, []) + ) class VolatilityTester: - def __init__(self, - images: List[VolatilityImage], - plugins: List[VolatilityPlugin], - frameworks: List[str], - output_dir: str, - vol2_path: str = None, - vol3_path: str = None, - rekall_path = None): + def __init__( + self, + images: List[VolatilityImage], + plugins: List[VolatilityPlugin], + frameworks: List[str], + output_dir: str, + vol2_path: str = None, + vol3_path: str = None, + rekall_path=None, + ): self.images = images self.plugins = plugins if not vol2_path: @@ -172,7 +231,7 @@ def __init__(self, Volatility3Test(vol3_path, output_dir), Volatility3PyPyTest(vol3_path, output_dir), Volatility2Test(vol2_path, output_dir), - RekallTest(rekall_path, output_dir) + RekallTest(rekall_path, output_dir), ] self.tests = [x for x in available_tests if x.short_name.lower() in frameworks] self.csv_writer = None @@ -183,7 +242,7 @@ def __init__(self, print(f"[?] Frameworks: {[x.long_name for x in self.tests]}") def run_tests(self): - with open("volatility-timings.csv", 'w') as csvfile: + with open("volatility-timings.csv", "w") as csvfile: self.csv_writer = csv.writer(csvfile) titles = ["Image Hash", "Image Path", "Plugin Name"] for test in self.tests: @@ -203,72 +262,121 @@ def run_test(self, plugin: VolatilityPlugin, image: VolatilityImage): self.csv_writer.writerow([image_hash, image.filepath, plugin.name] + results) -if __name__ == '__main__': +if __name__ == "__main__": plugins = [ - VolatilityPlugin(name = "pslist", - vol2_plugin_parameters = ["pslist"], - vol3_plugin_parameters = ["windows.pslist"]), - VolatilityPlugin(name = "psscan", - vol2_plugin_parameters = ["psscan"], - vol3_plugin_parameters = ["windows.psscan"], - rekall_plugin_parameters = ["psscan", "--scan_kernel"]), - VolatilityPlugin(name = "driverscan", - vol2_plugin_parameters = ["driverscan"], - vol3_plugin_parameters = ["windows.driverscan"], - rekall_plugin_parameters = ["driverscan", "--scan_kernel"]), - VolatilityPlugin(name = "handles", - vol2_plugin_parameters = ["handles"], - vol3_plugin_parameters = ["windows.handles"]), - VolatilityPlugin(name = "modules", - vol2_plugin_parameters = ["modules"], - vol3_plugin_parameters = ["windows.modules"]), - VolatilityPlugin(name = "hivelist", - vol2_plugin_parameters = ["hivelist"], - vol3_plugin_parameters = ["registry.hivelist"], - rekall_plugin_parameters = ["hives"]), - VolatilityPlugin(name = "vadinfo", - vol2_plugin_parameters = ["vadinfo"], - vol3_plugin_parameters = ["windows.vadinfo"], - rekall_plugin_parameters = ["vad"]), - VolatilityPlugin(name = "modscan", - vol2_plugin_parameters = ["modscan"], - vol3_plugin_parameters = ["windows.modscan"], - rekall_plugin_parameters = ["modscan", "--scan_kernel"]), - VolatilityPlugin(name = "svcscan", - vol2_plugin_parameters = ["svcscan"], - vol3_plugin_parameters = ["windows.svcscan"], - rekall_plugin_parameters = ["svcscan"]), - VolatilityPlugin(name = "ssdt", vol2_plugin_parameters = ["ssdt"], vol3_plugin_parameters = ["windows.ssdt"]), - VolatilityPlugin(name = "printkey", - vol2_plugin_parameters = ["printkey", "-K", "Classes"], - vol3_plugin_parameters = ["registry.printkey", "--key", "Classes"], - rekall_plugin_parameters = ["printkey", "--key", "Classes"]) + VolatilityPlugin( + name="pslist", + vol2_plugin_parameters=["pslist"], + vol3_plugin_parameters=["windows.pslist"], + ), + VolatilityPlugin( + name="psscan", + vol2_plugin_parameters=["psscan"], + vol3_plugin_parameters=["windows.psscan"], + rekall_plugin_parameters=["psscan", "--scan_kernel"], + ), + VolatilityPlugin( + name="driverscan", + vol2_plugin_parameters=["driverscan"], + vol3_plugin_parameters=["windows.driverscan"], + rekall_plugin_parameters=["driverscan", "--scan_kernel"], + ), + VolatilityPlugin( + name="handles", + vol2_plugin_parameters=["handles"], + vol3_plugin_parameters=["windows.handles"], + ), + VolatilityPlugin( + name="modules", + vol2_plugin_parameters=["modules"], + vol3_plugin_parameters=["windows.modules"], + ), + VolatilityPlugin( + name="hivelist", + vol2_plugin_parameters=["hivelist"], + vol3_plugin_parameters=["registry.hivelist"], + rekall_plugin_parameters=["hives"], + ), + VolatilityPlugin( + name="vadinfo", + vol2_plugin_parameters=["vadinfo"], + vol3_plugin_parameters=["windows.vadinfo"], + rekall_plugin_parameters=["vad"], + ), + VolatilityPlugin( + name="modscan", + vol2_plugin_parameters=["modscan"], + vol3_plugin_parameters=["windows.modscan"], + rekall_plugin_parameters=["modscan", "--scan_kernel"], + ), + VolatilityPlugin( + name="svcscan", + vol2_plugin_parameters=["svcscan"], + vol3_plugin_parameters=["windows.svcscan"], + rekall_plugin_parameters=["svcscan"], + ), + VolatilityPlugin( + name="ssdt", + vol2_plugin_parameters=["ssdt"], + vol3_plugin_parameters=["windows.ssdt"], + ), + VolatilityPlugin( + name="printkey", + vol2_plugin_parameters=["printkey", "-K", "Classes"], + vol3_plugin_parameters=["registry.printkey", "--key", "Classes"], + rekall_plugin_parameters=["printkey", "--key", "Classes"], + ), ] parser = argparse.ArgumentParser() - parser.add_argument("--output-dir", type = str, default = os.getcwd(), help = "Directory to store all results") - parser.add_argument("--vol3path", - type = str, - default = os.path.join(os.getcwd(), 'volatility3'), - help = "Path ot the volatility 3 directory") - parser.add_argument("--vol2path", - type = str, - default = os.path.join(os.getcwd(), 'volatility'), - help = "Path to the volatility 2 directory") - parser.add_argument("--rekallpath", - type = str, - default = os.path.join(os.getcwd(), 'rekall'), - help = "Path to the rekall directory") - parser.add_argument("--frameworks", - nargs = "+", - type = str, - choices = [x.short_name.lower() for x in VolatilityTest.__subclasses__()], - default = [x.short_name.lower() for x in VolatilityTest.__subclasses__()], - help = "A comma separated list of frameworks to test") - parser.add_argument('images', metavar = 'IMAGE', type = str, nargs = '+', help = 'The list of images to compare') + parser.add_argument( + "--output-dir", + type=str, + default=os.getcwd(), + help="Directory to store all results", + ) + parser.add_argument( + "--vol3path", + type=str, + default=os.path.join(os.getcwd(), "volatility3"), + help="Path ot the volatility 3 directory", + ) + parser.add_argument( + "--vol2path", + type=str, + default=os.path.join(os.getcwd(), "volatility"), + help="Path to the volatility 2 directory", + ) + parser.add_argument( + "--rekallpath", + type=str, + default=os.path.join(os.getcwd(), "rekall"), + help="Path to the rekall directory", + ) + parser.add_argument( + "--frameworks", + nargs="+", + type=str, + choices=[x.short_name.lower() for x in VolatilityTest.__subclasses__()], + default=[x.short_name.lower() for x in VolatilityTest.__subclasses__()], + help="A comma separated list of frameworks to test", + ) + parser.add_argument( + "images", + metavar="IMAGE", + type=str, + nargs="+", + help="The list of images to compare", + ) args = parser.parse_args() - vt = VolatilityTester([VolatilityImage(filepath = x) for x in args.images], plugins, - [x.lower() for x in args.frameworks], args.output_dir, args.vol2path, args.vol3path, - args.rekallpath) + vt = VolatilityTester( + [VolatilityImage(filepath=x) for x in args.images], + plugins, + [x.lower() for x in args.frameworks], + args.output_dir, + args.vol2path, + args.vol3path, + args.rekallpath, + ) vt.run_tests() diff --git a/development/mac-kdk/parse_pbzx2.py b/development/mac-kdk/parse_pbzx2.py index 173a4d6482..b175539b30 100644 --- a/development/mac-kdk/parse_pbzx2.py +++ b/development/mac-kdk/parse_pbzx2.py @@ -7,11 +7,12 @@ # Cleaned up C version (as the basis for my code) here, thanks to Pepijn Bruienne / @bruienne # https://gist.github.com/bruienne/029494bbcfb358098b41 +import os import struct import sys -def seekread(f, offset = None, length = 0, relative = True): +def seekread(f, offset=None, length=0, relative=True): if offset is not None: # offset provided, let's seek f.seek(offset, [0, 1, 2][relative]) @@ -22,55 +23,57 @@ def seekread(f, offset = None, length = 0, relative = True): def parse_pbzx(pbzx_path): section = 0 - xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section) - with open(pbzx_path, 'rb') as f: + xar_out_path = f"{pbzx_path}.part{section:02d}.cpio.xz" + with open(pbzx_path, "rb") as f: # pbzx = f.read() # f.close() - magic = seekread(f, length = 4) - if magic != 'pbzx': + magic = seekread(f, length=4) + if magic != "pbzx": raise RuntimeError("Error: Not a pbzx file") # Read 8 bytes for initial flags - flags = seekread(f, length = 8) + flags = seekread(f, length=8) # Interpret the flags as a 64-bit big-endian unsigned int - flags = struct.unpack('>Q', flags)[0] + flags = struct.unpack(">Q", flags)[0] while flags & (1 << 24): - with open(xar_out_path, 'wb') as xar_f: + with open(xar_out_path, "wb") as xar_f: xar_f.seek(0, os.SEEK_END) # Read in more flags - flags = seekread(f, length = 8) - flags = struct.unpack('>Q', flags)[0] + flags = seekread(f, length=8) + flags = struct.unpack(">Q", flags)[0] # Read in length - f_length = seekread(f, length = 8) - f_length = struct.unpack('>Q', f_length)[0] - xzmagic = seekread(f, length = 6) - if xzmagic != '\xfd7zXZ\x00': + f_length = seekread(f, length=8) + f_length = struct.unpack(">Q", f_length)[0] + xzmagic = seekread(f, length=6) + if xzmagic != "\xfd7zXZ\x00": # This isn't xz content, this is actually _raw decompressed cpio_ chunk of 16MB in size... # Let's back up ... - seekread(f, offset = -6, length = 0) + seekread(f, offset=-6, length=0) # ... and split it out ... - f_content = seekread(f, length = f_length) + f_content = seekread(f, length=f_length) section += 1 - decomp_out = '%s.part%02d.cpio' % (pbzx_path, section) - with open(decomp_out, 'wb') as g: + decomp_out = f"{pbzx_path}.part{section:02d}.cpio" + with open(decomp_out, "wb") as g: g.write(f_content) # Now to start the next section, which should hopefully be .xz (we'll just assume it is ...) section += 1 - xar_out_path = '%s.part%02d.cpio.xz' % (pbzx_path, section) + xar_out_path = f"{pbzx_path}.part{section:02d}.cpio.xz" else: f_length -= 6 # This part needs buffering - f_content = seekread(f, length = f_length) - tail = seekread(f, offset = -2, length = 2) + f_content = seekread(f, length=f_length) + tail = seekread(f, offset=-2, length=2) xar_f.write(xzmagic) xar_f.write(f_content) - if tail != 'YZ': + if tail != "YZ": raise RuntimeError("Error: Footer is not xar file footer") def main(): parse_pbzx(sys.argv[1]) - print("Now xz decompress the .xz chunks, then 'cat' them all together in order into a single new.cpio file") + print( + "Now xz decompress the .xz chunks, then 'cat' them all together in order into a single new.cpio file" + ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/development/pdbparse-to-json.py b/development/pdbparse-to-json.py index 819e44e15d..49b4da009d 100644 --- a/development/pdbparse-to-json.py +++ b/development/pdbparse-to-json.py @@ -13,10 +13,10 @@ logger = logging.getLogger(__name__) logger.setLevel(1) -if __name__ == '__main__': +if __name__ == "__main__": console = logging.StreamHandler() console.setLevel(1) - formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') + formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s") console.setFormatter(formatter) logger.addHandler(console) @@ -25,19 +25,19 @@ class PDBRetreiver: def retreive_pdb(self, guid: str, file_name: str) -> Optional[str]: logger.info("Download PDB file...") - file_name = ".".join(file_name.split(".")[:-1] + ['pdb']) - for sym_url in ['http://msdl.microsoft.com/download/symbols']: + file_name = ".".join(file_name.split(".")[:-1] + ["pdb"]) + for sym_url in ["http://msdl.microsoft.com/download/symbols"]: url = sym_url + f"/{file_name}/{guid}/" result = None - for suffix in [file_name[:-1] + '_', file_name]: + for suffix in [file_name[:-1] + "_", file_name]: try: - logger.debug(f"Attempting to retrieve {url + suffix}") + logger.debug("Attempting to retrieve %s", url + suffix) result, _ = request.urlretrieve(url + suffix) except request.HTTPError as excp: - logger.debug(f"Failed with {excp}") + logger.debug("Failed with %s", excp) if result: - logger.debug(f"Successfully written to {result}") + logger.debug("Successfully written to %s", result) break return result @@ -69,7 +69,7 @@ class PDBConvertor: "float": "float", "double": "float", "long double": "float", - "void": "void" + "void": "void", } base_type_size = { @@ -122,13 +122,18 @@ def lookup_ctype(self, ctype: str) -> str: self._seen_ctypes.add(ctype) return self.ctype[ctype] - def lookup_ctype_pointers(self, ctype_pointer: str) -> Dict[str, Union[str, Dict[str, str]]]: - base_type = ctype_pointer.replace('32P', '').replace('64P', '') + def lookup_ctype_pointers( + self, ctype_pointer: str + ) -> Dict[str, Union[str, Dict[str, str]]]: + base_type = ctype_pointer.replace("32P", "").replace("64P", "") if base_type == ctype_pointer: # We raise a KeyError, because we've been asked about a type that isn't a pointer raise KeyError self._seen_ctypes.add(base_type) - return {"kind": "pointer", "subtype": {"kind": "base", "name": self.ctype[base_type]}} + return { + "kind": "pointer", + "subtype": {"kind": "base", "name": self.ctype[base_type]}, + } def read_pdb(self) -> Dict: """Reads in the PDB file and forms essentially a python dictionary of necessary data""" @@ -137,32 +142,31 @@ def read_pdb(self) -> Dict: "enums": self.read_enums(), "metadata": self.generate_metadata(), "symbols": self.read_symbols(), - "base_types": self.read_basetypes() + "base_types": self.read_basetypes(), } return output def generate_metadata(self) -> Dict[str, Any]: """Generates the metadata necessary for this object""" dbg = self._pdb.STREAM_DBI - last_bytes = str(binascii.hexlify(self._pdb.STREAM_PDB.GUID.Data4), 'ascii')[-16:] - guidstr = u'{:08x}{:04x}{:04x}{}'.format(self._pdb.STREAM_PDB.GUID.Data1, self._pdb.STREAM_PDB.GUID.Data2, - self._pdb.STREAM_PDB.GUID.Data3, last_bytes) + last_bytes = str(binascii.hexlify(self._pdb.STREAM_PDB.GUID.Data4), "ascii")[ + -16: + ] + guidstr = f"{self._pdb.STREAM_PDB.GUID.Data1:08x}{self._pdb.STREAM_PDB.GUID.Data2:04x}{self._pdb.STREAM_PDB.GUID.Data3:04x}{last_bytes}" pdb_data = { "GUID": guidstr.upper(), "age": self._pdb.STREAM_PDB.Age, "database": "ntkrnlmp.pdb", - "machine_type": int(dbg.machine) + "machine_type": int(dbg.machine), } result = { "format": "6.0.0", "producer": { "datetime": datetime.datetime.now().isoformat(), "name": "pdbconv", - "version": "0.1.0" + "version": "0.1.0", }, - "windows": { - "pdb": pdb_data - } + "windows": {"pdb": pdb_data}, } return result @@ -173,16 +177,21 @@ def read_enums(self) -> Dict: stream = self._pdb.STREAM_TPI for type_index in stream.types: user_type = stream.types[type_index] - if (user_type.leaf_type == "LF_ENUM" and not user_type.prop.fwdref): + if user_type.leaf_type == "LF_ENUM" and not user_type.prop.fwdref: output.update(self._format_enum(user_type)) return output def _format_enum(self, user_enum): output = { user_enum.name: { - 'base': self.lookup_ctype(user_enum.utype), - 'size': self._determine_size(user_enum.utype), - 'constants': dict([(enum.name, enum.enum_value) for enum in user_enum.fieldlist.substructs]) + "base": self.lookup_ctype(user_enum.utype), + "size": self._determine_size(user_enum.utype), + "constants": dict( + [ + (enum.name, enum.enum_value) + for enum in user_enum.fieldlist.substructs + ] + ), } } return output @@ -195,14 +204,14 @@ def read_symbols(self) -> Dict: try: sects = self._pdb.STREAM_SECT_HDR_ORIG.sections omap = self._pdb.STREAM_OMAP_FROM_SRC - except AttributeError as e: + except AttributeError: # In this case there is no OMAP, so we use the given section # headers and use the identity function for omap.remap sects = self._pdb.STREAM_SECT_HDR.sections omap = None for sym in self._pdb.STREAM_GSYM.globals: - if not hasattr(sym, 'offset'): + if not hasattr(sym, "offset"): continue try: virt_base = sects[sym.segment - 1].VirtualAddress @@ -223,9 +232,9 @@ def read_usertypes(self) -> Dict: stream = self._pdb.STREAM_TPI for type_index in stream.types: user_type = stream.types[type_index] - if (user_type.leaf_type == "LF_STRUCTURE" and not user_type.prop.fwdref): + if user_type.leaf_type == "LF_STRUCTURE" and not user_type.prop.fwdref: output.update(self._format_usertype(user_type, "struct")) - elif (user_type.leaf_type == "LF_UNION" and not user_type.prop.fwdref): + elif user_type.leaf_type == "LF_UNION" and not user_type.prop.fwdref: output.update(self._format_usertype(user_type, "union")) return output @@ -233,16 +242,22 @@ def _format_usertype(self, usertype, kind) -> Dict: """Produces a single usertype""" fields: Dict[str, Dict[str, Any]] = {} [fields.update(self._format_field(s)) for s in usertype.fieldlist.substructs] - return {usertype.name: {'fields': fields, 'kind': kind, 'size': usertype.size}} + return {usertype.name: {"fields": fields, "kind": kind, "size": usertype.size}} def _format_field(self, field) -> Dict[str, Dict[str, Any]]: - return {field.name: {"offset": field.offset, "type": self._format_kind(field.index)}} + return { + field.name: {"offset": field.offset, "type": self._format_kind(field.index)} + } def _determine_size(self, field): output = None if isinstance(field, str): output = self.base_type_size[field] - elif (field.leaf_type == "LF_STRUCTURE" or field.leaf_type == "LF_ARRAY" or field.leaf_type == "LF_UNION"): + elif ( + field.leaf_type == "LF_STRUCTURE" + or field.leaf_type == "LF_ARRAY" + or field.leaf_type == "LF_UNION" + ): output = field.size elif field.leaf_type == "LF_POINTER": output = self.base_type_size[field.ptr_attr.type] @@ -256,6 +271,7 @@ def _determine_size(self, field): output = self._determine_size(field.index) if output is None: import pdb + pdb.set_trace() raise ValueError(f"Unknown size for field: {field.name}") return output @@ -267,36 +283,37 @@ def _format_kind(self, kind): output = self.lookup_ctype_pointers(kind) except KeyError: try: - output = {'kind': 'base', 'name': self.lookup_ctype(kind)} + output = {"kind": "base", "name": self.lookup_ctype(kind)} except KeyError: - output = {'kind': 'base', 'name': kind} - elif kind.leaf_type == 'LF_MODIFIER': + output = {"kind": "base", "name": kind} + elif kind.leaf_type == "LF_MODIFIER": output = self._format_kind(kind.modified_type) - elif kind.leaf_type == 'LF_STRUCTURE': - output = {'kind': 'struct', 'name': kind.name} - elif kind.leaf_type == 'LF_UNION': - output = {'kind': 'union', 'name': kind.name} - elif kind.leaf_type == 'LF_BITFIELD': + elif kind.leaf_type == "LF_STRUCTURE": + output = {"kind": "struct", "name": kind.name} + elif kind.leaf_type == "LF_UNION": + output = {"kind": "union", "name": kind.name} + elif kind.leaf_type == "LF_BITFIELD": output = { - 'kind': 'bitfield', - 'type': self._format_kind(kind.base_type), - 'bit_length': kind.length, - 'bit_position': kind.position + "kind": "bitfield", + "type": self._format_kind(kind.base_type), + "bit_length": kind.length, + "bit_position": kind.position, } - elif kind.leaf_type == 'LF_POINTER': - output = {'kind': 'pointer', 'subtype': self._format_kind(kind.utype)} - elif kind.leaf_type == 'LF_ARRAY': + elif kind.leaf_type == "LF_POINTER": + output = {"kind": "pointer", "subtype": self._format_kind(kind.utype)} + elif kind.leaf_type == "LF_ARRAY": output = { - 'kind': 'array', - 'count': kind.size // self._determine_size(kind.element_type), - 'subtype': self._format_kind(kind.element_type) + "kind": "array", + "count": kind.size // self._determine_size(kind.element_type), + "subtype": self._format_kind(kind.element_type), } - elif kind.leaf_type == 'LF_ENUM': - output = {'kind': 'enum', 'name': kind.name} - elif kind.leaf_type == 'LF_PROCEDURE': - output = {'kind': "function"} + elif kind.leaf_type == "LF_ENUM": + output = {"kind": "enum", "name": kind.name} + elif kind.leaf_type == "LF_PROCEDURE": + output = {"kind": "function"} else: import pdb + pdb.set_trace() return output @@ -306,40 +323,70 @@ def read_basetypes(self) -> Dict: if "64" in self._pdb.STREAM_DBI.machine: ptr_size = 8 - output = {"pointer": {"endian": "little", "kind": "int", "signed": False, "size": ptr_size}} + output = { + "pointer": { + "endian": "little", + "kind": "int", + "signed": False, + "size": ptr_size, + } + } for index in self._seen_ctypes: output[self.ctype[index]] = { "endian": "little", "kind": self.ctype_python_types.get(self.ctype[index], "int"), "signed": False if "_U" in index else True, - "size": self.base_type_size[index] + "size": self.base_type_size[index], } return output -if __name__ == '__main__': - parser = argparse.ArgumentParser(description = "Convertor for PDB files to Volatility 3 Intermediate Symbol Format") - parser.add_argument("-o", "--output", metavar = "OUTPUT", help = "Filename for data output", required = True) - file_group = parser.add_argument_group("file", description = "File-based conversion of PDB to ISF") - file_group.add_argument("-f", "--file", metavar = "FILE", help = "PDB file to translate to ISF") - data_group = parser.add_argument_group("data", description = "Convert based on a GUID and filename pattern") - data_group.add_argument("-p", "--pattern", metavar = "PATTERN", help = "Filename pattern to recover PDB file") - data_group.add_argument("-g", - "--guid", - metavar = "GUID", - help = "GUID + Age string for the required PDB file", - default = None) - data_group.add_argument("-k", - "--keep", - action = "store_true", - default = False, - help = "Keep the downloaded PDB file") +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Convertor for PDB files to Volatility 3 Intermediate Symbol Format" + ) + parser.add_argument( + "-o", + "--output", + metavar="OUTPUT", + help="Filename for data output", + required=True, + ) + file_group = parser.add_argument_group( + "file", description="File-based conversion of PDB to ISF" + ) + file_group.add_argument( + "-f", "--file", metavar="FILE", help="PDB file to translate to ISF" + ) + data_group = parser.add_argument_group( + "data", description="Convert based on a GUID and filename pattern" + ) + data_group.add_argument( + "-p", + "--pattern", + metavar="PATTERN", + help="Filename pattern to recover PDB file", + ) + data_group.add_argument( + "-g", + "--guid", + metavar="GUID", + help="GUID + Age string for the required PDB file", + default=None, + ) + data_group.add_argument( + "-k", + "--keep", + action="store_true", + default=False, + help="Keep the downloaded PDB file", + ) args = parser.parse_args() delfile = False filename = None if args.guid is not None and args.pattern is not None: - filename = PDBRetreiver().retreive_pdb(guid = args.guid, file_name = args.pattern) + filename = PDBRetreiver().retreive_pdb(guid=args.guid, file_name=args.pattern) delfile = True elif args.file: filename = args.file @@ -352,7 +399,7 @@ def read_basetypes(self) -> Dict: convertor = PDBConvertor(filename) with open(args.output, "w") as f: - json.dump(convertor.read_pdb(), f, indent = 2, sort_keys = True) + json.dump(convertor.read_pdb(), f, indent=2, sort_keys=True) if args.keep: print(f"Temporary PDB file: {filename}") diff --git a/development/schema_validate.py b/development/schema_validate.py index 0908e934f3..cf9565d687 100644 --- a/development/schema_validate.py +++ b/development/schema_validate.py @@ -1,34 +1,33 @@ import argparse import json +import logging import os import sys # TODO: Rather nasty hack, when volatility's actually installed this would be unnecessary sys.path += ".." -import logging - console = logging.StreamHandler() console.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') +formatter = logging.Formatter("%(levelname)-8s %(name)-12s: %(message)s") console.setFormatter(formatter) logger = logging.getLogger("") logger.addHandler(console) logger.setLevel(logging.DEBUG) -from volatility3 import schemas +from volatility3 import schemas # noqa: E402 -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser("Validates ") - parser.add_argument("-s", "--schema", dest = "schema", default = None) - parser.add_argument("filenames", metavar = "FILE", nargs = '+') + parser.add_argument("-s", "--schema", dest="schema", default=None) + parser.add_argument("filenames", metavar="FILE", nargs="+") args = parser.parse_args() schema = None if args.schema: - with open(os.path.abspath(args.schema), 'r') as s: + with open(os.path.abspath(args.schema)) as s: schema = json.load(s) failures = [] @@ -36,7 +35,7 @@ try: if os.path.exists(filename): print(f"[?] Validating file: {filename}") - with open(filename, 'r') as t: + with open(filename) as t: test = json.load(t) if args.schema: diff --git a/development/stock-linux-json.py b/development/stock-linux-json.py index 877f78e1cc..967cc7e183 100644 --- a/development/stock-linux-json.py +++ b/development/stock-linux-json.py @@ -9,7 +9,7 @@ import rpmfile from debian import debfile -DWARF2JSON = './dwarf2json' +DWARF2JSON = "./dwarf2json" class Downloader: @@ -17,7 +17,7 @@ class Downloader: def __init__(self, url_lists: List[List[str]]) -> None: self.url_lists = url_lists - def download_lists(self, keep = False): + def download_lists(self, keep=False): for url_list in self.url_lists: print("Downloading files...") files_for_processing = self.download_list(url_list) @@ -35,43 +35,45 @@ def download_list(self, urls: List[str]) -> Dict[str, str]: with tempfile.NamedTemporaryFile() as archivedata: archivedata.write(data.content) archivedata.seek(0) - if url.endswith('.rpm'): + if url.endswith(".rpm"): processed_files[url] = self.process_rpm(archivedata) - elif url.endswith('.deb'): + elif url.endswith(".deb"): processed_files[url] = self.process_deb(archivedata) return processed_files def process_rpm(self, archivedata) -> Optional[str]: - rpm = rpmfile.RPMFile(fileobj = archivedata) + rpm = rpmfile.RPMFile(fileobj=archivedata) member = None extracted = None for member in rpm.getmembers(): - if 'vmlinux' in member.name or 'System.map' in member.name: + if "vmlinux" in member.name or "System.map" in member.name: print(f" - Extracting {member.name}") extracted = rpm.extractfile(member) break if not member or not extracted: return None - with tempfile.NamedTemporaryFile(delete = False, - prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: + with tempfile.NamedTemporaryFile( + delete=False, prefix="vmlinux" if "vmlinux" in member.name else "System.map" + ) as output: print(f" - Writing to {output.name}") output.write(extracted.read()) return output.name def process_deb(self, archivedata) -> Optional[str]: - deb = debfile.DebFile(fileobj = archivedata) + deb = debfile.DebFile(fileobj=archivedata) member = None extracted = None for member in deb.data.tgz().getmembers(): - if member.name.endswith('vmlinux') or 'System.map' in member.name: + if member.name.endswith("vmlinux") or "System.map" in member.name: print(f" - Extracting {member.name}") extracted = deb.data.get_file(member.name) break if not member or not extracted: return None - with tempfile.NamedTemporaryFile(delete = False, - prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: + with tempfile.NamedTemporaryFile( + delete=False, prefix="vmlinux" if "vmlinux" in member.name else "System.map" + ) as output: print(f" - Writing to {output.name}") output.write(extracted.read()) return output.name @@ -83,43 +85,55 @@ def process_files(self, named_files: Dict[str, str]): if named_files[i] is None: print(f"FAILURE: None encountered for {i}") return - args = [DWARF2JSON, 'linux'] - output_filename = 'unknown-kernel.json' + args = [DWARF2JSON, "linux"] + output_filename = "unknown-kernel.json" for named_file in named_files: - prefix = '--system-map' - if 'System' not in named_files[named_file]: - prefix = '--elf' - output_filename = './' + '-'.join((named_file.split('/')[-1]).split('-')[2:])[:-4] + '.json.xz' + prefix = "--system-map" + if "System" not in named_files[named_file]: + prefix = "--elf" + output_filename = ( + "./" + + "-".join((named_file.split("/")[-1]).split("-")[2:])[:-4] + + ".json.xz" + ) args += [prefix, named_files[named_file]] print(f" - Running {args}") - proc = subprocess.run(args, capture_output = True) + proc = subprocess.run(args, capture_output=True) print(f" - Writing to {output_filename}") - with lzma.open(output_filename, 'w') as f: + with lzma.open(output_filename, "w") as f: f.write(proc.stdout) -if __name__ == '__main__': - parser = argparse.ArgumentParser(description = "Takes a list of URLs for Centos and downloads them") - parser.add_argument("-f", - "--file", - dest = 'filename', - metavar = "FILENAME", - help = "Filename to be read", - required = True) - parser.add_argument("-d", - "--dwarf2json", - dest = 'dwarfpath', - metavar = "PATH", - default = DWARF2JSON, - help = "Path to the dwarf2json binary", - required = True) - parser.add_argument("-k", - "--keep", - dest = 'keep', - action = 'store_true', - help = 'Keep extracted temporary files after completion', - default = False) +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Takes a list of URLs for Centos and downloads them" + ) + parser.add_argument( + "-f", + "--file", + dest="filename", + metavar="FILENAME", + help="Filename to be read", + required=True, + ) + parser.add_argument( + "-d", + "--dwarf2json", + dest="dwarfpath", + metavar="PATH", + default=DWARF2JSON, + help="Path to the dwarf2json binary", + required=True, + ) + parser.add_argument( + "-k", + "--keep", + dest="keep", + action="store_true", + help="Keep extracted temporary files after completion", + default=False, + ) args = parser.parse_args() DWARF2JSON = args.dwarfpath @@ -132,4 +146,4 @@ def process_files(self, named_files: Dict[str, str]): urls += [[lines[2 * i].strip(), lines[(2 * i) + 1].strip()]] d = Downloader(urls) - d.download_lists(keep = args.keep) + d.download_lists(keep=args.keep) diff --git a/doc/requirements.txt b/doc/requirements.txt deleted file mode 100644 index d3ba512245..0000000000 --- a/doc/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -# These packages are required for building the documentation. -sphinx>=4.0.0,<7 -sphinx_autodoc_typehints>=1.4.0 -sphinx-rtd-theme>=0.4.3 - -yara-python -yara-x -pycryptodome -pefile diff --git a/doc/source/basics.rst b/doc/source/basics.rst index 1b8e647808..278ef4d730 100644 --- a/doc/source/basics.rst +++ b/doc/source/basics.rst @@ -14,7 +14,7 @@ Memory layers ------------- A memory layer is a body of data that can be accessed by requesting data at a specific address. At its lowest level -this data is stored on a phyiscal medium (RAM) and very early computers addresses locations in memory directly. However, +this data is stored on a phyiscal medium (RAM) and very early computers addressed locations in memory directly. However, as the size of memory increased and it became more difficult to manage memory most architectures moved to a "paged" model of memory, where the available memory is cut into specific fixed-sized pages. To help further, programs can ask for any address and the processor will look up their (virtual) address in a map, to find out where the (physical) address that it lives at is, @@ -25,8 +25,8 @@ address `9`). The automagic that runs at the start of every volatility session a kernel virtual layer, which allows for kernel addresses to be looked up and the correct data returned. There can, however, be several maps, and in general there is a different map for each process (although a portion of the operating system's memory is usually mapped to the same location across all processes). The maps may take the same address but point to a different part of -physical memory. It also means that two processes could theoretically share memory, but having an virtual address mapped to the -same physical address as another process. See the worked example below for more information. +physical memory. It also means that two processes could theoretically share memory, both having a virtual address mapped to the +same physical address. See the worked example below for more information. To translate an address on a layer, call :py:meth:`layer.mapping(offset, length, ignore_errors) ` and it will return a list of chunks without overlap, in order, for the requested range. If a portion cannot be mapped, an exception will be thrown unless `ignore_errors` is true. Each @@ -61,7 +61,7 @@ mean they each see something different: 4 -> 2 16 - Free In this example, part of the operating system is visible across all processes (although not all processes can write to the memory, there -is a permissions model for intel addressing which is not discussed further here).) +is a permissions model for Intel addressing which is not discussed further here). In Volatility 3 mappings are represented by a directed graph of layers, whose end nodes are :py:class:`DataLayers ` and whose internal nodes are :py:class:`TranslationLayers `. @@ -69,13 +69,13 @@ In this way, a raw memory image in the LiME file format and a page file can be c memory layer. When requesting addresses from the Intel layer, it will use the Intel memory mapping algorithm, along with the address of the directory table base or page table map, to translate that address into a physical address, which will then either be directed towards the swap layer or the LiME layer. Should it -be directed towards the LiME layer, the LiME file format algorithm will be translate the new address to determine where +be directed towards the LiME layer, the LiME file format algorithm will translate the new address to determine where within the file the data is stored. When the :py:meth:`layer.read() ` method is called, the translation is done automatically and the correct data gathered and combined. .. note:: Volatility 2 had a similar concept, called address spaces, but these could only stack linearly one on top of another. -The list of layers supported by volatility can be determined by running the `frameworkinfo` plugin. +The list of layers supported by Volatility can be determined by running the `frameworkinfo` plugin. Templates and Objects --------------------- @@ -167,8 +167,7 @@ There are certain setup tasks that establish the context in a way favorable to a several tasks that are repetitive and also easy to get wrong. These are called :py:class:`Automagic `, since they do things like magically taking a raw memory image and automatically providing the plugin with an appropriate Intel translation layer and an -accurate symbol table without either the plugin or the calling program having to specify all the necessary details. +accurate symbol table without either the plugin or the calling program having to specify all the necessary details. Automagics are a core component which consumers of the library can call or not at their discretion. .. note:: Volatility 2 used to do this as well, but it wasn't a particularly modular mechanism, and was used only for stacking address spaces (rather than identifying profiles), and it couldn't really be disabled/configured easily. - Automagics in Volatility 3 are a core component which consumers of the library can call or not at their discretion. diff --git a/doc/source/conf.py b/doc/source/conf.py index cabfdc3279..7a9a72891d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -19,6 +19,8 @@ import sphinx.ext.apidoc +from importlib.util import find_spec + def setup(app): volatility_directory = os.path.abspath( @@ -124,7 +126,7 @@ def setup(app): # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath("../..")) -from volatility3.framework import constants +from volatility3.framework import constants # noqa: E402 # -- General configuration ------------------------------------------------ @@ -147,13 +149,9 @@ def setup(app): autosectionlabel_prefix_document = True -try: - import sphinx_autodoc_typehints - +if find_spec("sphinx_autodoc_typehints") is not None: extensions.append("sphinx_autodoc_typehints") -except ImportError: - # If the autodoc typehints extension isn't available, carry on regardless - pass +# If the autodoc typehints extension isn't available, carry on regardless # Add any paths that contain templates here, relative to this directory. # templates_path = ['tools/templates'] diff --git a/doc/source/getting-started-linux-tutorial.rst b/doc/source/getting-started-linux-tutorial.rst index d4b40d0532..a1aad235db 100644 --- a/doc/source/getting-started-linux-tutorial.rst +++ b/doc/source/getting-started-linux-tutorial.rst @@ -27,7 +27,7 @@ To create a symbol table please refer to :ref:`symbol-tables:Mac or Linux symbol Listing plugins --------------- -The following is a sample of the linux plugins available for volatility3, it is not complete and more more plugins may +The following is a sample of the linux plugins available for volatility3, it is not complete and more plugins may be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. For plugin requests, please create an issue with a description of the requested plugin. @@ -40,7 +40,7 @@ For plugin requests, please create an issue with a description of the requested linux.check_creds.Check_creds linux.check_idt.Check_idt -.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of linux plugins. +.. note:: Here the the command is piped to grep and head to provide the start of the list of linux plugins. Using plugins @@ -80,9 +80,9 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's kernel version and the distribution version. Now using the above banner we can search for the needed ISF file from the ISF server. -If ISF file cannot be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. +If an ISF file cannot be found then, follow the instructions on :ref:`getting-started-linux-tutorial:Procedure to create symbol tables for linux`. After that, place the ISF file under the ``volatility3/symbols/linux`` directory. -.. tip:: Use the banner text which is most repeated to search from ISF Server. +.. tip:: Use the banner text which is most repeated to search on the ISF Server. linux.pslist ~~~~~~~~~~~~ @@ -157,7 +157,7 @@ linux.pstree ***** 1548 1266 gsd-keyboard ***** 1550 1266 gsd-media-keys -``linux.pstree`` helps us to display the parent child relationships between processes. +``linux.pstree`` helps us to display the parent-child relationships between processes. linux.bash ~~~~~~~~~~ diff --git a/doc/source/getting-started-mac-tutorial.rst b/doc/source/getting-started-mac-tutorial.rst index 42e58c0d58..61af7089b0 100644 --- a/doc/source/getting-started-mac-tutorial.rst +++ b/doc/source/getting-started-mac-tutorial.rst @@ -37,7 +37,7 @@ For plugin requests, please create an issue with a description of the requested mac.check_sysctl.Check_sysctl mac.check_trap_table.Check_trap_table -.. note:: Here the the command is piped to grep and head in-order to provide the start of the list of macOS plugins. +.. note:: Here the the command is piped to grep and head to provide the start of the list of macOS plugins. Using plugins @@ -78,7 +78,7 @@ Thanks go to `stuxnet `_ for providing this memo The above command helps us to find the memory dump's Darwin kernel version. Now using the above banner we can search for the needed ISF file. -If ISF file cannot be found then, follow the instructions on :ref:`getting-started-mac-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols`` directory. +If an ISF file cannot be found then, follow the instructions on :ref:`getting-started-mac-tutorial:Procedure to create symbol tables for macOS`. After that, place the ISF file under the ``volatility3/symbols`` directory. mac.pslist ~~~~~~~~~~ @@ -125,7 +125,7 @@ mac.pstree 337 1 system_installd * 455 337 update_dyld_shar -``mac.pstree`` helps us to display the parent child relationships between processes. +``mac.pstree`` helps us to display the parent-child relationships between processes. mac.ifconfig ~~~~~~~~~~~~ @@ -150,4 +150,4 @@ mac.ifconfig utun0 False utun0 fe80:5::2a95:bb15:87e3:977c False -we can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. +We can use the ``mac.ifconfig`` plugin to get information about the configuration of the network interfaces of the host under investigation. diff --git a/doc/source/getting-started-windows-tutorial.rst b/doc/source/getting-started-windows-tutorial.rst index c89b065f51..979cf1d966 100644 --- a/doc/source/getting-started-windows-tutorial.rst +++ b/doc/source/getting-started-windows-tutorial.rst @@ -15,19 +15,19 @@ Memory can be acquired using a number of tools, below are some examples but othe Listing Plugins --------------- -The following is a sample of the windows plugins available for volatility3, it is not complete and more more plugins may +The following is a sample of the windows plugins available for volatility3, it is not complete and more plugins may be added. For a complete reference, please see the volatility 3 :doc:`list of plugins `. For plugin requests, please create an issue with a description of the requested plugin. .. code-block:: shell-session - $ python3 vol.py --help | grep windows | head -n 5 + $ python3 vol.py --help | grep windows | head -n 4 windows.bigpools.BigPools windows.cmdline.CmdLine windows.crashinfo.Crashinfo windows.dlllist.DllList -.. note:: Here the the command is piped to grep and head in-order to provide the start of a list of the available windows plugins. +.. note:: Here the the command is piped to grep and head to provide the start of a list of the available windows plugins. Using plugins ------------- @@ -95,9 +95,9 @@ windows.pstree ** 616 504 svchost.exe 0xfa8002b86ab0 13 314 0 False 2022-02-07 16:32:16.000000 N/A ** 624 504 svchost.exe 0xfa8002410630 10 350 0 False 2022-02-07 16:30:14.000000 N/A -``windows.pstree`` helps to display the parent child relationships between processes. +``windows.pstree`` helps to display the parent-child relationships between processes. -.. note:: Here the the command is piped to head in-order to provide smaller output, here listing only the first 20. +.. note:: Here the the command is piped to head to provide smaller output, here listing only the first 20. windows.hashdump ~~~~~~~~~~~~~~~~ @@ -116,9 +116,3 @@ windows.hashdump Dennis 1003 aad3b435b51404eeaad3b435b51404ee cf96684bbc7877920adaa9663698bf54 ``windows.hashdump`` helps to list the hashes of the users in the system. - - - - - - diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 66dabfafea..d3bc9613aa 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -23,7 +23,7 @@ Alignment .. _Array: Array - This represents a list of items, which can be access by an index, which is zero-based (meaning the first + This represents a list of items, which can be accessed by an index, which is zero-based (meaning the first element has index 0). Items in arrays are almost always the same size (it is not a generic list, as in python) even if they are :ref:`pointers` to different sized objects. @@ -43,7 +43,14 @@ Dereference .. _Domain: Domain - This the grouping for input values for a mapping or mathematical function. + The set of input values for a mapping or mathematical function. + +I +- +.. _Intermediate Symbol File (ISF): + +Intermediate Symbol File (ISF) + They contain kernel structures and specific offsets formatted as JSON. For macOS and Linux analysis, the kernel needs to be added as an ISF file to the volatility 3 symbols directory. For Windows, the required ISF file can often be generated from PDB files automatically downloaded from Microsoft servers, and therefore does not require manual intervention. M - @@ -54,9 +61,7 @@ Map, mapping of the :ref:`Range`). Mappings can be seen as a mathematical function, and therefore volatility 3 attempts to use mathematical functional notation where possible. Within volatility a mapping is most often used to refer to the function for translating addresses from a higher layer (domain) to a lower layer (range). - For further information, please see - `Function (mathematics) in wikipedia https://en.wikipedia.org/wiki/Function_(mathematics)` - + For further information, please see `Function (mathematics) in Wikipedia_`. .. _Member: @@ -69,7 +74,7 @@ O .. _Object: Object - This has a specific meaning within computer programming (as in Object Oriented Programming), but within the world + This has a specific meaning within computer programming (as in object-oriented programming), but within the world of Volatility it is used to refer to a type that has been associated with a chunk of data, or a specific instance of a type. See also :ref:`Type`. @@ -116,6 +121,11 @@ Page Table possible to use them as a way to map a particular address within a (potentially larger, but sparsely populated) virtual space to a concrete (and usually contiguous) physical space, through the process of :ref:`mapping`. +.. _Plugin: + +Plugin + Plugins are the "functions" of the volatility framework. They carry out algorithms on data stored in layers using objects constructed from symbols. Broadly, plugins take in a number of TranslationLayers (the data, which is a representation of part of an image, in a specified type described by templates) and outputs a TreeGrid. + .. _Pointer: Pointer @@ -145,9 +155,9 @@ Struct, Structure Symbol This is used in many different contexts, as a short term for many things. Within Volatility, a symbol is a - construct that usually encompasses a specific type :ref:`type` at a specific :ref:`offset`, + construct that usually encompasses a specific :ref:`type` at a specific :ref:`offset`, representing a particular instance of that type within the memory of a compiled and running program. An example - would be the location in memory of a list of active tcp endpoints maintained by the networking stack + would be the location in memory of a list of active TCP endpoints maintained by the networking stack within an operating system. T diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index b7c26e0463..59c1febcc1 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -9,7 +9,7 @@ How Volatility finds symbol tables All files are stored as JSON data, they can be in pure JSON files as ``.json``, or compressed as ``.json.gz`` or ``.json.xz``. Volatility will automatically decompress them on use. It will also cache their contents (compressed) when used, located -under the user's home directory, in :file:`.cache/volatility3`, along with other useful data. The cache directory currently +under the user's home directory, in :file:`.cache/volatility3` or when `XDG_CACHE_HOME` is set in :file:`${XDG_CACHE_HOME}/volatility3`, along with other useful data. The cache directory currently cannot be altered. Symbol table JSON files live, by default, under the :file:`volatility3/symbols` directory. The symbols directory is @@ -25,9 +25,9 @@ as long as the symbol files stay in the same location. Windows symbol tables --------------------- -For Windows systems, Volatility accepts a string made up of the GUID and Age of the required PDB file. It then +For Windows systems, Volatility accepts a string made up of the GUID and age of the required PDB file. It then searches all files under the configured symbol directories under the windows subdirectory. Any that contain metadata -which matches the pdb name and GUID/age (or any compressed variant) will be used. If such a symbol table cannot be found, then +which matches the PDB name and GUID/age (or any compressed variant) will be used. If such a symbol table cannot be found, then the associated PDB file will be downloaded from Microsoft's Symbol Server and converted into the appropriate JSON format, and will be saved in the correct location. @@ -54,8 +54,8 @@ most Volatility plugins. Note that in most linux distributions, the standard ke and the kernel with debugging information is stored in a package that must be acquired separately. A generic table isn't guaranteed to produce accurate results, and would reduce the number of structures -that all plugins could rely on. As such, and because linux kernels with different configurations can produce different structures, -volatility 3 requires that the banners in the JSON file match the banners found in the image *exactly*, not just the version +that all plugins could rely on. As such, and because Linux kernels with different configurations can produce different structures, +Volatility 3 requires that the banners in the JSON file match the banners found in the image *exactly*, not just the version number. This can include elements such as the compilation time and even the version of gcc used for the compilation. The exact match is required to ensure that the results volatility returns are accurate, therefore there is no simple means provided to get the wrong JSON ISF file to easily match. @@ -63,8 +63,8 @@ provided to get the wrong JSON ISF file to easily match. To determine the string for a particular memory image, use the `banners` plugin. Once the specific banner is known, try to locate that exact kernel debugging package for the operating system. Unfortunately each distribution provides its debugging packages under different package names and there are so many that the distribution may not keep all old -versions of the debugging symbols, and therefore **it may not be possible to find the right symbols to analyze a linux -memory image with volatility**. With Macs there are far fewer kernels and only one distribution, making it easier to +versions of the debugging symbols, and therefore **it may not be possible to find the right symbols to analyze a Linux +memory image with Volatility**. With Macs there are far fewer kernels and only one distribution, making it easier to ensure that the right symbols can be found. Once a kernel with debugging symbols/appropriate DWARF file has been located, `dwarf2json `_ will convert it into an @@ -75,7 +75,7 @@ symbol offsets within the DWARF data, which dwarf2json can extract into the JSON The banners available for volatility to use can be found using the `isfinfo` plugin, but this will potentially take a long time to run depending on the number of JSON files available. This will list all the JSON (ISF) files that -volatility3 is aware of, and for linux/mac systems what banner string they search for. For volatility to use the JSON +Volatility 3 is aware of, and for linux/mac systems what banner string they search for. For volatility to use the JSON file, the banners must match exactly (down to the compilation date). .. note:: diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index 4acf35f987..144cae644e 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -3,7 +3,7 @@ Using Volatility 3 as a Library This portion of the documentation discusses how to access the Volatility 3 framework from an external application. -The general process of using volatility as a library is to as follows: +The general process of using volatility as a library is as follows: 1. :ref:`create_context` 2. (Optional) :ref:`available_plugins` @@ -21,7 +21,7 @@ Creating a context First we make sure the volatility framework works the way we expect it (and is the version we expect). The versioning used is semantic versioning, meaning any version with the same major number and a higher or equal minor number will satisfy the requirement. An example is below since the CLI doesn't need any of the features -from versions 1.1 or 1.2: +from version 1.1 or later: :: @@ -86,7 +86,7 @@ List requirements are a list of simple types (integers, booleans, floats and str options, multiple requirements needs all their subrequirements fulfilled and the other types require the names of valid translation layers or symbol tables within the context, respectively. Luckily, each of these requirements can tell you whether they've been fulfilled or not later in the process. For now, they can be used to ask the user to -fill in any parameters they made need to. Some requirements are optional, others are not. +fill in any parameters they may need to. Some requirements are optional, others are not. The plugin is essentially a multiple requirement. It should also be noted that automagic classes can have requirements (as can translation layers). @@ -100,7 +100,7 @@ Once you know what requirements the plugin will need, you can populate them with The configuration is essentially a hierarchical tree of values, much like the windows registry. Each plugin is instantiated at a particular branch within the hierarchy and will look for its configuration options under that hierarchy (if it holds any configurable items, it will likely instantiate those at a point -underneaths its own branch). To set the hierarchy, you'll need to know where the configurables will be constructed. +underneath its own branch). To set the hierarchy, you'll need to know where the configurables will be constructed. For this example, we'll assume plugins' base_config_path is set as `plugins`, and that automagics are configured under the `automagic` tree. We'll see later how to ensure this matches up with the plugins and automagic when they're @@ -139,7 +139,7 @@ A suitable list of automagics for a particular plugin (based on operating system This will take the plugin module, extract the operating system (first level of the hierarchy) and then return just the automagics which apply to the operating system. Each automagic can exclude itself from being used for specific -operating systems, so that an automagic designed for linux is not used for windows or mac plugins. +operating systems, such that an automagic designed for linux is not used for windows or mac plugins. These automagics can then be run by providing the list, the context, the plugin to be run, the hierarchy name that the plugin will be constructed on ('plugins' by default) and a progress_callback. This is a callable which takes @@ -157,8 +157,8 @@ Any exceptions that occur during the execution of the automagic will be returned Run the plugin -------------- -Firstly, we should check whether the plugin will be able to run (ie, whether the configuration options it needs -have been successfully set). We do this as follow (where plugin_config_path is the base_config_path (which defaults +Firstly, we should check whether the plugin will be able to run (i.e., whether the configuration options it needs +have been successfully set). We do this as follows, where plugin_config_path is the base_config_path (which defaults to `plugins` and then the name of the class itself): :: @@ -166,7 +166,7 @@ to `plugins` and then the name of the class itself): unsatisfied = plugin.unsatisfied(context, plugin_config_path) If unsatisfied is an empty list, then the plugin has been given everything it requires. If not, it will be a -Dictionary of the hierarchy paths and their associated requirements that weren't satisfied. +dict of the hierarchy paths and their associated requirements that weren't satisfied. The plugin can then be instantiated with the context (containing the plugin's configuration) and the path that the plugin can find its configuration at. This configuration path only needs to be a unique value to identify where the diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index 7b91e815d6..43ca33f04c 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -58,7 +58,7 @@ Options EXTEND. Extensions must be of the form **configuration.item.name=value** -p PLUGIN_DIRS, --plugin-dirs PLUGIN_DIRS - Specified a semi-colon separated list of paths that contain directories + Specified as a semi-colon separated list of paths that contain directories where plugins may be found. These paths are searched before the default paths when loading python files for plugins. This can therefore be used to override built-in plugins. NOTE: All python code within this directory @@ -67,12 +67,12 @@ Options -s SYMBOL_DIRS, --symbol-dirs SYMBOL_DIRS SYMBOL_DIRS is a semi-colon separated list of paths that contain symbol files or symbol zip packs. Symbols must be within a particular directory - structure if they depending on the operating system of the symbols, + structure if they depend on the operating system of the symbols, whilst symbol packs must be in the root of the directory and named after - the after the operating system to which they apply. + the operating system to which they apply. -v, --verbose - A flag which can be used multiple times, each time increasing the level of + A flag which can be used multiple times (up to six, -vvvvvv), each time increasing the level of detail in the logs produced. -l LOG, --log LOG @@ -87,7 +87,7 @@ Options -q, --quiet When present, this flag mutes the progress feedback for operations. This can be beneficial when piping the output directly to a file or another - tool. This also removes the + tool. -r RENDERER, --renderer RENDERER Specifies the output format in which to display results. The default is @@ -120,9 +120,7 @@ Options Change the default path used to store the cache. --offline - Do not search online for additional JSON files. - Run offline mode (defaults to false) and for - remote windows symbol tables, linux/mac banner repositories. + Run offline mode (defaults to false). Do not search online for additional JSON files, remote windows symbol tables, nor linux/mac banner repositories. --single-location SINGLE_LOCATION This specifies a URL which will be downloaded if necessary, and built @@ -152,7 +150,7 @@ but can be overridden by creating a JSON file (`%APPDATA%/volatility3/vol.json` systems, or `~/.config/volatility3/vol.json` or `volshell.json` for all others). The format of this file is a JSON dictionary, containing the options above and their value. -It should be noted that the ordering is (`<` means is overridden by): +It should be noted that the ordering is (`x < y` means `x` is overridden by `y`): `in-built default value < config file value < command line parameter` diff --git a/doc/source/vol2to3.rst b/doc/source/vol2to3.rst index e768df0c2b..9b5a739f8f 100644 --- a/doc/source/vol2to3.rst +++ b/doc/source/vol2to3.rst @@ -27,7 +27,7 @@ The object model has changed as well, objects now inherit directly from their Py object is actually a Python integer (and has all the associated methods, and can be used wherever a normal int could). In Volatility 2, a complex proxy object was constructed which tried to emulate all the methods of the host object, but ultimately it was a different type and could not be used in the same places (critically, it could make the ordering of -operations important, since a + b might not work, but b + a might work fine). +operations important, since x + y might not work, but y + x might work fine). Volatility 3 has also had significant speed improvements, where Volatility 2 was designed to allow access to live memory images and situations in which the underlying data could change during the run of the plugin, in Volatility 3 the data @@ -36,11 +36,11 @@ This was because live memory analysis was barely ever used, and this feature cou re-read many times over for no benefit (particularly since each re-read could result in many additional image reads from following page table translations). -Finally, in order to provide Volatility specific information without impact on the ability for structures to have members +Further, in order to provide Volatility specific information without impact on the ability for structures to have members with arbitrary names, all the metadata about the object (such as its layer or offset) have been moved to a read-only :py:meth:`~volatility3.framework.interfaces.objects.ObjectInterface.vol` dictionary. -Further the distinction between a :py:class:`~volatility3.framework.interfaces.objects.Template` (the thing that +Finally, the distinction between a :py:class:`~volatility3.framework.interfaces.objects.Template` (the thing that constructs an object) and the :py:class:`Object ` itself has been made more explicit. In Volatility 2, some information (such as size) could only be determined from a constructed object, leading to instantiating a template on an empty buffer, just to determine the size. In Volatility 3, templates contain @@ -56,15 +56,14 @@ Volatility 2 were strictly limited to a stack, one on top of one other. In Vola Automagic --------- -In Volatility 2, we often tried to make this simpler for both users and developers. This resulted in something was -referred to as automagic, in that it was magic that happened automatically. We've now codified that more, so that the +In Volatility 2, we often tried to make this simpler for both users and developers. This resulted in something referred to as automagic, in that it was magic that happened automatically. We've now codified that more, so that the automagic processes are clearly defined and can be enabled or disabled as necessary for any particular run. We also included a stacker automagic to emulate the most common feature of Volatility 2, automatically stacking address spaces (now translation layers) on top of each other. -By default the automagic chosen to be run are determined based on the plugin requested, so that linux plugins get linux -specific automagic and windows plugins get windows specific automagic. This should reduce unnecessarily searching for -linux kernels in a windows image, for example. At the moment this is not user configurableS. +By default the automagic chosen to be run are determined based on the plugin requested, so that Linux plugins get Linux +specific automagic and Windows plugins get Windows specific automagic. This should reduce unnecessarily searching for +Linux kernels in a Windows image, for example. At the moment this is not user configurable. Searching and Scanning ---------------------- diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst index 5a4b21adeb..47ea2e905a 100644 --- a/doc/source/volshell.rst +++ b/doc/source/volshell.rst @@ -36,7 +36,7 @@ operating system mode for volshell, and the current layer available for use. (primary) >>> -Volshell itself in essentially a plugin, but an interactive one. As such, most values are accessed through `self` +Volshell itself is essentially a plugin, but an interactive one. As such, most values are accessed through `self` although there is also a `context` object whenever a context must be provided. The prompt for the tool will indicate the name of the current layer (which can be accessed as `self.current_layer` @@ -92,7 +92,7 @@ It can also be provided with an object and will interpret the data for each in t 0x2e8 : UniqueProcessId symbol_table_name1!pointer 4 ... -These values can be accessed directory as attributes +These values can be accessed directly as attributes :: @@ -144,12 +144,12 @@ We can provide arguments via the `dpo` method call: 356 4 smss.exe 0x8c0bccf8d040 3 - N/A False 2021-03-13 17:25:33.000000 N/A Disabled ... -Here's we've provided the kernel name that was requested by the volshell plugin itself (the generic volshell does not +Here we've provided the kernel name that was requested by the volshell plugin itself (the generic volshell does not load a kernel module, and instead only has a TranslationLayerRequirement). A different module could be created and provided instead. The context used by the `dpo` method is always `context`. -Instead of print the results directly to screen, they can be gathered into a TreeGrid objects for direct access by +Instead of printing the results directly to screen, they can be gathered into a TreeGrid objects for direct access by using the `generate_treegrid` or `gt` command. :: @@ -180,15 +180,68 @@ used: layer = cc(mynewlayer.MyNewLayer, on_top_of = 'primary', other_parameter = 'important') with open('output.dmp', 'wb') as fp: - for i in range(0, 1073741824, 0x1000): + for i in range(0, 0x4000000, 0x1000): data = layer.read(i, 0x1000, pad = True) fp.write(data) As this demonstrates, all of the python is accessible, as are the volshell built in functions (such as `cc` which creates a constructable, like a layer or a symbol table). +User Convenience +---------------- + +There are functions available that make often-done tasks easier, and generally provide a shell-like experience. These can be listed using `help()` which, as already mentioned, is advertised when volshell starts. + Loading files -------------- +^^^^^^^^^^^^^ Files can be loaded as physical layers using the `load_file` or `lf` command, which takes a filename or a URI. This will be added to `context.layers` and can be accessed by the name returned by `lf`. + +Regex +^^^^^ + +It is easy to scan for some bytes or a pattern using `regex_scan` or `rx`. + +:: + + (layer_name) >>> rx(rb"(Linux version|Darwin Kernel Version) [0-9]+\.[0-9]+\.[0-9]+") + 0x880001400070 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0x880001400080 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0x880001400090 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0x8800014000a0 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0x8800014000b0 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0x8800014000c0 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0x8800014000d0 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0x8800014000e0 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0x880001769027 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0x880001769037 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0x880001769047 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0x880001769057 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0x880001769067 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0x880001769077 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0x880001769087 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0x880001769097 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0xffff81400070 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0xffff81400080 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0xffff81400090 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0xffff814000a0 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0xffff814000b0 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0xffff814000c0 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0xffff814000d0 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0xffff814000e0 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + + 0xffff81769027 4c 69 6e 75 78 20 76 65 72 73 69 6f 6e 20 33 2e Linux.version.3. + 0xffff81769037 32 2e 30 2d 34 2d 61 6d 64 36 34 20 28 64 65 62 2.0-4-amd64.(deb + 0xffff81769047 69 61 6e 2d 6b 65 72 6e 65 6c 40 6c 69 73 74 73 ian-kernel@lists + 0xffff81769057 2e 64 65 62 69 61 6e 2e 6f 72 67 29 20 28 67 63 .debian.org).(gc + 0xffff81769067 63 20 76 65 72 73 69 6f 6e 20 34 2e 36 2e 33 20 c.version.4.6.3. + 0xffff81769077 28 44 65 62 69 61 6e 20 34 2e 36 2e 33 2d 31 34 (Debian.4.6.3-14 + 0xffff81769087 29 20 29 20 23 31 20 53 4d 50 20 44 65 62 69 61 ).).#1.SMP.Debia + 0xffff81769097 6e 20 33 2e 32 2e 35 37 2d 33 2b 64 65 62 37 75 n.3.2.57-3+deb7u + +An optional size can be given for the displayed results as with the other fuctions (db, dw, dd, dq, etc). + +You can, of course, specify a different layer name as well. diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 6fb9f9ed3d..0000000000 --- a/mypy.ini +++ /dev/null @@ -1,4 +0,0 @@ -[mypy] -mypy_path = ./stubs -show_traceback = True -ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index c8b2971e1f..86e3921d23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dev = [ "jsonschema>=4.23.0,<5", "pyinstaller>=6.11.0,<7", "pyinstaller-hooks-contrib>=2024.9", + "types-jsonschema>=4.23.0,<5", ] test = [ @@ -72,9 +73,6 @@ include = ["volatility3*"] mypy_path = "./stubs" show_traceback = true -[tool.mypy.overrides] -ignore_missing_imports = true - [tool.ruff] line-length = 88 target-version = "py38" diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index ae3482290b..0000000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,9 +0,0 @@ --r requirements.txt - -# This can improve error messages regarding improperly configured ISF files, -# but is only recommended for development -jsonschema>=2.3.0 - -# Used to build executable file -pyinstaller>=6.5.0 -pyinstaller-hooks-contrib>=2024.3 \ No newline at end of file diff --git a/requirements-minimal.txt b/requirements-minimal.txt deleted file mode 100644 index c030b332dd..0000000000 --- a/requirements-minimal.txt +++ /dev/null @@ -1,2 +0,0 @@ -# These packages are required for core functionality. -pefile>=2023.2.7 #foo \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 3af0331602..0000000000 --- a/setup.py +++ /dev/null @@ -1,24 +0,0 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 -# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -# - -import setuptools - - -def get_requires(filename): - requirements = [] - with open(filename, "r", encoding="utf-8") as fh: - for line in fh.readlines(): - stripped_line = line.strip() - if stripped_line == "" or stripped_line.startswith(("#", "-r")): - continue - requirements.append(stripped_line) - return requirements - - -setuptools.setup( - extras_require={ - "dev": get_requires("requirements-dev.txt"), - "full": get_requires("requirements.txt"), - }, -) diff --git a/test/README.md b/test/README.md index dcbe289b0c..5891d9508b 100644 --- a/test/README.md +++ b/test/README.md @@ -2,14 +2,12 @@ ## Requirements -The Volatility 3 Testing Framework requires the same version of Python as Volatility3 itself. To install the current set of dependencies that the framework requires, use a command like this: +The Volatility 3 Testing Framework requires the same version of Python as Volatility 3 itself. To install the current set of dependencies that the framework requires, use a command like this: ```shell -pip3 install -r requirements-testing.txt +pip3 install -e .[test] ``` -NOTE: `requirements-testing.txt` can be found in this current `test/` directory. - ## Quick Start: Manual Testing 1. To test Volatility 3 on an image, first download one with a command such as: diff --git a/test/conftest.py b/test/conftest.py index 4ad63065bb..0115fade93 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -35,16 +35,39 @@ def pytest_addoption(parser): def pytest_generate_tests(metafunc): """Parameterize tests based on image names""" - images = metafunc.config.getoption("image") + images = metafunc.config.getoption("image").copy() for image_dir in metafunc.config.getoption("image_dir"): - images = images + [ - os.path.join(image_dir, dir) for dir in os.listdir(image_dir) + images += [ + os.path.join(image_dir, dir_name) for dir_name in os.listdir(image_dir) ] - # tests with "image" parameter are run against images + # tests with "image" parameter are run against image if "image" in metafunc.fixturenames: + filtered_images = [] + ids = [] + for image in images: + image_base = os.path.basename(image) + test_name = metafunc.definition.originalname + if test_name.startswith("test_windows_") and not image_base.startswith( + "win-" + ): + continue + elif test_name.startswith("test_linux_") and not image_base.startswith( + "linux-" + ): + continue + elif test_name.startswith("test_mac_") and not image_base.startswith( + "mac-" + ): + continue + + filtered_images.append(image) + ids.append(image_base) + metafunc.parametrize( - "image", images, ids=[os.path.basename(image) for image in images] + "image", + filtered_images, + ids=ids, ) diff --git a/test/plugins/windows/test_scheduled_tasks.py b/test/plugins/windows/test_scheduled_tasks.py index 8f771b323c..15d7f79a65 100644 --- a/test/plugins/windows/test_scheduled_tasks.py +++ b/test/plugins/windows/test_scheduled_tasks.py @@ -2,9 +2,11 @@ import struct import traceback import unittest + sys.path.insert(0, "../../volatility3") from volatility3.plugins.windows import scheduled_tasks + class TestActionsDecoding(unittest.TestCase): def test_decode_exe_action(self): # fmt: off @@ -84,8 +86,7 @@ def test_decode_exe_action(self): self.assertEqual(actions[0].action_type, scheduled_tasks.ActionType.Exe) except Exception: self.fail( - "ActionDecoder.decode should not raise exception:\n%s" - % traceback.format_exc() + f"ActionDecoder.decode should not raise exception:\n{traceback.format_exc()}" ) diff --git a/test/requirements-testing.txt b/test/requirements-testing.txt deleted file mode 100644 index 51c8f602c2..0000000000 --- a/test/requirements-testing.txt +++ /dev/null @@ -1,11 +0,0 @@ -# These packages are required for core functionality. -pefile>=2017.8.1 #foo - -# The following packages are optional. -# If certain packages are not necessary, place a comment (#) at the start of the line. - -# This is required for the yara plugins -yara-python>=3.8.0 -yara-x>=0.5.0 - -pytest>=7.0.0 diff --git a/test/test_volatility.py b/test/test_volatility.py index 847be88d9b..bb7c9a851f 100644 --- a/test/test_volatility.py +++ b/test/test_volatility.py @@ -14,6 +14,7 @@ import hashlib import ntpath import json +import contextlib # # HELPER FUNCTIONS @@ -38,7 +39,9 @@ def runvol(args, volatility, python): return p.returncode, stdout, stderr -def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]): +def runvol_plugin(plugin, img, volatility, python, pluginargs=None, globalargs=None): + pluginargs = pluginargs or [] + globalargs = globalargs or [] args = ( globalargs + [ @@ -53,15 +56,70 @@ def runvol_plugin(plugin, img, volatility, python, pluginargs=[], globalargs=[]) return runvol(args, volatility, python) +def runvolshell(img, volshell, python, volshellargs=None, globalargs=None): + volshellargs = volshellargs or [] + globalargs = globalargs or [] + args = ( + globalargs + + [ + "--single-location", + img, + "-q", + ] + + volshellargs + ) + + return runvol(args, volshell, python) + + # # TESTS # + +def basic_volshell_test(image, volatility, python, globalargs): + # Basic VolShell test to verify requirements and ensure VolShell runs without crashing + + volshell_commands = [ + "print(ps())", + "exit()", + ] + + # FIXME: When the minimum Python version includes 3.12, replace the following with: + # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... + fd, filename = tempfile.mkstemp(suffix=".txt") + try: + volshell_script = "\n".join(volshell_commands) + with os.fdopen(fd, "w") as f: + f.write(volshell_script) + + rc, out, _err = runvolshell( + img=image, + volshell=volatility, + python=python, + volshellargs=["--script", filename], + globalargs=globalargs, + ) + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + + assert rc == 0 + assert out.count(b"\n") >= 4 + + return out + + # WINDOWS +def test_windows_volshell(image, volatility, python): + out = basic_volshell_test(image, volatility, python, globalargs=["-w"]) + assert out.count(b" 40 + + def test_windows_pslist(image, volatility, python): - rc, out, err = runvol_plugin("windows.pslist.PsList", image, volatility, python) + rc, out, _err = runvol_plugin("windows.pslist.PsList", image, volatility, python) out = out.lower() assert out.find(b"system") != -1 assert out.find(b"csrss.exe") != -1 @@ -69,7 +127,7 @@ def test_windows_pslist(image, volatility, python): assert out.count(b"\n") > 10 assert rc == 0 - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.pslist.PsList", image, volatility, python, pluginargs=["--pid", "4"] ) out = out.lower() @@ -79,7 +137,7 @@ def test_windows_pslist(image, volatility, python): def test_windows_psscan(image, volatility, python): - rc, out, err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) + rc, out, _err = runvol_plugin("windows.psscan.PsScan", image, volatility, python) out = out.lower() assert out.find(b"system") != -1 assert out.find(b"csrss.exe") != -1 @@ -89,21 +147,21 @@ def test_windows_psscan(image, volatility, python): def test_windows_dlllist(image, volatility, python): - rc, out, err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) + rc, out, _err = runvol_plugin("windows.dlllist.DllList", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 def test_windows_modules(image, volatility, python): - rc, out, err = runvol_plugin("windows.modules.Modules", image, volatility, python) + rc, out, _err = runvol_plugin("windows.modules.Modules", image, volatility, python) out = out.lower() assert out.count(b"\n") > 10 assert rc == 0 def test_windows_hivelist(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.registry.hivelist.HiveList", image, volatility, python ) out = out.lower() @@ -136,7 +194,7 @@ def test_windows_dumpfiles(image, volatility, python): path = tempfile.mkdtemp() - rc, out, err = runvol_plugin( + rc, _out, _err = runvol_plugin( "windows.dumpfiles.DumpFiles", image, volatility, @@ -166,7 +224,7 @@ def test_windows_dumpfiles(image, volatility, python): def test_windows_handles(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.handles.Handles", image, volatility, python, pluginargs=["--pid", "4"] ) @@ -183,7 +241,7 @@ def test_windows_handles(image, volatility, python): def test_windows_svcscan(image, volatility, python): - rc, out, err = runvol_plugin("windows.svcscan.SvcScan", image, volatility, python) + rc, out, _err = runvol_plugin("windows.svcscan.SvcScan", image, volatility, python) assert out.find(b"Microsoft ACPI Driver") != -1 assert out.count(b"\n") > 250 @@ -191,17 +249,19 @@ def test_windows_svcscan(image, volatility, python): def test_windows_thrdscan(image, volatility, python): - rc, out, err = runvol_plugin("windows.thrdscan.ThrdScan", image, volatility, python) + rc, out, _err = runvol_plugin( + "windows.thrdscan.ThrdScan", image, volatility, python + ) # find pid 4 (of system process) which starts with lowest tids assert out.find(b"\t4\t8") != -1 assert out.find(b"\t4\t12") != -1 assert out.find(b"\t4\t16") != -1 - #assert out.find(b"this raieses AssertionError") != -1 + # assert out.find(b"this raieses AssertionError") != -1 assert rc == 0 def test_windows_privileges(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.privileges.Privs", image, volatility, python, pluginargs=["--pid", "4"] ) @@ -213,7 +273,7 @@ def test_windows_privileges(image, volatility, python): def test_windows_getsids(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.getsids.GetSIDs", image, volatility, python, pluginargs=["--pid", "4"] ) @@ -225,7 +285,7 @@ def test_windows_getsids(image, volatility, python): def test_windows_envars(image, volatility, python): - rc, out, err = runvol_plugin("windows.envars.Envars", image, volatility, python) + rc, out, _err = runvol_plugin("windows.envars.Envars", image, volatility, python) assert out.find(b"PATH") != -1 assert out.find(b"PROCESSOR_ARCHITECTURE") != -1 @@ -237,7 +297,7 @@ def test_windows_envars(image, volatility, python): def test_windows_callbacks(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.callbacks.Callbacks", image, volatility, python ) @@ -249,7 +309,7 @@ def test_windows_callbacks(image, volatility, python): def test_windows_vadwalk(image, volatility, python): - rc, out, err = runvol_plugin("windows.vadwalk.VadWalk", image, volatility, python) + rc, out, _err = runvol_plugin("windows.vadwalk.VadWalk", image, volatility, python) assert out.find(b"Vad") != -1 assert out.find(b"VadS") != -1 @@ -260,7 +320,7 @@ def test_windows_vadwalk(image, volatility, python): def test_windows_devicetree(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "windows.devicetree.DeviceTree", image, volatility, python ) @@ -273,117 +333,511 @@ def test_windows_devicetree(image, volatility, python): assert rc == 0 +def test_windows_vadyarascan_yara_rule(image, volatility, python): + yara_rule_01 = r""" + rule fullvadyarascan + { + strings: + $s1 = "!This program cannot be run in DOS mode." + $s2 = "Qw))Pw" + $s3 = "W_wD)Pw" + $s4 = "1Xw+2Xw" + $s5 = "xd`wh``w" + $s6 = "0g`w0g`w8g`w8g`w@g`w@g`wHg`wHg`wPg`wPg`wXg`wXg`w`g`w`g`whg`whg`wpg`wpg`wxg`wxg`w" + condition: + all of them + } + """ + + # FIXME: When the minimum Python version includes 3.12, replace the following with: + # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... + fd, filename = tempfile.mkstemp(suffix=".yar") + try: + with os.fdopen(fd, "w") as f: + f.write(yara_rule_01) + + rc, out, _err = runvol_plugin( + "windows.vadyarascan.VadYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "4012", "--yara-file", filename], + ) + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + + out = out.lower() + assert out.count(b"\n") > 4 + assert rc == 0 + + +def test_windows_vadyarascan_yara_string(image, volatility, python): + rc, out, _err = runvol_plugin( + "windows.vadyarascan.VadYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "4012", "--yara-string", "MZ"], + ) + out = out.lower() + + assert out.count(b"\n") > 10 + assert rc == 0 + + # LINUX +def test_linux_volshell(image, volatility, python): + out = basic_volshell_test(image, volatility, python, globalargs=["-l"]) + assert out.count(b" 100 + + def test_linux_pslist(image, volatility, python): - rc, out, err = runvol_plugin("linux.pslist.PsList", image, volatility, python) - out = out.lower() + rc, out, _err = runvol_plugin("linux.pslist.PsList", image, volatility, python) + assert rc == 0 + out = out.lower() assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) assert out.find(b"watchdog") != -1 assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_check_idt(image, volatility, python): - rc, out, err = runvol_plugin("linux.check_idt.Check_idt", image, volatility, python) - out = out.lower() + rc, out, _err = runvol_plugin( + "linux.check_idt.Check_idt", image, volatility, python + ) + assert rc == 0 + out = out.lower() assert out.count(b"__kernel__") >= 10 assert out.count(b"\n") > 10 - assert rc == 0 def test_linux_check_syscall(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "linux.check_syscall.Check_syscall", image, volatility, python ) - out = out.lower() + assert rc == 0 + out = out.lower() assert out.find(b"sys_close") != -1 assert out.find(b"sys_open") != -1 assert out.count(b"\n") > 100 - assert rc == 0 def test_linux_lsmod(image, volatility, python): - rc, out, err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) - out = out.lower() + rc, out, _err = runvol_plugin("linux.lsmod.Lsmod", image, volatility, python) - assert out.count(b"\n") > 10 assert rc == 0 + out = out.lower() + assert out.count(b"\n") > 10 def test_linux_lsof(image, volatility, python): - rc, out, err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) - out = out.lower() + rc, out, _err = runvol_plugin("linux.lsof.Lsof", image, volatility, python) + assert rc == 0 + out = out.lower() assert out.count(b"socket:") >= 10 assert out.count(b"\n") > 35 - assert rc == 0 def test_linux_proc_maps(image, volatility, python): - rc, out, err = runvol_plugin("linux.proc.Maps", image, volatility, python) - out = out.lower() + rc, out, _err = runvol_plugin("linux.proc.Maps", image, volatility, python) + assert rc == 0 + out = out.lower() assert out.count(b"anonymous mapping") >= 10 assert out.count(b"\n") > 100 - assert rc == 0 def test_linux_tty_check(image, volatility, python): - rc, out, err = runvol_plugin("linux.tty_check.tty_check", image, volatility, python) - out = out.lower() + rc, out, _err = runvol_plugin( + "linux.tty_check.tty_check", image, volatility, python + ) + assert rc == 0 + out = out.lower() assert out.find(b"__kernel__") != -1 assert out.count(b"\n") >= 5 - assert rc == 0 + def test_linux_sockstat(image, volatility, python): - rc, out, err = runvol_plugin("linux.sockstat.Sockstat", image, volatility, python) + rc, out, _err = runvol_plugin("linux.sockstat.Sockstat", image, volatility, python) + assert rc == 0 assert out.count(b"AF_UNIX") >= 354 assert out.count(b"AF_BLUETOOTH") >= 5 assert out.count(b"AF_INET") >= 32 assert out.count(b"AF_INET6") >= 20 assert out.count(b"AF_PACKET") >= 1 assert out.count(b"AF_NETLINK") >= 43 - assert rc == 0 def test_linux_library_list(image, volatility, python): - rc, out, err = runvol_plugin( - "linux.library_list.LibraryList", image, volatility, python + rc, out, _err = runvol_plugin( + "linux.library_list.LibraryList", + image, + volatility, + python, + pluginargs=["--pids", "2363"], ) + assert rc == 0 assert re.search( rb"NetworkManager\s2363\s0x7f52cdda0000\s/lib/x86_64-linux-gnu/libnss_files.so.2", out, ) - assert re.search( - rb"gnome-settings-\s3807\s0x7f7e660b5000\s/lib/x86_64-linux-gnu/libbz2.so.1.0", - out, + + assert out.count(b"\n") > 10 + + +def test_linux_pstree(image, volatility, python): + rc, out, _err = runvol_plugin("linux.pstree.PsTree", image, volatility, python) + + assert rc == 0 + out = out.lower() + assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) + assert out.count(b"\n") > 10 + + +def test_linux_pidhashtable(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.pidhashtable.PIDHashTable", image, volatility, python ) - assert re.search( - rb"gdu-notificatio\s3878\s0x7f25ce33e000\s/usr/lib/x86_64-linux-gnu/libXau.so.6", - out, + + assert rc == 0 + out = out.lower() + assert (out.find(b"init") != -1) or (out.find(b"systemd") != -1) + assert out.count(b"\n") > 10 + + +def test_linux_bash(image, volatility, python): + rc, out, _err = runvol_plugin("linux.bash.Bash", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 10 + + +def test_linux_boottime(image, volatility, python): + rc, out, _err = runvol_plugin("linux.boottime.Boottime", image, volatility, python) + + assert rc == 0 + out = out.lower() + assert out.count(b"utc") >= 1 + + +def test_linux_capabilities(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.capabilities.Capabilities", + image, + volatility, + python, + globalargs=["-vvv"], + ) + + if rc != 0 and err.count(b"Unsupported kernel capabilities implementation") > 0: + # The linux-sample-1.bin kernel implementation isn't supported. + # However, we can still check that the plugin requirements are met. + return None + + assert rc == 0 + assert out.count(b"\n") > 10 + + +def test_linux_check_creds(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.check_creds.Check_creds", image, volatility, python ) + + # linux-sample-1.bin has no processes sharing credentials. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_elfs(image, volatility, python): + rc, out, _err = runvol_plugin("linux.elfs.Elfs", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 10 + + +def test_linux_envars(image, volatility, python): + rc, out, _err = runvol_plugin("linux.envars.Envars", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 10 + + +def test_linux_kthreads(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.kthreads.Kthreads", + image, + volatility, + python, + globalargs=["-vvv"], + ) + + if rc != 0 and err.count(b"Unsupported kthread implementation") > 0: + # The linux-sample-1.bin kernel implementation isn't supported. + # However, we can still check that the plugin requirements are met. + return None + + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_malfind(image, volatility, python): + rc, out, _err = runvol_plugin("linux.malfind.Malfind", image, volatility, python) + + # linux-sample-1.bin has no process memory ranges with potential injected code. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_mountinfo(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.mountinfo.MountInfo", image, volatility, python + ) + + assert rc == 0 + assert out.count(b"\n") > 10 + + +def test_linux_psaux(image, volatility, python): + rc, out, _err = runvol_plugin("linux.psaux.PsAux", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 50 + + +def test_linux_ptrace(image, volatility, python): + rc, out, _err = runvol_plugin("linux.ptrace.Ptrace", image, volatility, python) + + # linux-sample-1.bin has no processes being ptraced. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_vmaregexscan(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.vmaregexscan.VmaRegExScan", + image, + volatility, + python, + pluginargs=["--pid", "1", "--pattern", "\\x7fELF"], + ) + + assert rc == 0 + assert out.count(b"\n") > 10 + + +def test_linux_vmayarascan_yara_rule(image, volatility, python): + yara_rule_01 = r""" + rule fullvmayarascan + { + strings: + $s1 = "_nss_files_parse_grent" + $s2 = "/lib64/ld-linux-x86-64.so.2" + $s3 = "(bufferend - (char *) 0) % sizeof (char *) == 0" + condition: + all of them + } + """ + + # FIXME: When the minimum Python version includes 3.12, replace the following with: + # with tempfile.NamedTemporaryFile(delete_on_close=False) as fd: ... + fd, filename = tempfile.mkstemp(suffix=".yar") + try: + with os.fdopen(fd, "w") as f: + f.write(yara_rule_01) + + rc, out, _err = runvol_plugin( + "linux.vmayarascan.VmaYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "8600", "--yara-file", filename], + ) + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(filename) + + assert rc == 0 + assert out.count(b"\n") > 4 + + +def test_linux_vmayarascan_yara_string(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.vmayarascan.VmaYaraScan", + image, + volatility, + python, + pluginargs=["--pid", "1", "--yara-string", "ELF"], + ) + + assert rc == 0 + assert out.count(b"\n") > 10 + + +def test_linux_page_cache_files(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.pagecache.Files", + image, + volatility, + python, + pluginargs=["--find", "/etc/passwd"], + ) + + assert rc == 0 + assert out.count(b"\n") > 4 + + # inode_num inode_addr ... file_path assert re.search( - rb"bash\s8600\s0x7fe78a85f000\s/lib/x86_64-linux-gnu/libnss_files.so.2", + rb"146829\s0x88001ab5c270.*?/etc/passwd", out, ) - assert out.count(b"\n") >= 2677 + +def test_linux_page_cache_inodepages(image, volatility, python): + + inode_address = hex(0x88001AB5C270) + inode_dump_filename = f"inode_{inode_address}.dmp" + try: + rc, out, _err = runvol_plugin( + "linux.pagecache.InodePages", + image, + volatility, + python, + pluginargs=["--inode", inode_address, "--dump"], + ) + + assert rc == 0 + assert out.count(b"\n") > 4 + + # PageVAddr PagePAddr MappingAddr .. DumpSafe + assert re.search( + rb"0xea000054c5f8\s0x18389000\s0x88001ab5c3b0.*?True", + out, + ) + assert os.path.exists(inode_dump_filename) + with open(inode_dump_filename, "rb") as fp: + inode_contents = fp.read() + assert inode_contents.count(b"\n") > 30 + assert inode_contents.count(b"root:x:0:0:root:/root:/bin/bash") > 0 + finally: + with contextlib.suppress(FileNotFoundError): + os.remove(inode_dump_filename) + + +def test_linux_check_afinfo(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.check_afinfo.Check_afinfo", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results. + # This validates that plugin requirements are met and exceptions are not raised. assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_check_modules(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.check_modules.Check_modules", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_ebpf_progs(image, volatility, python): + rc, out, err = runvol_plugin( + "linux.ebpf.EBPF", + image, + volatility, + python, + globalargs=["-vvv"], + ) + + if rc != 0 and err.count(b"Unsupported kernel") > 0: + # The linux-sample-1.bin kernel implementation isn't supported. + # However, we can still check that the plugin requirements are met. + return None + + assert rc == 0 + assert out.count(b"\n") > 4 + + +def test_linux_iomem(image, volatility, python): + rc, out, _err = runvol_plugin("linux.iomem.IOMem", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 100 + + +def test_linux_keyboard_notifiers(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.keyboard_notifiers.Keyboard_notifiers", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results for this plugin. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_kmesg(image, volatility, python): + rc, out, _err = runvol_plugin("linux.kmsg.Kmsg", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 100 + + +def test_linux_netfilter(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.netfilter.Netfilter", image, volatility, python + ) + + # linux-sample-1.bin has no suspicious results for this plugin. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 + + +def test_linux_psscan(image, volatility, python): + rc, out, _err = runvol_plugin("linux.psscan.PsScan", image, volatility, python) + + assert rc == 0 + assert out.count(b"\n") > 100 + + +def test_linux_hidden_modules(image, volatility, python): + rc, out, _err = runvol_plugin( + "linux.hidden_modules.Hidden_modules", image, volatility, python + ) + + # linux-sample-1.bin has no hidden modules. + # This validates that plugin requirements are met and exceptions are not raised. + assert rc == 0 + assert out.count(b"\n") >= 4 # MAC +def test_mac_volshell(image, volatility, python): + basic_volshell_test(image, volatility, python, globalargs=["-m"]) + + def test_mac_pslist(image, volatility, python): - rc, out, err = runvol_plugin("mac.pslist.PsList", image, volatility, python) + rc, out, _err = runvol_plugin("mac.pslist.PsList", image, volatility, python) out = out.lower() assert (out.find(b"kernel_task") != -1) or (out.find(b"launchd") != -1) @@ -392,7 +846,7 @@ def test_mac_pslist(image, volatility, python): def test_mac_check_syscall(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "mac.check_syscall.Check_syscall", image, volatility, python ) out = out.lower() @@ -405,7 +859,7 @@ def test_mac_check_syscall(image, volatility, python): def test_mac_check_sysctl(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "mac.check_sysctl.Check_sysctl", image, volatility, python ) out = out.lower() @@ -416,7 +870,7 @@ def test_mac_check_sysctl(image, volatility, python): def test_mac_check_trap_table(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "mac.check_trap_table.Check_trap_table", image, volatility, python ) out = out.lower() @@ -427,7 +881,7 @@ def test_mac_check_trap_table(image, volatility, python): def test_mac_ifconfig(image, volatility, python): - rc, out, err = runvol_plugin("mac.ifconfig.Ifconfig", image, volatility, python) + rc, out, _err = runvol_plugin("mac.ifconfig.Ifconfig", image, volatility, python) out = out.lower() assert out.find(b"127.0.0.1") != -1 @@ -437,7 +891,7 @@ def test_mac_ifconfig(image, volatility, python): def test_mac_lsmod(image, volatility, python): - rc, out, err = runvol_plugin("mac.lsmod.Lsmod", image, volatility, python) + rc, out, _err = runvol_plugin("mac.lsmod.Lsmod", image, volatility, python) out = out.lower() assert out.find(b"com.apple") != -1 @@ -446,7 +900,7 @@ def test_mac_lsmod(image, volatility, python): def test_mac_lsof(image, volatility, python): - rc, out, err = runvol_plugin("mac.lsof.Lsof", image, volatility, python) + rc, out, _err = runvol_plugin("mac.lsof.Lsof", image, volatility, python) out = out.lower() assert out.count(b"\n") > 50 @@ -454,7 +908,7 @@ def test_mac_lsof(image, volatility, python): def test_mac_malfind(image, volatility, python): - rc, out, err = runvol_plugin("mac.malfind.Malfind", image, volatility, python) + rc, out, _err = runvol_plugin("mac.malfind.Malfind", image, volatility, python) out = out.lower() assert out.count(b"\n") > 20 @@ -462,7 +916,7 @@ def test_mac_malfind(image, volatility, python): def test_mac_mount(image, volatility, python): - rc, out, err = runvol_plugin("mac.mount.Mount", image, volatility, python) + rc, out, _err = runvol_plugin("mac.mount.Mount", image, volatility, python) out = out.lower() assert out.find(b"/dev") != -1 @@ -471,7 +925,7 @@ def test_mac_mount(image, volatility, python): def test_mac_netstat(image, volatility, python): - rc, out, err = runvol_plugin("mac.netstat.Netstat", image, volatility, python) + rc, out, _err = runvol_plugin("mac.netstat.Netstat", image, volatility, python) assert out.find(b"TCP") != -1 assert out.find(b"UDP") != -1 @@ -481,7 +935,7 @@ def test_mac_netstat(image, volatility, python): def test_mac_proc_maps(image, volatility, python): - rc, out, err = runvol_plugin("mac.proc_maps.Maps", image, volatility, python) + rc, out, _err = runvol_plugin("mac.proc_maps.Maps", image, volatility, python) out = out.lower() assert out.find(b"[heap]") != -1 @@ -490,7 +944,7 @@ def test_mac_proc_maps(image, volatility, python): def test_mac_psaux(image, volatility, python): - rc, out, err = runvol_plugin("mac.psaux.Psaux", image, volatility, python) + rc, out, _err = runvol_plugin("mac.psaux.Psaux", image, volatility, python) out = out.lower() assert out.find(b"executable_path") != -1 @@ -499,7 +953,7 @@ def test_mac_psaux(image, volatility, python): def test_mac_socket_filters(image, volatility, python): - rc, out, err = runvol_plugin( + rc, out, _err = runvol_plugin( "mac.socket_filters.Socket_filters", image, volatility, python ) out = out.lower() @@ -509,7 +963,7 @@ def test_mac_socket_filters(image, volatility, python): def test_mac_timers(image, volatility, python): - rc, out, err = runvol_plugin("mac.timers.Timers", image, volatility, python) + rc, out, _err = runvol_plugin("mac.timers.Timers", image, volatility, python) out = out.lower() assert out.count(b"\n") > 6 @@ -517,7 +971,9 @@ def test_mac_timers(image, volatility, python): def test_mac_trustedbsd(image, volatility, python): - rc, out, err = runvol_plugin("mac.trustedbsd.Trustedbsd", image, volatility, python) + rc, out, _err = runvol_plugin( + "mac.trustedbsd.Trustedbsd", image, volatility, python + ) out = out.lower() assert out.count(b"\n") > 10 diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 75b62abf6a..6172a17f3c 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -19,7 +19,7 @@ import sys import tempfile import traceback -from typing import Any, Dict, List, Tuple, Type, Union +from typing import Any, Dict, List, Optional, Tuple, Type, Union from urllib import parse, request try: @@ -57,14 +57,14 @@ console.setFormatter(formatter) -class PrintedProgress(object): +class PrintedProgress: """A progress handler that prints the progress value and the description onto the command line.""" def __init__(self): self._max_message_len = 0 - def __call__(self, progress: Union[int, float], description: str = None): + def __call__(self, progress: Union[int, float], description: Optional[str] = None): """A simple function for providing text-based feedback. .. warning:: Only for development use. @@ -81,14 +81,14 @@ def __call__(self, progress: Union[int, float], description: str = None): class MuteProgress(PrintedProgress): """A dummy progress handler that produces no output when called.""" - def __call__(self, progress: Union[int, float], description: str = None): + def __call__(self, progress: Union[int, float], description: Optional[str] = None): pass class CommandLine: """Constructs a command-line interface object for users to run plugins.""" - CLI_NAME = "volatility" + CLI_NAME = os.path.basename(sys.argv[0]) # vol or volatility def __init__(self): self.setup_logging() @@ -126,9 +126,7 @@ def run(self): "--help", action="help", default=argparse.SUPPRESS, - help="Show this help message and exit, for specific plugin options use '{} --help'".format( - parser.prog - ), + help=f"Show this help message and exit, for specific plugin options use '{parser.prog} --help'", ) parser.add_argument( "-c", @@ -360,10 +358,9 @@ def run(self): subparser = parser.add_subparsers( title="Plugins", dest="plugin", - description="For plugin specific options, run '{} --help'".format( - self.CLI_NAME - ), + description=f"For plugin specific options, run '{self.CLI_NAME} --help'", action=volargparse.HelpfulSubparserAction, + metavar="PLUGIN", ) for plugin in sorted(plugin_list): plugin_parser = subparser.add_parser( @@ -385,7 +382,9 @@ def run(self): argcomplete.autocomplete(parser) args = parser.parse_args() if args.plugin is None: - parser.error("Please select a plugin to run") + parser.error( + f"Please select a plugin to run (see '{self.CLI_NAME} --help' for options" + ) vollog.log( constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}" @@ -413,7 +412,7 @@ def run(self): # UI fills in the config, here we load it from the config file and do it before we process the CL parameters if args.config: - with open(args.config, "r") as f: + with open(args.config) as f: json_val = json.load(f) ctx.config.splice( plugin_config_path, @@ -719,9 +718,7 @@ def populate_config( if isinstance(requirement, requirements.ListRequirement): if not isinstance(value, list): raise TypeError( - "Configuration for ListRequirement was not a list: {}".format( - requirement.name - ) + f"Configuration for ListRequirement was not a list: {requirement.name}" ) value = [requirement.element_type(x) for x in value] if not inspect.isclass(configurables_list[configurable]): @@ -794,7 +791,7 @@ def __init__(self, filename: str): fd, self._name = tempfile.mkstemp( suffix=".vol3", prefix="tmp_", dir=output_dir ) - self._file = io.open(fd, mode="w+b") + self._file = open(fd, mode="w+b") CLIFileHandler.__init__(self, filename) for item in dir(self._file): if not item.startswith("_") and item not in ( @@ -867,9 +864,7 @@ def populate_requirements_argparse( requirement, interfaces.configuration.RequirementInterface ): raise TypeError( - "Plugin contains requirements that are not RequirementInterfaces: {}".format( - configurable.__name__ - ) + f"Plugin contains requirements that are not RequirementInterfaces: {configurable.__name__}" ) if isinstance(requirement, interfaces.configuration.SimpleTypeRequirement): additional["type"] = requirement.instance_type @@ -884,7 +879,7 @@ def populate_requirements_argparse( volatility3.framework.configuration.requirements.ListRequirement, ): # Allow a list of integers, specified with the convenient 0x hexadecimal format - if requirement.element_type == int: + if requirement.element_type is int: additional["type"] = lambda x: int(x, 0) else: additional["type"] = requirement.element_type diff --git a/volatility3/cli/text_filter.py b/volatility3/cli/text_filter.py index 3d69934e99..6bd6878a52 100644 --- a/volatility3/cli/text_filter.py +++ b/volatility3/cli/text_filter.py @@ -74,9 +74,9 @@ def find(self, item) -> bool: """Identifies whether an item is found in the appropriate column""" try: if self.regex: - return re.search(self.pattern, f"{item}") + return bool(re.search(self.pattern, f"{item}")) return self.pattern in f"{item}" - except IOError: + except OSError: return False def found(self, row: List[Any]) -> bool: diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index 96970d3cef..b1944ae5af 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -49,10 +49,12 @@ def hex_bytes_as_text(value: bytes, width: int = 16) -> str: output += "\n" printables = "" - # Handle leftovers when the lenght is not mutiple of width + # Handle leftovers when the length is not mutiple of width if printables: - output += " " * (width - len(printables)) + padding = width - len(printables) + output += " " * padding output += printables + output += " " * padding return output @@ -130,7 +132,7 @@ def display_disassembly(disasm: interfaces.renderers.Disassembly) -> str: for i in disasm_types[disasm.architecture].disasm( disasm.data, disasm.offset ): - output += f"\n0x{i.address:x}:\t{i.mnemonic}\t{i.op_str}" + output += f"\n{i.address:#x}:\t{i.mnemonic}\t{i.op_str}" return output return QuickTextRenderer._type_renderers[bytes](disasm.data) @@ -174,7 +176,7 @@ class QuickTextRenderer(CLIRenderer): format_hints.HexBytes: optional(hex_bytes_as_text), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), interfaces.renderers.Disassembly: optional(display_disassembly), - bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + bytes: optional(lambda x: " ".join(f"{b:02x}" for b in x)), datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), "default": optional(lambda x: f"{x}"), } @@ -254,7 +256,7 @@ class CSVRenderer(CLIRenderer): format_hints.HexBytes: optional(hex_bytes_as_text), format_hints.MultiTypeData: optional(multitypedata_as_text), interfaces.renderers.Disassembly: optional(display_disassembly), - bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + bytes: optional(lambda x: " ".join(f"{b:02x}" for b in x)), datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), "default": optional(lambda x: f"{x}"), } @@ -340,7 +342,7 @@ def render(self, grid: interfaces.renderers.TreeGrid) -> None: column_separator = " | " tree_indent_column = "".join( - random.choice(string.ascii_uppercase + string.digits) for _ in range(20) + random.choices(string.ascii_uppercase + string.digits, k=20) ) max_column_widths = dict( [(column.name, len(column.name)) for column in grid.columns] @@ -448,7 +450,7 @@ class JsonRenderer(CLIRenderer): format_hints.HexBytes: quoted_optional(hex_bytes_as_text), interfaces.renderers.Disassembly: quoted_optional(display_disassembly), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), - bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), + bytes: optional(lambda x: " ".join(f"{b:02x}" for b in x)), datetime.datetime: lambda x: ( x.isoformat() if not isinstance(x, interfaces.renderers.BaseAbsentValue) @@ -465,7 +467,7 @@ def get_render_options(self) -> List[interfaces.renderers.RenderOption]: def output_result(self, outfd, result): """Outputs the JSON data to a file in a particular format""" - outfd.write("{}\n".format(json.dumps(result, indent=2, sort_keys=True))) + outfd.write(f"{json.dumps(result, indent=2, sort_keys=True)}\n") def render(self, grid: interfaces.renderers.TreeGrid): outfd = sys.stdout diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index fd61ddce02..dce9cafa6e 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -5,7 +5,7 @@ import argparse import gettext import re -from typing import List, Optional, Sequence, Any, Union +from typing import Optional, Sequence, Any, Union # This effectively overrides/monkeypatches the core argparse module to provide more helpful output around choices diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 5172c5363f..0affe5d597 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -49,7 +49,7 @@ class VolShell(cli.CommandLine): python terminal with all the volatility support calls available. """ - CLI_NAME = "volshell" + CLI_NAME = os.path.basename(sys.argv[0]) # volshell def __init__(self): super().__init__() @@ -282,9 +282,7 @@ def run(self): for plugin in volshell_plugin_list: subparser = parser.add_argument_group( title=plugin.capitalize(), - description="Configuration options based on {} options".format( - plugin.capitalize() - ), + description=f"Configuration options based on {plugin.capitalize()} options", ) self.populate_requirements_argparse(subparser, volshell_plugin_list[plugin]) configurables_list[plugin] = volshell_plugin_list[plugin] @@ -331,7 +329,7 @@ def run(self): # UI fills in the config, here we load it from the config file and do it before we process the CL parameters if args.config: - with open(args.config, "r") as f: + with open(args.config) as f: json_val = json.load(f) ctx.config.splice( plugin_config_path, diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 82c470e1af..12f5499f60 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -203,7 +203,7 @@ def _display_data( connector = " " if chunk_size < 2: connector = "" - ascii_data = connector.join([self._ascii_bytes(x) for x in valid_data]) + ascii_data = connector.join(self._ascii_bytes(x) for x in valid_data) print(hex(offset), " ", hex_data, " ", ascii_data) offset += 16 @@ -240,7 +240,7 @@ def kernel(self): return None return self.context.modules[self.current_kernel_name] - def change_layer(self, layer_name: str = None): + def change_layer(self, layer_name: Optional[str] = None): """Changes the current default layer""" if not layer_name: layer_name = self.current_layer @@ -250,7 +250,7 @@ def change_layer(self, layer_name: str = None): self.__current_layer = layer_name sys.ps1 = f"({self.current_layer}) >>> " - def change_symbol_table(self, symbol_table_name: str = None): + def change_symbol_table(self, symbol_table_name: Optional[str] = None): """Changes the current_symbol_table""" if not symbol_table_name: print("No symbol table provided, not changing current symbol table") @@ -262,7 +262,7 @@ def change_symbol_table(self, symbol_table_name: str = None): self.__current_symbol_table = symbol_table_name print(f"Current Symbol Table: {self.current_symbol_table}") - def change_kernel(self, kernel_name: str = None): + def change_kernel(self, kernel_name: Optional[str] = None): if not kernel_name: print("No kernel module name provided, not changing current kernel") if kernel_name not in self.context.modules: @@ -347,7 +347,7 @@ def display_type( object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if not isinstance( @@ -479,7 +479,7 @@ def display_plugin_output( if treegrid is not None: self.render_treegrid(treegrid) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: print("No symbol table provided") @@ -585,7 +585,6 @@ def __init__(self, preferred_name: str): def writelines(self, lines: Iterable[bytes]): """Dummy method""" - pass def write(self, b: bytes): """Dummy method""" diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index c5e555ec7d..cc58fa1c24 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -20,7 +20,7 @@ def get_requirements(cls): name="kernel", description="Linux kernel module" ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.IntRequirement( name="pid", description="Process ID", optional=True @@ -61,7 +61,7 @@ def display_type( object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -69,7 +69,7 @@ def display_type( object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 2b32ad6776..0ed35eb272 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -63,7 +63,7 @@ def display_type( object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -71,7 +71,7 @@ def display_type( object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 5c2190c027..303d4d5c3b 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Any, List, Tuple, Union +from typing import Any, List, Optional, Tuple, Union from volatility3.cli.volshell import generic from volatility3.framework import constants, interfaces @@ -60,7 +60,7 @@ def display_type( object: Union[ str, interfaces.objects.ObjectInterface, interfaces.objects.Template ], - offset: int = None, + offset: Optional[int] = None, ): """Display Type describes the members of a particular object in alphabetical order""" if isinstance(object, str): @@ -68,7 +68,7 @@ def display_type( object = self.current_symbol_table + constants.BANG + object return super().display_type(object, offset) - def display_symbols(self, symbol_table: str = None): + def display_symbols(self, symbol_table: Optional[str] = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: symbol_table = self.current_symbol_table diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 51310bfa29..a1925faef0 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -6,28 +6,12 @@ import glob import sys import zipfile - -required_python_version = (3, 8, 0) -if ( - sys.version_info.major != required_python_version[0] - or sys.version_info.minor < required_python_version[1] - or ( - sys.version_info.minor == required_python_version[1] - and sys.version_info.micro < required_python_version[2] - ) -): - raise RuntimeError( - "Volatility framework requires python version {}.{}.{} or greater".format( - *required_python_version - ) - ) - import importlib import inspect import logging import os import traceback -from typing import Any, Dict, Generator, List, Tuple, Type, TypeVar +from typing import Any, Dict, Generator, List, Optional, Tuple, Type, TypeVar from volatility3.framework import constants, interfaces @@ -56,27 +40,25 @@ def require_interface_version(*args) -> None: if len(args): if args[0] != interface_version()[0]: raise RuntimeError( - "Framework interface version {} is incompatible with required version {}".format( - interface_version()[0], args[0] - ) + f"Framework interface version {interface_version()[0]} is incompatible with required version {args[0]}" ) if len(args) > 1: if args[1] > interface_version()[1]: raise RuntimeError( "Framework interface version {} is an older revision than the required version {}".format( - ".".join([str(x) for x in interface_version()[0:2]]), - ".".join([str(x) for x in args[0:2]]), + ".".join(str(x) for x in interface_version()[0:2]), + ".".join(str(x) for x in args[0:2]), ) ) -class NonInheritable(object): +class NonInheritable: def __init__(self, value: Any, cls: Type) -> None: self.default_value = value self.cls = cls - def __get__(self, obj: Any, get_type: Type = None) -> Any: - if type == self.cls: + def __get__(self, obj: Any, get_type: Optional[Type] = None) -> Any: + if type is self.cls: if hasattr(self.default_value, "__get__"): return self.default_value.__get__(obj, get_type) return self.default_value @@ -99,8 +81,7 @@ def class_subclasses(cls: Type[T]) -> Generator[Type[T], None, None]: # The typing system is not clever enough to realize that clazz has a hidden attr after the hasattr check if not hasattr(clazz, "hidden") or not clazz.hidden: # type: ignore yield clazz - for return_value in class_subclasses(clazz): - yield return_value + yield from class_subclasses(clazz) def import_files(base_module, ignore_errors: bool = False) -> List[str]: @@ -161,11 +142,7 @@ def import_files(base_module, ignore_errors: bool = False) -> List[str]: def _filter_files(filename: str): """Ensures that a filename traversed is an importable python file""" - return ( - filename.endswith(".py") - or filename.endswith(".pyc") - or filename.endswith(".pyo") - ) and not filename.startswith("__") + return (filename.endswith((".py", ".pyc"))) and not filename.startswith("__") def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str]: @@ -189,9 +166,7 @@ def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str traceback.TracebackException.from_exception(e).format(chain=True) ) ) - vollog.debug( - "Failed to import module {} based on file: {}".format(module, path) - ) + vollog.debug(f"Failed to import module {module} based on file: {path}") failures.append(module) if not ignore_errors: raise @@ -209,8 +184,7 @@ def _zipwalk(path: str): zip_results[os.path.join(path, os.path.dirname(file.filename))] = ( dirlist ) - for value in zip_results: - yield value, zip_results[value] + yield from zip_results.items() def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 52a73f45a8..f22cae0128 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -3,12 +3,10 @@ # import logging -import os -from typing import Optional, Tuple, Type +from typing import Optional, Tuple from volatility3.framework import constants, interfaces from volatility3.framework.automagic import symbol_cache, symbol_finder -from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import linux @@ -27,16 +25,6 @@ def stack( progress_callback: constants.ProgressCallback = None, ) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify linux within this layer.""" - # Version check the SQlite cache - required = (1, 0, 0) - if not requirements.VersionRequirement.matches_required( - required, symbol_cache.SqliteCache.version - ): - vollog.info( - f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}" - ) - return None - # Bail out by default unless we can stack properly layer = context.layers[layer_name] join = interfaces.configuration.path_join @@ -46,12 +34,9 @@ def stack( if isinstance(layer, intel.Intel): return None - identifiers_path = os.path.join( - constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + linux_banners = symbol_cache.load_cache_manager().get_identifier_dictionary( + operating_system="linux" ) - linux_banners = symbol_cache.SqliteCache( - identifiers_path - ).get_identifier_dictionary(operating_system="linux") # If we have no banners, don't bother scanning if not linux_banners: vollog.info( @@ -80,14 +65,14 @@ def stack( context, table_name, layer_name, progress_callback=progress_callback ) - layer_class: Type = intel.Intel if "init_top_pgt" in table.symbols: - layer_class = intel.Intel32e + layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_top_pgt" elif "init_level4_pgt" in table.symbols: - layer_class = intel.Intel32e + layer_class = intel.LinuxIntel32e dtb_symbol_name = "init_level4_pgt" else: + layer_class = intel.LinuxIntel dtb_symbol_name = "swapper_pg_dir" dtb = cls.virtual_to_physical_address( @@ -156,6 +141,18 @@ def find_aslr( and init_task.state.cast("unsigned int") != 0 ): continue + elif init_task.active_mm.cast("long unsigned int") == module.get_symbol( + "init_mm" + ).address and init_task.tasks.next.cast( + "long unsigned int" + ) == init_task.tasks.prev.cast( + "long unsigned int" + ): + # The idle task steals `mm` from previously running task, i.e., + # `init_mm` is only used as long as no CPU has ever been idle. + # This catches cases where we found a fragment of the + # unrelocated ELF file instead of the running kernel. + continue # This we get for free aslr_shift = ( @@ -174,9 +171,7 @@ def find_aslr( if aslr_shift & 0xFFF != 0 or kaslr_shift & 0xFFF != 0: continue vollog.debug( - "Linux ASLR shift values determined: physical {:0x} virtual {:0x}".format( - kaslr_shift, aslr_shift - ) + f"Linux ASLR shift values determined: physical {kaslr_shift:0x} virtual {aslr_shift:0x}" ) return kaslr_shift, aslr_shift @@ -199,5 +194,8 @@ class LinuxSymbolFinder(symbol_finder.SymbolFinder): banner_config_key = "kernel_banner" operating_system = "linux" symbol_class = "volatility3.framework.symbols.linux.LinuxKernelIntermedSymbols" - find_aslr = lambda cls, *args: LinuxIntelStacker.find_aslr(*args)[1] exclusion_list = ["mac", "windows"] + + @classmethod + def find_aslr(cls, *args): + return LinuxIntelStacker.find_aslr(*args)[1] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index e517531394..f3679d1608 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -3,13 +3,11 @@ # import logging -import os import struct from typing import Optional from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.automagic import symbol_cache, symbol_finder -from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import mac @@ -28,16 +26,6 @@ def stack( progress_callback: constants.ProgressCallback = None, ) -> Optional[interfaces.layers.DataLayerInterface]: """Attempts to identify mac within this layer.""" - # Version check the SQlite cache - required = (1, 0, 0) - if not requirements.VersionRequirement.matches_required( - required, symbol_cache.SqliteCache.version - ): - vollog.info( - f"SQLiteCache version not suitable: required {required} found {symbol_cache.SqliteCache.version}" - ) - return None - # Bail out by default unless we can stack properly layer = context.layers[layer_name] new_layer = None @@ -48,12 +36,9 @@ def stack( if isinstance(layer, intel.Intel): return None - identifiers_path = os.path.join( - constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME + mac_banners = symbol_cache.load_cache_manager().get_identifier_dictionary( + operating_system="mac" ) - mac_banners = symbol_cache.SqliteCache( - identifiers_path - ).get_identifier_dictionary(operating_system="mac") # If we have no banners, don't bother scanning if not mac_banners: vollog.info( @@ -197,7 +182,7 @@ def find_aslr( aslr_shift = 0 for offset, banner in offset_generator: - banner_major, banner_minor = [int(x) for x in banner[22:].split(b".")[0:2]] + banner_major, banner_minor = (int(x) for x in banner[22:].split(b".")[0:2]) tmp_aslr_shift = offset - cls.virtual_to_physical_address( version_json_address diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 7f38a23e1a..729c48063a 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -215,9 +215,7 @@ def test_physical_kernel( return (virtual_layer_name, kvo, kernel) else: vollog.debug( - "Potential kernel_virtual_offset did not map to expected location: {}".format( - hex(kvo) - ) + f"Potential kernel_virtual_offset did not map to expected location: {hex(kvo)}" ) except exceptions.InvalidAddressException: vollog.debug( @@ -270,6 +268,10 @@ def _method_layer_pdb_scan( progress_callback=progress_callback, ) for kernel in kernels: + vollog.log( + constants.LOGLEVEL_VVVV, + f"Testing potential kernel for {kernel.get('pdb_name', 'Unknown')} at {kernel.get('signature_offset', -1):x} with MZ offset at {(kernel.get('mz_offset', -1) or -1):x}", + ) valid_kernel = test_kernel(physical_layer_name, virtual_layer_name, kernel) if valid_kernel is not None: break diff --git a/volatility3/framework/automagic/stacker.py b/volatility3/framework/automagic/stacker.py index c251d3c46b..5968642646 100644 --- a/volatility3/framework/automagic/stacker.py +++ b/volatility3/framework/automagic/stacker.py @@ -166,7 +166,9 @@ def stack_layer( cls, context: interfaces.context.ContextInterface, initial_layer: str, - stack_set: List[Type[interfaces.automagic.StackerLayerInterface]] = None, + stack_set: Optional[ + List[Type[interfaces.automagic.StackerLayerInterface]] + ] = None, progress_callback: constants.ProgressCallback = None, ): """Stacks as many possible layers on top of the initial layer as can be done. diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 22f1c94f34..065eb6d43d 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -104,10 +104,11 @@ def __init__(self, filename: str): for subclazz in framework.class_subclasses(IdentifierProcessor): self._classifiers[subclazz.operating_system] = subclazz + @abstractmethod def add_identifier(self, location: str, operating_system: str, identifier: str): """Adds an identifier to the store""" - pass + @abstractmethod def find_location( self, identifier: bytes, operating_system: Optional[str] ) -> Optional[str]: @@ -120,19 +121,19 @@ def find_location( Returns: The location of the symbols file that matches the identifier """ - pass + @abstractmethod def get_local_locations(self) -> Iterable[str]: """Returns a list of all the local locations""" - pass + @abstractmethod def update(self): """Locates all files under the symbol directories. Updates the cache with additions, modifications and removals. This also updates remote locations based on a cache timeout. """ - pass + @abstractmethod def get_identifier_dictionary( self, operating_system: Optional[str] = None, local_only: bool = False ) -> Dict[bytes, str]: @@ -145,16 +146,16 @@ def get_identifier_dictionary( Returns: A dictionary of identifiers mapped to a location """ - pass + @abstractmethod def get_identifier(self, location: str) -> Optional[bytes]: """Returns an identifier based on a specific location or None""" - pass + @abstractmethod def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: """Returns all identifiers for a particular operating system""" - pass + @abstractmethod def get_location_statistics( self, location: str ) -> Optional[Tuple[int, int, int, int]]: @@ -164,6 +165,7 @@ def get_location_statistics( A tuple of base_types, types, enums, symbols, or None is location not found """ + @abstractmethod def get_hash(self, location: str) -> Optional[str]: """Returns the hash of the JSON from within a location ISF""" @@ -492,6 +494,21 @@ def get_identifiers(self, operating_system: Optional[str]) -> List[bytes]: return output +def load_cache_manager(cache_file: Optional[str] = None) -> CacheManagerInterface: + """Loads a cache manager based on a specific cache file""" + if cache_file is None: + cache_file = os.path.join(constants.CACHE_PATH, constants.IDENTIFIERS_FILENAME) + # Different implementations of cache + if not os.path.exists(cache_file): + raise ValueError("Non-existant cache file provided") + with open(cache_file, "rb") as fp: + header = fp.read(4) + if header not in [b"SQLi"]: + raise ValueError("Identifier file not in recognized format") + # Currently only one choice, so use that + return SqliteCache(cache_file) + + ### Automagic @@ -557,6 +574,6 @@ def process_v1( try: subrbf = RemoteIdentifierFormat(location) yield from subrbf.process(identifiers, operating_system) - except IOError: + except OSError: vollog.debug(f"Remote file not found: {location}") return identifiers diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index 21e594549e..1d30f3f51d 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -4,7 +4,7 @@ import logging import os -from typing import Any, Callable, Iterable, List, Optional, Tuple +from typing import Callable, List, Optional, Tuple from volatility3.framework import constants, interfaces, layers from volatility3.framework.automagic import symbol_cache @@ -142,11 +142,11 @@ def _banner_scan( ) for _, banner in banner_list: - vollog.debug(f"Identified banner: {repr(banner)}") - symbol_files = self.banners.get(banner, None) - if symbol_files: - isf_path = symbol_files - vollog.debug(f"Using symbol library: {symbol_files}") + vollog.debug(f"Identified banner: {banner!r}") + symbols_file = self.banners.get(banner, None) + if symbols_file: + isf_path = symbols_file + vollog.debug(f"Using symbol library: {symbols_file}") clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join @@ -160,8 +160,31 @@ def _banner_scan( path_join(config_path, requirement.name, "symbol_mask") ] = layer.address_mask + # Keep track of the existing table names so we know which ones were added + old_table_names = set(context.symbol_space) + # Construct the appropriate symbol table requirement.construct(context, config_path) + + new_table_names = set(context.symbol_space) - old_table_names + # It should add only one symbol table. Ignore the next steps if it doesn't + if len(new_table_names) == 1: + new_table_name = new_table_names.pop() + symbol_table = context.symbol_space[new_table_name] + producer_metadata = symbol_table.producer + vollog.debug( + f"producer_name: {producer_metadata.name}, producer_version: {producer_metadata.version_string}" + ) + + symbol_metadata = symbol_table.metadata + vollog.debug("Types:") + for types_source_dict in symbol_metadata.get_types_sources(): + vollog.debug(f"\t{types_source_dict}") + + vollog.debug("Symbols:") + for symbol_source_dict in symbol_metadata.get_symbols_sources(): + vollog.debug(f"\t{symbol_source_dict}") + break else: vollog.debug(f"Symbol library path not found for: {banner}") diff --git a/volatility3/framework/check_python_version.py b/volatility3/framework/check_python_version.py new file mode 100644 index 0000000000..f2d284f2a7 --- /dev/null +++ b/volatility3/framework/check_python_version.py @@ -0,0 +1,14 @@ +import sys + +required_python_version = (3, 8, 0) +if ( + sys.version_info.major != required_python_version[0] + or sys.version_info.minor < required_python_version[1] + or ( + sys.version_info.minor == required_python_version[1] + and sys.version_info.micro < required_python_version[2] + ) +): + raise RuntimeError( + f"Volatility framework requires python version {required_python_version[0]}.{required_python_version[1]}.{required_python_version[2]} or greater" + ) diff --git a/volatility3/framework/configuration/__init__.py b/volatility3/framework/configuration/__init__.py index 7a84ee4550..7b914cf164 100644 --- a/volatility3/framework/configuration/__init__.py +++ b/volatility3/framework/configuration/__init__.py @@ -2,4 +2,4 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework.configuration import requirements +from volatility3.framework.configuration import requirements as requirements diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 86e1aac529..3e3608000e 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -11,7 +11,7 @@ import abc import logging import os -from typing import Any, ClassVar, Dict, List, Optional, Tuple, Type +from typing import Any, ClassVar, Dict, List, Optional, Set, Tuple, Type from urllib import parse, request from volatility3.framework import constants, interfaces @@ -111,7 +111,7 @@ def __init__( Args: element_type: The (requirement) type of each element within the list - max_elements; The maximum number of acceptable elements this list can contain + max_elements: The maximum number of acceptable elements this list can contain min_elements: The minimum number of acceptable elements this list can contain """ super().__init__(*args, **kwargs) @@ -314,11 +314,11 @@ class TranslationLayerRequirement( def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: interfaces.configuration.ConfigSimpleType = None, optional: bool = False, - oses: List = None, - architectures: List = None, + oses: Optional[List] = None, + architectures: Optional[List[str]] = None, ) -> None: """Constructs a Translation Layer Requirement. @@ -526,19 +526,19 @@ def __init__( description: Optional[str] = None, default: bool = False, optional: bool = False, - component: Type[interfaces.configuration.VersionableInterface] = None, + component: Optional[Type[interfaces.configuration.VersionableInterface]] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: + if version is None: + raise TypeError("Version cannot be None") + if component is None: + raise TypeError("Component cannot be None") if description is None: - description = f"Version {'.'.join([str(x) for x in version])} dependency on {component.__module__}.{component.__name__} unmet" + description = f"Version {'.'.join(str(x) for x in version)} dependency on {component.__module__}.{component.__name__} unmet" super().__init__( name=name, description=description, default=default, optional=optional ) - if component is None: - raise TypeError("Component cannot be None") self._component: Type[interfaces.configuration.VersionableInterface] = component - if version is None: - raise TypeError("Version cannot be None") self._version = version def unsatisfied( @@ -546,7 +546,7 @@ def unsatisfied( context: interfaces.context.ContextInterface, config_path: str, accumulator: Optional[ - List[interfaces.configuration.VersionableInterface] + Set[interfaces.configuration.VersionableInterface] ] = None, ) -> Dict[str, interfaces.configuration.RequirementInterface]: # Mypy doesn't appreciate our classproperty implementation, self._plugin.version has no type @@ -580,7 +580,7 @@ def unsatisfied( ) if result: - result.update({config_path: self}) + result[config_path] = self return result context.config[interfaces.configuration.path_join(config_path, self.name)] = ( @@ -604,10 +604,10 @@ class PluginRequirement(VersionRequirement): def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: bool = False, optional: bool = False, - plugin: Type[interfaces.plugins.PluginInterface] = None, + plugin: Optional[Type[interfaces.plugins.PluginInterface]] = None, version: Optional[Tuple[int, ...]] = None, ) -> None: super().__init__( @@ -627,7 +627,7 @@ class ModuleRequirement( def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: bool = False, architectures: Optional[List[str]] = None, optional: bool = False, @@ -664,9 +664,7 @@ def unsatisfied( if value is not None: vollog.log( constants.LOGLEVEL_V, - "TypeError - Module Requirement only accepts string labels: {}".format( - repr(value) - ), + f"TypeError - Module Requirement only accepts string labels: {repr(value)}", ) return {config_path: self} diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index 27fae4ba1f..23cc2dde59 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -6,20 +6,21 @@ Stores all the constant values that are generally fixed throughout volatility This includes default scanning block sizes, etc. """ + import enum import os.path import sys import warnings from typing import Callable, Optional -import volatility3.framework.constants.linux -import volatility3.framework.constants.windows +from volatility3.framework.constants import linux as linux +from volatility3.framework.constants import windows as windows from volatility3.framework.constants._version import ( - PACKAGE_VERSION, - VERSION_MAJOR, - VERSION_MINOR, - VERSION_PATCH, - VERSION_SUFFIX, + PACKAGE_VERSION as PACKAGE_VERSION, + VERSION_MAJOR as VERSION_MAJOR, + VERSION_MINOR as VERSION_MINOR, + VERSION_PATCH as VERSION_PATCH, + VERSION_SUFFIX as VERSION_SUFFIX, ) PLUGINS_PATH = [ @@ -65,7 +66,11 @@ LOGLEVEL_VVVV = 6 """Logging level for four levels of detail: -vvvvvv""" -CACHE_PATH = os.path.join(os.path.expanduser("~"), ".cache", "volatility3") + +CACHE_PATH = os.path.join( + os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache"), + "volatility3", +) """Default path to store cached data""" SQLITE_CACHE_PERIOD = "-3 days" diff --git a/volatility3/framework/constants/_version.py b/volatility3/framework/constants/_version.py index ce803a6877..11edc07d81 100644 --- a/volatility3/framework/constants/_version.py +++ b/volatility3/framework/constants/_version.py @@ -1,11 +1,11 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change -VERSION_MINOR = 11 # Number of changes that only add to the interface +VERSION_MINOR = 13 # Number of changes that only add to the interface VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" PACKAGE_VERSION = ( - ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) + ".".join(str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]) + VERSION_SUFFIX ) """The canonical version of the volatility3 package""" diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index 6961d93282..f527544c06 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -229,7 +229,7 @@ def create( def object( self, object_type: str, - offset: int = None, + offset: Optional[int] = None, native_layer_name: Optional[str] = None, absolute: bool = False, **kwargs, @@ -356,7 +356,7 @@ def size(self) -> int: return size or 0 @property # type: ignore # FIXME: mypy #5107 - @functools.lru_cache() + @functools.lru_cache def hash(self) -> str: """Hashes the module for equality checks. diff --git a/volatility3/framework/interfaces/__init__.py b/volatility3/framework/interfaces/__init__.py index 51d81d63a5..fd6b1e0625 100644 --- a/volatility3/framework/interfaces/__init__.py +++ b/volatility3/framework/interfaces/__init__.py @@ -13,12 +13,12 @@ # This will also avoid namespace issues, because people can use interfaces.layers to # avoid clashing with the layers package from volatility3.framework.interfaces import ( - renderers, - configuration, - context, - layers, - objects, - plugins, - symbols, - automagic, + renderers as renderers, + configuration as configuration, + context as context, + layers as layers, + objects as objects, + plugins as plugins, + symbols as symbols, + automagic as automagic, ) diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index 0867b1608f..4ac386fc0d 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -42,7 +42,7 @@ class AutomagicInterface( priority = 10 """An ordering to indicate how soon this automagic should be run""" - exclusion_list = [] + exclusion_list: List[str] = [] """A list of plugin categories (typically operating systems) which the plugin will not operate on""" def __init__( diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index da0a4556c6..2e4f580a7d 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -82,7 +82,7 @@ class HierarchicalDict(collections.abc.Mapping): def __init__( self, - initial_dict: Dict[str, "SimpleTypeRequirement"] = None, + initial_dict: Optional[Dict[str, "SimpleTypeRequirement"]] = None, separator: str = CONFIG_SEPARATOR, ) -> None: """ @@ -94,7 +94,7 @@ def __init__( raise TypeError(f"Separator must be a one character string: {separator}") self._separator = separator self._data: Dict[str, ConfigSimpleType] = {} - self._subdict: Dict[str, "HierarchicalDict"] = {} + self._subdict: Dict[str, HierarchicalDict] = {} if isinstance(initial_dict, str): initial_dict = json.loads(initial_dict) if isinstance(initial_dict, dict): @@ -182,9 +182,7 @@ def _setitem(self, key: str, value: Any, is_data: bool = True) -> None: else: if not isinstance(value, HierarchicalDict): raise TypeError( - "HierarchicalDicts can only store HierarchicalDicts within their structure: {}".format( - type(value) - ) + f"HierarchicalDicts can only store HierarchicalDicts within their structure: {type(value)}" ) self._subdict[key] = value @@ -330,7 +328,7 @@ class RequirementInterface(metaclass=ABCMeta): def __init__( self, name: str, - description: str = None, + description: Optional[str] = None, default: ConfigSimpleType = None, optional: bool = False, ) -> None: @@ -498,9 +496,7 @@ def unsatisfied( if not isinstance(value, self.instance_type): vollog.log( constants.LOGLEVEL_V, - "TypeError - {} requirements only accept {} type: {}".format( - self.name, self.instance_type.__name__, repr(value) - ), + f"TypeError - {self.name} requirements only accept {self.instance_type.__name__} type: {repr(value)}", ) return {config_path: self} return {} @@ -622,7 +618,7 @@ def _construct_class( self, context: "interfaces.context.ContextInterface", config_path: str, - requirement_dict: Dict[str, object] = None, + requirement_dict: Optional[Dict[str, object]] = None, ) -> Optional["interfaces.objects.ObjectInterface"]: """Constructs the class, handing args and the subrequirements as parameters to __init__""" @@ -656,6 +652,7 @@ def _construct_class( class ConfigurableRequirementInterface(RequirementInterface): """Simple Abstract class to provide build_required_config.""" + @abstractmethod def build_configuration( self, context: "interfaces.context.ContextInterface", diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 8b5e816e8a..a87e0f1e8c 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -85,7 +85,7 @@ def object( object_type: Union[str, "interfaces.objects.Template"], layer_name: str, offset: int, - native_layer_name: str = None, + native_layer_name: Optional[str] = None, **arguments, ) -> "interfaces.objects.ObjectInterface": """Object factory, takes a context, symbol, offset and optional @@ -114,6 +114,7 @@ def clone(self) -> "ContextInterface": """ return copy.deepcopy(self) + @abstractmethod def module( self, module_name: str, @@ -232,7 +233,7 @@ def symbol_table_name(self) -> str: def object( self, object_type: str, - offset: int = None, + offset: Optional[int] = None, native_layer_name: Optional[str] = None, absolute: bool = False, **kwargs, @@ -277,27 +278,35 @@ def get_absolute_symbol_address(self, name: str) -> int: symbol = self.get_symbol(name) return self.offset + symbol.address + @abstractmethod def get_type(self, name: str) -> "interfaces.objects.Template": """Returns a type from the module's symbol table.""" + @abstractmethod def get_symbol(self, name: str) -> "interfaces.symbols.SymbolInterface": """Returns a symbol object from the module's symbol table.""" + @abstractmethod def get_enumeration(self, name: str) -> "interfaces.objects.Template": """Returns an enumeration from the module's symbol table.""" + @abstractmethod def has_type(self, name: str) -> bool: """Determines whether a type is present in the module's symbol table.""" + @abstractmethod def has_symbol(self, name: str) -> bool: """Determines whether a symbol is present in the module's symbol table.""" + @abstractmethod def has_enumeration(self, name: str) -> bool: """Determines whether an enumeration is present in the module's symbol table.""" + @abstractmethod def symbols(self) -> List: """Lists the symbols contained in the symbol table for this module""" + @abstractmethod def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: """Returns the symbols within table_name (or this module if not specified) that live at the specified absolute offset provided.""" @@ -343,6 +352,7 @@ def __len__(self) -> int: def __iter__(self): return iter(self._modules) + @abstractmethod def free_module_name(self, prefix: str = "module") -> str: """Returns an unused table name to ensure no collision occurs when inserting a symbol table.""" diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index e2a68780a7..a90a786674 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -136,7 +136,7 @@ def maximum_address(self) -> int: def minimum_address(self) -> int: """Returns the minimum valid address of the space.""" - @property + @functools.cached_property def address_mask(self) -> int: """Returns a mask which encapsulates all the active bits of an address for this layer.""" @@ -188,7 +188,6 @@ def destroy(self) -> None: the object unreadable (exceptions will be thrown using a DataLayer after destruction) """ - pass @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -211,7 +210,7 @@ def scan( context: interfaces.context.ContextInterface, scanner: ScannerInterface, progress_callback: constants.ProgressCallback = None, - sections: Iterable[Tuple[int, int]] = None, + sections: Optional[Iterable[Tuple[int, int]]] = None, ) -> Iterable[Any]: """Scans a Translation layer by chunk. @@ -361,9 +360,7 @@ def _scan_chunk( data += self.context.layers[layer_name].read(address, chunk_size) except exceptions.InvalidAddressException: vollog.debug( - "Invalid address in layer {} found scanning {} at address {:x}".format( - layer_name, self.name, address - ) + f"Invalid address in layer {layer_name} found scanning {self.name} at address {address:x}" ) if len(data) > scanner.chunk_size + scanner.overlap: @@ -721,7 +718,7 @@ def check_cycles(self) -> None: raise NotImplementedError("Cycle checking has not yet been implemented") -class DummyProgress(object): +class DummyProgress: """A class to emulate Multiprocessing/threading Value objects.""" def __init__(self) -> None: diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 51d25510d2..23c90b13b8 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -374,6 +374,7 @@ def __getattr__(self, attr: str) -> Any: f"{self.__class__.__name__} object has no attribute {attr}" ) + @abc.abstractmethod def __call__( self, context: "interfaces.context.ContextInterface", diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 697e4cdc3d..f763815a67 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -46,7 +46,7 @@ def preferred_filename(self): def preferred_filename(self, filename: str): """Sets the preferred filename""" if self.closed: - raise IOError("FileHandler name cannot be changed once closed") + raise OSError("FileHandler name cannot be changed once closed") if not isinstance(filename, str): raise TypeError("FileHandler preferred filenames must be strings") if os.path.sep in filename: @@ -59,14 +59,14 @@ def close(self): @staticmethod def sanitize_filename(filename: str) -> str: - """Sanititizes the filename to ensure only a specific whitelist of characters is allowed through""" - allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]{}!$%^:#~?<>,|" + """Sanititizes the filename to ensure only a specific allow list of characters is allowed through""" + allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]{}!$%^#~," result = "" for char in filename: if char in allowed: result += char else: - result += "?" + result += "_" # change unwanted chars to an underscore return result def __enter__(self): diff --git a/volatility3/framework/interfaces/renderers.py b/volatility3/framework/interfaces/renderers.py index b13de18349..e26164ee79 100644 --- a/volatility3/framework/interfaces/renderers.py +++ b/volatility3/framework/interfaces/renderers.py @@ -26,7 +26,11 @@ Union, ) -Column = NamedTuple("Column", [("name", str), ("type", Any)]) + +class Column(NamedTuple): + name: str + type: Any + RenderOption = Any @@ -98,11 +102,11 @@ def path_changed(self, path: str, added: bool = False) -> None: """ -class BaseAbsentValue(object): +class BaseAbsentValue: """Class that represents values which are not present for some reason.""" -class Disassembly(object): +class Disassembly: """A class to indicate that the bytes provided should be disassembled (based on the architecture)""" @@ -137,7 +141,7 @@ def __init__( VisitorSignature = Callable[[TreeNode, _Type], _Type] -class TreeGrid(object, metaclass=ABCMeta): +class TreeGrid(metaclass=ABCMeta): """Class providing the interface for a TreeGrid (which contains TreeNodes) The structure of a TreeGrid is designed to maintain the structure of the tree in a single object. @@ -179,7 +183,7 @@ def sanitize_name(text: str) -> str: @abstractmethod def populate( self, - function: VisitorSignature = None, + function: Optional[VisitorSignature] = None, initial_accumulator: Any = None, fail_on_errors: bool = True, ) -> Optional[Exception]: @@ -231,7 +235,7 @@ def visit( node: Optional[TreeNode], function: VisitorSignature, initial_accumulator: _Type, - sort_key: ColumnSortKey = None, + sort_key: Optional[ColumnSortKey] = None, ) -> None: """Visits all the nodes in a tree, calling function on each one. diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index b645f5cd12..b8712e38d8 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -250,13 +250,13 @@ def get_symbols_by_location(self, offset: int, size: int = 0) -> Iterable[str]: def clear_symbol_cache(self) -> None: """Clears the symbol cache of this symbol table.""" - pass class SymbolSpaceInterface(collections.abc.Mapping): """An interface for the container that holds all the symbol-containing tables for use within a context.""" + @abstractmethod def free_table_name(self, prefix: str = "layer") -> str: """Returns an unused table name to ensure no collision occurs when inserting a symbol table.""" @@ -378,7 +378,7 @@ def enumerations(self) -> Iterable[str]: return [] -class MetadataInterface(object): +class MetadataInterface: """Interface for accessing metadata stored within a symbol table.""" def __init__(self, json_data: Dict) -> None: diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 3cfc0a25bd..a5b25d178b 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -1,7 +1,6 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import contextlib import logging import struct from typing import Tuple, Optional @@ -138,7 +137,7 @@ def _load_segments(self) -> None: ulong_bitmap_array = summary_header.get_buffer_long() # outer_index points to a 32 bits array inside a list of arrays, # each bit indicating a page mapping state - for outer_index in range(0, ulong_bitmap_array.vol.count): + for outer_index in range(ulong_bitmap_array.vol.count): ulong_bitmap = ulong_bitmap_array[outer_index] # All pages in this 32 bits array are mapped (speedup iteration process) if ulong_bitmap == 0xFFFFFFFF: @@ -166,7 +165,7 @@ def _load_segments(self) -> None: seg_first_bit = None # Some pages in this 32 bits array are mapped and some aren't else: - for inner_bit_position in range(0, 32): + for inner_bit_position in range(32): current_bit = outer_index * 32 + inner_bit_position page_mapped = ulong_bitmap & (1 << inner_bit_position) if page_mapped: @@ -220,9 +219,7 @@ def _load_segments(self) -> None: for idx, (start_position, mapped_offset, length, _) in enumerate(segments): vollog.log( constants.LOGLEVEL_VVVV, - "Segment {}: Position {:#x} Offset {:#x} Length {:#x}".format( - idx, start_position, mapped_offset, length - ), + f"Segment {idx}: Position {start_position:#x} Offset {mapped_offset:#x} Length {length:#x}", ) self._segments = segments diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 2b0df53728..7c2c72ac17 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -22,6 +22,16 @@ class Intel(linear.LinearlyMappedLayer): """Translation Layer for the Intel IA32 memory mapping.""" + _PAGE_BIT_PRESENT = 0 + _PAGE_BIT_PSE = 7 # Page Size Extension: 4 MB (or 2MB) page + _PAGE_BIT_PROTNONE = 8 + _PAGE_BIT_PAT_LARGE = 12 # 2MB or 1GB pages + + _PAGE_PRESENT = 1 << _PAGE_BIT_PRESENT + _PAGE_PSE = 1 << _PAGE_BIT_PSE + _PAGE_PROTNONE = 1 << _PAGE_BIT_PROTNONE + _PAGE_PAT_LARGE = 1 << _PAGE_BIT_PAT_LARGE + _entry_format = " int: """Page shift for the intel memory layers.""" return cls._page_size_in_bits @classproperty - @functools.lru_cache() + @functools.lru_cache def page_size(cls) -> int: """Page size for the intel memory layers. @@ -83,25 +91,25 @@ def page_size(cls) -> int: return 1 << cls._page_size_in_bits @classproperty - @functools.lru_cache() + @functools.lru_cache def page_mask(cls) -> int: """Page mask for the intel memory layers.""" return ~(cls.page_size - 1) @classproperty - @functools.lru_cache() + @functools.lru_cache def bits_per_register(cls) -> int: """Returns the bits_per_register to determine the range of an IntelTranslationLayer.""" return cls._bits_per_register @classproperty - @functools.lru_cache() + @functools.lru_cache def minimum_address(cls) -> int: return 0 @classproperty - @functools.lru_cache() + @functools.lru_cache def maximum_address(cls) -> int: return (1 << cls._maxvirtaddr) - 1 @@ -115,7 +123,6 @@ def _mask(value: int, high_bit: int, low_bit: int) -> int: high_mask = (1 << (high_bit + 1)) - 1 low_mask = (1 << low_bit) - 1 mask = high_mask ^ low_mask - # print(high_bit, low_bit, bin(mask), bin(value)) return value & mask @staticmethod @@ -137,7 +144,7 @@ def canonicalize(self, addr: int) -> int: return self._mask(addr, self._maxvirtaddr, 0) + self._canonical_prefix def decanonicalize(self, addr: int) -> int: - """Removes canonicalization to ensure an adress fits within the correct range if it has been canonicalized + """Removes canonicalization to ensure an address fits within the correct range if it has been canonicalized This will produce an address outside the range if the canonicalization is incorrect """ @@ -163,12 +170,17 @@ def _translate(self, offset: int) -> Tuple[int, int, str]: entry, f"Page Fault at entry {hex(entry)} in page entry", ) - page = self._mask(entry, self._maxphyaddr - 1, position + 1) | self._mask( - offset, position, 0 - ) + + pfn = self._pte_pfn(entry) + page_offset = self._mask(offset, position, 0) + page = pfn << self.page_shift | page_offset return page, 1 << (position + 1), self._base_layer + def _pte_pfn(self, entry: int) -> int: + """Extracts the page frame number (PFN) from the page table entry (PTE) entry""" + return self._mask(entry, self._maxphyaddr - 1, 0) >> self.page_shift + def _translate_entry(self, offset: int) -> Tuple[int, int]: """Translates a specific offset based on paging tables. @@ -203,10 +215,10 @@ def _translate_entry(self, offset: int) -> Tuple[int, int]: "Page Fault at entry " + hex(entry) + " in table " + name, ) # Check if we're a large page - if large_page and (entry & (1 << 7)): + if large_page and (entry & self._PAGE_PSE): # Mask off the PAT bit - if entry & (1 << 12): - entry -= 1 << 12 + if entry & self._PAGE_PAT_LARGE: + entry -= self._PAGE_PAT_LARGE # We're a large page, the rest is finished below # If we want to implement PSE-36, it would need to be done here break @@ -239,12 +251,7 @@ def _translate_entry(self, offset: int) -> Tuple[int, int]: if INTEL_TRANSLATION_DEBUGGING: vollog.log( constants.LOGLEVEL_VVVV, - "Entry {} at index {} gives data {} as {}".format( - hex(entry), - hex(index), - hex(struct.unpack(self._entry_format, entry_data)[0]), - name, - ), + f"Entry {hex(entry)} at index {hex(index)} gives data {hex(struct.unpack(self._entry_format, entry_data)[0])} as {name}", ) # Read out the new entry from memory @@ -252,7 +259,7 @@ def _translate_entry(self, offset: int) -> Tuple[int, int]: return entry, position - @functools.lru_cache(1025) + @functools.lru_cache(maxsize=1025) def _get_valid_table(self, base_address: int) -> Optional[bytes]: """Extracts the table, validates it and returns it if it's valid.""" table = self._context.layers.read( @@ -501,3 +508,85 @@ class WindowsIntel32e(WindowsMixin, Intel32e): def _translate(self, offset: int) -> Tuple[int, int, str]: return self._translate_swap(self, offset, self._bits_per_register // 2) + + +class LinuxMixin(Intel): + @functools.cached_property + def _register_mask(self) -> int: + return (1 << self._bits_per_register) - 1 + + @functools.cached_property + def _physical_mask(self) -> int: + # From kernels 4.18 the physical mask is dynamic: See AMD SME, Intel Multi-Key Total + # Memory Encryption and CONFIG_DYNAMIC_PHYSICAL_MASK: 94d49eb30e854c84d1319095b5dd0405a7da9362 + physical_mask = (1 << self._maxphyaddr) - 1 + # TODO: Come back once SME support is available in the framework + return physical_mask + + @functools.cached_property + def page_mask(self) -> int: + # Note that within the Intel class it's a class method. However, since it uses + # complement operations and we are working in Python, it would be more careful to + # limit it to the architecture's pointer size. + return ~(self.page_size - 1) & self._register_mask + + @functools.cached_property + def _physical_page_mask(self) -> int: + return self.page_mask & self._physical_mask + + @functools.cached_property + def _pte_pfn_mask(self) -> int: + return self._physical_page_mask + + @functools.cached_property + def _pte_flags_mask(self) -> int: + return ~self._pte_pfn_mask & self._register_mask + + def _pte_flags(self, pte) -> int: + return pte & self._pte_flags_mask + + def _is_pte_present(self, entry: int) -> bool: + return ( + self._pte_flags(entry) & (self._PAGE_PRESENT | self._PAGE_PROTNONE) + ) != 0 + + def _page_is_valid(self, entry: int) -> bool: + # Overrides the Intel static method with the Linux-specific implementation + return self._is_pte_present(entry) + + def _pte_needs_invert(self, entry) -> bool: + # Entries that were set to PROT_NONE (PAGE_PRESENT) are inverted + # A clear PTE shouldn't be inverted. See f19f5c4 + return entry and not (entry & self._PAGE_PRESENT) + + def _protnone_mask(self, entry: int) -> int: + """Gets a mask to XOR with the page table entry to get the correct PFN""" + return self._register_mask if self._pte_needs_invert(entry) else 0 + + def _pte_pfn(self, entry: int) -> int: + """Extracts the page frame number from the page table entry""" + pfn = entry ^ self._protnone_mask(entry) + return (pfn & self._pte_pfn_mask) >> self.page_shift + + +class LinuxIntel(LinuxMixin, Intel): + pass + + +class LinuxIntelPAE(LinuxMixin, IntelPAE): + pass + + +class LinuxIntel32e(LinuxMixin, Intel32e): + # In the Linux kernel, the __PHYSICAL_MASK_SHIFT is a mask used to extract the + # physical address from a PTE. In Volatility3, this is referred to as _maxphyaddr. + # + # Until kernel version 4.17, Linux x86-64 used a 46-bit mask. With commit + # b83ce5ee91471d19c403ff91227204fb37c95fb2, this was extended to 52 bits, + # applying to both 4 and 5-level page tables. + # + # We initially used 52 bits for all Intel 64-bit systems, but this produced incorrect + # results for PROT_NONE pages. Since the mask value is defined by a preprocessor macro, + # it's difficult to detect the exact bit shift used in the current kernel. + # Using 46 bits has proven reliable for our use case, as seen in tools like crashtool. + _maxphyaddr = 46 diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py index 542fd6ca2d..eeede1673b 100644 --- a/volatility3/framework/layers/leechcore.py +++ b/volatility3/framework/layers/leechcore.py @@ -48,7 +48,7 @@ def handle(self): try: self._handle = leechcorepyc.LeechCore(self._device) except TypeError: - raise IOError(f"Unable to open LeechCore device {self._device}") + raise OSError(f"Unable to open LeechCore device {self._device}") return self._handle def fileno(self): diff --git a/volatility3/framework/layers/msf.py b/volatility3/framework/layers/msf.py index 8d84a774b8..03e144e25d 100644 --- a/volatility3/framework/layers/msf.py +++ b/volatility3/framework/layers/msf.py @@ -225,7 +225,7 @@ def mapping( returned = 0 page_size = self._pdb_layer.page_size while length > 0: - page = math.floor((offset + returned) / page_size) + page = (offset + returned) // page_size page_position = (offset + returned) % page_size chunk_size = min(page_size - page_position, length) if page >= self._pages_len: diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index ff483291c9..a8127e9545 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -236,7 +236,7 @@ def _load_segments(self): if self._architecture is None: vollog.log( constants.LOGLEVEL_VV, - f"QEVM architecture could not be determined", + "QEVM architecture could not be determined", ) # Once all segments have been read, determine the PCI hole if any diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 6098328867..cc364ad50f 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -156,9 +156,7 @@ def get_node(self, cell_offset: int) -> "objects.StructType": else: # It doesn't matter that we use KeyNode, we're just after the first two bytes vollog.debug( - "Unknown Signature {} (0x{:x}) at offset {}".format( - signature, cell.u.KeyNode.Signature, cell_offset - ) + f"Unknown Signature {signature} (0x{cell.u.KeyNode.Signature:x}) at offset {cell_offset}" ) return cell @@ -178,9 +176,7 @@ def get_key( if not root_node.vol.type_name.endswith(constants.BANG + "_CM_KEY_NODE"): raise RegistryFormatException( self.name, - "Encountered {} instead of _CM_KEY_NODE".format( - root_node.vol.type_name - ), + f"Encountered {root_node.vol.type_name} instead of _CM_KEY_NODE", ) node_key = [root_node] if key.endswith("\\"): diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index 2dba7caa8f..236d256f1e 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -29,7 +29,7 @@ try: # Import so that the handler is found by the framework.class_subclasses callc - import smb.SMBHandler # lgtm [py/unused-import] + from smb import SMBHandler as SMBHandler # lgtm [py/unused-import] except ImportError: # If we fail to import this, it means that SMB handling won't be available pass @@ -57,7 +57,7 @@ def close(): return new_fp -class ResourceAccessor(object): +class ResourceAccessor: """Object for opening URLs as files (downloading locally first if necessary)""" diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index dd8dc46be1..be9f1c39a3 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -5,7 +5,7 @@ from typing import Generator, List, Tuple, Dict, Optional from volatility3.framework.interfaces import layers -from volatility3.framework.layers.scanners import multiregexp +from volatility3.framework.layers.scanners import multiregexp as multiregexp class BytesScanner(layers.ScannerInterface): @@ -72,7 +72,7 @@ def _process_pattern(self, value: bytes) -> None: return None for char in value: - trie[char] = trie.get(char, {}) + trie.setdefault(char, {}) trie = trie[char] # Mark the end of a string diff --git a/volatility3/framework/layers/scanners/multiregexp.py b/volatility3/framework/layers/scanners/multiregexp.py index be3581f052..9831a9d8ed 100644 --- a/volatility3/framework/layers/scanners/multiregexp.py +++ b/volatility3/framework/layers/scanners/multiregexp.py @@ -6,7 +6,7 @@ from typing import Generator, List, Tuple -class MultiRegexp(object): +class MultiRegexp: """Algorithm for multi-string matching.""" def __init__(self) -> None: diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index e7aa0ccecd..c0a5e1a7df 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -54,6 +54,7 @@ def _load_segments(self) -> None: segments = [] self._segment_headers = [] + segment_names = None for sindex in range(ehdr.e_shnum): shdr = self.context.object( diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index b65277067a..869d4dae62 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -35,13 +35,13 @@ def convert_data_to_value( data_format: DataFormatInfo, ) -> TUnion[int, float, bytes, str, bool]: """Converts a series of bytes to a particular type of value.""" - if struct_type == int: + if struct_type is int: return int.from_bytes( data, byteorder=data_format.byteorder, signed=data_format.signed ) - if struct_type == bool: + if struct_type is bool: struct_format = "?" - elif struct_type == float: + elif struct_type is float: float_vals = "zzezfzzzd" if ( data_format.length > len(float_vals) @@ -70,7 +70,7 @@ def convert_value_to_data( f"Written value is not of the correct type for {struct_type.__name__}" ) - if struct_type == int and isinstance(value, int): + if struct_type is int and isinstance(value, int): # Doubling up on the isinstance is for mypy return int.to_bytes( value, @@ -78,9 +78,9 @@ def convert_value_to_data( byteorder=data_format.byteorder, signed=data_format.signed, ) - if struct_type == bool: + if struct_type is bool: struct_format = "?" - elif struct_type == float: + elif struct_type is float: float_vals = "zzezfzzzd" if ( data_format.length > len(float_vals) @@ -152,7 +152,7 @@ def __new__( type_name: str, object_info: interfaces.objects.ObjectInformation, data_format: DataFormatInfo, - new_value: TUnion[int, float, bool, bytes, str] = None, + new_value: Optional[TUnion[int, float, bool, bytes, str]] = None, **kwargs, ) -> "PrimitiveObject": """Creates the appropriate class and returns it so that the native type @@ -601,7 +601,7 @@ def _generate_inverse_choices(cls, choices: Dict[str, int]) -> Dict[int, str]: inverse_choices[v] = k return inverse_choices - def lookup(self, value: int = None) -> str: + def lookup(self, value: Optional[int] = None) -> str: """Looks up an individual value and returns the associated name. If multiple identifiers map to the same value, the first matching identifier will be returned @@ -690,7 +690,7 @@ def __init__( type_name: str, object_info: interfaces.objects.ObjectInformation, count: int = 0, - subtype: templates.ObjectTemplate = None, + subtype: Optional[templates.ObjectTemplate] = None, ) -> None: super().__init__(context=context, type_name=type_name, object_info=object_info) self._vol["count"] = count diff --git a/volatility3/framework/objects/utility.py b/volatility3/framework/objects/utility.py index 8aa527cdbe..b241ed56a9 100644 --- a/volatility3/framework/objects/utility.py +++ b/volatility3/framework/objects/utility.py @@ -22,8 +22,8 @@ def bswap_32(value: int) -> int: def bswap_64(value: int) -> int: - low = bswap_32((value >> 32)) - high = bswap_32((value & 0xFFFFFFFF)) + low = bswap_32(value >> 32) + high = bswap_32(value & 0xFFFFFFFF) return ((high << 32) | low) & 0xFFFFFFFFFFFFFFFF diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 4f07bd5a8d..78e78fb9e2 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -7,6 +7,7 @@ import pathlib import zipfile from typing import Generator, List +from importlib.util import find_spec from volatility3 import schemas, symbols from volatility3.framework import constants, interfaces, renderers @@ -96,16 +97,12 @@ def _generator(self): if filter_item in isf_file: filtered_list.append(isf_file) - try: - import jsonschema - - if not self.config["validate"]: - raise ImportError # Act as if we couldn't import if validation is turned off + if find_spec("jsonschema") and self.config["validate"]: def check_valid(data): return "True" if schemas.validate(data, True) else "False" - except ImportError: + else: def check_valid(data): return "Unknown" diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 24149a390b..10e7a7a726 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -119,7 +119,7 @@ def _generator(self): # Update the filename, which may have changed if a file # with the same name already existed. output_name = file_handle.preferred_filename - except IOError as excp: + except OSError as excp: yield 0, (f"Layer cannot be written to {output_name}: {excp}",) yield 0, (f"Layer has been written to {output_name}",) diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index ce4567ca60..056e3cd511 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -22,6 +22,7 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): """Recovers bash command history from memory.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -32,7 +33,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/boottime.py b/volatility3/framework/plugins/linux/boottime.py index 8f63ee7f87..c57bdd65a2 100644 --- a/volatility3/framework/plugins/linux/boottime.py +++ b/volatility3/framework/plugins/linux/boottime.py @@ -15,8 +15,7 @@ class Boottime(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface) """Shows the time the system was started""" _required_framework_version = (2, 11, 0) - - _version = (1, 0, 0) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -27,7 +26,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 3, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index bfdb69aba2..1d0c60c11e 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -49,9 +49,8 @@ def astuple(self) -> Tuple: class Capabilities(plugins.PluginInterface): """Lists process capabilities""" - _required_framework_version = (2, 0, 0) - - _version = (1, 0, 0) + _required_framework_version = (2, 13, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -62,7 +61,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pids", @@ -87,7 +86,7 @@ def _check_capabilities_support( try: kernel_cap_last_cap = vmlinux.object_from_symbol(symbol_name="cap_last_cap") except exceptions.SymbolError: - # It should be a kernel < 3.2 + # It should be a kernel < 3.2 See 73efc0394e148d0e15583e13712637831f926720 return None vol2_last_cap = extensions.kernel_cap_struct.get_last_cap_value() @@ -137,7 +136,7 @@ def get_task_capabilities( comm=utility.array_to_string(task.comm), pid=int(task.pid), tgid=int(task.tgid), - ppid=int(task.parent.pid), + ppid=int(task.get_parent_pid()), euid=int(task.cred.euid), ) diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index b7f73c3eb0..96f77ce4d5 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -12,8 +12,7 @@ class Check_creds(interfaces.plugins.PluginInterface): """Checks if any processes are sharing credential structures""" _required_framework_version = (2, 0, 0) - - _version = (2, 0, 0) + _version = (2, 0, 2) @classmethod def get_requirements(cls): @@ -24,7 +23,7 @@ def get_requirements(cls): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), ] @@ -56,7 +55,7 @@ def _generator(self): for cred_addr, pids in creds.items(): if len(pids) > 1: - pid_str = ", ".join([str(pid) for pid in pids]) + pid_str = ", ".join(str(pid) for pid in pids) fields = [ format_hints.Hex(cred_addr), diff --git a/volatility3/framework/plugins/linux/check_idt.py b/volatility3/framework/plugins/linux/check_idt.py index cc3a08933f..07582e2c15 100644 --- a/volatility3/framework/plugins/linux/check_idt.py +++ b/volatility3/framework/plugins/linux/check_idt.py @@ -53,7 +53,7 @@ def _generator(self): address_mask = self.context.layers[vmlinux.layer_name].address_mask # hw handlers + system call - check_idxs = list(range(0, 20)) + [128] + check_idxs = list(range(20)) + [128] if is_32bit: if vmlinux.has_type("gate_struct"): diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index b6634d612d..3537a9fa12 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -103,7 +103,7 @@ def _get_table_info_disassembly(self, ptr_sz, vmlinux): try: func_addr = vmlinux.get_symbol(syscall_entry_func).address - except exceptions.SymbolError as e: + except exceptions.SymbolError: # if we can't find the disassemble function then bail and rely on a different method return 0 diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 22e39d1276..2fd7409417 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -25,7 +25,7 @@ class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 1) + _version = (2, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -36,7 +36,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/envars.py b/volatility3/framework/plugins/linux/envars.py index 5cbf0f5020..8cdbfe493b 100644 --- a/volatility3/framework/plugins/linux/envars.py +++ b/volatility3/framework/plugins/linux/envars.py @@ -3,8 +3,9 @@ # import logging +from typing import Iterable, Tuple -from volatility3.framework import exceptions, renderers +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -16,7 +17,8 @@ class Envars(plugins.PluginInterface): """Lists processes with their environment variables""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls): @@ -28,7 +30,7 @@ def get_requirements(cls): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", @@ -38,84 +40,98 @@ def get_requirements(cls): ), ] + @staticmethod + def get_task_env_variables( + context: interfaces.context.ContextInterface, + task: interfaces.objects.ObjectInterface, + env_area_max_size: int = 8192, + ) -> Iterable[Tuple[str, str]]: + """Yields environment variables for a given task. + + Args: + context: The plugin's operational context. + task: The task object from which to extract environment variables. + env_area_max_size: Maximum allowable size for the environment variables area. + Tasks exceeding this size will be skipped. Default is 8192. + + Yields: + Tuples of (key, value) representing each environment variable. + """ + + task_name = utility.array_to_string(task.comm) + task_pid = task.pid + env_start = task.mm.env_start + env_end = task.mm.env_end + env_area_size = env_end - env_start + if not (0 < env_area_size <= env_area_max_size): + vollog.debug( + f"Task {task_pid} {task_name} appears to have environment variables of size " + f"{env_area_size} bytes which fails the sanity checking, will not extract " + "any envars." + ) + return None + + # Get process layer to read envars from + proc_layer_name = task.add_process_layer() + if proc_layer_name is None: + return None + proc_layer = context.layers[proc_layer_name] + + # Ensure the entire buffer is readable to prevent relying on exception handling + if not proc_layer.is_valid(env_start, env_area_size): + # Not mapped / swapped out + vollog.debug( + f"Unable to read environment variables for {task_pid} {task_name} starting at " + f" virtual address 0x{env_start:x} for {env_area_size} bytes, will not " + "extract any envars." + ) + return None + + # Read the full task environment variable buffer. + envar_data = proc_layer.read(env_start, env_area_size) + + # Parse envar data, envars are null terminated, keys and values are separated by '=' + envar_data = envar_data.rstrip(b"\x00") + for envar_pair in envar_data.split(b"\x00"): + try: + env_key, env_value = envar_pair.decode().split("=", 1) + except ValueError: + # Some legitimate programs, like 'avahi-daemon', avoid reallocating the args + # and instead exploit the fact that the environment variables area is contiguous + # to the args. This allows them to include a longer process name in the listing, + # causing overwrites and incorrect results. In such cases, it's better to abort + # the current task rather than displaying misleading or incorrect output. + break + + yield env_key, env_value + def _generator(self, tasks): """Generates a listing of processes along with environment variables""" # walk the process list and return the envars for task in tasks: - pid = task.pid - - # get process name as string - name = utility.array_to_string(task.comm) - - # try and get task parent - try: - ppid = task.parent.pid - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read parent pid for task {pid} {name}, setting ppid to 0." - ) - ppid = 0 - - # kernel threads never have an mm as they do not have userland mappings - try: - mm = task.mm - except exceptions.InvalidAddressException: - # no mm so cannot get envars - vollog.debug( - f"Unable to access mm for task {pid} {name} it is likely a kernel thread, will not extract any envars." - ) - mm = None + if task.is_kernel_thread: continue - # if mm exists attempt to get envars - if mm: - # get process layer to read envars from - proc_layer_name = task.add_process_layer() - if proc_layer_name is None: - vollog.debug( - f"Unable to construct process layer for task {pid} {name}, will not extract any envars." - ) - continue - proc_layer = self.context.layers[proc_layer_name] - - # get the size of the envars with sanity checking - envars_size = task.mm.env_end - task.mm.env_start - if not (0 < envars_size <= 8192): - vollog.debug( - f"Task {pid} {name} appears to have envars of size {envars_size} bytes which fails the sanity checking, will not extract any envars." - ) - continue - - # attempt to read all envars data - try: - envar_data = proc_layer.read(task.mm.env_start, envars_size) - except exceptions.InvalidAddressException: - vollog.debug( - f"Unable to read full envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)} for {envars_size} bytes, will not extract any envars." - ) - continue - - # parse envar data, envars are null terminated, keys and values are separated by '=' - envar_data = envar_data.rstrip(b"\x00") - for envar_pair in envar_data.split(b"\x00"): - try: - key, value = envar_pair.decode().split("=", 1) - except ValueError: - vollog.debug( - f"Unable to extract envars for {pid} {name} starting at virtual offset {hex(task.mm.env_start)}, they don't appear to be '=' separated" - ) - continue - yield (0, (pid, ppid, name, key, value)) + task_pid = task.pid + task_name = utility.array_to_string(task.comm) + task_ppid = task.get_parent_pid() + + for env_key, env_value in self.get_task_env_variables(self.context, task): + yield (0, (task_pid, task_ppid, task_name, env_key, env_value)) def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) - - return renderers.TreeGrid( - [("PID", int), ("PPID", int), ("COMM", str), ("KEY", str), ("VALUE", str)], - self._generator( - pslist.PsList.list_tasks( - self.context, self.config["kernel"], filter_func=filter_func - ) - ), + tasks = pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func ) + + headers = [ + ("PID", int), + ("PPID", int), + ("COMM", str), + ("KEY", str), + ("VALUE", str), + ] + + return renderers.TreeGrid(headers, self._generator(tasks)) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index e26d695438..d66e3b9cae 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -149,7 +149,7 @@ def nsec_to_sec_str(self, nsec: int) -> str: # This might seem insignificant but it could cause some issues # when compared with userland tool results or when used in # timelines. - return "%lu.%06lu" % (nsec / 1000000000, (nsec % 1000000000) / 1000) + return f"{nsec / 1000000000:lu}.{(nsec % 1000000000) / 1000:06lu}" def get_timestamp_in_sec_str(self, obj) -> str: # obj could be log, printk_log or printk_info @@ -166,7 +166,7 @@ def get_caller(self, obj): def get_caller_text(self, caller_id): caller_name = "CPU" if caller_id & 0x80000000 else "Task" - caller = "%s(%u)" % (caller_name, caller_id & ~0x80000000) + caller = f"{caller_name}({caller_id & ~0x80000000:u})" return caller def get_prefix(self, obj) -> Tuple[int, int, str, str]: diff --git a/volatility3/framework/plugins/linux/kthreads.py b/volatility3/framework/plugins/linux/kthreads.py index 2e51b4688c..40e992069f 100644 --- a/volatility3/framework/plugins/linux/kthreads.py +++ b/volatility3/framework/plugins/linux/kthreads.py @@ -20,8 +20,7 @@ class Kthreads(plugins.PluginInterface): """Enumerates kthread functions""" _required_framework_version = (2, 11, 0) - - _version = (1, 0, 0) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -35,7 +34,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 3, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.PluginRequirement( name="lsmod", plugin=lsmod.Lsmod, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/library_list.py b/volatility3/framework/plugins/linux/library_list.py index 062ed078eb..e251b5689d 100644 --- a/volatility3/framework/plugins/linux/library_list.py +++ b/volatility3/framework/plugins/linux/library_list.py @@ -21,8 +21,7 @@ class LibraryList(interfaces.plugins.PluginInterface): """Enumerate libraries loaded into processes""" _required_framework_version = (2, 0, 0) - - _version = (1, 0, 0) + _version = (1, 0, 2) @classmethod def get_requirements(cls): @@ -33,7 +32,7 @@ def get_requirements(cls): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pids", diff --git a/volatility3/framework/plugins/linux/lsmod.py b/volatility3/framework/plugins/linux/lsmod.py index a65b0d00bc..49e990e933 100644 --- a/volatility3/framework/plugins/linux/lsmod.py +++ b/volatility3/framework/plugins/linux/lsmod.py @@ -54,8 +54,7 @@ def list_modules( table_name = modules.vol.type_name.split(constants.BANG)[0] - for module in modules.to_list(table_name + constants.BANG + "module", "list"): - yield module + yield from modules.to_list(table_name + constants.BANG + "module", "list") def _generator(self): try: diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 42b447dfb9..daa8e5a3d3 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -110,7 +110,7 @@ class Lsof(plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists open files for each processes.""" _required_framework_version = (2, 0, 0) - _version = (2, 0, 0) + _version = (2, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -121,7 +121,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 18f3dcd56b..e45688e975 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -4,7 +4,7 @@ from typing import List import logging -from volatility3.framework import constants, interfaces +from volatility3.framework import interfaces from volatility3.framework import renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility @@ -18,6 +18,7 @@ class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" _required_framework_version = (2, 0, 0) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -28,7 +29,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", diff --git a/volatility3/framework/plugins/linux/mountinfo.py b/volatility3/framework/plugins/linux/mountinfo.py index 2499f009e2..b4f80e4f5f 100644 --- a/volatility3/framework/plugins/linux/mountinfo.py +++ b/volatility3/framework/plugins/linux/mountinfo.py @@ -36,8 +36,7 @@ class MountInfo(plugins.PluginInterface): """Lists mount points on processes mount namespaces""" _required_framework_version = (2, 2, 0) - - _version = (1, 2, 1) + _version = (1, 2, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -48,7 +47,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) diff --git a/volatility3/framework/plugins/linux/pagecache.py b/volatility3/framework/plugins/linux/pagecache.py index 005fc9acc0..3822685158 100644 --- a/volatility3/framework/plugins/linux/pagecache.py +++ b/volatility3/framework/plugins/linux/pagecache.py @@ -389,7 +389,7 @@ class InodePages(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -412,9 +412,10 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] description="Inode address", optional=True, ), - requirements.StringRequirement( + requirements.BooleanRequirement( name="dump", - description="Output file path", + description="Extract inode content", + default=False, optional=True, ), ] @@ -436,7 +437,7 @@ def write_inode_content_to_file( """ if not inode.is_reg: vollog.error("The inode is not a regular file") - return + return None # By using truncate/seek, provided the filesystem supports it, a sparse file will be # created, saving both disk space and I/O time. @@ -461,7 +462,7 @@ def write_inode_content_to_file( f.seek(current_fp) f.write(page_bytes) - except IOError as e: + except OSError as e: vollog.error("Unable to write to file (%s): %s", filename, e) def _generator(self): @@ -471,7 +472,7 @@ def _generator(self): if self.config["inode"] and self.config["find"]: vollog.error("Cannot use --inode and --find simultaneously") - return + return None if self.config["find"]: inodes_iter = Files.get_inodes( @@ -482,20 +483,23 @@ def _generator(self): if inode_in.path == self.config["find"]: inode = inode_in.inode break # Only the first match + else: + vollog.error("Unable to find inode with path %s", self.config["find"]) + return None elif self.config["inode"]: inode = vmlinux.object("inode", self.config["inode"], absolute=True) else: vollog.error("You must use either --inode or --find") - return + return None if not inode.is_valid(): vollog.error("Invalid inode at 0x%x", inode.vol.offset) - return + return None if not inode.is_reg: vollog.error("The inode is not a regular file") - return + return None inode_size = inode.i_size for page_obj in inode.get_pages(): @@ -519,9 +523,13 @@ def _generator(self): yield 0, fields if self.config["dump"]: - filename = self.config["dump"] - vollog.info("[*] Writing inode at 0x%x to '%s'", inode.vol.offset, filename) - self.write_inode_content_to_file(inode, filename, self.open, vmlinux_layer) + open_method = self.open + inode_address = inode.vol.offset + filename = open_method.sanitize_filename(f"inode_0x{inode_address:x}.dmp") + vollog.info("[*] Writing inode at 0x%x to '%s'", inode_address, filename) + self.write_inode_content_to_file( + inode, filename, open_method, vmlinux_layer + ) def run(self): headers = [ diff --git a/volatility3/framework/plugins/linux/pidhashtable.py b/volatility3/framework/plugins/linux/pidhashtable.py index edafe97e05..060b3928e0 100644 --- a/volatility3/framework/plugins/linux/pidhashtable.py +++ b/volatility3/framework/plugins/linux/pidhashtable.py @@ -3,7 +3,7 @@ # import logging -from typing import List +from typing import List, Iterable from volatility3.framework import renderers, interfaces, constants from volatility3.framework.symbols import linux @@ -19,8 +19,7 @@ class PIDHashTable(plugins.PluginInterface): """Enumerates processes through the PID hash table""" _required_framework_version = (2, 0, 0) - - _version = (1, 0, 1) + _version = (1, 0, 3) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -31,7 +30,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 1, 0) @@ -219,7 +218,7 @@ def _determine_pid_func(self): return None - def get_tasks(self) -> interfaces.objects.ObjectInterface: + def get_tasks(self) -> Iterable[interfaces.objects.ObjectInterface]: """Enumerates processes through the PID hash table Yields: @@ -232,14 +231,16 @@ def get_tasks(self) -> interfaces.objects.ObjectInterface: yield from sorted(pid_func(), key=lambda t: (t.tgid, t.pid)) - def _generator( - self, decorate_comm: bool = False - ) -> interfaces.objects.ObjectInterface: + def _generator(self, decorate_comm: bool = False): for task in self.get_tasks(): - offset, pid, tid, ppid, name = pslist.PsList.get_task_fields( - task, decorate_comm + task_fields = pslist.PsList.get_task_fields(task, decorate_comm) + fields = ( + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, ) - fields = format_hints.Hex(offset), pid, tid, ppid, name yield 0, fields def run(self): diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index e7d38b1072..441c6bc930 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -21,7 +21,8 @@ class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 2) + MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb @classmethod @@ -34,7 +35,7 @@ def get_requirements(cls): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", @@ -124,9 +125,7 @@ def vma_dump( proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - pid, excp.invalid_address, excp.layer_name - ) + f"Process {pid}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None vm_size = vm_end - vm_start @@ -164,7 +163,9 @@ def _generator(self, tasks): address_list = self.config.get("address", None) if not address_list: # do not filter as no address_list was supplied - vma_filter_func = lambda _: True + def vma_filter_func(_): + return True + else: # filter for any vm_start that matches the supplied address config def vma_filter_function(x: interfaces.objects.ObjectInterface) -> bool: diff --git a/volatility3/framework/plugins/linux/psaux.py b/volatility3/framework/plugins/linux/psaux.py index a4a23498fe..a544c9d67c 100644 --- a/volatility3/framework/plugins/linux/psaux.py +++ b/volatility3/framework/plugins/linux/psaux.py @@ -14,7 +14,8 @@ class PsAux(plugins.PluginInterface): """Lists processes with their command line arguments""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls): @@ -26,7 +27,7 @@ def get_requirements(cls): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", @@ -97,14 +98,8 @@ def _generator(self, tasks): # walk the process list and report the arguments for task in tasks: pid = task.pid - - try: - ppid = task.parent.pid - except exceptions.InvalidAddressException: - ppid = 0 - + ppid = task.get_parent_pid() name = utility.array_to_string(task.comm) - args = self._get_command_line_args(task, name) yield (0, (pid, ppid, name, args)) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index b05d69c7ac..37cf000fcd 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -2,7 +2,9 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import datetime -from typing import Any, Callable, Iterable, List, Tuple +import dataclasses +import contextlib +from typing import Any, Callable, Iterable, List, Optional from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements @@ -14,12 +16,25 @@ from volatility3.plugins.linux import elfs +@dataclasses.dataclass +class TaskFields: + offset: int + user_pid: int + user_tid: int + user_ppid: int + name: str + uid: Optional[int] + gid: Optional[int] + euid: Optional[int] + egid: Optional[int] + creation_time: Optional[datetime.datetime] + + class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular linux memory image.""" - _required_framework_version = (2, 0, 0) - - _version = (2, 3, 0) + _required_framework_version = (2, 13, 0) + _version = (4, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -59,7 +74,9 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] ] @classmethod - def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[Any], bool]: + def create_pid_filter( + cls, pid_list: Optional[List[int]] = None + ) -> Callable[[Any], bool]: """Constructs a filter function for process IDs. Args: @@ -83,7 +100,7 @@ def filter_func(x): @classmethod def get_task_fields( cls, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False - ) -> Tuple[int, int, int, int, str, datetime.datetime]: + ) -> TaskFields: """Extract the fields needed for the final output Args: @@ -92,21 +109,34 @@ def get_task_fields( and of Kernel threads in square brackets. Defaults to False. Returns: - A tuple with the fields to show in the plugin output. + A TaskFields object with the fields to show in the plugin output. """ - pid = task.tgid - tid = task.pid - ppid = task.parent.tgid if task.parent else 0 name = utility.array_to_string(task.comm) - start_time = task.get_create_time() if decorate_comm: if task.is_kernel_thread: name = f"[{name}]" elif task.is_user_thread: name = f"{{{name}}}" - task_fields = (task.vol.offset, pid, tid, ppid, name, start_time) - return task_fields + # This function may be called with a partially initialized/uninitialized task. + # Ensure it always returns a valid TaskFields object, ready for use in a plugin. + valid_cred = task.cred and task.cred.is_readable() + creation_time = None + with contextlib.suppress(Exception): + creation_time = task.get_create_time() + + return TaskFields( + offset=task.vol.offset, + user_pid=task.tgid, + user_tid=task.pid, + user_ppid=task.get_parent_pid(), + name=name, + uid=task.cred.uid if valid_cred else None, + gid=task.cred.gid if valid_cred else None, + euid=task.cred.euid if valid_cred else None, + egid=task.cred.egid if valid_cred else None, + creation_time=creation_time, + ) def _get_file_output(self, task: interfaces.objects.ObjectInterface) -> str: """Extract the elf for the process if requested @@ -180,17 +210,19 @@ def _generator( else: file_output = "Disabled" - offset, pid, tid, ppid, name, creation_time = self.get_task_fields( - task, decorate_comm - ) + task_fields = self.get_task_fields(task, decorate_comm) yield 0, ( - format_hints.Hex(offset), - pid, - tid, - ppid, - name, - creation_time or renderers.NotAvailableValue(), + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, + task_fields.uid or renderers.NotAvailableValue(), + task_fields.gid or renderers.NotAvailableValue(), + task_fields.euid or renderers.NotAvailableValue(), + task_fields.egid or renderers.NotAvailableValue(), + task_fields.creation_time or renderers.NotAvailableValue(), file_output, ) @@ -239,6 +271,10 @@ def run(self): ("TID", int), ("PPID", int), ("COMM", str), + ("UID", int), + ("GID", int), + ("EUID", int), + ("EGID", int), ("CREATION TIME", datetime.datetime), ("File output", str), ] @@ -252,10 +288,11 @@ def generate_timeline(self): for task in self.list_tasks( self.context, self.config["kernel"], filter_func, include_threads=True ): - offset, user_pid, user_tid, _user_ppid, name, creation_time = ( - self.get_task_fields(task) - ) + task_fields = self.get_task_fields(task) + description = f"Process {task_fields.user_pid}/{task_fields.user_tid} {task_fields.name} ({task_fields.offset})" - description = f"Process {user_pid}/{user_tid} {name} ({offset})" - - yield (description, timeliner.TimeLinerType.CREATED, creation_time) + yield ( + description, + timeliner.TimeLinerType.CREATED, + task_fields.creation_time, + ) diff --git a/volatility3/framework/plugins/linux/psscan.py b/volatility3/framework/plugins/linux/psscan.py index 40784a647b..ba68c48567 100644 --- a/volatility3/framework/plugins/linux/psscan.py +++ b/volatility3/framework/plugins/linux/psscan.py @@ -2,15 +2,15 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import Iterable, List, Tuple +from typing import Iterable, List import struct from enum import Enum from volatility3.framework import renderers, interfaces, symbols, constants, exceptions from volatility3.framework.configuration import requirements -from volatility3.framework.objects import utility from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints +from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) @@ -27,8 +27,8 @@ class DescExitStateEnum(Enum): class PsScan(interfaces.plugins.PluginInterface): """Scans for processes present in a particular linux image.""" - _required_framework_version = (2, 0, 0) - _version = (1, 0, 1) + _required_framework_version = (2, 13, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -38,37 +38,11 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) + ), ] - def _get_task_fields( - self, task: interfaces.objects.ObjectInterface - ) -> Tuple[int, int, int, str, str]: - """Extract the fields needed for the final output - - Args: - task: A task object from where to get the fields. - Returns: - A tuple with the fields to show in the plugin output. - """ - pid = task.tgid - tid = task.pid - ppid = 0 - - if task.parent.is_readable(): - ppid = task.parent.tgid - name = utility.array_to_string(task.comm) - exit_state = DescExitStateEnum(task.exit_state).name - - task_fields = ( - format_hints.Hex(task.vol.offset), - pid, - tid, - ppid, - name, - exit_state, - ) - return task_fields - def _generator(self): """Generates the tasks found from scanning.""" @@ -78,8 +52,18 @@ def _generator(self): for task in self.scan_tasks( self.context, vmlinux_module_name, vmlinux.layer_name ): - row = self._get_task_fields(task) - yield (0, row) + task_fields = pslist.PsList.get_task_fields(task) + exit_state = DescExitStateEnum(task.exit_state).name + fields = ( + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, + exit_state, + ) + + yield (0, fields) @classmethod def scan_tasks( @@ -133,7 +117,7 @@ def scan_tasks( ) elif len(kernel_layer.dependencies) == 0: vollog.error( - f"Kernel layer has no dependencies, meaning there is no memory layer for this plugin to scan." + "Kernel layer has no dependencies, meaning there is no memory layer for this plugin to scan." ) raise exceptions.LayerException( kernel_layer_name, f"Layer {kernel_layer_name} has no dependencies" diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index efe5223dfd..74e1721392 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -12,7 +12,8 @@ class PsTree(interfaces.plugins.PluginInterface): """Plugin for listing processes in a tree based on their parent process ID.""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 13, 0) + _version = (1, 1, 1) @classmethod def get_requirements(cls): @@ -24,7 +25,7 @@ def get_requirements(cls): architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.ListRequirement( name="pid", @@ -55,9 +56,9 @@ def find_level(self, pid: int) -> None: seen = set([pid]) level = 0 proc = self._tasks.get(pid) - while proc and proc.parent and proc.parent.pid not in seen: + while proc and proc.get_parent_pid() not in seen: if proc.is_thread_group_leader: - parent_pid = proc.parent.pid + parent_pid = proc.get_parent_pid() else: parent_pid = proc.tgid @@ -100,15 +101,17 @@ def _generator( def yield_processes(pid): task = self._tasks[pid] - row = pslist.PsList.get_task_fields(task, decorate_comm) - # update the first element, the offset, in the row tuple to use format_hints.Hex - # as a simple int is returned from get_task_fields. - row = (format_hints.Hex(row[0]),) + row[1:] - - tid = task.pid - yield (self._levels[tid] - 1, row) - - for child_pid in sorted(self._children.get(tid, [])): + task_fields = pslist.PsList.get_task_fields(task, decorate_comm) + fields = ( + format_hints.Hex(task_fields.offset), + task_fields.user_pid, + task_fields.user_tid, + task_fields.user_ppid, + task_fields.name, + ) + yield (self._levels[task_fields.user_tid] - 1, fields) + + for child_pid in sorted(self._children.get(task_fields.user_tid, [])): yield from yield_processes(child_pid) for pid, level in self._levels.items(): diff --git a/volatility3/framework/plugins/linux/ptrace.py b/volatility3/framework/plugins/linux/ptrace.py index e467ee6440..6493f22b9b 100644 --- a/volatility3/framework/plugins/linux/ptrace.py +++ b/volatility3/framework/plugins/linux/ptrace.py @@ -19,7 +19,7 @@ class Ptrace(plugins.PluginInterface): """Enumerates ptrace's tracer and tracee tasks""" _required_framework_version = (2, 10, 0) - _version = (1, 0, 0) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -30,7 +30,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] architectures=architectures.LINUX_ARCHS, ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), ] diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index 0ddd3e26d6..7376bcbee5 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -12,6 +12,7 @@ from volatility3.framework.objects import utility from volatility3.framework.symbols import linux from volatility3.plugins.linux import lsof +from volatility3.plugins.linux import pslist vollog = logging.getLogger(__name__) @@ -21,7 +22,6 @@ class SockHandlers(interfaces.configuration.VersionableInterface): """Handles several socket families extracting the sockets information.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 0) def __init__(self, vmlinux, task, *args, **kwargs): @@ -372,7 +372,7 @@ def _bluetooth_sock( bt_sock = sock.cast("bt_sock") def bt_addr(addr): - return ":".join(reversed(["%02x" % x for x in addr.b])) + return ":".join(reversed([f"{x:02x}" for x in addr.b])) src_addr = src_port = dst_addr = dst_port = None bt_protocol = bt_sock.get_protocol() @@ -438,8 +438,7 @@ class Sockstat(plugins.PluginInterface): """Lists all network connections for all processes.""" _required_framework_version = (2, 0, 0) - - _version = (3, 0, 0) + _version = (3, 0, 2) @classmethod def get_requirements(cls): @@ -455,6 +454,9 @@ def get_requirements(cls): requirements.PluginRequirement( name="lsof", plugin=lsof.Lsof, version=(2, 0, 0) ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) + ), requirements.VersionRequirement( name="linuxutils", component=linux.LinuxUtilities, version=(2, 0, 0) ), @@ -591,7 +593,7 @@ def _generator(self, pids: List[int], netns_id_arg: int, symbol_table: str): tasks: String with a list of tasks and FDs using a socket. It can also have extended information such as socket filters, bpf info, etc. """ - filter_func = lsof.pslist.PsList.create_pid_filter(pids) + filter_func = pslist.PsList.create_pid_filter(pids) socket_generator = self.list_sockets( self.context, symbol_table, filter_func=filter_func ) diff --git a/volatility3/framework/plugins/linux/vmaregexscan.py b/volatility3/framework/plugins/linux/vmaregexscan.py new file mode 100644 index 0000000000..8fb96da1e8 --- /dev/null +++ b/volatility3/framework/plugins/linux/vmaregexscan.py @@ -0,0 +1,129 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import re +from typing import List + +from volatility3.framework import renderers, interfaces +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.layers import scanners +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.plugins.linux import pslist + +vollog = logging.getLogger(__name__) + + +class VmaRegExScan(plugins.PluginInterface): + """Scans all virtual memory areas for tasks using RegEx.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 2) + + MAXSIZE_DEFAULT = 128 + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.StringRequirement( + name="pattern", description="RegEx pattern", optional=False + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size in bytes for displayed context", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), + ] + + def _generator(self, regex_pattern, tasks): + regex_pattern = bytes(regex_pattern, "UTF-8") + vollog.debug(f"RegEx Pattern: {regex_pattern}") + + for task in tasks: + + if not task.mm: + continue + name = utility.array_to_string(task.comm) + + # attempt to create a process layer for each task and skip those + # that cannot (e.g. kernel threads) + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + continue + + # get the proc_layer object from the context + proc_layer = self.context.layers[proc_layer_name] + + # get process sections for scanning + sections = [ + (start, size) for (start, size) in task.get_process_memory_sections() + ] + + for offset in proc_layer.scan( + context=self.context, + scanner=scanners.RegExScanner(regex_pattern), + sections=sections, + progress_callback=self._progress_callback, + ): + result_data = proc_layer.read(offset, self.MAXSIZE_DEFAULT, pad=True) + + # reapply the regex in order to extact just the match + regex_result = re.match(regex_pattern, result_data) + + if regex_result: + # the match is within the results_data (e.g. it fits within MAXSIZE_DEFAULT) + # extract just the match itself + regex_match = regex_result.group(0) + text_result = str(regex_match, encoding="UTF-8", errors="replace") + bytes_result = regex_match + else: + # the match is not with the results_data (e.g. it doesn't fit within MAXSIZE_DEFAULT) + text_result = str(result_data, encoding="UTF-8", errors="replace") + bytes_result = result_data + + user_pid = task.tgid + yield 0, ( + user_pid, + name, + format_hints.Hex(offset), + text_result, + bytes_result, + ) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Offset", format_hints.Hex), + ("Text", str), + ("Hex", bytes), + ], + self._generator( + self.config.get("pattern"), + pslist.PsList.list_tasks( + self.context, self.config["kernel"], filter_func=filter_func + ), + ), + ) diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py index 9fe06b0c84..4db23e50b2 100644 --- a/volatility3/framework/plugins/linux/vmayarascan.py +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import logging from typing import Iterable, List, Tuple from volatility3.framework import interfaces, renderers @@ -10,12 +11,14 @@ from volatility3.plugins import yarascan from volatility3.plugins.linux import pslist +vollog = logging.getLogger(__name__) + class VmaYaraScan(interfaces.plugins.PluginInterface): """Scans all virtual memory areas for tasks using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 0) + _version = (1, 0, 2) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -28,11 +31,14 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] optional=True, ), requirements.PluginRequirement( - name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + name="pslist", plugin=pslist.PsList, version=(4, 0, 0) ), requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), requirements.ModuleRequirement( name="kernel", description="Linux kernel", @@ -50,6 +56,8 @@ def _generator(self): # use yarascan to parse the yara options provided and create the rules rules = yarascan.YaraScan.process_yara_options(dict(self.config)) + sanity_check = 1024 * 1024 * 1024 # 1 GB + # filter based on the pid option if provided filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) for task in pslist.PsList.list_tasks( @@ -66,29 +74,36 @@ def _generator(self): # get the proc_layer object from the context proc_layer = self.context.layers[proc_layer_name] - for start, end in self.get_vma_maps(task): - for match in rules.match( - data=proc_layer.read(start, end - start, True) + max_vma_size = 0 + vma_maps_to_scan = [] + for start, size in self.get_vma_maps(task): + if size > sanity_check: + vollog.debug( + f"VMA at 0x{start:x} over sanity-check size, not scanning" + ) + continue + max_vma_size = max(max_vma_size, size) + vma_maps_to_scan.append((start, size)) + + if not vma_maps_to_scan: + vollog.warning(f"No VMAs were found for task {task.tgid}, not scanning") + continue + + scanner = yarascan.YaraScanner(rules=rules) + scanner.chunk_size = max_vma_size + + # scan the VMA data (in one contiguous block) with the yarascanner + for start, size in vma_maps_to_scan: + for offset, rule_name, name, value in scanner( + proc_layer.read(start, size, pad=True), start ): - if yarascan.YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: - yield 0, ( - format_hints.Hex(instance.offset + start), - task.UniqueProcessId, - match.rule, - match_string.identifier, - instance.matched_data, - ) - else: - for offset, name, value in match.strings: - yield 0, ( - format_hints.Hex(offset + start), - task.tgid, - match.rule, - name, - value, - ) + yield 0, ( + format_hints.Hex(offset), + task.tgid, + rule_name, + name, + value, + ) @staticmethod def get_vma_maps( diff --git a/volatility3/framework/plugins/mac/check_sysctl.py b/volatility3/framework/plugins/mac/check_sysctl.py index 4f64eaed80..ed3e34aea0 100644 --- a/volatility3/framework/plugins/mac/check_sysctl.py +++ b/volatility3/framework/plugins/mac/check_sysctl.py @@ -60,7 +60,7 @@ def _parse_global_variable_sysctls(self, kernel, name): return var_str def _process_sysctl_list(self, kernel, sysctl_list, recursive=0): - if type(sysctl_list) == volatility3.framework.objects.Pointer: + if type(sysctl_list) is volatility3.framework.objects.Pointer: sysctl_list = sysctl_list.dereference().cast("sysctl_oid_list") sysctl = sysctl_list.slh_first @@ -93,10 +93,9 @@ def _process_sysctl_list(self, kernel, sysctl_list, recursive=0): val = self._parse_global_variable_sysctls(kernel, name) elif ctltype == "CTLTYPE_NODE": if sysctl.oid_handler == 0: - for info in self._process_sysctl_list( + yield from self._process_sysctl_list( kernel, sysctl.oid_arg1, recursive=1 - ): - yield info + ) val = "Node" diff --git a/volatility3/framework/plugins/mac/kauth_scopes.py b/volatility3/framework/plugins/mac/kauth_scopes.py index afb320a07d..c2c473eace 100644 --- a/volatility3/framework/plugins/mac/kauth_scopes.py +++ b/volatility3/framework/plugins/mac/kauth_scopes.py @@ -80,7 +80,7 @@ def _generator(self): ( identifier, format_hints.Hex(scope.ks_idata), - len([l for l in scope.get_listeners()]), + len([listener for listener in scope.get_listeners()]), format_hints.Hex(callback), module_name, symbol_name, diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 2a8692b772..41fde31ca0 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -119,8 +119,7 @@ def _walk_klist_array(cls, kernel, fdp, array_pointer_member, array_size_member) return None for klist in klist_array: - for kn in mac.MacUtilities.walk_slist(klist, "kn_link"): - yield kn + yield from mac.MacUtilities.walk_slist(klist, "kn_link") @classmethod def _get_task_kevents(cls, kernel, task): diff --git a/volatility3/framework/plugins/mac/mount.py b/volatility3/framework/plugins/mac/mount.py index ff654e1a74..1a1e33571f 100644 --- a/volatility3/framework/plugins/mac/mount.py +++ b/volatility3/framework/plugins/mac/mount.py @@ -49,8 +49,7 @@ def list_mounts( list_head = kernel.object_from_symbol(symbol_name="mountlist") - for mount in mac.MacUtilities.walk_tailq(list_head, "mnt_list"): - yield mount + yield from mac.MacUtilities.walk_tailq(list_head, "mnt_list") def _generator(self): for mount in self.list_mounts(self.context, self.config["kernel"]): diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index fe5179dfa7..bd905615d6 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -115,9 +115,7 @@ def vma_dump( proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - pid, excp.invalid_address, excp.layer_name - ) + f"Process {pid}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None vm_size = vm_end - vm_start @@ -154,7 +152,9 @@ def _generator(self, tasks): address_list = self.config.get("address", None) if not address_list: # do not filter as no address_list was supplied - vma_filter_func = lambda _: True + def vma_filter_func(_): + return True + else: # filter for any vm_start that matches the supplied address config def vma_filter_function(task: interfaces.objects.ObjectInterface) -> bool: diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 9b570f3f9c..8c5e5c1a5d 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Callable, Dict, Iterable, List +from typing import Callable, Dict, Iterable, List, Optional from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements @@ -82,8 +82,12 @@ def get_list_tasks(cls, method: str) -> Callable[ return list_tasks @classmethod - def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[int], bool]: - filter_func = lambda _: False + def create_pid_filter( + cls, pid_list: Optional[List[int]] = None + ) -> Callable[[int], bool]: + def filter_func(_): + return False + # FIXME: mypy #4973 or #2608 pid_list = pid_list or [] filter_list = [x for x in pid_list if x is not None] diff --git a/volatility3/framework/plugins/regexscan.py b/volatility3/framework/plugins/regexscan.py new file mode 100644 index 0000000000..c526b16979 --- /dev/null +++ b/volatility3/framework/plugins/regexscan.py @@ -0,0 +1,78 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import re +from typing import List + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.layers import scanners +from volatility3.framework.renderers import format_hints + +vollog = logging.getLogger(__name__) + + +class RegExScan(plugins.PluginInterface): + """Scans kernel memory using RegEx patterns.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + MAXSIZE_DEFAULT = 128 + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.StringRequirement( + name="pattern", description="RegEx pattern", optional=False + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size in bytes for displayed context", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), + ] + + def _generator(self, regex_pattern): + regex_pattern = bytes(regex_pattern, "UTF-8") + vollog.debug(f"RegEx Pattern: {regex_pattern}") + + layer = self.context.layers[self.config["primary"]] + for offset in layer.scan( + context=self.context, scanner=scanners.RegExScanner(regex_pattern) + ): + result_data = layer.read(offset, self.MAXSIZE_DEFAULT, pad=True) + + # reapply the regex in order to extact just the match + regex_result = re.match(regex_pattern, result_data) + + if regex_result: + # the match is within the results_data (e.g. it fits within MAXSIZE_DEFAULT) + # extract just the match itself + regex_match = regex_result.group(0) + text_result = str(regex_match, encoding="UTF-8", errors="replace") + bytes_result = regex_match + else: + # the match is not with the results_data (e.g. it doesn't fit within MAXSIZE_DEFAULT) + text_result = str(result_data, encoding="UTF-8", errors="replace") + bytes_result = result_data + + yield 0, (format_hints.Hex(offset), text_result, bytes_result) + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Text", str), + ("Hex", bytes), + ], + self._generator(self.config.get("pattern")), + ) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index c754e43eff..0f4064d794 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -54,7 +54,9 @@ def __init__(self, *args, **kwargs): self.automagics: Optional[List[interfaces.automagic.AutomagicInterface]] = None @classmethod - def get_usable_plugins(cls, selected_list: List[str] = None) -> List[Type]: + def get_usable_plugins( + cls, selected_list: Optional[List[str]] = None + ) -> List[Type]: # Initialize for the run plugin_list = list(framework.class_subclasses(TimeLinerInterface)) @@ -143,9 +145,7 @@ def _generator( times = self.timeline.get((plugin_name, item), {}) if times.get(timestamp_type, None) is not None: vollog.debug( - "Multiple timestamps for the same plugin/file combination found: {} {}".format( - plugin_name, item - ) + f"Multiple timestamps for the same plugin/file combination found: {plugin_name} {item}" ) times[timestamp_type] = timestamp self.timeline[(plugin_name, item)] = times @@ -206,8 +206,7 @@ def _generator( ) vollog.log(logging.DEBUG, traceback.format_exc()) - for data_item in sorted(data, key=self._sort_function): - yield data_item + yield from sorted(data, key=self._sort_function) # Write out a body file if necessary if self.config.get("create-bodyfile", True): diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 562846def5..414a8814ab 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -48,7 +48,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] name="driverirp", plugin=driverirp.DriverIrp, version=(1, 0, 0) ), requirements.PluginRequirement( - name="handles", plugin=handles.Handles, version=(1, 0, 0) + name="handles", plugin=handles.Handles, version=(2, 0, 0) ), ] diff --git a/volatility3/framework/plugins/windows/cmdline.py b/volatility3/framework/plugins/windows/cmdline.py index 8cfb5576c2..9bd9eda0e5 100644 --- a/volatility3/framework/plugins/windows/cmdline.py +++ b/volatility3/framework/plugins/windows/cmdline.py @@ -84,9 +84,7 @@ def _generator(self, procs): result_text = f"Required memory at {exp.invalid_address:#x} is not valid (process exited?)" except exceptions.InvalidAddressException as exp: - result_text = "Process {}: Required memory at {:#x} is not valid (incomplete layer {}?)".format( - proc_id, exp.invalid_address, exp.layer_name - ) + result_text = f"Process {proc_id}: Required memory at {exp.invalid_address:#x} is not valid (incomplete layer {exp.layer_name}?)" yield (0, (proc.UniqueProcessId, process_name, result_text)) diff --git a/volatility3/framework/plugins/windows/consoles.py b/volatility3/framework/plugins/windows/consoles.py index ad1c9d4bde..a448989c0e 100644 --- a/volatility3/framework/plugins/windows/consoles.py +++ b/volatility3/framework/plugins/windows/consoles.py @@ -95,9 +95,7 @@ def find_conhost_proc( except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) @classmethod @@ -176,12 +174,7 @@ def determine_conhost_version( ) vollog.debug( - "Determined OS Version: {}.{} {}.{}".format( - kuser.NtMajorVersion, - kuser.NtMinorVersion, - vers.MajorVersion, - vers.MinorVersion, - ) + f"Determined OS Version: {kuser.NtMajorVersion}.{kuser.NtMinorVersion} {vers.MajorVersion}.{vers.MinorVersion}" ) if nt_major_version == 10 and arch == "x64": @@ -260,9 +253,7 @@ def determine_conhost_version( if ver: conhost_mod_version = ver[3] vollog.debug( - "Determined conhost.exe's FileVersion: {}".format( - conhost_mod_version - ) + f"Determined conhost.exe's FileVersion: {conhost_mod_version}" ) else: vollog.debug("Could not determine conhost.exe's FileVersion.") @@ -311,12 +302,7 @@ def determine_conhost_version( else: raise NotImplementedError( - "This version of Windows is not supported: {}.{} {}.{}!".format( - nt_major_version, - nt_minor_version, - vers.MajorVersion, - vers_minor_version, - ) + f"This version of Windows is not supported: {nt_major_version}.{nt_minor_version} {vers.MajorVersion}.{vers_minor_version}!" ) vollog.debug(f"Determined symbol filename: {filename}") diff --git a/volatility3/framework/plugins/windows/direct_system_calls.py b/volatility3/framework/plugins/windows/direct_system_calls.py new file mode 100644 index 0000000000..b0c162f465 --- /dev/null +++ b/volatility3/framework/plugins/windows/direct_system_calls.py @@ -0,0 +1,467 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging + +from collections import namedtuple +from typing import List, Tuple, Optional, Generator, Callable + +from volatility3.framework.objects import utility +from volatility3.framework import interfaces, renderers, symbols, exceptions +from volatility3.framework.configuration import requirements +from volatility3.plugins import yarascan +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist + +vollog = logging.getLogger(__name__) + +try: + import capstone + + has_capstone = True +except ImportError: + has_capstone = False + +# Full details on the techniques used in these plugins to detect EDR-evading malware +# can be found in our 20 page whitepaper submitted to DEFCON along with the presentation +# https://www.volexity.com/wp-content/uploads/2024/08/Defcon24_EDR_Evasion_Detection_White-Paper_Andrew-Case.pdf + +syscall_finder_type = namedtuple( + "syscall_finder_type", + [ + "get_syscall_target_address", + "wants_syscall_inst", + "rule_str", + "invalid_ops", + "termination_ops", + ], +) + +syscall_finder_type.__doc__ = """ +This type is used to specify how malicious system call invocations should be found. + +`get_syscall_target_address` is optionally used to extract the address containing the malicious 'syscall' instruction +`wants_syscall_inst` whether or not this method expects the 'syscall' instrunction directly within the malicious code block +`rule` the opcode string to search for the malicious syscall instructions +`invalid_ops` instructions that only appear in invalid code blocks. Stops processing of the code block when encountered. +`termination_ops` instructions that are expected to be present in the code block and that stop processing +""" + + +class DirectSystemCalls(interfaces.plugins.PluginInterface): + """Detects the Direct System Call technique used to bypass EDRs""" + + _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) + + # DLLs that are expected to host system call invocations + valid_syscall_handlers = ("ntdll.dll", "win32u.dll") + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.syscall_finder = syscall_finder_type( + # for direct system calls, we find the `syscall` instruction directly, so we already know the address + None, + # yes, we want the syscall instruction present as it is what this technique looks for + True, + # regex to find "\x0f\x05" (syscall) followed later by "\xc3" (ret) + # we allow spacing in between to break naive anti-analysis forms (e.g., TarTarus Gate) + # Standard techniques, such as HellsGate, look like: + # mov r10, rcx + # mov eax, + # syscall + # ret + "/\\x0f\\x05[^\\xc3]{,24}\\xc3/", + # any of these will not be in a workable, malicious direct system call block + ["jmp", "call", "leave", "int3"], + # the expected form is to end with a "ret" back to the calling code + ["ret"], + ) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # create a list of requirements for vadyarascan + vadyarascan_requirements = [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) + ), + requirements.PluginRequirement( + name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) + ), + ] + + # get base yarascan requirements for command line options + yarascan_requirements = yarascan.YaraScan.get_yarascan_option_requirements() + + # return the combined requirements + return yarascan_requirements + vadyarascan_requirements + + @staticmethod + def _is_syscall_block( + disasm_func: Callable, + syscall_finder: syscall_finder_type, + data: bytes, + address: int, + ) -> Optional[Tuple[str, "capstone._cs_insn"]]: + """ + Determines if the bytes starting at `data` represent a valid syscall instrunction invocation block + + To maliciously invoke the system call instruction, malware must do each of the following: + + 1) update RAX to the system call number + 2) update R10 to the first parameter + 3) hit the 'termination' instruction set in `syscall_finder_type` + + We also track whether the 'syscall' instruction was encountered while parsing + + This function is reusable for every technique we found and studied during the DEFCON research timeframe + + Args: + disasm_func: capstone disassembly function gathered from `get_disasm_function` + syscall_finder: the method and constraints on the malicious system call blocks that the calling plugin knows how to find + data: the bytes from memory to search for malicious syscall invocations + address: the address from where `data` came from in the particular process + Returns: + Optional[Tuple[str, capstone._cs_insn]]: For valid blocks, the disassembled bytes in string from and the last (termination) instruction + """ + found_movr10 = False + found_movreax = False + found_syscall = False + found_end = False + end_inst = None + + disasm_bytes = "" + + for inst in disasm_func(data, address): + disasm_bytes += f"{inst.address:#x}: {inst.mnemonic} {inst.op_str}; " + + # an instruction of all 0x00 opcodes + if inst.opcode.count(0) == len(inst.opcode): + break + + op = inst.mnemonic + + # invalid op, bail + if op in syscall_finder.invalid_ops: + break + + # found the end instruction wanted by the caller + elif op in syscall_finder.termination_ops: + found_end = True + end_inst = inst + break + + # track this no matter what to make code more re-usable + elif op == "syscall": + found_syscall = True + + # if we hit a 'syscall' but RAX or R10 haven't been touched + # then we are in an invalid path, so bail + if not syscall_finder.wants_syscall_inst or ( + not (found_movr10 and found_movreax) + ): + break + + else: + # attempt to see if any other instruction type wrote to registers + try: + _, regs_written = inst.regs_access() + except capstone.CsError: + continue + + if regs_written: + for r in regs_written: + # track writes to eax/rax or R10 + reg = inst.reg_name(r) + if reg in ["eax", "rax"]: + found_movreax = True + + elif reg == "r10": + found_movr10 = True + + # if any of these are missing, the block is invalid regardless of + # the technique we are trying to detect now or in the future + if not (found_movr10 and found_movreax and found_end): + return None + + # if the finder requires a 'syscall' instruction then bail now if we didn't find one + if syscall_finder.wants_syscall_inst and not found_syscall: + return None + + return disasm_bytes, end_inst + + @staticmethod + def get_disasm_function(architecture: str) -> Callable: + """ + Returns the disassembly handler for the given architecture + .detail is used to get full instruction information + + Args: + architecture: the name of the architecture for the process being disassembled + Returns: + The disasm function from capstone for the given architecture + """ + disasm_types = { + "intel": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32), + "intel64": capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64), + } + + disasm_type = disasm_types[architecture] + disasm_type.detail = True + return disasm_type.disasm + + @classmethod + def _is_valid_syscall( + cls, + syscall_finder: syscall_finder_type, + proc_layer: interfaces.layers.DataLayerInterface, + architecture: str, + vads: List[Tuple[int, int, str]], + address: int, + ) -> Optional[Tuple[int, str]]: + """ + Args: + syscall_finder: + proc_layer: the memory layer of the process being scanned + architecture: the name of the architecture for the process being disassembled + vads: the ranges of this process under 10MB + address: the starting address to check for malicious syscall code blocks + + Returns: + Optional[Tuple[int, str]]: For valid code blocks, the starting address of the block and the disassembly string + """ + # the number bytes behind the yara rule hit to scan + behind = 32 + + address = address - behind + + try: + data = proc_layer.read(address, behind * 2) + except exceptions.InvalidAddressException: + return None + + disasm_func = cls.get_disasm_function(architecture) + + # since Intel does not have fixed-size instructions, we have to scan + # each byte offset and re-disassemble the remaining block + for offset in range(behind): + # if this looks like a system call back (r10, rax, ret/jmp) + syscall_info = cls._is_syscall_block( + disasm_func, syscall_finder, data[offset:], address + offset + ) + if syscall_info: + disasm_bytes, end_inst = syscall_info + + # if we can recover (and require) a target address for this malware technique + if syscall_finder.get_syscall_target_address: + target_address = syscall_finder.get_syscall_target_address( + proc_layer, end_inst + ) + + # could not determine the address -> invalid basic block + if not target_address: + continue + + # we only care about calls to system call DLLs + path = cls.get_range_path(vads, target_address) + if not isinstance(path, str) or not path.lower().endswith( + cls.valid_syscall_handlers + ): + continue + + # return the address and disassembly string if all checks pass + return address + offset, disasm_bytes + + return None + + @staticmethod + def get_vad_maps( + task: interfaces.objects.ObjectInterface, + ) -> List[Tuple[int, int, str]]: + """Creates a map of start/end addresses within a virtual address + descriptor tree. + + Args: + task: The EPROCESS object of which to traverse the vad tree + + Returns: + An iterable of tuples containing start and end addresses for each descriptor + """ + vads: List[Tuple[int, int, str]] = [] + + # scan regions under 10MB + scan_max = 10 * 1000 * 1000 + + vad_root = task.get_vad_root() + + for vad in vad_root.traverse(): + if vad.get_size() < scan_max: + vads.append((vad.get_start(), vad.get_size(), vad.get_file_name())) + + return vads + + @staticmethod + def get_range_path( + ranges: List[Tuple[int, int, str]], address: int + ) -> Optional[str]: + """ + Returns the path for the range holding `address`, if found + + Args: + ranges: VADs collected from `get_vad_maps` + address: the address to find + Returns: + The path holding the address, if any + """ + for start, size, path in ranges: + if start <= address < start + size: + return path + + return None + + @classmethod + def get_tasks_to_scan( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table_name: str, + ) -> Generator[ + Tuple[interfaces.objects.ObjectInterface, str, str, str], None, None + ]: + """ + Gathers active processes with the extra information needed + to detect malicious syscall instructions + + Returns: + Generator of the process object, name, memory layer, and architecture + """ + + # gather active processes + filter_func = pslist.PsList.create_active_process_filter() + + is_32bit_arch = not symbols.symbol_table_is_64bit(context, symbol_table_name) + + for proc in pslist.PsList.list_processes( + context=context, + layer_name=layer_name, + symbol_table=symbol_table_name, + filter_func=filter_func, + ): + proc_name = utility.array_to_string(proc.ImageFileName) + + # skip Defender + if proc_name in ["MsMpEng.exe"]: + continue + + try: + proc_layer_name = proc.add_process_layer() + except exceptions.InvalidAddressException: + continue + + if is_32bit_arch or proc.get_is_wow64(): + architecture = "intel" + else: + architecture = "intel64" + + yield proc, proc_name, proc_layer_name, architecture + + @classmethod + def _get_rule_hits( + cls, + context: interfaces.objects.ObjectInterface, + proc_layer: interfaces.layers.DataLayerInterface, + vads: List[Tuple[int, int, str]], + pattern: str, + ) -> Generator[Tuple[int, Optional[str]], None, None]: + """ + Runs the given opcode rule through Yara and returns the address and file path of hits + + Args: + context: + proc_layer: the layer to scan + vads: the ranges inside of the process being scanned + pattern: the opcodes rule from the plugin to detect a particular EDR-bypass technique + + Returns: + Generator of the address and file path of hits + """ + sections = [(vad[0], vad[1]) for vad in vads] + + rule = yarascan.YaraScanner.get_rule(pattern) + + for hit in proc_layer.scan( + context=context, + scanner=yarascan.YaraScanner(rules=rule), + sections=sections, + ): + address = hit[0] + + path = cls.get_range_path(vads, address) + + # ignore hits in the system call DLLs + if isinstance(path, str) and path.lower().endswith( + cls.valid_syscall_handlers + ): + continue + + yield address, path + + def _generator( + self, + ) -> Generator[Tuple[int, Tuple[str, int, Optional[str], int, str]], None, None]: + if not has_capstone: + vollog.warning( + "capstone is not installed. This plugin requires capstone to operate." + ) + return + + kernel = self.context.modules[self.config["kernel"]] + + for proc, proc_name, proc_layer_name, architecture in self.get_tasks_to_scan( + self.context, kernel.layer_name, kernel.symbol_table_name + ): + proc_layer = self.context.layers[proc_layer_name] + + vads = self.get_vad_maps(proc) + + # for each valid process, look for malicious syscall invocations + for address, vad_path in self._get_rule_hits( + self.context, proc_layer, vads, self.syscall_finder.rule_str + ): + syscall_info = self._is_valid_syscall( + self.syscall_finder, proc_layer, architecture, vads, address + ) + if not syscall_info: + continue + + address, disasm_bytes = syscall_info + + yield 0, ( + proc_name, + proc.UniqueProcessId, + vad_path, + format_hints.Hex(address), + disasm_bytes, + ) + + def run(self) -> renderers.TreeGrid: + return renderers.TreeGrid( + [ + ("Process", str), + ("PID", int), + ("Range", str), + ("Address", format_hints.Hex), + ("Disasm", str), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 5a1b37fcf8..1dafb6bf58 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -5,9 +5,9 @@ import datetime import logging import re -from typing import List, Optional, Type +from typing import List -from volatility3.framework import constants, exceptions, interfaces, renderers +from volatility3.framework import exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import conversion, format_hints from volatility3.framework.symbols import intermed @@ -19,7 +19,7 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): - """Lists the loaded modules in a particular windows memory image.""" + """Lists the loaded DLLs in a particular windows memory image.""" _required_framework_version = (2, 0, 0) _version = (3, 0, 0) @@ -39,6 +39,9 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] requirements.VersionRequirement( name="psscan", component=psscan.PsScan, version=(1, 1, 0) ), + requirements.VersionRequirement( + name="pedump", component=pedump.PEDump, version=(1, 0, 0) + ), requirements.VersionRequirement( name="info", component=info.Info, version=(1, 0, 0) ), @@ -53,16 +56,16 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] description="Process offset in the physical address space", optional=True, ), - requirements.StringRequirement( - name="name", - description="Specify a regular expression to match dll name(s)", - optional=True, - ), requirements.IntRequirement( name="base", description="Specify a base virtual address in process memory", optional=True, ), + requirements.StringRequirement( + name="name", + description="Specify a regular expression to match dll name(s)", + optional=True, + ), requirements.BooleanRequirement( name="ignore-case", description="Specify case insensitivity for the regular expression name matching", @@ -75,9 +78,6 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] default=False, optional=True, ), - requirements.VersionRequirement( - name="pedump", component=pedump.PEDump, version=(1, 0, 0) - ), ] def _generator(self, procs): @@ -90,12 +90,15 @@ def _generator(self, procs): kuser = info.Info.get_kuser_structure( self.context, kernel.layer_name, kernel.symbol_table_name ) + nt_major_version = int(kuser.NtMajorVersion) nt_minor_version = int(kuser.NtMinorVersion) + # LoadTime only applies to versions higher or equal to Window 7 (6.1 and higher) dll_load_time_field = (nt_major_version > 6) or ( nt_major_version == 6 and nt_minor_version >= 1 ) + for proc in procs: proc_id = proc.UniqueProcessId proc_layer_name = proc.add_process_layer() @@ -135,7 +138,7 @@ def _generator(self, procs): if dll_load_time_field: # Versions prior to 6.1 won't have the LoadTime attribute - # and 32bit version shouldn't have the Quadpart according to MSDN + # and 32-bit version shouldn't have the Quadpart according to MSDN try: DllLoadTime = conversion.wintime_to_datetime( entry.LoadTime.QuadPart @@ -199,16 +202,7 @@ def generate_timeline(self): _depth, row_data = row if not isinstance(row_data[6], datetime.datetime): continue - description = ( - "DLL Load: Process {} {} Loaded {} ({}) Size {} Offset {}".format( - row_data[0], - row_data[1], - row_data[4], - row_data[5], - row_data[3], - row_data[2], - ) - ) + description = f"DLL Load: Process {row_data[0]} {row_data[1]} Loaded {row_data[4]} ({row_data[5]}) Size {row_data[3]} Offset {row_data[2]}" yield (description, timeliner.TimeLinerType.CREATED, row_data[6]) def run(self): diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 33d2d0d41d..64d9be4db5 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -69,7 +69,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] name="pslist", component=pslist.PsList, version=(2, 0, 0) ), requirements.VersionRequirement( - name="handles", component=handles.Handles, version=(1, 0, 0) + name="handles", component=handles.Handles, version=(2, 0, 0) ), ] @@ -192,13 +192,7 @@ def process_file_object( for memory_object, layer, extension in dump_parameters: cache_name = EXTENSION_CACHE_MAP[extension] - desired_file_name = "file.{0:#x}.{1:#x}.{2}.{3}.{4}".format( - file_obj.vol.offset, - memory_object.vol.offset, - cache_name, - ntpath.basename(obj_name), - extension, - ) + desired_file_name = f"file.{file_obj.vol.offset:#x}.{memory_object.vol.offset:#x}.{cache_name}.{ntpath.basename(obj_name)}.{extension}" file_handle = cls.dump_file_producer( file_obj, memory_object, open_method, layer, desired_file_name diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index 66db03c9c2..cac4ecf400 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -92,7 +92,7 @@ def _get_silent_vars(self) -> List[str]: except ( exceptions.InvalidAddressException, registry.RegistryFormatException, - ) as excp: + ): vollog.log( constants.LOGLEVEL_VVV, "Error while parsing global environment variables keys (some keys might be excluded)", @@ -113,7 +113,7 @@ def _get_silent_vars(self) -> List[str]: except ( exceptions.InvalidAddressException, registry.RegistryFormatException, - ) as excp: + ): vollog.log( constants.LOGLEVEL_VVV, "Error while parsing user environment variables keys (some keys might be excluded)", @@ -134,7 +134,7 @@ def _get_silent_vars(self) -> List[str]: except ( exceptions.InvalidAddressException, registry.RegistryFormatException, - ) as excp: + ): vollog.log( constants.LOGLEVEL_VVV, "Error while parsing volatile environment variables keys (some keys might be excluded)", diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index 9b20ed2d0f..eece7fb6ce 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -26,7 +26,7 @@ def createservicesid(svc) -> str: ## The use of struct here is OK. It doesn't make much sense ## to leverage obj.Object inside this loop. dec.append(struct.unpack(" Dict[str, str]: except ( exceptions.InvalidAddressException, layers.registry.RegistryFormatException, - ) as excp: + ): continue try: value_data = node.decode_data() @@ -156,7 +156,7 @@ def lookup_user_sids(self) -> Dict[str, str]: ValueError, exceptions.InvalidAddressException, layers.registry.RegistryFormatException, - ) as excp: + ): continue except (KeyError, exceptions.InvalidAddressException): continue diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index 3e5a2fd826..62eceb973d 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -3,9 +3,9 @@ # import logging -from typing import List, Optional, Dict +from typing import Dict, List, Optional -from volatility3.framework import constants, exceptions, renderers, interfaces, symbols +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints @@ -13,23 +13,15 @@ vollog = logging.getLogger(__name__) -try: - import capstone - - has_capstone = True -except ImportError: - has_capstone = False - class Handles(interfaces.plugins.PluginInterface): """Lists process open handles.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 2) + _version = (2, 0, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._sar_value = None self._type_map = None self._cookie = None self._level_mask = 7 @@ -62,21 +54,6 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] ), ] - def _decode_pointer(self, value, magic): - """Windows encodes pointers to objects and decodes them on the fly - before using them. - - This function mimics the decoding routine so we can generate the - proper pointer values as well. - """ - - value = value & 0xFFFFFFFFFFFFFFF8 - value = value >> magic - # if (value & (1 << 47)): - # value = value | 0xFFFF000000000000 - - return value - def _get_item(self, handle_table_entry, handle_value): """Given a handle table entry (_HANDLE_TABLE_ENTRY) structure from a process' handle table, determine where the corresponding object's @@ -100,24 +77,11 @@ def _get_item(self, handle_table_entry, handle_value): ) if is_64bit: - if handle_table_entry.LowValue == 0: + if handle_table_entry.ObjectPointerBits == 0: return None - magic = self.find_sar_value() - - # is this the right thing to raise here? - if magic is None: - if has_capstone: - raise AttributeError( - "Unable to find the SAR value for decoding handle table pointers" - ) - else: - raise exceptions.MissingModuleException( - "capstone", - "Requires capstone to find the SAR value for decoding handle table pointers", - ) + offset = handle_table_entry.ObjectPointerBits << 4 - offset = self._decode_pointer(handle_table_entry.LowValue, magic) else: if handle_table_entry.InfoTable == 0: return None @@ -135,78 +99,6 @@ def _get_item(self, handle_table_entry, handle_value): object_header.HandleValue = handle_value return object_header - def find_sar_value(self): - """Locate ObpCaptureHandleInformationEx if it exists in the sample. - - Once found, parse it for the SAR value that we need to decode - pointers in the _HANDLE_TABLE_ENTRY which allows us to find the - associated _OBJECT_HEADER. - """ - DEFAULT_SAR_VALUE = 0x10 # to be used only when decoding fails - - if self._sar_value is None: - if not has_capstone: - vollog.debug( - "capstone module is missing, unable to create disassembly of ObpCaptureHandleInformationEx" - ) - return None - kernel = self.context.modules[self.config["kernel"]] - - virtual_layer_name = kernel.layer_name - kvo = self.context.layers[virtual_layer_name].config[ - "kernel_virtual_offset" - ] - ntkrnlmp = self.context.module( - kernel.symbol_table_name, layer_name=virtual_layer_name, offset=kvo - ) - - try: - func_addr = ntkrnlmp.get_symbol("ObpCaptureHandleInformationEx").address - except exceptions.SymbolError: - vollog.debug("Unable to locate ObpCaptureHandleInformationEx symbol") - return None - - try: - func_addr_to_read = kvo + func_addr - num_bytes_to_read = 0x200 - vollog.debug( - f"ObpCaptureHandleInformationEx symbol located at {hex(func_addr_to_read)}" - ) - data = self.context.layers.read( - virtual_layer_name, func_addr_to_read, num_bytes_to_read - ) - except exceptions.InvalidAddressException: - vollog.warning( - f"Failed to read {hex(num_bytes_to_read)} bytes at symbol {hex(func_addr_to_read)}. Unable to decode SAR value. Failing back to a common value of {hex(DEFAULT_SAR_VALUE)}" - ) - self._sar_value = DEFAULT_SAR_VALUE - return self._sar_value - - md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) - - instruction_count = 0 - for address, size, mnemonic, op_str in md.disasm_lite( - data, kvo + func_addr - ): - # print("{} {} {} {}".format(address, size, mnemonic, op_str)) - instruction_count += 1 - if mnemonic.startswith("sar"): - # if we don't want to parse op strings, we can disasm the - # single sar instruction again, but we use disasm_lite for speed - self._sar_value = int(op_str.split(",")[1].strip(), 16) - vollog.debug( - f"SAR located at {hex(address)} with value of {hex(self._sar_value)}" - ) - break - - if self._sar_value is None: - vollog.warning( - f"Failed to to locate SAR value having parsed {instruction_count} instructions, failing back to a common value of {hex(DEFAULT_SAR_VALUE)}" - ) - self._sar_value = DEFAULT_SAR_VALUE - - return self._sar_value - @classmethod def get_type_map( cls, @@ -335,8 +227,7 @@ def _make_handle_array(self, offset, level, depth=0): for entry in table: if level > 0: - for x in self._make_handle_array(entry, level - 1, depth): - yield x + yield from self._make_handle_array(entry, level - 1, depth) depth += 1 else: handle_multiplier = 4 @@ -372,8 +263,7 @@ def handles(self, handle_table): ) return None - for handle_table_entry in self._make_handle_array(TableCode, table_levels): - yield handle_table_entry + yield from self._make_handle_array(TableCode, table_levels) def _generator(self, procs): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/windows/hollowprocesses.py b/volatility3/framework/plugins/windows/hollowprocesses.py index 69fa94f063..30d4b602c6 100644 --- a/volatility3/framework/plugins/windows/hollowprocesses.py +++ b/volatility3/framework/plugins/windows/hollowprocesses.py @@ -12,20 +12,15 @@ vollog = logging.getLogger(__name__) -VadData = NamedTuple( - "VadData", - [ - ("protection", str), - ("path", str), - ], -) - -DLLData = NamedTuple( - "DLLData", - [ - ("path", str), - ], -) + +class VadData(NamedTuple): + protection: str + path: str + + +class DLLData(NamedTuple): + path: str + ### Useful references on process hollowing # https://cysinfo.com/detecting-deceptive-hollowing-techniques/ @@ -146,9 +141,7 @@ def _check_load_address(self, proc, _, __) -> Generator[str, None, None]: """ image_base = self._get_image_base(proc) if image_base is not None and image_base != proc.SectionBaseAddress: - yield "The ImageBaseAddress reported from the PEB ({:#x}) does not match the process SectionBaseAddress ({:#x})".format( - image_base, proc.SectionBaseAddress - ) + yield f"The ImageBaseAddress reported from the PEB ({image_base:#x}) does not match the process SectionBaseAddress ({proc.SectionBaseAddress:#x})" def _check_exe_protection( self, proc, vads: Dict[int, VadData], __ @@ -166,13 +159,9 @@ def _check_exe_protection( base = proc.SectionBaseAddress if base not in vads: - yield "There is no VAD starting at the base address of the process executable ({:#x})".format( - base - ) + yield f"There is no VAD starting at the base address of the process executable ({base:#x})" elif vads[base].protection != "PAGE_EXECUTE_WRITECOPY": - yield "Unexpected protection ({}) for VAD hosting the process executable ({:#x}) with path {}".format( - vads[base].protection, base, vads[base].path - ) + yield f"Unexpected protection ({vads[base].protection}) for VAD hosting the process executable ({base:#x}) with path {vads[base].path}" def _check_dlls_protection( self, _, vads: Dict[int, VadData], dlls: Dict[int, DLLData] @@ -184,9 +173,7 @@ def _check_dlls_protection( # PAGE_EXECUTE_WRITECOPY is the only valid permission for mapped DLLs and .exe files if vads[dll_base].protection != "PAGE_EXECUTE_WRITECOPY": - yield "Unexpected protection ({}) for DLL in the PEB's load order list ({:#x}) with path {}".format( - vads[dll_base].protection, dll_base, dlls[dll_base].path - ) + yield f"Unexpected protection ({vads[dll_base].protection}) for DLL in the PEB's load order list ({dll_base:#x}) with path {dlls[dll_base].path}" def _generator(self, procs): checks = [ diff --git a/volatility3/framework/plugins/windows/iat.py b/volatility3/framework/plugins/windows/iat.py index d2fdc0ad89..3bf7f57ed6 100644 --- a/volatility3/framework/plugins/windows/iat.py +++ b/volatility3/framework/plugins/windows/iat.py @@ -1,7 +1,9 @@ # This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 -import logging, io, pefile +import logging +import io +import pefile from volatility3.framework.symbols import intermed from volatility3.framework import renderers, interfaces, exceptions, constants from volatility3.framework.configuration import requirements @@ -119,9 +121,7 @@ def _generator(self, procs): ) except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/plugins/windows/indirect_system_calls.py b/volatility3/framework/plugins/windows/indirect_system_calls.py new file mode 100644 index 0000000000..dac0f9c4af --- /dev/null +++ b/volatility3/framework/plugins/windows/indirect_system_calls.py @@ -0,0 +1,122 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import struct +import logging +from typing import List, Optional + +from volatility3.framework import interfaces, exceptions +from volatility3.framework.configuration import requirements +from volatility3.plugins import yarascan +from volatility3.plugins.windows import pslist, direct_system_calls + +vollog = logging.getLogger(__name__) + + +class IndirectSystemCalls(direct_system_calls.DirectSystemCalls): + _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.syscall_finder = direct_system_calls.syscall_finder_type( + # gets the target address of a indirect jmp + self._indirect_syscall_block_target, + # we are looking for indirect system calls, so we don't want 'syscall' instructions in our code block + False, + # jmp [address]; ret + "/\\xff\\x25[^\\xc3]{,24}\\xc3/", + # any of these mean we aren't in a malicious indirect call + ["call", "leave", "int3", "ret"], + # stop at jmp, this should reference the system call instruction + ["jmp"], + ) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # create a list of requirements for vadyarascan + vadyarascan_requirements = [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 1, 0) + ), + requirements.PluginRequirement( + name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="direct_system_calls", + plugin=direct_system_calls.DirectSystemCalls, + version=(1, 0, 0), + ), + ] + + # get base yarascan requirements for command line options + yarascan_requirements = yarascan.YaraScan.get_yarascan_option_requirements() + + # return the combined requirements + return yarascan_requirements + vadyarascan_requirements + + @staticmethod + def _indirect_syscall_block_target( + proc_layer: interfaces.layers.DataLayerInterface, inst + ) -> Optional[int]: + """ + This function determines the address of a jmp in the following form: + + jmp [address] + + To determine this, we must: + 1) Pull the 4 byte relative offset of 'address' inside the instruction + 2) Compute the full address of this relative offset + 3) Read from the address as it is being dereferenced + 4) Ensure the target address points to a 'syscall' instruction + + Args: + proc_layer: the layer of the potential syscall block + inst: the terminating instruction of the syscall block check + Returns: + The target address of the jump if it can be computed + """ + + try: + jmp_address_str = proc_layer.read(inst.address, 6) + except exceptions.InvalidAddressException: + return None + + # Should be an jmp... + if jmp_address_str[0:2] != b"\xff\x25": + return None + + # get the address of the 'jmp [address]' instruction + relative_offset = struct.unpack(" interfaces.objects.ObjectInterface: + try: + primary = context.layers[primary_layer_name] + except KeyError: + vollog.error( + "Unable to obtain primary layer for scanning. Please file a bug on GitHub about this issue." + ) + return + + try: + phys_layer = primary.config["memory_layer"] + except KeyError: + vollog.error( + "Unable to obtain memory layer from primary layer. Please file a bug on GitHub about this issue." + ) + return + + layer = context.layers[phys_layer] # Yara Rule to scan for MFT Header Signatures rules = yarascan.YaraScan.process_yara_options( @@ -43,30 +77,34 @@ def _generator(self): # Read in the Symbol File symbol_table = intermed.IntermediateSymbolTable.create( - context=self.context, - config_path=self.config_path, + context=context, + config_path=config_path, sub_path="windows", filename="mft", - class_types={"FILE_NAME_ENTRY": mft.MFTFileName, "MFT_ENTRY": mft.MFTEntry}, + class_types={ + "FILE_NAME_ENTRY": mft.MFTFileName, + "MFT_ENTRY": mft.MFTEntry, + "ATTRIBUTE": mft.MFTAttribute, + }, ) # get each of the individual Field Sets mft_object = symbol_table + constants.BANG + "MFT_ENTRY" attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" - si_object = symbol_table + constants.BANG + "STANDARD_INFORMATION_ENTRY" - fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + record_map = {} # Scan the layer for Raw MFT records and parse the fields for offset, _rule_name, _name, _value in layer.scan( - context=self.context, scanner=yarascan.YaraScanner(rules=rules) + context=context, scanner=yarascan.YaraScanner(rules=rules) ): - with contextlib.suppress(exceptions.PagedInvalidAddressException): - mft_record = self.context.object( + with contextlib.suppress(exceptions.InvalidAddressException): + mft_record = context.object( mft_object, offset=offset, layer_name=layer.name ) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset - attr = self.context.object( + attr = context.object( attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, @@ -74,60 +112,8 @@ def _generator(self): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while attr.Attr_Header.AttrType.is_valid_choice: - vollog.debug(f"Attr Type: {attr.Attr_Header.AttrType.lookup()}") - - # MFT Flags determine the file type or dir - # If we don't have a valid enum, coerce to hex so we can keep the record - try: - mft_flag = mft_record.Flags.lookup() - except ValueError: - mft_flag = hex(mft_record.Flags) - - # Standard Information Attribute - if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": - attr_data = attr.Attr_Data.cast(si_object) - yield 0, ( - format_hints.Hex(attr_data.vol.offset), - mft_record.get_signature(), - mft_record.RecordNumber, - mft_record.LinkCount, - mft_flag, - renderers.NotApplicableValue(), - attr.Attr_Header.AttrType.lookup(), - conversion.wintime_to_datetime(attr_data.CreationTime), - conversion.wintime_to_datetime(attr_data.ModifiedTime), - conversion.wintime_to_datetime(attr_data.UpdatedTime), - conversion.wintime_to_datetime(attr_data.AccessedTime), - renderers.NotApplicableValue(), - ) - - # File Name Attribute - if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - attr_data = attr.Attr_Data.cast(fn_object) - file_name = attr_data.get_full_name() - - # If we don't have a valid enum, coerce to hex so we can keep the record - try: - permissions = attr_data.Flags.lookup() - except ValueError: - permissions = hex(attr_data.Flags) - - yield 1, ( - format_hints.Hex(attr_data.vol.offset), - mft_record.get_signature(), - mft_record.RecordNumber, - mft_record.LinkCount, - mft_flag, - permissions, - attr.Attr_Header.AttrType.lookup(), - conversion.wintime_to_datetime(attr_data.CreationTime), - conversion.wintime_to_datetime(attr_data.ModifiedTime), - conversion.wintime_to_datetime(attr_data.UpdatedTime), - conversion.wintime_to_datetime(attr_data.AccessedTime), - file_name, - ) + yield from attr_callback(record_map, mft_record, attr, symbol_table) # If there's no advancement the loop will never end, so break it now if attr.Attr_Header.Length == 0: @@ -135,12 +121,172 @@ def _generator(self): # Update the base offset to point to the next attribute attr_base_offset += attr.Attr_Header.Length - attr = self.context.object( + # Get the next attribute + attr = context.object( attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) + @staticmethod + def parse_mft_records( + record_map: Dict[int, Tuple[str, int, int]], + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + symbol_table_name: str, + ): + # MFT Flags determine the file type or dir + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + mft_flag = mft_record.Flags.lookup() + except ValueError: + mft_flag = hex(mft_record.Flags) + + # Standard Information Attribute + if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": + si_object = ( + symbol_table_name + constants.BANG + "STANDARD_INFORMATION_ENTRY" + ) + attr_data = attr.Attr_Data.cast(si_object) + yield 0, ( + format_hints.Hex(attr_data.vol.offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + mft_flag, + renderers.NotApplicableValue(), + attr.Attr_Header.AttrType.lookup(), + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + renderers.NotApplicableValue(), + ) + + # File Name Attribute + elif attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + fn_object = symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" + + attr_data = attr.Attr_Data.cast(fn_object) + file_name = attr_data.get_full_name() + + # If we don't have a valid enum, coerce to hex so we can keep the record + try: + permissions = attr_data.Flags.lookup() + except ValueError: + permissions = hex(attr_data.Flags) + + yield 1, ( + format_hints.Hex(attr_data.vol.offset), + mft_record.get_signature(), + mft_record.RecordNumber, + mft_record.LinkCount, + mft_flag, + permissions, + attr.Attr_Header.AttrType.lookup(), + conversion.wintime_to_datetime(attr_data.CreationTime), + conversion.wintime_to_datetime(attr_data.ModifiedTime), + conversion.wintime_to_datetime(attr_data.UpdatedTime), + conversion.wintime_to_datetime(attr_data.AccessedTime), + file_name, + ) + + @staticmethod + def parse_data_record( + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + record_map: Dict[int, Tuple[str, int, int]], + return_first_record: bool, + ) -> Generator[Iterable, None, None]: + """ + Returns the parsed data from a MFT record + """ + # we only care about resident data + if attr.Attr_Header.NonResidentFlag: + return + + # we aren't looking ADS when we want the first data record + if return_first_record: + ads_name = renderers.NotApplicableValue() + + # skip records without a name if we want ADS entries + elif attr.Attr_Header.NameLength == 0: + return + + else: + # past the first $DATA record, attempt to get the ADS name + # NotAvailableValue = > 1st Data, but name was not parsable + ads_name = attr.get_resident_filename() or renderers.NotAvailableValue() + + content = attr.get_resident_filecontent() + if content: + content = format_hints.HexBytes(content) + else: + content = renderers.NotAvailableValue() + + yield ( + format_hints.Hex(record_map[mft_record.vol.offset][2]), + mft_record.get_signature(), + mft_record.RecordNumber, + attr.Attr_Header.AttrType.lookup(), + record_map[mft_record.vol.offset][0], + ads_name, + content, + ) + + @classmethod + def parse_data_records( + cls, + record_map: Dict[int, Tuple[str, int, int]], + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + symbol_table_name: str, + return_first_record: bool, + ) -> Generator[Iterable, None, None]: + """ + Parses DATA records while maintaining the FILE_NAME association + from previous parsing of the record + Suports returning the first/main $DATA as well as however many + ADS records a file might have + """ + if mft_record.vol.offset not in record_map: + # file name, DATA count, offset + record_map[mft_record.vol.offset] = [renderers.NotAvailableValue(), 0, None] + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + fn_object = symbol_table_name + constants.BANG + "FILE_NAME_ENTRY" + attr_data = attr.Attr_Data.cast(fn_object) + rec_name = attr_data.get_full_name() + record_map[mft_record.vol.offset][0] = rec_name + elif attr.Attr_Header.AttrType.lookup() == "DATA": + # first data + record_map[mft_record.vol.offset][2] = attr.Attr_Data.vol.offset + + display_data = False + + # first DATA attribute of this record + if record_map[mft_record.vol.offset][1] == 0: + if return_first_record: + display_data = True + + record_map[mft_record.vol.offset][1] = 1 + + # at the second DATA attribute of this record + elif record_map[mft_record.vol.offset][1] == 1 and not return_first_record: + display_data = True + + if display_data: + yield from cls.parse_data_record( + mft_record, attr, record_map, return_first_record + ) + + def _generator(self): + yield from self.enumerate_mft_records( + self.context, + self.config_path, + self.config["primary"], + self.parse_mft_records, + ) + def generate_timeline(self): for row in self._generator(): _depth, row_data = row @@ -177,11 +323,16 @@ def run(self): class ADS(interfaces.plugins.PluginInterface): """Scans for Alternate Data Stream""" - _required_framework_version = (2, 0, 0) + _required_framework_version = (2, 7, 0) + + _version = (1, 0, 0) @classmethod def get_requirements(cls): return [ + requirements.PluginRequirement( + name="MFTScan", plugin=MFTScan, version=(2, 0, 0) + ), requirements.TranslationLayerRequirement( name="primary", description="Memory layer for the kernel", @@ -192,108 +343,102 @@ def get_requirements(cls): ), ] + @staticmethod + def parse_ads_data_records( + record_map: Dict[int, Tuple[str, int, int]], + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + symbol_table_name: str, + ): + return MFTScan.parse_data_records( + record_map, mft_record, attr, symbol_table_name, False + ) + def _generator(self): - layer = self.context.layers[self.config["primary"]] + for ( + offset, + rec_type, + rec_num, + attr_type, + file_name, + ads_name, + content, + ) in MFTScan.enumerate_mft_records( + self.context, + self.config_path, + self.config["primary"], + self.parse_ads_data_records, + ): + yield ( + 0, + (offset, rec_type, rec_num, attr_type, file_name, ads_name, content), + ) - # Yara Rule to scan for MFT Header Signatures - rules = yarascan.YaraScan.process_yara_options( - {"yara_string": "/FILE0|FILE\\*|BAAD/"} + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Record Type", str), + ("Record Number", int), + ("MFT Type", str), + ("Filename", str), + ("ADS Filename", str), + ("Hexdump", format_hints.HexBytes), + ], + self._generator(), ) - # Read in the Symbol File - symbol_table = intermed.IntermediateSymbolTable.create( - context=self.context, - config_path=self.config_path, - sub_path="windows", - filename="mft", - class_types={ - "MFT_ENTRY": mft.MFTEntry, - "FILE_NAME_ENTRY": mft.MFTFileName, - "ATTRIBUTE": mft.MFTAttribute, - }, - ) - # get each of the individual Field Sets - mft_object = symbol_table + constants.BANG + "MFT_ENTRY" - attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" - fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" +class ResidentData(interfaces.plugins.PluginInterface): + """Scans for MFT Records with Resident Data""" - # Scan the layer for Raw MFT records and parse the fields - for offset, _rule_name, _name, _value in layer.scan( - context=self.context, scanner=yarascan.YaraScanner(rules=rules) - ): - with contextlib.suppress(exceptions.PagedInvalidAddressException): - mft_record = self.context.object( - mft_object, offset=offset, layer_name=layer.name - ) - # We will update this on each pass in the next loop and use it as the new offset. - attr_base_offset = mft_record.FirstAttrOffset + _required_framework_version = (2, 7, 0) - attr = self.context.object( - attribute_object, - offset=offset + attr_base_offset, - layer_name=layer.name, - ) + _version = (1, 0, 0) - # There is no field that has a count of Attributes - # Keep Attempting to read attributes until we get an invalid attr.AttrType - is_ads = False - file_name = renderers.NotAvailableValue - # The First $DATA Attr is the 'principal' file itself not the ADS - while attr.Attr_Header.AttrType.is_valid_choice: - if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": - attr_data = attr.Attr_Data.cast(fn_object) - file_name = attr_data.get_full_name() - if attr.Attr_Header.AttrType.lookup() == "DATA": - if is_ads: - if not attr.Attr_Header.NonResidentFlag: - # Resident files are the most interesting. - if attr.Attr_Header.NameLength > 0: - ads_name = attr.get_resident_filename() - if not ads_name: - ads_name = renderers.NotAvailableValue - - content = attr.get_resident_filecontent() - if content: - # Preparing for Disassembly - disasm = interfaces.renderers.BaseAbsentValue - architecture = layer.metadata.get( - "architecture", None - ) - if architecture: - disasm = interfaces.renderers.Disassembly( - content, 0, architecture.lower() - ) - content = format_hints.HexBytes(content) - else: - content = renderers.NotAvailableValue() - disasm = interfaces.renderers.BaseAbsentValue() - - yield 0, ( - format_hints.Hex(attr_data.vol.offset), - mft_record.get_signature(), - mft_record.RecordNumber, - attr.Attr_Header.AttrType.lookup(), - file_name, - ads_name, - content, - disasm, - ) - else: - is_ads = True + @classmethod + def get_requirements(cls): + return [ + requirements.PluginRequirement( + name="MFTScan", plugin=MFTScan, version=(2, 0, 0) + ), + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + ] - # If there's no advancement the loop will never end, so break it now - if attr.Attr_Header.Length == 0: - break + @staticmethod + def parse_first_data_records( + record_map: Dict[int, Tuple[str, int, int]], + mft_record: interfaces.objects.ObjectInterface, + attr: interfaces.objects.ObjectInterface, + symbol_table_name: str, + ): + return MFTScan.parse_data_records( + record_map, mft_record, attr, symbol_table_name, True + ) - # Update the base offset to point to the next attribute - attr_base_offset += attr.Attr_Header.Length - # Get the next attribute - attr = self.context.object( - attribute_object, - offset=offset + attr_base_offset, - layer_name=layer.name, - ) + def _generator(self): + for ( + offset, + rec_type, + rec_num, + attr_type, + file_name, + _, + content, + ) in MFTScan.enumerate_mft_records( + self.context, + self.config_path, + self.config["primary"], + self.parse_first_data_records, + ): + yield (0, (offset, rec_type, rec_num, attr_type, file_name, content)) def run(self): return renderers.TreeGrid( @@ -303,9 +448,7 @@ def run(self): ("Record Number", int), ("MFT Type", str), ("Filename", str), - ("ADS Filename", str), ("Hexdump", format_hints.HexBytes), - ("Disasm", interfaces.renderers.Disassembly), ], self._generator(), ) diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index ba45834d5c..85eb474a8b 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -2,14 +2,14 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import logging -from typing import List, Iterable, Generator +from typing import Generator, Iterable, List, Optional -from volatility3.framework import exceptions, interfaces, constants, renderers +from volatility3.framework import constants, exceptions, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe -from volatility3.plugins.windows import pslist, pedump +from volatility3.plugins.windows import pedump, pslist vollog = logging.getLogger(__name__) @@ -104,12 +104,11 @@ def _generator(self): try: BaseDllName = mod.BaseDllName.get_string() + if self.config["name"] and self.config["name"] not in BaseDllName: + continue except exceptions.InvalidAddressException: BaseDllName = interfaces.renderers.BaseAbsentValue() - if self.config["name"] and self.config["name"] not in BaseDllName: - continue - try: FullDllName = mod.FullDllName.get_string() except exceptions.InvalidAddressException: @@ -134,7 +133,7 @@ def get_session_layers( context: interfaces.context.ContextInterface, layer_name: str, symbol_table: str, - pids: List[int] = None, + pids: Optional[List[int]] = None, ) -> Generator[str, None, None]: """Build a cache of possible virtual layers, in priority starting with the primary/kernel layer. Then keep one layer per session by cycling @@ -165,26 +164,42 @@ def get_session_layers( # create the session space object in the process' own layer. # not all processes have a valid session pointer. - session_space = context.object( - symbol_table + constants.BANG + "_MM_SESSION_SPACE", - layer_name=layer_name, - offset=proc.Session, - ) - - if session_space.SessionId in seen_ids: + try: + session_space = context.object( + symbol_table + constants.BANG + "_MM_SESSION_SPACE", + layer_name=layer_name, + offset=proc.Session, + ) + session_id = session_space.SessionId + + except exceptions.SymbolError: + # In Windows 11 24H2, the _MM_SESSION_SPACE type was + # replaced with _PSP_SESSION_SPACE, and the kernel PDB + # doesn't contain information about its members (otherwise, + # we would just fall back to the new type). However, it + # appears to be, for our purposes, functionally identical + # to the _MM_SESSION_SPACE. Because _MM_SESSION_SPACE + # stores its session ID at offset 8 as an unsigned long, we + # create an unsigned long at that offset and use that + # instead. + session_id = context.object( + layer_name=layer_name, + object_type=symbol_table + constants.BANG + "unsigned long", + offset=proc.Session + 8, + ) + + if session_id in seen_ids: continue except exceptions.InvalidAddressException: vollog.log( constants.LOGLEVEL_VVV, - "Process {} does not have a valid Session or a layer could not be constructed for it".format( - proc_id - ), + f"Process {proc_id} does not have a valid Session or a layer could not be constructed for it", ) continue # save the layer if we haven't seen the session yet - seen_ids.append(session_space.SessionId) + seen_ids.append(session_id) yield proc_layer_name @classmethod @@ -250,8 +265,7 @@ def list_modules( object_type=type_name, offset=list_entry.vol.offset - reloff, absolute=True ) - for mod in module.InLoadOrderLinks: - yield mod + yield from module.InLoadOrderLinks def run(self): return renderers.TreeGrid( diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 66a24da5ac..1620311045 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -161,20 +161,15 @@ def determine_tcpip_version( raise NotImplementedError( "Kernel Debug Structure version format not supported!" ) - except: - # unsure what to raise here. Also, it might be useful to add some kind of fallback, + except Exception: + # FIXME: unsure what to raise here. Also, it might be useful to add some kind of fallback, # either to a user-provided version or to another method to determine tcpip.sys's version raise exceptions.VolatilityException( "Kernel Debug Structure missing VERSION/KUSER structure, unable to determine Windows version!" ) vollog.debug( - "Determined OS Version: {}.{} {}.{}".format( - kuser.NtMajorVersion, - kuser.NtMinorVersion, - vers.MajorVersion, - vers.MinorVersion, - ) + f"Determined OS Version: {kuser.NtMajorVersion}.{kuser.NtMinorVersion} {vers.MajorVersion}.{vers.MinorVersion}" ) if nt_major_version == 10 and arch == "x64": @@ -272,9 +267,7 @@ def determine_tcpip_version( if ver: tcpip_mod_version = ver[3] vollog.debug( - "Determined tcpip.sys's FileVersion: {}".format( - tcpip_mod_version - ) + f"Determined tcpip.sys's FileVersion: {tcpip_mod_version}" ) else: vollog.debug("Could not determine tcpip.sys's FileVersion.") @@ -316,12 +309,7 @@ def determine_tcpip_version( else: raise NotImplementedError( - "This version of Windows is not supported: {}.{} {}.{}!".format( - nt_major_version, - nt_minor_version, - vers.MajorVersion, - vers_minor_version, - ) + f"This version of Windows is not supported: {nt_major_version}.{nt_minor_version} {vers.MajorVersion}.{vers_minor_version}!" ) vollog.debug(f"Determined symbol filename: {filename}") @@ -510,17 +498,8 @@ def generate_timeline(self): for i in row_data ] description = ( - "Network connection: Process {} {} Local Address {}:{} " - "Remote Address {}:{} State {} Protocol {} ".format( - row_data[7], - row_data[8], - row_data[2], - row_data[3], - row_data[4], - row_data[5], - row_data[6], - row_data[1], - ) + f"Network connection: Process {row_data[7]} {row_data[8]} Local Address {row_data[2]}:{row_data[3]} " + f"Remote Address {row_data[4]}:{row_data[5]} State {row_data[6]} Protocol {row_data[1]} " ) yield (description, timeliner.TimeLinerType.CREATED, row_data[9]) diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index 0908767fc7..a1521a8c6a 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -311,9 +311,7 @@ def parse_partitions( part_table.Partitions.count = part_count vollog.debug( - "Found TCP connection PartitionTable @ 0x{:x} (partition count: {})".format( - part_table_addr, part_count - ) + f"Found TCP connection PartitionTable @ 0x{part_table_addr:x} (partition count: {part_count})" ) entry_offset = context.symbol_space.get_type(obj_name).relative_child_offset( "ListEntry" @@ -490,14 +488,13 @@ def list_sockets( """ # first, TCP endpoints by parsing the partition table - for endpoint in cls.parse_partitions( + yield from cls.parse_partitions( context, layer_name, net_symbol_table, tcpip_symbol_table, tcpip_module_offset, - ): - yield endpoint + ) # then, towards the UDP and TCP port pools # first, find their addresses @@ -624,9 +621,7 @@ def _generator(self, show_corrupt_results: Optional[bool] = None): proto = "TCPv6" else: vollog.debug( - "TCP Endpoint @ 0x{:2x} has unknown address family 0x{:x}".format( - netw_obj.vol.offset, netw_obj.get_address_family() - ) + f"TCP Endpoint @ 0x{netw_obj.vol.offset:2x} has unknown address family 0x{netw_obj.get_address_family():x}" ) proto = "TCPv?" diff --git a/volatility3/framework/plugins/windows/pe_symbols.py b/volatility3/framework/plugins/windows/pe_symbols.py index 955098d6bf..21e657ab33 100644 --- a/volatility3/framework/plugins/windows/pe_symbols.py +++ b/volatility3/framework/plugins/windows/pe_symbols.py @@ -158,7 +158,7 @@ def _do_get_address(self, name: str) -> Optional[int]: class PDBSymbolFinder(PESymbolFinder): """ - PESymbolFinder implementation for PDB modules + PESymbolFinder implementation for PDB modules """ def _do_get_address(self, name: str) -> Optional[int]: @@ -195,7 +195,7 @@ def _do_get_name(self, address: int) -> Optional[str]: class ExportSymbolFinder(PESymbolFinder): """ - PESymbolFinder implementation for PDB modules + PESymbolFinder implementation for PDB modules """ def _get_name(self, export: pefile.ExportData) -> Optional[str]: @@ -300,7 +300,7 @@ def _get_pefile_obj( base_address: int, ) -> Optional[pefile.PE]: """ - Attempts to pefile object from the bytes of the PE file + Attempts to create a pefile object from the bytes of the PE file Args: pe_table_name: name of the pe types table @@ -645,7 +645,7 @@ def _get_symbol_value( and wanted_addresses_identifier not in wanted_symbols ): vollog.warning( - f"Invalid `wanted_symbols` sent to `find_symbols`. addresses and names keys both misssing." + "Invalid `wanted_symbols` sent to `find_symbols`. addresses and names keys both misssing." ) return diff --git a/volatility3/framework/plugins/windows/pedump.py b/volatility3/framework/plugins/windows/pedump.py index 858d0615a5..5107cb48bf 100644 --- a/volatility3/framework/plugins/windows/pedump.py +++ b/volatility3/framework/plugins/windows/pedump.py @@ -64,30 +64,27 @@ def dump_pe( """ Returns the filename of the dump file or None """ - try: - file_handle = open_method(file_name) - - dos_header = context.object( - pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", - offset=base, - layer_name=layer_name, - ) - - for offset, data in dos_header.reconstruct(): - file_handle.seek(offset) - file_handle.write(data) - except ( - IOError, - exceptions.VolatilityException, - OverflowError, - ValueError, - ) as excp: - vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") - return None - finally: - file_handle.close() - - return file_handle.preferred_filename + with open_method(file_name) as file_handle: + try: + dos_header = context.object( + pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset=base, + layer_name=layer_name, + ) + + for offset, data in dos_header.reconstruct(): + file_handle.seek(offset) + file_handle.write(data) + except ( + OSError, + exceptions.VolatilityException, + OverflowError, + ValueError, + ) as excp: + vollog.debug(f"Unable to dump PE file at offset {base}: {excp}") + return None + + return file_handle.preferred_filename @classmethod def dump_ldr_entry( @@ -96,7 +93,7 @@ def dump_ldr_entry( pe_table_name: str, ldr_entry: interfaces.objects.ObjectInterface, open_method: Type[interfaces.plugins.FileHandlerInterface], - layer_name: str = None, + layer_name: Optional[str] = None, prefix: str = "", ) -> Optional[str]: """Extracts the PE file referenced an LDR_DATA_TABLE_ENTRY (DLL, kernel module) instance @@ -119,12 +116,7 @@ def dump_ldr_entry( if layer_name is None: layer_name = ldr_entry.vol.layer_name - file_name = "{}{}.{:#x}.{:#x}.dmp".format( - prefix, - ntpath.basename(name), - ldr_entry.vol.offset, - ldr_entry.DllBase, - ) + file_name = f"{prefix}{ntpath.basename(name)}.{ldr_entry.vol.offset:#x}.{ldr_entry.DllBase:#x}.dmp" return cls.dump_pe( context, @@ -146,11 +138,7 @@ def dump_pe_at_base( pid: int, base: int, ) -> Optional[str]: - file_name = "PE.{:#x}.{:d}.{:#x}.dmp".format( - proc_offset, - pid, - base, - ) + file_name = f"PE.{proc_offset:#x}.{pid:d}.{base:#x}.dmp" return PEDump.dump_pe( context, pe_table_name, layer_name, open_method, file_name, base @@ -227,11 +215,11 @@ def _generator(self): ) if self.config["kernel_module"] and self.config["pid"]: - vollog.error("Only --kernel_module or --pid should be set. Not both") + vollog.error("Only 'kernel-module' or 'pid' should be set, not both") return if not self.config["kernel_module"] and not self.config["pid"]: - vollog.error("--kernel_module or --pid must be set") + vollog.error("Either 'kernel-module' or 'pid' argument must be set") return if self.config["kernel_module"]: diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index 1f70cfb8c7..efde096387 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -139,7 +139,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] architectures=["Intel32", "Intel64"], ), requirements.PluginRequirement( - name="handles", plugin=handles.Handles, version=(1, 0, 0) + name="handles", plugin=handles.Handles, version=(2, 0, 0) ), ] @@ -183,7 +183,7 @@ def _generator(self): @staticmethod def builtin_constraints( - symbol_table: str, tags_filter: List[bytes] = None + symbol_table: str, tags_filter: Optional[List[bytes]] = None ) -> List[PoolConstraint]: """Get built-in PoolConstraints given a list of pool tags. diff --git a/volatility3/framework/plugins/windows/privileges.py b/volatility3/framework/plugins/windows/privileges.py index 0370dfc921..7b4d002056 100644 --- a/volatility3/framework/plugins/windows/privileges.py +++ b/volatility3/framework/plugins/windows/privileges.py @@ -39,7 +39,7 @@ def __init__(self, *args, **kwargs): ) # Get service sids dictionary (we need only the service sids). - with open(sids_json_file_name, "r") as file_handle: + with open(sids_json_file_name) as file_handle: temp_json = json.load(file_handle)["privileges"] self.privilege_info = { int(priv_num): temp_json[priv_num] for priv_num in temp_json diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 478cc8b1b6..579a235d8c 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import Callable, Iterator, List, Type +from typing import Callable, Iterator, List, Optional, Type from volatility3.framework import renderers, interfaces, layers, exceptions, constants from volatility3.framework.configuration import requirements @@ -114,7 +114,7 @@ def process_dump( @classmethod def create_pid_filter( - cls, pid_list: List[int] = None, exclude: bool = False + cls, pid_list: Optional[List[int]] = None, exclude: bool = False ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process IDs. @@ -126,15 +126,24 @@ def create_pid_filter( Returns: Filter function for passing to the `list_processes` method """ - filter_func = lambda _: False + + def filter_func(_): + return False + # FIXME: mypy #4973 or #2608 pid_list = pid_list or [] filter_list = [x for x in pid_list if x is not None] if filter_list: if exclude: - filter_func = lambda x: x.UniqueProcessId in filter_list + + def filter_func(x): + return x.UniqueProcessId in filter_list + else: - filter_func = lambda x: x.UniqueProcessId not in filter_list + + def filter_func(x): + return x.UniqueProcessId not in filter_list + return filter_func @classmethod @@ -162,7 +171,7 @@ def create_active_process_filter( @classmethod def create_name_filter( - cls, name_list: List[str] = None, exclude: bool = False + cls, name_list: Optional[List[str]] = None, exclude: bool = False ) -> Callable[[interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process names. @@ -173,20 +182,24 @@ def create_name_filter( Returns: Filter function for passing to the `list_processes` method """ - filter_func = lambda _: False + + def filter_func(_): + return False + # FIXME: mypy #4973 or #2608 name_list = name_list or [] filter_list = [x for x in name_list if x is not None] if filter_list: if exclude: - filter_func = ( - lambda x: utility.array_to_string(x.ImageFileName) in filter_list - ) + + def filter_func(x): + return utility.array_to_string(x.ImageFileName) in filter_list + else: - filter_func = ( - lambda x: utility.array_to_string(x.ImageFileName) - not in filter_list - ) + + def filter_func(x): + return utility.array_to_string(x.ImageFileName) not in filter_list + return filter_func @classmethod diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index 5ce470cd86..cdf344ee63 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -89,7 +89,7 @@ def create_offset_filter( cls, context: interfaces.context.ContextInterface, layer_name: str, - offset: int = None, + offset: Optional[int] = None, physical: bool = True, exclude: bool = False, ) -> Callable[[interfaces.objects.ObjectInterface], bool]: @@ -102,29 +102,38 @@ def create_offset_filter( Returns: Filter function to be passed to the list of processes. """ - filter_func = lambda _: False + + def filter_func(_): + return False if offset: if physical: if exclude: - filter_func = ( - lambda proc: cls.physical_offset_from_virtual( - context, layer_name, proc + + def filter_func(proc): + return ( + cls.physical_offset_from_virtual(context, layer_name, proc) + == offset ) - == offset - ) + else: - filter_func = ( - lambda proc: cls.physical_offset_from_virtual( - context, layer_name, proc + + def filter_func(proc): + return ( + cls.physical_offset_from_virtual(context, layer_name, proc) + != offset ) - != offset - ) + else: if exclude: - filter_func = lambda proc: proc.vol.offset == offset + + def filter_func(proc): + return proc.vol.offset == offset + else: - filter_func = lambda proc: proc.vol.offset != offset + + def filter_func(proc): + return proc.vol.offset != offset return filter_func diff --git a/volatility3/framework/plugins/windows/psxview.py b/volatility3/framework/plugins/windows/psxview.py index a8d185a2c0..b5ddd2ee5f 100644 --- a/volatility3/framework/plugins/windows/psxview.py +++ b/volatility3/framework/plugins/windows/psxview.py @@ -14,7 +14,6 @@ info, pslist, psscan, - sessions, thrdscan, ) @@ -26,7 +25,7 @@ class PsXView(plugins.PluginInterface): identify processes that are trying to hide themselves. I recommend using -r pretty if you are looking at this plugin's output in a terminal.""" - # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the funcitonality + # I've omitted the desktop thread scanning method because Volatility3 doesn't appear to have the functionality # which the original plugin used to do it. # The sessions method is omitted because it begins with the list of processes found by Pslist anyway. @@ -62,7 +61,7 @@ def get_requirements(cls): name="thrdscan", component=thrdscan.ThrdScan, version=(1, 0, 0) ), requirements.VersionRequirement( - name="handles", component=handles.Handles, version=(1, 0, 0) + name="handles", component=handles.Handles, version=(2, 0, 0) ), requirements.BooleanRequirement( name="physical-offsets", @@ -219,7 +218,7 @@ def _generator(self): name = self._proc_name_to_string(proc) exit_time = proc.get_exit_time() - if type(exit_time) != datetime.datetime: + if type(exit_time) is not datetime.datetime: exit_time = "" else: exit_time = str(exit_time) diff --git a/volatility3/framework/plugins/windows/registry/hivelist.py b/volatility3/framework/plugins/windows/registry/hivelist.py index ddc9c18555..91a99a9fb2 100644 --- a/volatility3/framework/plugins/windows/registry/hivelist.py +++ b/volatility3/framework/plugins/windows/registry/hivelist.py @@ -232,10 +232,8 @@ def list_hive_objects( for hive in hg: if hive.vol.offset in seen: vollog.debug( - "Hivelist found an already seen offset {} while " - "traversing forwards, this should not occur".format( - hex(hive.vol.offset) - ) + f"Hivelist found an already seen offset {hex(hive.vol.offset)} while " + "traversing forwards, this should not occur" ) break seen.add(hive.vol.offset) @@ -249,18 +247,14 @@ def list_hive_objects( forward_invalid = hg.invalid if forward_invalid: vollog.debug( - "Hivelist failed traversing the list forwards at {}, traversing backwards".format( - hex(forward_invalid) - ) + f"Hivelist failed traversing the list forwards at {hex(forward_invalid)}, traversing backwards" ) hg = HiveGenerator(cmhive, forward=False) for hive in hg: if hive.vol.offset in seen: vollog.debug( - "Hivelist found an already seen offset {} while " - "traversing backwards, list walking met in the middle".format( - hex(hive.vol.offset) - ) + f"Hivelist found an already seen offset {hex(hive.vol.offset)} while " + "traversing backwards, list walking met in the middle" ) break seen.add(hive.vol.offset) @@ -281,10 +275,8 @@ def list_hive_objects( # by walking the list, so revert to scanning, and walk the list forwards and backwards from each # found hive vollog.debug( - "Hivelist failed traversing backwards at {}, a different " - "location from forwards, revert to scanning".format( - hex(backward_invalid) - ) + f"Hivelist failed traversing backwards at {hex(backward_invalid)}, a different " + "location from forwards, revert to scanning" ) for hive in hivescan.HiveScan.scan_hives( context, layer_name, symbol_table @@ -320,9 +312,7 @@ def list_hive_objects( yield linked_hive except exceptions.InvalidAddressException: vollog.debug( - "InvalidAddressException when traversing hive {} found from scan, skipping".format( - hex(hive.vol.offset) - ) + f"InvalidAddressException when traversing hive {hex(hive.vol.offset)} found from scan, skipping" ) def run(self) -> renderers.TreeGrid: diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index 4fe3f97fb8..ed926805bb 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -4,7 +4,7 @@ import datetime import logging -from typing import List, Sequence, Iterable, Tuple, Union +from typing import List, Optional, Sequence, Iterable, Tuple, Union from volatility3.framework import objects, renderers, exceptions, interfaces, constants from volatility3.framework.configuration import requirements @@ -51,7 +51,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] def key_iterator( cls, hive: RegistryHive, - node_path: Sequence[objects.StructType] = None, + node_path: Optional[Sequence[objects.StructType]] = None, recurse: bool = False, ) -> Iterable[ Tuple[ @@ -121,7 +121,7 @@ def key_iterator( def _printkey_iterator( self, hive: RegistryHive, - node_path: Sequence[objects.StructType] = None, + node_path: Optional[Sequence[objects.StructType]] = None, recurse: bool = False, ): """Method that wraps the more generic key_iterator, to provide output @@ -242,8 +242,8 @@ def _registry_walker( self, layer_name: str, symbol_table: str, - hive_offsets: List[int] = None, - key: str = None, + hive_offsets: Optional[List[int]] = None, + key: Optional[str] = None, recurse: bool = False, ): for hive in hivelist.HiveList.list_hives( diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index bd832b20c1..932ee9d6f9 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -39,7 +39,7 @@ def __init__(self, *args, **kwargs): os.path.join(os.path.dirname(__file__), "userassist.json"), "rb" ) as fp: self._folder_guids = json.load(fp) - except IOError: + except OSError: vollog.error("Usersassist data file not found") @classmethod @@ -308,9 +308,7 @@ def _generator(self): ) except exceptions.InvalidAddressException as excp: vollog.debug( - "Invalid address identified in lower layer {}: {}".format( - excp.layer_name, excp.invalid_address - ) + f"Invalid address identified in lower layer {excp.layer_name}: {excp.invalid_address}" ) except KeyError: vollog.debug( diff --git a/volatility3/framework/plugins/windows/scheduled_tasks.py b/volatility3/framework/plugins/windows/scheduled_tasks.py index 277a0d8563..6dd5613c47 100644 --- a/volatility3/framework/plugins/windows/scheduled_tasks.py +++ b/volatility3/framework/plugins/windows/scheduled_tasks.py @@ -270,7 +270,6 @@ def read_bstring(self, aligned=False) -> Optional[str]: return val def read_aligned_bstring_expand_sz(self) -> Optional[str]: - # type: () -> Optional[str] sz = self.read_aligned_u4() if sz is None: return None diff --git a/volatility3/framework/plugins/windows/shimcachemem.py b/volatility3/framework/plugins/windows/shimcachemem.py index 6afaf43560..b8e9b5bd7f 100644 --- a/volatility3/framework/plugins/windows/shimcachemem.py +++ b/volatility3/framework/plugins/windows/shimcachemem.py @@ -17,8 +17,6 @@ from volatility3.plugins import timeliner from volatility3.plugins.windows import modules, pslist, vadinfo -# from volatility3.plugins.windows import pslist, vadinfo, modules - vollog = logging.getLogger(__name__) @@ -146,7 +144,7 @@ def find_shimcache_win_xp( context, layer_name, kernel_symbol_table ): pid = process.UniqueProcessId - vollog.debug("checking process %d" % pid) + vollog.debug("checking process %d", pid) for vad in vadinfo.VadInfo.list_vads( process, lambda x: x.get_tag() == b"Vad " and x.Protection == 4 ): @@ -285,10 +283,9 @@ def find_shimcache_win_2k3_to_7( if not shim_head: return - for shim_entry in shim_head.ListEntry.to_list( + yield from shim_head.ListEntry.to_list( shimcache_symbol_table + constants.BANG + "SHIM_CACHE_ENTRY", "ListEntry" - ): - yield shim_entry + ) @classmethod def try_get_shim_head_at_offset( @@ -333,7 +330,7 @@ def try_get_shim_head_at_offset( eresource_rel_off = ersrc_size + ((offset - ersrc_size) % ersrc_alignment) eresource_offset = offset - eresource_rel_off - vollog.debug("Constructing ERESOURCE at %s" % hex(eresource_offset)) + vollog.debug(f"Constructing ERESOURCE at {hex(eresource_offset)}") eresource = context.object( kernel_symbol_table + constants.BANG + "_ERESOURCE", layer_name, diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index d321c2cc0c..f5d7e1b3ac 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -172,9 +172,7 @@ def _construct_ecrypt_array( except exceptions.InvalidAddressException: vollog.debug( - "Unable to construct cSystems array at given offset: {:x}".format( - array_start - ) + f"Unable to construct cSystems array at given offset: {array_start:x}" ) array = None @@ -291,9 +289,7 @@ def _find_lsass_proc( except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None, None diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index 0eaa65884b..b8dea0cdd3 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -170,9 +170,7 @@ def generate_mapping( proc_layer_name = process.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/plugins/windows/svclist.py b/volatility3/framework/plugins/windows/svclist.py index a59581063b..ea73247ce2 100644 --- a/volatility3/framework/plugins/windows/svclist.py +++ b/volatility3/framework/plugins/windows/svclist.py @@ -85,9 +85,7 @@ def service_list( layer_name = proc.add_process_layer() except exceptions.InvalidAddressException: vollog.warning( - "Unable to access memory of services.exe running with PID: {}".format( - proc.UniqueProcessId - ) + f"Unable to access memory of services.exe running with PID: {proc.UniqueProcessId}" ) continue @@ -105,11 +103,10 @@ def service_list( scanner=scanners.BytesScanner(needle=b"Sc27"), sections=exe_range, ): - for record in cls.enumerate_vista_or_later_header( + yield from cls.enumerate_vista_or_later_header( context, service_table_name, service_binary_dll_map, layer_name, offset, - ): - yield record + ) diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 52ed5e759e..bd477ba274 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -20,26 +20,22 @@ from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import versions from volatility3.framework.symbols.windows.extensions import services as services_types -from volatility3.plugins.windows import poolscanner, pslist, vadyarascan +from volatility3.plugins.windows import poolscanner, pslist from volatility3.plugins.windows.registry import hivelist vollog = logging.getLogger(__name__) -ServiceBinaryInfo = NamedTuple( - "ServiceBinaryInfo", - [ - ("dll", Union[str, interfaces.renderers.BaseAbsentValue]), - ("binary", Union[str, interfaces.renderers.BaseAbsentValue]), - ], -) +class ServiceBinaryInfo(NamedTuple): + dll: Union[str, interfaces.renderers.BaseAbsentValue] + binary: Union[str, interfaces.renderers.BaseAbsentValue] class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" _required_framework_version = (2, 0, 0) - _version = (3, 0, 0) + _version = (3, 0, 1) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -60,9 +56,6 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] requirements.PluginRequirement( name="poolscanner", plugin=poolscanner.PoolScanner, version=(1, 0, 0) ), - requirements.PluginRequirement( - name="vadyarascan", plugin=vadyarascan.VadYaraScan, version=(1, 0, 0) - ), requirements.PluginRequirement( name="hivelist", plugin=hivelist.HiveList, version=(1, 0, 0) ), @@ -309,18 +302,23 @@ def service_scan( proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue layer = context.layers[proc_layer_name] + # get process sections for scanning + sections = [] + for vad in task.get_vad_root().traverse(): + base = vad.get_start() + if vad.get_size(): + sections.append((base, vad.get_size())) + for offset in layer.scan( context=context, scanner=scanners.BytesScanner(needle=service_tag), - sections=vadyarascan.VadYaraScan.get_vad_maps(task), + sections=sections, ): if not is_vista_or_later: service_record = context.object( diff --git a/volatility3/framework/plugins/windows/thrdscan.py b/volatility3/framework/plugins/windows/thrdscan.py index b812a15ffb..c0963e754c 100644 --- a/volatility3/framework/plugins/windows/thrdscan.py +++ b/volatility3/framework/plugins/windows/thrdscan.py @@ -82,7 +82,7 @@ def gather_thread_info(cls, ethread): ethread.get_exit_time() ) # datetime.datetime object / volatility3.framework.renderers.UnparsableValue object except exceptions.InvalidAddressException: - vollog.debug("Thread invalid address {:#x}".format(ethread.vol.offset)) + vollog.debug(f"Thread invalid address {ethread.vol.offset:#x}") return None return ( diff --git a/volatility3/framework/plugins/windows/threads.py b/volatility3/framework/plugins/windows/threads.py index 98a3169a5b..84daa85952 100644 --- a/volatility3/framework/plugins/windows/threads.py +++ b/volatility3/framework/plugins/windows/threads.py @@ -3,7 +3,7 @@ # import logging -from typing import Callable, Iterable, List, Generator +from typing import Iterable, List, Generator from volatility3.framework import interfaces, constants from volatility3.framework.configuration import requirements @@ -82,5 +82,4 @@ def list_process_threads( symbol_table=symbol_table_name, filter_func=filter_func, ): - for thread in cls.list_threads(module, proc): - yield thread + yield from cls.list_threads(module, proc) diff --git a/volatility3/framework/plugins/windows/timers.py b/volatility3/framework/plugins/windows/timers.py index d49c287842..cd8101a957 100644 --- a/volatility3/framework/plugins/windows/timers.py +++ b/volatility3/framework/plugins/windows/timers.py @@ -14,7 +14,7 @@ ) from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints -from volatility3.framework.symbols.windows import versions +from volatility3.framework.symbols.windows import versions, extensions from volatility3.plugins.windows import ssdt, kpcrs vollog = logging.getLogger(__name__) @@ -24,7 +24,7 @@ class Timers(interfaces.plugins.PluginInterface): """Print kernel timers and associated module DPCs""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -49,7 +49,7 @@ def list_timers( kernel_module_name: str, layer_name: str, symbol_table: str, - ) -> Iterable[Tuple[str, int, str]]: + ) -> Iterable[extensions.KTIMER]: """Lists all kernel timers. Args: @@ -141,7 +141,7 @@ def _generator(self) -> Iterator[Tuple]: if dpc.DeferredRoutine == 0: continue deferred_routine = dpc.DeferredRoutine - except Exception as e: + except Exception: continue module_symbols = list( diff --git a/volatility3/framework/plugins/windows/unhooked_system_calls.py b/volatility3/framework/plugins/windows/unhooked_system_calls.py index 1a1e599407..5b21225c87 100644 --- a/volatility3/framework/plugins/windows/unhooked_system_calls.py +++ b/volatility3/framework/plugins/windows/unhooked_system_calls.py @@ -191,7 +191,7 @@ def _generator(self) -> Generator[Tuple[int, Tuple[str, str, int]], None, None]: # gather processes on small_idx since these are the malware infected ones for pid, pname in cb[small_idx]: - ps.append("{:d}:{}".format(pid, pname)) + ps.append(f"{pid:d}:{pname}") proc_names = ", ".join(ps) diff --git a/volatility3/framework/plugins/windows/unloadedmodules.py b/volatility3/framework/plugins/windows/unloadedmodules.py index 01e5758189..077fe33cbf 100644 --- a/volatility3/framework/plugins/windows/unloadedmodules.py +++ b/volatility3/framework/plugins/windows/unloadedmodules.py @@ -116,8 +116,7 @@ def list_unloadedmodules( ) unloadedmodules_array.UnloadedDrivers.count = unloaded_count - for mod in unloadedmodules_array.UnloadedDrivers: - yield mod + yield from unloadedmodules_array.UnloadedDrivers def _generator(self): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index 2c6ed4dafc..0c4a8aaca4 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -169,9 +169,7 @@ def vad_dump( proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) return None diff --git a/volatility3/framework/plugins/windows/vadregexscan.py b/volatility3/framework/plugins/windows/vadregexscan.py new file mode 100644 index 0000000000..0d35cd6589 --- /dev/null +++ b/volatility3/framework/plugins/windows/vadregexscan.py @@ -0,0 +1,130 @@ +# This file is Copyright 2024 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import re +from typing import List + +from volatility3.framework import renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins, configuration +from volatility3.framework.layers import scanners +from volatility3.framework.renderers import format_hints +from volatility3.plugins.windows import pslist + +vollog = logging.getLogger(__name__) + + +class VadRegExScan(plugins.PluginInterface): + """Scans all virtual memory areas for tasks using RegEx.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + MAXSIZE_DEFAULT = 128 + + @classmethod + def get_requirements(cls) -> List[configuration.RequirementInterface]: + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Windows kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.StringRequirement( + name="pattern", description="RegEx pattern", optional=False + ), + requirements.IntRequirement( + name="maxsize", + description="Maximum size in bytes for displayed context", + default=cls.MAXSIZE_DEFAULT, + optional=True, + ), + ] + + def _generator(self, regex_pattern, procs): + regex_pattern = bytes(regex_pattern, "UTF-8") + vollog.debug(f"RegEx Pattern: {regex_pattern}") + + for proc in procs: + + # attempt to create a process layer for each proc + proc_layer_name = proc.add_process_layer() + if not proc_layer_name: + continue + + # get the proc_layer object from the context + proc_layer = self.context.layers[proc_layer_name] + + # get process sections for scanning + sections = [] + for vad in proc.get_vad_root().traverse(): + base = vad.get_start() + if vad.get_size(): + sections.append((base, vad.get_size())) + + for offset in proc_layer.scan( + context=self.context, + scanner=scanners.RegExScanner(regex_pattern), + sections=sections, + progress_callback=self._progress_callback, + ): + result_data = proc_layer.read(offset, self.MAXSIZE_DEFAULT, pad=True) + + # reapply the regex in order to extact just the match + regex_result = re.match(regex_pattern, result_data) + + if regex_result: + # the match is within the results_data (e.g. it fits within MAXSIZE_DEFAULT) + # extract just the match itself + regex_match = regex_result.group(0) + text_result = str(regex_match, encoding="UTF-8", errors="replace") + bytes_result = regex_match + else: + # the match is not with the results_data (e.g. it doesn't fit within MAXSIZE_DEFAULT) + text_result = str(result_data, encoding="UTF-8", errors="replace") + bytes_result = result_data + + proc_id = proc.UniqueProcessId + process_name = proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ) + yield 0, ( + proc_id, + process_name, + format_hints.Hex(offset), + text_result, + bytes_result, + ) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + kernel = self.context.modules[self.config["kernel"]] + procs = pslist.PsList.list_processes( + self.context, + kernel.layer_name, + kernel.symbol_table_name, + filter_func=filter_func, + ) + return renderers.TreeGrid( + [ + ("PID", int), + ("Process", str), + ("Offset", format_hints.Hex), + ("Text", str), + ("Hex", bytes), + ], + self._generator(self.config.get("pattern"), procs), + ) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index efcc70d07f..2e9cc44ea0 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -32,6 +32,9 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] requirements.PluginRequirement( name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), requirements.PluginRequirement( name="yarascan", plugin=yarascan.YaraScan, version=(2, 0, 0) ), @@ -66,49 +69,40 @@ def _generator(self): ): layer_name = task.add_process_layer() layer = self.context.layers[layer_name] + + max_vad_size = 0 + vad_maps_to_scan = [] + for start, size in self.get_vad_maps(task): if size > sanity_check: vollog.debug( f"VAD at 0x{start:x} over sanity-check size, not scanning" ) continue - - data = layer.read(start, size, True) - if not yarascan.YaraScan._yara_x: - for match in rules.match(data=data): - if yarascan.YaraScan.yara_returns_instances(): - for match_string in match.strings: - for instance in match_string.instances: - yield 0, ( - format_hints.Hex(instance.offset + start), - task.UniqueProcessId, - match.rule, - match_string.identifier, - instance.matched_data, - ) - else: - for offset, name, value in match.strings: - yield 0, ( - format_hints.Hex(offset + start), - task.UniqueProcessId, - match.rule, - name, - value, - ) - else: - for match in rules.scan(data).matching_rules: - for match_string in match.patterns: - for instance in match_string.matches: - yield 0, ( - format_hints.Hex(instance.offset + start), - task.UniqueProcessId, - f"{match.namespace}.{match.identifier}", - match_string.identifier, - data[ - instance.offset : instance.offset - + instance.length - ], - ) + max_vad_size = max(max_vad_size, size) + vad_maps_to_scan.append((start, size)) + + if not vad_maps_to_scan: + vollog.warning( + f"No VADs were found for task {task.UniqueProcessID}, not scanning" + ) + continue + + scanner = yarascan.YaraScanner(rules=rules) + scanner.chunk_size = max_vad_size + + # scan the VAD data (in one contiguous block) with the yarascanner + for start, size in vad_maps_to_scan: + for offset, rule_name, name, value in scanner( + layer.read(start, size, pad=True), start + ): + yield 0, ( + format_hints.Hex(offset), + task.UniqueProcessId, + rule_name, + name, + value, + ) @staticmethod def get_vad_maps( diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index 4a06ed0c95..4930789d26 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -212,9 +212,7 @@ def _generator( proc_layer_name = proc.add_process_layer() except exceptions.InvalidAddressException as excp: vollog.debug( - "Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name - ) + f"Process {proc_id}: invalid address {excp.invalid_address} in layer {excp.layer_name}" ) continue diff --git a/volatility3/framework/plugins/windows/virtmap.py b/volatility3/framework/plugins/windows/virtmap.py index 3f3f270e2e..e02cca89ec 100644 --- a/volatility3/framework/plugins/windows/virtmap.py +++ b/volatility3/framework/plugins/windows/virtmap.py @@ -138,8 +138,7 @@ def scannable_sections( mapping = cls.determine_map(module) for entry in mapping: if "Unused" not in entry: - for value in mapping[entry]: - yield value + yield from mapping[entry] def run(self): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 43bb59a210..112e937511 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -88,9 +88,7 @@ def _validate_values(self, values: List[interfaces.renderers.BaseTypes]) -> None val = values[index] if not isinstance(val, (column.type, interfaces.renderers.BaseAbsentValue)): raise TypeError( - "Values item with index {} is the wrong type for column {} (got {} but expected {})".format( - index, column.name, type(val), column.type - ) + f"Values item with index {index} is the wrong type for column {column.name} (got {type(val)} but expected {column.type})" ) # TODO: Consider how to deal with timezone naive/aware datetimes (and alert plugin uses to be precise) # if isinstance(val, datetime.datetime): @@ -189,9 +187,7 @@ def __init__( is_simple_type = issubclass(column_type, self.base_types) if not is_simple_type: raise TypeError( - "Column {}'s type is not a simple type: {}".format( - name, column_type.__class__.__name__ - ) + f"Column {name}'s type is not a simple type: {column_type.__class__.__name__}" ) converted_columns.append(interfaces.renderers.Column(name, column_type)) self.RowStructure = RowStructureConstructor( @@ -218,7 +214,7 @@ def sanitize_name(text: str) -> str: def populate( self, - function: interfaces.renderers.VisitorSignature = None, + function: Optional[interfaces.renderers.VisitorSignature] = None, initial_accumulator: Any = None, fail_on_errors: bool = True, ) -> Optional[Exception]: @@ -434,10 +430,10 @@ def __call__(self, values: List[Any]) -> Any: value = datetime.datetime.min elif self._type in [int, float]: value = -1 - elif self._type == bool: + elif self._type is bool: value = False elif self._type in [str, renderers.Disassembly]: value = "-" - elif self._type == bytes: + elif self._type is bytes: value = b"" return value diff --git a/volatility3/framework/renderers/format_hints.py b/volatility3/framework/renderers/format_hints.py index 194e38099c..d57c7e9f19 100644 --- a/volatility3/framework/renderers/format_hints.py +++ b/volatility3/framework/renderers/format_hints.py @@ -70,15 +70,21 @@ def __eq__(self, other): ) -BinOrAbsent = lambda x: ( - Bin(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) -HexOrAbsent = lambda x: ( - Hex(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) -HexBytesOrAbsent = lambda x: ( - HexBytes(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) -MultiTypeDataOrAbsent = lambda x: ( - MultiTypeData(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x -) +def BinOrAbsent(x): + return Bin(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x + + +def HexOrAbsent(x): + return Hex(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x + + +def HexBytesOrAbsent(x): + return HexBytes(x) if not isinstance(x, interfaces.renderers.BaseAbsentValue) else x + + +def MultiTypeDataOrAbsent(x): + return ( + MultiTypeData(x) + if not isinstance(x, interfaces.renderers.BaseAbsentValue) + else x + ) diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index a8753bd4db..87f2288d79 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -53,10 +53,10 @@ def __init__(self) -> None: self._resolved: Dict[str, interfaces.objects.Template] = {} self._resolved_symbols: Dict[str, interfaces.objects.Template] = {} - def clear_symbol_cache(self, table_name: str = None) -> None: + def clear_symbol_cache(self, table_name: Optional[str] = None) -> None: """Clears the symbol cache for the specified table name. If no table name is specified, the caches of all symbol tables are cleared.""" - table_list: List[interfaces.symbols.BaseSymbolTableInterface] = list() + table_list: List[interfaces.symbols.BaseSymbolTableInterface] = [] if table_name is None: table_list = list(self._dict.values()) else: @@ -81,7 +81,7 @@ def get_symbols_by_type(self, type_name: str) -> Iterable[str]: yield table + constants.BANG + symbol_name def get_symbols_by_location( - self, offset: int, size: int = 0, table_name: str = None + self, offset: int, size: int = 0, table_name: Optional[str] = None ) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" table_list: Iterable[interfaces.symbols.BaseSymbolTableInterface] = ( @@ -128,7 +128,7 @@ def verify_table_versions( self, producer: str, validator: Callable[[Optional[Tuple], Optional[datetime.datetime]], bool], - tables: List[str] = None, + tables: Optional[List[str]] = None, ) -> bool: """Verifies the producer metadata and version of tables diff --git a/volatility3/framework/symbols/generic/__init__.py b/volatility3/framework/symbols/generic/__init__.py index 9d6da5aa41..7dd00fa75e 100644 --- a/volatility3/framework/symbols/generic/__init__.py +++ b/volatility3/framework/symbols/generic/__init__.py @@ -4,7 +4,7 @@ import random import string -from typing import Union +from typing import Optional, Union from volatility3.framework import objects, interfaces @@ -14,8 +14,8 @@ def _add_process_layer( self, context: interfaces.context.ContextInterface, dtb: Union[int, interfaces.objects.ObjectInterface], - config_prefix: str = None, - preferred_name: str = None, + config_prefix: Optional[str] = None, + preferred_name: Optional[str] = None, ) -> str: """Constructs a new layer based on the process's DirectoryTableBase.""" diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 751f88e39c..5b4aa22b89 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -86,7 +86,7 @@ def __init__( config_path: str, name: str, isf_url: str, - native_types: interfaces.symbols.NativeTableInterface = None, + native_types: Optional[interfaces.symbols.NativeTableInterface] = None, table_mapping: Optional[Dict[str, str]] = None, validate: bool = True, class_types: Optional[ @@ -171,7 +171,7 @@ def _closest_version( (indicating that only additive changes have been made) than the consumer (in this case, the file reader). """ - major, minor, patch = [int(x) for x in version.split(".")] + major, minor, patch = (int(x) for x in version.split(".")) supported_versions = [x for x in versions if x[0] == major and x[1] >= minor] if not supported_versions: raise ValueError( @@ -319,7 +319,7 @@ def __init__( config_path: str, name: str, json_object: Any, - native_types: interfaces.symbols.NativeTableInterface = None, + native_types: Optional[interfaces.symbols.NativeTableInterface] = None, table_mapping: Optional[Dict[str, str]] = None, ) -> None: self._json_object = json_object @@ -738,10 +738,17 @@ class Version6Format(Version5Format): @property def metadata(self) -> Optional[interfaces.symbols.MetadataInterface]: """Returns a MetadataInterface object.""" - if self._json_object.get("metadata", {}).get("windows"): - return metadata.WindowsMetadata(self._json_object["metadata"]["windows"]) - if self._json_object.get("metadata", {}).get("linux"): - return metadata.LinuxMetadata(self._json_object["metadata"]["linux"]) + if "metadata" not in self._json_object: + return None + + json_metadata = self._json_object["metadata"] + if "windows" in json_metadata: + return metadata.WindowsMetadata(json_metadata["windows"]) + if "linux" in json_metadata: + return metadata.LinuxMetadata(json_metadata["linux"]) + if "mac" in json_metadata: + return metadata.MacMetadata(json_metadata["mac"]) + return None diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 573bc66d89..ba223f9798 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -645,8 +645,7 @@ def _iter_node(self, nodep, height) -> Iterator[int]: if self.is_valid_node(nodep): yield nodep else: - for child_node in self._iter_node(nodep, height - 1): - yield child_node + yield from self._iter_node(nodep, height - 1) def get_entries(self, root: interfaces.objects.ObjectInterface) -> Iterator[int]: """Walks the tree data structure @@ -675,8 +674,7 @@ def get_entries(self, root: interfaces.objects.ObjectInterface) -> Iterator[int] if self.is_valid_node(nodep): yield nodep else: - for child_node in self._iter_node(nodep, height): - yield child_node + yield from self._iter_node(nodep, height) class XArray(IDStorage): @@ -814,7 +812,7 @@ def is_valid_node(self, nodep) -> bool: return True -class PageCache(object): +class PageCache: """Linux Page Cache abstraction""" def __init__( diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 927f767e2a..34d0fcba90 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -200,8 +200,7 @@ def get_sections(self): count=num_sects, ) - for attr in arr: - yield attr + yield from arr def get_elf_table_name(self): elf_table_name = intermed.IntermediateSymbolTable.create( @@ -237,8 +236,7 @@ def get_symbols(self): count=self.num_symtab + 1, ) if self.section_strtab: - for sym in syms: - yield sym + yield from syms def get_symbols_names_and_addresses(self) -> Iterable[Tuple[str, int]]: """Get names and addresses for each symbol of the module @@ -310,7 +308,7 @@ def section_strtab(self): class task_struct(generic.GenericIntelProcess): def add_process_layer( - self, config_prefix: str = None, preferred_name: str = None + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: """Constructs a new layer based on the process's DTB. @@ -635,6 +633,20 @@ def get_create_time(self) -> datetime.datetime: # root time namespace, not within the task's own time namespace return boottime + task_start_time_timedelta + def get_parent_pid(self) -> int: + """Returns the parent process ID (PPID) + + This method replicates the Linux kernel's `getppid` syscall behavior. + Avoid using `task.parent`; instead, use this function for accurate results. + """ + + if self.real_parent and self.real_parent.is_readable(): + ppid = self.real_parent.tgid + else: + ppid = 0 + + return ppid + class fs_struct(objects.StructType): def get_root_dentry(self): @@ -1107,9 +1119,16 @@ def get_subdirs(self) -> Iterable[interfaces.objects.ObjectInterface]: walk_member = "d_sib" list_head_member = self.d_children elif self.has_member("d_child") and self.has_member("d_subdirs"): - # 2.5.0 <= kernels < 6.8 + # 3.19.0 <= kernels < 6.8 walk_member = "d_child" list_head_member = self.d_subdirs + elif self.has_member("d_u") and self.has_member("d_subdirs"): + # kernels < 3.19 + + # Actually, 'd_u.d_child' but to_list() doesn't support something like that. + # Since, it's an union, everything is at the same offset than 'd_u'. + walk_member = "d_u" + list_head_member = self.d_subdirs else: raise exceptions.VolatilityException("Unsupported dentry type") @@ -1462,7 +1481,7 @@ def is_path_reachable(self, current_dentry, root): def next_peer(self): table_name = self.vol.type_name.split(constants.BANG)[0] - mount_struct = "{0}{1}mount".format(table_name, constants.BANG) + mount_struct = f"{table_name}{constants.BANG}mount" offset = self._context.symbol_space.get_type( mount_struct ).relative_child_offset("mnt_share") @@ -2060,14 +2079,41 @@ def _get_cred_int_value(self, member: str) -> int: return int(value) @property - def euid(self): + def uid(self) -> int: + """Returns the real user ID + + Returns: + The real user ID value + """ + return self._get_cred_int_value("uid") + + @property + def gid(self) -> int: + """Returns the real user ID + + Returns: + The real user ID value + """ + return self._get_cred_int_value("gid") + + @property + def euid(self) -> int: """Returns the effective user ID Returns: - int: the effective user ID value + The effective user ID value """ return self._get_cred_int_value("euid") + @property + def egid(self) -> int: + """Returns the effective group ID + + Returns: + int: the effective user ID value + """ + return self._get_cred_int_value("egid") + class kernel_cap_struct(objects.StructType): # struct kernel_cap_struct exists from 2.1.92 <= kernels < 6.3 @@ -2480,7 +2526,7 @@ def i_pages(self): class page(objects.StructType): @property - @functools.lru_cache() + @functools.lru_cache def pageflags_enum(self) -> Dict: """Returns 'pageflags' enumeration key/values @@ -2658,8 +2704,7 @@ def _new_kernel_get_entries(self) -> Iterable[int]: id_storage = linux.IDStorage.choose_id_storage( self._context, kernel_module_name="kernel" ) - for page_addr in id_storage.get_entries(root=self.idr_rt): - yield page_addr + yield from id_storage.get_entries(root=self.idr_rt) def get_entries(self) -> Iterable[int]: """Walks the IDR and yield a pointer associated with each element. @@ -2677,8 +2722,7 @@ def get_entries(self) -> Iterable[int]: # Kernels < 4.11 get_entries_func = self._old_kernel_get_entries - for page_addr in get_entries_func(): - yield page_addr + yield from get_entries_func() class rb_root(objects.StructType): diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index c695ca77a7..dc54a83714 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Iterator, Any, Iterable, List, Tuple, Set +from typing import Iterator, Any, Iterable, List, Optional, Tuple, Set from volatility3.framework import interfaces, objects, exceptions, constants from volatility3.framework.symbols import intermed @@ -97,7 +97,7 @@ def lookup_module_address( context: interfaces.context.ContextInterface, handlers: Iterator[Any], target_address, - kernel_module_name: str = None, + kernel_module_name: Optional[str] = None, ): mod_name = "UNKNOWN" symbol_name = "N/A" @@ -232,10 +232,9 @@ def walk_tailq( next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( + yield from cls._walk_iterable( queue, "tqh_first", "tqe_next", next_member, max_elements - ): - yield element + ) @classmethod def walk_list_head( @@ -244,10 +243,9 @@ def walk_list_head( next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( + yield from cls._walk_iterable( queue, "lh_first", "le_next", next_member, max_elements - ): - yield element + ) @classmethod def walk_slist( @@ -256,7 +254,6 @@ def walk_slist( next_member: str, max_elements: int = 4096, ) -> Iterable[interfaces.objects.ObjectInterface]: - for element in cls._walk_iterable( + yield from cls._walk_iterable( queue, "slh_first", "sle_next", next_member, max_elements - ): - yield element + ) diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index 15fe7aeda9..cc700f2097 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -18,7 +18,7 @@ def get_task(self): return self.task.dereference().cast("task") def add_process_layer( - self, config_prefix: str = None, preferred_name: str = None + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None ) -> Optional[str]: """Constructs a new layer based on the process's DTB. @@ -237,7 +237,7 @@ def get_special_path(self): def get_path(self, context, config_prefix): node = self.get_vnode(context, config_prefix) - if type(node) == str and node == "sub_map": + if type(node) is str and node == "sub_map": ret = node elif node: path = [] diff --git a/volatility3/framework/symbols/metadata.py b/volatility3/framework/symbols/metadata.py index 95f542f079..ea635f1f1c 100644 --- a/volatility3/framework/symbols/metadata.py +++ b/volatility3/framework/symbols/metadata.py @@ -4,8 +4,7 @@ import datetime import logging -from typing import Optional, Tuple, Union - +from typing import Optional, Tuple, Union, List, Dict from volatility3.framework import constants, interfaces vollog = logging.getLogger(__name__) @@ -19,9 +18,16 @@ def name(self) -> Optional[str]: return self._json_data.get("name", None) @property - def version(self) -> Optional[Tuple[int]]: + def version_string(self) -> str: + """Returns the ISF file producer's version as a string. + If no version is present, an empty string is returned. + """ + return self._json_data.get("version", "") + + @property + def version(self) -> Optional[Tuple[int, ...]]: """Returns the version of the ISF file producer""" - version = self._json_data.get("version", None) + version = self.version_string if not version: return None if all(x in "0123456789." for x in version): @@ -79,5 +85,21 @@ def pdb_age(self) -> Optional[int]: return self._json_data.get("pdb", {}).get("age", None) -class LinuxMetadata(interfaces.symbols.MetadataInterface): +class PosixMetadata(interfaces.symbols.MetadataInterface): + """Base class to handle metadata of Posix-based ISF sources""" + + def get_types_sources(self) -> List[Optional[Dict]]: + """Returns the types sources metadata""" + return self._json_data.get("types", []) + + def get_symbols_sources(self) -> List[Optional[Dict]]: + """Returns the symbols sources metadata""" + return self._json_data.get("symbols", []) + + +class LinuxMetadata(PosixMetadata): """Class to handle the metadata from a Linux symbol table.""" + + +class MacMetadata(PosixMetadata): + """Class to handle the metadata from a Mac symbol table.""" diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ecfc2f1632..d63f138b66 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -692,7 +692,9 @@ def is_valid(self) -> bool: return True - def add_process_layer(self, config_prefix: str = None, preferred_name: str = None): + def add_process_layer( + self, config_prefix: Optional[str] = None, preferred_name: Optional[str] = None + ): """Constructs a new layer based on the process's DirectoryTableBase.""" parent_layer = self._context.layers[self.vol.layer_name] @@ -749,11 +751,10 @@ def load_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: try: peb = self.get_peb() - for entry in peb.Ldr.InLoadOrderModuleList.to_list( + yield from peb.Ldr.InLoadOrderModuleList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InLoadOrderLinks", - ): - yield entry + ) except exceptions.InvalidAddressException: return None @@ -762,11 +763,10 @@ def init_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: try: peb = self.get_peb() - for entry in peb.Ldr.InInitializationOrderModuleList.to_list( + yield from peb.Ldr.InInitializationOrderModuleList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InInitializationOrderLinks", - ): - yield entry + ) except exceptions.InvalidAddressException: return None @@ -775,11 +775,10 @@ def mem_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: try: peb = self.get_peb() - for entry in peb.Ldr.InMemoryOrderModuleList.to_list( + yield from peb.Ldr.InMemoryOrderModuleList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InMemoryOrderLinks", - ): - yield entry + ) except exceptions.InvalidAddressException: return None @@ -797,7 +796,7 @@ def get_handle_count(self): return renderers.UnreadableValue() - def get_session_id(self): + def get_session_id(self) -> Union[int, interfaces.renderers.BaseAbsentValue]: try: if self.has_member("Session"): if self.Session == 0: @@ -813,12 +812,30 @@ def get_session_id(self): offset=kvo, native_layer_name=self.vol.native_layer_name, ) - session = ntkrnlmp.object( - object_type="_MM_SESSION_SPACE", offset=self.Session, absolute=True - ) - - if session.has_member("SessionId"): - return session.SessionId + try: + session = ntkrnlmp.object( + object_type="_MM_SESSION_SPACE", + offset=self.Session, + absolute=True, + ) + if session.has_member("SessionId"): + return session.SessionId + except exceptions.SymbolError: + # In Windows 11 24H2, the _MM_SESSION_SPACE type was + # replaced with _PSP_SESSION_SPACE, and the kernel PDB + # doesn't contain information about its members (otherwise, + # we would just fall back to the new type). However, it + # appears to be, for our purposes, functionally identical + # to the _MM_SESSION_SPACE. Because _MM_SESSION_SPACE + # stores its session ID at offset 8 as an unsigned long, we + # create an unsigned long at that offset and use that + # instead. + session_id = ntkrnlmp.object( + object_type="unsigned long", + offset=self.Session + 8, + absolute=True, + ) + return session_id except exceptions.InvalidAddressException: vollog.log( @@ -1076,7 +1093,7 @@ def valid_type(self): return self.Header.Type in self.VALID_TYPES def get_due_time(self): - return "{0:#010x}:{1:#010x}".format(self.DueTime.HighPart, self.DueTime.LowPart) + return f"{self.DueTime.HighPart:#010x}:{self.DueTime.LowPart:#010x}" def get_dpc(self): """Return Dpc, and if Windows 7 or later, decode it""" @@ -1383,7 +1400,7 @@ def process_index_array( ) # Iterate through the entries - for counter in range(0, self.VACB_ARRAY): + for counter in range(self.VACB_ARRAY): # Check if the VACB entry is in use if not vacb_array[counter]: continue @@ -1467,7 +1484,7 @@ def get_available_pages(self) -> List: if not section_size > self.VACB_SIZE_OF_FIRST_LEVEL: array_head = vacb_obj - for counter in range(0, full_blocks): + for counter in range(full_blocks): vacb_entry = self._context.object( symbol_table_name + constants.BANG + "pointer", layer_name=self.vol.layer_name, @@ -1526,7 +1543,7 @@ def get_available_pages(self) -> List: # Walk the array and if any entry points to the shared cache map object then we extract it. # Otherwise, if it is non-zero, then traverse to the next level. - for counter in range(0, self.VACB_ARRAY): + for counter in range(self.VACB_ARRAY): if not vacb_array[counter]: continue diff --git a/volatility3/framework/symbols/windows/extensions/consoles.py b/volatility3/framework/symbols/windows/extensions/consoles.py index 2312149c71..9666fd79c9 100644 --- a/volatility3/framework/symbols/windows/extensions/consoles.py +++ b/volatility3/framework/symbols/windows/extensions/consoles.py @@ -73,7 +73,7 @@ def get_text(self, truncate: bool = True) -> str: ) for i in range(0, len(char_row), 3) ) - except Exception as e: + except Exception: line = "" if truncate: @@ -107,11 +107,10 @@ def get_exename(self) -> Union[str, None]: def get_aliases(self) -> Generator[interfaces.objects.ObjectInterface, None, None]: """Generator for the individual aliases for a particular executable.""" - for alias in self.AliasList.to_list( + yield from self.AliasList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_ALIAS", "ListEntry", - ): - yield alias + ) class SCREEN_INFORMATION(objects.StructType): @@ -245,11 +244,10 @@ def get_screens(self) -> Generator[interfaces.objects.ObjectInterface, None, Non def get_histories( self, ) -> Generator[interfaces.objects.ObjectInterface, None, None]: - for cmd_hist in self.HistoryList.to_list( + yield from self.HistoryList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_COMMAND_HISTORY", "ListEntry", - ): - yield cmd_hist + ) def get_exe_aliases( self, @@ -258,20 +256,18 @@ def get_exe_aliases( # Windows 10 22000 and Server 20348 made this a Pointer if isinstance(exe_alias_list, objects.Pointer): exe_alias_list = exe_alias_list.dereference() - for exe_alias_list_item in exe_alias_list.to_list( + yield from exe_alias_list.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_EXE_ALIAS_LIST", "ListEntry", - ): - yield exe_alias_list_item + ) def get_processes( self, ) -> Generator[interfaces.objects.ObjectInterface, None, None]: - for proc in self.ConsoleProcessList.to_list( + yield from self.ConsoleProcessList.to_list( f"{self.get_symbol_table_name()}{constants.BANG}_CONSOLE_PROCESS_LIST", "ListEntry", - ): - yield proc + ) def get_title(self) -> Union[str, None]: try: @@ -393,8 +389,7 @@ def get_commands( rest are coalesced. """ - for i, cmd in self.scan_command_bucket(self.CommandBucket.End): - yield i, cmd + yield from self.scan_command_bucket(self.CommandBucket.End) win10_x64_class_types = { diff --git a/volatility3/framework/symbols/windows/extensions/mbr.py b/volatility3/framework/symbols/windows/extensions/mbr.py index afdc73a172..078c4beb0c 100644 --- a/volatility3/framework/symbols/windows/extensions/mbr.py +++ b/volatility3/framework/symbols/windows/extensions/mbr.py @@ -8,12 +8,7 @@ class PARTITION_TABLE(objects.StructType): def get_disk_signature(self) -> str: """Get Disk Signature (GUID).""" - return "{0:02x}-{1:02x}-{2:02x}-{3:02x}".format( - self.DiskSignature[0], - self.DiskSignature[1], - self.DiskSignature[2], - self.DiskSignature[3], - ) + return f"{self.DiskSignature[0]:02x}-{self.DiskSignature[1]:02x}-{self.DiskSignature[2]:02x}-{self.DiskSignature[3]:02x}" class PARTITION_ENTRY(objects.StructType): diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index 9b7573c2e8..478deab6b9 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -4,7 +4,7 @@ import logging import socket -from typing import Dict, Tuple, List, Union +from typing import Dict, Tuple, List, Union, Optional from volatility3.framework import exceptions from volatility3.framework import objects, interfaces @@ -22,7 +22,7 @@ def inet_ntop(address_family: int, packed_ip: Union[List[int], Array]) -> str: raise RuntimeError( "This version of python does not have socket.inet_ntop, please upgrade" ) - raise socket.error("[Errno 97] Address family not supported by protocol") + raise OSError("[Errno 97] Address family not supported by protocol") # Python's socket.AF_INET6 is 0x1e but Microsoft defines it @@ -86,19 +86,29 @@ def get_owner(self): except exceptions.InvalidAddressException: return None - def get_owner_pid(self): - if self.get_owner().is_valid(): - if self.get_owner().has_valid_member("UniqueProcessId"): - return self.get_owner().UniqueProcessId + def get_owner_pid(self) -> Optional[int]: + owner = self.get_owner() + + if owner is None: + return None + + if owner.is_valid(): + if owner.has_valid_member("UniqueProcessId"): + return owner.UniqueProcessId return None - def get_owner_procname(self): - if self.get_owner().is_valid(): - if self.get_owner().has_valid_member("ImageFileName"): - return self.get_owner().ImageFileName.cast( + def get_owner_procname(self) -> Optional[str]: + owner = self.get_owner() + + if owner is None: + return None + + if owner.is_valid(): + if owner.has_valid_member("ImageFileName"): + return owner.ImageFileName.cast( "string", - max_length=self.get_owner().ImageFileName.vol.count, + max_length=owner.ImageFileName.vol.count, errors="replace", ) @@ -167,11 +177,9 @@ def dual_stack_sockets(self): def is_valid(self): try: - if not self.get_address_family() in (AF_INET, AF_INET6): + if self.get_address_family() not in (AF_INET, AF_INET6): vollog.debug( - "netw obj 0x{:x} invalid due to invalid address_family {}".format( - self.vol.offset, self.get_address_family() - ) + f"netw obj 0x{self.vol.offset:x} invalid due to invalid address_family {self.get_address_family()}" ) return False diff --git a/volatility3/framework/symbols/windows/extensions/pe.py b/volatility3/framework/symbols/windows/extensions/pe.py index 3f34fc3dd4..2c7400f259 100644 --- a/volatility3/framework/symbols/windows/extensions/pe.py +++ b/volatility3/framework/symbols/windows/extensions/pe.py @@ -101,9 +101,9 @@ def fix_image_base( ) except OverflowError: vollog.warning( - "Volatility was unable to fix the image base for the PE file at base address {:#x}. " + f"Volatility was unable to fix the image base for the PE file at base address {self.vol.offset:#x}. " "This will cause issues with many static analysis tools if you do not inform the " - "tool of the in-memory load address.".format(self.vol.offset) + "tool of the in-memory load address." ) new_pe = raw_data diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index b761ddad88..de5c8271b7 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -217,7 +217,7 @@ def get_object( yield mem_object @classmethod - @functools.lru_cache() + @functools.lru_cache def _calculate_optional_header_lengths( cls, context: interfaces.context.ContextInterface, symbol_table_name: str ) -> Tuple[List[str], List[int]]: @@ -362,7 +362,7 @@ def is_valid(self) -> bool: return True def get_object_type( - self, type_map: Dict[int, str], cookie: int = None + self, type_map: Dict[int, str], cookie: Optional[int] = None ) -> Optional[str]: """Across all Windows versions, the _OBJECT_HEADER embeds details on the type of object (i.e. process, file) but the way its embedded @@ -430,9 +430,7 @@ def NameInfo(self) -> interfaces.objects.ObjectInterface: if header_offset == 0: raise ValueError( - "Could not find _OBJECT_HEADER_NAME_INFO for object at {} of layer {}".format( - self.vol.offset, self.vol.layer_name - ) + f"Could not find _OBJECT_HEADER_NAME_INFO for object at {self.vol.offset} of layer {self.vol.layer_name}" ) header = ntkrnlmp.object( diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index bebfaea891..9e2f8df3bd 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -196,9 +196,7 @@ def _get_subkeys_recursive( yield cast("CM_KEY_NODE", node) else: vollog.debug( - "Unexpected node type encountered when traversing subkeys: {}, signature: {}".format( - node.vol.type_name, signature - ) + f"Unexpected node type encountered when traversing subkeys: {node.vol.type_name}, signature: {signature}" ) if listjump: diff --git a/volatility3/framework/symbols/windows/extensions/services.py b/volatility3/framework/symbols/windows/extensions/services.py index e14de761d8..0a2194e075 100644 --- a/volatility3/framework/symbols/windows/extensions/services.py +++ b/volatility3/framework/symbols/windows/extensions/services.py @@ -14,13 +14,16 @@ class SERVICE_RECORD(objects.StructType): def is_valid(self) -> bool: """Determine if the structure is valid.""" - if self.Order < 0 or self.Order > 0xFFFF: - return False - try: - _ = self.State.description - _ = self.Start.description - except ValueError: + if self.Order < 0 or self.Order > 0xFFFF: + return False + + try: + _ = self.State.description + _ = self.Start.description + except ValueError: + return False + except exceptions.InvalidAddressException: return False return True diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index 82ec31ccbc..248ef7d0c3 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -128,7 +128,10 @@ def __init__( self._layer_name, self._context = self.load_pdb_layer(context, location) self._dbiheader: Optional[interfaces.objects.ObjectInterface] = None if not progress_callback: - progress_callback = lambda x, y: None + + def progress_callback(x, y): + return None + self._progress_callback = progress_callback self.types: List[ Tuple[ @@ -263,9 +266,7 @@ def _read_info_stream(self, stream_number, stream_name, info_list): ) if header.index_max < header.index_min: raise ValueError( - "Maximum {} index is smaller than minimum TPI index, found: {} < {} ".format( - stream_name, header.index_max, header.index_min - ) + f"Maximum {stream_name} index is smaller than minimum TPI index, found: {header.index_max} < {header.index_min} " ) # Reset the state info_references: Dict[str, int] = {} @@ -976,14 +977,16 @@ def retreive_pdb( if __name__ == "__main__": import argparse - class PrintedProgress(object): + class PrintedProgress: """A progress handler that prints the progress value and the description onto the command line.""" def __init__(self): self._max_message_len = 0 - def __call__(self, progress: Union[int, float], description: str = None): + def __call__( + self, progress: Union[int, float], description: Optional[str] = None + ): """A simple function for providing text-based feedback. .. warning:: Only for development use. diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 3816312cd5..b5e8ca70ad 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -36,7 +36,7 @@ def symbol_table_from_offset( layer_name: str, offset: int, symbol_table_class: str = "volatility3.framework.symbols.intermed.IntermediateSymbolTable", - config_path: str = None, + config_path: Optional[str] = None, progress_callback: constants.ProgressCallback = None, ) -> Optional[str]: """Produces the name of a symbol table loaded from the offset for an MZ header @@ -94,7 +94,7 @@ def load_windows_symbol_table( if not requirements.VersionRequirement.matches_required( (1, 0, 0), symbol_cache.SqliteCache.version ): - vollog.debug(f"Required version of SQLiteCache not found") + vollog.debug("Required version of SQLiteCache not found") return None identifiers_path = os.path.join( @@ -291,9 +291,7 @@ def download_pdb_isf( break except PermissionError: vollog.warning( - "Cannot write necessary symbol file, please check permissions on {}".format( - potential_output_filename - ) + f"Cannot write necessary symbol file, please check permissions on {potential_output_filename}" ) continue finally: @@ -390,8 +388,8 @@ def symbol_table_from_pdb( config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, ) -> str: """Creates symbol table for a module in the specified layer_name. @@ -420,8 +418,8 @@ def _modtable_from_pdb( config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, create_module: bool = False, ) -> Tuple[Optional[str], Optional[str]]: if module_offset is None: @@ -480,8 +478,8 @@ def module_from_pdb( config_path: str, layer_name: str, pdb_name: str, - module_offset: int = None, - module_size: int = None, + module_offset: Optional[int] = None, + module_size: Optional[int] = None, ) -> str: """Creates a module in the specified layer_name based on a pdb name. diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 5ef840f324..8587b37198 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -60,7 +60,7 @@ def dump_certificate( open_method: Type[interfaces.plugins.FileHandlerInterface], ) -> Optional[interfaces.plugins.FileHandlerInterface]: try: - dump_name = "{}-{}-{}.crt".format(hive_offset, reg_section, key_hash) + dump_name = f"{hive_offset}-{reg_section}-{key_hash}.crt" file_handle = open_method(dump_name) file_handle.write(certificate_data) return file_handle diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index 7f56b75f8a..e7557dc0c7 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -64,9 +64,7 @@ def _generator(self): other_invalid += 1 page_size = expected_page_size vollog.debug( - "A non-page lookup invalid address exception occurred at: {} in layer {}".format( - hex(excp.invalid_address), excp.layer_name - ) + f"A non-page lookup invalid address exception occurred at: {hex(excp.invalid_address)} in layer {excp.layer_name}" ) page_addr += page_size diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 3ca00e5dc6..90cfaba481 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -20,7 +20,7 @@ def load_cached_validations() -> Set[str]: to revalidate them.""" validhashes: Set = set() if os.path.exists(cached_validation_filepath): - with open(cached_validation_filepath, "r") as f: + with open(cached_validation_filepath) as f: validhashes.update(json.load(f)) return validhashes @@ -46,7 +46,7 @@ def validate(input: Dict[str, Any], use_cache: bool = True) -> bool: if not os.path.exists(schema_path): vollog.debug(f"Schema for format not found: {schema_path}") return False - with open(schema_path, "r") as s: + with open(schema_path) as s: schema = json.load(s) return valid(input, schema, use_cache) @@ -66,7 +66,7 @@ def create_json_hash( if not os.path.exists(schema_path): vollog.debug(f"Schema for format not found: {schema_path}") return None - with open(schema_path, "r") as s: + with open(schema_path) as s: schema = json.load(s) return hashlib.sha1( bytes(json.dumps((input, schema), sort_keys=True), "utf-8")