From 0860441c2fc5a5a97902a7582473d8462c457bc3 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 17:11:02 +0000 Subject: [PATCH 1/5] Core: Improve speed to JSONSchema validation --- pyproject.toml | 3 +-- volatility3/framework/plugins/isfinfo.py | 2 +- volatility3/schemas/__init__.py | 16 +++++++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86e3921d2..af22cbe0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,10 +33,9 @@ cloud = [ dev = [ "volatility3[full,cloud]", - "jsonschema>=4.23.0,<5", + "fastjsonschema>=2.21.1,<3", "pyinstaller>=6.11.0,<7", "pyinstaller-hooks-contrib>=2024.9", - "types-jsonschema>=4.23.0,<5", ] test = [ diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 1c2ac52e9..34b0a5653 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -97,7 +97,7 @@ def _generator(self): if filter_item in isf_file: filtered_list.append(isf_file) - if find_spec("jsonschema") and self.config["validate"]: + if find_spec("fastjsonschema") and self.config["validate"]: def check_valid(data): return "True" if schemas.validate(data, True) else "False" diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 90cfaba48..3964e29a6 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -14,6 +14,8 @@ cached_validation_filepath = os.path.join(constants.CACHE_PATH, "valid_isf.hashcache") +validators = {} + def load_cached_validations() -> Set[str]: """Loads up the list of successfully cached json objects, so we don't need @@ -92,7 +94,12 @@ def valid( if input_hash in cached_validations and use_cache: return True try: - import jsonschema + import fastjsonschema + + schema_key = json.dumps(schema, sort_keys=True) + if schema_key not in validators: + validator = fastjsonschema.compile(schema) + validators[schema_key] = validator except ImportError: vollog.info("Dependency for validation unavailable: jsonschema") vollog.debug("All validations will report success, even with malformed input") @@ -100,10 +107,13 @@ def valid( try: vollog.debug("Validating JSON against schema...") - jsonschema.validate(input, schema) + validators[schema_key](input) + import pdb + + pdb.set_trace() cached_validations.add(input_hash) vollog.debug("JSON validated against schema (result cached)") - except jsonschema.exceptions.SchemaError: + except fastjsonschema.JsonSchemaValueException: vollog.debug("Schema validation error", exc_info=True) return False From e676e6179a3cd9c8d6afb838560d7a9e4d3a5420 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 18:08:22 +0000 Subject: [PATCH 2/5] Swap fastjsonschema for jsonschema because of date-time validation issues --- volatility3/schemas/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index 3964e29a6..b94e0831d 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -94,11 +94,13 @@ def valid( if input_hash in cached_validations and use_cache: return True try: - import fastjsonschema + import jsonschema schema_key = json.dumps(schema, sort_keys=True) if schema_key not in validators: - validator = fastjsonschema.compile(schema) + validator_class = jsonschema.validators.validator_for(schema) + validator_class.check_schema(schema) + validator = validator_class(schema) validators[schema_key] = validator except ImportError: vollog.info("Dependency for validation unavailable: jsonschema") @@ -107,10 +109,7 @@ def valid( try: vollog.debug("Validating JSON against schema...") - validators[schema_key](input) - import pdb - - pdb.set_trace() + validators[schema_key].validate(input) cached_validations.add(input_hash) vollog.debug("JSON validated against schema (result cached)") except fastjsonschema.JsonSchemaValueException: From 21decf13708d882369cc1f8fa2884f5a8ae5494d Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 18:12:36 +0000 Subject: [PATCH 3/5] Core: Put the dependencies back for jsonschema --- pyproject.toml | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index af22cbe0a..fbdbf0a8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,15 @@ [project] name = "volatility3" description = "Memory forensics framework" -keywords = ["volatility", "memory", "forensics", "framework", "windows", "linux", "volshell"] +keywords = [ + "volatility", + "memory", + "forensics", + "framework", + "windows", + "linux", + "volshell", +] readme = "README.md" authors = [ { name = "Volatility Foundation", email = "volatility@volatilityfoundation.org" }, @@ -10,9 +18,7 @@ requires-python = ">=3.8.0" license = { text = "VSL" } dynamic = ["version"] -dependencies = [ - "pefile>=2024.8.26", -] +dependencies = ["pefile>=2024.8.26"] [project.optional-dependencies] full = [ @@ -26,16 +32,14 @@ full = [ "pillow>=10.0.0,<11.0.0", ] -cloud = [ - "gcsfs>=2024.10.0", - "s3fs>=2024.10.0", -] +cloud = ["gcsfs>=2024.10.0", "s3fs>=2024.10.0"] dev = [ "volatility3[full,cloud]", - "fastjsonschema>=2.21.1,<3", + "jsonschema>=4.23.0,<5", "pyinstaller>=6.11.0,<7", "pyinstaller-hooks-contrib>=2024.9", + "types-jsonschema>=4.23.0,<5", ] test = [ @@ -78,16 +82,16 @@ target-version = "py38" [tool.ruff.lint] select = [ - "F", # pyflakes - "E", # pycodestyle errors - "W", # pycodestyle warnings - "G", # flake8-logging-format - "PIE", # flake8-pie - "UP", # pyupgrade + "F", # pyflakes + "E", # pycodestyle errors + "W", # pycodestyle warnings + "G", # flake8-logging-format + "PIE", # flake8-pie + "UP", # pyupgrade ] ignore = [ - "E501", # ignore due to conflict with formatter + "E501", # ignore due to conflict with formatter ] [build-system] From d07d31047b2ed49a570aa4b3698f6a64145c2d83 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 9 Jan 2025 18:24:48 +0000 Subject: [PATCH 4/5] Core: Revert the exception catching too --- volatility3/schemas/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index b94e0831d..e894def9f 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -112,7 +112,7 @@ def valid( validators[schema_key].validate(input) cached_validations.add(input_hash) vollog.debug("JSON validated against schema (result cached)") - except fastjsonschema.JsonSchemaValueException: + except jsonschema.exceptions.SchemaError: vollog.debug("Schema validation error", exc_info=True) return False From 749a0f9c656d8da0f5087faf25f225a6097ac8b4 Mon Sep 17 00:00:00 2001 From: Mike Auty Date: Thu, 16 Jan 2025 12:06:22 +0000 Subject: [PATCH 5/5] Core: Fix up ISFinfo looking for fastjsonschema --- volatility3/framework/plugins/isfinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index 34b0a5653..1c2ac52e9 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -97,7 +97,7 @@ def _generator(self): if filter_item in isf_file: filtered_list.append(isf_file) - if find_spec("fastjsonschema") and self.config["validate"]: + if find_spec("jsonschema") and self.config["validate"]: def check_valid(data): return "True" if schemas.validate(data, True) else "False"