From 9912b24c77c4f069b43dd1a6cf8d747d0a89e88e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 21 Jul 2021 16:18:25 -0400 Subject: [PATCH 1/3] Add hash of retrieved URL as a CSV output This adds a "VDP Hash" field to the domain CSV output. This field will contain the hash of the retrieved and parsed VDP per the cisagov/hash-http-content project. This field contains the hash on an HTTP OK response for the resolved URL and an empty string for anything else. --- src/vdp_scanner.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/vdp_scanner.py b/src/vdp_scanner.py index 288f68f..a303c2c 100644 --- a/src/vdp_scanner.py +++ b/src/vdp_scanner.py @@ -54,6 +54,7 @@ class DomainResult(NamedTuple): visited_url: str is_redirect: bool vdp_present: bool + vdp_hash: str class VdpScanner: @@ -82,6 +83,7 @@ class VdpScanner: "Visited URL", "Was it Redirected", "VDP is Published", + "VDP Hash", ] def __init__(self, hasher: UrlHasher): @@ -105,7 +107,7 @@ def _log_vdp_failure(domain: str, err: Exception) -> None: logging.debug("Caught %s", type(err).__name__) logging.debug(err) - def check_for_vdp(self, domain: str) -> Tuple[str, bool, bool]: + def check_for_vdp(self, domain: str) -> Tuple[str, bool, bool, str]: """Check for a VDP at the given domain and return the relavent information.""" url = urlparse(f"https://{domain}/vulnerability-disclosure-policy") result: Optional[UrlResult] = None @@ -149,12 +151,12 @@ def check_for_vdp(self, domain: str) -> Tuple[str, bool, bool]: self._log_vdp_failure(domain, err) if not result: - return ("", False, False) + return ("", False, False, "") if result.status == 200: - return (result.visited_url, result.is_redirect, True) + return (result.visited_url, result.is_redirect, True, result.hash) - return (result.visited_url, result.is_redirect, False) + return (result.visited_url, result.is_redirect, False, "") def process_domain(self, domain_info: Dict[str, Any]) -> None: """Process a domain entry from the DotGov CSV.""" @@ -181,6 +183,7 @@ def add_domain_result(self, result: DomainResult) -> None: "Visited URL": result.visited_url, "Was it Redirected": result.is_redirect, "VDP is Published": result.vdp_present, + "VDP Hash": result.vdp_hash, } self.domain_results.append(result_dict) From 19b9d98ebb4cbd8d2220decb4cf40f4803505ac0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 21 Jul 2021 16:44:58 -0400 Subject: [PATCH 2/3] Replace manual dictionary creation with zip Switch to using zip() and casting to a dict to populate the results dictionary. This will allow existing definition to be the "source of truth" for the contents of the dictionary. This will reduce human error (or at least make it consistent) and improve maintainability. --- src/vdp_scanner.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/vdp_scanner.py b/src/vdp_scanner.py index a303c2c..94a3247 100644 --- a/src/vdp_scanner.py +++ b/src/vdp_scanner.py @@ -175,17 +175,11 @@ def process_domain(self, domain_info: Dict[str, Any]) -> None: def add_domain_result(self, result: DomainResult) -> None: """Process the provided results for a domain.""" - result_dict = { - "Domain": result.domain, - "Agency": result.agency, - "Organization": result.organization, - "Security Contact Email": result.security_contact, - "Visited URL": result.visited_url, - "Was it Redirected": result.is_redirect, - "VDP is Published": result.vdp_present, - "VDP Hash": result.vdp_hash, - } - self.domain_results.append(result_dict) + # Create a dict with the values of domain_csv_header as keys and the + # contents of result as values. This leverages the fact that the + # DomainResult NamedTuple is positionally aligned with the contents of + # the domain_csv_header list. + self.domain_results.append(dict(zip(self.domain_csv_header, result))) self.agency_results[result.agency]["Total Domains"] += 1 From 6c17c51b67fdc83605a49d79cad3ca46c2ebb2c3 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 23 Jul 2021 11:22:10 -0400 Subject: [PATCH 3/3] Bump version from 0.0.2 to 0.0.3 --- README.md | 14 +++++++------- src/version.txt | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index fee5aae..7da171d 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Python library. Then it will output CSVs with agency and domain level results. To run the `cisagov/vdp-scanner` image via Docker: ```console -docker run cisagov/vdp-scanner:0.0.2 +docker run cisagov/vdp-scanner:0.0.3 ``` ### Running with Docker Compose ### @@ -38,7 +38,7 @@ docker run cisagov/vdp-scanner:0.0.2 services: vdp-scanner: - image: 'cisagov/vdp-scanner:0.0.2' + image: 'cisagov/vdp-scanner:0.0.3' volumes: - .:/task/host_mount ``` @@ -76,7 +76,7 @@ docker run cisagov/vdp-scanner:0.0.2 1. Pull the new image: ```console - docker pull cisagov/vdp-scanner:0.0.2 + docker pull cisagov/vdp-scanner:0.0.3 ``` 1. Recreate and run the container by following the [previous instructions](#running-with-docker). @@ -85,11 +85,11 @@ docker run cisagov/vdp-scanner:0.0.2 The images of this container are tagged with [semantic versions](https://semver.org). It is recommended that most users use -a version tag (e.g. `:0.0.2`). +a version tag (e.g. `:0.0.3`). | Image:tag | Description | |-----------|-------------| -|`cisagov/vdp-scanner:0.0.2`| An exact release version. | +|`cisagov/vdp-scanner:0.0.3`| An exact release version. | |`cisagov/vdp-scanner:0.0`| The most recent release matching the major and minor version numbers. | |`cisagov/vdp-scanner:0`| The most recent release matching the major version number. | |`cisagov/vdp-scanner:edge` | The most recent image built from a merge into the `develop` branch of this repository. | @@ -155,7 +155,7 @@ Build the image locally using this git repository as the [build context](https:/ ```console docker build \ - --tag cisagov/vdp-scanner:0.0.2 \ + --tag cisagov/vdp-scanner:0.0.3 \ https://github.com/cisagov/vdp-scanner-docker.git#develop ``` @@ -186,7 +186,7 @@ Docker: --file Dockerfile-x \ --platform linux/amd64 \ --output type=docker \ - --tag cisagov/vdp-scanner:0.0.2 . + --tag cisagov/vdp-scanner:0.0.3 . ``` ## Contributing ## diff --git a/src/version.txt b/src/version.txt index 3b93d0b..27fdca4 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.0.2" +__version__ = "0.0.3"