From 5161905997d154361dfe2f59e2823bbe4c9eb04d Mon Sep 17 00:00:00 2001 From: Ashley Song <165685692+ashsong-nv@users.noreply.github.com> Date: Thu, 7 Nov 2024 00:04:13 -0500 Subject: [PATCH] Merge `develop` branch into `main` (#1) * Initial commit of code base --------- Co-authored-by: Michael Demoret --- .gitattributes | 1 + .gitignore | 197 + .tmp/.gitkeep | 0 Dockerfile | 97 + README.md | 863 ++- ci/scripts/copyright.py | 555 ++ ci/scripts/gitutils.py | 645 +++ configs/from_file.json | 73 + configs/from_http.json | 74 + configs/from_manual.json | 142 + configs/openapi/openapi.json | 1223 +++++ configs/schemas/config.schema.json | 1335 +++++ .../morpheus:23.11-runtime.json | 64 + .../morpheus:24.03-runtime.json | 64 + .../rapidsai:23.08-cuda11.8-py3.10.json | 282 + .../rapidsai:24.06-cuda12.0-py3.9.json | 247 + data/sboms/README.md | 59 + .../morpheus/morpheus:23.07-runtime.sbom | 494 ++ .../morpheus/morpheus:23.11-runtime.sbom | 465 ++ .../morpheus/morpheus:24.03-runtime.sbom | 493 ++ .../morpheus/morpheus:v23.07.03-runtime.sbom | 502 ++ .../morpheus/morpheus:v23.11.01-runtime.sbom | 465 ++ .../morpheus/morpheus:v24.03.02-runtime.sbom | 493 ++ .../rapidsai/base:23.08-cuda11.8-py3.10.sbom | 621 +++ .../rapidsai/base:24.06-cuda12.2-py3.10.sbom | 599 +++ .../cve_validation_formatted_openai.json | 147 + docker-compose.nim.yml | 76 + docker-compose.yml | 132 + docker/scripts/entrypoint.sh | 27 + images/hosting_options.png | 3 + ...nalysis_ai_workflow_conceptual_diagram.jpg | 3 + nginx/.gitignore | 6 + nginx/nginx_cache.conf | 120 + nginx/nginx_ssl.conf | 24 + nginx/templates/routes/intel.conf.template | 123 + nginx/templates/routes/nemo.conf.template | 24 + nginx/templates/routes/nim.conf.template | 45 + nginx/templates/routes/nvidia.conf.template | 47 + nginx/templates/routes/openai.conf.template | 23 + .../template-variables.conf.template | 57 + quick_start/.gitkeep | 0 quick_start/quick_start_guide.ipynb | 4688 +++++++++++++++++ requirements.yaml | 72 + scripts/run_all_messages.sh | 37 + src/__init__.py | 14 + src/cli.py | 81 + src/cve/__init__.py | 21 + src/cve/data_models/common.py | 77 + src/cve/data_models/config.py | 313 ++ src/cve/data_models/cve_intel.py | 221 + src/cve/data_models/dependencies.py | 60 + src/cve/data_models/info.py | 64 + src/cve/data_models/input.py | 162 + src/cve/data_models/output.py | 89 + src/cve/data_models/plugin.py | 58 + src/cve/data_models/vulnerability_level.py | 44 + src/cve/nodes/__init__.py | 14 + src/cve/nodes/cve_checklist_node.py | 154 + src/cve/nodes/cve_justification_node.py | 155 + src/cve/nodes/cve_langchain_agent_node.py | 184 + src/cve/nodes/cve_summary_node.py | 91 + src/cve/pipeline/__init__.py | 20 + src/cve/pipeline/engine.py | 239 + src/cve/pipeline/input.py | 324 ++ src/cve/pipeline/output.py | 74 + src/cve/pipeline/pipeline.py | 227 + src/cve/run.py | 409 ++ src/cve/stages/__init__.py | 14 + src/cve/stages/build_vdb_stage.py | 143 + src/cve/stages/convert_to_output_object.py | 146 + src/cve/stages/pydantic_http_stage.py | 233 + src/cve/stages/write_pydantic_to_file.py | 122 + src/cve/utils/__init__.py | 14 + src/cve/utils/async_http_utils.py | 101 + src/cve/utils/clients/__init__.py | 14 + src/cve/utils/clients/first_client.py | 71 + src/cve/utils/clients/ghsa_client.py | 93 + src/cve/utils/clients/intel_client.py | 85 + src/cve/utils/clients/nvd_client.py | 290 + src/cve/utils/clients/rhsa_client.py | 74 + src/cve/utils/clients/ubuntu_client.py | 70 + src/cve/utils/code_searcher.py | 104 + src/cve/utils/data_utils.py | 49 + src/cve/utils/document_embedding.py | 478 ++ src/cve/utils/embedding_loader.py | 86 + src/cve/utils/intel_retriever.py | 191 + src/cve/utils/intel_utils.py | 132 + src/cve/utils/output_formatter.py | 280 + src/cve/utils/prompting.py | 258 + src/cve/utils/sbom_parsers/__init__.py | 14 + src/cve/utils/sbom_parsers/sbom_parser.py | 84 + src/cve/utils/sbom_parsers/syft_parser.py | 45 + src/cve/utils/serp_api_wrapper.py | 82 + src/cve/utils/source_code_git_loader.py | 194 + src/cve/utils/string_utils.py | 125 + src/cve/utils/tools.py | 175 + src/cve/utils/url_utils.py | 27 + .../utils/vulnerable_dependency_checker.py | 679 +++ src/main.py | 35 + 99 files changed, 23308 insertions(+), 2 deletions(-) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 .tmp/.gitkeep create mode 100755 Dockerfile create mode 100755 ci/scripts/copyright.py create mode 100755 ci/scripts/gitutils.py create mode 100644 configs/from_file.json create mode 100644 configs/from_http.json create mode 100644 configs/from_manual.json create mode 100644 configs/openapi/openapi.json create mode 100644 configs/schemas/config.schema.json create mode 100644 data/input_messages/morpheus:23.11-runtime.json create mode 100644 data/input_messages/morpheus:24.03-runtime.json create mode 100644 data/input_messages/rapidsai:23.08-cuda11.8-py3.10.json create mode 100644 data/input_messages/rapidsai:24.06-cuda12.0-py3.9.json create mode 100644 data/sboms/README.md create mode 100644 data/sboms/nvcr.io/nvidia/morpheus/morpheus:23.07-runtime.sbom create mode 100644 data/sboms/nvcr.io/nvidia/morpheus/morpheus:23.11-runtime.sbom create mode 100644 data/sboms/nvcr.io/nvidia/morpheus/morpheus:24.03-runtime.sbom create mode 100644 data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.07.03-runtime.sbom create mode 100644 data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom create mode 100644 data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom create mode 100644 data/sboms/nvcr.io/nvidia/rapidsai/base:23.08-cuda11.8-py3.10.sbom create mode 100644 data/sboms/nvcr.io/nvidia/rapidsai/base:24.06-cuda12.2-py3.10.sbom create mode 100644 data/validation_output/cve_validation_formatted_openai.json create mode 100755 docker-compose.nim.yml create mode 100755 docker-compose.yml create mode 100755 docker/scripts/entrypoint.sh create mode 100644 images/hosting_options.png create mode 100644 images/security_vulnerability_analysis_ai_workflow_conceptual_diagram.jpg create mode 100644 nginx/.gitignore create mode 100644 nginx/nginx_cache.conf create mode 100644 nginx/nginx_ssl.conf create mode 100644 nginx/templates/routes/intel.conf.template create mode 100644 nginx/templates/routes/nemo.conf.template create mode 100644 nginx/templates/routes/nim.conf.template create mode 100644 nginx/templates/routes/nvidia.conf.template create mode 100644 nginx/templates/routes/openai.conf.template create mode 100644 nginx/templates/variables/template-variables.conf.template create mode 100644 quick_start/.gitkeep create mode 100644 quick_start/quick_start_guide.ipynb create mode 100644 requirements.yaml create mode 100755 scripts/run_all_messages.sh create mode 100644 src/__init__.py create mode 100644 src/cli.py create mode 100644 src/cve/__init__.py create mode 100644 src/cve/data_models/common.py create mode 100644 src/cve/data_models/config.py create mode 100644 src/cve/data_models/cve_intel.py create mode 100644 src/cve/data_models/dependencies.py create mode 100644 src/cve/data_models/info.py create mode 100644 src/cve/data_models/input.py create mode 100644 src/cve/data_models/output.py create mode 100644 src/cve/data_models/plugin.py create mode 100644 src/cve/data_models/vulnerability_level.py create mode 100644 src/cve/nodes/__init__.py create mode 100644 src/cve/nodes/cve_checklist_node.py create mode 100644 src/cve/nodes/cve_justification_node.py create mode 100644 src/cve/nodes/cve_langchain_agent_node.py create mode 100644 src/cve/nodes/cve_summary_node.py create mode 100644 src/cve/pipeline/__init__.py create mode 100644 src/cve/pipeline/engine.py create mode 100644 src/cve/pipeline/input.py create mode 100644 src/cve/pipeline/output.py create mode 100644 src/cve/pipeline/pipeline.py create mode 100644 src/cve/run.py create mode 100644 src/cve/stages/__init__.py create mode 100644 src/cve/stages/build_vdb_stage.py create mode 100644 src/cve/stages/convert_to_output_object.py create mode 100644 src/cve/stages/pydantic_http_stage.py create mode 100644 src/cve/stages/write_pydantic_to_file.py create mode 100644 src/cve/utils/__init__.py create mode 100644 src/cve/utils/async_http_utils.py create mode 100644 src/cve/utils/clients/__init__.py create mode 100644 src/cve/utils/clients/first_client.py create mode 100644 src/cve/utils/clients/ghsa_client.py create mode 100644 src/cve/utils/clients/intel_client.py create mode 100644 src/cve/utils/clients/nvd_client.py create mode 100644 src/cve/utils/clients/rhsa_client.py create mode 100644 src/cve/utils/clients/ubuntu_client.py create mode 100644 src/cve/utils/code_searcher.py create mode 100644 src/cve/utils/data_utils.py create mode 100644 src/cve/utils/document_embedding.py create mode 100644 src/cve/utils/embedding_loader.py create mode 100644 src/cve/utils/intel_retriever.py create mode 100644 src/cve/utils/intel_utils.py create mode 100644 src/cve/utils/output_formatter.py create mode 100644 src/cve/utils/prompting.py create mode 100644 src/cve/utils/sbom_parsers/__init__.py create mode 100644 src/cve/utils/sbom_parsers/sbom_parser.py create mode 100644 src/cve/utils/sbom_parsers/syft_parser.py create mode 100644 src/cve/utils/serp_api_wrapper.py create mode 100644 src/cve/utils/source_code_git_loader.py create mode 100644 src/cve/utils/string_utils.py create mode 100644 src/cve/utils/tools.py create mode 100644 src/cve/utils/url_utils.py create mode 100644 src/cve/utils/vulnerable_dependency_checker.py create mode 100644 src/main.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..0dc6018 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +images/** filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..aa5db81 --- /dev/null +++ b/.gitignore @@ -0,0 +1,197 @@ +###### Place new entries directly below this line! ###### + +# Ignore anything in the ./.tmp directory +.tmp/ + +# Ignore user-generated fiass databases +vdb/ + +# Explicitly ignore .vscode/. Shared settings should go in morpheus.code-workspace +# and user settings will go in .vscode/ +.vscode/ + +# Ignore .sqlite cache files +*.sqlite + + +##### Do not alter the items below this line! ##### +########## They are managed by a script! ########## + +# Created by https://www.gitignore.io/api/vim,c++,cmake,python,synology + +### C++ ### +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +### CMake ### +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +### Synology ### +# Thumbnails +@eaDir +# Recycle bin +\#recycle + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + + +# End of https://www.gitignore.io/api/vim,c++,cmake,python,synology diff --git a/.tmp/.gitkeep b/.tmp/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/Dockerfile b/Dockerfile new file mode 100755 index 0000000..9690890 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE_IMG=nvcr.io/nvidia/cuda +ARG BASE_IMG_TAG=12.5.1-base-ubuntu22.04 + +FROM ${BASE_IMG}:${BASE_IMG_TAG} as base + +# Install necessary dependencies using apt-get +RUN apt-get update && apt-get install -y \ + git \ + git-lfs \ + wget \ + && apt-get clean + +# Install miniconda +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/miniconda.sh \ + && bash /tmp/miniconda.sh -b -p /opt/conda \ + && rm /tmp/miniconda.sh + +# Add conda to the PATH +ENV PATH=/opt/conda/bin:$PATH + +# Install Mamba, a faster alternative to conda, within the base environment +RUN --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \ + conda install -y mamba -n base -c conda-forge + +# Create base environment +RUN --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \ + conda create -y --name morpheus-vuln-analysis + +# Activate the environment (make it default for subsequent commands) +RUN echo "source activate morpheus-vuln-analysis" >> ~/.bashrc + +# Set default shell to bash +SHELL ["/bin/bash", "-c"] + +# Add conda channels required for the Morpheus dependencies +RUN source activate morpheus-vuln-analysis \ + && conda config --env --add channels conda-forge \ + && conda config --env --add channels nvidia \ + && conda config --env --add channels rapidsai \ + && conda config --env --add channels pytorch + +RUN --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \ + source activate morpheus-vuln-analysis &&\ + mamba install -y -c conda-forge tini=0.19 + +WORKDIR /workspace_examples/ + +# Copy custom entrypoint script +# Copy just the conda env first to minimize cache busting +COPY requirements.yaml /workspace_examples/requirements.yaml + +SHELL ["/bin/bash", "-c"] + +# Install dependencies +RUN --mount=type=cache,id=pip_cache,target=/root/.cache/pip,sharing=locked \ + --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \ + source activate morpheus-vuln-analysis &&\ + mamba env update morpheus-vuln-analysis -f ./requirements.yaml + +# If any changes have been made from the base image, recopy the sources +COPY . /workspace_examples/ + +# Mark all git repos as safe to avoid git errors +RUN git config --global --add safe.directory "*" + +ENTRYPOINT [ "/opt/conda/envs/morpheus-vuln-analysis/bin/tini", "--", "/workspace_examples/docker/scripts/entrypoint.sh"] + +# ===== Setup for development ===== +FROM base as runtime + +RUN --mount=type=cache,id=pip_cache,target=/root/.cache/pip,sharing=locked \ + --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \ + source activate morpheus-vuln-analysis &&\ + mamba install -y -c conda-forge \ + ipywidgets \ + jupyter_contrib_nbextensions \ + # notebook v7 is incompatible with jupyter_contrib_nbextensions + notebook=6 &&\ + jupyter contrib nbextension install --user &&\ + pip install jupyterlab_nvdashboard==0.9 + +CMD ["jupyter-lab", "--no-browser", "--allow-root", "--ip='*'", "--NotebookApp.token=''", "--NotebookApp.password=''"] diff --git a/README.md b/README.md index 11e11b6..6231503 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,861 @@ -# vulnerability-analysis -Rapidly identify and mitigate container security vulnerabilities with generative AI. + + +

NIM Agent Blueprint: Vulnerability Analysis for Container Security

+ +## Table of Contents +- [Table of Contents](#table-of-contents) +- [Overview](#overview) +- [Software components](#software-components) +- [Target audience](#target-audience) +- [Prerequisites](#prerequisites) +- [Hardware requirements](#hardware-requirements) +- [API definition](#api-definition) +- [Use case description](#use-case-description) + - [How it works](#how-it-works) + - [Key components](#key-components) + - [NIM microservices](#nim-microservices) +- [Getting started](#getting-started) + - [Prerequisites](#prerequisites-1) + - [Install system requirements](#install-system-requirements) + - [Obtain API keys](#obtain-api-keys) + - [Set up the workflow repository](#set-up-the-workflow-repository) + - [Environment setup](#environment-setup) + - [Set up the environment file](#set-up-the-environment-file) + - [Build the Docker containers](#build-the-docker-containers) +- [Running the workflow](#running-the-workflow) + - [Run the workflow using NVIDIA-hosted NIMs](#run-the-workflow-using-nvidia-hosted-nims) + - [Run the workflow using self-hosted NIMs](#run-the-workflow-using-self-hosted-nims) + - [Starting the workflow](#starting-the-workflow) + - [From the quick start user guide notebook](#from-the-quick-start-user-guide-notebook) + - [From the command line](#from-the-command-line) + - [Command line interface](#command-line-interface) + - [Command line interface (CLI)](#command-line-interface-cli) + - [Overriding options on the command line](#overriding-options-on-the-command-line) + - [Example 1: Override `max_retries`](#example-1-override-max_retries) + - [Example 2: Override input type](#example-2-override-input-type) + - [Example 3: Combine multiple overrides](#example-3-combine-multiple-overrides) + - [Full list of options](#full-list-of-options) + - [Configuration files](#configuration-files) + - [Configuration file breakdown](#configuration-file-breakdown) + - [NGINX caching server](#nginx-caching-server) +- [Customizing the Workflow](#customizing-the-workflow) + - [Customizing the input](#customizing-the-input) + - [Customizing the embedding model](#customizing-the-embedding-model) + - [Customizing the LLM models](#customizing-the-llm-models) + - [Supported LLM services](#supported-llm-services) + - [Customizing the Output](#customizing-the-output) + - [Integrating self-hosted LLM NIM](#integrating-self-hosted-llm-nim) +- [Troubleshooting](#troubleshooting) + - [Git LFS issues](#git-lfs-issues) + - [Container build issues](#container-build-issues) + - [Device error](#device-error) + - [Deploy.Resources.Reservations.devices error](#deployresourcesreservationsdevices-error) + - [NGINX caching server](#nginx-caching-server-1) + - [Resetting the entire cache](#resetting-the-entire-cache) + - [Resetting just the LLM cache or the services cache](#resetting-just-the-llm-cache-or-the-services-cache) + - [Vector databases](#vector-databases) + - [Service outages](#service-outages) + - [National Vulnerability Database (NVD)](#national-vulnerability-database-nvd) + - [Running out of credits](#running-out-of-credits) +- [Testing and validation](#testing-and-validation) +- [License](#license) +- [Terms of Use](#terms-of-use) + + +## Overview +This repository is what powers the [build experience](https://build.nvidia.com/nvidia/vulnerability-analysis-for-container-security), showcasing vulnerability analysis for container security using NVIDIA NIM microservices and [NVIDIA Morpheus](https://github.com/nv-morpheus/Morpheus). + +The NIM Agent Blueprint demonstrates accelerated analysis on common vulnerabilities and exposures (CVE) at an enterprise scale, reducing mitigation from days and hours to just seconds. While traditional methods require substantial manual effort to pinpoint solutions for vulnerabilities, these technologies enable quick, automatic, and actionable CVE risk analysis using large language models (LLMs) and retrieval-augmented generation (RAG). With this blueprint, security analysts can expedite the process of determining whether a software package includes exploitable and vulnerable components using LLMs and event-driven RAG triggered by the creation of a new software package or the detection of a CVE. + +## Software components +The following are used by this blueprint: +- [NIM of meta/llama3-70b-instruct](https://build.nvidia.com/meta/llama3-70b) +- [NIM of nvidia/nv-embedqa-e5-v5](https://build.nvidia.com/nvidia/nv-embedqa-e5-v5) +- [NVIDIA Morpheus Cybersecurity AI SDK](https://developer.nvidia.com/morpheus-cybersecurity) + +## Target audience +This blueprint is for: +- **Security analysts and IT engineers**: People analyzing vulnerabilities and ensuring the security of containerized environments. +- **AI practitioners in cybersecurity**: People applying AI to enhance cybersecurity, particularly those interested in using the Morpheus SDK and NIMs for faster vulnerability detection and analysis. + +## Prerequisites +- NVAIE developer licence +- API keys for vulnerability databases, search engines, and LLM model service(s). + - Details can be found in this later section: [Obtain API keys](#obtain-api-keys) + +## Hardware requirements +The vulnerability analysis pipeline supports the following hardware: + +- **Pipeline operation**: 1x L40 GPU recommended +- **LLM NIM**: [Meta Llama 3 70B Instruct Support Matrix](https://docs.nvidia.com/nim/large-language-models/latest/support-matrix.html#meta-llama-3-70b-instruct) (Optional) + - For improved paralleled performance, we recommend 8x or more H100s for LLM inference. + - The pipeline can share the GPU with the LLM NIM, but it is recommended to have a separate GPU for the LLM NIM for optimal performance. +- **Embedding NIM**: [NV-EmbedQA-E5-v5 Support Matrix](https://docs.nvidia.com/nim/nemo-retriever/text-embedding/latest/support-matrix.html#nv-embedqa-e5-v5) (Optional) + - The pipeline can share the GPU with the Embedding NIM, but it is recommended to have a separate GPU for the Embedding NIM for optimal performance. + + +## API definition + +[OpenAPI Specification](./configs/openapi/openapi.json) + +## Use case description +Determining the impact of a documented CVE on a specific project or container is a labor-intensive and manual task, especially as the rate of new reports into the [CVE database](https://www.cve.org/) accelerates. This process involves the collection, comprehension, and synthesis of various pieces of information to ascertain whether immediate remediation is necessary upon the identification of a new CVE. + +**Current challenges in CVE analysis:** +- **Information collection**: The process involves significant manual labor to collect and synthesize relevant information. +- **Decision complexity**: Decisions on whether to update a library impacted by a CVE often hinge on various considerations, including: + - **Scan false positives**: Occasionally, vulnerability scans may incorrectly flag a library as vulnerable, leading to a false alarm. + - **Mitigating factors**: In some cases, existing safeguards within the environment may reduce or negate the risk posed by a CVE. + - **Lack of required environments or dependencies**: For an exploit to succeed, specific conditions must be met. The absence of these necessary elements can render a vulnerability irrelevant. +- **Manual documentation**: Once an analyst has determined the library is not affected, a Vulnerability Exploitability eXchange (VEX) document must be created to standardize and distribute the results. + +The efficiency of this process can be significantly enhanced through the deployment of an automated LLM agent pipeline, leveraging generative AI to improve vulnerability defense while decreasing the load on security teams. + +### How it works +The workflow operates using a Plan-and-Execute-style LLM pipeline for CVE impact analysis. The process begins with an LLM planner that generates a context-sensitive task checklist. This checklist is then executed by an LLM agent equipped with Retrieval-Augmented Generation (RAG) capabilities. The gathered information and the agent's findings are subsequently summarized and categorized by additional LLM nodes to provide a final verdict. + +> [!TIP] +> The pipeline is adaptable, supporting various LLM services that conform to the `LLMService` interface, including OpenAI, NeMo, or local execution with llama-cpp-python. + +### Key components +The detailed architecture consists of the following components: + +

+ +

+ +- **Security scan result**: The workflow begins by inputting the identified CVEs from a container security scan as input. This can be generated from a container image scanner of your choosing such as [Anchore](https://anchore.com/container-vulnerability-scanning/). + +- **PreProcessing**: All the below actions are encapsulated by multiple Morpheus preprocessing pipeline stages to prepare the data for use with the LLM engine. (See `src/cve/pipeline/input.py`.) + - **Code repository and documentation**: The blueprint pulls code repositories and documentation provided by the user. These repositories are processed through an embedding model, and the resulting embeddings are stored in vector databases (VDBs) for the agent's reference. + - **Vector database**: Various vector databases can be used for the embedding. We currently utilize FAISS for the VDB because it does not require an external service and is simple to use. Any vector store can be used, such as NVIDIA cuVS, which would provide accelerated indexing and search. + - **Lexical search**: As an alternative, a lexical search is available for use cases where creating an embedding is impractical due to a large number of source files in the target container. + - **Software Bill of Materials (SBOM)**: The provided SBOM document is processed into a software-ingestible format for the agent's reference. SBOMs can be generated for any container using the open-source tool [Syft](https://github.com/anchore/syft). + - **Web vulnerability intel**: The system collects detailed information about each CVE through web scraping and data retrieval from various public security databases, including GHSA, Redhat, Ubuntu, and NIST CVE records, as well as tailored threat intelligence feeds. + +- **Core LLM engine**: (See `src/cve/pipeline/engine.py`.) + - **Checklist generation**: Leveraging the gathered information about each vulnerability, the checklist generation node creates a tailored, context-sensitive task checklist designed to guide the impact analysis. (See `src/cve/nodes/cve_checklist_node.py`.) + + - **Task agent**: At the core of the process is an LLM agent iterating through each item in the checklist. For each item, the agent answers the question using a set of tools which provide information about the target container. The tools tap into various data sources (web intel, vector DB, search etc.), retrieving relevant information to address each checklist item. The loop continues until the agent resolves each checklist item satisfactorily. (See `src/cve/nodes/cve_langchain_agent_node.py`.) + + - **Summarization**: Once the agent has compiled findings for each checklist item, these results are condensed by the summarization node into a concise, human-readable paragraph. (See `src/cve/nodes/cve_summary_node.py`.) + + - **Justification Assignment**: Given the summary, the justification status categorization node then assigns a resulting VEX (Vulnerability Exploitability eXchange) status to the CVE. We provided a set of predefined categories for the model to choose from. (See `src/cve/nodes/cve_justification_node.py`.) If the CVE is deemed exploitable, the reasoning category is "vulnerable." If it is not exploitable, there are 10 different reasoning categories to explain why the vulnerability is not exploitable in the given environment: + - `false_positive` + - `code_not_present` + - `code_not_reachable` + - `requires_configuration` + - `requires_dependency` + - `requires_environment` + - `protected_by_compiler` + - `protected_at_runtime` + - `protected_by_perimeter` + - `protected_by_mitigating_control` + +- **Output**: At the end of the pipeline run, an output file including all the gathered and generated information is prepared for security analysts for a final review. (See `src/cve/pipeline/output.py`.) + + > [!WARNING] + > All output should be vetted by a security analyst before being used in a cybersecurity application. + +### NIM microservices +The Morpheus SDK can utilize various embedding model and LLM endpoints, and is optimized to use [NVIDIA NIM microservices](https://developer.nvidia.com/nim) (NIMs). NIMs are pre-built containers for the latest AI models that provide industry-standard APIs and optimized inference for the given model and hardware. Using NIMs enables easy deployment and scaling for self-hosted model inference. + +The current default embedding NIM model is `nv-embedqa-e5-v5`, which was selected to balance speed and overall pipeline accuracy. The current default LLM model is the `llama3-70b-instruct` NIM, with specifically tailored prompt engineering and edge case handling. Other models are able to be substituted for either the embedding or LLM model, such as smaller, fine-tuned NIM LLM models or other external LLM inference services. Subsequent updates will provide more details about fine-tuning and data flywheel techniques. + +> [!NOTE] +> The LangChain library is employed to deploy all LLM agents within a Morpheus pipeline, streamlining efficiency and reducing the need for duplicative efforts. + +> [!TIP] +> Routinely checked validation datasets are critical to ensuring proper and consistent outputs. Learn more about our test-driven development approach in the section on [testing and validation](#testing-and-validation). + +## Getting started + +### Prerequisites + +#### Install system requirements + +* [git](https://git-scm.com/) +* [git-lfs](https://git-lfs.com/) +* Since the workflow uses an [NVIDIA Morpheus](https://developer.nvidia.com/morpheus-cybersecurity) pipeline, the [Morpheus requirements](https://docs.nvidia.com/morpheus/getting_started.html#requirements) also need to be installed. + +#### Obtain API keys +To run the pipeline you need to obtain API keys for the following APIs. These will be needed in a later step to [Set up the environment file](#set-up-the-environment-file). + +- **Required API Keys**: These APIs are required by the pipeline to retrieve vulnerability information from databases, perform online searches, and execute LLM queries. + + - GitHub Security Advisory (GHSA) Database + - Follow [these](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token) instructions to create a personal access token. No repository access or permissions are required for this API. + - This will be used in the `GHSA_API_KEY` environment variable. + - National Vulnerability Database (NVD) + - Follow [these](https://nvd.nist.gov/developers/request-an-api-key) instructions to create an API key. + - This will be used in the `NVD_API_KEY` environment variable. + - SerpApi + - Go to https://serpapi.com/ and create a SerpApi account. Once signed in, navigate to Your Account > Api Key. + - This will be used in the `SERPAPI_API_KEY` environment variable. + - NVIDIA Inference Microservices (NIM) + - There are two possible methods to generate an API key for NIM: + - Sign in to the [NVIDIA Build](https://build.nvidia.com/explore/discover?signin=true) portal with your email. + - Click on any [model](https://build.nvidia.com/meta/llama3-70b), then click "Get API Key", and finally click "Generate Key". + - Sign in to the [NVIDIA NGC](https://ngc.nvidia.com/) portal with your email. + - Select your organization from the dropdown menu after logging in. You must select an organization which has NVIDIA AI Enterprise (NVAIE) enabled. + - Click on your account in the top right, select "Setup" from the dropdown. + - Click the "Generate Personal Key" option and then the "+ Generate Personal Key" button to create your API key. + - This will be used in the `NVIDIA_API_KEY` environment variable. + +The workflow can be configured to use other LLM services as well, see the [Customizing the LLM models](#customizing-the-llm-models) section for more info. + + +#### Set up the workflow repository + +Clone the repository and set an environment variable for the path to the repository root. + +```bash +export REPO_ROOT=$(git rev-parse --show-toplevel) +``` + +All commands are run from the repository root unless otherwise specified. + +### Environment setup + +#### Set up the environment file + +First we need to create an `.env` file in the `REPO_ROOT`, and add the API keys you created in the earlier [Obtain API keys](#obtain-api-keys) step. + +```bash +cd $REPO_ROOT +cat < .env +GHSA_API_KEY="your GitHub personal access token" +NVD_API_KEY="your National Vulnerability Database API key" +NVIDIA_API_KEY="your NVIDIA Inference Microservices API key" +SERPAPI_API_KEY="your SerpApi API key" +EOF +``` + +These variables need to be exported to the environment: +```bash +export $(cat .env | xargs) +``` + +#### Build the Docker containers +Next we need to build the necessary containers. +```bash +cd $REPO_ROOT + +# Build the morpheus-vuln-analysis container +docker compose build morpheus-vuln-analysis +``` + +## Running the workflow + +To run the workflow, there are two supported configurations. Both configurations utilize `docker compose` to start the service: +1. [**NVIDIA-hosted NIMs**](#run-the-workflow-using-nvidia-hosted-nims): The workflow is run with all computation being performed by NIMs hosted in NVIDIA GPU Cloud. This is the default configuration and is recommended for most users getting started with the workflow. + 1. When using NVIDIA-hosted NIMs, only the `docker-compose.yml` configuration file is required. +2. [**Self-hosted NIMs**](#run-the-workflow-using-self-hosted-nims): The workflow is run using self-hosted LLM NIM services. This configuration is more advanced and requires additional setup to run the NIM services locally. + 1. When using self-hosted NIMs, both the `docker-compose.yml` and `docker-compose.nim.yml` configuration files are required. + +These two configurations are illustrated by the following diagram: + +

+Workflow configurations +

+ +Before beginning, ensure that the environment variables are set correctly. Both configurations require the same environment variables to be set. More information on setting these variables can be found in the [Obtain API keys](#obtain-api-keys) section. + +>[!TIP] +The container binds to port 8080 by default. If you encounter a port collision error (e.g. `Bind for 0.0.0.0:8080 failed: port is already allocated`), you can set the environment variable `NGINX_HOST_HTTP_PORT` to specify a custom port before launching `docker compose`. For example: + +```bash +export NGINX_HOST_HTTP_PORT=8081 + +#... docker compose commands... +``` + +#### Run the workflow using NVIDIA-hosted NIMs +When running the workflow in this configuration, only the `morpheus-vuln-analysis` service needs to be started since we will utilize NIMs hosted by NVIDIA. The `morpheus-vuln-analysis` container can be started using the following command: + +```bash +cd ${REPO_ROOT} +docker compose up -d +``` + +The command above starts the container in the background using the detached mode, `-d`. We can confirm the container is running via the following command: +```bash +docker compose ps +``` + +Continue to the [Starting the workflow](#starting-the-workflow) section to run the workflow. + +#### Run the workflow using self-hosted NIMs +To run the workflow using self-hosted NIMs, we use a second `docker compose` configuration file, `docker-compose.nim.yml`, which adds the self-hosted NIM services to the workflow. Utilizing a second configuration file allows for easy switching between the two configurations while keeping the base configuration file the same. + +>[!NOTE] +> The self-hosted NIM services require additional GPU resources to run. With this configuration, the LLM NIM, embedding model NIM, and the `morpheus-vuln-analysis` service will all be launched on the same machine. Ensure that you have the necessary hardware requirements for all three services before proceeding (multiple services can share the same GPU). + +To use multiple configuration files, we need to specify both configuration files when running the `docker compose` command. You will need to specify both configuration files for every `docker compose` command. For example: + +```bash +docker compose -f docker-compose.yml -f docker-compose.nim.yml [NORMAL DOCKER COMPOSE COMMAND] +``` + +For example, to start the `morpheus-vuln-analysis` service with the self-hosted NIMs, you would run: +```bash +cd ${REPO_ROOT} +docker compose -f docker-compose.yml -f docker-compose.nim.yml up -d +``` + +Continue to the [Starting the workflow](#starting-the-workflow) section to run the workflow. + +### Starting the workflow + +Once the services have been started, the workflow can be run using an [interactive notebook](#from-the-quick-start-user-guide-notebook) (for a guided, step-by-step process) or from the [command line](#from-the-command-line). + +### From the quick start user guide notebook +To run the workflow in an interactive notebook, connect to the Jupyter notebook at http://localhost:8888/lab. Once connected, navigate to the notebook located at [`quick_start/quick_start_guide.ipynb`](./quick_start/quick_start_guide.ipynb) and follow the instructions. + +> [!TIP] +> If you are running the workflow on a remote machine, you can forward the port to your local machine using SSH. For example, to forward port 8888 from the remote machine to your local machine, you can run the following command from your local machine: +> +> ```bash +> ssh -L 8888:127.0.0.1:8888 +> ``` + +### From the command line + +To run the workflow from the command line, you can use the provided CLI tool. In order to use the CLI tool, we must first attach to the `morpheus-vuln-analysis` container: + +```bash +docker compose exec -it morpheus-vuln-analysis bash +``` + +If you are using self-hosted NIMs, make sure to use both configuration files: +```bash +docker compose -f docker-compose.yml -f docker-compose.nim.yml exec -it morpheus-vuln-analysis bash +``` + +From here, you can run the workflow using the provided CLI tool outlined in the [following section](#command-line-interface). + +### Command line interface + +The pipeline can be started using the following command: +```bash +python src/main.py --log_level DEBUG \ + cve pipeline --config_file=${CONFIG_FILE} +``` + +Where `${CONFIG_FILE}` is the path to the configuration file you want to use. For example, to run the pipeline with the `from_manual.json` configuration file, you would run: +```bash +python src/main.py --log_level DEBUG \ + cve pipeline --config_file=configs/from_manual.json +``` + +This pipeline will run to completion and the output is stashed by default at `.tmp/output.json`. Sample logs are as follows: +``` +Message elapsed time: 28.240849 sec +Vulnerability 'GHSA-3f63-hfp8-52jq' affected status: FALSE. Label: code_not_reachable +Vulnerability 'CVE-2023-50782' affected status: FALSE. Label: requires_configuration +Vulnerability 'CVE-2023-36632' affected status: FALSE. Label: code_not_present +Vulnerability 'CVE-2023-43804' affected status: TRUE. Label: vulnerable +Vulnerability 'GHSA-cxfr-5q3r-2rc2' affected status: TRUE. Label: vulnerable +Vulnerability 'GHSA-554w-xh4j-8w64' affected status: TRUE. Label: vulnerable +Vulnerability 'GHSA-3ww4-gg4f-jr7f' affected status: FALSE. Label: requires_configuration +Vulnerability 'CVE-2023-31147' affected status: FALSE. Label: code_not_present +Source[Complete]: 7 messages [00:14, 2.02s/ messages] +LLM[Complete]: 7 messages [00:42, 6.05s/ messages] +====Pipeline Complete==== +Total time: 45.61 sec +Pipeline runtime: 42.69 sec + +``` + +In addition to the raw JSON output in `.tmp/output.json`, you can also view a Markdown-formatted report for each CVE in the `.tmp/vulnerability_markdown_reports directory`. + +> [!WARNING] +> The output you receive from the pipeline may not be identical as the output in the example above. The output may vary due to the non-deterministic nature of the LLM models. + +Similarly, to run the pipeline with the `from_http.json` configuration file, you would run: +```bash +python src/main.py --log_level DEBUG cve pipeline --config_file=configs/from_http.json +``` + +This command starts an HTTP server that listens on port `26466` and runs the workflow indefinitely, waiting for incoming data to process. This is useful if you want to trigger the workflow on demand via HTTP requests. + +Once the server is running, you can send a `POST` request to the `/scan` endpoint with the input parameters in the request body. The pipeline will process the input data and return the output in the terminal and the given output path in the config file. + +Here's an example using `curl` to send a `POST` request. From a new terminal outside of the container, go to the root of the cloned git repository, and run: +```bash +curl -X POST http://localhost:26466/scan -d @data/input_messages/morpheus:24.03-runtime.json +``` +In this command: +- `http://localhost:26466/scan` is the URL of the server and endpoint. +- The `-d` option specifies the data file being sent in the request body. In this case, it's pointing to the input file `morpheus:24.03-runtime.json` under the `data/input_messages/` directory. You can refer to this file as an example of the expected data format. + - Since it uses a relative path, it's important to run the `curl` command from the root of the git repository. Alternatively, you can modify the relative path in the command to directly reference the example json file. + +Note that the results of the pipeline are not returned to the curl request. After processing the request, the server will save the results to the output path specified in the configuration file. The server will also display log and summary results from the workflow as it's running. Additional submissions to the server will append the results to the specified output file. + +### Command line interface (CLI) + +The top level entrypoint to each of the LLM example pipelines is `src/main.py`. The main entrypoint is a CLI tool with a built documentation using the `--help` command. For example, to see what commands are available, you can run: +``` +(morpheus) root@58145366033a:/workspace_examples# python src/main.py --help +Usage: morpheus_llm [OPTIONS] COMMAND [ARGS]... + + Main entrypoint for the Vulnerability Analysis for Container Security + +Options: + --log_level [CRITICAL|FATAL|ERROR|WARN|WARNING|INFO|DEBUG] + Specify the logging level to use. [default: + INFO] + --use_cpp BOOLEAN Whether or not to use C++ node and message + types or to prefer python. Only use as a + last resort if bugs are encountered + [default: True] + --version Show the version and exit. + --help Show this message and exit. + +Commands: + cve Run the Vulnerability Analysis for Container Security pipeline +``` + +#### Overriding options on the command line + +It's common to want to override some options in the configuration file on the command line. This is useful to use some common options but tweak the input/output of the pipeline or to change a single setting without modifying the original config file. + +##### Example 1: Override `max_retries` +For example, to override the `max_retries` option in the configuration file, you can run: +```bash +python src/main.py --log_level=DEBUG \ + cve pipeline --config_file=configs/from_manual.json \ + config \ + general --max_retries=3 +``` +This will run the pipeline with the `from_manual.json` configuration file, but with the `max_retries` option set to `3`. + +##### Example 2: Override input type +It's also possible to change the input type. For example, to use a different input message with the `from_file.json` configuration file, you can run: +```bash +python src/main.py --log_level=DEBUG \ + cve pipeline --config_file=configs/from_file.json \ + config \ + input-file --file=data/input_messages/rapidsai:23.08-cuda11.8-py3.10.json +``` + +##### Example 3: Combine multiple overrides +It's possible to combine multiple overrides in a single command. For example, to run the pipeline with the `from_manual.json` configuration file, but with the `max_retries` option set to `3`, the input message set to `rapidsai:23.08-cuda11.8-py3.10.json`, and the output destination set to `output_rapids.json` you can run: +```bash +python src/main.py --log_level=DEBUG \ + cve pipeline --config_file=configs/from_manual.json \ + config \ + general --max_retries=3 \ + input-file --file=data/input_messages/rapidsai:23.08-cuda11.8-py3.10.json + output-file --file_path=output_rapids.json --markdown_dir=rapids_reports +``` + +#### Full list of options +For the full list of possible options, use the `--help` options from the CLI. + +### Configuration files + +The pipeline settings are controlled using configuration files. These files are JSON files that define the pipeline settings, such as the input data, the LLM models used, and the output format. Several example configuration files are located in the `configs/` folder. A brief description of each configuration file is as follows: +- `from_manual.json`: This configuration file is used to demonstrate the pipeline using manually provided data. All pipeline inputs are manually specified directly in the configuration file. +- `from_file.json`: This configuration file is used to demonstrate the pipeline using data fetched from a file. The pipeline fetches the input data from a file and processes it. Very similar to `from_manual.json`, but the input data is read from a file instead of being directly specified in the config file. +- `from_http.json`: This configuration file starts an HTTP server to turn the workflow into a microservice. The pipeline fetches the input data from an HTTP source and processes it. To trigger the pipeline, you can send a POST request to the `/scan` endpoint with the input data in the request body. The pipeline will process the input data. + +#### Configuration file breakdown +The configuration defines how the workflow operates, including model settings, input sources, and output options. + +1. **Schema** + - `$schema`: Specifies the schema for validating the configuration file. This ensures the correct structure and data types are used throughout the config file. +2. **LLM engine configuration (`engine`)**: The `engine` section configures various models for the LLM nodes. + - LLM processing nodes: `agent`, `checklist_model`, `justification_model`, `summary_model` + - `model_name`: The name of the LLM model used by the node. + - `service`: Specifies the service for running the LLM inference. (Set to `nvfoundation` if using NIM.) + - `max_tokens`: Defines the maximum number of tokens that can be generated in one output step. + - `temperature`: Controls randomness in the output. A lower temperature produces more deterministic results. + - `top_p`: Limits the diversity of token sampling based on cumulative probability. + - Settings specific to the `agent` node: + - `verbose`: Toggles detailed logging for the agent during execution. + - `return_intermediate_steps`: Controls whether to return intermediate steps taken by the agent, and include them in the output file. Helpful for troubleshooting agent responses. + - `return_source_documents`: Controls whether to return source documents from the VDB tools, and include them in the intermediate steps output. Helpful for identifying the source files used in agent responses. + - Note: enabling this will also include source documents in the agent's memory and increase the agent's prompt length. + - Embedding model for generating VDB for RAG: `rag_embedding` + - `_type`: Defines the source of the model used for generating embeddings (e.g., `nim`, `huggingface`, `openai`). + - Other model-dependent parameters, such as `model`/`model_name`, `api_key`, `truncate`, or `encode_kwargs`: see the [embedding model customization](#customizing-the-embedding-model) section below for more details. +3. **General configuration**: The `general` section contains settings that influence the workflow's general behavior, including cache settings, batch sizes, and retry policies. + - `cache_dir`: The directory where the node's cache should be stored. If None, caching is not used. + - `base_vdb_dir`: The directory used for storing vector database files. + - `base_git_dir`: The directory for storing pulled git repositories used for code analysis. + - `max_retries`: Sets the number of retry attempts for failed operations. + - `model_max_batch_size`: Specifies the maximum number of messages to send to the model for inference in a single batch. + - `pipeline_batch_size`: Determines the number of messages per batch for the pipeline. + - `use_uvloop`: Toggles the use of `uvloop`, an optimized event loop for improved performance. + - `code_search_tool`: Enables or disables the use of the code search tool. +4. **Input configuration**: The `input` section defines how and where the input data (container images and vulnerabilities) is sourced. + - `_type`: Defines the input type + - `manual`: input data is provided manually in the config file + - `http`: input data is provided through http `POST` call pointing to an input source file + - `file`: input data is fetched from a provided source file + - `message`: Contains details about the input image and its associated vulnerabilities. Required only for the `manual` input type. + - `image`: Specifies the container image to be analyzed. + - `name`: Specifies the name of the container image. + - `tag`: Specifies the tag of the container image. + - `source_info`: Specifies the sources (e.g., git repositories) used to retrieve code or documentation for analysis. + - `include`: Specifies the file patterns to be included in the analysis. + - `exclude`: Specifies the file patterns to exclude from analysis. + - `sbom_info`: Specifies the Software Bill of Materials (SBOM) file to be analyzed. + - `scan`: Provides a list of vulnerabilities (by ID) to be analyzed for the input image. +5. **Output configuration**: The `output` section defines where the output results of the workflow should be stored and how they should be managed. + - `_type`: Specifies the output type. Use `file` to write the results to a file. + - `file_path`: Defines the path to the file where the output will be saved. + - `markdown_dir`: Defines the path to the directory where the output will be saved in individual navigable markdown files per CVE-ID. + - `overwrite`: Indicates whether the output file should be overwritten when the pipeline starts if it already exists. Will throw an error if set to `False` and the file already exists. Note that the overwrite behavior only occurs on pipeline + initialization. For pipelines started in HTTP mode, each new request will append the existing file until the pipeline is restarted. + +### NGINX caching server + +The docker compose file includes an `nginx-cache` proxy server container that enables caching for API requests made by the workflow. It is highly recommend to route API requests through the proxy server to reduce API calls for duplicate requests and improve workflow speed. This is especially useful when running the pipeline multiple times with the same configuration (e.g., for debugging) and can help keep costs down when using paid APIs. + +The NGINX proxy server is started by default when running the `morpheus-vuln-analysis` service. However, it can be started separately using the following command: +```bash +cd ${REPO_ROOT} +docker compose up --detach nginx-cache +``` + +To use the proxy server for API calls in the workflow, you can set environment variables for each base URL used by the workflow to point to `http://localhost:${NGINX_HOST_HTTP_PORT}/`. These are set automatically when running the `morpheus-vuln-analysis` service, but can be set manually in the `.env` file as follows: +```bash +CVE_DETAILS_BASE_URL="http://localhost:8080/cve-details" +CWE_DETAILS_BASE_URL="http://localhost:8080/cwe-details" +DEPSDEV_BASE_URL="http://localhost:8080/depsdev" +FIRST_BASE_URL="http://localhost:8080/first" +GHSA_BASE_URL="http://localhost:8080/ghsa" +NGC_API_BASE="http://localhost:8080/nemo/v1" +NIM_EMBED_BASE_URL="http://localhost:8080/nim_embed/v1" +NVD_BASE_URL="http://localhost:8080/nvd" +NVIDIA_API_BASE="http://localhost:8080/nim_llm/v1" +OPENAI_API_BASE="http://localhost:8080/openai/v1" +OPENAI_BASE_URL="http://localhost:8080/openai/v1" +RHSA_BASE_URL="http://localhost:8080/rhsa" +SERPAPI_BASE_URL="http://localhost:8080/serpapi" +UBUNTU_BASE_URL="http://localhost:8080/ubuntu" +``` + +## Customizing the Workflow + +The primary method for customizing the workflow is to generate a new configuration file with new options. The configuration file defines the pipeline settings, such as the input data, the LLM models used, and the output format. The configuration file is a JSON file that can be modified to suit your needs. + +### Customizing the input + +Currently, there are 3 types of input sources supported by the pipeline: +- **Manual input**: The input data is directly specified in the configuration file. +- **File input**: The input data is read from a file. +- **HTTP input**: The input data is fetched from an HTTP source. + +To customize the input, modify the configuration file accordingly. In any configuration file, locate the `input` section to see the input source used by the pipeline. For example, in the configuration file `configs/from_manual.json`, the following snippet defines the input source as manual: +``` + "input": { + "_type": "manual", + "message": { + ...Contents of the input message... + } + } +``` + +To use a file as the input source, update the JSON object in the config file to: +``` + "input": { + "_type": "file", + "file": "data/input_messages/rapidsai:23.08-cuda11.8-py3.10.json" + } +``` + +To use an HTTP source as the input, update the JSON object in the config file to: +``` + "input": { + "_type": "http", + "address": "127.0.0.1", + "endpoint": "/scan", + "http_method": "POST", + "port": 26466 + } +``` + +### Customizing the embedding model +Vector databases are used by the agent to fetch relevant information for impact analysis investigations. The embedding model used to vectorize your documents can significantly affect the agent's performance. The default embedding model used by the pipeline is the NIM [nvidia/nv-embedqa-e5-v5](https://build.nvidia.com/nvidia/nv-embedqa-e5-v5) model, but you can experiment with different embedding models of your choice. + +To test a custom embedding model, modify the configuration file in the `engine.rag_embedding` section. For example, in the `from_manual.json` configuration file, the following snippet defines the settings for the default embedding model: +``` + "rag_embedding": { + "_type": "nim", + "model": "nvidia/nv-embedqa-e5-v5", + "truncate": "END", + "max_batch_size": 128 + }, +``` +* `rag_embedding._type`: specifies the embedding provider. The current [supported options](./src/cve/utils/embedding_loader.py) are `nim`, `huggingface` and `openai`. +* `model_name`: specifies the model name for the embedding provider. Refer to the embedding provider's documentation to determine the available models. +* `truncate`: specifies how inputs longer than the maximum token length of the model are handled. Passing `START` discards the start of the input. `END` discards the end of the input. In both cases, input is discarded until the remaining input is exactly the maximum input token length for the model. If `NONE` is selected, when the input exceeds the maximum input token length an error will be returned. +* `max_batch_size`: specifies the batch size to use when generating embeddings. We recommend setting this to 128 (default) or lower when using the cloud-hosted embedding NIM. When using a local NIM, this value can be tuned based on throughput/memory performance on your hardware. + +**Steps to configure an alternate embedding provider** +1. If using OpenAI embeddings, first obtain an API key, then update the `.env` file with the auth and base URL environment variables for the service as indicated in the [Supported LLM Services](#supported-llm-services) table. Otherwise, proceed to step 2. +2. Update the `rag_embedding` section of the config file as described above. + + - Example HuggingFace embedding configuration: + ``` + "rag_embedding": { + "_type": "huggingface", + "model_name": "intfloat/e5-large-v2", + "encode_kwargs": { + "batch_size": 128 + } + } + ``` + - Example OpenAI embedding configuration: + ``` + "rag_embedding": { + "_type": "openai", + "model_name": "text-embedding-3-small", + "encode_kwargs": { + "max_retries": 5, + "chunk_size": 256 + } + } + ``` + - For HuggingFace embeddings, all parameters from LangChain's [HuggingFaceEmbeddings](https://api.python.langchain.com/en/latest/embeddings/langchain_huggingface.embeddings.huggingface.HuggingFaceEmbeddings.html) class are supported. However, for OpenAI models, only a subset of parameters are supported. The full set of available parameters can be found in the config definitions [here](./src/cve/data_models/config.py?#L182-203). Any non-supported parameters provided in the configuration will be ignored. + +The current pipeline uses [FAISS](https://ai.meta.com/tools/faiss/) to create the vector databases. Interested users can customize the source code to use other vector databases such as [cuVS](https://rapids.ai/cuvs/). + +### Customizing the LLM models + +The configuration file also allows customizing the LLM model and parameters for each component of the workflow, as well which LLM service is used when invoking the model. + +In any configuration file, locate the `engine` section to see the current settings. For example, in the `from_manual.json` configuration file, the following snippet defines the LLM used for the checklist model: +``` + "checklist_model": { + "service": { + "_type": "nvfoundation" + }, + "model_name": "meta/llama3-70b-instruct", + "temperature": 0, + "max_tokens": 2000, + "top_p": 0.01 + } +``` +* `service._type`: specifies the LLM service to use. Refer to the [Supported LLM Services](#supported-llm-services) table for available options. +* `model_name`: specifies the model name within the LLM service. Refer to the service's API documentation to determine the available models. +* `temperature`, `max_tokens`, `top_p`, ...: specifies the model parameters. Note that by default, the config supports only a subset of parameters provided by each LLM service. The available parameters can be found in the configuration object's definition [here](./src/cve/data_models/config.py?#L122-127]). Any non-supported parameters provided in the configuration will be ignored. + +#### Supported LLM services +| Name | `_type` | Auth Env Var(s) | Base URL Env Var(s) | Proxy Server Route | +| ------------------------------------------------------------------------------------------------------------ | -------------- | ----------------------------- | ------------------------------------------------------------------------------- | ------------------ | +| [NVIDIA Inference Microservices (NIMs)](http://build.nvidia.com/) (Default) | `nvfoundation` | `NVIDIA_API_KEY` | `NVIDIA_API_BASE` | `/nim_llm/v1` | +| [NVIDIA GPU Cloud (NGC)](https://docs.nvidia.com/ngc/gpu-cloud/ngc-user-guide/index.html#generating-api-key) | `nemo` | `NGC_API_KEY`
`NGC_ORG_ID` | `NGC_API_BASE` | `/nemo/v1` | +| [OpenAI](https://platform.openai.com/docs/api-reference/authentication) | `openai` | `OPENAI_API_KEY` | `OPENAI_API_BASE` (used by `langchain`)
`OPENAI_BASE_URL` (used by `openai`) | `/openai/v1` | + + +**Steps to configure an LLM model** +1. Obtain an API key and any other required auth info for the selected service. +2. Update the `.env` file with the auth and base URL environment variables for the service as indicated in the [Supported LLM Services](#supported-llm-services) table. +3. Update the config file as described above. For example, if you want to use OpenAI's `gpt-4o` model for checklist generation, update the above json object in the config file to: +``` + "checklist_model": { + "service": { + "_type": "openai" + }, + "model_name": "gpt-4o", + "temperature": 0, + "top_p": 0.01, + "seed": 0, + "max_retries": 5 + }, +``` +Please note that the prompts have been tuned to work best with the Llama 3 70B NIM and that when using other LLM models it may be necessary to adjust the prompting. + +### Customizing the Output + +Currently, there are 2 types of outputs supported by the pipeline: +- **File output**: The output data is written to a file in JSON format. +- **Print output**: The output data is printed to the console. + +To customize the output, modify the configuration file accordingly. In any configuration file, locate the `output` section to see the output destination used by the pipeline. For example, in the configuration file `configs/from_manual.json`, the following snippet defines the output destination as a single json file and individual markdown files per CVE-ID: +``` + "output": { + "_type": "file", + "file_path": ".tmp/output.json", + "markdown_dir": ".tmp/vulnerability_markdown_reports" + } +``` + +To print the output without saving to a file, update the JSON object in the config file to: +``` + "output": { + "_type": "print" + } +``` + +Additional output options will be added in the future. + +### Integrating self-hosted LLM NIM + +The worklow is configured by default to use an NVIDIA-hosted LLM NIM for which an NVIDIA API key (`NVIDIA_API_KEY`) is required. The workflow can also be used with a self-hosted LLM NIM. You can start [here](https://build.nvidia.com/meta/llama3-70b?integrate_nim=true&self_hosted_api=true) for more information on how to pull and run the NIM locally. + +Once NIM is deployed, update your pipeline configuration (e.g. `from_manual.json`) to now use the your NIM. For every component under `engine` that uses `nvfoundation`, update to now use `openai` and model name of your NIM (i.e. meta/llama3-70b-instruct). For example: +``` +"agent": { + "model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "openai" + }, + "max_tokens": 2000 + }, + "verbose": true +} +``` + +If using nginx cache, update `openai_upstream` in [nginx_cache.conf](./nginx/nginx_cache.conf) to point to your NIM URL: +``` +set $openai_upstream http://llm-nim:8000; +``` + +Here `llm-nim` is the configured service name for the NIM if using helm chart of docker compose. Otherwise, set to actual host name or IP address of your NIM. + +Now set `OPENAI_BASE_URL` environment variable to NIM or nginx URL depending on your configuration. The `OPENAI_API_KEY` must also be set but only to prevent a check error (not for authentication). This can be set to any string. + + +## Troubleshooting + +Several common issues can arise when running the pipeline. Here are some common issues and their solutions. + +### Git LFS issues + +If you encounter issues with Git LFS, ensure that you have Git LFS installed and that it is enabled for the repository. You can check if Git LFS is enabled by running the following command: +```bash +git lfs install +``` + +Verifying that all files are being tracked by Git LFS can be done by running the following command: +```bash +git lfs ls-files +``` +Files which are missing will show a `-` next to their name. To ensure all LFS files have been pulled correctly, you can run the following command: +```bash +git lfs fetch --all +git lfs checkout * +``` + +### Container build issues + +When building containers for self-hosted NIMs, certain issues may occur. Below are common troubleshooting steps to help resolve them. + +#### Device error + +If you encounter an error resembling the following during the [container build process for self-hosted NIMs](#run-the-workflow-using-self-hosted-nims): + +```bash +nvidia-container-cli: device error: {n}: unknown device: unknown +``` + +This error typically indicates that the container is attempting to access GPUs that are either unavailable or non-existent on the host. To resolve this, verify the GPU count specified in the [docker-compose.nim.yml](./docker-compose.nim.yml) configuration file: +- Navigate to the `deploy.resources.reservations.devices` section and check the count parameter. +- Set the environment variable `NIM_LLM_GPU_COUNT` to the actual number of GPUs available on the host machine before building the container. Note that the default value is set to 4. + +This adjustment ensures the container accurately matches the available GPU resources, preventing access errors during deployment. + +#### Deploy.Resources.Reservations.devices error + +If you encounter an error resembling the following during the [container build process for self-hosted NIMs](#run-the-workflow-using-self-hosted-nims) process: +``` +1 error(s) decoding: + +* error decoding 'Deploy.Resources.Reservations.devices[0]': invalid string value for 'count' (the only value allowed is 'all') +``` + +This is likely caused by an [outdated Docker Compose version](https://github.com/docker/compose/issues/11097). Please upgrade Docker Compose to at least `v2.21.0`. + +### NGINX caching server + +Because the workflow makes such heavy use of the caching server to speed up API requests, it is important to ensure that the server is running correctly. If you encounter issues with the caching server, you can reset the cache. + +#### Resetting the entire cache + +To reset the entire cache, you can run the following command: +```bash +docker compose down -v +``` +This will delete all the volumes associated with the containers, including the cache. + +#### Resetting just the LLM cache or the services cache + +If you want to reset just the LLM cache or the services cache, you can run the following commands: +```bash +docker compose down + +# To remove the LLM cache +docker volume rm ${COMPOSE_PROJECT_NAME:-morpheus_vuln_analysis}-llm-cache + +# To remove the services cache +docker volume rm ${COMPOSE_PROJECT_NAME:-morpheus_vuln_analysis}-service-cache +``` + +#### Vector databases +We've integrated VDB and embedding creation directly into the pipeline with caching included for expediency. However, in a production environment, it's better to use a separately managed VDB service. + +NVIDIA offers optimized models and tools like NIMs ([build.nvidia.com/explore/retrieval](https://build.nvidia.com/explore/retrieval)) and cuVS ([github.com/rapidsai/cuvs](https://github.com/rapidsai/cuvs)). + +### Service outages + +#### National Vulnerability Database (NVD) +These typically resolve on their own. Please wait and try running the pipeline again later. Example errors: + +404 +``` +Error requesting [1/10]: (Retry 0.1 sec) https://services.nvd.nist.gov/rest/json/cves/2.0: 404, message='', url=URL('https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=CVE-2023-6709') +``` + +503 +``` +Error requesting [1/10]: (Retry 0.1 sec) https://services.nvd.nist.gov/rest/json/cves/2.0: 503, message='Service Unavailable', url=URL('https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=CVE-2023-50447') +``` + +### Running out of credits + +If you run out of credits for the NVIDIA API Catalog, you will need to obtain more credits to continue using the API. Please contact your NVIDIA representative to get more credits added. + +## Testing and validation +Test-driven development is essential for building reliable LLM-based agentic systems, especially when deploying or scaling them in production environments. + +In our development process, we use the Morpheus public container as a case study. We perform security scans and collaborate with developers and security analysts to assess the exploitability of identified CVEs. Each CVE is labeled as either vulnerable or not vulnerable. For non-vulnerable CVEs, we provide a justification based on one of the ten VEX statuses. Team members document their investigative steps and findings to validate and compare results at different stages of the system. + +We have collected labels for 38 CVEs, which serve several purposes: +- Human-generated checklists, findings, and summaries are used as ground truth during various stages of prompt engineering to refine LLM output. +- The justification status for each CVE is used as a label to measure end-to-end pipeline accuracy. Every time there is a change to the system, such as adding a new agent tool, modifying a prompt, or introducing an engineering optimization, we run the labeled dataset through the updated pipeline to detect performance regressions. + +As a next step, we plan to integrate this process into our CI/CD pipeline to automate testing. While LLMs' non-deterministic nature makes it difficult to assert exact results for each test case, we can adopt a statistical approach, where we run the pipeline multiple times and ensure that the average accuracy stays within an acceptable range. + +We recommend that teams looking to test or optimize their CVE analysis system curate a similar dataset for testing and validation. Note that in test-driven development, it's important that the model has not achieved perfect accuracy on the test set, as this may indicate overfitting or that the set lacks sufficient complexity to expose areas for improvement. The test set should be representative of the problem space, covering both scenarios where the model performs well and where further refinement is needed. Investing in a robust dataset ensures long-term reliability and drives continued performance improvements. + +## License + +By using this software or microservice, you are agreeing to the [terms and conditions](https://www.nvidia.com/en-us/data-center/products/nvidia-ai-enterprise/eula/) of the license and acceptable use policy. + + +## Terms of Use + +GOVERNING TERMS: The NIM container is governed by the [NVIDIA Software License Agreement](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-software-license-agreement/) and [Product-Specific Terms for AI Products](https://www.nvidia.com/en-us/agreements/enterprise-software/product-specific-terms-for-ai-products/); and use of this model is governed by the [NVIDIA AI Foundation Models Community License Agreement](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-ai-foundation-models-community-license-agreement/). + +ADDITIONAL Terms: [Meta Llama 3 Community License](https://www.llama.com/llama3/license/), Built with Meta Llama 3. diff --git a/ci/scripts/copyright.py b/ci/scripts/copyright.py new file mode 100755 index 0000000..95c31ba --- /dev/null +++ b/ci/scripts/copyright.py @@ -0,0 +1,555 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import datetime +import io +import logging +import os +import re +import sys +import typing + +# Now import gitutils. Ignore flake8 error here since there is no other way to +# set up imports +import gitutils # noqa: E402 + +# pylint: disable=global-statement + +FilesToCheck = [ + # Get all of these extensions and templates (*.in) + re.compile(r"[.](cmake|cpp|cc|cu|cuh|h|hpp|md|rst|sh|pxd|py|pyx|yml|yaml)(\.in)?$"), + # And files with a particular file/extension combo + re.compile(r"CMakeLists[.]txt$"), + re.compile(r"setup[.]cfg$"), + re.compile(r"[.]flake8[.]cython$"), + re.compile(r"meta[.]yaml$"), + re.compile(r"[^ \/\n]*Dockerfile[^ \/\n]*") +] + +# Nothing in a build folder or .cache +ExemptFiles: typing.List[re.Pattern] = [ + re.compile(r"(_version|versioneer)\.py"), # Skip versioning files + re.compile(r"^[^ \/\n]*\.cache[^ \/\n]*\/.*$"), # Ignore .cache folder + re.compile(r"^[^ \/\n]*build[^ \/\n]*\/.*$"), # Ignore any build*/ folder + re.compile(r"^external\/.*$"), # Ignore external + re.compile(r"[^ \/\n]*docs/source/(_lib|_modules|_templates)/.*$"), + re.compile(r"PULL_REQUEST_TEMPLATE.md"), # Ignore the PR template, + re.compile(r"[^ \/\n]*conda/environments/.*\.yaml$"), # Ignore generated environment files +] + +# this will break starting at year 10000, which is probably OK :) +CheckSimple = re.compile(r"Copyright *(?:\(c\))? *(\d{4}),? *NVIDIA C(?:ORPORATION|orporation)") +CheckDouble = re.compile(r"Copyright *(?:\(c\))? *(\d{4})-(\d{4}),? *NVIDIA C(?:ORPORATION|orporation)") +CHECK_APACHE_LIC = 'Licensed under the Apache License, Version 2.0 (the "License");' + + +def is_file_empty(f): + return os.stat(f).st_size == 0 + + +def check_this_file(f): + # This check covers things like symlinks which point to files that DNE + if not (os.path.exists(f)): + return False + if is_file_empty(f): + return False + for exempt in ExemptFiles: + if exempt.search(f): + return False + for checker in FilesToCheck: + if checker.search(f): + return True + return False + + +def get_copyright_years(line): + res = CheckSimple.search(line) + if res: + return (int(res.group(1)), int(res.group(1))) + res = CheckDouble.search(line) + if res: + return (int(res.group(1)), int(res.group(2))) + return (None, None) + + +def replace_current_year(line, start, end): + # first turn a simple regex into double (if applicable). then update years + res = CheckSimple.sub(r"Copyright (c) \1-\1, NVIDIA CORPORATION", line) + + # pylint: disable=consider-using-f-string + res = CheckDouble.sub(r"Copyright (c) {:04d}-{:04d}, NVIDIA CORPORATION".format(start, end), res) + return res + + +def insert_license(f, this_year, first_line): + ext = os.path.splitext(f)[1].lstrip('.') + + try: + license_text = EXT_LIC_MAPPING[ext][0].format(YEAR=this_year) + except KeyError: + return [ + f, + 0, + f"Unsupported extension {ext} for automatic insertion, " + "please manually insert an Apache v2.0 header or add the file to " + "excempted from this check add it to the 'ExemptFiles' list in " + "the 'ci/scripts/copyright.py' file (manual fix required)", + None + ] + + # If the file starts with a #! keep it as the first line + if first_line.startswith("#!"): + replace_line = first_line + license_text + else: + replace_line = f"{license_text}\n{first_line}" + + return [f, 1, "License inserted", replace_line] + + +def remove_proprietary(f): + with io.open(f, "r", encoding="utf-8") as file: + content = file.read() + + ext = os.path.splitext(f)[1].lstrip('.') + + try: + prop_re = EXT_LIC_MAPPING[ext][1] + except KeyError: + return [f, 0, f"Unsupported extension {ext} for proprietary license removal", None] + + match = prop_re.search(content) + if match is not None: + content = content.replace(match.group(0), "") + + with io.open(f, "w", encoding="utf-8") as file: + file.write(content) + + return [f, 1, "Proprietary license replaced", ""] + + +def check_copyright(f, + update_current_year, + verify_apache_v2=False, + update_start_year=False, + do_insert_license=False, + git_add=False): + """ + Checks for copyright headers and their years + """ + errs = [] + this_year = datetime.datetime.now().year + line_num = 0 + cr_found = False + apache_lic_found = not verify_apache_v2 + year_matched = False + with io.open(f, "r", encoding="utf-8") as file: + lines = file.readlines() + for line in lines: + line_num += 1 + if not apache_lic_found: + apache_lic_found = CHECK_APACHE_LIC in line + + start, end = get_copyright_years(line) + if start is None: + continue + + cr_found = True + if update_start_year: + try: + git_start = gitutils.get_file_add_date(f).year + if start > git_start: + e = [ + f, + line_num, + "Current year not included in the " + "copyright header", + replace_current_year(line, git_start, this_year) + ] + errs.append(e) + continue + + except Exception as excp: + e = [f, line_num, f"Error determining start year from git: {excp}", None] + errs.append(e) + continue + + if start > end: + e = [f, line_num, "First year after second year in the copyright header (manual fix required)", None] + errs.append(e) + if this_year < start or this_year > end: + e = [f, line_num, "Current year not included in the copyright header", None] + if this_year < start: + e[-1] = replace_current_year(line, this_year, end) + if this_year > end: + e[-1] = replace_current_year(line, start, this_year) + errs.append(e) + else: + year_matched = True + file.close() + + if not apache_lic_found: + if len(lines) > 0: + if do_insert_license and len(lines): + e = insert_license(f, this_year, lines[0]) + cr_found = True + year_matched = True + else: + e = [ + f, + 0, + "Apache copyright header missing, if this file needs to be " + "excempted from this check add it to the 'ExemptFiles' list in " + "the 'ci/scripts/copyright.py' file.", + True + ] + errs.append(e) + + # copyright header itself not found + if not cr_found: + e = [f, 0, "Copyright header missing or formatted incorrectly (manual fix required)", None] + errs.append(e) + + # even if the year matches a copyright header, make the check pass + if year_matched and apache_lic_found: + errs = [] + + if update_current_year or update_start_year or do_insert_license: + errs_update = [x for x in errs if x[-1] is not None] + if len(errs_update) > 0: + logging.info("File: %s. Changing line(s) %s", f, ', '.join(str(x[1]) for x in errs if x[-1] is not None)) + for _, line_num, __, replacement in errs_update: + if line_num > 0: + line_num -= 1 + lines[line_num] = replacement + with io.open(f, "w", encoding="utf-8") as out_file: + for new_line in lines: + out_file.write(new_line) + + if git_add: + gitutils.add_files(f) + + errs = [x for x in errs if x[-1] is None] + + return errs + + +def _main(): + """ + Checks for copyright headers in all the modified files. In case of local + repo, this script will just look for uncommitted files and in case of CI + it compares between branches "$PR_TARGET_BRANCH" and "current-pr-branch" + """ + log_level = logging.getLevelName(os.environ.get("MORPHEUS_LOG_LEVEL", "INFO")) + logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) + + ret_val = 0 + global ExemptFiles + + argparser = argparse.ArgumentParser("Checks for a consistent copyright header in git's modified files") + argparser.add_argument("--update-start-year", + dest='update_start_year', + action="store_true", + required=False, + help="If set, " + "update the start year based on a start date parsed " + "on the earliest entry from `git log --follow` will " + "only set the year if it is less than the current " + "copyright year") + argparser.add_argument("--update-current-year", + dest='update_current_year', + action="store_true", + required=False, + help="If set, " + "update the current year if a header is already " + "present and well formatted.") + + argparser.add_argument("--insert", + dest='insert', + action="store_true", + required=False, + help="If set, " + "inserts an Apache v2.0 license into a files " + "without a license, implies --verify-apache-v2") + argparser.add_argument("--remove-proprietary", + dest='remove_proprietary', + action="store_true", + required=False, + help="If set, removes the proprietary license from the file") + argparser.add_argument( + "--fix-all", + dest='fix_all', + action="store_true", + required=False, + help="Shortcut for setting --update-start-year --update-current-year --insert and --remove-proprietary") + + git_group = argparser.add_mutually_exclusive_group() + git_group.add_argument("--git-modified-only", + dest='git_modified_only', + action="store_true", + required=False, + help="If set, " + "only files seen as modified by git will be " + "processed. Cannot be combined with --git-diff-commits or --git-diff-staged") + git_group.add_argument("--git-diff-commits", + dest='git_diff_commits', + required=False, + nargs=2, + metavar='hash', + help="If set, " + "only files modified between the two given commit hashes. " + "Cannot be combined with --git-modified-only or --git-diff-staged") + git_group.add_argument("--git-diff-staged", + dest='git_diff_staged', + required=False, + nargs="?", + metavar='HEAD', + default=None, + const='HEAD', + help="If set, " + "only files staged for commit. " + "Cannot be combined with --git-modified-only or --git-diff-commits") + + argparser.add_argument("--git-add", + dest='git_add', + action="store_true", + required=False, + help="If set, " + "any files auto-fixed will have `git add` run on them. ") + + argparser.add_argument("--verify-apache-v2", + dest='verify_apache_v2', + action="store_true", + required=False, + help="If set, " + "verifies all files contain the Apache license " + "in their header") + argparser.add_argument("--exclude", + dest='exclude', + action="append", + required=False, + default=["_version\\.py"], + help=("Exclude the paths specified (regexp). " + "Can be specified multiple times.")) + + (args, dirs) = argparser.parse_known_args() + try: + ExemptFiles = ExemptFiles + [re.compile(pathName) for pathName in args.exclude] + except re.error as re_exception: + logging.exception("Regular expression error: %s", re_exception, exc_info=True) + return 1 + + if args.git_modified_only: + files = gitutils.modified_files() + elif args.git_diff_commits: + files = gitutils.changed_files(*args.git_diff_commits) + elif args.git_diff_staged: + files = gitutils.staged_files(args.git_diff_staged) + else: + files = gitutils.all_files(*dirs) + + logging.debug("File count before filter(): %s", len(files)) + + # Now filter the files down based on the exclude/include + files = gitutils.filter_files(files, path_filter=check_this_file) + + logging.info("Checking files (%s):\n %s", len(files), "\n ".join(files)) + + errors = [] + for f in files: + if args.remove_proprietary or args.fix_all: + remove_proprietary(f) + + errors += check_copyright(f, + args.update_current_year, + verify_apache_v2=(args.verify_apache_v2 or args.insert or args.fix_all), + update_start_year=(args.update_start_year or args.fix_all), + do_insert_license=(args.insert or args.fix_all), + git_add=args.git_add) + + if len(errors) > 0: + logging.info("Copyright headers incomplete in some of the files!") + for e in errors: + logging.error(" %s:%d Issue: %s", e[0], e[1], e[2]) + logging.info("") + n_fixable = sum(1 for e in errors if e[-1] is not None) + path_parts = os.path.abspath(__file__).split(os.sep) + file_from_repo = os.sep.join(path_parts[path_parts.index("ci"):]) + if n_fixable > 0: + logging.info(("You can run `python %s --git-modified-only " + "--update-current-year --insert` to fix %s of these " + "errors.\n"), + file_from_repo, + n_fixable) + ret_val = 1 + else: + logging.info("Copyright check passed") + + return ret_val + + +A2_LIC_HASH = """# SPDX-FileCopyrightText: Copyright (c) {YEAR}, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +A2_LIC_C = """/* + * SPDX-FileCopyrightText: Copyright (c) {YEAR}, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +""" + +A2_LIC_MD = """ +""" + +A2_LIC_RST = """.. + SPDX-FileCopyrightText: Copyright (c) {YEAR}, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +""" + +NV_PROP_LIC_HASH_RE = re.compile( + r"""# SPDX-FileCopyrightText: Copyright \(c\) ((\d{4})|(\d{4})-(\d{4})),? NVIDIA CORPORATION & AFFILIATES\. All rights reserved\. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto\. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited\. +""", + re.MULTILINE) + +NV_PROP_LIC_C_RE = re.compile( + r"""/\* + \* SPDX-FileCopyrightText: Copyright \(c\) ((\d{4})|(\d{4})-(\d{4})),? NVIDIA CORPORATION & AFFILIATES\. All rights reserved\. + \* SPDX-License-Identifier: LicenseRef-NvidiaProprietary + \* + \* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + \* property and proprietary rights in and to this material, related + \* documentation and any modifications thereto\. Any use, reproduction, + \* disclosure or distribution of this material and related documentation + \* without an express license agreement from NVIDIA CORPORATION or + \* its affiliates is strictly prohibited\. + \*/ +""", + re.MULTILINE) + +NV_PROP_LIC_MD_RE = re.compile( + r""" +""", + re.MULTILINE) + +NV_PROP_LIC_RST_RE = re.compile( + r"""\.\. + SPDX-FileCopyrightText: Copyright \(c\) ((\d{4})|(\d{4})-(\d{4})),? NVIDIA CORPORATION & AFFILIATES\. All rights reserved\. + SPDX-License-Identifier: LicenseRef-NvidiaProprietary + + NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + property and proprietary rights in and to this material, related + documentation and any modifications thereto\. Any use, reproduction, + disclosure or distribution of this material and related documentation + without an express license agreement from NVIDIA CORPORATION or + its affiliates is strictly prohibited\. + +""", + re.MULTILINE) + +# FilesToCheck list will allow us to assume Cmake for the txt extension +EXT_LIC_MAPPING = { + 'c': (A2_LIC_C, NV_PROP_LIC_C_RE), + 'cc': (A2_LIC_C, NV_PROP_LIC_C_RE), + 'cmake': (A2_LIC_HASH, NV_PROP_LIC_HASH_RE), + 'cpp': (A2_LIC_C, NV_PROP_LIC_C_RE), + 'cu': (A2_LIC_C, NV_PROP_LIC_C_RE), + 'cuh': (A2_LIC_C, NV_PROP_LIC_C_RE), + 'h': (A2_LIC_C, NV_PROP_LIC_C_RE), + 'hpp': (A2_LIC_C, NV_PROP_LIC_C_RE), + 'md': (A2_LIC_MD, NV_PROP_LIC_MD_RE), + 'pxd': (A2_LIC_HASH, NV_PROP_LIC_HASH_RE), + 'py': (A2_LIC_HASH, NV_PROP_LIC_HASH_RE), + 'pyx': (A2_LIC_HASH, NV_PROP_LIC_HASH_RE), + 'rst': (A2_LIC_RST, NV_PROP_LIC_RST_RE), + 'sh': (A2_LIC_HASH, NV_PROP_LIC_HASH_RE), + 'txt': (A2_LIC_HASH, NV_PROP_LIC_HASH_RE), + 'yaml': (A2_LIC_HASH, NV_PROP_LIC_HASH_RE), + 'yml': (A2_LIC_HASH, NV_PROP_LIC_HASH_RE), +} + +if __name__ == "__main__": + sys.exit(_main()) diff --git a/ci/scripts/gitutils.py b/ci/scripts/gitutils.py new file mode 100755 index 0000000..b9cc585 --- /dev/null +++ b/ci/scripts/gitutils.py @@ -0,0 +1,645 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import datetime +import functools +import json +import logging +import os +import re +import subprocess +import typing + + +def _run_cmd(exe: str, *args: str): + """Runs a command with args and returns its output""" + + cmd_list = (exe, ) + args + + # Join the args to make the command string (for logging only) + cmd_str = " ".join(cmd_list) + + # If we only passed in one executable (could be piping commands together) then use a shell + shell = len(args) <= 0 + + if (shell): + # For logging purposes, if we only passed in one executable, then clear the exe name to make logging better + exe = "" + + try: + ret = subprocess.check_output(cmd_list, stderr=subprocess.PIPE, shell=shell) + output = ret.decode("UTF-8").rstrip("\n") + + logging.debug("Running %s command: `%s`, Output: '%s'", exe, cmd_str, output) + + return output + except subprocess.CalledProcessError as e: + logging.warning("Running %s command [ERRORED]: `%s`, Output: '%s'", + exe, + cmd_str, + e.stderr.decode("UTF-8").rstrip("\n")) + raise + + +def _gh(*args): + """Runs a Github CLI command and returns its output""" + + return _run_cmd("gh", *args) + + +def _git(*args): + """Runs a git command and returns its output""" + return _run_cmd("git", *args) + + +class GitWrapper: + + @functools.lru_cache + @staticmethod + def get_closest_tag(): + """ + Determines the version of the repo by using `git describe` + + Returns + ------- + str + The full version of the repo in the format 'v#.#.#{a|b|rc}' + """ + return _git("describe", "--tags", "--abbrev=0") + + @functools.lru_cache + @staticmethod + def get_repo_version(): + """ + Determines the version of the repo using `git describe` and returns only + the major and minor portion + + Returns + ------- + str + The partial version of the repo in the format '{major}.{minor}' + """ + + full_repo_version = GitWrapper.get_closest_tag() + + match = re.match(r"^v?(?P[0-9]+)(?:\.(?P[0-9]+))?", full_repo_version) + + if (match is None): + logging.debug("Could not determine repo major minor version. Full repo version: %s.", full_repo_version) + return None + + out_version = match.group("major") + + if (match.group("minor")): + out_version += "." + match.group("minor") + + return out_version + + @functools.lru_cache + @staticmethod + def get_repo_owner_name(): + + # pylint: disable=anomalous-backslash-in-string + return "nv-morpheus/" + _run_cmd("git remote -v | grep -oP '/\K\w*(?=\.git \(fetch\))' | head -1") # noqa: W605 + + @functools.lru_cache + @staticmethod + def get_repo_remote_name(repo_owner_and_name: str): + + return _run_cmd(f"git remote -v | grep :{repo_owner_and_name} | grep \"(fetch)\" | head -1 | cut -f1") + + @functools.lru_cache + @staticmethod + def is_ref_valid(git_ref: str): + + try: + return _git("rev-parse", "--verify", git_ref) != "" + except subprocess.CalledProcessError: + return False + + @functools.lru_cache + @staticmethod + def get_remote_branch(local_branch_ref: str, *, repo_owner_and_name: str = None): + + if (repo_owner_and_name is None): + repo_owner_and_name = GitWrapper.get_repo_owner_name() + + remote_name = GitWrapper.get_repo_remote_name(repo_owner_and_name) + + remote_branch_ref = f"{remote_name}/{local_branch_ref}" + + if (GitWrapper.is_ref_valid(remote_branch_ref)): + return remote_branch_ref + + logging.info("Remote branch '%s' for repo '%s' does not exist. Falling back to rev-parse", + remote_branch_ref, + repo_owner_and_name) + + remote_branch_ref = _git("rev-parse", "--abbrev-ref", "--symbolic-full-name", local_branch_ref + "@{upstream}") + + return remote_branch_ref + + @functools.lru_cache + @staticmethod + def get_target_remote_branch(): + try: + # Try and guess the target branch as "branch-." + version = GitWrapper.get_repo_version() + + if (version is None): + return None + + base_ref = f"branch-{version}" + + # If our current branch and the base ref are the same, then use main + if (base_ref == GitWrapper.get_current_branch()): + logging.warning("Current branch is the same as the tagged branch: %s. Falling back to 'main'", base_ref) + base_ref = "main" + + except Exception: + logging.debug("Could not determine branch version falling back to main") + base_ref = "main" + + return GitWrapper.get_remote_branch(base_ref) + + @functools.lru_cache + @staticmethod + def get_repo_dir(): + """ + Returns the top level directory for this git repo + """ + return _git("rev-parse", "--show-toplevel") + + @functools.lru_cache + @staticmethod + def get_current_branch(): + """Returns the name of the current branch""" + name = _git("rev-parse", "--abbrev-ref", "HEAD") + name = name.rstrip() + return name + + @staticmethod + def add_files(*files_to_add): + """Runs git add on file""" + return _git("add", *files_to_add) + + @functools.lru_cache + @staticmethod + def get_file_add_date(file_path): + """Return the date a given file was added to git""" + date_str = _git("log", "--follow", "--format=%as", "--", file_path, "|", "tail", "-n 1") + return datetime.datetime.strptime(date_str, "%Y-%m-%d") + + @staticmethod + def get_uncommitted_files(): + """ + Returns a list of all changed files that are not yet committed. This + means both untracked/unstaged as well as uncommitted files too. + """ + files = _git("status", "-u", "-s") + ret = [] + for f in files.splitlines(): + f = f.strip(" ") + f = re.sub(r"\s+", " ", f) # noqa: W605 + tmp = f.split(" ", 1) + # only consider staged files or uncommitted files + # in other words, ignore untracked files + if tmp[0] == "M" or tmp[0] == "A": + ret.append(tmp[1]) + return ret + + @staticmethod + def diff(target_ref: str, base_ref: str, merge_base: bool = False, staged: bool = False): + + assert base_ref is not None or base_ref != "", "base_ref must be a valid ref" + assert target_ref is not None or target_ref != "", "target_ref must be a valid ref" + + args = ["--no-pager", "diff", "--name-only", "--ignore-submodules"] + + if (merge_base): + args.append("--merge-base") + + if (staged): + args.append("--cached") + + args += [target_ref, base_ref] + + return _git(*args).splitlines() + + @staticmethod + def diff_index(target_ref: str, merge_base: bool = False, staged: bool = False): + + assert target_ref is not None or target_ref != "", "target_ref must be a valid ref" + + args = ["--no-pager", "diff-index", "--name-only", "--ignore-submodules"] + + if (merge_base): + args.append("--merge-base") + + if (staged): + args.append("--cached") + + args += [target_ref] + + return _git(*args).splitlines() + + @staticmethod + def merge_base(target_ref: str, base_ref: str = "HEAD"): + + assert base_ref is not None or base_ref != "", "base_ref must be a valid ref" + assert target_ref is not None or target_ref != "", "target_ref must be a valid ref" + + return _git("merge-base", target_ref, base_ref) + + +class GithubWrapper: + + @functools.lru_cache + @staticmethod + def has_cli(): + try: + _gh("--version") + + # Run a test function + repo_name = _gh("repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner") + + logging.debug("Github CLI is installed. Using repo: %s", repo_name) + return True + except (FileNotFoundError, subprocess.CalledProcessError): + logging.debug("Github CLI is not installed") + return False + + @functools.lru_cache + @staticmethod + def get_repo_owner_name(): + + # Make sure we have the CLI + if (not GithubWrapper.has_cli()): + return None + + return _gh("repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner") + + @functools.lru_cache + @staticmethod + def get_pr_info() -> dict | None: + + # Make sure we have the CLI + if (not GithubWrapper.has_cli()): + return None + + # List of fields to get from the PR + fields = [ + "baseRefName", + "number", + ] + + json_output = _gh("pr", "status", "--json", ",".join(fields), "--jq", ".currentBranch") + + if (json_output == ""): + return None + + return json.loads(json_output) + + @functools.lru_cache + @staticmethod + def is_pr(): + + return GithubWrapper.get_pr_info() is not None + + @functools.lru_cache + @staticmethod + def get_pr_number(): + + pr_info = GithubWrapper.get_pr_info() + + if (pr_info is None): + return None + + return pr_info["number"] + + @functools.lru_cache + @staticmethod + def get_pr_base_ref_name(): + + pr_info = GithubWrapper.get_pr_info() + + if (pr_info is None): + return None + + return pr_info["baseRefName"] + + @functools.lru_cache + @staticmethod + def get_pr_target_remote_branch(): + + # Make sure we are in a PR + if (not GithubWrapper.is_pr()): + return None + + # Get the PR base reference + base_ref = GithubWrapper.get_pr_base_ref_name() + + # Now determine the remote ref name matching our repository + remote_name = GitWrapper.get_remote_branch(base_ref, repo_owner_and_name=GithubWrapper.get_repo_owner_name()) + + return remote_name + + +def _is_repo_relative(f: str, git_root: str = None): + if (git_root is None): + git_root = GitWrapper.get_repo_dir() + + abs_f = os.path.abspath(f) + + rel_path = os.path.relpath(abs_f, git_root) + + return not rel_path.startswith("../") + + +def get_merge_target(): + """ + Returns the merge target branch for the current branch as if it were a PR/MR + + Order of operations: + 1. Try to determine the target branch from GitLab CI (assuming were in a PR) + 2. Try to guess the target branch as "branch-." using the most recent tag (assuming we have a remote + pointing to the base repo) + 3. Try to determine the target branch by finding a head reference that matches "branch-*" and is in this history + 4. Fall back to "main" if all else fails or the target branch and current branch are the same + + Returns + ------- + str + Ref name of the target branch + """ + # + + remote_branch = GithubWrapper.get_pr_target_remote_branch() + + if (remote_branch is None): + # Try to use tags + remote_branch = GitWrapper.get_target_remote_branch() + + if (remote_branch is None): + + raise RuntimeError("Could not determine remote_branch. Manually set TARGET_BRANCH to continue") + + return remote_branch + + +def determine_merge_commit(current_branch="HEAD"): + """ + When running outside of CI, this will estimate the target merge commit hash of `current_branch` by finding a common + ancester with the remote branch 'branch-{major}.{minor}' where {major} and {minor} are determined from the repo + version. + + Parameters + ---------- + current_branch : str, optional + Which branch to consider as the current branch, by default "HEAD" + + Returns + ------- + str + The common commit hash ID + """ + + remote_branch = get_merge_target() + + common_commit = GitWrapper.merge_base(remote_branch, current_branch) + + logging.info("Determined TARGET_BRANCH as: '%s'. With merge-commit: %s", remote_branch, common_commit) + + return common_commit + + +def filter_files(files: typing.Union[str, typing.List[str]], + path_filter: typing.Callable[[str], bool] = None) -> list[str]: + """ + Filters out the input files according to a predicate + + Parameters + ---------- + files : typing.Union[str, typing.List[str]] + List of files to filter + path_filter : typing.Callable[[str], bool], optional + Predicate that returns True/False for each file, by default None + + Returns + ------- + list[str] + Filtered list of files + """ + + # Convert all to array of strings + if (isinstance(files, str)): + files = files.splitlines() + + git_root = GitWrapper.get_repo_dir() + + ret_files: list[str] = [] + + for file in files: + # Check that we are relative to the git repo + assert _is_repo_relative(file, git_root=git_root), f"Path {file} must be relative to git root: {git_root}" + + if (path_filter is None or path_filter(file)): + ret_files.append(file) + + return ret_files + + +def changed_files(target_ref: str = None, + base_ref="HEAD", + *, + merge_base: bool = True, + staged=False, + path_filter: typing.Callable[[str], bool] = None): + """ + Comparison between 2 commits in the repo. Returns a list of files that have been filtered by `path_filter` + + Parameters + ---------- + target_ref : str, optional + The branch name to use as the target. If set to None, it will use the value in $TARGET_BRANCH + base_ref : str, optional + The base branch name, by default "HEAD" + merge_base : bool, optional + Setting this to True will calculate the diff to the merge-base between `taget_ref` and `base_ref`. Setting to + False will compre the HEAD of each ref + staged : bool, optional + Whether or not to include staged, but not committed, files, by default False + path_filter : typing.Callable[[str], bool], optional + A predicate to apply to the list of files, by default None + + Returns + ------- + list[str] + The list of files that have changed between the refs filtered by `path_filter` + """ + + if (target_ref is None): + target_ref = os.environ.get("TARGET_BRANCH", None) + + if (target_ref is None): + target_ref = get_merge_target() + + logging.info("Comparing %s..%s with merge_base: %s, staged: %s", target_ref, base_ref, merge_base, staged) + + diffs = GitWrapper.diff(target_ref, base_ref, merge_base=merge_base, staged=staged) + + return filter_files(diffs, path_filter=path_filter) + + +def modified_files(target_ref: str = None, + *, + merge_base: bool = True, + staged=False, + path_filter: typing.Callable[[str], bool] = None): + """ + Comparison between the working tree and a target branch. Returns a list of files that have been filtered by + `path_filter` + + Parameters + ---------- + target_ref : str, optional + The branch name to use as the target. If set to None, it will use the value in $TARGET_BRANCH + merge_base : bool, optional + Setting this to True will calculate the diff to the merge-base between `taget_ref` and `base_ref`. Setting to + False will compre the HEAD of each ref + staged : bool, optional + Whether or not to include staged, but not committed, files, by default False + path_filter : typing.Callable[[str], bool], optional + A predicate to apply to the list of files, by default None + + Returns + ------- + list[str] + The list of files that have changed between the refs filtered by `path_filter` + """ + + if (target_ref is None): + target_ref = os.environ.get("TARGET_BRANCH", None) + + if (target_ref is None): + target_ref = get_merge_target() + + logging.info("Comparing index to %s with merge_base: %s, staged: %s", target_ref, merge_base, staged) + + diffs = GitWrapper.diff_index(target_ref, merge_base=merge_base, staged=staged) + + return filter_files(diffs, path_filter=path_filter) + + +def staged_files(base_ref="HEAD", *, path_filter: typing.Callable[[str], bool] = None): + """ + Calculates the different between the working tree and the index including staged files. Returns a list of files that + have been filtered by `path_filter`. + + Identical to `modified_files` with `staged=True` + + Parameters + ---------- + base_ref : str, optional + The base branch name, by default "HEAD" + path_filter : typing.Callable[[str], bool], optional + A predicate to apply to the list of files, by default None + + Returns + ------- + list[str] + The list of files that have changed between the refs filtered by `path_filter` + """ + + return modified_files(target_ref=base_ref, merge_base=False, staged=True, path_filter=path_filter) + + +def all_files(*paths, base_ref="HEAD", path_filter: typing.Callable[[str], bool] = None): + """ + Returns a list of all files in the repo that have been filtered by `path_filter`. + + Parameters + ---------- + paths : typing.List[str] + The list of paths to include in the search + base_ref : str, optional + The base branch name, by default "HEAD" + path_filter : typing.Callable[[str], bool], optional + A predicate to apply to the list of files, by default None + + Returns + ------- + list[str] + The list of files in the repo filtered by `path_filter` + """ + + git_args = ["ls-tree", "-r", "--name-only", base_ref] + list(paths) + + ls_files = _git(*git_args) + + return filter_files(ls_files, path_filter=path_filter) + + +def add_files(*files_to_add): + """ + Calls `git add` on the input files + + Returns + ------- + str + Output of the git command + """ + return GitWrapper.add_files(*files_to_add) + + +def get_file_add_date(filename: str): + """ + Returns the date a given file was added to git. + + Parameters + ---------- + filename : str + Filename in question + + Returns + ------- + datetime.datetime + Time the file was added. + """ + + return GitWrapper.get_file_add_date(filename) + + +def _parse_args(): + argparser = argparse.ArgumentParser("Executes a gitutil action") + argparser.add_argument("action", choices=['get_merge_target'], help="Action to execute") + args = argparser.parse_args() + return args + + +def _main(): + log_level = logging.getLevelName(os.environ.get("MORPHEUS_LOG_LEVEL", "INFO")) + logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) + + args = _parse_args() + + if args.action == 'get_merge_target': + print(determine_merge_commit()) + + +if __name__ == '__main__': + _main() diff --git a/configs/from_file.json b/configs/from_file.json new file mode 100644 index 0000000..21fc9f9 --- /dev/null +++ b/configs/from_file.json @@ -0,0 +1,73 @@ +{ + "$schema": "./schemas/config.schema.json", + "engine": { + "agent": { + "model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 2000, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "verbose": false + }, + "checklist_model": { + "service": { + "_type": "nvfoundation" + }, + "model_name": "meta/llama3-70b-instruct", + "max_tokens": 2000, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "justification_model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "rag_embedding": { + "_type": "nim", + "model": "nvidia/nv-embedqa-e5-v5", + "truncate": "END", + "max_batch_size": 128 + }, + "summary_model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + } + }, + "general": { + "cache_dir": null, + "base_vdb_dir": ".cache/am_cache/vdb", + "base_git_dir": ".cache/am_cache/git", + "max_retries": 5, + "model_max_batch_size": 64, + "pipeline_batch_size": 1024, + "use_uvloop": true + }, + "input": { + "_type": "file", + "file": "data/input_messages/rapidsai:23.08-cuda11.8-py3.10.json" + }, + "output": { + "_type": "file", + "file_path": ".tmp/output.json", + "markdown_dir": ".tmp/vulnerability_markdown_reports", + "overwrite": true + } +} diff --git a/configs/from_http.json b/configs/from_http.json new file mode 100644 index 0000000..ed004d4 --- /dev/null +++ b/configs/from_http.json @@ -0,0 +1,74 @@ +{ + "$schema": "./schemas/config.schema.json", + "engine": { + "agent": { + "model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 2000, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "verbose": false + }, + "checklist_model": { + "service": { + "_type": "nvfoundation" + }, + "model_name": "meta/llama3-70b-instruct", + "max_tokens": 2000, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "justification_model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "rag_embedding": { + "_type": "nim", + "model": "nvidia/nv-embedqa-e5-v5", + "truncate": "END", + "max_batch_size": 128 + }, + "summary_model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + } + }, + "general": { + "cache_dir": null, + "base_vdb_dir": ".cache/am_cache/vdb", + "base_git_dir": ".cache/am_cache/git", + "max_retries": 5, + "model_max_batch_size": 64, + "pipeline_batch_size": 1024, + "use_uvloop": true + }, + "input": { + "_type": "http", + "port": 26466, + "address": "0.0.0.0" + }, + "output": { + "_type": "file", + "file_path": ".tmp/output.json", + "markdown_dir": ".tmp/vulnerability_markdown_reports", + "overwrite": true + } +} diff --git a/configs/from_manual.json b/configs/from_manual.json new file mode 100644 index 0000000..d8549ae --- /dev/null +++ b/configs/from_manual.json @@ -0,0 +1,142 @@ +{ + "$schema": "./schemas/config.schema.json", + "engine": { + "agent": { + "model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 2000, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "verbose": false, + "return_intermediate_steps": false, + "return_source_documents": false + }, + "checklist_model": { + "service": { + "_type": "nvfoundation" + }, + "model_name": "meta/llama3-70b-instruct", + "max_tokens": 2000, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "justification_model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + }, + "rag_embedding": { + "_type": "nim", + "model": "nvidia/nv-embedqa-e5-v5", + "truncate": "END", + "max_batch_size": 128 + }, + "summary_model": { + "model_name": "meta/llama3-70b-instruct", + "service": { + "_type": "nvfoundation" + }, + "max_tokens": 1024, + "temperature": 0, + "top_p": 0.01, + "seed": 42 + } + }, + "general": { + "cache_dir": null, + "base_vdb_dir": ".cache/am_cache/vdb", + "base_git_dir": ".cache/am_cache/git", + "max_retries": 5, + "model_max_batch_size": 64, + "pipeline_batch_size": 1024, + "use_uvloop": true, + "code_search_tool": false + }, + "input": { + "_type": "manual", + "message": { + "image": { + "name": "nvcr.io/nvidia/morpheus/morpheus", + "tag": "23.11-runtime", + "source_info": [ + { + "type": "code", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "tag": "v23.11.01", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ], + "exclude": [ + "tests/**/*" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "tag": "v23.11.01", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom" + } + }, + "scan": { + "vulns": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq" + }, + { + "vuln_id": "CVE-2023-50782" + }, + { + "vuln_id": "CVE-2023-36632" + }, + { + "vuln_id": "CVE-2023-43804" + }, + { + "vuln_id": "GHSA-cxfr-5q3r-2rc2" + }, + { + "vuln_id": "GHSA-554w-xh4j-8w64" + }, + { + "vuln_id": "GHSA-3ww4-gg4f-jr7f" // Same as CVE-2023-50782 + }, + { + "vuln_id": "CVE-2023-31147" // No vulnerable dependencies for this CVE + } + ] + } + } + }, + "output": { + "_type": "file", + "file_path": ".tmp/output.json", + "markdown_dir": ".tmp/vulnerability_markdown_reports", + "overwrite": true + } +} diff --git a/configs/openapi/openapi.json b/configs/openapi/openapi.json new file mode 100644 index 0000000..1e88ffe --- /dev/null +++ b/configs/openapi/openapi.json @@ -0,0 +1,1223 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Agent Morpheus", + "description": "API to determine the impact of documented CVEs on a specific project or container.", + "version": "1.0.0", + "termsOfService": "https://nvidia.com/legal/terms-of-use", + "contact": { + "name": "NVIDIA Support", + "url": "https://help.nvidia.com/" + } + }, + "servers": [ + { + "url": "https://ai.api.nvidia.com/v1/" + } + ], + "paths": { + "/scan": { + "post": { + "summary": "Run CVE workflow", + "operationId": "morpheus-vuln-analysis-scan", + "tags": [ + "nvidia / morpheus-vuln-analysis" + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusInput" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentMorpheusOutput" + } + } + } + } + } + }, + "required": true, + "x-nvai-meta": {} + } + }, + "security": [ + { + "Token": [] + } + ], + "components": { + "securitySchemes": { + "Token": { + "type": "http", + "scheme": "bearer" + } + }, + "schemas": { + "AgentMorpheusEngineOutput": { + "properties": { + "vuln_id": { + "type": "string", + "title": "Vuln Id" + }, + "checklist": { + "items": { + "$ref": "#/components/schemas/ChecklistItemOutput" + }, + "type": "array", + "title": "Checklist" + }, + "summary": { + "type": "string", + "title": "Summary" + }, + "justification": { + "$ref": "#/components/schemas/JustificationOutput" + } + }, + "type": "object", + "required": [ + "vuln_id", + "checklist", + "summary", + "justification" + ], + "title": "AgentMorpheusEngineOutput", + "description": "Contains all output generated by the main Agent Morpheus LLM Engine for a given vulnerability.\n\n- vuln_id: the ID of the vulnerability being processed by the LLM engine.\n- checklist: a list of ChecklistItemOutput objects, each containing an input and a response from the LLM agent.\n- summary: a short summary of the checklist inputs and responses, generated by an LLM.\n- justification: a justification" + }, + "AgentMorpheusInfo": { + "properties": { + "vdb": { + "anyOf": [ + { + "$ref": "#/components/schemas/VdbPaths" + }, + { + "type": "null" + } + ] + }, + "intel": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/CveIntel" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Intel" + }, + "sbom": { + "anyOf": [ + { + "$ref": "#/components/schemas/SBOMInfo" + }, + { + "type": "null" + } + ] + } + }, + "type": "object", + "title": "AgentMorpheusInfo", + "description": "Information used for decisioning in the Agent Morpheus engine. These information can all be automatically\ngenerated or retrieved by the pipeline from the input information.\n\n- vdb: paths to source code and documentation vector databases (VDBs) used to understand whether a vulnerability\n is exploitable in the source code.\n- intel: list of CveIntel objects representing intelligence for each vulnerability pulled from various vulnerability\n databases and APIs.\n- sbom: software bill of materials listing the packages and versions in the container image, used to understand\n whether the vulnerable package exists in the image." + }, + "AgentMorpheusInput": { + "properties": { + "scan": { + "$ref": "#/components/schemas/ScanInfoInput" + }, + "image": { + "$ref": "#/components/schemas/ImageInfoInput" + } + }, + "type": "object", + "required": [ + "scan", + "image" + ], + "title": "AgentMorpheusInput", + "description": "Inputs required by the Agent Morpheus pipeline." + }, + "AgentMorpheusOutput": { + "properties": { + "input": { + "$ref": "#/components/schemas/AgentMorpheusInput" + }, + "info": { + "$ref": "#/components/schemas/AgentMorpheusInfo" + }, + "output": { + "items": { + "$ref": "#/components/schemas/AgentMorpheusEngineOutput" + }, + "type": "array", + "title": "Output" + } + }, + "type": "object", + "required": [ + "input", + "info", + "output" + ], + "title": "AgentMorpheusOutput", + "description": "\"\nThe final output of the Agent Morpheus pipeline.\nContains all fields in the AgentMorpheusEngineInput, plus the AgentMorpheusEngineOuput for each input vulnerability." + }, + "Bugzilla": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + } + }, + "type": "object", + "title": "Bugzilla" + }, + "CVSS": { + "properties": { + "score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Score" + }, + "vector_string": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Vector String" + } + }, + "type": "object", + "title": "CVSS" + }, + "CWE": { + "properties": { + "cwe_id": { + "type": "string", + "title": "Cwe Id" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + } + }, + "type": "object", + "required": [ + "cwe_id" + ], + "title": "CWE" + }, + "ChecklistItemOutput": { + "properties": { + "input": { + "type": "string", + "title": "Input" + }, + "response": { + "type": "string", + "title": "Response" + } + }, + "type": "object", + "required": [ + "input", + "response" + ], + "title": "ChecklistItemOutput", + "description": "Input-response pair for a single checklist item provided to the LLM agent." + }, + "CveIntel": { + "properties": { + "vuln_id": { + "type": "string", + "title": "Vuln Id" + }, + "ghsa": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelGhsa" + }, + { + "type": "null" + } + ] + }, + "nvd": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelNvd" + }, + { + "type": "null" + } + ] + }, + "rhsa": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelRhsa" + }, + { + "type": "null" + } + ] + }, + "ubuntu": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelUbuntu" + }, + { + "type": "null" + } + ] + }, + "epss": { + "anyOf": [ + { + "$ref": "#/components/schemas/CveIntelEpss" + }, + { + "type": "null" + } + ] + } + }, + "type": "object", + "required": [ + "vuln_id" + ], + "title": "CveIntel", + "description": "Information about a CVE (Common Vulnerabilities and Exposures) entry." + }, + "CveIntelEpss": { + "properties": { + "epss": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Epss" + }, + "percentile": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Percentile" + }, + "date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Date" + } + }, + "additionalProperties": true, + "type": "object", + "title": "CveIntelEpss", + "description": "Information about an EPSS (Elastic Product Security Service) entry." + }, + "CveIntelGhsa": { + "properties": { + "ghsa_id": { + "type": "string", + "title": "Ghsa Id" + }, + "cve_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cve Id" + }, + "summary": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Summary" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "severity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Severity" + }, + "vulnerabilities": { + "anyOf": [ + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Vulnerabilities" + }, + "cvss": { + "anyOf": [ + { + "$ref": "#/components/schemas/CVSS" + }, + { + "type": "null" + } + ] + }, + "cwes": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/CWE" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Cwes" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "ghsa_id" + ], + "title": "CveIntelGhsa", + "description": "Information about a GHSA (GitHub Security Advisory) entry." + }, + "CveIntelNvd": { + "properties": { + "cve_id": { + "type": "string", + "title": "Cve Id" + }, + "cve_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cve Description" + }, + "cvss_vector": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cvss Vector" + }, + "cwe_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cwe Name" + }, + "cwe_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cwe Description" + }, + "cwe_extended_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cwe Extended Description" + }, + "configurations": { + "anyOf": [ + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Configurations" + }, + "vendor_names": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Vendor Names" + } + }, + "additionalProperties": true, + "type": "object", + "required": [ + "cve_id" + ], + "title": "CveIntelNvd", + "description": "Information about an NVD (National Vulnerability Database) entry." + }, + "CveIntelRhsa": { + "properties": { + "bugzilla": { + "$ref": "#/components/schemas/Bugzilla" + }, + "details": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Details" + }, + "statement": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Statement" + }, + "package_state": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/PackageState" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Package State" + }, + "upstream_fix": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Upstream Fix" + } + }, + "additionalProperties": true, + "type": "object", + "title": "CveIntelRhsa", + "description": "Information about a RHSA (Red Hat Security Advisory) entry." + }, + "CveIntelUbuntu": { + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "notes": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/Note" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Notes" + }, + "priority": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Priority" + }, + "ubuntu_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Ubuntu Description" + } + }, + "additionalProperties": true, + "type": "object", + "title": "CveIntelUbuntu", + "description": "Information about a Ubuntu CVE entry." + }, + "FileSBOMInfoInput": { + "properties": { + "_type": { + "const": "file", + "title": "Type", + "description": "The type of the object", + "default": "file" + }, + "file_path": { + "type": "string", + "title": "File Path" + } + }, + "type": "object", + "required": [ + "file_path" + ], + "title": "FileSBOMInfoInput", + "description": "A file path pointing to a Software Bill of Materials file." + }, + "ImageInfoInput": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Tag" + }, + "digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Digest" + }, + "platform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Platform" + }, + "feed_group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feed Group" + }, + "source_info": { + "items": { + "$ref": "#/components/schemas/SourceDocumentsInfo" + }, + "type": "array", + "title": "Source Info" + }, + "sbom_info": { + "oneOf": [ + { + "$ref": "#/components/schemas/ManualSBOMInfoInput" + }, + { + "$ref": "#/components/schemas/FileSBOMInfoInput" + } + ], + "title": "Sbom Info" + } + }, + "type": "object", + "required": [ + "source_info", + "sbom_info" + ], + "title": "ImageInfoInput", + "description": "Information about a container image, including the source information and sbom information." + }, + "JustificationOutput": { + "properties": { + "label": { + "type": "string", + "title": "Label" + }, + "reason": { + "type": "string", + "title": "Reason" + }, + "status": { + "type": "string", + "enum": [ + "TRUE", + "FALSE", + "UNKNOWN" + ], + "title": "Status" + } + }, + "type": "object", + "required": [ + "label", + "reason", + "status" + ], + "title": "JustificationOutput", + "description": "Final justification for the vulnerability.\n\n- label: a categorical justification label classifying the status of an image against a given vulnerability, e.g.\n code_not_present, code_not_reachable, false_positive.\n- reason: a human-readable explanation for why justification label was selected.\n- status: a ternary status (TRUE, FALSE, OR UNKNOWN) that indicates whether the image can be exploited for a given\n vulnerability. Determined based on a mapping from the justification label." + }, + "ManualSBOMInfoInput": { + "properties": { + "_type": { + "const": "manual", + "title": "Type", + "description": "The type of the object", + "default": "manual" + }, + "packages": { + "items": { + "$ref": "#/components/schemas/SBOMPackage" + }, + "type": "array", + "title": "Packages" + } + }, + "type": "object", + "required": [ + "packages" + ], + "title": "ManualSBOMInfoInput", + "description": "Manually provided Software Bill of Materials, consisting of a list of SBOMPackage objects." + }, + "Note": { + "properties": { + "author": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Author" + }, + "note": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Note" + } + }, + "type": "object", + "title": "Note" + }, + "PackageState": { + "properties": { + "product_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Product Name" + }, + "fix_state": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Fix State" + }, + "package_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Package Name" + }, + "cpe": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Cpe" + } + }, + "type": "object", + "title": "PackageState" + }, + "SBOMInfo": { + "properties": { + "packages": { + "items": { + "$ref": "#/components/schemas/SBOMPackage" + }, + "type": "array", + "title": "Packages" + } + }, + "type": "object", + "required": [ + "packages" + ], + "title": "SBOMInfo", + "description": "List of SBOMPackage objects representing the packages found in the input image." + }, + "SBOMPackage": { + "properties": { + "name": { + "type": "string", + "title": "Name" + }, + "version": { + "type": "string", + "title": "Version" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Path" + } + }, + "type": "object", + "required": [ + "name", + "version" + ], + "title": "SBOMPackage", + "description": "Information about a single package in the container image's Software Bill of Materials (SBOM)." + }, + "ScanInfoInput": { + "properties": { + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Id" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + }, + "started_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Started At" + }, + "completed_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Completed At" + }, + "vulns": { + "items": { + "$ref": "#/components/schemas/VulnInfo" + }, + "type": "array", + "title": "Vulns" + } + }, + "type": "object", + "required": [ + "vulns" + ], + "title": "ScanInfoInput", + "description": "Information about a unique scan for a container image against a list of vulnerabilies." + }, + "SourceDocumentsInfo": { + "properties": { + "type": { + "type": "string", + "enum": [ + "code", + "doc" + ], + "title": "Type" + }, + "git_repo": { + "type": "string", + "title": "Git Repo" + }, + "tag": { + "type": "string", + "title": "Tag" + }, + "include": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Include", + "default": [ + "*.py", + "*.ipynb" + ] + }, + "exclude": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Exclude", + "default": [] + } + }, + "type": "object", + "required": [ + "type", + "git_repo", + "tag" + ], + "title": "SourceDocumentsInfo", + "description": "Information about the source documents for the container image.\n\n- type: document type.\n- git_repo: git repo URL where the source documents can be cloned.\n- tag: git tag.\n- include: file extensions to include when indexing the source documents.\n- exclude: file extensions to exclude when indexing the source documents." + }, + "VdbPaths": { + "properties": { + "code_vdb_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Code Vdb Path" + }, + "doc_vdb_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Doc Vdb Path" + } + }, + "type": "object", + "title": "VdbPaths", + "description": "Paths to where the generated VDBs are stored." + }, + "VulnInfo": { + "properties": { + "vuln_id": { + "type": "string", + "title": "Vuln Id" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Score" + }, + "severity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Severity" + }, + "published_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published Date" + }, + "last_modified_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Last Modified Date" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Source" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Url" + }, + "feed_group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Feed Group" + } + }, + "type": "object", + "required": [ + "vuln_id" + ], + "title": "VulnInfo", + "description": "Information about a vulnerability." + } + } + } +} diff --git a/configs/schemas/config.schema.json b/configs/schemas/config.schema.json new file mode 100644 index 0000000..c572369 --- /dev/null +++ b/configs/schemas/config.schema.json @@ -0,0 +1,1335 @@ +{ + "$defs": { + "AgentMorpheusInput": { + "description": "Inputs required by the Agent Morpheus pipeline.", + "properties": { + "scan": { + "$ref": "#/$defs/ScanInfoInput" + }, + "image": { + "$ref": "#/$defs/ImageInfoInput" + } + }, + "required": [ + "scan", + "image" + ], + "title": "AgentMorpheusInput", + "type": "object" + }, + "EngineAgentConfig": { + "properties": { + "model": { + "oneOf": [ + { + "$ref": "#/$defs/NeMoLLMModelConfig" + }, + { + "$ref": "#/$defs/OpenAIModelConfig" + }, + { + "$ref": "#/$defs/NVFoundationLLMModelConfig" + } + ], + "title": "Model" + }, + "version_compare_tool": { + "default": false, + "title": "Version Compare Tool", + "type": "boolean" + }, + "prompt_examples": { + "default": false, + "title": "Prompt Examples", + "type": "boolean" + }, + "verbose": { + "default": false, + "title": "Verbose", + "type": "boolean" + }, + "return_intermediate_steps": { + "default": false, + "title": "Return Intermediate Steps", + "type": "boolean" + }, + "return_source_documents": { + "default": false, + "title": "Return Source Documents", + "type": "boolean" + } + }, + "required": [ + "model" + ], + "title": "EngineAgentConfig", + "type": "object" + }, + "EngineConfig": { + "properties": { + "agent": { + "$ref": "#/$defs/EngineAgentConfig" + }, + "checklist_model": { + "oneOf": [ + { + "$ref": "#/$defs/NeMoLLMModelConfig" + }, + { + "$ref": "#/$defs/OpenAIModelConfig" + }, + { + "$ref": "#/$defs/NVFoundationLLMModelConfig" + } + ], + "title": "Checklist Model" + }, + "justification_model": { + "oneOf": [ + { + "$ref": "#/$defs/NeMoLLMModelConfig" + }, + { + "$ref": "#/$defs/OpenAIModelConfig" + }, + { + "$ref": "#/$defs/NVFoundationLLMModelConfig" + } + ], + "title": "Justification Model" + }, + "rag_embedding": { + "oneOf": [ + { + "$ref": "#/$defs/HuggingFaceEmbeddingModelConfig" + }, + { + "$ref": "#/$defs/OpenAIEmbeddingModelConfig" + }, + { + "$ref": "#/$defs/NIMEmbeddingModelConfig" + } + ], + "title": "Rag Embedding" + }, + "sbom_faiss_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sbom Faiss Dir" + }, + "summary_model": { + "oneOf": [ + { + "$ref": "#/$defs/NeMoLLMModelConfig" + }, + { + "$ref": "#/$defs/OpenAIModelConfig" + }, + { + "$ref": "#/$defs/NVFoundationLLMModelConfig" + } + ], + "title": "Summary Model" + } + }, + "required": [ + "agent", + "checklist_model", + "justification_model", + "rag_embedding", + "summary_model" + ], + "title": "EngineConfig", + "type": "object" + }, + "FileInputConfig": { + "properties": { + "_type": { + "const": "file", + "default": "file", + "description": "The type of the object", + "title": "Type" + }, + "file": { + "title": "File", + "type": "string" + }, + "repeat_count": { + "default": 1, + "title": "Repeat Count", + "type": "integer" + } + }, + "required": [ + "file" + ], + "title": "FileInputConfig", + "type": "object" + }, + "FileSBOMInfoInput": { + "description": "A file path pointing to a Software Bill of Materials file.", + "properties": { + "_type": { + "const": "file", + "default": "file", + "description": "The type of the object", + "title": "Type" + }, + "file_path": { + "title": "File Path", + "type": "string" + } + }, + "required": [ + "file_path" + ], + "title": "FileSBOMInfoInput", + "type": "object" + }, + "GeneralConfig": { + "properties": { + "base_vdb_dir": { + "default": "/tmp/am_cache/vdb", + "title": "Base Vdb Dir", + "type": "string" + }, + "base_git_dir": { + "default": "/tmp/am_cache/git", + "title": "Base Git Dir", + "type": "string" + }, + "enable_llm_list_parsing": { + "default": false, + "title": "Enable Llm List Parsing", + "type": "boolean" + }, + "cache_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Cache Dir" + }, + "ignore_build_vdb_errors": { + "default": false, + "title": "Ignore Build Vdb Errors", + "type": "boolean" + }, + "max_retries": { + "default": 10, + "minimum": 0, + "title": "Max Retries", + "type": "integer" + }, + "model_max_batch_size": { + "default": 64, + "exclusiveMinimum": 0, + "title": "Model Max Batch Size", + "type": "integer" + }, + "num_threads": { + "exclusiveMinimum": 0, + "title": "Num Threads", + "type": "integer" + }, + "pipeline_batch_size": { + "default": 1024, + "exclusiveMinimum": 0, + "title": "Pipeline Batch Size", + "type": "integer" + }, + "use_uvloop": { + "default": true, + "title": "Use Uvloop", + "type": "boolean" + }, + "code_search_tool": { + "default": false, + "title": "Code Search Tool", + "type": "boolean" + } + }, + "title": "GeneralConfig", + "type": "object" + }, + "HTTPMethod": { + "description": "Not a complete list of HTTP methods, just the ones we use.", + "enum": [ + "GET", + "PATCH", + "POST", + "PUT" + ], + "title": "HTTPMethod", + "type": "string" + }, + "HttpInputConfig": { + "properties": { + "_type": { + "const": "http", + "default": "http", + "description": "The type of the object", + "title": "Type" + }, + "address": { + "default": "127.0.0.1", + "title": "Address", + "type": "string" + }, + "endpoint": { + "default": "/scan", + "title": "Endpoint", + "type": "string" + }, + "port": { + "default": 8080, + "title": "Port", + "type": "integer" + }, + "http_method": { + "allOf": [ + { + "$ref": "#/$defs/HTTPMethod" + } + ], + "default": "POST" + }, + "stop_after": { + "default": 0, + "title": "Stop After", + "type": "integer" + } + }, + "title": "HttpInputConfig", + "type": "object" + }, + "HuggingFaceEmbeddingModelConfig": { + "properties": { + "_type": { + "const": "huggingface", + "default": "huggingface", + "description": "The type of the object", + "title": "Type" + }, + "model_name": { + "title": "Model Name", + "type": "string" + }, + "model_kwargs": { + "default": {}, + "title": "Model Kwargs", + "type": "object" + }, + "encode_kwargs": { + "default": {}, + "title": "Encode Kwargs", + "type": "object" + } + }, + "required": [ + "model_name" + ], + "title": "HuggingFaceEmbeddingModelConfig", + "type": "object" + }, + "ImageInfoInput": { + "description": "Information about a container image, including the source information and sbom information.", + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "tag": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tag" + }, + "digest": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Digest" + }, + "platform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Platform" + }, + "feed_group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Feed Group" + }, + "source_info": { + "items": { + "$ref": "#/$defs/SourceDocumentsInfo" + }, + "title": "Source Info", + "type": "array" + }, + "sbom_info": { + "oneOf": [ + { + "$ref": "#/$defs/ManualSBOMInfoInput" + }, + { + "$ref": "#/$defs/FileSBOMInfoInput" + } + ], + "title": "Sbom Info" + } + }, + "required": [ + "source_info", + "sbom_info" + ], + "title": "ImageInfoInput", + "type": "object" + }, + "ManualInputConfig": { + "properties": { + "_type": { + "const": "manual", + "default": "manual", + "description": "The type of the object", + "title": "Type" + }, + "message": { + "$ref": "#/$defs/AgentMorpheusInput" + }, + "repeat_count": { + "default": 1, + "title": "Repeat Count", + "type": "integer" + } + }, + "required": [ + "message" + ], + "title": "ManualInputConfig", + "type": "object" + }, + "ManualSBOMInfoInput": { + "description": "Manually provided Software Bill of Materials, consisting of a list of SBOMPackage objects.", + "properties": { + "_type": { + "const": "manual", + "default": "manual", + "description": "The type of the object", + "title": "Type" + }, + "packages": { + "items": { + "$ref": "#/$defs/SBOMPackage" + }, + "title": "Packages", + "type": "array" + } + }, + "required": [ + "packages" + ], + "title": "ManualSBOMInfoInput", + "type": "object" + }, + "NIMEmbeddingModelConfig": { + "properties": { + "_type": { + "const": "nim", + "default": "nim", + "description": "The type of the object", + "title": "Type" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Api Key" + }, + "model": { + "title": "Model", + "type": "string" + }, + "truncate": { + "default": "END", + "enum": [ + "NONE", + "START", + "END" + ], + "title": "Truncate", + "type": "string" + }, + "max_batch_size": { + "default": 128, + "title": "Max Batch Size", + "type": "integer" + } + }, + "required": [ + "model" + ], + "title": "NIMEmbeddingModelConfig", + "type": "object" + }, + "NVFoundationLLMModelConfig": { + "properties": { + "_type": { + "const": "nvfoundation", + "default": "nvfoundation", + "description": "The type of the object", + "title": "Type" + }, + "service": { + "$ref": "#/$defs/NVFoundationLLMServiceConfig" + }, + "model_name": { + "title": "Model Name", + "type": "string" + }, + "temperature": { + "default": 0.0, + "title": "Temperature", + "type": "number" + }, + "top_p": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top P" + }, + "max_tokens": { + "default": 300, + "exclusiveMinimum": 0, + "title": "Max Tokens", + "type": "integer" + }, + "seed": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seed" + } + }, + "required": [ + "service", + "model_name" + ], + "title": "NVFoundationLLMModelConfig", + "type": "object" + }, + "NVFoundationLLMServiceConfig": { + "properties": { + "_type": { + "const": "nvfoundation", + "default": "nvfoundation", + "description": "The type of the object", + "title": "Type" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Api Key" + }, + "base_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Base Url" + } + }, + "title": "NVFoundationLLMServiceConfig", + "type": "object" + }, + "NeMoLLMModelConfig": { + "properties": { + "service": { + "$ref": "#/$defs/NeMoLLMServiceConfig" + }, + "model_name": { + "title": "Model Name", + "type": "string" + }, + "customization_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Customization Id" + }, + "temperature": { + "default": 0.0, + "maximum": 1.0, + "minimum": 0.0, + "title": "Temperature", + "type": "number" + }, + "top_k": { + "default": 0, + "minimum": 0, + "title": "Top K", + "type": "integer" + }, + "top_p": { + "default": 1, + "title": "Top P", + "type": "number" + }, + "random_seed": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Random Seed" + }, + "tokens_to_generate": { + "default": 300, + "title": "Tokens To Generate", + "type": "integer" + }, + "beam_search_diversity_rate": { + "default": 0.0, + "title": "Beam Search Diversity Rate", + "type": "number" + }, + "beam_width": { + "default": 1, + "title": "Beam Width", + "type": "integer" + }, + "repetition_penalty": { + "default": 1.0, + "title": "Repetition Penalty", + "type": "number" + }, + "length_penalty": { + "default": 1.0, + "title": "Length Penalty", + "type": "number" + }, + "logprobs": { + "default": true, + "title": "Logprobs", + "type": "boolean" + } + }, + "required": [ + "service", + "model_name" + ], + "title": "NeMoLLMModelConfig", + "type": "object" + }, + "NeMoLLMServiceConfig": { + "properties": { + "_type": { + "const": "nemo", + "default": "nemo", + "description": "The type of the object", + "title": "Type" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Api Key" + }, + "org_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Org Id" + }, + "retry_count": { + "default": 5, + "minimum": 0, + "title": "Retry Count", + "type": "integer" + } + }, + "title": "NeMoLLMServiceConfig", + "type": "object" + }, + "OpenAIEmbeddingModelConfig": { + "properties": { + "_type": { + "const": "openai", + "default": "openai", + "description": "The type of the object", + "title": "Type" + }, + "openai_api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Openai Api Key" + }, + "max_retries": { + "default": 10, + "title": "Max Retries", + "type": "integer" + }, + "chunk_size": { + "default": 128, + "title": "Chunk Size", + "type": "integer" + } + }, + "title": "OpenAIEmbeddingModelConfig", + "type": "object" + }, + "OpenAIModelConfig": { + "properties": { + "service": { + "$ref": "#/$defs/OpenAIServiceConfig" + }, + "model_name": { + "title": "Model Name", + "type": "string" + }, + "temperature": { + "default": 0.0, + "title": "Temperature", + "type": "number" + }, + "top_p": { + "default": 1.0, + "title": "Top P", + "type": "number" + }, + "seed": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seed" + }, + "max_retries": { + "default": 10, + "title": "Max Retries", + "type": "integer" + }, + "json": { + "default": false, + "title": "Json", + "type": "boolean" + } + }, + "required": [ + "service", + "model_name" + ], + "title": "OpenAIModelConfig", + "type": "object" + }, + "OpenAIServiceConfig": { + "properties": { + "_type": { + "const": "openai", + "default": "openai", + "description": "The type of the object", + "title": "Type" + }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Api Key" + }, + "base_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Base Url" + } + }, + "title": "OpenAIServiceConfig", + "type": "object" + }, + "OutputFileConfig": { + "properties": { + "_type": { + "const": "file", + "default": "file", + "description": "The type of the object", + "title": "Type" + }, + "file_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "./.tmp/agent_morpheus_output.json", + "title": "File Path" + }, + "markdown_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "./.tmp/vulnerability_markdown_reports", + "title": "Markdown Dir" + }, + "overwrite": { + "default": false, + "title": "Overwrite", + "type": "boolean" + } + }, + "title": "OutputFileConfig", + "type": "object" + }, + "OutputPluginConfig": { + "properties": { + "_type": { + "const": "plugin", + "default": "plugin", + "description": "The type of the object", + "title": "Type" + }, + "plugin_name": { + "title": "Plugin Name", + "type": "string" + }, + "plugin_config": { + "default": {}, + "title": "Plugin Config", + "type": "object" + } + }, + "required": [ + "plugin_name" + ], + "title": "OutputPluginConfig", + "type": "object" + }, + "OutputPrintConfig": { + "properties": { + "_type": { + "const": "print", + "default": "print", + "description": "The type of the object", + "title": "Type" + } + }, + "title": "OutputPrintConfig", + "type": "object" + }, + "PluginInputConfig": { + "properties": { + "_type": { + "const": "plugin", + "default": "plugin", + "description": "The type of the object", + "title": "Type" + }, + "plugin_name": { + "title": "Plugin Name", + "type": "string" + }, + "plugin_config": { + "default": {}, + "title": "Plugin Config", + "type": "object" + } + }, + "required": [ + "plugin_name" + ], + "title": "PluginInputConfig", + "type": "object" + }, + "SBOMPackage": { + "description": "Information about a single package in the container image's Software Bill of Materials (SBOM).", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "version": { + "title": "Version", + "type": "string" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "system": { + "title": "System", + "type": "string" + } + }, + "required": [ + "name", + "version", + "system" + ], + "title": "SBOMPackage", + "type": "object" + }, + "ScanInfoInput": { + "description": "Information about a unique scan for a container image against a list of vulnerabilies.", + "properties": { + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Id" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Type" + }, + "started_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Started At" + }, + "completed_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Completed At" + }, + "vulns": { + "items": { + "$ref": "#/$defs/VulnInfo" + }, + "minItems": 1, + "title": "Vulns", + "type": "array" + } + }, + "required": [ + "vulns" + ], + "title": "ScanInfoInput", + "type": "object" + }, + "SourceDocumentsInfo": { + "description": "Information about the source documents for the container image.\n\n- type: document type.\n- git_repo: git repo URL where the source documents can be cloned.\n- tag: git tag.\n- include: file extensions to include when indexing the source documents.\n- exclude: file extensions to exclude when indexing the source documents.", + "properties": { + "type": { + "enum": [ + "code", + "doc" + ], + "title": "Type", + "type": "string" + }, + "git_repo": { + "minLength": 1, + "title": "Git Repo", + "type": "string" + }, + "tag": { + "minLength": 1, + "title": "Tag", + "type": "string" + }, + "include": { + "default": [ + "*.py", + "*.ipynb" + ], + "items": { + "type": "string" + }, + "title": "Include", + "type": "array" + }, + "exclude": { + "default": [], + "items": { + "type": "string" + }, + "title": "Exclude", + "type": "array" + } + }, + "required": [ + "type", + "git_repo", + "tag" + ], + "title": "SourceDocumentsInfo", + "type": "object" + }, + "VulnInfo": { + "additionalProperties": true, + "description": "Information about a vulnerability.", + "properties": { + "vuln_id": { + "title": "Vuln Id", + "type": "string" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Score" + }, + "severity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Severity" + }, + "published_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Published Date" + }, + "last_modified_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Last Modified Date" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Url" + }, + "feed_group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Feed Group" + }, + "package": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Package" + }, + "package_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Package Version" + }, + "package_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Package Name" + }, + "package_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Package Type" + } + }, + "required": [ + "vuln_id" + ], + "title": "VulnInfo", + "type": "object" + } + }, + "properties": { + "general": { + "allOf": [ + { + "$ref": "#/$defs/GeneralConfig" + } + ], + "default": { + "base_vdb_dir": "/tmp/am_cache/vdb", + "base_git_dir": "/tmp/am_cache/git", + "enable_llm_list_parsing": false, + "cache_dir": null, + "ignore_build_vdb_errors": false, + "max_retries": 10, + "model_max_batch_size": 64, + "num_threads": 64, + "pipeline_batch_size": 1024, + "use_uvloop": true, + "code_search_tool": false + } + }, + "input": { + "oneOf": [ + { + "$ref": "#/$defs/ManualInputConfig" + }, + { + "$ref": "#/$defs/FileInputConfig" + }, + { + "$ref": "#/$defs/HttpInputConfig" + }, + { + "$ref": "#/$defs/PluginInputConfig" + } + ], + "title": "Input" + }, + "engine": { + "$ref": "#/$defs/EngineConfig" + }, + "output": { + "oneOf": [ + { + "$ref": "#/$defs/OutputPrintConfig" + }, + { + "$ref": "#/$defs/OutputFileConfig" + }, + { + "$ref": "#/$defs/OutputPluginConfig" + } + ], + "title": "Output" + } + }, + "required": [ + "input", + "engine", + "output" + ], + "title": "RunConfig", + "type": "object" +} \ No newline at end of file diff --git a/data/input_messages/morpheus:23.11-runtime.json b/data/input_messages/morpheus:23.11-runtime.json new file mode 100644 index 0000000..1a91fb4 --- /dev/null +++ b/data/input_messages/morpheus:23.11-runtime.json @@ -0,0 +1,64 @@ +{ + "image": { + "name": "nvcr.io/nvidia/morpheus/morpheus", + "tag": "23.11-runtime", + "source_info": [ + { + "type": "code", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "tag": "v23.11.01", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ], + "exclude": [ + "tests/**/*" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "tag": "v23.11.01", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom" + } + }, + "scan": { + "vulns": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq" + }, + { + "vuln_id": "CVE-2023-50782" + }, + { + "vuln_id": "CVE-2023-36632" + }, + { + "vuln_id": "CVE-2023-43804" + }, + { + "vuln_id": "GHSA-cxfr-5q3r-2rc2" + }, + { + "vuln_id": "GHSA-554w-xh4j-8w64" + }, + { + "vuln_id": "GHSA-3ww4-gg4f-jr7f" + } + ] + } +} diff --git a/data/input_messages/morpheus:24.03-runtime.json b/data/input_messages/morpheus:24.03-runtime.json new file mode 100644 index 0000000..87d9be8 --- /dev/null +++ b/data/input_messages/morpheus:24.03-runtime.json @@ -0,0 +1,64 @@ +{ + "image": { + "name": "nvcr.io/nvidia/morpheus/morpheus", + "tag": "v24.03.02-runtime", + "source_info": [ + { + "type": "code", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "tag": "v24.03.02", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ], + "exclude": [ + "tests/**/*" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/nv-morpheus/Morpheus.git", + "tag": "v24.03.02", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom" + } + }, + "scan": { + "vulns": [ + { + "vuln_id": "GHSA-3f63-hfp8-52jq" + }, + { + "vuln_id": "CVE-2023-50782" + }, + { + "vuln_id": "CVE-2023-36632" + }, + { + "vuln_id": "CVE-2023-43804" + }, + { + "vuln_id": "GHSA-cxfr-5q3r-2rc2" + }, + { + "vuln_id": "GHSA-554w-xh4j-8w64" + }, + { + "vuln_id": "GHSA-3ww4-gg4f-jr7f" + } + ] + } +} diff --git a/data/input_messages/rapidsai:23.08-cuda11.8-py3.10.json b/data/input_messages/rapidsai:23.08-cuda11.8-py3.10.json new file mode 100644 index 0000000..60a83d5 --- /dev/null +++ b/data/input_messages/rapidsai:23.08-cuda11.8-py3.10.json @@ -0,0 +1,282 @@ +{ + "image": { + "name": "nvcr.io/nvidia/rapidsai/base", + "tag": "24.06-cuda12.2-py3.10", + "source_info": [ + // cuDF + { + "type": "code", + "git_repo": "https://github.com/rapidsai/cudf.git", + "tag": "v23.08.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/cudf.git", + "tag": "v23.08.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // cuML + { + "type": "code", + "git_repo": "https://github.com/rapidsai/cuml.git", + "tag": "v23.08.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/cuml.git", + "tag": "v23.08.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // cuGraph + { + "type": "code", + "git_repo": "https://github.com/rapidsai/cugraph.git", + "tag": "v23.08.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/cugraph.git", + "tag": "v23.08.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // RMM + { + "type": "code", + "git_repo": "https://github.com/rapidsai/RMM.git", + "tag": "v23.08.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/RMM.git", + "tag": "v23.08.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // RAFT + { + "type": "code", + "git_repo": "https://github.com/rapidsai/RAFT.git", + "tag": "v23.08.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/RAFT.git", + "tag": "v23.08.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // cuSpatial + { + "type": "code", + "git_repo": "https://github.com/rapidsai/cuSpatial.git", + "tag": "v23.08.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/cuSpatial.git", + "tag": "v23.08.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // cuxfilter + { + "type": "code", + "git_repo": "https://github.com/rapidsai/cuxfilter.git", + "tag": "v23.08.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/cuxfilter.git", + "tag": "v23.08.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // cuCIM + { + "type": "code", + "git_repo": "https://github.com/rapidsai/cuCIM.git", + "tag": "v23.08.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/cuCIM.git", + "tag": "v23.08.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/rapidsai/base:23.08-cuda11.8-py3.10.sbom" + } + }, + "scan": { + "vulns": [ + { + "vuln_id": "GHSA-5wvp-7f3h-6wmm" + }, + { + "vuln_id": "GHSA-w3vc-fx9p-wp4v" + }, + { + "vuln_id": "CVE-2024-0985" + }, + { + "vuln_id": "GHSA-94vc-p8w7-5p49" + }, + { + "vuln_id": "GHSA-q3qx-c6g2-7pw2" + }, + { + "vuln_id": "CVE-2023-5869" + }, + { + "vuln_id": "GHSA-2jv5-9r88-3w3p" + }, + { + "vuln_id": "CVE-2023-5363" + }, + { + "vuln_id": "GHSA-2jv5-9r88-3w3p" + }, + { + "vuln_id": "GHSA-3f63-hfp8-52jq" + }, + { + "vuln_id": "CVE-2023-4911" + }, + { + "vuln_id": "GHSA-56pw-mpj4-fxww" + }, + { + "vuln_id": "CVE-2023-4911" + }, + { + "vuln_id": "GHSA-6673-4983-2vx5" + }, + { + "vuln_id": "CVE-2023-4911" + }, + { + "vuln_id": "CVE-2023-4807" + }, + { + "vuln_id": "CVE-2023-36632" + }, + { + "vuln_id": "GHSA-6vqw-3v5j-54x4" + }, + { + "vuln_id": "GHSA-j7hp-h8jx-5ppr" + }, + { + "vuln_id": "CVE-2023-36632" + }, + { + "vuln_id": "GHSA-5m98-qgg9-wh84" + }, + { + "vuln_id": "GHSA-3ww4-gg4f-jr7f" + } + ] + } +} diff --git a/data/input_messages/rapidsai:24.06-cuda12.0-py3.9.json b/data/input_messages/rapidsai:24.06-cuda12.0-py3.9.json new file mode 100644 index 0000000..5bc7528 --- /dev/null +++ b/data/input_messages/rapidsai:24.06-cuda12.0-py3.9.json @@ -0,0 +1,247 @@ +{ + "image": { + "name": "nvcr.io/nvidia/rapidsai/base", + "tag": "24.06-cuda12.2-py3.10", + "source_info": [ + // cuDF + { + "type": "code", + "git_repo": "https://github.com/rapidsai/cudf.git", + "tag": "v24.06.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/cudf.git", + "tag": "v24.06.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // cuML + { + "type": "code", + "git_repo": "https://github.com/rapidsai/cuml.git", + "tag": "v24.06.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/cuml.git", + "tag": "v24.06.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // cuGraph + { + "type": "code", + "git_repo": "https://github.com/rapidsai/cugraph.git", + "tag": "v24.06.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/cugraph.git", + "tag": "v24.06.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // cuVS + { + "type": "code", + "git_repo": "https://github.com/rapidsai/cuvs.git", + "tag": "v24.06.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/cuvs.git", + "tag": "v24.06.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // RMM + { + "type": "code", + "git_repo": "https://github.com/rapidsai/RMM.git", + "tag": "v24.06.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/RMM.git", + "tag": "v24.06.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // RAFT + { + "type": "code", + "git_repo": "https://github.com/rapidsai/RAFT.git", + "tag": "v24.06.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/RAFT.git", + "tag": "v24.06.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // cuSpatial + { + "type": "code", + "git_repo": "https://github.com/rapidsai/cuSpatial.git", + "tag": "v24.06.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/cuSpatial.git", + "tag": "v24.06.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // cuxfilter + { + "type": "code", + "git_repo": "https://github.com/rapidsai/cuxfilter.git", + "tag": "v24.06.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/cuxfilter.git", + "tag": "v24.06.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + }, + // cuCIM + { + "type": "code", + "git_repo": "https://github.com/rapidsai/cuCIM.git", + "tag": "v24.06.00", + "include": [ + "**/*.cpp", + "**/*.cu", + "**/*.cuh", + "**/*.h", + "**/*.hpp", + "**/*.ipynb", + "**/*.py", + "**/*Dockerfile" + ] + }, + { + "type": "doc", + "git_repo": "https://github.com/rapidsai/cuCIM.git", + "tag": "v24.06.00", + "include": [ + "**/*.md", + "docs/**/*.rst" + ] + } + ], + "sbom_info": { + "_type": "file", + "file_path": "data/sboms/nvcr.io/nvidia/rapidsai/base:24.06-cuda12.2-py3.10.sbom" + } + }, + "scan": { + "vulns": [ + { + "vuln_id": "CVE-2023-36632" + }, + { + "vuln_id": "GHSA-f5x3-32g6-xq36" + } + ] + } +} diff --git a/data/sboms/README.md b/data/sboms/README.md new file mode 100644 index 0000000..bf49303 --- /dev/null +++ b/data/sboms/README.md @@ -0,0 +1,59 @@ + + + +This directory contains the SBOMs for the containers used in the LLM example. To generate an SBOM for a container, you can use [syft](https://github.com/anchore/syft). + +To install syft, you can use the following command: + +```bash +mamba install -c conda-forge syft +``` + +The following steps show how to generate an SBOM for the Morpheus container. + +```bash +# Save the Morpheus repo directory +export MORPHEUS_ROOT=$(git rev-parse --show-toplevel) + +# Change directory to the SBOMs directory +cd ${MORPHEUS_ROOT}/data/sboms + +# Disable colors for syft +export NO_COLORS=y + +# Specify which container to generate an SBOM for +export CONTAINER="nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime" + +# Generate SBOM +syft scan ${CONTAINER} -o syft-table=${CONTAINER}.sbom +``` + +To generate an SBOM for a list of containers, you can use the following script: + +```bash +# Specify which containers to generate SBOMs for +export CONTAINERS=( + "nvcr.io/nvidia/morpheus/morpheus:24.03-runtime" + "nvcr.io/nvidia/morpheus/morpheus:23.11-runtime" +) + +# Generate SBOMs +for CONTAINER in "${CONTAINERS[@]}"; do + syft scan ${CONTAINER} -o syft-table=${CONTAINER}.sbom +done +``` diff --git a/data/sboms/nvcr.io/nvidia/morpheus/morpheus:23.07-runtime.sbom b/data/sboms/nvcr.io/nvidia/morpheus/morpheus:23.07-runtime.sbom new file mode 100644 index 0000000..79026ec --- /dev/null +++ b/data/sboms/nvcr.io/nvidia/morpheus/morpheus:23.07-runtime.sbom @@ -0,0 +1,494 @@ +NAME VERSION TYPE +@jupyter-widgets/jupyterlab-manager 5.0.8 npm  +@jupyterlab/application-top 4.0.3 npm (+2 duplicates) +@jupyterlab/galata-extension 5.0.2 npm  +@jupyterlab/mock-consumer 4.0.3 npm  +@jupyterlab/mock-extension 4.0.3 npm  +@jupyterlab/mock-incompat 0.1.0 npm  +@jupyterlab/mock-mime-extension 0.3.0 npm  +@jupyterlab/mock-package 0.1.0 npm  +@jupyterlab/mock-provider 4.0.3 npm  +@jupyterlab/mock-token 4.0.3 npm  +Babel 2.12.1 python  +Brotli 1.0.9 python (+1 duplicate) +ConfigArgParse 1.5.5 python  +Flask 2.3.2 python  +GitPython 3.1.32 python  +Jinja2 3.1.2 python (+1 duplicate) +Mako 1.2.4 python  +Markdown 3.4.4 python  +MarkupSafe 2.1.3 python (+1 duplicate) +Pillow 10.0.0 python  +PyJWT 2.8.0 python  +PySocks 1.7.1 python (+1 duplicate) +PyYAML 6.0 python (+1 duplicate) +Pygments 2.15.1 python (+1 duplicate) +SQLAlchemy 2.0.0 python  +Send2Trash 1.8.2 python  +Simple Launcher 1.1.0.14 dotnet (+5 duplicates) +Sphinx 7.0.1 python  +Werkzeug 2.3.6 python  +absl-py 1.4.0 python  +adduser 3.118ubuntu5 deb  +alabaster 0.7.13 python  +alembic 1.11.1 python  +anyio 3.7.1 python (+1 duplicate) +appdirs 1.4.4 python  +apt 2.4.9 deb  +argon2-cffi 21.3.0 python  +argon2-cffi-bindings 21.2.0 python  +arrow 1.2.3 python  +asn1crypto 1.5.1 python  +asttokens 2.2.1 python  +async-lru 2.0.4 python  +attrs 23.1.0 python (+1 duplicate) +backcall 0.2.0 python  +backports.functools-lru-cache 1.6.5 python  +base-files 12ubuntu4.3 deb  +base-passwd 3.5.52build1 deb  +bash 5.1-6ubuntu1 deb  +bc 1.07.1-3build1 deb  +beautifulsoup4 4.12.2 python (+1 duplicate) +betterproto 1.2.5 python  +binutils 2.38-4ubuntu2.2 deb  +binutils-common 2.38-4ubuntu2.2 deb  +binutils-x86-64-linux-gnu 2.38-4ubuntu2.2 deb  +bleach 6.0.0 python  +blinker 1.6.2 python  +boa 0.15.1 python  +bokeh 2.4.3 python  +bsdutils 1:2.37.2-4ubuntu3 deb  +build-essential 12.9ubuntu3 deb  +bytecode 0.13.0.dev0 python  +bzip2 1.0.8-5build1 deb  +ca-certificates 20230311ubuntu0.22.04.1 deb  +cachetools 5.3.1 python  +certifi 2023.7.22 python (+1 duplicate) +cffi 1.15.1 python (+1 duplicate) +chardet 5.1.0 python  +charset-normalizer 3.2.0 python (+1 duplicate) +click 8.1.6 python (+1 duplicate) +cloudpickle 2.2.1 python  +colorama 0.4.6 python (+1 duplicate) +comm 0.1.3 python  +conda 23.1.0 python  +conda-build 3.25.0 python  +conda-package-handling 2.0.2 python  +conda_index 0.2.3 python  +conda_package_streaming 0.8.0 python  +configparser 5.3.0 python  +confluent-kafka 1.9.2 python  +contourpy 1.1.0 python  +coreutils 8.32-4.1ubuntu1 deb  +cpp 4:11.2.0-1ubuntu1 deb  +cpp-11 11.3.0-1ubuntu1~22.04.1 deb  +cryptography 41.0.2 python (+1 duplicate) +cubinlinker 0.3.0 python  +cuda-cccl-11-8 11.8.89-1 deb  +cuda-compat-11-8 520.61.05-1 deb  +cuda-compiler-11-8 11.8.0-1 deb  +cuda-cudart-11-8 11.8.89-1 deb  +cuda-cudart-dev-11-8 11.8.89-1 deb  +cuda-cuobjdump-11-8 11.8.86-1 deb  +cuda-cupti-11-8 11.8.87-1 deb  +cuda-cuxxfilt-11-8 11.8.86-1 deb  +cuda-driver-dev-11-8 11.8.89-1 deb  +cuda-gdb-11-8 11.8.86-1 deb  +cuda-keyring 1.1-1 deb  +cuda-nvcc-11-8 11.8.89-1 deb  +cuda-nvdisasm-11-8 11.8.86-1 deb  +cuda-nvprune-11-8 11.8.86-1 deb  +cuda-python 11.8.2 python  +cuda-toolkit-11-8-config-common 11.8.89-1 deb  +cuda-toolkit-11-config-common 11.8.89-1 deb  +cuda-toolkit-config-common 12.2.53-1 deb  +cudf 23.6.1 python  +cudf-kafka 23.6.1 python  +cupy 12.1.0 python  +curl 7.81.0-1ubuntu1.13 deb  +cycler 0.11.0 python  +cytoolz 0.12.2 python  +dash 0.5.11+git20210903+057cd650a4ed-3build1 deb  +dask 2023.3.2 python  +dask-cuda 23.6.0 python  +dask-cudf 23.6.1 python  +databricks-cli 0.17.7 python  +dataclasses 0.8 python (+1 duplicate) +datacompy 0.8.4 python  +debconf 1.5.79ubuntu1 deb  +debianutils 5.5-1ubuntu2 deb  +debugpy 1.6.7 python  +decorator 5.1.1 python  +defusedxml 0.7.1 python  +diffutils 1:3.8-0ubuntu2 deb  +dill 0.3.7 python  +dirmngr 2.2.27-3ubuntu2.1 deb  +distributed 2023.3.2.1 python  +docker 5.0.3 python  +docker-pycreds 0.4.0 python  +docutils 0.20.1 python  +dpkg 1.21.1ubuntu2.2 deb  +dpkg-dev 1.21.1ubuntu2.2 deb  +e2fsprogs 1.46.5-2ubuntu1.1 deb  +entrypoints 0.4 python  +exceptiongroup 1.1.2 python (+1 duplicate) +executing 1.2.0 python  +fastjsonschema 2.18.0 python  +fastrlock 0.8 python  +filelock 3.12.2 python (+1 duplicate) +findutils 4.8.0-1ubuntu3 deb  +fonttools 4.41.1 python  +fqdn 1.5.1 python  +fsspec 2023.6.0 python  +g++ 4:11.2.0-1ubuntu1 deb  +g++-11 11.3.0-1ubuntu1~22.04.1 deb  +gcc 4:11.2.0-1ubuntu1 deb  +gcc-11 11.3.0-1ubuntu1~22.04.1 deb  +gcc-11-base 11.3.0-1ubuntu1~22.04.1 deb  +gcc-12-base 12.1.0-2ubuntu1~22.04 deb  +gitdb 4.0.10 python  +glob2 0.7 python  +gmpy2 2.1.2 python  +gnupg 2.2.27-3ubuntu2.1 deb  +gnupg-l10n 2.2.27-3ubuntu2.1 deb  +gnupg-utils 2.2.27-3ubuntu2.1 deb  +gnupg2 2.2.27-3ubuntu2.1 deb  +googleapis-common-protos 1.59.1 python  +gpg 2.2.27-3ubuntu2.1 deb  +gpg-agent 2.2.27-3ubuntu2.1 deb  +gpg-wks-client 2.2.27-3ubuntu2.1 deb  +gpg-wks-server 2.2.27-3ubuntu2.1 deb  +gpgconf 2.2.27-3ubuntu2.1 deb  +gpgsm 2.2.27-3ubuntu2.1 deb  +gpgv 2.2.27-3ubuntu2.1 deb  +greenlet 2.0.2 python  +grep 3.7-1build1 deb  +grpcio 1.54.2 python  +grpclib 0.4.5 python  +gunicorn 20.1.0 python  +gzip 1.10-4ubuntu4.1 deb  +h2 4.1.0 python  +hostname 3.23ubuntu2 deb  +hpack 4.0.0 python  +html5lib 1.1 python  +hyperframe 6.0.1 python  +idna 3.4 python (+1 duplicate) +imagesize 1.4.1 python  +importlib-metadata 6.8.0 python  +importlib-resources 6.0.0 python (+1 duplicate) +init-system-helpers 1.62 deb  +ipykernel 6.25.0 python  +ipython 8.14.0 python  +ipywidgets 8.0.7 python  +isoduration 20.11.0 python  +itsdangerous 2.1.2 python  +jedi 0.18.2 python  +joblib 1.3.0 python (+1 duplicate) +jq 1.6-2.1ubuntu3 deb  +js 1.0.0 npm  +json5 0.9.14 python (+1 duplicate) +jsonpointer 2.4 python  +jsonschema 4.18.4 python (+1 duplicate) +jsonschema-specifications 2023.7.1 python (+1 duplicate) +jupyter-events 0.6.3 python  +jupyter-lsp 2.2.0 python  +jupyter_client 8.3.0 python  +jupyter_core 4.12.0 python  +jupyter_server 2.7.0 python  +jupyter_server_terminals 0.4.4 python  +jupyterlab 4.0.3 python  +jupyterlab-pygments 0.2.2 python  +jupyterlab-widgets 3.0.8 python  +jupyterlab_pygments 0.2.2 npm (+1 duplicate) +jupyterlab_server 2.24.0 python  +kiwisolver 1.4.4 python  +libacl1 2.3.1-1 deb  +libapt-pkg6.0 2.4.9 deb  +libarchive-c 5.0 python  +libasan6 11.3.0-1ubuntu1~22.04.1 deb  +libassuan0 2.5.5-1build1 deb  +libatomic1 12.1.0-2ubuntu1~22.04 deb  +libattr1 1:2.5.1-1build1 deb  +libaudit-common 1:3.0.7-1build1 deb  +libaudit1 1:3.0.7-1build1 deb  +libbinutils 2.38-4ubuntu2.2 deb  +libblkid1 2.37.2-4ubuntu3 deb  +libbrotli1 1.0.9-2build6 deb  +libbsd0 0.11.5-1 deb  +libbz2-1.0 1.0.8-5build1 deb  +libc-bin 2.35-0ubuntu3.1 deb  +libc-dev-bin 2.35-0ubuntu3.1 deb  +libc6 2.35-0ubuntu3.1 deb  +libc6-dev 2.35-0ubuntu3.1 deb  +libcap-ng0 0.7.9-2.2build3 deb  +libcap2 1:2.44-1ubuntu0.22.04.1 deb  +libcbor0.8 0.8.0-2ubuntu1 deb  +libcc1-0 12.1.0-2ubuntu1~22.04 deb  +libcom-err2 1.46.5-2ubuntu1.1 deb  +libcrypt-dev 1:4.4.27-1 deb  +libcrypt1 1:4.4.27-1 deb  +libctf-nobfd0 2.38-4ubuntu2.2 deb  +libctf0 2.38-4ubuntu2.2 deb  +libcublas-11-8 11.11.3.6-1 deb  +libcufft-11-8 10.9.0.58-1 deb  +libcurand-11-8 10.3.0.86-1 deb  +libcurl4 7.81.0-1ubuntu1.13 deb  +libcusolver-11-8 11.4.1.48-1 deb  +libdb5.3 5.3.28+dfsg1-0.8ubuntu3 deb  +libdebconfclient0 0.261ubuntu1 deb  +libdpkg-perl 1.21.1ubuntu2.2 deb  +libedit2 3.1-20210910-1build1 deb  +libext2fs2 1.46.5-2ubuntu1.1 deb  +libffi8 3.4.2-4 deb  +libfido2-1 1.10.0-1 deb  +libgcc-11-dev 11.3.0-1ubuntu1~22.04.1 deb  +libgcc-s1 12.1.0-2ubuntu1~22.04 deb  +libgcrypt20 1.9.4-3ubuntu3 deb  +libgdbm-compat4 1.23-1 deb  +libgdbm6 1.23-1 deb  +libglib2.0-0 2.72.4-0ubuntu2.2 deb  +libgmp10 2:6.2.1+dfsg-3ubuntu1 deb  +libgnutls30 3.7.3-4ubuntu1.2 deb  +libgomp1 12.1.0-2ubuntu1~22.04 deb  +libgpg-error0 1.43-3 deb  +libgssapi-krb5-2 1.19.2-2ubuntu0.2 deb  +libhogweed6 3.7.3-1build2 deb  +libidn2-0 2.3.2-2build1 deb  +libisl23 0.24-2build1 deb  +libitm1 12.1.0-2ubuntu1~22.04 deb  +libjq1 1.6-2.1ubuntu3 deb  +libk5crypto3 1.19.2-2ubuntu0.2 deb  +libkeyutils1 1.6.1-2ubuntu3 deb  +libkrb5-3 1.19.2-2ubuntu0.2 deb  +libkrb5support0 1.19.2-2ubuntu0.2 deb  +libksba8 1.6.0-2ubuntu0.2 deb  +libldap-2.5-0 2.5.14+dfsg-0ubuntu0.22.04.2 deb  +liblsan0 12.1.0-2ubuntu1~22.04 deb  +liblz4-1 1.9.3-2build2 deb  +liblzma5 5.2.5-2ubuntu1 deb  +libmambapy 1.4.2 python  +libmd0 1.0.4-1build1 deb  +libmount1 2.37.2-4ubuntu3 deb  +libmpc3 1.2.1-2build1 deb  +libmpfr6 4.1.0-3build3 deb  +libncurses6 6.3-2ubuntu0.1 deb  +libncursesw6 6.3-2ubuntu0.1 deb  +libnettle8 3.7.3-1build2 deb  +libnghttp2-14 1.43.0-1build3 deb  +libnpth0 1.6-3build2 deb  +libnsl-dev 1.3.0-2build2 deb  +libnsl2 1.3.0-2build2 deb  +libnuma1 2.0.14-3ubuntu2 deb  +libonig5 6.9.7.1-2build1 deb  +libp11-kit0 0.24.0-6build1 deb  +libpam-modules 1.4.0-11ubuntu2.3 deb  +libpam-modules-bin 1.4.0-11ubuntu2.3 deb  +libpam-runtime 1.4.0-11ubuntu2.3 deb  +libpam0g 1.4.0-11ubuntu2.3 deb  +libpcre2-8-0 10.39-3ubuntu0.1 deb  +libpcre3 2:8.39-13ubuntu0.22.04.1 deb  +libperl5.34 5.34.0-3ubuntu1.2 deb  +libprocps8 2:3.3.17-6ubuntu2 deb  +libpsl5 0.21.0-1.2build2 deb  +libquadmath0 12.1.0-2ubuntu1~22.04 deb  +libreadline8 8.1.2-1 deb  +librtmp1 2.4+20151223.gitfa8646d.1-2build4 deb  +libsasl2-2 2.1.27+dfsg2-3ubuntu1.2 deb  +libsasl2-modules-db 2.1.27+dfsg2-3ubuntu1.2 deb  +libseccomp2 2.5.3-2ubuntu2 deb  +libselinux1 3.3-1build2 deb  +libsemanage-common 3.3-1build2 deb  +libsemanage2 3.3-1build2 deb  +libsepol2 3.3-1build1 deb  +libsmartcols1 2.37.2-4ubuntu3 deb  +libsqlite3-0 3.37.2-2ubuntu0.1 deb  +libss2 1.46.5-2ubuntu1.1 deb  +libssh-4 0.9.6-2ubuntu0.22.04.1 deb  +libssl3 3.0.2-0ubuntu1.10 deb  +libstdc++-11-dev 11.3.0-1ubuntu1~22.04.1 deb  +libstdc++6 12.1.0-2ubuntu1~22.04 deb  +libsystemd0 249.11-0ubuntu3.9 deb  +libtasn1-6 4.18.0-4build1 deb  +libtinfo6 6.3-2ubuntu0.1 deb  +libtirpc-common 1.3.2-2ubuntu0.1 deb  +libtirpc-dev 1.3.2-2ubuntu0.1 deb  +libtirpc3 1.3.2-2ubuntu0.1 deb  +libtsan0 11.3.0-1ubuntu1~22.04.1 deb  +libubsan1 12.1.0-2ubuntu1~22.04 deb  +libudev1 249.11-0ubuntu3.9 deb  +libunistring2 1.0-1 deb  +libuuid1 2.37.2-4ubuntu3 deb  +libxxhash0 0.8.1-1 deb  +libzstd1 1.4.8+dfsg-3build1 deb  +linux-libc-dev 5.15.0-78.85 deb  +llvmlite 0.40.1 python  +locket 1.0.0 python  +login 1:4.8.1-2ubuntu2.1 deb  +logsave 1.46.5-2ubuntu1.1 deb  +lsb-base 11.1.0ubuntu4 deb  +lto-disabled-list 24 deb  +lz4 4.3.2 python  +make 4.3-4.1build1 deb  +mamba 1.4.2 python  +markdown-it-py 3.0.0 python  +matplotlib 3.7.1 python  +matplotlib-inline 0.1.6 python  +mawk 1.3.4.20200120-3 deb  +mdurl 0.1.0 python  +merlin-core 23.6.0 python  +merlin-dataloader 23.6.0 python  +mistune 3.0.1 python  +mlflow 2.5.0 python  +more-itertools 10.0.0 python  +morpheus 23.7.2 python  +mount 2.37.2-4ubuntu3 deb  +mpmath 1.3.0 python  +mrc 23.7.0 python  +msgpack 1.0.5 python  +multidict 6.0.4 python  +munkres 1.1.4 python  +nb-conda-kernels 2.3.1 python  +nbclient 0.8.0 python  +nbconvert 7.7.3 python  +nbformat 5.9.1 python  +ncurses-base 6.3-2ubuntu0.1 deb  +ncurses-bin 6.3-2ubuntu0.1 deb  +nest-asyncio 1.5.7 python  +networkx 3.1 python  +notebook_shim 0.2.3 python  +npy-append-array 0.9.16 python  +numba 0.57.1 python  +numpy 1.24.4 python  +numpydoc 1.4.0 python  +nvidia-pyindex 1.0.9 python  +nvtabular 23.6.0 python  +nvtx 0.2.5 python  +oauthlib 3.2.2 python  +openssh-client 1:8.9p1-3ubuntu0.3 deb  +openssl 3.0.2-0ubuntu1.10 deb  +openssl 3.1.1 binary  +ordered-set 4.1.0 python  +overrides 7.3.1 python  +packaging 23.1 python (+1 duplicate) +pandas 1.3.5 python  +pandocfilters 1.5.0 python  +parso 0.8.3 python  +partd 1.4.0 python  +passwd 1:4.8.1-2ubuntu2.1 deb  +patch 2.7.6-7build2 deb  +perl 5.34.0-3ubuntu1.2 deb  +perl-base 5.34.0-3ubuntu1.2 deb  +perl-modules-5.34 5.34.0-3ubuntu1.2 deb  +pexpect 4.8.0 python  +pickleshare 0.7.5 python  +pinentry-curses 1.1.1-1build2 deb  +pip 23.2.1 python  +pkg-config 0.29.2-1ubuntu3 deb  +pkginfo 1.9.6 python  +pkgutil_resolve_name 1.3.10 python  +platformdirs 3.9.1 python  +pluggy 1.0.0 python  +pluggy 1.2.0 python  +pooch 1.7.0 python  +procps 2:3.3.17-6ubuntu2 deb  +prometheus-client 0.17.1 python  +prometheus-flask-exporter 0.22.4 python  +prompt-toolkit 3.0.39 python (+1 duplicate) +protobuf 4.21.12 python  +psutil 5.9.5 python (+1 duplicate) +ptxcompiler 0.8.1 python  +ptyprocess 0.7.0 python  +pure-eval 0.2.2 python  +pyOpenSSL 23.2.0 python (+1 duplicate) +pyarrow 11.0.0 python  +pycosat 0.6.4 python  +pycparser 2.21 python (+1 duplicate) +pynvml 11.4.1 python  +pyparsing 3.1.0 python  +python 3.10.12 binary  +python-dateutil 2.8.2 python  +python-json-logger 2.0.7 python  +python-rapidjson 1.10 python  +pytz 2023.3 python (+1 duplicate) +pyzmq 25.1.0 python  +querystring-parser 1.2.4 python  +readline-common 8.1.2-1 deb  +referencing 0.30.0 python (+1 duplicate) +requests 2.31.0 python (+1 duplicate) +rfc3339-validator 0.1.4 python  +rfc3986-validator 0.1.1 python  +rich 13.4.2 python  +rmm 23.6.0 python  +rpcsvc-proto 1.4.2-0ubuntu6 deb  +rpds-py 0.9.2 python (+1 duplicate) +ruamel.yaml 0.17.32 python  +ruamel.yaml.clib 0.2.7 python  +scikit-learn 1.2.2 python  +scipy 1.11.1 python  +sed 4.8-1ubuntu2 deb  +sensible-utils 0.0.17 deb  +setuptools 59.8.0 python  +setuptools 68.0.0 python  +six 1.16.0 python (+1 duplicate) +smmap 3.0.5 python  +sniffio 1.3.0 python (+1 duplicate) +snowballstemmer 2.2.0 python  +sortedcontainers 2.4.0 python  +soupsieve 2.3.2.post1 python  +soupsieve 2.4.1 python  +sphinxcontrib-applehelp 1.0.4 python  +sphinxcontrib-devhelp 1.0.2 python  +sphinxcontrib-htmlhelp 2.0.1 python  +sphinxcontrib-jsmath 1.0.1 python  +sphinxcontrib-qthelp 1.0.3 python  +sphinxcontrib-serializinghtml 1.1.5 python  +sqlparse 0.4.4 python  +stack-data 0.6.2 python  +stringcase 1.2.0 python  +sympy 1.12 python  +sysvinit-utils 3.01-1ubuntu1 deb  +tabulate 0.9.0 python  +tar 1.34+dfsg-1ubuntu0.1.22.04.1 deb  +tblib 1.7.0 python  +tensorflow-metadata 1.13.1 python  +terminado 0.17.1 python  +test-hyphens 3.0.2 npm  +test-hyphens-underscore 3.0.2 npm  +test_no_hyphens 3.0.2 npm  +threadpoolctl 3.2.0 python  +tinycss2 1.2.1 python  +tomli 2.0.1 python (+1 duplicate) +toolz 0.12.0 python (+1 duplicate) +torch 2.0.1 python  +tornado 6.3.2 python  +tqdm 4.65.0 python (+1 duplicate) +traitlets 5.9.0 python  +triton 2.0.0 python  +tritonclient 2.26.0 python  +typing-utils 0.1.0 python  +typing_extensions 4.7.1 python (+1 duplicate) +ubuntu-keyring 2021.03.26 deb  +unicodedata2 15.0.0 python  +unzip 6.0-26ubuntu3.1 deb  +uri-template 1.3.0 python  +urllib3 2.0.3 python  +urllib3 2.0.4 python  +usrmerge 25ubuntu2 deb  +util-linux 2.37.2-4ubuntu3 deb  +watchdog 2.1.9 python  +watchgod 0.8.2 python  +wcwidth 0.2.6 python (+1 duplicate) +webcolors 1.13 python  +webencodings 0.5.1 python  +websocket-client 1.6.1 python  +websockets 11.0.3 python  +wget 1.21.2-2ubuntu1 deb  +wheel 0.40.0 python  +wheel 0.41.0 python  +widgetsnbextension 4.0.8 python  +xz-utils 5.2.5-2ubuntu1 deb  +zict 3.0.0 python  +zip 3.0-12build2 deb  +zipp 3.16.2 python (+1 duplicate) +zlib1g 1:1.2.11.dfsg-2ubuntu9.2 deb  +zstandard 0.19.0 python  diff --git a/data/sboms/nvcr.io/nvidia/morpheus/morpheus:23.11-runtime.sbom b/data/sboms/nvcr.io/nvidia/morpheus/morpheus:23.11-runtime.sbom new file mode 100644 index 0000000..cd469cc --- /dev/null +++ b/data/sboms/nvcr.io/nvidia/morpheus/morpheus:23.11-runtime.sbom @@ -0,0 +1,465 @@ +NAME VERSION TYPE +Babel 2.13.1 python  +Brotli 1.0.9 python  +Brotli 1.1.0 python  +ConfigArgParse 1.5.5 python  +Flask 3.0.0 python  +GitPython 3.1.40 python  +Jinja2 3.1.2 python (+1 duplicate) +Mako 1.3.0 python  +Markdown 3.5.1 python  +MarkupSafe 2.1.3 python (+1 duplicate) +Pillow 10.1.0 python  +PyJWT 2.8.0 python  +PySocks 1.7.1 python (+1 duplicate) +PyYAML 6.0.1 python (+1 duplicate) +Pygments 2.17.2 python (+1 duplicate) +SQLAlchemy 2.0.0 python  +SciPy 1.11.4 python  +Simple Launcher 1.1.0.14 dotnet (+5 duplicates) +Sphinx 7.2.6 python  +Werkzeug 3.0.1 python  +absl-py 1.4.0 python  +adduser 3.118ubuntu5 deb  +alabaster 0.7.13 python  +alembic 1.13.0 python  +anyio 3.7.1 python  +appdirs 1.4.4 python  +apt 2.4.11 deb  +asn1crypto 1.5.1 python  +attrs 23.1.0 python (+1 duplicate) +base-files 12ubuntu4.4 deb  +base-passwd 3.5.52build1 deb  +bash 5.1-6ubuntu1 deb  +bc 1.07.1-3build1 deb  +beautifulsoup4 4.12.2 python  +betterproto 1.2.5 python  +binutils 2.38-4ubuntu2.3 deb  +binutils-common 2.38-4ubuntu2.3 deb  +binutils-x86-64-linux-gnu 2.38-4ubuntu2.3 deb  +blinker 1.7.0 python  +boa 0.16.0 python  +bokeh 2.4.3 python  +boltons 23.0.0 python  +bsdutils 1:2.37.2-4ubuntu3 deb  +build-essential 12.9ubuntu3 deb  +bzip2 1.0.8-5build1 deb  +ca-certificates 20230311ubuntu0.22.04.1 deb  +ca-certificates-java java-archive  +ca-certificates-java 20190909ubuntu1.2 deb  +cachetools 5.3.2 python  +cattrs 23.2.3 python  +certifi 2023.11.17 python (+1 duplicate) +cffi 1.15.1 python  +cffi 1.16.0 python  +chardet 5.2.0 python  +charset-normalizer 3.2.0 python  +charset-normalizer 3.3.2 python  +click 8.1.7 python (+1 duplicate) +cloudpickle 3.0.0 python  +colorama 0.4.6 python (+1 duplicate) +conda 23.3.1 python  +conda-build 3.28.1 python  +conda-libmamba-solver 23.3.0 python  +conda-package-handling 2.2.0 python  +conda_index 0.3.0 python  +conda_package_streaming 0.9.0 python  +configparser 5.3.0 python  +confluent-kafka 1.9.2 python  +contourpy 1.2.0 python  +coreutils 8.32-4.1ubuntu1 deb  +cpp 4:11.2.0-1ubuntu1 deb  +cpp-11 11.4.0-1ubuntu1~22.04 deb  +cryptography 41.0.3 python  +cryptography 41.0.7 python  +cubinlinker 0.3.0 python  +cuda-cccl-11-8 11.8.89-1 deb  +cuda-compat-11-8 520.61.05-1 deb  +cuda-compiler-11-8 11.8.0-1 deb  +cuda-cudart-11-8 11.8.89-1 deb  +cuda-cudart-dev-11-8 11.8.89-1 deb  +cuda-cuobjdump-11-8 11.8.86-1 deb  +cuda-cupti-11-8 11.8.87-1 deb  +cuda-cuxxfilt-11-8 11.8.86-1 deb  +cuda-driver-dev-11-8 11.8.89-1 deb  +cuda-gdb-11-8 11.8.86-1 deb  +cuda-keyring 1.1-1 deb  +cuda-nvcc-11-8 11.8.89-1 deb  +cuda-nvdisasm-11-8 11.8.86-1 deb  +cuda-nvprune-11-8 11.8.86-1 deb  +cuda-python 11.8.3 python  +cuda-toolkit-11-8-config-common 11.8.89-1 deb  +cuda-toolkit-11-config-common 11.8.89-1 deb  +cuda-toolkit-config-common 12.3.101-1 deb  +cudf 23.6.1 python  +cudf-kafka 23.6.1 python  +cupy 12.2.0 python  +curl 7.81.0-1ubuntu1.15 deb  +cycler 0.12.1 python  +cytoolz 0.12.2 python  +dash 0.5.11+git20210903+057cd650a4ed-3build1 deb  +dask 2023.3.2 python  +dask-cuda 23.6.0 python  +dask-cudf 23.6.1 python  +databricks-cli 0.18.0 python  +dataclasses 0.8 python  +datacompy 0.8.4 python  +debconf 1.5.79ubuntu1 deb  +debianutils 5.5-1ubuntu2 deb  +diffutils 1:3.8-0ubuntu2 deb  +dill 0.3.7 python  +dirmngr 2.2.27-3ubuntu2.1 deb  +distributed 2023.3.2.1 python  +docker 5.0.3 python  +docker-pycreds 0.4.0 python  +docutils 0.20.1 python  +dpkg 1.21.1ubuntu2.2 deb  +dpkg-dev 1.21.1ubuntu2.2 deb  +e2fsprogs 1.46.5-2ubuntu1.1 deb  +entrypoints 0.4 python  +exceptiongroup 1.2.0 python (+1 duplicate) +fastrlock 0.8.2 python  +filelock 3.13.1 python (+1 duplicate) +findutils 4.8.0-1ubuntu3 deb  +fontconfig-config 2.13.1-4.2ubuntu5 deb  +fonts-dejavu-core 2.37-2build1 deb  +fonttools 4.46.0 python  +fsspec 2023.12.1 python  +g++ 4:11.2.0-1ubuntu1 deb  +g++-11 11.4.0-1ubuntu1~22.04 deb  +gcc 4:11.2.0-1ubuntu1 deb  +gcc-11 11.4.0-1ubuntu1~22.04 deb  +gcc-11-base 11.4.0-1ubuntu1~22.04 deb  +gcc-12-base 12.3.0-1ubuntu1~22.04 deb  +gitdb 4.0.11 python  +gmpy2 2.1.2 python  +gnupg 2.2.27-3ubuntu2.1 deb  +gnupg-l10n 2.2.27-3ubuntu2.1 deb  +gnupg-utils 2.2.27-3ubuntu2.1 deb  +gnupg2 2.2.27-3ubuntu2.1 deb  +googleapis-common-protos 1.61.0 python  +gpg 2.2.27-3ubuntu2.1 deb  +gpg-agent 2.2.27-3ubuntu2.1 deb  +gpg-wks-client 2.2.27-3ubuntu2.1 deb  +gpg-wks-server 2.2.27-3ubuntu2.1 deb  +gpgconf 2.2.27-3ubuntu2.1 deb  +gpgsm 2.2.27-3ubuntu2.1 deb  +gpgv 2.2.27-3ubuntu2.1 deb  +greenlet 3.0.1 python  +grep 3.7-1build1 deb  +grpcio 1.54.2 python  +grpclib 0.4.6 python  +gunicorn 21.2.0 python  +gzip 1.10-4ubuntu4.1 deb  +h2 4.1.0 python  +hostname 3.23ubuntu2 deb  +hpack 4.0.0 python  +hyperframe 6.0.1 python  +idna 3.4 python  +idna 3.6 python  +imagesize 1.4.1 python  +importlib-metadata 7.0.0 python  +importlib-resources 6.1.1 python (+1 duplicate) +init-system-helpers 1.62 deb  +itsdangerous 2.1.2 python  +java-common 0.72build2 deb  +joblib 1.3.2 python (+1 duplicate) +jq 1.6-2.1ubuntu3 deb  +jrt-fs 11.0.21 java-archive  +js 1.0.0 npm  +json5 0.9.14 python  +jsonpatch 1.32 python  +jsonpointer 2.0 python  +jsonschema 4.20.0 python  +jsonschema-specifications 2023.11.2 python  +jupyter_client 8.6.0 python  +jupyter_core 5.5.0 python  +kiwisolver 1.4.5 python  +libacl1 2.3.1-1 deb  +libapt-pkg6.0 2.4.11 deb  +libarchive-c 5.0 python  +libasan6 11.4.0-1ubuntu1~22.04 deb  +libasound2 1.2.6.1-1ubuntu1 deb  +libasound2-data 1.2.6.1-1ubuntu1 deb  +libassuan0 2.5.5-1build1 deb  +libatomic1 12.3.0-1ubuntu1~22.04 deb  +libattr1 1:2.5.1-1build1 deb  +libaudit-common 1:3.0.7-1build1 deb  +libaudit1 1:3.0.7-1build1 deb  +libavahi-client3 0.8-5ubuntu5.2 deb  +libavahi-common-data 0.8-5ubuntu5.2 deb  +libavahi-common3 0.8-5ubuntu5.2 deb  +libbinutils 2.38-4ubuntu2.3 deb  +libblkid1 2.37.2-4ubuntu3 deb  +libbrotli1 1.0.9-2build6 deb  +libbsd0 0.11.5-1 deb  +libbz2-1.0 1.0.8-5build1 deb  +libc-bin 2.35-0ubuntu3.5 deb  +libc-dev-bin 2.35-0ubuntu3.5 deb  +libc6 2.35-0ubuntu3.5 deb  +libc6-dev 2.35-0ubuntu3.5 deb  +libcap-ng0 0.7.9-2.2build3 deb  +libcap2 1:2.44-1ubuntu0.22.04.1 deb  +libcbor0.8 0.8.0-2ubuntu1 deb  +libcc1-0 12.3.0-1ubuntu1~22.04 deb  +libcom-err2 1.46.5-2ubuntu1.1 deb  +libcrypt-dev 1:4.4.27-1 deb  +libcrypt1 1:4.4.27-1 deb  +libctf-nobfd0 2.38-4ubuntu2.3 deb  +libctf0 2.38-4ubuntu2.3 deb  +libcublas-11-8 11.11.3.6-1 deb  +libcufft-11-8 10.9.0.58-1 deb  +libcups2 2.4.1op1-1ubuntu4.7 deb  +libcurand-11-8 10.3.0.86-1 deb  +libcurl4 7.81.0-1ubuntu1.15 deb  +libcusolver-11-8 11.4.1.48-1 deb  +libdb5.3 5.3.28+dfsg1-0.8ubuntu3 deb  +libdbus-1-3 1.12.20-2ubuntu4.1 deb  +libdebconfclient0 0.261ubuntu1 deb  +libdpkg-perl 1.21.1ubuntu2.2 deb  +libedit2 3.1-20210910-1build1 deb  +libexpat1 2.4.7-1ubuntu0.2 deb  +libext2fs2 1.46.5-2ubuntu1.1 deb  +libffi8 3.4.2-4 deb  +libfido2-1 1.10.0-1 deb  +libfontconfig1 2.13.1-4.2ubuntu5 deb  +libfreetype6 2.11.1+dfsg-1ubuntu0.2 deb  +libgcc-11-dev 11.4.0-1ubuntu1~22.04 deb  +libgcc-s1 12.3.0-1ubuntu1~22.04 deb  +libgcrypt20 1.9.4-3ubuntu3 deb  +libgdbm-compat4 1.23-1 deb  +libgdbm6 1.23-1 deb  +libglib2.0-0 2.72.4-0ubuntu2.2 deb  +libgmp10 2:6.2.1+dfsg-3ubuntu1 deb  +libgnutls30 3.7.3-4ubuntu1.3 deb  +libgomp1 12.3.0-1ubuntu1~22.04 deb  +libgpg-error0 1.43-3 deb  +libgraphite2-3 1.3.14-1build2 deb  +libgssapi-krb5-2 1.19.2-2ubuntu0.3 deb  +libharfbuzz0b 2.7.4-1ubuntu3.1 deb  +libhogweed6 3.7.3-1build2 deb  +libidn2-0 2.3.2-2build1 deb  +libisl23 0.24-2build1 deb  +libitm1 12.3.0-1ubuntu1~22.04 deb  +libjpeg-turbo8 2.1.2-0ubuntu1 deb  +libjpeg8 8c-2ubuntu10 deb  +libjq1 1.6-2.1ubuntu3 deb  +libk5crypto3 1.19.2-2ubuntu0.3 deb  +libkeyutils1 1.6.1-2ubuntu3 deb  +libkrb5-3 1.19.2-2ubuntu0.3 deb  +libkrb5support0 1.19.2-2ubuntu0.3 deb  +libksba8 1.6.0-2ubuntu0.2 deb  +liblcms2-2 2.12~rc1-2build2 deb  +libldap-2.5-0 2.5.16+dfsg-0ubuntu0.22.04.1 deb  +liblsan0 12.3.0-1ubuntu1~22.04 deb  +liblz4-1 1.9.3-2build2 deb  +liblzma5 5.2.5-2ubuntu1 deb  +libmambapy 1.5.0 python  +libmd0 1.0.4-1build1 deb  +libmount1 2.37.2-4ubuntu3 deb  +libmpc3 1.2.1-2build1 deb  +libmpfr6 4.1.0-3build3 deb  +libncurses6 6.3-2ubuntu0.1 deb  +libncursesw6 6.3-2ubuntu0.1 deb  +libnettle8 3.7.3-1build2 deb  +libnghttp2-14 1.43.0-1ubuntu0.1 deb  +libnpth0 1.6-3build2 deb  +libnsl-dev 1.3.0-2build2 deb  +libnsl2 1.3.0-2build2 deb  +libnspr4 2:4.32-3build1 deb  +libnss3 2:3.68.2-0ubuntu1.2 deb  +libnuma1 2.0.14-3ubuntu2 deb  +libonig5 6.9.7.1-2build1 deb  +libp11-kit0 0.24.0-6build1 deb  +libpam-modules 1.4.0-11ubuntu2.3 deb  +libpam-modules-bin 1.4.0-11ubuntu2.3 deb  +libpam-runtime 1.4.0-11ubuntu2.3 deb  +libpam0g 1.4.0-11ubuntu2.3 deb  +libpcre2-8-0 10.39-3ubuntu0.1 deb  +libpcre3 2:8.39-13ubuntu0.22.04.1 deb  +libpcsclite1 1.9.5-3ubuntu1 deb  +libperl5.34 5.34.0-3ubuntu1.3 deb  +libpng16-16 1.6.37-3build5 deb  +libprocps8 2:3.3.17-6ubuntu2.1 deb  +libpsl5 0.21.0-1.2build2 deb  +libquadmath0 12.3.0-1ubuntu1~22.04 deb  +libreadline8 8.1.2-1 deb  +librtmp1 2.4+20151223.gitfa8646d.1-2build4 deb  +libsasl2-2 2.1.27+dfsg2-3ubuntu1.2 deb  +libsasl2-modules-db 2.1.27+dfsg2-3ubuntu1.2 deb  +libseccomp2 2.5.3-2ubuntu2 deb  +libselinux1 3.3-1build2 deb  +libsemanage-common 3.3-1build2 deb  +libsemanage2 3.3-1build2 deb  +libsepol2 3.3-1build1 deb  +libsmartcols1 2.37.2-4ubuntu3 deb  +libsqlite3-0 3.37.2-2ubuntu0.1 deb  +libss2 1.46.5-2ubuntu1.1 deb  +libssh-4 0.9.6-2ubuntu0.22.04.1 deb  +libssl3 3.0.2-0ubuntu1.12 deb  +libstdc++-11-dev 11.4.0-1ubuntu1~22.04 deb  +libstdc++6 12.3.0-1ubuntu1~22.04 deb  +libsystemd0 249.11-0ubuntu3.11 deb  +libtasn1-6 4.18.0-4build1 deb  +libtinfo6 6.3-2ubuntu0.1 deb  +libtirpc-common 1.3.2-2ubuntu0.1 deb  +libtirpc-dev 1.3.2-2ubuntu0.1 deb  +libtirpc3 1.3.2-2ubuntu0.1 deb  +libtsan0 11.4.0-1ubuntu1~22.04 deb  +libubsan1 12.3.0-1ubuntu1~22.04 deb  +libudev1 249.11-0ubuntu3.11 deb  +libunistring2 1.0-1 deb  +libuuid1 2.37.2-4ubuntu3 deb  +libxxhash0 0.8.1-1 deb  +libzstd1 1.4.8+dfsg-3build1 deb  +linux-libc-dev 5.15.0-89.99 deb  +llvmlite 0.41.1 python  +locket 1.0.0 python  +login 1:4.8.1-2ubuntu2.1 deb  +logsave 1.46.5-2ubuntu1.1 deb  +lsb-base 11.1.0ubuntu4 deb  +lto-disabled-list 24 deb  +lz4 4.3.2 python  +make 4.3-4.1build1 deb  +mamba 1.5.0 python  +markdown-it-py 3.0.0 python  +matplotlib 3.8.2 python  +mawk 1.3.4.20200120-3 deb  +mdurl 0.1.0 python  +menuinst 2.0.0 python  +merlin-core 23.6.0 python  +merlin-dataloader 23.6.0 python  +mlflow 2.9.1 python  +more-itertools 10.1.0 python  +morpheus 23.11.1 python  +mount 2.37.2-4ubuntu3 deb  +mpmath 1.3.0 python  +mrc 23.11.0 python  +msgpack 1.0.7 python  +multidict 6.0.4 python  +munkres 1.1.4 python  +nb-conda-kernels 2.3.1 python  +ncurses-base 6.3-2ubuntu0.1 deb  +ncurses-bin 6.3-2ubuntu0.1 deb  +networkx 3.2 python  +npy-append-array 0.9.16 python  +numba 0.58.1 python  +numpy 1.26.2 python  +numpydoc 1.4.0 python  +nvtabular 23.6.0 python  +nvtx 0.2.8 python  +oauthlib 3.2.2 python  +openjdk-11-jre-headless 11.0.21+9-0ubuntu1~22.04 deb  +openssh-client 1:8.9p1-3ubuntu0.4 deb  +openssl 3.0.2-0ubuntu1.12 deb  +openssl 3.2.0 binary  +ordered-set 4.1.0 python  +packaging 23.1 python  +packaging 23.2 python  +pandas 1.3.5 python  +partd 1.4.1 python  +passwd 1:4.8.1-2ubuntu2.1 deb  +patch 2.7.6-7build2 deb  +perl 5.34.0-3ubuntu1.3 deb  +perl-base 5.34.0-3ubuntu1.3 deb  +perl-modules-5.34 5.34.0-3ubuntu1.3 deb  +pinentry-curses 1.1.1-1build2 deb  +pip 23.3.1 python  +pkg-config 0.29.2-1ubuntu3 deb  +pkginfo 1.9.6 python  +pkgutil_resolve_name 1.3.10 python  +platformdirs 4.1.0 python  +pluggy 1.0.0 python  +pluggy 1.3.0 python  +procps 2:3.3.17-6ubuntu2.1 deb  +prometheus-client 0.19.0 python  +prometheus-flask-exporter 0.23.0 python  +prompt-toolkit 3.0.41 python  +protobuf 4.21.12 python  +psutil 5.9.5 python (+1 duplicate) +ptxcompiler 0.8.1 python  +pyOpenSSL 23.2.0 python  +pyOpenSSL 23.3.0 python  +pyarrow 11.0.0 python  +pyarrow-hotfix 0.6 python  +pycosat 0.6.4 python  +pycparser 2.21 python (+1 duplicate) +pynvml 11.4.1 python  +pyparsing 3.1.1 python  +python 3.10.12 binary  +python 3.10.13 binary  +python-dateutil 2.8.2 python  +python-rapidjson 1.13 python  +pytz 2023.3.post1 python (+1 duplicate) +pyzmq 25.1.2 python  +querystring-parser 1.2.4 python  +readline-common 8.1.2-1 deb  +referencing 0.31.1 python  +requests 2.31.0 python (+1 duplicate) +requests-cache 1.1.1 python  +rich 13.7.0 python  +rmm 23.6.0 python  +rpcsvc-proto 1.4.2-0ubuntu6 deb  +rpds-py 0.13.2 python  +ruamel.yaml 0.17.32 python  +ruamel.yaml.clib 0.2.7 python  +scikit-learn 1.2.2 python  +sed 4.8-1ubuntu2 deb  +sensible-utils 0.0.17 deb  +setuptools 59.8.0 python  +setuptools 68.1.2 python  +six 1.16.0 python  +smmap 5.0.0 python  +sniffio 1.3.0 python  +snowballstemmer 2.2.0 python  +sortedcontainers 2.4.0 python  +soupsieve 2.5 python  +sphinxcontrib-applehelp 1.0.7 python  +sphinxcontrib-devhelp 1.0.5 python  +sphinxcontrib-htmlhelp 2.0.4 python  +sphinxcontrib-jsmath 1.0.1 python  +sphinxcontrib-qthelp 1.0.6 python  +sphinxcontrib-serializinghtml 1.1.9 python  +sqlparse 0.4.4 python  +stringcase 1.2.0 python  +sympy 1.12 python  +sysvinit-utils 3.01-1ubuntu1 deb  +tabulate 0.9.0 python  +tar 1.34+dfsg-1ubuntu0.1.22.04.1 deb  +tblib 2.0.0 python  +tensorflow-metadata 1.13.1 python  +threadpoolctl 3.2.0 python  +tomli 2.0.1 python  +toolz 0.12.0 python (+1 duplicate) +torch 2.0.1 python  +tornado 6.3.3 python  +tqdm 4.66.1 python (+1 duplicate) +traitlets 5.14.0 python  +triton 2.0.0 python  +tritonclient 2.26.0 python  +typing-utils 0.1.0 python  +typing_extensions 4.8.0 python (+1 duplicate) +ubuntu-keyring 2021.03.26 deb  +ucf 3.0043 deb  +ujson 5.8.0 python  +unicodedata2 15.1.0 python  +unzip 6.0-26ubuntu3.1 deb  +url-normalize 1.4.3 python  +urllib3 2.0.4 python  +urllib3 2.1.0 python  +usrmerge 25ubuntu2 deb  +util-linux 2.37.2-4ubuntu3 deb  +watchdog 2.1.9 python  +watchgod 0.8.2 python  +wcwidth 0.2.12 python  +websocket-client 1.7.0 python  +websockets 12.0 python  +wget 1.21.2-2ubuntu1 deb  +wheel 0.41.2 python  +wheel 0.42.0 python  +xz-utils 5.2.5-2ubuntu1 deb  +zict 3.0.0 python  +zip 3.0-12build2 deb  +zipp 3.17.0 python (+1 duplicate) +zlib1g 1:1.2.11.dfsg-2ubuntu9.2 deb  +zstandard 0.19.0 python  diff --git a/data/sboms/nvcr.io/nvidia/morpheus/morpheus:24.03-runtime.sbom b/data/sboms/nvcr.io/nvidia/morpheus/morpheus:24.03-runtime.sbom new file mode 100644 index 0000000..ceef72f --- /dev/null +++ b/data/sboms/nvcr.io/nvidia/morpheus/morpheus:24.03-runtime.sbom @@ -0,0 +1,493 @@ +NAME VERSION TYPE +Babel 2.14.0 python  +Brotli 1.1.0 python (+1 duplicate) +ConfigArgParse 1.5.5 python  +Flask 3.0.3 python  +GitPython 3.1.43 python  +Jinja2 3.1.3 python (+1 duplicate) +Mako 1.3.3 python  +Markdown 3.6 python  +MarkupSafe 2.1.5 python (+1 duplicate) +PyJWT 2.8.0 python  +PySocks 1.7.1 python (+1 duplicate) +PyYAML 6.0.1 python (+1 duplicate) +Pygments 2.17.2 python (+1 duplicate) +SQLAlchemy 1.4.49 python  +Simple Launcher 1.1.0.14 dotnet (+5 duplicates) +Sphinx 7.3.7 python  +Werkzeug 3.0.2 python  +absl-py 1.4.0 python  +adagio 0.2.4 python  +adduser 3.118ubuntu5 deb  +alabaster 0.7.16 python  +alembic 1.13.1 python  +aniso8601 9.0.1 python  +annotated-types 0.6.0 python  +antlr4-python3-runtime 4.11.1 python  +anyio 3.7.1 python  +appdirs 1.4.4 python  +apt 2.4.12 deb  +archspec 0.2.3 python  +argon2-cffi 23.1.0 python  +argon2-cffi-bindings 21.2.0 python  +asn1crypto 1.5.1 python  +attrs 23.2.0 python (+1 duplicate) +base-files 12ubuntu4.6 deb  +base-passwd 3.5.52build1 deb  +bash 5.1-6ubuntu1.1 deb  +bc 1.07.1-3build1 deb  +beautifulsoup4 4.12.3 python  +betterproto 1.2.5 python  +binutils 2.38-4ubuntu2.6 deb  +binutils-common 2.38-4ubuntu2.6 deb  +binutils-x86-64-linux-gnu 2.38-4ubuntu2.6 deb  +blinker 1.7.0 python  +boa 0.17.0 python  +bokeh 3.4.1 python  +boltons 24.0.0 python  +bsdutils 1:2.37.2-4ubuntu3.4 deb  +build-essential 12.9ubuntu3 deb  +bzip2 1.0.8-5build1 deb  +ca-certificates 20230311ubuntu0.22.04.1 deb  +ca-certificates-java java-archive  +ca-certificates-java 20190909ubuntu1.2 deb  +cachetools 5.3.3 python  +cattrs 23.2.3 python  +certifi 2024.2.2 python (+1 duplicate) +cffi 1.16.0 python (+1 duplicate) +chardet 5.2.0 python  +charset-normalizer 3.3.2 python (+1 duplicate) +click 8.1.7 python (+1 duplicate) +cloudpickle 3.0.0 python  +colorama 0.4.6 python (+1 duplicate) +conda 24.3.0 python  +conda-build 24.3.0 python  +conda-libmamba-solver 24.1.0 python  +conda-package-handling 2.2.0 python  +conda_index 0.4.0 python  +conda_package_streaming 0.9.0 python  +confluent-kafka 1.9.2 python  +contourpy 1.2.1 python  +coreutils 8.32-4.1ubuntu1.2 deb  +cpp 4:11.2.0-1ubuntu1 deb  +cpp-11 11.4.0-1ubuntu1~22.04 deb  +cryptography 42.0.5 python  +cuda-cccl-12-1 12.1.109-1 deb  +cuda-compat-12-1 530.30.02-1 deb  +cuda-compiler-12-1 12.1.1-1 deb  +cuda-cudart-12-1 12.1.105-1 deb  +cuda-cudart-dev-12-1 12.1.105-1 deb  +cuda-cuobjdump-12-1 12.1.111-1 deb  +cuda-cupti-12-1 12.1.105-1 deb  +cuda-cuxxfilt-12-1 12.1.105-1 deb  +cuda-driver-dev-12-1 12.1.105-1 deb  +cuda-gdb-12-1 12.1.105-1 deb  +cuda-keyring 1.1-1 deb  +cuda-nvcc-12-1 12.1.105-1 deb  +cuda-nvdisasm-12-1 12.1.105-1 deb  +cuda-nvprune-12-1 12.1.105-1 deb  +cuda-python 12.4.0 python  +cuda-toolkit-12-1-config-common 12.1.105-1 deb  +cuda-toolkit-12-config-common 12.4.127-1 deb  +cuda-toolkit-config-common 12.4.127-1 deb  +cudf 24.2.2 python  +cupy 13.1.0 python  +curl 7.81.0-1ubuntu1.16 deb  +cycler 0.12.1 python  +cytoolz 0.12.3 python  +dash 0.5.11+git20210903+057cd650a4ed-3build1 deb  +dask 2024.1.1 python  +dask-cuda 24.2.0 python  +dask-cudf 24.2.2 python  +databricks-cli 0.18.0 python  +databricks-connect 14.3.1 python  +databricks-sdk 0.26.0 python  +dataclasses 0.8 python  +datacompy 0.10.5 python  +debconf 1.5.79ubuntu1 deb  +debianutils 5.5-1ubuntu2 deb  +diffutils 1:3.8-0ubuntu2 deb  +dill 0.3.7 python  +dirmngr 2.2.27-3ubuntu2.1 deb  +distributed 2024.1.1 python  +distro 1.9.0 python  +docker 5.0.3 python  +docker-pycreds 0.4.0 python  +docutils 0.21.2 python  +dpkg 1.21.1ubuntu2.3 deb  +dpkg-dev 1.21.1ubuntu2.3 deb  +e2fsprogs 1.46.5-2ubuntu1.1 deb  +elastic-transport 8.13.0 python  +elasticsearch 8.9.0 python  +entrypoints 0.4 python  +environs 9.5.0 python  +exceptiongroup 1.2.0 python (+1 duplicate) +fastrlock 0.8.2 python  +feedparser 6.0.10 python  +filelock 3.13.4 python (+1 duplicate) +findutils 4.8.0-1ubuntu3 deb  +fontconfig-config 2.13.1-4.2ubuntu5 deb  +fonts-dejavu-core 2.37-2build1 deb  +fonttools 4.51.0 python  +fs 2.4.16 python  +fsspec 2024.3.1 python  +fugue 0.8.7 python  +fugue-sql-antlr 0.2.0 python  +g++ 4:11.2.0-1ubuntu1 deb  +g++-11 11.4.0-1ubuntu1~22.04 deb  +gcc 4:11.2.0-1ubuntu1 deb  +gcc-11 11.4.0-1ubuntu1~22.04 deb  +gcc-11-base 11.4.0-1ubuntu1~22.04 deb  +gcc-12-base 12.3.0-1ubuntu1~22.04 deb  +gitdb 4.0.11 python  +gmpy2 2.1.5 python  +gnupg 2.2.27-3ubuntu2.1 deb  +gnupg-l10n 2.2.27-3ubuntu2.1 deb  +gnupg-utils 2.2.27-3ubuntu2.1 deb  +gnupg2 2.2.27-3ubuntu2.1 deb  +google-auth 2.29.0 python  +googleapis-common-protos 1.63.0 python  +gpg 2.2.27-3ubuntu2.1 deb  +gpg-agent 2.2.27-3ubuntu2.1 deb  +gpg-wks-client 2.2.27-3ubuntu2.1 deb  +gpg-wks-server 2.2.27-3ubuntu2.1 deb  +gpgconf 2.2.27-3ubuntu2.1 deb  +gpgsm 2.2.27-3ubuntu2.1 deb  +gpgv 2.2.27-3ubuntu2.1 deb  +graphene 3.3 python  +graphql-core 3.2.3 python  +graphql-relay 3.2.0 python  +graphviz 0.20.3 python  +greenlet 3.0.3 python  +grep 3.7-1build1 deb  +grpcio 1.60.0 python  +grpcio-status 1.60.0 python  +grpclib 0.4.7 python  +gunicorn 21.2.0 python  +gzip 1.10-4ubuntu4.1 deb  +h2 4.1.0 python  +hostname 3.23ubuntu2 deb  +hpack 4.0.0 python  +hyperframe 6.0.1 python  +idna 3.6 python  +idna 3.7 python  +imagesize 1.4.1 python  +importlib_metadata 7.1.0 python  +importlib_resources 6.4.0 python (+1 duplicate) +init-system-helpers 1.62 deb  +itsdangerous 2.2.0 python  +java-common 0.72build2 deb  +joblib 1.4.0 python (+1 duplicate) +jq 1.6-2.1ubuntu3 deb  +jrt-fs 11.0.22 java-archive  +js 1.0.0 npm  +json5 0.9.25 python  +jsonpatch 1.33 python  +jsonpointer 2.4 python  +jsonschema 4.21.1 python  +jsonschema-specifications 2023.12.1 python  +kiwisolver 1.4.5 python  +libacl1 2.3.1-1 deb  +libapt-pkg6.0 2.4.12 deb  +libarchive-c 5.1 python  +libasan6 11.4.0-1ubuntu1~22.04 deb  +libasound2 1.2.6.1-1ubuntu1 deb  +libasound2-data 1.2.6.1-1ubuntu1 deb  +libassuan0 2.5.5-1build1 deb  +libatomic1 12.3.0-1ubuntu1~22.04 deb  +libattr1 1:2.5.1-1build1 deb  +libaudit-common 1:3.0.7-1build1 deb  +libaudit1 1:3.0.7-1build1 deb  +libavahi-client3 0.8-5ubuntu5.2 deb  +libavahi-common-data 0.8-5ubuntu5.2 deb  +libavahi-common3 0.8-5ubuntu5.2 deb  +libbinutils 2.38-4ubuntu2.6 deb  +libblkid1 2.37.2-4ubuntu3.4 deb  +libbrotli1 1.0.9-2build6 deb  +libbsd0 0.11.5-1 deb  +libbz2-1.0 1.0.8-5build1 deb  +libc-bin 2.35-0ubuntu3.7 deb  +libc-dev-bin 2.35-0ubuntu3.7 deb  +libc6 2.35-0ubuntu3.7 deb  +libc6-dev 2.35-0ubuntu3.7 deb  +libcap-ng0 0.7.9-2.2build3 deb  +libcap2 1:2.44-1ubuntu0.22.04.1 deb  +libcbor0.8 0.8.0-2ubuntu1 deb  +libcc1-0 12.3.0-1ubuntu1~22.04 deb  +libcom-err2 1.46.5-2ubuntu1.1 deb  +libcrypt-dev 1:4.4.27-1 deb  +libcrypt1 1:4.4.27-1 deb  +libctf-nobfd0 2.38-4ubuntu2.6 deb  +libctf0 2.38-4ubuntu2.6 deb  +libcublas-12-1 12.1.3.1-1 deb  +libcufft-12-1 11.0.2.54-1 deb  +libcups2 2.4.1op1-1ubuntu4.8 deb  +libcurand-12-1 10.3.2.106-1 deb  +libcurl4 7.81.0-1ubuntu1.16 deb  +libcusolver-12-1 11.4.5.107-1 deb  +libdb5.3 5.3.28+dfsg1-0.8ubuntu3 deb  +libdbus-1-3 1.12.20-2ubuntu4.1 deb  +libdebconfclient0 0.261ubuntu1 deb  +libdpkg-perl 1.21.1ubuntu2.3 deb  +libedit2 3.1-20210910-1build1 deb  +libexpat1 2.4.7-1ubuntu0.3 deb  +libext2fs2 1.46.5-2ubuntu1.1 deb  +libffi8 3.4.2-4 deb  +libfido2-1 1.10.0-1 deb  +libfontconfig1 2.13.1-4.2ubuntu5 deb  +libfreetype6 2.11.1+dfsg-1ubuntu0.2 deb  +libgcc-11-dev 11.4.0-1ubuntu1~22.04 deb  +libgcc-s1 12.3.0-1ubuntu1~22.04 deb  +libgcrypt20 1.9.4-3ubuntu3 deb  +libgdbm-compat4 1.23-1 deb  +libgdbm6 1.23-1 deb  +libglib2.0-0 2.72.4-0ubuntu2.2 deb  +libgmp10 2:6.2.1+dfsg-3ubuntu1 deb  +libgnutls30 3.7.3-4ubuntu1.5 deb  +libgomp1 12.3.0-1ubuntu1~22.04 deb  +libgpg-error0 1.43-3 deb  +libgraphite2-3 1.3.14-1build2 deb  +libgssapi-krb5-2 1.19.2-2ubuntu0.3 deb  +libharfbuzz0b 2.7.4-1ubuntu3.1 deb  +libhogweed6 3.7.3-1build2 deb  +libidn2-0 2.3.2-2build1 deb  +libisl23 0.24-2build1 deb  +libitm1 12.3.0-1ubuntu1~22.04 deb  +libjpeg-turbo8 2.1.2-0ubuntu1 deb  +libjpeg8 8c-2ubuntu10 deb  +libjq1 1.6-2.1ubuntu3 deb  +libk5crypto3 1.19.2-2ubuntu0.3 deb  +libkeyutils1 1.6.1-2ubuntu3 deb  +libkrb5-3 1.19.2-2ubuntu0.3 deb  +libkrb5support0 1.19.2-2ubuntu0.3 deb  +libksba8 1.6.0-2ubuntu0.2 deb  +liblcms2-2 2.12~rc1-2build2 deb  +libldap-2.5-0 2.5.17+dfsg-0ubuntu0.22.04.1 deb  +liblsan0 12.3.0-1ubuntu1~22.04 deb  +liblz4-1 1.9.3-2build2 deb  +liblzma5 5.2.5-2ubuntu1 deb  +libmambapy 1.5.8 python  +libmd0 1.0.4-1build1 deb  +libmount1 2.37.2-4ubuntu3.4 deb  +libmpc3 1.2.1-2build1 deb  +libmpfr6 4.1.0-3build3 deb  +libncurses6 6.3-2ubuntu0.1 deb  +libncursesw6 6.3-2ubuntu0.1 deb  +libnettle8 3.7.3-1build2 deb  +libnghttp2-14 1.43.0-1ubuntu0.1 deb  +libnpth0 1.6-3build2 deb  +libnsl-dev 1.3.0-2build2 deb  +libnsl2 1.3.0-2build2 deb  +libnspr4 2:4.35-0ubuntu0.22.04.1 deb  +libnss3 2:3.98-0ubuntu0.22.04.2 deb  +libnuma1 2.0.14-3ubuntu2 deb  +libonig5 6.9.7.1-2build1 deb  +libp11-kit0 0.24.0-6build1 deb  +libpam-modules 1.4.0-11ubuntu2.4 deb  +libpam-modules-bin 1.4.0-11ubuntu2.4 deb  +libpam-runtime 1.4.0-11ubuntu2.4 deb  +libpam0g 1.4.0-11ubuntu2.4 deb  +libpcre2-8-0 10.39-3ubuntu0.1 deb  +libpcre3 2:8.39-13ubuntu0.22.04.1 deb  +libpcsclite1 1.9.5-3ubuntu1 deb  +libperl5.34 5.34.0-3ubuntu1.3 deb  +libpng16-16 1.6.37-3build5 deb  +libprocps8 2:3.3.17-6ubuntu2.1 deb  +libpsl5 0.21.0-1.2build2 deb  +libquadmath0 12.3.0-1ubuntu1~22.04 deb  +libreadline8 8.1.2-1 deb  +librtmp1 2.4+20151223.gitfa8646d.1-2build4 deb  +libsasl2-2 2.1.27+dfsg2-3ubuntu1.2 deb  +libsasl2-modules-db 2.1.27+dfsg2-3ubuntu1.2 deb  +libseccomp2 2.5.3-2ubuntu2 deb  +libselinux1 3.3-1build2 deb  +libsemanage-common 3.3-1build2 deb  +libsemanage2 3.3-1build2 deb  +libsepol2 3.3-1build1 deb  +libsmartcols1 2.37.2-4ubuntu3.4 deb  +libsqlite3-0 3.37.2-2ubuntu0.3 deb  +libss2 1.46.5-2ubuntu1.1 deb  +libssh-4 0.9.6-2ubuntu0.22.04.3 deb  +libssl3 3.0.2-0ubuntu1.15 deb  +libstdc++-11-dev 11.4.0-1ubuntu1~22.04 deb  +libstdc++6 12.3.0-1ubuntu1~22.04 deb  +libsystemd0 249.11-0ubuntu3.12 deb  +libtasn1-6 4.18.0-4build1 deb  +libtinfo6 6.3-2ubuntu0.1 deb  +libtirpc-common 1.3.2-2ubuntu0.1 deb  +libtirpc-dev 1.3.2-2ubuntu0.1 deb  +libtirpc3 1.3.2-2ubuntu0.1 deb  +libtsan0 11.4.0-1ubuntu1~22.04 deb  +libubsan1 12.3.0-1ubuntu1~22.04 deb  +libudev1 249.11-0ubuntu3.12 deb  +libunistring2 1.0-1 deb  +libuuid1 2.37.2-4ubuntu3.4 deb  +libxxhash0 0.8.1-1 deb  +libzstd1 1.4.8+dfsg-3build1 deb  +lief 0.14.1 python  +linux-libc-dev 5.15.0-105.115 deb  +llvmlite 0.42.0 python  +locket 1.0.0 python  +login 1:4.8.1-2ubuntu2.2 deb  +logsave 1.46.5-2ubuntu1.1 deb  +lsb-base 11.1.0ubuntu4 deb  +lto-disabled-list 24 deb  +lz4 4.3.3 python  +make 4.3-4.1build1 deb  +mamba 1.5.8 python  +markdown-it-py 3.0.0 python (+1 duplicate) +marshmallow 3.21.1 python  +matplotlib 3.8.4 python  +mawk 1.3.4.20200120-3 deb  +mdurl 0.1.2 python (+1 duplicate) +menuinst 2.0.2 python  +merlin-core 23.8.0 python  +merlin-dataloader 23.8.0 python  +milvus 2.3.5 python  +minio 7.2.5 python  +mlflow 2.12.1 python  +mlflow-skinny 2.12.1 python  +more-itertools 10.2.0 python  +morpheus 24.3.2 python  +mount 2.37.2-4ubuntu3.4 deb  +mpmath 1.3.0 python  +mrc 24.3.1 python  +msgpack 1.0.7 python  +multidict 6.0.5 python  +munkres 1.1.4 python  +ncurses-base 6.3-2ubuntu0.1 deb  +ncurses-bin 6.3-2ubuntu0.1 deb  +networkx 2.8.8 python  +npy-append-array 0.9.16 python  +numba 0.59.1 python  +numpy 1.24.4 python  +numpydoc 1.5.0 python  +nvtabular 23.8.0 python  +nvtx 0.2.10 python  +oauthlib 3.2.2 python  +openjdk-11-jre-headless 11.0.22+7-0ubuntu2~22.04.1 deb  +openssh-client 1:8.9p1-3ubuntu0.7 deb  +openssl 3.0.2-0ubuntu1.15 deb  +openssl 3.2.1 binary  +ordered-set 4.1.0 python  +packaging 24.0 python (+1 duplicate) +pandas 1.5.3 python  +partd 1.4.1 python  +passwd 1:4.8.1-2ubuntu2.2 deb  +patch 2.7.6-7build2 deb  +perl 5.34.0-3ubuntu1.3 deb  +perl-base 5.34.0-3ubuntu1.3 deb  +perl-modules-5.34 5.34.0-3ubuntu1.3 deb  +pillow 10.3.0 python  +pinentry-curses 1.1.1-1build2 deb  +pip 24.0 python  +pkg-config 0.29.2-1ubuntu3 deb  +pkginfo 1.10.0 python  +pkgutil_resolve_name 1.3.10 python  +platformdirs 4.2.0 python  +platformdirs 4.2.1 python  +pluggy 1.3.0 python  +pluggy 1.4.0 python  +procps 2:3.3.17-6ubuntu2.1 deb  +prometheus-flask-exporter 0.23.0 python  +prometheus_client 0.20.0 python  +prompt-toolkit 3.0.42 python  +protobuf 4.24.4 python  +psutil 5.9.8 python (+1 duplicate) +py4j 0.10.9.7 python  +py4j0 10.9.7 java-archive  +pyOpenSSL 24.0.0 python  +pyarrow 14.0.2 python  +pyarrow-hotfix 0.6 python  +pyasn1 0.6.0 python  +pyasn1_modules 0.4.0 python  +pycosat 0.6.6 python  +pycparser 2.22 python (+1 duplicate) +pycryptodome 3.20.0 python  +pydantic 2.7.1 python  +pydantic_core 2.18.2 python  +pymilvus 2.3.6 python  +pynvjitlink 0.2.1 python  +pynvml 11.4.1 python  +pyparsing 3.1.2 python  +python 3.10.14 binary  +python-dateutil 2.9.0 python  +python-dotenv 1.0.1 python  +python-rapidjson 1.16 python  +pytz 2024.1 python (+1 duplicate) +qpd 0.4.4 python  +querystring-parser 1.2.4 python  +readline-common 8.1.2-1 deb  +referencing 0.34.0 python  +requests 2.31.0 python (+1 duplicate) +requests-cache 1.1.1 python  +requests-toolbelt 1.0.0 python  +rich 13.7.1 python (+1 duplicate) +rmm 24.2.0 python  +rpcsvc-proto 1.4.2-0ubuntu6 deb  +rpds-py 0.18.0 python  +rsa 4.9 python  +ruamel.yaml 0.18.6 python  +ruamel.yaml.clib 0.2.8 python  +scikit-learn 1.3.2 python  +scipy 1.13.0 python  +sed 4.8-1ubuntu2 deb  +sensible-utils 0.0.17 deb  +setuptools 69.5.1 python (+1 duplicate) +sgmllib3k 1.0.0 python  +six 1.16.0 python  +smmap 5.0.0 python  +sniffio 1.3.1 python  +snowballstemmer 2.2.0 python  +sortedcontainers 2.4.0 python  +soupsieve 2.5 python  +sphinxcontrib-applehelp 1.0.8 python  +sphinxcontrib-devhelp 1.0.6 python  +sphinxcontrib-htmlhelp 2.0.5 python  +sphinxcontrib-jsmath 1.0.1 python  +sphinxcontrib-qthelp 1.0.7 python  +sphinxcontrib-serializinghtml 1.1.10 python  +sqlglot 23.11.2 python  +sqlparse 0.4.4 python  +stringcase 1.2.0 python  +sympy 1.12 python  +sysvinit-utils 3.01-1ubuntu1 deb  +tabulate 0.9.0 python  +tar 1.34+dfsg-1ubuntu0.1.22.04.2 deb  +tblib 3.0.0 python  +tensorflow-metadata 1.13.1 python  +threadpoolctl 3.4.0 python  +tomli 2.0.1 python (+1 duplicate) +toolz 0.12.1 python  +torch 2.3.0 python  +tornado 6.4 python  +tqdm 4.66.2 python (+1 duplicate) +triad 0.9.6 python  +triton 2.3.0 python  +tritonclient 2.34.0 python  +truststore 0.8.0 python  +typing-utils 0.1.0 python  +typing_extensions 4.11.0 python (+1 duplicate) +ubuntu-keyring 2021.03.26 deb  +ucf 3.0043 deb  +ujson 5.9.0 python  +unicodedata2 15.1.0 python  +unzip 6.0-26ubuntu3.2 deb  +url-normalize 1.4.3 python  +urllib3 2.2.1 python (+1 duplicate) +usrmerge 25ubuntu2 deb  +util-linux 2.37.2-4ubuntu3.4 deb  +watchdog 3.0.0 python  +watchgod 0.8.2 python  +wcwidth 0.2.13 python  +websocket-client 1.8.0 python  +websockets 12.0 python  +wget 1.21.2-2ubuntu1 deb  +wheel 0.43.0 python (+1 duplicate) +xyzservices 2024.4.0 python  +xz-utils 5.2.5-2ubuntu1 deb  +zict 3.0.0 python  +zip 3.0-12build2 deb  +zipp 3.17.0 python (+1 duplicate) +zlib1g 1:1.2.11.dfsg-2ubuntu9.2 deb  +zstandard 0.22.0 python  diff --git a/data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.07.03-runtime.sbom b/data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.07.03-runtime.sbom new file mode 100644 index 0000000..bc80a8b --- /dev/null +++ b/data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.07.03-runtime.sbom @@ -0,0 +1,502 @@ +NAME VERSION TYPE +@jupyter-widgets/jupyterlab-manager 5.0.9 npm  +@jupyterlab/application-top 4.0.7 npm (+2 duplicates) +@jupyterlab/galata-extension 5.0.6 npm  +@jupyterlab/mock-consumer 4.0.7 npm  +@jupyterlab/mock-extension 4.0.7 npm  +@jupyterlab/mock-incompat 0.1.0 npm  +@jupyterlab/mock-mime-extension 0.3.0 npm  +@jupyterlab/mock-package 0.1.0 npm  +@jupyterlab/mock-provider 4.0.7 npm  +@jupyterlab/mock-token 4.0.7 npm  +Babel 2.13.0 python  +Brotli 1.0.9 python  +Brotli 1.1.0 python  +ConfigArgParse 1.5.5 python  +Flask 2.3.3 python  +GitPython 3.1.37 python  +Jinja2 3.1.2 python (+1 duplicate) +Mako 1.2.4 python  +Markdown 3.5 python  +MarkupSafe 2.1.3 python (+1 duplicate) +Pillow 10.0.1 python  +PyJWT 2.8.0 python  +PySocks 1.7.1 python (+1 duplicate) +PyYAML 6.0.1 python (+1 duplicate) +Pygments 2.16.1 python (+1 duplicate) +SQLAlchemy 2.0.0 python  +Send2Trash 1.8.2 python  +Simple Launcher 1.1.0.14 dotnet (+5 duplicates) +Sphinx 7.2.6 python  +Werkzeug 3.0.0 python  +absl-py 1.4.0 python  +adduser 3.118ubuntu5 deb  +alabaster 0.7.13 python  +alembic 1.12.0 python  +anyio 3.7.1 python  +anyio 4.0.0 python  +appdirs 1.4.4 python  +apt 2.4.10 deb  +argon2-cffi 23.1.0 python  +argon2-cffi-bindings 21.2.0 python  +arrow 1.3.0 python  +asn1crypto 1.5.1 python  +asttokens 2.4.0 python  +async-lru 2.0.4 python  +attrs 23.1.0 python (+1 duplicate) +backcall 0.2.0 python  +backports.functools-lru-cache 1.6.5 python  +base-files 12ubuntu4.4 deb  +base-passwd 3.5.52build1 deb  +bash 5.1-6ubuntu1 deb  +bc 1.07.1-3build1 deb  +beautifulsoup4 4.12.2 python (+1 duplicate) +betterproto 1.2.5 python  +binutils 2.38-4ubuntu2.3 deb  +binutils-common 2.38-4ubuntu2.3 deb  +binutils-x86-64-linux-gnu 2.38-4ubuntu2.3 deb  +bleach 6.1.0 python  +blinker 1.6.3 python  +boa 0.15.1 python  +bokeh 2.4.3 python  +boltons 23.0.0 python  +bsdutils 1:2.37.2-4ubuntu3 deb  +build-essential 12.9ubuntu3 deb  +bytecode 0.13.0.dev0 python  +bzip2 1.0.8-5build1 deb  +ca-certificates 20230311ubuntu0.22.04.1 deb  +cachetools 5.3.1 python  +certifi 2023.7.22 python (+1 duplicate) +cffi 1.15.1 python  +cffi 1.16.0 python  +chardet 5.2.0 python  +charset-normalizer 3.2.0 python  +charset-normalizer 3.3.0 python  +click 8.1.7 python (+1 duplicate) +cloudpickle 2.2.1 python  +colorama 0.4.6 python (+1 duplicate) +comm 0.1.4 python  +conda 23.3.1 python  +conda-build 3.25.0 python  +conda-libmamba-solver 23.3.0 python  +conda-package-handling 2.2.0 python  +conda_index 0.3.0 python  +conda_package_streaming 0.9.0 python  +configparser 5.3.0 python  +confluent-kafka 1.9.2 python  +contourpy 1.1.1 python  +coreutils 8.32-4.1ubuntu1 deb  +cpp 4:11.2.0-1ubuntu1 deb  +cpp-11 11.4.0-1ubuntu1~22.04 deb  +cryptography 41.0.3 python  +cryptography 41.0.4 python  +cubinlinker 0.3.0 python  +cuda-cccl-11-8 11.8.89-1 deb  +cuda-compat-11-8 520.61.05-1 deb  +cuda-compiler-11-8 11.8.0-1 deb  +cuda-cudart-11-8 11.8.89-1 deb  +cuda-cudart-dev-11-8 11.8.89-1 deb  +cuda-cuobjdump-11-8 11.8.86-1 deb  +cuda-cupti-11-8 11.8.87-1 deb  +cuda-cuxxfilt-11-8 11.8.86-1 deb  +cuda-driver-dev-11-8 11.8.89-1 deb  +cuda-gdb-11-8 11.8.86-1 deb  +cuda-keyring 1.1-1 deb  +cuda-nvcc-11-8 11.8.89-1 deb  +cuda-nvdisasm-11-8 11.8.86-1 deb  +cuda-nvprune-11-8 11.8.86-1 deb  +cuda-python 11.8.2 python  +cuda-toolkit-11-8-config-common 11.8.89-1 deb  +cuda-toolkit-11-config-common 11.8.89-1 deb  +cuda-toolkit-config-common 12.2.140-1 deb  +cudf 23.6.1 python  +cudf-kafka 23.6.1 python  +cupy 12.2.0 python  +curl 7.81.0-1ubuntu1.14 deb  +cycler 0.12.1 python  +cytoolz 0.12.2 python  +dash 0.5.11+git20210903+057cd650a4ed-3build1 deb  +dask 2023.3.2 python  +dask-cuda 23.6.0 python  +dask-cudf 23.6.1 python  +databricks-cli 0.18.0 python  +dataclasses 0.8 python (+1 duplicate) +datacompy 0.8.4 python  +debconf 1.5.79ubuntu1 deb  +debianutils 5.5-1ubuntu2 deb  +debugpy 1.8.0 python  +decorator 5.1.1 python  +defusedxml 0.7.1 python  +diffutils 1:3.8-0ubuntu2 deb  +dill 0.3.7 python  +dirmngr 2.2.27-3ubuntu2.1 deb  +distributed 2023.3.2.1 python  +docker 5.0.3 python  +docker-pycreds 0.4.0 python  +docutils 0.20.1 python  +dpkg 1.21.1ubuntu2.2 deb  +dpkg-dev 1.21.1ubuntu2.2 deb  +e2fsprogs 1.46.5-2ubuntu1.1 deb  +entrypoints 0.4 python  +exceptiongroup 1.1.3 python (+1 duplicate) +executing 2.0.0 python  +fastjsonschema 2.18.1 python  +fastrlock 0.8.2 python  +filelock 3.12.4 python (+1 duplicate) +findutils 4.8.0-1ubuntu3 deb  +fonttools 4.43.1 python  +fqdn 1.5.1 python  +fsspec 2023.9.2 python  +g++ 4:11.2.0-1ubuntu1 deb  +g++-11 11.4.0-1ubuntu1~22.04 deb  +gcc 4:11.2.0-1ubuntu1 deb  +gcc-11 11.4.0-1ubuntu1~22.04 deb  +gcc-11-base 11.4.0-1ubuntu1~22.04 deb  +gcc-12-base 12.3.0-1ubuntu1~22.04 deb  +gitdb 4.0.10 python  +glob2 0.7 python  +gmpy2 2.1.2 python  +gnupg 2.2.27-3ubuntu2.1 deb  +gnupg-l10n 2.2.27-3ubuntu2.1 deb  +gnupg-utils 2.2.27-3ubuntu2.1 deb  +gnupg2 2.2.27-3ubuntu2.1 deb  +googleapis-common-protos 1.60.0 python  +gpg 2.2.27-3ubuntu2.1 deb  +gpg-agent 2.2.27-3ubuntu2.1 deb  +gpg-wks-client 2.2.27-3ubuntu2.1 deb  +gpg-wks-server 2.2.27-3ubuntu2.1 deb  +gpgconf 2.2.27-3ubuntu2.1 deb  +gpgsm 2.2.27-3ubuntu2.1 deb  +gpgv 2.2.27-3ubuntu2.1 deb  +greenlet 3.0.0 python  +grep 3.7-1build1 deb  +grpcio 1.54.2 python  +grpclib 0.4.6 python  +gunicorn 21.2.0 python  +gzip 1.10-4ubuntu4.1 deb  +h2 4.1.0 python  +hostname 3.23ubuntu2 deb  +hpack 4.0.0 python  +html5lib 1.1 python  +hyperframe 6.0.1 python  +idna 3.4 python (+1 duplicate) +imagesize 1.4.1 python  +importlib-metadata 6.8.0 python  +importlib-resources 6.1.0 python (+1 duplicate) +init-system-helpers 1.62 deb  +ipykernel 6.25.2 python  +ipython 8.16.1 python  +ipywidgets 8.1.1 python  +isoduration 20.11.0 python  +itsdangerous 2.1.2 python  +jedi 0.19.1 python  +joblib 1.3.2 python (+1 duplicate) +jq 1.6-2.1ubuntu3 deb  +js 1.0.0 npm  +json5 0.9.14 python (+1 duplicate) +jsonpatch 1.32 python  +jsonpointer 2.0 python  +jsonpointer 2.4 python  +jsonschema 4.19.1 python (+1 duplicate) +jsonschema-specifications 2023.7.1 python (+1 duplicate) +jupyter-events 0.7.0 python  +jupyter-lsp 2.2.0 python  +jupyter_client 8.4.0 python  +jupyter_core 4.12.0 python  +jupyter_server 2.7.3 python  +jupyter_server_terminals 0.4.4 python  +jupyterlab 4.0.7 python  +jupyterlab-pygments 0.2.2 python  +jupyterlab-widgets 3.0.9 python  +jupyterlab_pygments 0.2.2 npm (+1 duplicate) +jupyterlab_server 2.25.0 python  +kiwisolver 1.4.5 python  +libacl1 2.3.1-1 deb  +libapt-pkg6.0 2.4.10 deb  +libarchive-c 5.0 python  +libasan6 11.4.0-1ubuntu1~22.04 deb  +libassuan0 2.5.5-1build1 deb  +libatomic1 12.3.0-1ubuntu1~22.04 deb  +libattr1 1:2.5.1-1build1 deb  +libaudit-common 1:3.0.7-1build1 deb  +libaudit1 1:3.0.7-1build1 deb  +libbinutils 2.38-4ubuntu2.3 deb  +libblkid1 2.37.2-4ubuntu3 deb  +libbrotli1 1.0.9-2build6 deb  +libbsd0 0.11.5-1 deb  +libbz2-1.0 1.0.8-5build1 deb  +libc-bin 2.35-0ubuntu3.4 deb  +libc-dev-bin 2.35-0ubuntu3.4 deb  +libc6 2.35-0ubuntu3.4 deb  +libc6-dev 2.35-0ubuntu3.4 deb  +libcap-ng0 0.7.9-2.2build3 deb  +libcap2 1:2.44-1ubuntu0.22.04.1 deb  +libcbor0.8 0.8.0-2ubuntu1 deb  +libcc1-0 12.3.0-1ubuntu1~22.04 deb  +libcom-err2 1.46.5-2ubuntu1.1 deb  +libcrypt-dev 1:4.4.27-1 deb  +libcrypt1 1:4.4.27-1 deb  +libctf-nobfd0 2.38-4ubuntu2.3 deb  +libctf0 2.38-4ubuntu2.3 deb  +libcublas-11-8 11.11.3.6-1 deb  +libcufft-11-8 10.9.0.58-1 deb  +libcurand-11-8 10.3.0.86-1 deb  +libcurl4 7.81.0-1ubuntu1.14 deb  +libcusolver-11-8 11.4.1.48-1 deb  +libdb5.3 5.3.28+dfsg1-0.8ubuntu3 deb  +libdebconfclient0 0.261ubuntu1 deb  +libdpkg-perl 1.21.1ubuntu2.2 deb  +libedit2 3.1-20210910-1build1 deb  +libext2fs2 1.46.5-2ubuntu1.1 deb  +libffi8 3.4.2-4 deb  +libfido2-1 1.10.0-1 deb  +libgcc-11-dev 11.4.0-1ubuntu1~22.04 deb  +libgcc-s1 12.3.0-1ubuntu1~22.04 deb  +libgcrypt20 1.9.4-3ubuntu3 deb  +libgdbm-compat4 1.23-1 deb  +libgdbm6 1.23-1 deb  +libglib2.0-0 2.72.4-0ubuntu2.2 deb  +libgmp10 2:6.2.1+dfsg-3ubuntu1 deb  +libgnutls30 3.7.3-4ubuntu1.2 deb  +libgomp1 12.3.0-1ubuntu1~22.04 deb  +libgpg-error0 1.43-3 deb  +libgssapi-krb5-2 1.19.2-2ubuntu0.2 deb  +libhogweed6 3.7.3-1build2 deb  +libidn2-0 2.3.2-2build1 deb  +libisl23 0.24-2build1 deb  +libitm1 12.3.0-1ubuntu1~22.04 deb  +libjq1 1.6-2.1ubuntu3 deb  +libk5crypto3 1.19.2-2ubuntu0.2 deb  +libkeyutils1 1.6.1-2ubuntu3 deb  +libkrb5-3 1.19.2-2ubuntu0.2 deb  +libkrb5support0 1.19.2-2ubuntu0.2 deb  +libksba8 1.6.0-2ubuntu0.2 deb  +libldap-2.5-0 2.5.16+dfsg-0ubuntu0.22.04.1 deb  +liblsan0 12.3.0-1ubuntu1~22.04 deb  +liblz4-1 1.9.3-2build2 deb  +liblzma5 5.2.5-2ubuntu1 deb  +libmambapy 1.4.2 python  +libmd0 1.0.4-1build1 deb  +libmount1 2.37.2-4ubuntu3 deb  +libmpc3 1.2.1-2build1 deb  +libmpfr6 4.1.0-3build3 deb  +libncurses6 6.3-2ubuntu0.1 deb  +libncursesw6 6.3-2ubuntu0.1 deb  +libnettle8 3.7.3-1build2 deb  +libnghttp2-14 1.43.0-1build3 deb  +libnpth0 1.6-3build2 deb  +libnsl-dev 1.3.0-2build2 deb  +libnsl2 1.3.0-2build2 deb  +libnuma1 2.0.14-3ubuntu2 deb  +libonig5 6.9.7.1-2build1 deb  +libp11-kit0 0.24.0-6build1 deb  +libpam-modules 1.4.0-11ubuntu2.3 deb  +libpam-modules-bin 1.4.0-11ubuntu2.3 deb  +libpam-runtime 1.4.0-11ubuntu2.3 deb  +libpam0g 1.4.0-11ubuntu2.3 deb  +libpcre2-8-0 10.39-3ubuntu0.1 deb  +libpcre3 2:8.39-13ubuntu0.22.04.1 deb  +libperl5.34 5.34.0-3ubuntu1.2 deb  +libprocps8 2:3.3.17-6ubuntu2 deb  +libpsl5 0.21.0-1.2build2 deb  +libquadmath0 12.3.0-1ubuntu1~22.04 deb  +libreadline8 8.1.2-1 deb  +librtmp1 2.4+20151223.gitfa8646d.1-2build4 deb  +libsasl2-2 2.1.27+dfsg2-3ubuntu1.2 deb  +libsasl2-modules-db 2.1.27+dfsg2-3ubuntu1.2 deb  +libseccomp2 2.5.3-2ubuntu2 deb  +libselinux1 3.3-1build2 deb  +libsemanage-common 3.3-1build2 deb  +libsemanage2 3.3-1build2 deb  +libsepol2 3.3-1build1 deb  +libsmartcols1 2.37.2-4ubuntu3 deb  +libsqlite3-0 3.37.2-2ubuntu0.1 deb  +libss2 1.46.5-2ubuntu1.1 deb  +libssh-4 0.9.6-2ubuntu0.22.04.1 deb  +libssl3 3.0.2-0ubuntu1.10 deb  +libstdc++-11-dev 11.4.0-1ubuntu1~22.04 deb  +libstdc++6 12.3.0-1ubuntu1~22.04 deb  +libsystemd0 249.11-0ubuntu3.10 deb  +libtasn1-6 4.18.0-4build1 deb  +libtinfo6 6.3-2ubuntu0.1 deb  +libtirpc-common 1.3.2-2ubuntu0.1 deb  +libtirpc-dev 1.3.2-2ubuntu0.1 deb  +libtirpc3 1.3.2-2ubuntu0.1 deb  +libtsan0 11.4.0-1ubuntu1~22.04 deb  +libubsan1 12.3.0-1ubuntu1~22.04 deb  +libudev1 249.11-0ubuntu3.10 deb  +libunistring2 1.0-1 deb  +libuuid1 2.37.2-4ubuntu3 deb  +libxxhash0 0.8.1-1 deb  +libzstd1 1.4.8+dfsg-3build1 deb  +linux-libc-dev 5.15.0-86.96 deb  +llvmlite 0.40.1 python  +locket 1.0.0 python  +login 1:4.8.1-2ubuntu2.1 deb  +logsave 1.46.5-2ubuntu1.1 deb  +lsb-base 11.1.0ubuntu4 deb  +lto-disabled-list 24 deb  +lz4 4.3.2 python  +make 4.3-4.1build1 deb  +mamba 1.4.2 python  +markdown-it-py 3.0.0 python  +matplotlib 3.8.0 python  +matplotlib-inline 0.1.6 python  +mawk 1.3.4.20200120-3 deb  +mdurl 0.1.0 python  +merlin-core 23.6.0 python  +merlin-dataloader 23.6.0 python  +mistune 3.0.2 python  +mlflow 2.6.0 python  +more-itertools 10.1.0 python  +morpheus 23.7.3 python  +mount 2.37.2-4ubuntu3 deb  +mpmath 1.3.0 python  +mrc 23.7.0 python  +msgpack 1.0.6 python  +multidict 6.0.4 python  +munkres 1.1.4 python  +nb-conda-kernels 2.3.1 python  +nbclient 0.8.0 python  +nbconvert 7.9.2 python  +nbformat 5.9.2 python  +ncurses-base 6.3-2ubuntu0.1 deb  +ncurses-bin 6.3-2ubuntu0.1 deb  +nest-asyncio 1.5.8 python  +networkx 3.1 python  +notebook_shim 0.2.3 python  +npy-append-array 0.9.16 python  +numba 0.57.1 python  +numpy 1.24.4 python  +numpydoc 1.4.0 python  +nvidia-pyindex 1.0.9 python  +nvtabular 23.6.0 python  +nvtx 0.2.8 python  +oauthlib 3.2.2 python  +openssh-client 1:8.9p1-3ubuntu0.4 deb  +openssl 3.0.2-0ubuntu1.10 deb  +openssl 3.1.3 binary  +ordered-set 4.1.0 python  +overrides 7.4.0 python  +packaging 23.1 python  +packaging 23.2 python  +pandas 1.3.5 python  +pandocfilters 1.5.0 python  +parso 0.8.3 python  +partd 1.4.1 python  +passwd 1:4.8.1-2ubuntu2.1 deb  +patch 2.7.6-7build2 deb  +perl 5.34.0-3ubuntu1.2 deb  +perl-base 5.34.0-3ubuntu1.2 deb  +perl-modules-5.34 5.34.0-3ubuntu1.2 deb  +pexpect 4.8.0 python  +pickleshare 0.7.5 python  +pinentry-curses 1.1.1-1build2 deb  +pip 23.2.1 python  +pkg-config 0.29.2-1ubuntu3 deb  +pkginfo 1.9.6 python  +pkgutil_resolve_name 1.3.10 python  +platformdirs 3.11.0 python  +pluggy 1.0.0 python  +pluggy 1.3.0 python  +procps 2:3.3.17-6ubuntu2 deb  +prometheus-client 0.17.1 python  +prometheus-flask-exporter 0.22.4 python  +prompt-toolkit 3.0.39 python (+1 duplicate) +protobuf 4.21.12 python  +psutil 5.9.5 python (+1 duplicate) +ptxcompiler 0.8.1 python  +ptyprocess 0.7.0 python  +pure-eval 0.2.2 python  +pyOpenSSL 23.2.0 python (+1 duplicate) +pyarrow 11.0.0 python  +pycosat 0.6.4 python  +pycparser 2.21 python (+1 duplicate) +pynvml 11.4.1 python  +pyparsing 3.1.1 python  +python 3.10.12 binary  +python-dateutil 2.8.2 python  +python-json-logger 2.0.7 python  +python-rapidjson 1.12 python  +pytz 2023.3.post1 python (+1 duplicate) +pyzmq 25.1.1 python  +querystring-parser 1.2.4 python  +readline-common 8.1.2-1 deb  +referencing 0.30.2 python (+1 duplicate) +requests 2.31.0 python (+1 duplicate) +rfc3339-validator 0.1.4 python  +rfc3986-validator 0.1.1 python  +rich 13.6.0 python  +rmm 23.6.0 python  +rpcsvc-proto 1.4.2-0ubuntu6 deb  +rpds-py 0.10.6 python (+1 duplicate) +ruamel.yaml 0.17.32 python  +ruamel.yaml.clib 0.2.7 python  +scikit-learn 1.2.2 python  +scipy 1.11.3 python  +sed 4.8-1ubuntu2 deb  +sensible-utils 0.0.17 deb  +setuptools 59.8.0 python  +setuptools 68.1.2 python  +six 1.16.0 python (+1 duplicate) +smmap 3.0.5 python  +sniffio 1.3.0 python (+1 duplicate) +snowballstemmer 2.2.0 python  +sortedcontainers 2.4.0 python  +soupsieve 2.5 python (+1 duplicate) +sphinxcontrib-applehelp 1.0.7 python  +sphinxcontrib-devhelp 1.0.5 python  +sphinxcontrib-htmlhelp 2.0.4 python  +sphinxcontrib-jsmath 1.0.1 python  +sphinxcontrib-qthelp 1.0.6 python  +sphinxcontrib-serializinghtml 1.1.9 python  +sqlparse 0.4.4 python  +stack-data 0.6.3 python  +stringcase 1.2.0 python  +sympy 1.12 python  +sysvinit-utils 3.01-1ubuntu1 deb  +tabulate 0.9.0 python  +tar 1.34+dfsg-1ubuntu0.1.22.04.1 deb  +tblib 2.0.0 python  +tensorflow-metadata 1.13.1 python  +terminado 0.17.1 python  +test-hyphens 3.0.2 npm  +test-hyphens-underscore 3.0.2 npm  +test_no_hyphens 3.0.2 npm  +threadpoolctl 3.2.0 python  +tinycss2 1.2.1 python  +tomli 2.0.1 python (+1 duplicate) +toolz 0.12.0 python (+1 duplicate) +torch 2.0.1 python  +tornado 6.3.3 python  +tqdm 4.66.1 python (+1 duplicate) +traitlets 5.11.2 python  +triton 2.0.0 python  +tritonclient 2.26.0 python  +types-python-dateutil 2.8.19.14 python  +typing-utils 0.1.0 python  +typing_extensions 4.8.0 python (+1 duplicate) +ubuntu-keyring 2021.03.26 deb  +unicodedata2 15.1.0 python  +unzip 6.0-26ubuntu3.1 deb  +uri-template 1.3.0 python  +urllib3 2.0.4 python  +urllib3 2.0.6 python  +usrmerge 25ubuntu2 deb  +util-linux 2.37.2-4ubuntu3 deb  +watchdog 2.1.9 python  +watchgod 0.8.2 python  +wcwidth 0.2.8 python (+1 duplicate) +webcolors 1.13 python  +webencodings 0.5.1 python  +websocket-client 1.6.4 python  +websockets 11.0.3 python  +wget 1.21.2-2ubuntu1 deb  +wheel 0.41.2 python (+1 duplicate) +widgetsnbextension 4.0.9 python  +xz-utils 5.2.5-2ubuntu1 deb  +zict 3.0.0 python  +zip 3.0-12build2 deb  +zipp 3.17.0 python (+1 duplicate) +zlib1g 1:1.2.11.dfsg-2ubuntu9.2 deb  +zstandard 0.19.0 python  diff --git a/data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom b/data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom new file mode 100644 index 0000000..cd469cc --- /dev/null +++ b/data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom @@ -0,0 +1,465 @@ +NAME VERSION TYPE +Babel 2.13.1 python  +Brotli 1.0.9 python  +Brotli 1.1.0 python  +ConfigArgParse 1.5.5 python  +Flask 3.0.0 python  +GitPython 3.1.40 python  +Jinja2 3.1.2 python (+1 duplicate) +Mako 1.3.0 python  +Markdown 3.5.1 python  +MarkupSafe 2.1.3 python (+1 duplicate) +Pillow 10.1.0 python  +PyJWT 2.8.0 python  +PySocks 1.7.1 python (+1 duplicate) +PyYAML 6.0.1 python (+1 duplicate) +Pygments 2.17.2 python (+1 duplicate) +SQLAlchemy 2.0.0 python  +SciPy 1.11.4 python  +Simple Launcher 1.1.0.14 dotnet (+5 duplicates) +Sphinx 7.2.6 python  +Werkzeug 3.0.1 python  +absl-py 1.4.0 python  +adduser 3.118ubuntu5 deb  +alabaster 0.7.13 python  +alembic 1.13.0 python  +anyio 3.7.1 python  +appdirs 1.4.4 python  +apt 2.4.11 deb  +asn1crypto 1.5.1 python  +attrs 23.1.0 python (+1 duplicate) +base-files 12ubuntu4.4 deb  +base-passwd 3.5.52build1 deb  +bash 5.1-6ubuntu1 deb  +bc 1.07.1-3build1 deb  +beautifulsoup4 4.12.2 python  +betterproto 1.2.5 python  +binutils 2.38-4ubuntu2.3 deb  +binutils-common 2.38-4ubuntu2.3 deb  +binutils-x86-64-linux-gnu 2.38-4ubuntu2.3 deb  +blinker 1.7.0 python  +boa 0.16.0 python  +bokeh 2.4.3 python  +boltons 23.0.0 python  +bsdutils 1:2.37.2-4ubuntu3 deb  +build-essential 12.9ubuntu3 deb  +bzip2 1.0.8-5build1 deb  +ca-certificates 20230311ubuntu0.22.04.1 deb  +ca-certificates-java java-archive  +ca-certificates-java 20190909ubuntu1.2 deb  +cachetools 5.3.2 python  +cattrs 23.2.3 python  +certifi 2023.11.17 python (+1 duplicate) +cffi 1.15.1 python  +cffi 1.16.0 python  +chardet 5.2.0 python  +charset-normalizer 3.2.0 python  +charset-normalizer 3.3.2 python  +click 8.1.7 python (+1 duplicate) +cloudpickle 3.0.0 python  +colorama 0.4.6 python (+1 duplicate) +conda 23.3.1 python  +conda-build 3.28.1 python  +conda-libmamba-solver 23.3.0 python  +conda-package-handling 2.2.0 python  +conda_index 0.3.0 python  +conda_package_streaming 0.9.0 python  +configparser 5.3.0 python  +confluent-kafka 1.9.2 python  +contourpy 1.2.0 python  +coreutils 8.32-4.1ubuntu1 deb  +cpp 4:11.2.0-1ubuntu1 deb  +cpp-11 11.4.0-1ubuntu1~22.04 deb  +cryptography 41.0.3 python  +cryptography 41.0.7 python  +cubinlinker 0.3.0 python  +cuda-cccl-11-8 11.8.89-1 deb  +cuda-compat-11-8 520.61.05-1 deb  +cuda-compiler-11-8 11.8.0-1 deb  +cuda-cudart-11-8 11.8.89-1 deb  +cuda-cudart-dev-11-8 11.8.89-1 deb  +cuda-cuobjdump-11-8 11.8.86-1 deb  +cuda-cupti-11-8 11.8.87-1 deb  +cuda-cuxxfilt-11-8 11.8.86-1 deb  +cuda-driver-dev-11-8 11.8.89-1 deb  +cuda-gdb-11-8 11.8.86-1 deb  +cuda-keyring 1.1-1 deb  +cuda-nvcc-11-8 11.8.89-1 deb  +cuda-nvdisasm-11-8 11.8.86-1 deb  +cuda-nvprune-11-8 11.8.86-1 deb  +cuda-python 11.8.3 python  +cuda-toolkit-11-8-config-common 11.8.89-1 deb  +cuda-toolkit-11-config-common 11.8.89-1 deb  +cuda-toolkit-config-common 12.3.101-1 deb  +cudf 23.6.1 python  +cudf-kafka 23.6.1 python  +cupy 12.2.0 python  +curl 7.81.0-1ubuntu1.15 deb  +cycler 0.12.1 python  +cytoolz 0.12.2 python  +dash 0.5.11+git20210903+057cd650a4ed-3build1 deb  +dask 2023.3.2 python  +dask-cuda 23.6.0 python  +dask-cudf 23.6.1 python  +databricks-cli 0.18.0 python  +dataclasses 0.8 python  +datacompy 0.8.4 python  +debconf 1.5.79ubuntu1 deb  +debianutils 5.5-1ubuntu2 deb  +diffutils 1:3.8-0ubuntu2 deb  +dill 0.3.7 python  +dirmngr 2.2.27-3ubuntu2.1 deb  +distributed 2023.3.2.1 python  +docker 5.0.3 python  +docker-pycreds 0.4.0 python  +docutils 0.20.1 python  +dpkg 1.21.1ubuntu2.2 deb  +dpkg-dev 1.21.1ubuntu2.2 deb  +e2fsprogs 1.46.5-2ubuntu1.1 deb  +entrypoints 0.4 python  +exceptiongroup 1.2.0 python (+1 duplicate) +fastrlock 0.8.2 python  +filelock 3.13.1 python (+1 duplicate) +findutils 4.8.0-1ubuntu3 deb  +fontconfig-config 2.13.1-4.2ubuntu5 deb  +fonts-dejavu-core 2.37-2build1 deb  +fonttools 4.46.0 python  +fsspec 2023.12.1 python  +g++ 4:11.2.0-1ubuntu1 deb  +g++-11 11.4.0-1ubuntu1~22.04 deb  +gcc 4:11.2.0-1ubuntu1 deb  +gcc-11 11.4.0-1ubuntu1~22.04 deb  +gcc-11-base 11.4.0-1ubuntu1~22.04 deb  +gcc-12-base 12.3.0-1ubuntu1~22.04 deb  +gitdb 4.0.11 python  +gmpy2 2.1.2 python  +gnupg 2.2.27-3ubuntu2.1 deb  +gnupg-l10n 2.2.27-3ubuntu2.1 deb  +gnupg-utils 2.2.27-3ubuntu2.1 deb  +gnupg2 2.2.27-3ubuntu2.1 deb  +googleapis-common-protos 1.61.0 python  +gpg 2.2.27-3ubuntu2.1 deb  +gpg-agent 2.2.27-3ubuntu2.1 deb  +gpg-wks-client 2.2.27-3ubuntu2.1 deb  +gpg-wks-server 2.2.27-3ubuntu2.1 deb  +gpgconf 2.2.27-3ubuntu2.1 deb  +gpgsm 2.2.27-3ubuntu2.1 deb  +gpgv 2.2.27-3ubuntu2.1 deb  +greenlet 3.0.1 python  +grep 3.7-1build1 deb  +grpcio 1.54.2 python  +grpclib 0.4.6 python  +gunicorn 21.2.0 python  +gzip 1.10-4ubuntu4.1 deb  +h2 4.1.0 python  +hostname 3.23ubuntu2 deb  +hpack 4.0.0 python  +hyperframe 6.0.1 python  +idna 3.4 python  +idna 3.6 python  +imagesize 1.4.1 python  +importlib-metadata 7.0.0 python  +importlib-resources 6.1.1 python (+1 duplicate) +init-system-helpers 1.62 deb  +itsdangerous 2.1.2 python  +java-common 0.72build2 deb  +joblib 1.3.2 python (+1 duplicate) +jq 1.6-2.1ubuntu3 deb  +jrt-fs 11.0.21 java-archive  +js 1.0.0 npm  +json5 0.9.14 python  +jsonpatch 1.32 python  +jsonpointer 2.0 python  +jsonschema 4.20.0 python  +jsonschema-specifications 2023.11.2 python  +jupyter_client 8.6.0 python  +jupyter_core 5.5.0 python  +kiwisolver 1.4.5 python  +libacl1 2.3.1-1 deb  +libapt-pkg6.0 2.4.11 deb  +libarchive-c 5.0 python  +libasan6 11.4.0-1ubuntu1~22.04 deb  +libasound2 1.2.6.1-1ubuntu1 deb  +libasound2-data 1.2.6.1-1ubuntu1 deb  +libassuan0 2.5.5-1build1 deb  +libatomic1 12.3.0-1ubuntu1~22.04 deb  +libattr1 1:2.5.1-1build1 deb  +libaudit-common 1:3.0.7-1build1 deb  +libaudit1 1:3.0.7-1build1 deb  +libavahi-client3 0.8-5ubuntu5.2 deb  +libavahi-common-data 0.8-5ubuntu5.2 deb  +libavahi-common3 0.8-5ubuntu5.2 deb  +libbinutils 2.38-4ubuntu2.3 deb  +libblkid1 2.37.2-4ubuntu3 deb  +libbrotli1 1.0.9-2build6 deb  +libbsd0 0.11.5-1 deb  +libbz2-1.0 1.0.8-5build1 deb  +libc-bin 2.35-0ubuntu3.5 deb  +libc-dev-bin 2.35-0ubuntu3.5 deb  +libc6 2.35-0ubuntu3.5 deb  +libc6-dev 2.35-0ubuntu3.5 deb  +libcap-ng0 0.7.9-2.2build3 deb  +libcap2 1:2.44-1ubuntu0.22.04.1 deb  +libcbor0.8 0.8.0-2ubuntu1 deb  +libcc1-0 12.3.0-1ubuntu1~22.04 deb  +libcom-err2 1.46.5-2ubuntu1.1 deb  +libcrypt-dev 1:4.4.27-1 deb  +libcrypt1 1:4.4.27-1 deb  +libctf-nobfd0 2.38-4ubuntu2.3 deb  +libctf0 2.38-4ubuntu2.3 deb  +libcublas-11-8 11.11.3.6-1 deb  +libcufft-11-8 10.9.0.58-1 deb  +libcups2 2.4.1op1-1ubuntu4.7 deb  +libcurand-11-8 10.3.0.86-1 deb  +libcurl4 7.81.0-1ubuntu1.15 deb  +libcusolver-11-8 11.4.1.48-1 deb  +libdb5.3 5.3.28+dfsg1-0.8ubuntu3 deb  +libdbus-1-3 1.12.20-2ubuntu4.1 deb  +libdebconfclient0 0.261ubuntu1 deb  +libdpkg-perl 1.21.1ubuntu2.2 deb  +libedit2 3.1-20210910-1build1 deb  +libexpat1 2.4.7-1ubuntu0.2 deb  +libext2fs2 1.46.5-2ubuntu1.1 deb  +libffi8 3.4.2-4 deb  +libfido2-1 1.10.0-1 deb  +libfontconfig1 2.13.1-4.2ubuntu5 deb  +libfreetype6 2.11.1+dfsg-1ubuntu0.2 deb  +libgcc-11-dev 11.4.0-1ubuntu1~22.04 deb  +libgcc-s1 12.3.0-1ubuntu1~22.04 deb  +libgcrypt20 1.9.4-3ubuntu3 deb  +libgdbm-compat4 1.23-1 deb  +libgdbm6 1.23-1 deb  +libglib2.0-0 2.72.4-0ubuntu2.2 deb  +libgmp10 2:6.2.1+dfsg-3ubuntu1 deb  +libgnutls30 3.7.3-4ubuntu1.3 deb  +libgomp1 12.3.0-1ubuntu1~22.04 deb  +libgpg-error0 1.43-3 deb  +libgraphite2-3 1.3.14-1build2 deb  +libgssapi-krb5-2 1.19.2-2ubuntu0.3 deb  +libharfbuzz0b 2.7.4-1ubuntu3.1 deb  +libhogweed6 3.7.3-1build2 deb  +libidn2-0 2.3.2-2build1 deb  +libisl23 0.24-2build1 deb  +libitm1 12.3.0-1ubuntu1~22.04 deb  +libjpeg-turbo8 2.1.2-0ubuntu1 deb  +libjpeg8 8c-2ubuntu10 deb  +libjq1 1.6-2.1ubuntu3 deb  +libk5crypto3 1.19.2-2ubuntu0.3 deb  +libkeyutils1 1.6.1-2ubuntu3 deb  +libkrb5-3 1.19.2-2ubuntu0.3 deb  +libkrb5support0 1.19.2-2ubuntu0.3 deb  +libksba8 1.6.0-2ubuntu0.2 deb  +liblcms2-2 2.12~rc1-2build2 deb  +libldap-2.5-0 2.5.16+dfsg-0ubuntu0.22.04.1 deb  +liblsan0 12.3.0-1ubuntu1~22.04 deb  +liblz4-1 1.9.3-2build2 deb  +liblzma5 5.2.5-2ubuntu1 deb  +libmambapy 1.5.0 python  +libmd0 1.0.4-1build1 deb  +libmount1 2.37.2-4ubuntu3 deb  +libmpc3 1.2.1-2build1 deb  +libmpfr6 4.1.0-3build3 deb  +libncurses6 6.3-2ubuntu0.1 deb  +libncursesw6 6.3-2ubuntu0.1 deb  +libnettle8 3.7.3-1build2 deb  +libnghttp2-14 1.43.0-1ubuntu0.1 deb  +libnpth0 1.6-3build2 deb  +libnsl-dev 1.3.0-2build2 deb  +libnsl2 1.3.0-2build2 deb  +libnspr4 2:4.32-3build1 deb  +libnss3 2:3.68.2-0ubuntu1.2 deb  +libnuma1 2.0.14-3ubuntu2 deb  +libonig5 6.9.7.1-2build1 deb  +libp11-kit0 0.24.0-6build1 deb  +libpam-modules 1.4.0-11ubuntu2.3 deb  +libpam-modules-bin 1.4.0-11ubuntu2.3 deb  +libpam-runtime 1.4.0-11ubuntu2.3 deb  +libpam0g 1.4.0-11ubuntu2.3 deb  +libpcre2-8-0 10.39-3ubuntu0.1 deb  +libpcre3 2:8.39-13ubuntu0.22.04.1 deb  +libpcsclite1 1.9.5-3ubuntu1 deb  +libperl5.34 5.34.0-3ubuntu1.3 deb  +libpng16-16 1.6.37-3build5 deb  +libprocps8 2:3.3.17-6ubuntu2.1 deb  +libpsl5 0.21.0-1.2build2 deb  +libquadmath0 12.3.0-1ubuntu1~22.04 deb  +libreadline8 8.1.2-1 deb  +librtmp1 2.4+20151223.gitfa8646d.1-2build4 deb  +libsasl2-2 2.1.27+dfsg2-3ubuntu1.2 deb  +libsasl2-modules-db 2.1.27+dfsg2-3ubuntu1.2 deb  +libseccomp2 2.5.3-2ubuntu2 deb  +libselinux1 3.3-1build2 deb  +libsemanage-common 3.3-1build2 deb  +libsemanage2 3.3-1build2 deb  +libsepol2 3.3-1build1 deb  +libsmartcols1 2.37.2-4ubuntu3 deb  +libsqlite3-0 3.37.2-2ubuntu0.1 deb  +libss2 1.46.5-2ubuntu1.1 deb  +libssh-4 0.9.6-2ubuntu0.22.04.1 deb  +libssl3 3.0.2-0ubuntu1.12 deb  +libstdc++-11-dev 11.4.0-1ubuntu1~22.04 deb  +libstdc++6 12.3.0-1ubuntu1~22.04 deb  +libsystemd0 249.11-0ubuntu3.11 deb  +libtasn1-6 4.18.0-4build1 deb  +libtinfo6 6.3-2ubuntu0.1 deb  +libtirpc-common 1.3.2-2ubuntu0.1 deb  +libtirpc-dev 1.3.2-2ubuntu0.1 deb  +libtirpc3 1.3.2-2ubuntu0.1 deb  +libtsan0 11.4.0-1ubuntu1~22.04 deb  +libubsan1 12.3.0-1ubuntu1~22.04 deb  +libudev1 249.11-0ubuntu3.11 deb  +libunistring2 1.0-1 deb  +libuuid1 2.37.2-4ubuntu3 deb  +libxxhash0 0.8.1-1 deb  +libzstd1 1.4.8+dfsg-3build1 deb  +linux-libc-dev 5.15.0-89.99 deb  +llvmlite 0.41.1 python  +locket 1.0.0 python  +login 1:4.8.1-2ubuntu2.1 deb  +logsave 1.46.5-2ubuntu1.1 deb  +lsb-base 11.1.0ubuntu4 deb  +lto-disabled-list 24 deb  +lz4 4.3.2 python  +make 4.3-4.1build1 deb  +mamba 1.5.0 python  +markdown-it-py 3.0.0 python  +matplotlib 3.8.2 python  +mawk 1.3.4.20200120-3 deb  +mdurl 0.1.0 python  +menuinst 2.0.0 python  +merlin-core 23.6.0 python  +merlin-dataloader 23.6.0 python  +mlflow 2.9.1 python  +more-itertools 10.1.0 python  +morpheus 23.11.1 python  +mount 2.37.2-4ubuntu3 deb  +mpmath 1.3.0 python  +mrc 23.11.0 python  +msgpack 1.0.7 python  +multidict 6.0.4 python  +munkres 1.1.4 python  +nb-conda-kernels 2.3.1 python  +ncurses-base 6.3-2ubuntu0.1 deb  +ncurses-bin 6.3-2ubuntu0.1 deb  +networkx 3.2 python  +npy-append-array 0.9.16 python  +numba 0.58.1 python  +numpy 1.26.2 python  +numpydoc 1.4.0 python  +nvtabular 23.6.0 python  +nvtx 0.2.8 python  +oauthlib 3.2.2 python  +openjdk-11-jre-headless 11.0.21+9-0ubuntu1~22.04 deb  +openssh-client 1:8.9p1-3ubuntu0.4 deb  +openssl 3.0.2-0ubuntu1.12 deb  +openssl 3.2.0 binary  +ordered-set 4.1.0 python  +packaging 23.1 python  +packaging 23.2 python  +pandas 1.3.5 python  +partd 1.4.1 python  +passwd 1:4.8.1-2ubuntu2.1 deb  +patch 2.7.6-7build2 deb  +perl 5.34.0-3ubuntu1.3 deb  +perl-base 5.34.0-3ubuntu1.3 deb  +perl-modules-5.34 5.34.0-3ubuntu1.3 deb  +pinentry-curses 1.1.1-1build2 deb  +pip 23.3.1 python  +pkg-config 0.29.2-1ubuntu3 deb  +pkginfo 1.9.6 python  +pkgutil_resolve_name 1.3.10 python  +platformdirs 4.1.0 python  +pluggy 1.0.0 python  +pluggy 1.3.0 python  +procps 2:3.3.17-6ubuntu2.1 deb  +prometheus-client 0.19.0 python  +prometheus-flask-exporter 0.23.0 python  +prompt-toolkit 3.0.41 python  +protobuf 4.21.12 python  +psutil 5.9.5 python (+1 duplicate) +ptxcompiler 0.8.1 python  +pyOpenSSL 23.2.0 python  +pyOpenSSL 23.3.0 python  +pyarrow 11.0.0 python  +pyarrow-hotfix 0.6 python  +pycosat 0.6.4 python  +pycparser 2.21 python (+1 duplicate) +pynvml 11.4.1 python  +pyparsing 3.1.1 python  +python 3.10.12 binary  +python 3.10.13 binary  +python-dateutil 2.8.2 python  +python-rapidjson 1.13 python  +pytz 2023.3.post1 python (+1 duplicate) +pyzmq 25.1.2 python  +querystring-parser 1.2.4 python  +readline-common 8.1.2-1 deb  +referencing 0.31.1 python  +requests 2.31.0 python (+1 duplicate) +requests-cache 1.1.1 python  +rich 13.7.0 python  +rmm 23.6.0 python  +rpcsvc-proto 1.4.2-0ubuntu6 deb  +rpds-py 0.13.2 python  +ruamel.yaml 0.17.32 python  +ruamel.yaml.clib 0.2.7 python  +scikit-learn 1.2.2 python  +sed 4.8-1ubuntu2 deb  +sensible-utils 0.0.17 deb  +setuptools 59.8.0 python  +setuptools 68.1.2 python  +six 1.16.0 python  +smmap 5.0.0 python  +sniffio 1.3.0 python  +snowballstemmer 2.2.0 python  +sortedcontainers 2.4.0 python  +soupsieve 2.5 python  +sphinxcontrib-applehelp 1.0.7 python  +sphinxcontrib-devhelp 1.0.5 python  +sphinxcontrib-htmlhelp 2.0.4 python  +sphinxcontrib-jsmath 1.0.1 python  +sphinxcontrib-qthelp 1.0.6 python  +sphinxcontrib-serializinghtml 1.1.9 python  +sqlparse 0.4.4 python  +stringcase 1.2.0 python  +sympy 1.12 python  +sysvinit-utils 3.01-1ubuntu1 deb  +tabulate 0.9.0 python  +tar 1.34+dfsg-1ubuntu0.1.22.04.1 deb  +tblib 2.0.0 python  +tensorflow-metadata 1.13.1 python  +threadpoolctl 3.2.0 python  +tomli 2.0.1 python  +toolz 0.12.0 python (+1 duplicate) +torch 2.0.1 python  +tornado 6.3.3 python  +tqdm 4.66.1 python (+1 duplicate) +traitlets 5.14.0 python  +triton 2.0.0 python  +tritonclient 2.26.0 python  +typing-utils 0.1.0 python  +typing_extensions 4.8.0 python (+1 duplicate) +ubuntu-keyring 2021.03.26 deb  +ucf 3.0043 deb  +ujson 5.8.0 python  +unicodedata2 15.1.0 python  +unzip 6.0-26ubuntu3.1 deb  +url-normalize 1.4.3 python  +urllib3 2.0.4 python  +urllib3 2.1.0 python  +usrmerge 25ubuntu2 deb  +util-linux 2.37.2-4ubuntu3 deb  +watchdog 2.1.9 python  +watchgod 0.8.2 python  +wcwidth 0.2.12 python  +websocket-client 1.7.0 python  +websockets 12.0 python  +wget 1.21.2-2ubuntu1 deb  +wheel 0.41.2 python  +wheel 0.42.0 python  +xz-utils 5.2.5-2ubuntu1 deb  +zict 3.0.0 python  +zip 3.0-12build2 deb  +zipp 3.17.0 python (+1 duplicate) +zlib1g 1:1.2.11.dfsg-2ubuntu9.2 deb  +zstandard 0.19.0 python  diff --git a/data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom b/data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom new file mode 100644 index 0000000..ceef72f --- /dev/null +++ b/data/sboms/nvcr.io/nvidia/morpheus/morpheus:v24.03.02-runtime.sbom @@ -0,0 +1,493 @@ +NAME VERSION TYPE +Babel 2.14.0 python  +Brotli 1.1.0 python (+1 duplicate) +ConfigArgParse 1.5.5 python  +Flask 3.0.3 python  +GitPython 3.1.43 python  +Jinja2 3.1.3 python (+1 duplicate) +Mako 1.3.3 python  +Markdown 3.6 python  +MarkupSafe 2.1.5 python (+1 duplicate) +PyJWT 2.8.0 python  +PySocks 1.7.1 python (+1 duplicate) +PyYAML 6.0.1 python (+1 duplicate) +Pygments 2.17.2 python (+1 duplicate) +SQLAlchemy 1.4.49 python  +Simple Launcher 1.1.0.14 dotnet (+5 duplicates) +Sphinx 7.3.7 python  +Werkzeug 3.0.2 python  +absl-py 1.4.0 python  +adagio 0.2.4 python  +adduser 3.118ubuntu5 deb  +alabaster 0.7.16 python  +alembic 1.13.1 python  +aniso8601 9.0.1 python  +annotated-types 0.6.0 python  +antlr4-python3-runtime 4.11.1 python  +anyio 3.7.1 python  +appdirs 1.4.4 python  +apt 2.4.12 deb  +archspec 0.2.3 python  +argon2-cffi 23.1.0 python  +argon2-cffi-bindings 21.2.0 python  +asn1crypto 1.5.1 python  +attrs 23.2.0 python (+1 duplicate) +base-files 12ubuntu4.6 deb  +base-passwd 3.5.52build1 deb  +bash 5.1-6ubuntu1.1 deb  +bc 1.07.1-3build1 deb  +beautifulsoup4 4.12.3 python  +betterproto 1.2.5 python  +binutils 2.38-4ubuntu2.6 deb  +binutils-common 2.38-4ubuntu2.6 deb  +binutils-x86-64-linux-gnu 2.38-4ubuntu2.6 deb  +blinker 1.7.0 python  +boa 0.17.0 python  +bokeh 3.4.1 python  +boltons 24.0.0 python  +bsdutils 1:2.37.2-4ubuntu3.4 deb  +build-essential 12.9ubuntu3 deb  +bzip2 1.0.8-5build1 deb  +ca-certificates 20230311ubuntu0.22.04.1 deb  +ca-certificates-java java-archive  +ca-certificates-java 20190909ubuntu1.2 deb  +cachetools 5.3.3 python  +cattrs 23.2.3 python  +certifi 2024.2.2 python (+1 duplicate) +cffi 1.16.0 python (+1 duplicate) +chardet 5.2.0 python  +charset-normalizer 3.3.2 python (+1 duplicate) +click 8.1.7 python (+1 duplicate) +cloudpickle 3.0.0 python  +colorama 0.4.6 python (+1 duplicate) +conda 24.3.0 python  +conda-build 24.3.0 python  +conda-libmamba-solver 24.1.0 python  +conda-package-handling 2.2.0 python  +conda_index 0.4.0 python  +conda_package_streaming 0.9.0 python  +confluent-kafka 1.9.2 python  +contourpy 1.2.1 python  +coreutils 8.32-4.1ubuntu1.2 deb  +cpp 4:11.2.0-1ubuntu1 deb  +cpp-11 11.4.0-1ubuntu1~22.04 deb  +cryptography 42.0.5 python  +cuda-cccl-12-1 12.1.109-1 deb  +cuda-compat-12-1 530.30.02-1 deb  +cuda-compiler-12-1 12.1.1-1 deb  +cuda-cudart-12-1 12.1.105-1 deb  +cuda-cudart-dev-12-1 12.1.105-1 deb  +cuda-cuobjdump-12-1 12.1.111-1 deb  +cuda-cupti-12-1 12.1.105-1 deb  +cuda-cuxxfilt-12-1 12.1.105-1 deb  +cuda-driver-dev-12-1 12.1.105-1 deb  +cuda-gdb-12-1 12.1.105-1 deb  +cuda-keyring 1.1-1 deb  +cuda-nvcc-12-1 12.1.105-1 deb  +cuda-nvdisasm-12-1 12.1.105-1 deb  +cuda-nvprune-12-1 12.1.105-1 deb  +cuda-python 12.4.0 python  +cuda-toolkit-12-1-config-common 12.1.105-1 deb  +cuda-toolkit-12-config-common 12.4.127-1 deb  +cuda-toolkit-config-common 12.4.127-1 deb  +cudf 24.2.2 python  +cupy 13.1.0 python  +curl 7.81.0-1ubuntu1.16 deb  +cycler 0.12.1 python  +cytoolz 0.12.3 python  +dash 0.5.11+git20210903+057cd650a4ed-3build1 deb  +dask 2024.1.1 python  +dask-cuda 24.2.0 python  +dask-cudf 24.2.2 python  +databricks-cli 0.18.0 python  +databricks-connect 14.3.1 python  +databricks-sdk 0.26.0 python  +dataclasses 0.8 python  +datacompy 0.10.5 python  +debconf 1.5.79ubuntu1 deb  +debianutils 5.5-1ubuntu2 deb  +diffutils 1:3.8-0ubuntu2 deb  +dill 0.3.7 python  +dirmngr 2.2.27-3ubuntu2.1 deb  +distributed 2024.1.1 python  +distro 1.9.0 python  +docker 5.0.3 python  +docker-pycreds 0.4.0 python  +docutils 0.21.2 python  +dpkg 1.21.1ubuntu2.3 deb  +dpkg-dev 1.21.1ubuntu2.3 deb  +e2fsprogs 1.46.5-2ubuntu1.1 deb  +elastic-transport 8.13.0 python  +elasticsearch 8.9.0 python  +entrypoints 0.4 python  +environs 9.5.0 python  +exceptiongroup 1.2.0 python (+1 duplicate) +fastrlock 0.8.2 python  +feedparser 6.0.10 python  +filelock 3.13.4 python (+1 duplicate) +findutils 4.8.0-1ubuntu3 deb  +fontconfig-config 2.13.1-4.2ubuntu5 deb  +fonts-dejavu-core 2.37-2build1 deb  +fonttools 4.51.0 python  +fs 2.4.16 python  +fsspec 2024.3.1 python  +fugue 0.8.7 python  +fugue-sql-antlr 0.2.0 python  +g++ 4:11.2.0-1ubuntu1 deb  +g++-11 11.4.0-1ubuntu1~22.04 deb  +gcc 4:11.2.0-1ubuntu1 deb  +gcc-11 11.4.0-1ubuntu1~22.04 deb  +gcc-11-base 11.4.0-1ubuntu1~22.04 deb  +gcc-12-base 12.3.0-1ubuntu1~22.04 deb  +gitdb 4.0.11 python  +gmpy2 2.1.5 python  +gnupg 2.2.27-3ubuntu2.1 deb  +gnupg-l10n 2.2.27-3ubuntu2.1 deb  +gnupg-utils 2.2.27-3ubuntu2.1 deb  +gnupg2 2.2.27-3ubuntu2.1 deb  +google-auth 2.29.0 python  +googleapis-common-protos 1.63.0 python  +gpg 2.2.27-3ubuntu2.1 deb  +gpg-agent 2.2.27-3ubuntu2.1 deb  +gpg-wks-client 2.2.27-3ubuntu2.1 deb  +gpg-wks-server 2.2.27-3ubuntu2.1 deb  +gpgconf 2.2.27-3ubuntu2.1 deb  +gpgsm 2.2.27-3ubuntu2.1 deb  +gpgv 2.2.27-3ubuntu2.1 deb  +graphene 3.3 python  +graphql-core 3.2.3 python  +graphql-relay 3.2.0 python  +graphviz 0.20.3 python  +greenlet 3.0.3 python  +grep 3.7-1build1 deb  +grpcio 1.60.0 python  +grpcio-status 1.60.0 python  +grpclib 0.4.7 python  +gunicorn 21.2.0 python  +gzip 1.10-4ubuntu4.1 deb  +h2 4.1.0 python  +hostname 3.23ubuntu2 deb  +hpack 4.0.0 python  +hyperframe 6.0.1 python  +idna 3.6 python  +idna 3.7 python  +imagesize 1.4.1 python  +importlib_metadata 7.1.0 python  +importlib_resources 6.4.0 python (+1 duplicate) +init-system-helpers 1.62 deb  +itsdangerous 2.2.0 python  +java-common 0.72build2 deb  +joblib 1.4.0 python (+1 duplicate) +jq 1.6-2.1ubuntu3 deb  +jrt-fs 11.0.22 java-archive  +js 1.0.0 npm  +json5 0.9.25 python  +jsonpatch 1.33 python  +jsonpointer 2.4 python  +jsonschema 4.21.1 python  +jsonschema-specifications 2023.12.1 python  +kiwisolver 1.4.5 python  +libacl1 2.3.1-1 deb  +libapt-pkg6.0 2.4.12 deb  +libarchive-c 5.1 python  +libasan6 11.4.0-1ubuntu1~22.04 deb  +libasound2 1.2.6.1-1ubuntu1 deb  +libasound2-data 1.2.6.1-1ubuntu1 deb  +libassuan0 2.5.5-1build1 deb  +libatomic1 12.3.0-1ubuntu1~22.04 deb  +libattr1 1:2.5.1-1build1 deb  +libaudit-common 1:3.0.7-1build1 deb  +libaudit1 1:3.0.7-1build1 deb  +libavahi-client3 0.8-5ubuntu5.2 deb  +libavahi-common-data 0.8-5ubuntu5.2 deb  +libavahi-common3 0.8-5ubuntu5.2 deb  +libbinutils 2.38-4ubuntu2.6 deb  +libblkid1 2.37.2-4ubuntu3.4 deb  +libbrotli1 1.0.9-2build6 deb  +libbsd0 0.11.5-1 deb  +libbz2-1.0 1.0.8-5build1 deb  +libc-bin 2.35-0ubuntu3.7 deb  +libc-dev-bin 2.35-0ubuntu3.7 deb  +libc6 2.35-0ubuntu3.7 deb  +libc6-dev 2.35-0ubuntu3.7 deb  +libcap-ng0 0.7.9-2.2build3 deb  +libcap2 1:2.44-1ubuntu0.22.04.1 deb  +libcbor0.8 0.8.0-2ubuntu1 deb  +libcc1-0 12.3.0-1ubuntu1~22.04 deb  +libcom-err2 1.46.5-2ubuntu1.1 deb  +libcrypt-dev 1:4.4.27-1 deb  +libcrypt1 1:4.4.27-1 deb  +libctf-nobfd0 2.38-4ubuntu2.6 deb  +libctf0 2.38-4ubuntu2.6 deb  +libcublas-12-1 12.1.3.1-1 deb  +libcufft-12-1 11.0.2.54-1 deb  +libcups2 2.4.1op1-1ubuntu4.8 deb  +libcurand-12-1 10.3.2.106-1 deb  +libcurl4 7.81.0-1ubuntu1.16 deb  +libcusolver-12-1 11.4.5.107-1 deb  +libdb5.3 5.3.28+dfsg1-0.8ubuntu3 deb  +libdbus-1-3 1.12.20-2ubuntu4.1 deb  +libdebconfclient0 0.261ubuntu1 deb  +libdpkg-perl 1.21.1ubuntu2.3 deb  +libedit2 3.1-20210910-1build1 deb  +libexpat1 2.4.7-1ubuntu0.3 deb  +libext2fs2 1.46.5-2ubuntu1.1 deb  +libffi8 3.4.2-4 deb  +libfido2-1 1.10.0-1 deb  +libfontconfig1 2.13.1-4.2ubuntu5 deb  +libfreetype6 2.11.1+dfsg-1ubuntu0.2 deb  +libgcc-11-dev 11.4.0-1ubuntu1~22.04 deb  +libgcc-s1 12.3.0-1ubuntu1~22.04 deb  +libgcrypt20 1.9.4-3ubuntu3 deb  +libgdbm-compat4 1.23-1 deb  +libgdbm6 1.23-1 deb  +libglib2.0-0 2.72.4-0ubuntu2.2 deb  +libgmp10 2:6.2.1+dfsg-3ubuntu1 deb  +libgnutls30 3.7.3-4ubuntu1.5 deb  +libgomp1 12.3.0-1ubuntu1~22.04 deb  +libgpg-error0 1.43-3 deb  +libgraphite2-3 1.3.14-1build2 deb  +libgssapi-krb5-2 1.19.2-2ubuntu0.3 deb  +libharfbuzz0b 2.7.4-1ubuntu3.1 deb  +libhogweed6 3.7.3-1build2 deb  +libidn2-0 2.3.2-2build1 deb  +libisl23 0.24-2build1 deb  +libitm1 12.3.0-1ubuntu1~22.04 deb  +libjpeg-turbo8 2.1.2-0ubuntu1 deb  +libjpeg8 8c-2ubuntu10 deb  +libjq1 1.6-2.1ubuntu3 deb  +libk5crypto3 1.19.2-2ubuntu0.3 deb  +libkeyutils1 1.6.1-2ubuntu3 deb  +libkrb5-3 1.19.2-2ubuntu0.3 deb  +libkrb5support0 1.19.2-2ubuntu0.3 deb  +libksba8 1.6.0-2ubuntu0.2 deb  +liblcms2-2 2.12~rc1-2build2 deb  +libldap-2.5-0 2.5.17+dfsg-0ubuntu0.22.04.1 deb  +liblsan0 12.3.0-1ubuntu1~22.04 deb  +liblz4-1 1.9.3-2build2 deb  +liblzma5 5.2.5-2ubuntu1 deb  +libmambapy 1.5.8 python  +libmd0 1.0.4-1build1 deb  +libmount1 2.37.2-4ubuntu3.4 deb  +libmpc3 1.2.1-2build1 deb  +libmpfr6 4.1.0-3build3 deb  +libncurses6 6.3-2ubuntu0.1 deb  +libncursesw6 6.3-2ubuntu0.1 deb  +libnettle8 3.7.3-1build2 deb  +libnghttp2-14 1.43.0-1ubuntu0.1 deb  +libnpth0 1.6-3build2 deb  +libnsl-dev 1.3.0-2build2 deb  +libnsl2 1.3.0-2build2 deb  +libnspr4 2:4.35-0ubuntu0.22.04.1 deb  +libnss3 2:3.98-0ubuntu0.22.04.2 deb  +libnuma1 2.0.14-3ubuntu2 deb  +libonig5 6.9.7.1-2build1 deb  +libp11-kit0 0.24.0-6build1 deb  +libpam-modules 1.4.0-11ubuntu2.4 deb  +libpam-modules-bin 1.4.0-11ubuntu2.4 deb  +libpam-runtime 1.4.0-11ubuntu2.4 deb  +libpam0g 1.4.0-11ubuntu2.4 deb  +libpcre2-8-0 10.39-3ubuntu0.1 deb  +libpcre3 2:8.39-13ubuntu0.22.04.1 deb  +libpcsclite1 1.9.5-3ubuntu1 deb  +libperl5.34 5.34.0-3ubuntu1.3 deb  +libpng16-16 1.6.37-3build5 deb  +libprocps8 2:3.3.17-6ubuntu2.1 deb  +libpsl5 0.21.0-1.2build2 deb  +libquadmath0 12.3.0-1ubuntu1~22.04 deb  +libreadline8 8.1.2-1 deb  +librtmp1 2.4+20151223.gitfa8646d.1-2build4 deb  +libsasl2-2 2.1.27+dfsg2-3ubuntu1.2 deb  +libsasl2-modules-db 2.1.27+dfsg2-3ubuntu1.2 deb  +libseccomp2 2.5.3-2ubuntu2 deb  +libselinux1 3.3-1build2 deb  +libsemanage-common 3.3-1build2 deb  +libsemanage2 3.3-1build2 deb  +libsepol2 3.3-1build1 deb  +libsmartcols1 2.37.2-4ubuntu3.4 deb  +libsqlite3-0 3.37.2-2ubuntu0.3 deb  +libss2 1.46.5-2ubuntu1.1 deb  +libssh-4 0.9.6-2ubuntu0.22.04.3 deb  +libssl3 3.0.2-0ubuntu1.15 deb  +libstdc++-11-dev 11.4.0-1ubuntu1~22.04 deb  +libstdc++6 12.3.0-1ubuntu1~22.04 deb  +libsystemd0 249.11-0ubuntu3.12 deb  +libtasn1-6 4.18.0-4build1 deb  +libtinfo6 6.3-2ubuntu0.1 deb  +libtirpc-common 1.3.2-2ubuntu0.1 deb  +libtirpc-dev 1.3.2-2ubuntu0.1 deb  +libtirpc3 1.3.2-2ubuntu0.1 deb  +libtsan0 11.4.0-1ubuntu1~22.04 deb  +libubsan1 12.3.0-1ubuntu1~22.04 deb  +libudev1 249.11-0ubuntu3.12 deb  +libunistring2 1.0-1 deb  +libuuid1 2.37.2-4ubuntu3.4 deb  +libxxhash0 0.8.1-1 deb  +libzstd1 1.4.8+dfsg-3build1 deb  +lief 0.14.1 python  +linux-libc-dev 5.15.0-105.115 deb  +llvmlite 0.42.0 python  +locket 1.0.0 python  +login 1:4.8.1-2ubuntu2.2 deb  +logsave 1.46.5-2ubuntu1.1 deb  +lsb-base 11.1.0ubuntu4 deb  +lto-disabled-list 24 deb  +lz4 4.3.3 python  +make 4.3-4.1build1 deb  +mamba 1.5.8 python  +markdown-it-py 3.0.0 python (+1 duplicate) +marshmallow 3.21.1 python  +matplotlib 3.8.4 python  +mawk 1.3.4.20200120-3 deb  +mdurl 0.1.2 python (+1 duplicate) +menuinst 2.0.2 python  +merlin-core 23.8.0 python  +merlin-dataloader 23.8.0 python  +milvus 2.3.5 python  +minio 7.2.5 python  +mlflow 2.12.1 python  +mlflow-skinny 2.12.1 python  +more-itertools 10.2.0 python  +morpheus 24.3.2 python  +mount 2.37.2-4ubuntu3.4 deb  +mpmath 1.3.0 python  +mrc 24.3.1 python  +msgpack 1.0.7 python  +multidict 6.0.5 python  +munkres 1.1.4 python  +ncurses-base 6.3-2ubuntu0.1 deb  +ncurses-bin 6.3-2ubuntu0.1 deb  +networkx 2.8.8 python  +npy-append-array 0.9.16 python  +numba 0.59.1 python  +numpy 1.24.4 python  +numpydoc 1.5.0 python  +nvtabular 23.8.0 python  +nvtx 0.2.10 python  +oauthlib 3.2.2 python  +openjdk-11-jre-headless 11.0.22+7-0ubuntu2~22.04.1 deb  +openssh-client 1:8.9p1-3ubuntu0.7 deb  +openssl 3.0.2-0ubuntu1.15 deb  +openssl 3.2.1 binary  +ordered-set 4.1.0 python  +packaging 24.0 python (+1 duplicate) +pandas 1.5.3 python  +partd 1.4.1 python  +passwd 1:4.8.1-2ubuntu2.2 deb  +patch 2.7.6-7build2 deb  +perl 5.34.0-3ubuntu1.3 deb  +perl-base 5.34.0-3ubuntu1.3 deb  +perl-modules-5.34 5.34.0-3ubuntu1.3 deb  +pillow 10.3.0 python  +pinentry-curses 1.1.1-1build2 deb  +pip 24.0 python  +pkg-config 0.29.2-1ubuntu3 deb  +pkginfo 1.10.0 python  +pkgutil_resolve_name 1.3.10 python  +platformdirs 4.2.0 python  +platformdirs 4.2.1 python  +pluggy 1.3.0 python  +pluggy 1.4.0 python  +procps 2:3.3.17-6ubuntu2.1 deb  +prometheus-flask-exporter 0.23.0 python  +prometheus_client 0.20.0 python  +prompt-toolkit 3.0.42 python  +protobuf 4.24.4 python  +psutil 5.9.8 python (+1 duplicate) +py4j 0.10.9.7 python  +py4j0 10.9.7 java-archive  +pyOpenSSL 24.0.0 python  +pyarrow 14.0.2 python  +pyarrow-hotfix 0.6 python  +pyasn1 0.6.0 python  +pyasn1_modules 0.4.0 python  +pycosat 0.6.6 python  +pycparser 2.22 python (+1 duplicate) +pycryptodome 3.20.0 python  +pydantic 2.7.1 python  +pydantic_core 2.18.2 python  +pymilvus 2.3.6 python  +pynvjitlink 0.2.1 python  +pynvml 11.4.1 python  +pyparsing 3.1.2 python  +python 3.10.14 binary  +python-dateutil 2.9.0 python  +python-dotenv 1.0.1 python  +python-rapidjson 1.16 python  +pytz 2024.1 python (+1 duplicate) +qpd 0.4.4 python  +querystring-parser 1.2.4 python  +readline-common 8.1.2-1 deb  +referencing 0.34.0 python  +requests 2.31.0 python (+1 duplicate) +requests-cache 1.1.1 python  +requests-toolbelt 1.0.0 python  +rich 13.7.1 python (+1 duplicate) +rmm 24.2.0 python  +rpcsvc-proto 1.4.2-0ubuntu6 deb  +rpds-py 0.18.0 python  +rsa 4.9 python  +ruamel.yaml 0.18.6 python  +ruamel.yaml.clib 0.2.8 python  +scikit-learn 1.3.2 python  +scipy 1.13.0 python  +sed 4.8-1ubuntu2 deb  +sensible-utils 0.0.17 deb  +setuptools 69.5.1 python (+1 duplicate) +sgmllib3k 1.0.0 python  +six 1.16.0 python  +smmap 5.0.0 python  +sniffio 1.3.1 python  +snowballstemmer 2.2.0 python  +sortedcontainers 2.4.0 python  +soupsieve 2.5 python  +sphinxcontrib-applehelp 1.0.8 python  +sphinxcontrib-devhelp 1.0.6 python  +sphinxcontrib-htmlhelp 2.0.5 python  +sphinxcontrib-jsmath 1.0.1 python  +sphinxcontrib-qthelp 1.0.7 python  +sphinxcontrib-serializinghtml 1.1.10 python  +sqlglot 23.11.2 python  +sqlparse 0.4.4 python  +stringcase 1.2.0 python  +sympy 1.12 python  +sysvinit-utils 3.01-1ubuntu1 deb  +tabulate 0.9.0 python  +tar 1.34+dfsg-1ubuntu0.1.22.04.2 deb  +tblib 3.0.0 python  +tensorflow-metadata 1.13.1 python  +threadpoolctl 3.4.0 python  +tomli 2.0.1 python (+1 duplicate) +toolz 0.12.1 python  +torch 2.3.0 python  +tornado 6.4 python  +tqdm 4.66.2 python (+1 duplicate) +triad 0.9.6 python  +triton 2.3.0 python  +tritonclient 2.34.0 python  +truststore 0.8.0 python  +typing-utils 0.1.0 python  +typing_extensions 4.11.0 python (+1 duplicate) +ubuntu-keyring 2021.03.26 deb  +ucf 3.0043 deb  +ujson 5.9.0 python  +unicodedata2 15.1.0 python  +unzip 6.0-26ubuntu3.2 deb  +url-normalize 1.4.3 python  +urllib3 2.2.1 python (+1 duplicate) +usrmerge 25ubuntu2 deb  +util-linux 2.37.2-4ubuntu3.4 deb  +watchdog 3.0.0 python  +watchgod 0.8.2 python  +wcwidth 0.2.13 python  +websocket-client 1.8.0 python  +websockets 12.0 python  +wget 1.21.2-2ubuntu1 deb  +wheel 0.43.0 python (+1 duplicate) +xyzservices 2024.4.0 python  +xz-utils 5.2.5-2ubuntu1 deb  +zict 3.0.0 python  +zip 3.0-12build2 deb  +zipp 3.17.0 python (+1 duplicate) +zlib1g 1:1.2.11.dfsg-2ubuntu9.2 deb  +zstandard 0.22.0 python  diff --git a/data/sboms/nvcr.io/nvidia/rapidsai/base:23.08-cuda11.8-py3.10.sbom b/data/sboms/nvcr.io/nvidia/rapidsai/base:23.08-cuda11.8-py3.10.sbom new file mode 100644 index 0000000..a6c64aa --- /dev/null +++ b/data/sboms/nvcr.io/nvidia/rapidsai/base:23.08-cuda11.8-py3.10.sbom @@ -0,0 +1,621 @@ +NAME VERSION TYPE +@colors/colors 1.5.0 npm  +@holoviz/panel 1.2.1 npm  +@isaacs/cliui 8.0.2 npm  +@isaacs/string-locale-compare 1.1.0 npm  +@jupyterhub/jupyter-server-proxy 4.0.0 npm  +@npmcli/arborist 6.3.0 npm  +@npmcli/config 6.2.1 npm  +@npmcli/disparity-colors 3.0.0 npm  +@npmcli/fs 3.1.0 npm  +@npmcli/git 4.1.0 npm  +@npmcli/installed-package-contents 2.0.2 npm  +@npmcli/map-workspaces 3.0.4 npm  +@npmcli/metavuln-calculator 5.0.1 npm  +@npmcli/name-from-folder 2.0.0 npm  +@npmcli/node-gyp 3.0.0 npm  +@npmcli/package-json 4.0.0 npm  +@npmcli/promise-spawn 6.0.2 npm  +@npmcli/query 3.0.0 npm  +@npmcli/run-script 6.0.2 npm  +@pkgjs/parseargs 0.11.0 npm  +@pyviz/jupyterlab_pyviz 3.0.0 npm (+1 duplicate) +@sigstore/protobuf-specs 0.1.0 npm  +@sigstore/tuf 1.0.2 npm  +@tootallnate/once 2.0.0 npm  +@tufjs/canonical-json 1.0.0 npm  +@tufjs/models 1.0.4 npm  +Brotli 1.0.9 python  +Fiona 1.9.4 python  +GDAL 3.7.1 python  +Jinja2 3.1.2 python  +Markdown 3.4.4 python  +MarkupSafe 2.1.3 python  +Pillow 10.0.0 python  +PySocks 1.7.1 python  +PyWavelets 1.4.1 python  +PyYAML 6.0.1 python  +Pygments 2.16.1 python  +Rtree 1.0.1 python  +Send2Trash 1.8.2 python  +Simple Launcher 1.1.0.14 dotnet (+5 duplicates) +abbrev 1.1.1 npm  +abbrev 2.0.0 npm  +abort-controller 3.0.0 npm  +adduser 3.118ubuntu5 deb  +agent-base 6.0.2 npm  +agentkeepalive 4.3.0 npm  +aggregate-error 3.1.0 npm  +aiohttp 3.8.5 python  +aiosignal 1.3.1 python  +annotated-types 0.5.0 python  +ansi-regex 5.0.1 npm  +ansi-regex 6.0.1 npm (+1 duplicate) +ansi-styles 4.3.0 npm  +ansi-styles 6.2.1 npm  +anyio 3.7.1 python  +appdirs 1.4.4 python  +aproba 2.0.0 npm  +apt 2.4.10 deb  +archy 1.0.0 npm  +are-we-there-yet 3.0.1 npm  +are-we-there-yet 4.0.0 npm  +argon2-cffi 23.1.0 python  +argon2-cffi-bindings 21.2.0 python  +arrow 1.2.3 python  +asttokens 2.2.1 python  +async-timeout 4.0.3 python  +attrs 23.1.0 python  +backcall 0.2.0 python  +backports.functools-lru-cache 1.6.5 python  +balanced-match 1.0.2 npm  +base-files 12ubuntu4.4 deb  +base-passwd 3.5.52build1 deb  +base64-js 1.5.1 npm  +bash 5.1-6ubuntu1 deb  +beautifulsoup4 4.12.2 python  +bin-links 4.0.1 npm  +binary-extensions 2.2.0 npm  +bleach 6.0.0 python  +bokeh 3.2.2 python  +boltons 23.0.0 python  +brace-expansion 1.1.11 npm (+1 duplicate) +brace-expansion 2.0.1 npm  +branca 0.6.0 python  +brotlipy 0.7.0 python  +bsdutils 1:2.37.2-4ubuntu3 deb  +buffer 6.0.3 npm  +builtins 5.0.1 npm  +ca-certificates 20230311ubuntu0.22.04.1 deb  +cacache 17.1.3 npm  +cached-property 1.5.2 python  +cachetools 5.3.1 python  +certifi 2023.7.22 python  +cffi 1.15.1 python  +chalk 5.2.0 npm  +charset-normalizer 3.2.0 python  +chownr 2.0.0 npm  +ci-info 3.8.0 npm  +cidr-regex 3.1.1 npm  +clean-stack 2.2.0 npm  +cli-columns 4.0.0 npm  +cli-table3 0.6.3 npm  +click 8.1.7 python  +click-plugins 1.1.1 python  +cligj 0.7.2 python  +clone 1.0.4 npm  +cloudpickle 2.2.1 python  +cmd-shim 6.0.1 npm  +color-convert 2.0.1 npm  +color-name 1.1.4 npm  +color-support 1.1.3 npm  +colorama 0.4.6 python  +colorcet 3.0.1 python  +columnify 1.6.0 npm  +common-ancestor-path 1.0.1 npm  +concat-map 0.0.1 npm  +conda 23.7.3 python  +conda-libmamba-solver 23.7.0 python  +conda-package-handling 2.2.0 python  +conda_package_streaming 0.9.0 python  +confluent-kafka 1.9.2 python  +console-control-strings 1.1.0 npm  +contourpy 1.1.0 python  +corepack 0.19.0 npm  +coreutils 8.32-4.1ubuntu1 deb  +cross-spawn 7.0.3 npm  +cryptography 41.0.3 python  +cssesc 3.0.0 npm  +cubinlinker 0.3.0 python  +cucim 23.8.0 python  +cuda-compat-11-8 520.61.05-1 deb  +cuda-cudart-11-8 11.8.89-1 deb  +cuda-keyring 1.1-1 deb  +cuda-python 11.8.2 python  +cuda-toolkit-11-8-config-common 11.8.89-1 deb  +cuda-toolkit-11-config-common 11.8.89-1 deb  +cuda-toolkit-config-common 12.2.128-1 deb  +cudf 23.8.0 python  +cudf-kafka 23.8.0 python  +cugraph 23.8.0 python  +cuml 23.8.0 python  +cuproj 23.8.0 python  +cupy 12.2.0 python  +cusignal 23.8.0 python  +cuspatial 23.8.0 python  +custreamz 23.8.0 python  +cuxfilter 23.8 python  +cycler 0.11.0 python  +cytoolz 0.12.2 python  +dash 0.5.11+git20210903+057cd650a4ed-3build1 deb  +dask 2023.7.1 python  +dask-cuda 23.8.0 python  +dask-cudf 23.8.0 python  +dask-sql 2023.8.0 python  +datashader 0.15.2 python  +datashape 0.5.4 python  +debconf 1.5.79ubuntu1 deb  +debianutils 5.5-1ubuntu2 deb  +debug 4.3.4 npm  +decorator 5.1.1 python  +defaults 1.0.4 npm  +defusedxml 0.7.1 python  +delegates 1.0.0 npm  +depd 2.0.0 npm  +diff 5.1.0 npm  +diffutils 1:3.8-0ubuntu2 deb  +dirmngr 2.2.27-3ubuntu2.1 deb  +distributed 2023.7.1 python  +dpkg 1.21.1ubuntu2.2 deb  +e2fsprogs 1.46.5-2ubuntu1.1 deb  +eastasianwidth 0.2.0 npm  +emoji-regex 8.0.0 npm  +emoji-regex 9.2.2 npm (+1 duplicate) +encoding 0.1.13 npm  +entrypoints 0.4 python  +env-paths 2.2.1 npm  +err-code 2.0.3 npm  +event-target-shim 5.0.1 npm  +events 3.3.0 npm  +exceptiongroup 1.1.3 python  +executing 1.2.0 python  +exponential-backoff 3.1.1 npm  +fastapi 0.101.1 python  +fastest-levenshtein 1.0.16 npm  +fastjsonschema 2.18.0 python  +fastrlock 0.8 python  +findutils 4.8.0-1ubuntu3 deb  +folium 0.14.0 python  +fonttools 4.42.1 python  +foreground-child 3.1.1 npm  +fqdn 1.5.1 python  +frozenlist 1.4.0 python  +fs-minipass 2.1.0 npm  +fs-minipass 3.0.2 npm  +fs.realpath 1.0.0 npm  +fsspec 2023.6.0 python  +function-bind 1.1.1 npm  +gauge 4.0.4 npm  +gauge 5.0.1 npm  +gcc-12-base 12.3.0-1ubuntu1~22.04 deb  +geopandas 0.13.2 python  +glob 10.2.7 npm  +glob 7.2.3 npm (+1 duplicate) +gnupg 2.2.27-3ubuntu2.1 deb  +gnupg-l10n 2.2.27-3ubuntu2.1 deb  +gnupg-utils 2.2.27-3ubuntu2.1 deb  +gnupg2 2.2.27-3ubuntu2.1 deb  +gpg 2.2.27-3ubuntu2.1 deb  +gpg-agent 2.2.27-3ubuntu2.1 deb  +gpg-wks-client 2.2.27-3ubuntu2.1 deb  +gpg-wks-server 2.2.27-3ubuntu2.1 deb  +gpgconf 2.2.27-3ubuntu2.1 deb  +gpgsm 2.2.27-3ubuntu2.1 deb  +gpgv 2.2.27-3ubuntu2.1 deb  +graceful-fs 4.2.11 npm  +grep 3.7-1build1 deb  +gzip 1.10-4ubuntu4.1 deb  +h11 0.14.0 python  +h2 4.1.0 python  +has 1.0.3 npm  +has-unicode 2.0.1 npm  +holoviews 1.17.1 python  +hosted-git-info 6.1.1 npm  +hostname 3.23ubuntu2 deb  +hpack 4.0.0 python  +html5lib 1.1 python  +http-cache-semantics 4.1.1 npm  +http-proxy-agent 5.0.0 npm  +httpcore 0.17.3 python  +https-proxy-agent 5.0.1 npm  +httpx 0.24.1 python  +humanize-ms 1.2.1 npm  +hyperframe 6.0.1 python  +iconv-lite 0.6.3 npm  +idna 3.4 python  +ieee754 1.2.1 npm  +ignore-walk 6.0.3 npm  +imagecodecs 2023.8.12 python  +imageio 2.31.1 python  +importlib-metadata 6.8.0 python  +importlib-resources 6.0.1 python  +imurmurhash 0.1.4 npm  +indent-string 4.0.0 npm  +inflight 1.0.6 npm  +inherits 2.0.4 npm  +ini 4.1.1 npm  +init-package-json 5.0.0 npm  +init-system-helpers 1.62 deb  +ip 2.0.0 npm  +ip-regex 4.3.0 npm  +ipython 8.14.0 python  +is-cidr 4.0.2 npm  +is-core-module 2.12.1 npm  +is-fullwidth-code-point 3.0.0 npm  +is-lambda 1.0.1 npm  +isexe 2.0.0 npm  +isoduration 20.11.0 python  +jackspeak 2.2.1 npm  +jedi 0.19.0 python  +joblib 1.3.2 python  +json-parse-even-better-errors 3.0.0 npm  +json-stringify-nice 1.1.4 npm  +jsonparse 1.3.1 npm  +jsonpatch 1.32 python  +jsonpointer 2.0 python  +jsonschema 4.19.0 python  +jsonschema-specifications 2023.7.1 python  +jupyter-events 0.7.0 python  +jupyter_client 8.3.0 python  +jupyter_core 5.3.1 python  +jupyter_server 2.7.1 python  +jupyter_server_proxy 4.0.0 python  +jupyter_server_terminals 0.4.4 python  +jupyterlab-pygments 0.2.2 python  +jupyterlab_pygments 0.2.2 npm (+1 duplicate) +just-diff 6.0.2 npm  +just-diff-apply 5.5.0 npm  +kiwisolver 1.4.5 python  +lazy_loader 0.3 python  +libacl1 2.3.1-1 deb  +libapt-pkg6.0 2.4.10 deb  +libassuan0 2.5.5-1build1 deb  +libattr1 1:2.5.1-1build1 deb  +libaudit-common 1:3.0.7-1build1 deb  +libaudit1 1:3.0.7-1build1 deb  +libblkid1 2.37.2-4ubuntu3 deb  +libbz2-1.0 1.0.8-5build1 deb  +libc-bin 2.35-0ubuntu3.1 deb  +libc-dev-bin 2.35-0ubuntu3.1 deb  +libc6 2.35-0ubuntu3.1 deb  +libc6-dev 2.35-0ubuntu3.1 deb  +libcap-ng0 0.7.9-2.2build3 deb  +libcap2 1:2.44-1ubuntu0.22.04.1 deb  +libcom-err2 1.46.5-2ubuntu1.1 deb  +libcrypt-dev 1:4.4.27-1 deb  +libcrypt1 1:4.4.27-1 deb  +libdb5.3 5.3.28+dfsg1-0.8ubuntu3 deb  +libdebconfclient0 0.261ubuntu1 deb  +libext2fs2 1.46.5-2ubuntu1.1 deb  +libffi8 3.4.2-4 deb  +libgcc-s1 12.3.0-1ubuntu1~22.04 deb  +libgcrypt20 1.9.4-3ubuntu3 deb  +libgmp10 2:6.2.1+dfsg-3ubuntu1 deb  +libgnutls30 3.7.3-4ubuntu1.2 deb  +libgpg-error0 1.43-3 deb  +libgssapi-krb5-2 1.19.2-2ubuntu0.2 deb  +libhogweed6 3.7.3-1build2 deb  +libidn2-0 2.3.2-2build1 deb  +libk5crypto3 1.19.2-2ubuntu0.2 deb  +libkeyutils1 1.6.1-2ubuntu3 deb  +libkrb5-3 1.19.2-2ubuntu0.2 deb  +libkrb5support0 1.19.2-2ubuntu0.2 deb  +libksba8 1.6.0-2ubuntu0.2 deb  +libldap-2.5-0 2.5.16+dfsg-0ubuntu0.22.04.1 deb  +liblz4-1 1.9.3-2build2 deb  +liblzma5 5.2.5-2ubuntu1 deb  +libmambapy 1.5.0 python  +libmount1 2.37.2-4ubuntu3 deb  +libncurses6 6.3-2ubuntu0.1 deb  +libncursesw6 6.3-2ubuntu0.1 deb  +libnettle8 3.7.3-1build2 deb  +libnpmaccess 7.0.2 npm  +libnpmdiff 5.0.19 npm  +libnpmexec 6.0.2 npm  +libnpmfund 4.0.19 npm  +libnpmhook 9.0.3 npm  +libnpmorg 5.0.4 npm  +libnpmpack 5.0.19 npm  +libnpmpublish 7.5.0 npm  +libnpmsearch 6.0.2 npm  +libnpmteam 5.0.3 npm  +libnpmversion 4.0.2 npm  +libnpth0 1.6-3build2 deb  +libnsl-dev 1.3.0-2build2 deb  +libnsl2 1.3.0-2build2 deb  +libnuma-dev 2.0.14-3ubuntu2 deb  +libnuma1 2.0.14-3ubuntu2 deb  +libp11-kit0 0.24.0-6build1 deb  +libpam-modules 1.4.0-11ubuntu2.3 deb  +libpam-modules-bin 1.4.0-11ubuntu2.3 deb  +libpam-runtime 1.4.0-11ubuntu2.3 deb  +libpam0g 1.4.0-11ubuntu2.3 deb  +libpcre2-8-0 10.39-3ubuntu0.1 deb  +libpcre3 2:8.39-13ubuntu0.22.04.1 deb  +libprocps8 2:3.3.17-6ubuntu2 deb  +libreadline8 8.1.2-1 deb  +libsasl2-2 2.1.27+dfsg2-3ubuntu1.2 deb  +libsasl2-modules-db 2.1.27+dfsg2-3ubuntu1.2 deb  +libseccomp2 2.5.3-2ubuntu2 deb  +libselinux1 3.3-1build2 deb  +libsemanage-common 3.3-1build2 deb  +libsemanage2 3.3-1build2 deb  +libsepol2 3.3-1build1 deb  +libsmartcols1 2.37.2-4ubuntu3 deb  +libsqlite3-0 3.37.2-2ubuntu0.1 deb  +libss2 1.46.5-2ubuntu1.1 deb  +libssl3 3.0.2-0ubuntu1.10 deb  +libstdc++6 12.3.0-1ubuntu1~22.04 deb  +libsystemd0 249.11-0ubuntu3.9 deb  +libtasn1-6 4.18.0-4build1 deb  +libtinfo6 6.3-2ubuntu0.1 deb  +libtirpc-common 1.3.2-2ubuntu0.1 deb  +libtirpc-dev 1.3.2-2ubuntu0.1 deb  +libtirpc3 1.3.2-2ubuntu0.1 deb  +libudev1 249.11-0ubuntu3.9 deb  +libunistring2 1.0-1 deb  +libuuid1 2.37.2-4ubuntu3 deb  +libxxhash0 0.8.1-1 deb  +libzstd1 1.4.8+dfsg-3build1 deb  +linkify-it-py 2.0.0 python  +linux-libc-dev 5.15.0-79.86 deb  +llvmlite 0.40.1 python  +locket 1.0.0 python  +login 1:4.8.1-2ubuntu2.1 deb  +logsave 1.46.5-2ubuntu1.1 deb  +lru-cache 6.0.0 npm  +lru-cache 7.18.3 npm  +lru-cache 9.1.1 npm  +lsb-base 11.1.0ubuntu4 deb  +lz4 4.3.2 python  +make-fetch-happen 11.1.1 npm  +mamba 1.5.0 python  +mapclassify 2.5.0 python  +markdown-it-py 3.0.0 python  +matplotlib 3.7.2 python  +matplotlib-inline 0.1.6 python  +mawk 1.3.4.20200120-3 deb  +mdit-py-plugins 0.4.0 python  +mdurl 0.1.0 python  +minimatch 3.1.2 npm (+1 duplicate) +minimatch 9.0.1 npm  +minipass 3.3.6 npm (+6 duplicates) +minipass 5.0.0 npm  +minipass-collect 1.0.2 npm  +minipass-fetch 3.0.3 npm  +minipass-flush 1.0.5 npm  +minipass-json-stream 1.0.1 npm  +minipass-pipeline 1.2.4 npm  +minipass-sized 1.0.3 npm  +minizlib 2.1.2 npm  +mistune 3.0.1 python  +mkdirp 1.0.4 npm  +mount 2.37.2-4ubuntu3 deb  +ms 2.1.2 npm  +ms 2.1.3 npm  +msgpack 1.0.5 python  +multidict 6.0.4 python  +multipledispatch 0.6.0 python  +munch 4.0.0 python  +munkres 1.1.4 python  +mute-stream 1.0.0 npm  +nbclient 0.8.0 python  +nbconvert 7.7.4 python  +nbformat 5.9.2 python  +ncurses-base 6.3-2ubuntu0.1 deb  +ncurses-bin 6.3-2ubuntu0.1 deb  +negotiator 0.6.3 npm  +networkx 3.1 python  +node-gyp 9.4.0 npm  +nopt 6.0.0 npm  +nopt 7.2.0 npm  +normalize-package-data 5.0.0 npm  +npm 9.8.0 npm  +npm-audit-report 5.0.0 npm  +npm-bundled 3.0.0 npm  +npm-install-checks 6.1.1 npm  +npm-normalize-package-bin 3.0.1 npm  +npm-package-arg 10.1.0 npm  +npm-packlist 7.0.4 npm  +npm-pick-manifest 8.0.1 npm  +npm-profile 7.0.1 npm  +npm-registry-fetch 14.0.5 npm  +npm-user-validate 2.0.0 npm  +npmlog 6.0.2 npm  +npmlog 7.0.1 npm  +numba 0.57.1 python  +numpy 1.24.4 python  +nvtx 0.2.5 python  +once 1.4.0 npm  +openssl 3.0.2-0ubuntu1.10 deb  +openssl 3.1.2 binary  +overrides 7.4.0 python  +p-map 4.0.0 npm  +packaging 23.1 python  +pacote 15.2.0 npm  +pandas 1.5.3 python  +pandocfilters 1.5.0 python  +panel 1.2.1 python  +param 1.13.0 python  +parse-conflict-json 3.0.1 npm  +parso 0.8.3 python  +partd 1.4.0 python  +passwd 1:4.8.1-2ubuntu2.1 deb  +path-is-absolute 1.0.1 npm  +path-key 3.1.1 npm  +path-scurry 1.9.2 npm  +perl-base 5.34.0-3ubuntu1.2 deb  +pexpect 4.8.0 python  +pickleshare 0.7.5 python  +pinentry-curses 1.1.1-1build2 deb  +pip 23.2.1 python  +pkgutil_resolve_name 1.3.10 python  +platformdirs 3.10.0 python  +pluggy 1.2.0 python  +pooch 1.7.0 python  +postcss-selector-parser 6.0.13 npm  +postgresql 15.4 binary  +proc-log 3.0.0 npm  +process 0.11.10 npm  +procps 2:3.3.17-6ubuntu2 deb  +prometheus-client 0.17.1 python  +promise-all-reject-late 1.0.1 npm  +promise-call-limit 1.0.2 npm  +promise-inflight 1.0.1 npm  +promise-retry 2.0.1 npm  +prompt-toolkit 3.0.39 python  +promzard 1.0.0 npm  +protobuf 4.23.3 python  +psutil 5.9.5 python  +ptxcompiler 0.8.1 python  +ptyprocess 0.7.0 python  +pure-eval 0.2.2 python  +pyOpenSSL 23.2.0 python  +pyarrow 11.0.0 python  +pycosat 0.6.4 python  +pycparser 2.21 python  +pyct 0.4.6 python  +pydantic 2.3.0 python  +pydantic_core 2.6.3 python  +pyee 8.1.0 python  +pylibcugraph 23.8.0 python  +pylibraft 23.8.0 python  +pynvml 11.4.1 python  +pyparsing 3.0.9 python  +pyppeteer 1.0.2 python  +pyproj 3.6.0 python  +python 3.10.12 binary  +python-dateutil 2.8.2 python  +python-json-logger 2.0.7 python  +pytz 2023.3 python  +pyviz_comms 3.0.0 python  +pyzmq 25.1.1 python  +qrcode-terminal 0.12.0 npm  +raft-dask 23.8.0 python  +read 2.1.0 npm  +read-cmd-shim 4.0.0 npm  +read-package-json 6.0.4 npm  +read-package-json-fast 3.0.2 npm  +readable-stream 3.6.2 npm  +readable-stream 4.4.0 npm  +readline-common 8.1.2-1 deb  +referencing 0.30.2 python  +requests 2.31.0 python  +retry 0.12.0 npm  +rfc3339-validator 0.1.4 python  +rfc3986-validator 0.1.1 python  +rimraf 3.0.2 npm  +rmm 23.8.0 python  +rpcsvc-proto 1.4.2-0ubuntu6 deb  +rpds-py 0.9.2 python  +ruamel.yaml 0.17.32 python  +ruamel.yaml.clib 0.2.7 python  +safe-buffer 5.2.1 npm  +safer-buffer 2.1.2 npm  +scikit-image 0.21.0 python  +scikit-learn 1.3.0 python  +scipy 1.11.2 python  +sed 4.8-1ubuntu2 deb  +semver 7.5.2 npm  +sensible-utils 0.0.17 deb  +set-blocking 2.0.0 npm  +setuptools 68.1.2 python  +shapely 2.0.1 python  +shebang-command 2.0.0 npm  +shebang-regex 3.0.0 npm  +signal-exit 3.0.7 npm  +signal-exit 4.0.2 npm  +sigstore 1.7.0 npm  +simpervisor 1.0.0 python  +six 1.16.0 python  +smart-buffer 4.2.0 npm  +sniffio 1.3.0 python  +socks 2.7.1 npm  +socks-proxy-agent 7.0.0 npm  +sortedcontainers 2.4.0 python  +soupsieve 2.3.2.post1 python  +spdx-correct 3.2.0 npm  +spdx-exceptions 2.3.0 npm  +spdx-expression-parse 3.0.1 npm  +spdx-license-ids 3.0.13 npm  +ssri 10.0.4 npm  +stack-data 0.6.2 python  +starlette 0.27.0 python  +streamz 0.6.4 python  +string-width 4.2.3 npm (+1 duplicate) +string-width 5.1.2 npm (+1 duplicate) +string_decoder 1.3.0 npm  +strip-ansi 6.0.1 npm (+1 duplicate) +strip-ansi 7.1.0 npm (+1 duplicate) +supports-color 9.3.1 npm  +sysvinit-utils 3.01-1ubuntu1 deb  +tabulate 0.9.0 python  +tar 1.34+dfsg-1ubuntu0.1.22.04.1 deb  +tar 6.1.15 npm  +tblib 1.7.0 python  +terminado 0.17.1 python  +text-table 0.2.0 npm  +threadpoolctl 3.2.0 python  +tifffile 2023.8.12 python  +tiny-relative-date 1.3.0 npm  +tinycss2 1.2.1 python  +toolz 0.12.0 python  +tornado 6.3.3 python  +tqdm 4.66.1 python  +traitlets 5.9.0 python  +treelite 3.2.0 python  +treelite-runtime 3.2.0 python  +treeverse 3.0.0 npm  +tuf-js 1.1.7 npm  +typing-utils 0.1.0 python  +typing_extensions 4.7.1 python  +tzdata 2023c-0ubuntu0.22.04.2 deb  +tzlocal 5.0.1 python  +ubuntu-keyring 2021.03.26 deb  +uc-micro-py 1.0.1 python  +ucx-py 0.33.0 python  +unicodedata2 15.0.0 python  +unique-filename 3.0.0 npm  +unique-slug 4.0.0 npm  +uri-template 1.3.0 python  +urllib3 1.26.15 python  +usrmerge 25ubuntu2 deb  +util-deprecate 1.0.2 npm  +util-linux 2.37.2-4ubuntu3 deb  +uvicorn 0.23.2 python  +validate-npm-package-license 3.0.4 npm  +validate-npm-package-name 5.0.0 npm  +walk-up-path 3.0.1 npm  +wcwidth 0.2.6 python  +wcwidth 1.0.1 npm  +webcolors 1.13 python  +webencodings 0.5.1 python  +websocket-client 1.6.2 python  +websockets 10.4 python  +wheel 0.41.2 python  +which 2.0.2 npm (+1 duplicate) +which 3.0.1 npm  +wide-align 1.1.5 npm  +wrap-ansi 7.0.0 npm  +wrap-ansi 8.1.0 npm  +wrappy 1.0.2 npm  +write-file-atomic 5.0.1 npm  +xarray 2023.8.0 python  +xgboost 1.7.4 python  +xyzservices 2023.7.0 python  +yallist 4.0.0 npm  +yarl 1.9.2 python  +zict 3.0.0 python  +zipp 3.16.2 python  +zlib1g 1:1.2.11.dfsg-2ubuntu9.2 deb  +zstandard 0.19.0 python  diff --git a/data/sboms/nvcr.io/nvidia/rapidsai/base:24.06-cuda12.2-py3.10.sbom b/data/sboms/nvcr.io/nvidia/rapidsai/base:24.06-cuda12.2-py3.10.sbom new file mode 100644 index 0000000..e53daca --- /dev/null +++ b/data/sboms/nvcr.io/nvidia/rapidsai/base:24.06-cuda12.2-py3.10.sbom @@ -0,0 +1,599 @@ +NAME VERSION TYPE +@colors/colors 1.5.0 npm  +@holoviz/panel 1.4.4 npm  +@isaacs/cliui 8.0.2 npm  +@isaacs/string-locale-compare 1.1.0 npm  +@jupyterhub/jupyter-server-proxy 4.2.0 npm  +@npmcli/agent 2.2.1 npm  +@npmcli/arborist 7.4.0 npm  +@npmcli/config 8.2.0 npm  +@npmcli/disparity-colors 3.0.0 npm  +@npmcli/fs 3.1.0 npm  +@npmcli/git 5.0.4 npm  +@npmcli/installed-package-contents 2.0.2 npm  +@npmcli/map-workspaces 3.0.4 npm  +@npmcli/metavuln-calculator 7.0.0 npm  +@npmcli/name-from-folder 2.0.0 npm  +@npmcli/node-gyp 3.0.0 npm  +@npmcli/package-json 5.0.0 npm  +@npmcli/promise-spawn 7.0.1 npm  +@npmcli/query 3.1.0 npm  +@npmcli/run-script 7.0.4 npm  +@pkgjs/parseargs 0.11.0 npm  +@pyviz/jupyterlab_pyviz 3.0.2 npm (+1 duplicate) +@sigstore/bundle 2.2.0 npm  +@sigstore/core 1.0.0 npm  +@sigstore/protobuf-specs 0.3.0 npm  +@sigstore/sign 2.2.3 npm  +@sigstore/tuf 2.3.1 npm  +@sigstore/verify 1.1.0 npm  +@tufjs/canonical-json 2.0.0 npm  +@tufjs/models 2.0.0 npm  +Brotli 1.1.0 python  +GDAL 3.8.1 python  +Jinja2 3.1.4 python  +Markdown 3.6 python  +MarkupSafe 2.1.5 python  +PySocks 1.7.1 python  +PyWavelets 1.4.1 python  +PyYAML 6.0.1 python  +Pygments 2.18.0 python  +Rtree 1.2.0 python  +Send2Trash 1.8.3 python  +Simple Launcher 1.1.0.14 dotnet (+5 duplicates) +abbrev 2.0.0 npm  +adduser 3.118ubuntu5 deb  +agent-base 7.1.0 npm  +aggregate-error 3.1.0 npm  +aiohttp 3.9.5 python  +aiosignal 1.3.1 python  +annotated-types 0.7.0 python  +ansi-regex 5.0.1 npm  +ansi-regex 6.0.1 npm (+1 duplicate) +ansi-styles 4.3.0 npm (+1 duplicate) +ansi-styles 6.2.1 npm  +anyio 4.4.0 python  +appdirs 1.4.4 python  +aproba 2.0.0 npm  +apt 2.4.12 deb  +archspec 0.2.3 python  +archy 1.0.0 npm  +are-we-there-yet 4.0.2 npm  +argon2-cffi 23.1.0 python  +argon2-cffi-bindings 21.2.0 python  +arrow 1.3.0 python  +asttokens 2.4.1 python  +async-timeout 4.0.3 python  +attrs 23.2.0 python  +balanced-match 1.0.2 npm  +base-files 12ubuntu4.6 deb  +base-passwd 3.5.52build1 deb  +bash 5.1-6ubuntu1.1 deb  +beautifulsoup4 4.12.3 python  +bin-links 4.0.3 npm  +binary-extensions 2.2.0 npm  +bleach 6.1.0 python  +bokeh 3.4.1 python  +boltons 24.0.0 python  +brace-expansion 2.0.1 npm  +branca 0.7.2 python  +bsdutils 1:2.37.2-4ubuntu3.4 deb  +builtins 5.0.1 npm  +ca-certificates 20230311ubuntu0.22.04.1 deb  +cacache 18.0.2 npm  +cached-property 1.5.2 python  +cachetools 5.3.3 python  +certifi 2024.6.2 python  +cffi 1.16.0 python  +chalk 5.3.0 npm  +charset-normalizer 3.3.2 python  +chownr 2.0.0 npm  +ci-info 4.0.0 npm  +cidr-regex 4.0.3 npm  +clean-stack 2.2.0 npm  +cli-columns 4.0.0 npm  +cli-table3 0.6.3 npm  +click 8.1.7 python  +click-plugins 1.1.1 python  +cligj 0.7.2 python  +clone 1.0.4 npm  +cloudpickle 3.0.0 python  +cmd-shim 6.0.2 npm  +color-convert 2.0.1 npm  +color-name 1.1.4 npm  +color-support 1.1.3 npm  +colorama 0.4.6 python  +colorcet 3.1.0 python  +columnify 1.6.0 npm  +common-ancestor-path 1.0.1 npm  +conda 24.5.0 python  +conda-libmamba-solver 24.1.0 python  +conda-package-handling 2.3.0 python  +conda_package_streaming 0.10.0 python  +confluent-kafka 1.9.2 python  +console-control-strings 1.1.0 npm  +contourpy 1.2.1 python  +corepack 0.25.2 npm  +coreutils 8.32-4.1ubuntu1.2 deb  +cross-spawn 7.0.3 npm  +cssesc 3.0.0 npm  +cucim 24.6.0 python  +cuda-compat-12-2 535.183.01-1 deb  +cuda-cudart-12-2 12.2.140-1 deb  +cuda-keyring 1.1-1 deb  +cuda-python 12.5.0 python  +cuda-toolkit-12-2-config-common 12.2.140-1 deb  +cuda-toolkit-12-config-common 12.5.39-1 deb  +cuda-toolkit-config-common 12.5.39-1 deb  +cudf 24.6.0 python  +cudf_kafka 24.6.0 python  +cugraph 24.6.1 python  +cuml 24.6.1 python  +cuproj 24.6.0 python  +cupy 13.2.0 python  +cuspatial 24.6.0 python  +custreamz 24.6.0 python  +cuvs 24.6.0 python  +cuxfilter 24.6.0 python  +cycler 0.12.1 python  +cytoolz 0.12.3 python  +dash 0.5.11+git20210903+057cd650a4ed-3build1 deb  +dask 2024.5.1 python  +dask-cuda 24.6.0 python  +dask-cudf 24.6.0 python  +dask-expr 1.1.1 python  +dask_sql 2024.5.0 python  +datashader 0.16.2 python  +debconf 1.5.79ubuntu1 deb  +debianutils 5.5-1ubuntu2 deb  +debug 4.3.4 npm  +decorator 5.1.1 python  +defaults 1.0.4 npm  +defusedxml 0.7.1 python  +diff 5.2.0 npm  +diffutils 1:3.8-0ubuntu2 deb  +dirmngr 2.2.27-3ubuntu2.1 deb  +distributed 2024.5.1 python  +distributed-ucxx 0.38.0 python  +distro 1.9.0 python  +dnspython 2.6.1 python  +dpkg 1.21.1ubuntu2.3 deb  +e2fsprogs 1.46.5-2ubuntu1.1 deb  +eastasianwidth 0.2.0 npm  +email_validator 2.1.1 python  +emoji-regex 8.0.0 npm  +emoji-regex 9.2.2 npm (+1 duplicate) +encoding 0.1.13 npm  +entrypoints 0.4 python  +env-paths 2.2.1 npm  +err-code 2.0.3 npm  +exceptiongroup 1.2.0 python  +executing 2.0.1 python  +exponential-backoff 3.1.1 npm  +fastapi 0.111.0 python  +fastapi-cli 0.0.4 python  +fastest-levenshtein 1.0.16 npm  +fastjsonschema 2.19.1 python  +fastrlock 0.8.2 python  +findutils 4.8.0-1ubuntu3 deb  +fiona 1.9.5 python  +folium 0.16.0 python  +fonttools 4.53.0 python  +foreground-child 3.1.1 npm  +fqdn 1.5.1 python  +frozendict 2.4.4 python  +frozenlist 1.4.1 python  +fs-minipass 2.1.0 npm  +fs-minipass 3.0.3 npm  +fsspec 2024.6.0 python  +function-bind 1.1.2 npm  +gauge 5.0.1 npm  +gcc-12-base 12.3.0-1ubuntu1~22.04 deb  +geopandas 0.14.4 python  +glob 10.3.10 npm  +gnupg 2.2.27-3ubuntu2.1 deb  +gnupg-l10n 2.2.27-3ubuntu2.1 deb  +gnupg-utils 2.2.27-3ubuntu2.1 deb  +gnupg2 2.2.27-3ubuntu2.1 deb  +gpg 2.2.27-3ubuntu2.1 deb  +gpg-agent 2.2.27-3ubuntu2.1 deb  +gpg-wks-client 2.2.27-3ubuntu2.1 deb  +gpg-wks-server 2.2.27-3ubuntu2.1 deb  +gpgconf 2.2.27-3ubuntu2.1 deb  +gpgsm 2.2.27-3ubuntu2.1 deb  +gpgv 2.2.27-3ubuntu2.1 deb  +graceful-fs 4.2.11 npm  +grep 3.7-1build1 deb  +gzip 1.10-4ubuntu4.1 deb  +h11 0.14.0 python  +h2 4.1.0 python  +has-unicode 2.0.1 npm  +hasown 2.0.1 npm  +holoviews 1.19.0 python  +hosted-git-info 7.0.1 npm  +hostname 3.23ubuntu2 deb  +hpack 4.0.0 python  +html5lib 1.1 python  +http-cache-semantics 4.1.1 npm  +http-proxy-agent 7.0.2 npm  +httpcore 1.0.5 python  +https-proxy-agent 7.0.4 npm  +httpx 0.27.0 python  +hyperframe 6.0.1 python  +iconv-lite 0.6.3 npm  +idna 3.7 python  +ignore-walk 6.0.4 npm  +imagecodecs-lite 2019.12.3 python  +imageio 2.34.1 python  +importlib_metadata 7.1.0 python  +importlib_resources 6.4.0 python  +imurmurhash 0.1.4 npm  +indent-string 4.0.0 npm  +ini 4.1.1 npm  +init-package-json 6.0.0 npm  +init-system-helpers 1.62 deb  +ip-address 9.0.5 npm  +ip-regex 5.0.0 npm  +ipython 8.25.0 python  +is-cidr 5.0.3 npm  +is-core-module 2.13.1 npm  +is-fullwidth-code-point 3.0.0 npm  +is-lambda 1.0.1 npm  +isexe 2.0.0 npm  +isexe 3.1.1 npm  +isoduration 20.11.0 python  +jackspeak 2.3.6 npm  +jedi 0.19.1 python  +joblib 1.4.2 python  +jsbn 1.1.0 npm  +json-parse-even-better-errors 3.0.1 npm  +json-stringify-nice 1.1.4 npm  +jsonparse 1.3.1 npm  +jsonpatch 1.33 python  +jsonpointer 3.0.0 python  +jsonschema 4.22.0 python  +jsonschema-specifications 2023.12.1 python  +jupyter-events 0.10.0 python  +jupyter_client 8.6.2 python  +jupyter_core 5.7.2 python  +jupyter_server 2.14.1 python  +jupyter_server_proxy 4.2.0 python  +jupyter_server_terminals 0.5.3 python  +jupyterlab_pygments 0.3.0 npm  +jupyterlab_pygments 0.3.0 python  +just-diff 6.0.2 npm  +just-diff-apply 5.5.0 npm  +kiwisolver 1.4.5 python  +lazy_loader 0.4 python  +libacl1 2.3.1-1 deb  +libapt-pkg6.0 2.4.12 deb  +libassuan0 2.5.5-1build1 deb  +libattr1 1:2.5.1-1build1 deb  +libaudit-common 1:3.0.7-1build1 deb  +libaudit1 1:3.0.7-1build1 deb  +libblkid1 2.37.2-4ubuntu3.4 deb  +libbz2-1.0 1.0.8-5build1 deb  +libc-bin 2.35-0ubuntu3.8 deb  +libc6 2.35-0ubuntu3.8 deb  +libcap-ng0 0.7.9-2.2build3 deb  +libcap2 1:2.44-1ubuntu0.22.04.1 deb  +libcom-err2 1.46.5-2ubuntu1.1 deb  +libcrypt1 1:4.4.27-1 deb  +libdb5.3 5.3.28+dfsg1-0.8ubuntu3 deb  +libdebconfclient0 0.261ubuntu1 deb  +libext2fs2 1.46.5-2ubuntu1.1 deb  +libffi8 3.4.2-4 deb  +libgcc-s1 12.3.0-1ubuntu1~22.04 deb  +libgcrypt20 1.9.4-3ubuntu3 deb  +libgmp10 2:6.2.1+dfsg-3ubuntu1 deb  +libgnutls30 3.7.3-4ubuntu1.5 deb  +libgpg-error0 1.43-3 deb  +libgssapi-krb5-2 1.19.2-2ubuntu0.3 deb  +libhogweed6 3.7.3-1build2 deb  +libidn2-0 2.3.2-2build1 deb  +libk5crypto3 1.19.2-2ubuntu0.3 deb  +libkeyutils1 1.6.1-2ubuntu3 deb  +libkrb5-3 1.19.2-2ubuntu0.3 deb  +libkrb5support0 1.19.2-2ubuntu0.3 deb  +libksba8 1.6.0-2ubuntu0.2 deb  +libldap-2.5-0 2.5.17+dfsg-0ubuntu0.22.04.1 deb  +liblz4-1 1.9.3-2build2 deb  +liblzma5 5.2.5-2ubuntu1 deb  +libmambapy 1.5.8 python  +libmount1 2.37.2-4ubuntu3.4 deb  +libncurses6 6.3-2ubuntu0.1 deb  +libncursesw6 6.3-2ubuntu0.1 deb  +libnettle8 3.7.3-1build2 deb  +libnpmaccess 8.0.2 npm  +libnpmdiff 6.0.7 npm  +libnpmexec 7.0.8 npm  +libnpmfund 5.0.5 npm  +libnpmhook 10.0.1 npm  +libnpmorg 6.0.2 npm  +libnpmpack 6.0.7 npm  +libnpmpublish 9.0.4 npm  +libnpmsearch 7.0.1 npm  +libnpmteam 6.0.1 npm  +libnpmversion 5.0.2 npm  +libnpth0 1.6-3build2 deb  +libnsl2 1.3.0-2build2 deb  +libp11-kit0 0.24.0-6build1 deb  +libpam-modules 1.4.0-11ubuntu2.4 deb  +libpam-modules-bin 1.4.0-11ubuntu2.4 deb  +libpam-runtime 1.4.0-11ubuntu2.4 deb  +libpam0g 1.4.0-11ubuntu2.4 deb  +libpcre2-8-0 10.39-3ubuntu0.1 deb  +libpcre3 2:8.39-13ubuntu0.22.04.1 deb  +libprocps8 2:3.3.17-6ubuntu2.1 deb  +libreadline8 8.1.2-1 deb  +libsasl2-2 2.1.27+dfsg2-3ubuntu1.2 deb  +libsasl2-modules-db 2.1.27+dfsg2-3ubuntu1.2 deb  +libseccomp2 2.5.3-2ubuntu2 deb  +libselinux1 3.3-1build2 deb  +libsemanage-common 3.3-1build2 deb  +libsemanage2 3.3-1build2 deb  +libsepol2 3.3-1build1 deb  +libsmartcols1 2.37.2-4ubuntu3.4 deb  +libsqlite3-0 3.37.2-2ubuntu0.3 deb  +libss2 1.46.5-2ubuntu1.1 deb  +libssl3 3.0.2-0ubuntu1.15 deb  +libstdc++6 12.3.0-1ubuntu1~22.04 deb  +libsystemd0 249.11-0ubuntu3.12 deb  +libtasn1-6 4.18.0-4build1 deb  +libtinfo6 6.3-2ubuntu0.1 deb  +libtirpc-common 1.3.2-2ubuntu0.1 deb  +libtirpc3 1.3.2-2ubuntu0.1 deb  +libudev1 249.11-0ubuntu3.12 deb  +libunistring2 1.0-1 deb  +libuuid1 2.37.2-4ubuntu3.4 deb  +libxxhash0 0.8.1-1 deb  +libzstd1 1.4.8+dfsg-3build1 deb  +linkify-it-py 2.0.3 python  +llvmlite 0.42.0 python  +locket 1.0.0 python  +login 1:4.8.1-2ubuntu2.2 deb  +logsave 1.46.5-2ubuntu1.1 deb  +lru-cache 10.2.0 npm  +lru-cache 6.0.0 npm  +lsb-base 11.1.0ubuntu4 deb  +lz4 4.3.3 python  +make-fetch-happen 13.0.0 npm  +mamba 1.5.8 python  +mapclassify 2.6.1 python  +markdown-it-py 3.0.0 python  +matplotlib 3.8.4 python  +matplotlib-inline 0.1.7 python  +mawk 1.3.4.20200120-3 deb  +mdit-py-plugins 0.4.1 python  +mdurl 0.1.2 python  +menuinst 2.1.1 python  +minimatch 9.0.3 npm  +minipass 3.3.6 npm (+5 duplicates) +minipass 5.0.0 npm  +minipass 7.0.4 npm  +minipass-collect 2.0.1 npm  +minipass-fetch 3.0.4 npm  +minipass-flush 1.0.5 npm  +minipass-json-stream 1.0.1 npm  +minipass-pipeline 1.2.4 npm  +minipass-sized 1.0.3 npm  +minizlib 2.1.2 npm  +mistune 3.0.2 python  +mkdirp 1.0.4 npm  +mount 2.37.2-4ubuntu3.4 deb  +ms 2.1.2 npm  +ms 2.1.3 npm  +msgpack 1.0.8 python  +multidict 6.0.5 python  +multipledispatch 0.6.0 python  +munkres 1.1.4 python  +mute-stream 1.0.0 npm  +nbclient 0.10.0 python  +nbconvert 7.16.4 python  +nbformat 5.10.4 python  +ncurses-base 6.3-2ubuntu0.1 deb  +ncurses-bin 6.3-2ubuntu0.1 deb  +negotiator 0.6.3 npm  +networkx 3.3 python  +node-gyp 10.0.1 npm  +nopt 7.2.0 npm  +normalize-package-data 6.0.0 npm  +npm 10.5.0 npm  +npm-audit-report 5.0.0 npm  +npm-bundled 3.0.0 npm  +npm-install-checks 6.3.0 npm  +npm-normalize-package-bin 3.0.1 npm  +npm-package-arg 11.0.1 npm  +npm-packlist 8.0.2 npm  +npm-pick-manifest 9.0.0 npm  +npm-profile 9.0.0 npm  +npm-registry-fetch 16.1.0 npm  +npm-user-validate 2.0.0 npm  +npmlog 7.0.1 npm  +numba 0.59.1 python  +numpy 1.26.4 python  +nvtx 0.2.10 python  +nx-cugraph 24.6.1 python  +openssl 3.0.2-0ubuntu1.15 deb  +openssl 3.3.1 binary  +orjson 3.10.4 python  +overrides 7.7.0 python  +p-map 4.0.0 npm  +packaging 24.1 python  +pacote 17.0.6 npm  +pandas 2.2.2 python  +pandocfilters 1.5.0 python  +panel 1.4.4 python  +param 2.1.0 python  +parse-conflict-json 3.0.1 npm  +parso 0.8.4 python  +partd 1.4.2 python  +passwd 1:4.8.1-2ubuntu2.2 deb  +path-key 3.1.1 npm  +path-scurry 1.10.1 npm  +perl-base 5.34.0-3ubuntu1.3 deb  +pexpect 4.9.0 python  +pickleshare 0.7.5 python  +pillow 10.3.0 python  +pinentry-curses 1.1.1-1build2 deb  +pip 24.0 python  +pkgutil_resolve_name 1.3.10 python  +platformdirs 4.2.2 python  +pluggy 1.5.0 python  +postcss-selector-parser 6.0.15 npm  +postgresql 16.3 binary  +proc-log 3.0.0 npm  +procps 2:3.3.17-6ubuntu2.1 deb  +prometheus_client 0.20.0 python  +promise-all-reject-late 1.0.1 npm  +promise-call-limit 3.0.1 npm  +promise-inflight 1.0.1 npm  +promise-retry 2.0.1 npm  +prompt_toolkit 3.0.47 python  +promzard 1.0.0 npm  +psutil 5.9.8 python  +ptyprocess 0.7.0 python  +pure-eval 0.2.2 python  +pyarrow 16.1.0 python  +pyarrow-hotfix 0.6 python  +pycosat 0.6.6 python  +pycparser 2.22 python  +pyct 0.5.0 python  +pydantic 2.7.4 python  +pydantic_core 2.18.4 python  +pyee 8.1.0 python  +pylibcugraph 24.6.1 python  +pylibraft 24.6.0 python  +pynvjitlink-cu12 0.2.4 python  +pynvml 11.4.1 python  +pyparsing 3.1.2 python  +pyppeteer 1.0.2 python  +pyproj 3.6.1 python  +python 3.10.14 binary  +python-dateutil 2.9.0 python  +python-json-logger 2.0.7 python  +python-multipart 0.0.9 python  +pytz 2024.1 python  +pyviz_comms 3.0.2 python  +pyzmq 26.0.3 python  +qrcode-terminal 0.12.0 npm  +raft-dask 24.6.0 python  +rapids-dask-dependency 24.6.0a0 python  +read 2.1.0 npm  +read-cmd-shim 4.0.0 npm  +read-package-json 7.0.0 npm  +read-package-json-fast 3.0.2 npm  +readline-common 8.1.2-1 deb  +referencing 0.35.1 python  +requests 2.32.3 python  +retry 0.12.0 npm  +rfc3339-validator 0.1.4 python  +rfc3986-validator 0.1.1 python  +rich 13.7.1 python  +rmm 24.6.0 python  +rpds-py 0.18.1 python  +ruamel.yaml 0.18.6 python  +ruamel.yaml.clib 0.2.8 python  +safer-buffer 2.1.2 npm  +scikit-image 0.20.0 python  +scikit-learn 1.5.0 python  +scipy 1.13.1 python  +sed 4.8-1ubuntu2 deb  +semver 7.6.0 npm  +sensible-utils 0.0.17 deb  +set-blocking 2.0.0 npm  +setuptools 70.0.0 python  +shapely 2.0.4 python  +shebang-command 2.0.0 npm  +shebang-regex 3.0.0 npm  +shellingham 1.5.4 python  +signal-exit 4.1.0 npm  +sigstore 2.2.2 npm  +simpervisor 1.0.0 python  +six 1.16.0 python  +smart-buffer 4.2.0 npm  +sniffio 1.3.1 python  +socks 2.8.0 npm  +socks-proxy-agent 8.0.2 npm  +sortedcontainers 2.4.0 python  +soupsieve 2.5 python  +spdx-correct 3.2.0 npm  +spdx-exceptions 2.5.0 npm  +spdx-expression-parse 3.0.1 npm  +spdx-license-ids 3.0.17 npm  +sprintf-js 1.1.3 npm  +ssri 10.0.5 npm  +stack-data 0.6.2 python  +starlette 0.37.2 python  +streamz 0.6.4 python  +string-width 4.2.3 npm (+1 duplicate) +string-width 5.1.2 npm (+1 duplicate) +strip-ansi 6.0.1 npm (+1 duplicate) +strip-ansi 7.1.0 npm (+1 duplicate) +supports-color 9.4.0 npm  +sysvinit-utils 3.01-1ubuntu1 deb  +tabulate 0.9.0 python  +tar 1.34+dfsg-1ubuntu0.1.22.04.2 deb  +tar 6.2.0 npm  +tblib 3.0.0 python  +terminado 0.18.1 python  +text-table 0.2.0 npm  +threadpoolctl 3.5.0 python  +tifffile 2020.6.3 python  +tiny-relative-date 1.3.0 npm  +tinycss2 1.3.0 python  +toolz 0.12.1 python  +tornado 6.4.1 python  +tqdm 4.66.4 python  +traitlets 5.14.3 python  +treelite 4.1.2 python  +treeverse 3.0.0 npm  +truststore 0.8.0 python  +tuf-js 2.2.0 npm  +typer 0.12.3 python  +typer-slim 0.12.3 python  +types-python-dateutil 2.9.0.20240316 python  +typing-utils 0.1.0 python  +typing_extensions 4.12.2 python  +tzdata 2024.1 python  +tzdata 2024a-0ubuntu0.22.04 deb  +tzlocal 5.2 python  +ubuntu-keyring 2021.03.26 deb  +uc-micro-py 1.0.3 python  +ucx-py 0.38.0 python  +ucxx 0.38.0 python  +ujson 5.10.0 python  +unicodedata2 15.1.0 python  +unique-filename 3.0.0 npm  +unique-slug 4.0.0 npm  +uri-template 1.3.0 python  +urllib3 1.26.18 python  +usrmerge 25ubuntu2 deb  +util-deprecate 1.0.2 npm  +util-linux 2.37.2-4ubuntu3.4 deb  +uvicorn 0.30.1 python  +validate-npm-package-license 3.0.4 npm  +validate-npm-package-name 5.0.0 npm  +walk-up-path 3.0.1 npm  +wcwidth 0.2.13 python  +wcwidth 1.0.1 npm  +webcolors 24.6.0 python  +webencodings 0.5.1 python  +websocket-client 1.8.0 python  +websockets 10.4 python  +wheel 0.43.0 python  +which 2.0.2 npm  +which 4.0.0 npm  +wide-align 1.1.5 npm  +wrap-ansi 7.0.0 npm  +wrap-ansi 8.1.0 npm  +write-file-atomic 5.0.1 npm  +xarray 2024.6.0 python  +xgboost 2.0.3 python  +xyzservices 2024.6.0 python  +yallist 4.0.0 npm  +yarl 1.9.4 python  +zict 3.0.0 python  +zipp 3.19.2 python  +zlib1g 1:1.2.11.dfsg-2ubuntu9.2 deb  +zstandard 0.19.0 python  diff --git a/data/validation_output/cve_validation_formatted_openai.json b/data/validation_output/cve_validation_formatted_openai.json new file mode 100644 index 0000000..7477efb --- /dev/null +++ b/data/validation_output/cve_validation_formatted_openai.json @@ -0,0 +1,147 @@ +[ + { + "cve": "CVE-2018-21028", + "input_check_list": [ + "Check the Boa version: The vulnerability affects Boa versions up to and including 0.94.14rc21. Verify the version of Boa running in the Docker container. If it is using a vulnerable version, consider updating to a patched version.", + "Identify potential memory leaks: The vulnerability involves missing calls to the free function, leading to memory leaks. Review the code and configuration of the applications running in the Docker container to identify any instances where resources are not being released properly.", + "Assess the impact: The vulnerability does not have a direct impact on confidentiality or integrity (CVSS:C:N/I:N). However, it can lead to a denial of service (CVSS:A:H). Evaluate the potential impact of a denial of service attack on the Docker container and the applications it hosts.", + "Evaluate network exposure: Determine if the Docker container or any services within it are exposed to the network. Assess the potential for remote attackers to exploit the vulnerability by triggering memory leaks.", + "Consider the CWE description: The CWE description highlights the importance of releasing resources after their effective lifetime has ended. Review the code and configuration of the applications running in the Docker container to ensure that resources are properly released when no longer needed.", + "Verify the disputed status: Check if there are any disputes regarding the vulnerability. It is important to consider the credibility and consensus among security researchers and vendors regarding the existence and severity of the vulnerability." + ], + "output_check_list": [ + "I don't know the version of Boa running in the Docker container.", + "To prevent memory leaks in a Docker container, you can use the --scale option when running docker-compose up to limit the number of instances running. In practice, 3 instances have been shown to work well. Use as many instances as required. Keep in mind each instance takes about 1 Gb of memory.", + "Based on the information gathered, the potential impact of a denial of service attack on the Docker container and the applications it hosts can vary depending on the specific vulnerabilities present in the software versions used. It is important to further investigate if any vulnerable functions or components are used in the Docker container's code or its dependencies to fully assess the impact.", + "The Docker container is not exposed to the network, reducing the potential for remote attackers to exploit the vulnerability by triggering memory leaks. However, to fully assess its vulnerability, we need more information about the libraries and dependencies used by the Docker container. Unfortunately, I don't have access to the Docker Container Software Bill of Materials QA System to gather this information. We can try using the Internet Search tool to find relevant resources or consult with Docker documentation for more details.", + "Based on the information gathered, it is important to review the code and configuration of the applications running in the Docker container to ensure that resources are properly released when no longer needed. The Docker Container Code QA System did not provide any specific information about the applications using proper resource release mechanisms. The Docker Container Developer Guide QA System mentioned that the Docker container is responsible for managing resources such as memory, CPU, and disk space, but did not provide specific details about resource release. The Docker Container Software Bill of Materials QA System directed me to the Docker website, but an internet search did not provide specific information about the libraries used in the Docker container and their resource release mechanisms. Therefore, further investigation may be required to determine how resources are released in the Docker container.", + "The disputed status of a vulnerability can be determined by considering information from security researchers and vendors, as well as the reasons why a CVE ID may be rejected or disputed." + ] + }, + { + "cve": "CVE-2023-36632", + "input_check_list": [ + "Check the Python version: The vulnerability affects Python versions through 3.11.4. If your Docker container is running Python, verify the version. If it is through 3.11.4, it may be vulnerable.", + "Identify usage of the legacy email.utils.parseaddr function: Check all applications within your Docker container for usage of the legacy email.utils.parseaddr function. This function is categorized as a Legacy API, and applications are recommended to use the email.parser.BytesParser or email.parser.Parser class instead. If any application is using the legacy function, it may be vulnerable.", + "Evaluate the input data: Determine if the applications input data includes a name and an email address. If so, consider whether the input data is untrusted and potentially crafted by an attacker. If untrusted input data is passed to the legacy email.utils.parseaddr function, it may trigger the vulnerability.", + "Assess resource consumption: The vulnerability can lead to a \"RecursionError: maximum recursion depth exceeded while calling a Python object.\" Consider whether the Docker container has resource limits in place to prevent excessive recursion and control resource consumption. If not, it may be more susceptible to this vulnerability.", + "Vendor perspective: Note that the vendors perspective is that this is neither a vulnerability nor a bug. However, it is still important to assess the impact and potential risks within your specific environment.", + "Network exposure: Determine if the Docker container has network exposure, especially if the vulnerable Python code or any service using it is exposed to the internet. This will help assess the potential attack surface.", + "CVSS score: The CVSS 3.1 score for this vulnerability is 4.13, indicating a moderate severity. Consider this score in relation to your organization's risk tolerance and prioritize accordingly." + ], + "output_check_list": [ + "The Python version in the Docker container is 3.10.6.", + "The legacy email.utils.parseaddr function is not used in any of the applications within the Docker container, so it is not vulnerable to this specific legacy API.", + "The input data in the Docker container does not include a name and an email address, and therefore does not trigger the vulnerability.", + "No, the Docker container does not have resource limits in place.", + "The Docker container includes the following libraries in its software bill of materials: libassuan0, libatomic1, libattr1, libbsd0, libbz2-1.0, libc-bin, libisl23, libitm1, libjq1, NETStandard.Library, and Newtonsoft.Json. The Docker container's software version is within the vulnerability range of 1.0.0 to 1.2.0. I have searched the CVE system and found information about the reported CVE.", + "To determine if the Docker container has network exposure, it is necessary to analyze the Docker API to check for network-related functions or components used within the container. This can provide information about the network configuration and help assess the potential attack surface.", + "The reported CVE has a CVSS score of 4.13, indicating a moderate severity. The Docker container uses the following libraries and components: libassuan0, libatomic1, libattr1, docker/cli-docs-tool, docker/cli/cmd/docker, docker/compose-switch, docker/compose/v2, docker/distribution, and docker/docker. The reported vulnerability may affect the Docker container, and further investigation is needed to gather more information about the specific CVE." + ] + }, + { + "cve": "CVE-2023-4033", + "input_check_list": [ + "Check the mlflow/mlflow version: The vulnerability affects mlflow/mlflow versions prior to 2.6.0. If your Docker container is running mlflow/mlflow, verify the version. If its prior to 2.6.0, its vulnerable and you should update to version 2.6.0 or later.", + "Evaluate the OS command injection risk: Since the vulnerability involves OS command injection, you should consider whether your applications are constructing OS commands using externally-influenced input. Check if any applications within your Docker container are accepting user-supplied inputs to construct OS commands.", + "Assess the impact: The vulnerability has a CVSS 3.1 score of 4.13, indicating a high severity. Consider the potential impact of an attacker executing unexpected, dangerous commands directly on the operating system. Assess the privileges of the compromised process and the potential damage it can cause.", + "Network exposure: Determine if your Docker container has network exposure, especially if the mlflow/mlflow service or any service using it is exposed to the internet. This will help assess the attack surface and potential entry points for exploitation.", + "Mitigation: If your Docker container is vulnerable, consider implementing mitigations such as input validation and sanitization to prevent OS command injection attacks. Additionally, ensure that the principle of least privilege is followed to limit the privileges of the compromised process.", + "Note: Its important to stay updated with the latest security advisories and patches from the mlflow/mlflow project to address any vulnerabilities promptly." + ], + "output_check_list": [ + "Agent stopped due to iteration limit or time limit.", + "The Docker container does not have any applications that accept user-supplied inputs to construct OS commands, so the OS command injection risk is low.", + "The privileges of the compromised process in the Docker container can be determined by using the docker inspect command with the appropriate container ID.", + "The mlflow/mlflow service or any service using it in the Docker container is likely exposed to the internet through port 5000, indicating potential network exposure and entry points for exploitation. Further investigation into the network configuration, settings, and specific libraries and dependencies used is necessary to fully assess the impact of the reported CVE on the Docker container.", + "Some mitigation strategies for a vulnerable Docker container include using multi-stage builds, rebuilding images, and checking the image for vulnerabilities.", + "The Docker container is not impacted by the reported CVE." + ] + }, + { + "cve": "CVE-2022-32212", + "input_check_list": [ + "Check the Node.js version: The vulnerability affects Node.js versions less than 14.20.0, less than 16.20.0, and less than 18.5.0. If your Docker container is running Node.js, verify the version. If it's below the mentioned versions, it's vulnerable, and you should update to a secure version.", + "Evaluate the command injection risk: Since the vulnerability involves OS command injection, you should consider whether your applications are constructing OS commands using externally-influenced input. Check if your applications accept user input that is used to construct OS commands. If so, this is a potential attack vector.", + "Assess the IsAllowedHost check: The vulnerability is due to an insufficient IsAllowedHost check that can be bypassed. Review your code to determine if it uses the IsAllowedHost check and if it properly validates IP addresses before making DBS requests. If the check is not implemented correctly, it could allow for rebinding attacks.", + "Network exposure: Consider whether your Docker container has network exposure, especially if the Node.js service or any service using it is exposed to the internet. This vulnerability has a network attack vector, so it's important to assess the potential impact.", + "Verify the CVSS score: The CVSS 3.1 score for this vulnerability is 4.13, indicating a high severity. Take this into account when prioritizing your remediation efforts.", + "Check for vendor patches: Check if the Node.js project has released any patches or updates to address this vulnerability. If patches are available, apply them to your Docker container.", + "Monitor for security advisories: Stay updated on any security advisories related to this vulnerability. Subscribe to relevant mailing lists or follow trusted sources to receive timely information about any new developments or mitigations.", + "Remember that this checklist is preliminary, and it's important to conduct a thorough assessment of your specific environment and applications to ensure accurate vulnerability management." + ], + "output_check_list": [ + "The Docker container does not use Node.js, so it is not vulnerable to the reported CVE.", + "There is no command injection risk in the Docker container.", + "The Docker container's code does not implement the IsAllowedHost check correctly, which could potentially allow for rebinding attacks.", + "Based on the information gathered, the reported CVE does not have a direct impact on the Docker container. The Docker container has network exposure, but the Node.js service or any service using it is not exposed to the internet. Additionally, none of the libraries used by the Docker container contain the vulnerable function or component related to the reported CVE. Therefore, the potential impact of the vulnerability on the Docker container is limited. However, it is still important to stay updated on any new information or patches related to the reported CVE.", + "Agent stopped due to iteration limit or time limit.", + "There are no vendor patches or updates available for the Node.js project that can be applied to the Docker container, as Node.js does not appear to be installed or used in the container.", + "The security advisory for the reported CVE on the Docker container mentions a vulnerability in Docker versions before 9.03.15 and 20.10.3, where pulling an intentionally malformed Docker image manifest can cause a crash.", + "None" + ] + }, + { + "cve": "CVE-2022-45142", + "input_check_list": [ + "Check the Heimdal version: The vulnerability affects Heimdal versions 7.7.1 and 7.8.0. If your Docker container is running Heimdal, verify the version. If it matches one of the vulnerable versions, it may be vulnerable.", + "Check for any software using vulnerable Heimdal versions: Other applications might use Heimdal and be vulnerable. Check all applications within your Docker container for dependencies on vulnerable versions of Heimdal.", + "Evaluate the impact of the vulnerability: The vulnerability affects the validation of message integrity codes in gssapi/arcfour. Consider whether your applications are using gssapi/arcfour and if the integrity of the messages is critical. If so, this vulnerability may pose a risk.", + "Check for the fix: The fix for CVE-2022-3437 included changing memcmp to be constant time and a workaround for a compiler bug. Verify if these patches were backported to the Heimdal version in your Docker container. If the fix is present, the vulnerability may be mitigated.", + "Assess the CVSS score: The CVSS score for this vulnerability is 3.1, indicating a moderate severity. Consider the impact and exploitability of the vulnerability based on this score.", + "Network exposure: Determine if your Docker container has network exposure, especially if the Heimdal service or any service using it is exposed to the internet. This will help assess the potential risk of exploitation.", + "Verify the CWE description: The vulnerability is related to improper validation of integrity check values. Review the CWE description and consider if your applications are susceptible to this type of vulnerability." + ], + "output_check_list": [ + "The Docker container is not vulnerable to the reported CVE because it does not use Heimdal.", + "I was unable to determine if any application within the Docker container uses vulnerable versions of Heimdal. However, you can find the Docker container's software bill of materials (SBOM) on the github.com/docker/cli-docs-tool page. Additionally, you can use the Internet Search to find information about vulnerable versions of the Heimdal library.", + "The Docker container under investigation does not use gssapi/arcfour, so the reported vulnerability does not pose a risk to the integrity of the messages.", + "Agent stopped due to iteration limit or time limit.", + "To assess the impact and exploitability of the vulnerability with a CVSS score of 3.1, I will use the CVSS calculator for version 3.1 to input the specific metrics and calculate its impact and exploitability.", + "Based on the information gathered, the purpose and functionality of the Docker container do not directly indicate network exposure or mention the Heimdal service. However, further investigation is needed to assess the potential risk of network exposure and any vulnerabilities associated with the Heimdal service.", + "The CWE description for improper validation of integrity check values is related to the product not validating or incorrectly validating the integrity check values or \"checksums\" of a message. This may prevent it from detecting if the data has been modified or corrupted in transmission." + ] + }, + { + "cve": "CVE-2021-33910", + "input_check_list": [ + "Check the version of systemd: The vulnerability affects systemd versions prior to 246.15, 247.8, 248.5, and 249.1. Verify the version of systemd running in the Docker container. If it is a vulnerable version, further investigation is required.", + "Evaluate the impact of the vulnerability: The vulnerability results in an operating system crash, indicating a potential denial of service (DoS) attack. Consider the potential impact of a DoS attack on the Docker container and the services running within it.", + "Assess the attack vector: The vulnerability has a local attack vector, meaning that an attacker needs local access to the system to exploit it. Determine if the Docker container is exposed to potential attackers with local access.", + "Review resource allocation practices: The vulnerability is related to the allocation of resources without limits or throttling. Assess the resource allocation practices within the Docker container and check if there are any instances where resources are allocated without appropriate restrictions.", + "Consider the severity: The CVSS 3.1 score for this vulnerability is 7.5, indicating a high severity. Take into account the severity level when prioritizing the vulnerability assessment and mitigation efforts.", + "Check for available patches or updates: Check if there are any patches or updates available for the vulnerable version of systemd. If patches are available, apply them to mitigate the vulnerability.", + "Monitor for any signs of exploitation: Keep an eye on system logs and monitor for any signs of exploitation or abnormal behavior that could indicate an attempted or successful attack.", + "Note: This checklist provides a starting point for assessing the vulnerability in a Docker container. It is important to conduct a thorough analysis and consider other factors specific to the environment and deployment to ensure accurate vulnerability assessment and mitigation." + ], + "output_check_list": [ + "The version of systemd running in the Docker container is not vulnerable.", + "A denial of service (DoS) attack on a Docker container can potentially disrupt the availability of the container and the services running within it, leading to service downtime and loss of productivity. The impact of the attack depends on the criticality of the services or applications hosted by the Docker container.", + "Based on the information gathered, if the Docker container is running an affected version of Docker Desktop (before 4.23.0), it may be exposed to potential attackers with local access.", + "Based on the information from the Docker Container Developer Guide QA System, it appears that the Docker container has resource allocation practices in place that require 1 Gb of memory for each instance.", + "Based on the available information, I cannot provide a definitive answer about the impact of the reported CVE on the Docker container. Further investigation is needed, including obtaining the Docker container's software bill of materials (SBOM) and analyzing its dependencies. Additionally, more specific information about the reported CVE and its impact on Docker containers may need to be obtained through other means.", + "There are no patches or updates available for the vulnerable version of systemd in the Docker container.", + "To monitor for signs of exploitation or abnormal behavior in a Docker container, you can use the output from detected anomalies published to an S3 bucket, directory, or Kafka topic and integrate it with a monitoring tool.", + "No action is required as this is not a question." + ] + }, + { + "cve": "CVE-2015-20107", + "input_check_list": [ + "Check the Python version: The vulnerability affects Python versions up to 3.10.8, as well as versions 3.7, 3.8, and 3.9. If your Docker container is running Python, verify the version. If its within the affected range, its vulnerable and you should update to a patched version.", + "Identify applications using Python: Determine if any applications within your Docker container are using Python. Check for dependencies on vulnerable versions of Python. If any applications are using the affected versions, they may be vulnerable.", + "Evaluate the usage of the mailcap module: Since the vulnerability involves the mailcap module, check if any applications within your Docker container are using this module. Consider whether these applications are using untrusted input as part of commands executed through the mailcap module. If so, this is a potential attack vector.", + "Assess the risk of command injection: Given that the vulnerability allows for command injection, evaluate whether the applications within your Docker container are susceptible to this type of attack. Consider if the applications accept untrusted input that is used to construct commands executed by the application. If so, this increases the risk of exploitation.", + "Network exposure: Determine if your Docker container has network exposure, especially if the Python service or any service using it is exposed to the internet. This will help assess the potential impact of an attack exploiting the vulnerability.", + "Check for patched versions: Verify if patched versions of Python are available for your Docker container. If patches are available, update to the latest version to mitigate the vulnerability." + ], + "output_check_list": [ + "Agent stopped due to iteration limit or time limit.", + "The Docker container does not provide any information about the applications within it or the libraries it contains. However, the Docker container is used to set up dependencies and ensure a consistent environment for the workflow. It can be built and run using specific commands. To investigate if any applications within the Docker container are using Python and if they are using vulnerable versions, further analysis is required.", + "Agent stopped due to iteration limit or time limit.", + "The risk of command injection in the Docker container is low.", + "Based on the information available, it is not possible to determine if the Docker container has network exposure.", + "I don't know the version of the Python library in the Docker container." + ] + } +] \ No newline at end of file diff --git a/docker-compose.nim.yml b/docker-compose.nim.yml new file mode 100755 index 0000000..5ce3e54 --- /dev/null +++ b/docker-compose.nim.yml @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +services: + + nginx-cache: + environment: + # Route Variables + - NGINX_UPSTREAM_NVAI=${NGINX_UPSTREAM_NVAI:-http://nim-llm:8000} + - NGINX_UPSTREAM_NIM_LLM=${NGINX_UPSTREAM_NIM_LLM:-http://nim-llm:8000} + - NGINX_UPSTREAM_NIM_EMBED=${NGINX_UPSTREAM_NIM_EMBED:-http://nim-embed:8000} + depends_on: + - nim-llm + - nim-embed + + nim-llm: + image: ${NGC_NIM_LLM_CONTAINER:-nvcr.io/nim/meta/llama3-70b-instruct:1.0} + # Increase the shared memory available to the container + shm_size: 16G + runtime: nvidia + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: ${NIM_LLM_GPU_COUNT:-4} + capabilities: [ gpu ] + networks: + - app_network + environment: + - NGC_API_KEY=${NVIDIA_API_KEY:?"NVIDIA_API_KEY is required"} + volumes: + - nim-llm-cache:/opt/nim/.cache + ports: + - "8081:8000" + + nim-embed: + image: ${NGC_NIM_EMBED_CONTAINER:-nvcr.io/nim/nvidia/nv-embedqa-e5-v5:1.0.1} + # Increase the shared memory available to the container + shm_size: 16G + runtime: nvidia + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: ${NIM_EMBED_GPU_COUNT:-1} + capabilities: [ gpu ] + networks: + - app_network + environment: + - NGC_API_KEY=${NVIDIA_API_KEY:?"NVIDIA_API_KEY is required"} + volumes: + - nim-embedding-cache:/opt/nim/.cache + ports: + - "8082:8000" + + +volumes: + nim-llm-cache: + driver: local + nim-embedding-cache: + driver: local diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100755 index 0000000..22dfcb9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Set project name environment variable to avoid container collision in shared environment +name: ${DOCKER_COMPOSE_PROJECT_NAME:-morpheus_vuln_analysis} + +services: + + morpheus-vuln-analysis: + image: nvcr.io/nvidia/morpheus/morpheus-vuln-analysis:24.10 + build: + context: ./ + dockerfile: ./Dockerfile + target: runtime + ports: + - "8888:8888" + - "26466:26466" + working_dir: /workspace_examples + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [ gpu ] + networks: + - app_network + environment: + - TERM=${TERM:-} + - HF_HUB_CACHE=/workspace_examples/.cache/huggingface + - XDG_CACHE_HOME=/workspace_examples/.cache/am_cache # Allow logs to be written back to host + + # Required API Keys + - NVD_API_KEY=${NVD_API_KEY:?"NVD_API_KEY is required"} + - NVIDIA_API_KEY=${NVIDIA_API_KEY:?"NVIDIA_API_KEY is required"} + - SERPAPI_API_KEY=${SERPAPI_API_KEY:?"SERPAPI_API_KEY is required"} + # Optional API Keys + - GHSA_API_KEY=${GHSA_API_KEY:-""} + - NGC_API_KEY=${NGC_API_KEY:-""} + - NGC_ORG_ID=${NGC_ORG_ID:-""} + - OPENAI_API_KEY=${OPENAI_API_KEY:-""} + + # Base URLs for API endpoints + - CVE_DETAILS_BASE_URL=${CVE_DETAILS_BASE_URL:-http://nginx-cache/cve-details} + - CWE_DETAILS_BASE_URL=${CWE_DETAILS_BASE_URL:-http://nginx-cache/cwe-details} + - DEPSDEV_BASE_URL=${CWE_DETAILS_BASE_URL:-http://nginx-cache/depsdev} + - FIRST_BASE_URL=${FIRST_BASE_URL:-http://nginx-cache/first} + - GHSA_BASE_URL=${GHSA_BASE_URL:-http://nginx-cache/ghsa} + - NGC_API_BASE=${NGC_API_BASE:-http://nginx-cache/nemo/v1} + - NIM_EMBED_BASE_URL=${NIM_EMBED_BASE_URL:-http://nginx-cache/nim_embed/v1} + - NVD_BASE_URL=${NVD_BASE_URL:-http://nginx-cache/nvd} + - NVIDIA_API_BASE=${NVIDIA_API_BASE:-http://nginx-cache/nim_llm/v1} + - OPENAI_API_BASE=${OPENAI_API_BASE:-http://nginx-cache/openai/v1} # Used by `langchain` for embedding generation + - OPENAI_BASE_URL=${OPENAI_BASE_URL:-http://nginx-cache/openai/v1} # Used by `openai` for LLM inference + - RHSA_BASE_URL=${RHSA_BASE_URL:-http://nginx-cache/rhsa} + - SERPAPI_BASE_URL=${SERPAPI_BASE_URL:-http://nginx-cache/serpapi} + - UBUNTU_BASE_URL=${UBUNTU_BASE_URL:-http://nginx-cache/ubuntu} + volumes: + - ./:/workspace_examples + cap_add: + - sys_nice + depends_on: + - nginx-cache + restart: always + + nginx-cache: + image: nginx + volumes: + - ./nginx/nginx_cache.conf:/etc/nginx/nginx.conf:ro + - ./nginx/logs:/var/log/nginx + - ./nginx/templates:/etc/nginx/templates:ro + - service-cache:/server_cache_intel:rw + - llm-cache:/server_cache_llm:rw + ports: + # Set custom ports in environment variables to avoid port collision + - "${NGINX_HOST_HTTP_PORT:-8080}:80" + environment: + # API Keys + - GHSA_API_KEY=${GHSA_API_KEY:-""} + - NGC_API_KEY=${NGC_API_KEY:-""} + - NGC_ORG_ID=${NGC_ORG_ID:-""} + - NVD_API_KEY=${NVD_API_KEY:-""} + - NVIDIA_API_KEY=${NVIDIA_API_KEY:-""} + - OPENAI_API_KEY=${OPENAI_API_KEY:-""} + # Route Variables + - NGINX_UPSTREAM_NVAI=${NGINX_UPSTREAM_NVAI:-https://api.nvcf.nvidia.com} + - NGINX_UPSTREAM_NIM_LLM=${NGINX_UPSTREAM_NIM_LLM:-https://integrate.api.nvidia.com} + - NGINX_UPSTREAM_NIM_EMBED=${NGINX_UPSTREAM_NIM_EMBED:-https://integrate.api.nvidia.com} + networks: + - app_network + restart: always + + nginx-ssl: + image: nginx + volumes: + - ./nginx/nginx_ssl.conf:/etc/nginx/nginx.conf:ro + - ./nginx/key.pem:/etc/nginx/ssl/key.pem + - ./nginx/cert.pem:/etc/nginx/ssl/cert.pem + + ports: + # Set custom ports in environment variables to avoid port collision + - "${NGINX_HOST_HTTPS_PORT:-443}:443" + networks: + - app_network + depends_on: + - nginx-cache + profiles: + - ssl + restart: always + +networks: + app_network: + driver: bridge + +volumes: + service-cache: + driver: local + llm-cache: + driver: local diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh new file mode 100755 index 0000000..45fc5d8 --- /dev/null +++ b/docker/scripts/entrypoint.sh @@ -0,0 +1,27 @@ +#!/bin/bash --login +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Activate the `morpheus` conda environment. +. /opt/conda/etc/profile.d/conda.sh +conda activate morpheus-vuln-analysis + +# Source "source" file if it exists +SRC_FILE="/opt/docker/bin/entrypoint_source" +[ -f "${SRC_FILE}" ] && source "${SRC_FILE}" + +# Run whatever the user wants. +exec "$@" diff --git a/images/hosting_options.png b/images/hosting_options.png new file mode 100644 index 0000000..0b951eb --- /dev/null +++ b/images/hosting_options.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96863ce9307e61f57bccb13f706aec93ffd04553ed0cc7c73e613a57ab20b107 +size 91482 diff --git a/images/security_vulnerability_analysis_ai_workflow_conceptual_diagram.jpg b/images/security_vulnerability_analysis_ai_workflow_conceptual_diagram.jpg new file mode 100644 index 0000000..6c5598f --- /dev/null +++ b/images/security_vulnerability_analysis_ai_workflow_conceptual_diagram.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e57a9847e380f841fb17e5ab2268a08c048cf89c1bcca8f7f0b77552a20a6fb +size 1110007 diff --git a/nginx/.gitignore b/nginx/.gitignore new file mode 100644 index 0000000..b305946 --- /dev/null +++ b/nginx/.gitignore @@ -0,0 +1,6 @@ + +# Ignore any secret certificate files +*.pem + +# Ignore the logs directory +logs/ diff --git a/nginx/nginx_cache.conf b/nginx/nginx_cache.conf new file mode 100644 index 0000000..293cd3f --- /dev/null +++ b/nginx/nginx_cache.conf @@ -0,0 +1,120 @@ +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + proxy_ssl_server_name on; + + proxy_cache_path /server_cache_llm levels=1:2 keys_zone=llm_cache:10m max_size=20g inactive=14d use_temp_path=off; + + proxy_cache_path /server_cache_intel levels=1:2 keys_zone=intel_cache:10m max_size=20g inactive=14d use_temp_path=off; + + error_log /dev/stdout info; + + log_format upstream_time '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"' + 'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'; + + log_format cache_log '[$time_local] ($upstream_cache_status) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization'; + + log_format no_cache_log '[$time_local] (BYPASSED) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization'; + + log_format mirror_log '[$time_local] (MIRROR) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization'; + + log_format nvai_cache_log '[$time_local] ($upstream_cache_status) "$request" $status - $body_bytes_sent bytes {$remote_addr} "$http_user_agent" $request_time - $connection_requests. Auth: $http_authorization. Final Auth: $nvai_http_authorization'; + + include /etc/nginx/conf.d/variables/*.conf; + + map $http_cache_control $cache_bypass { + no-cache 1; + } + + # Log to stdout and a file for searchability + access_log /dev/stdout cache_log; + access_log /var/log/nginx/access.log cache_log; + + error_log /dev/stdout info; + error_log /var/log/nginx/error.log info; + + server { + listen 80; + server_name localhost; + + proxy_http_version 1.1; + + # Headers to Add + # proxy_set_header Host $host; + proxy_set_header Connection ''; + + # Headers to Remove + proxy_ignore_headers Cache-Control; + proxy_ignore_headers "Set-Cookie"; + proxy_hide_header "Set-Cookie"; + + # Proxy Buffer Config + proxy_busy_buffers_size 1024k; + proxy_buffers 4 512k; + proxy_buffer_size 1024k; + + # Proxy validity + proxy_cache_valid 200 202 14d; + proxy_read_timeout 8m; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_bypass $cache_bypass; + + # Configure a resolver to use for DNS resolution. This uses the Docker DNS resolver + # See https://tenzer.dk/nginx-with-dynamic-upstreams/ for why this is necessary + # When considering what the "base_url" should be, consider the following: + # - The base_url should be the unchangable part of the URL for any request tho that API + # - If the API uses versioning, the version should be included in the base_url + # - If the API is a subpath of a larger API, the base_url should be the path to the API + # - Examples: + # - GET `https://api.first.org/data/v1/epss` => base_url=`https://api.first.org/data/v1` + # - GET `https://services.nvd.nist.gov/rest/json/cves/2.0` => base_url=`https://services.nvd.nist.gov/rest` + + resolver 127.0.0.11 [::1]:5353 valid=60s; + + # rewrite_log on; + + ################ Docker Compose Services ################# + + # Force nginx to resolve morpheus-vuln-analysis each call to allow starting this before starting the service + set $morpheus_vuln_analysis_upstream "http://morpheus-vuln-analysis:26466"; + + location /scan { + proxy_pass $morpheus_vuln_analysis_upstream; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + } + + # Include any additional routes from the routes directory + include /etc/nginx/conf.d/routes/*.conf; + + + ################### Redirect Handling #################### + + location @handle_redirects { + # store the current state of the world so we can reuse it in a minute + # We need to capture these values now, because as soon as we invoke + # the proxy_* directives, these will disappear + set $original_uri $uri; + set $orig_loc $upstream_http_location; + + # nginx goes to fetch the value from the upstream Location header + proxy_pass $orig_loc; + proxy_cache llm_cache; + + # But we store the result with the cache key of the original request URI + # so that future clients don't need to follow the redirect too + proxy_cache_key $original_uri; + proxy_cache_valid 200 206 14d; + } + } +} diff --git a/nginx/nginx_ssl.conf b/nginx/nginx_ssl.conf new file mode 100644 index 0000000..a3dfae3 --- /dev/null +++ b/nginx/nginx_ssl.conf @@ -0,0 +1,24 @@ +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + server { + listen 443 ssl; + listen [::]:443 ssl; + server_name localhost; + ssl_certificate /etc/nginx/ssl/cert.pem; + ssl_certificate_key /etc/nginx/ssl/key.pem; + + + location / { + proxy_pass http://nginx-cache:80; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } +} diff --git a/nginx/templates/routes/intel.conf.template b/nginx/templates/routes/intel.conf.template new file mode 100644 index 0000000..f848a64 --- /dev/null +++ b/nginx/templates/routes/intel.conf.template @@ -0,0 +1,123 @@ +####################### Intel APIs ####################### + +set $serpapi_upstream https://serpapi.com; + +location /serpapi { + rewrite ^\/serpapi(\/.*)$ $1 break; + proxy_pass $serpapi_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $nvd_upstream https://services.nvd.nist.gov; + +location /nvd { + rewrite ^\/nvd(\/.*)$ /rest$1 break; + proxy_pass $nvd_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $cve_details_upstream https://www.cvedetails.com; + +location /cve-details { + rewrite ^\/cve-details(\/.*)$ $1 break; + proxy_pass $cve_details_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $cwe_details_upstream https://cwe.mitre.org; + +location /cwe-details { + rewrite ^\/cwe-details(\/.*)$ $1 break; + proxy_pass $cwe_details_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + + error_page 301 302 307 = @handle_redirects; +} + +set $epss_upstream https://api.first.org; + +location /first { + rewrite ^\/first(\/.*)$ /data/v1$1 break; + proxy_pass $epss_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $recf_upstream https://api.recordedfuture.com; + +location /recf { + rewrite ^\/recf(\/.*)$ /v2$1 break; + proxy_pass $recf_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $ghsa_upstream https://api.github.com; + +location /ghsa { + rewrite ^\/ghsa(\/.*)$ $1 break; + proxy_pass $ghsa_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $rhsa_upstream https://access.redhat.com; + +location /rhsa { + rewrite ^\/rhsa(\/.*)$ /hydra/rest$1 break; + proxy_pass $rhsa_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $ubuntu_upstream https://ubuntu.com; + +location /ubuntu { + rewrite ^\/ubuntu(\/.*)$ $1 break; + proxy_pass $ubuntu_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} + +set $depsdev_upstream https://api.deps.dev; + +location /depsdev { + rewrite ^\/depsdev(\/.*)$ $1 break; + proxy_pass $depsdev_upstream; + proxy_cache intel_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; +} diff --git a/nginx/templates/routes/nemo.conf.template b/nginx/templates/routes/nemo.conf.template new file mode 100644 index 0000000..a3c7aef --- /dev/null +++ b/nginx/templates/routes/nemo.conf.template @@ -0,0 +1,24 @@ +set $nemo_upstream https://api.llm.ngc.nvidia.com; + +location /nemo { + + location ~* ^\/nemo\/v1\/models(\/.+\/completions)?$ { + rewrite ^\/nemo(\/.*)$ $1 break; + proxy_pass $nemo_upstream; + proxy_set_header Connection ''; + proxy_set_header Authorization $nemo_http_authorization; + proxy_set_header Organization-ID $nemo_http_organization_id; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + } + + location /nemo/v1 { + rewrite ^\/nemo(\/.*)$ $1 break; + proxy_pass $nemo_upstream; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/nginx/templates/routes/nim.conf.template b/nginx/templates/routes/nim.conf.template new file mode 100644 index 0000000..b04c02c --- /dev/null +++ b/nginx/templates/routes/nim.conf.template @@ -0,0 +1,45 @@ + +set $upstream_nim_llm ${NGINX_UPSTREAM_NIM_LLM}; +set $upstream_nim_embed ${NGINX_UPSTREAM_NIM_EMBED}; + +location /nim_llm { + location ~* ^\/nim_llm\/v1\/(?:chat\/completions|completions|edits|moderations|answers)$ { + rewrite ^\/nim_llm(\/.*)$ $1 break; + proxy_pass $upstream_nim_llm; + proxy_set_header Connection ''; + proxy_set_header Authorization $nim_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + } + + location /nim_llm/v1 { + rewrite ^\/nim_llm(\/.*)$ $1 break; + proxy_pass $upstream_nim_llm; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} + +location /nim_embed { + location ~* ^\/nim_embed\/v1\/(?:embeddings)$ { + rewrite ^\/nim_embed(\/.*)$ $1 break; + proxy_pass $upstream_nim_embed; + proxy_set_header Connection ''; + proxy_set_header Authorization $nim_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + } + + location /nim_embed/v1 { + rewrite ^\/nim_embed(\/.*)$ $1 break; + proxy_pass $upstream_nim_embed; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/nginx/templates/routes/nvidia.conf.template b/nginx/templates/routes/nvidia.conf.template new file mode 100644 index 0000000..2bdf18b --- /dev/null +++ b/nginx/templates/routes/nvidia.conf.template @@ -0,0 +1,47 @@ + +set $nvai_upstream ${NGINX_UPSTREAM_NVAI}; + +location /nvai { + + location ~* ^\/nvai\/v2\/nvcf\/((pexec\/functions\/.+)|(functions))$ { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass $nvai_upstream; + proxy_set_header Connection ''; + proxy_set_header Authorization $nvai_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + + # Only cache 200/202, not 404. + proxy_cache_valid 200 14d; + + add_header X-Cache-Status $upstream_cache_status always; + client_body_buffer_size 4m; + + access_log /dev/stdout nvai_cache_log; + access_log /var/log/nginx/access.log nvai_cache_log; + } + + location ~* ^\/nvai\/v2\/nvcf\/(pexec\/status\/.+)$ { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass $nvai_upstream; + proxy_set_header Connection ''; + proxy_set_header Authorization $nvai_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods GET; + proxy_cache_key "$request_method|$request_uri"; + + # Only cache 200. Not 202 as it will cause a never ending loop + proxy_cache_valid 200 202 14d; + + add_header X-Cache-Status $upstream_cache_status always; + client_body_buffer_size 4m; + } + + location /nvai/v2 { + rewrite ^\/nvai(\/.*)$ $1 break; + proxy_pass $nvai_upstream; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/nginx/templates/routes/openai.conf.template b/nginx/templates/routes/openai.conf.template new file mode 100644 index 0000000..8a8a0cb --- /dev/null +++ b/nginx/templates/routes/openai.conf.template @@ -0,0 +1,23 @@ +set $openai_upstream https://api.openai.com; + +location /openai { + + location ~* ^\/openai\/v1\/((engines\/.+\/)?(?:chat\/completions|completions|edits|moderations|answers|embeddings))$ { + rewrite ^\/openai(\/.*)$ $1 break; + proxy_pass $openai_upstream; + proxy_set_header Connection ''; + proxy_set_header Authorization $openai_http_authorization; + proxy_cache llm_cache; + proxy_cache_methods POST; + proxy_cache_key "$request_method|$request_uri|$request_body"; + add_header X-Cache-Status $upstream_cache_status; + client_body_buffer_size 4m; + } + + location /openai/v1 { + rewrite ^\/openai(\/.*)$ $1 break; + proxy_pass $openai_upstream; + access_log /dev/stdout no_cache_log; + access_log /var/log/nginx/access.log no_cache_log; + } +} diff --git a/nginx/templates/variables/template-variables.conf.template b/nginx/templates/variables/template-variables.conf.template new file mode 100644 index 0000000..0af689d --- /dev/null +++ b/nginx/templates/variables/template-variables.conf.template @@ -0,0 +1,57 @@ +map $http_authorization $nemo_http_authorization { + 'Bearer AGENT_MORPHEUS' 'Bearer ${NGC_API_KEY}'; + 'Bearer "AGENT_MORPHEUS"' 'Bearer ${NGC_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${NGC_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' 'Bearer ${NGC_API_KEY}'; + default $http_authorization; +} + +map $http_organization_id $nemo_http_organization_id { + 'AGENT_MORPHEUS' '${NGC_ORG_ID}'; + '"AGENT_MORPHEUS"' '${NGC_ORG_ID}'; + 'CYBER_DEV_DAY' '${NGC_ORG_ID}'; + '"CYBER_DEV_DAY"' '${NGC_ORG_ID}'; + default $http_organization_id; +} + +map $http_authorization $nim_http_authorization { + 'Bearer AGENT_MORPHEUS' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer "AGENT_MORPHEUS"' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer CYBER_DEV_DAY' '${NVIDIA_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' '${NVIDIA_API_KEY}'; + default $http_authorization; +} + +map $http_authorization $nvai_http_authorization { + 'Bearer AGENT_MORPHEUS' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer nvapi-AGENT_MORPHEUS' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer "nvapi-AGENT_MORPHEUS"' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer nvapi-CYBER_DEV_DAY' 'Bearer ${NVIDIA_API_KEY}'; + 'Bearer "nvapi-CYBER_DEV_DAY"' 'Bearer ${NVIDIA_API_KEY}'; + default $http_authorization; +} + +map $http_authorization $openai_http_authorization { + 'Bearer AGENT_MORPHEUS' 'Bearer ${OPENAI_API_KEY}'; + 'Bearer "AGENT_MORPHEUS"' 'Bearer ${OPENAI_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${OPENAI_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' 'Bearer ${OPENAI_API_KEY}'; + default $http_authorization; +} + +map $http_authorization $ghsa_http_authorization { + 'Bearer AGENT_MORPHEUS' 'Bearer ${GHSA_API_KEY}'; + 'Bearer "AGENT_MORPHEUS"' 'Bearer ${GHSA_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${GHSA_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' 'Bearer ${GHSA_API_KEY}'; + default $http_authorization; +} + +map $http_authorization $nvd_http_authorization { + 'Bearer AGENT_MORPHEUS' 'Bearer ${NVD_API_KEY}'; + 'Bearer "AGENT_MORPHEUS"' 'Bearer ${NVD_API_KEY}'; + 'Bearer CYBER_DEV_DAY' 'Bearer ${NVD_API_KEY}'; + 'Bearer "CYBER_DEV_DAY"' 'Bearer ${NVD_API_KEY}'; + default $http_authorization; +} diff --git a/quick_start/.gitkeep b/quick_start/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/quick_start/quick_start_guide.ipynb b/quick_start/quick_start_guide.ipynb new file mode 100644 index 0000000..3298d2a --- /dev/null +++ b/quick_start/quick_start_guide.ipynb @@ -0,0 +1,4688 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Overview\n", + "This notebook contains a quick start guide for the NIM Agent Blueprint: Vulnerability Analysis for Container Security. By the end of the notebook, you will be able to run the blueprint on an example input, and view the results.\n", + "\n", + "For full instructions on running and customizing the blueprint, please see the [README](../README.md).\n", + "\n", + "## Prerequisites\n", + "* [Build and launch Morpheus Vulnerability Analysis container](../README.md#prerequisites)\n", + " + Be sure to keep the container bash shell handy for launching the HTTP server\n", + "* From the host machine (outside the container), open this notebook in a [Jupyter server](https://docs.jupyter.org/en/latest/running.html), or from an IDE that supports notebook execution, such as [VS Code](https://code.visualstudio.com/docs/datascience/jupyter-notebooks).\n", + " + Please keep this notebook in its original directory, and ensure the repository structure has not been modified before running the notebook.\n", + "* [Install `pandas`](https://pandas.pydata.org/docs/getting_started/install.html)\n", + "* [Install Syft](https://github.com/anchore/syft) (optional prerequisite for cell 4, Generating a custom SBOM)\n", + "\n", + "## Other notes\n", + "* All files and directories in the notebook are relative to the repository root.\n", + "\n", + "# Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "import pandas as pd\n", + "import re\n", + "import requests" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Change the notebook's working directory to the repository root. All the directory and file paths below will now be relative to `REPO_ROOT`." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "REPO_ROOT = !echo $(git rev-parse --show-toplevel)\n", + "os.chdir(REPO_ROOT[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Preparing the workflow" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Select container and obtain SBOM" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using pre-generated SBOM" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sbom_file = \"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generating a custom SBOM\n", + "\n", + "Be aware that depending on the size of the container, the sbom generation process can take several minutes to complete." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# import os\n", + "# import subprocess\n", + "#\n", + "# image_url = \"nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime\"\n", + "# sbom_file = \".tmp/morpheus231101.sbom\"\n", + "# if not os.path.exists(sbom_file):\n", + "# cmd = ['syft', image_url, '-o', 'syft-table='+sbom_file]\n", + "# sp = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=False)\n", + "# (out, err) = sp.communicate()\n", + "# print(out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load up example HTTP input config" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"$schema\": \"./schemas/config.schema.json\",\n", + " \"engine\": {\n", + " \"agent\": {\n", + " \"model\": {\n", + " \"model_name\": \"meta/llama3-70b-instruct\",\n", + " \"service\": {\n", + " \"_type\": \"nvfoundation\"\n", + " },\n", + " \"max_tokens\": 2000,\n", + " \"temperature\": 0,\n", + " \"top_p\": 0.01,\n", + " \"seed\": 42\n", + " },\n", + " \"verbose\": false\n", + " },\n", + " \"checklist_model\": {\n", + " \"service\": {\n", + " \"_type\": \"nvfoundation\"\n", + " },\n", + " \"model_name\": \"meta/llama3-70b-instruct\",\n", + " \"max_tokens\": 2000,\n", + " \"temperature\": 0,\n", + " \"top_p\": 0.01,\n", + " \"seed\": 42\n", + " },\n", + " \"justification_model\": {\n", + " \"model_name\": \"meta/llama3-70b-instruct\",\n", + " \"service\": {\n", + " \"_type\": \"nvfoundation\"\n", + " },\n", + " \"max_tokens\": 1024,\n", + " \"temperature\": 0,\n", + " \"top_p\": 0.01,\n", + " \"seed\": 42\n", + " },\n", + " \"rag_embedding\": {\n", + " \"_type\": \"nim\",\n", + " \"model\": \"nvidia/nv-embedqa-e5-v5\",\n", + " \"truncate\": \"END\",\n", + " \"max_batch_size\": 128\n", + " },\n", + " \"summary_model\": {\n", + " \"model_name\": \"meta/llama3-70b-instruct\",\n", + " \"service\": {\n", + " \"_type\": \"nvfoundation\"\n", + " },\n", + " \"max_tokens\": 1024,\n", + " \"temperature\": 0,\n", + " \"top_p\": 0.01,\n", + " \"seed\": 42\n", + " }\n", + " },\n", + " \"general\": {\n", + " \"cache_dir\": null,\n", + " \"base_vdb_dir\": \".cache/am_cache/vdb\",\n", + " \"base_git_dir\": \".cache/am_cache/git\",\n", + " \"max_retries\": 5,\n", + " \"model_max_batch_size\": 64,\n", + " \"pipeline_batch_size\": 1024,\n", + " \"use_uvloop\": true\n", + " },\n", + " \"input\": {\n", + " \"_type\": \"http\",\n", + " \"port\": 26466,\n", + " \"address\": \"0.0.0.0\"\n", + " },\n", + " \"output\": {\n", + " \"_type\": \"file\",\n", + " \"file_path\": \".tmp/output.json\",\n", + " \"markdown_dir\": \".tmp/vulnerability_markdown_reports\",\n", + " \"overwrite\": true\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "with open('configs/from_http.json', 'r') as config_in:\n", + " http_config = json.load(config_in)\n", + "print(json.dumps(http_config, indent=2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configuring the HTTP server" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# The default container port is 26466. If another port is required this can be adjusted in the Docker compose file\n", + "# and the container can be re-built or the desired port mapping can be specified when starting the container.\n", + "input_port = 26466\n", + "input_address = \"0.0.0.0\"\n", + "output_type = \"file\"\n", + "output_file_path = \".tmp/quick_start_results.json\"\n", + "\n", + "# Update the default http_configuration with our custom values\n", + "http_config['input']['port'] = input_port\n", + "http_config['input']['address'] = input_address\n", + "\n", + "http_config['output']['_type'] = output_type\n", + "http_config['output']['file_path'] = output_file_path\n", + "\n", + "# Save the updated http configuration file to the .tmp/ directory\n", + "config_path = \".tmp/http_config.json\"\n", + "os.makedirs(os.path.dirname(config_path), exist_ok=True)\n", + "with open(config_path, \"w\") as config_out:\n", + " json.dump(http_config, config_out, indent=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Starting the HTTP server" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Pull up the bash shell for the container and run the following command:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```bash\n", + "python src/main.py --log_level DEBUG cve pipeline --config_file=.tmp/http_config.json\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Once you see the \n", + "\n", + "`====Building Segment Complete!====` \n", + "\n", + "message, HTTP requests can be sent to the server." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Sending Workflow Requests" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup input message for HTTP server" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"image\": {\n", + " \"name\": \"nvcr.io/nvidia/morpheus/morpheus\",\n", + " \"tag\": \"23.11-runtime\",\n", + " \"source_info\": [\n", + " {\n", + " \"type\": \"code\",\n", + " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", + " \"tag\": \"v23.11.01\",\n", + " \"include\": [\n", + " \"**/*.cpp\",\n", + " \"**/*.cu\",\n", + " \"**/*.cuh\",\n", + " \"**/*.h\",\n", + " \"**/*.hpp\",\n", + " \"**/*.ipynb\",\n", + " \"**/*.py\",\n", + " \"**/*Dockerfile\"\n", + " ],\n", + " \"exclude\": [\n", + " \"tests/**/*\"\n", + " ]\n", + " },\n", + " {\n", + " \"type\": \"doc\",\n", + " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", + " \"tag\": \"v23.11.01\",\n", + " \"include\": [\n", + " \"**/*.md\",\n", + " \"docs/**/*.rst\"\n", + " ]\n", + " }\n", + " ],\n", + " \"sbom_info\": {\n", + " \"_type\": \"file\",\n", + " \"file_path\": \"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom\"\n", + " }\n", + " },\n", + " \"scan\": {\n", + " \"vulns\": [\n", + " {\n", + " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\"\n", + " },\n", + " {\n", + " \"vuln_id\": \"CVE-2023-50782\"\n", + " },\n", + " {\n", + " \"vuln_id\": \"CVE-2023-36632\"\n", + " },\n", + " {\n", + " \"vuln_id\": \"CVE-2023-43804\"\n", + " },\n", + " {\n", + " \"vuln_id\": \"GHSA-cxfr-5q3r-2rc2\"\n", + " },\n", + " {\n", + " \"vuln_id\": \"GHSA-554w-xh4j-8w64\"\n", + " },\n", + " {\n", + " \"vuln_id\": \"GHSA-3ww4-gg4f-jr7f\"\n", + " }\n", + " ]\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "with open('data/input_messages/morpheus:23.11-runtime.json', 'r') as input_message_in:\n", + " input_message = json.load(input_message_in)\n", + "print(json.dumps(input_message, indent=2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Set up source code and documentation for vector database" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "source_code_repo = \"https://github.com/nv-morpheus/Morpheus.git\"\n", + "source_code_tag = \"v23.11.01\"\n", + "doc_repo = \"https://github.com/nv-morpheus/Morpheus.git\"\n", + "doc_tag = \"v23.11.01\"\n", + "\n", + "input_message[\"image\"][\"source_info\"] = [\n", + " {\n", + " \"type\": \"code\",\n", + " \"git_repo\": source_code_repo,\n", + " \"tag\": source_code_tag,\n", + " \"include\": [\"**/*.cpp\", \"**/*.cu\", \"**/*.cuh\", \"**/*.h\", \"**/*.hpp\", \"**/*.ipynb\", \"**/*.py\", \"**/*Dockerfile\"],\n", + " \"exclude\": [\"tests/**/*\"]\n", + " },\n", + " {\n", + " \"type\": \"doc\", \"git_repo\": doc_repo, \"tag\": doc_tag, \"include\": [\"**/*.md\", \"docs/**/*.rst\"]\n", + " },\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Set up SBOM and list of vulnerabilities to analyze" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "input_message[\"image\"][\"sbom_info\"][\"file_path\"] = sbom_file\n", + "\n", + "vuln_list = [\n", + " \"CVE-2024-21762\", # irrelevant to the Morpheus container\n", + " \"GHSA-hh8p-p8mp-gqhm\", # vulnerable package in use\n", + " \"GHSA-3f63-hfp8-52jq\", # vulnerable code included in the environment but not executed during runtime\n", + "]\n", + "vuln_entry = [{\"vuln_id\": vuln} for vuln in vuln_list]\n", + "input_message[\"scan\"][\"vulns\"] = vuln_entry" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"image\": {\n", + " \"name\": \"nvcr.io/nvidia/morpheus/morpheus\",\n", + " \"tag\": \"23.11-runtime\",\n", + " \"source_info\": [\n", + " {\n", + " \"type\": \"code\",\n", + " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", + " \"tag\": \"v23.11.01\",\n", + " \"include\": [\n", + " \"**/*.cpp\",\n", + " \"**/*.cu\",\n", + " \"**/*.cuh\",\n", + " \"**/*.h\",\n", + " \"**/*.hpp\",\n", + " \"**/*.ipynb\",\n", + " \"**/*.py\",\n", + " \"**/*Dockerfile\"\n", + " ],\n", + " \"exclude\": [\n", + " \"tests/**/*\"\n", + " ]\n", + " },\n", + " {\n", + " \"type\": \"doc\",\n", + " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", + " \"tag\": \"v23.11.01\",\n", + " \"include\": [\n", + " \"**/*.md\",\n", + " \"docs/**/*.rst\"\n", + " ]\n", + " }\n", + " ],\n", + " \"sbom_info\": {\n", + " \"_type\": \"file\",\n", + " \"file_path\": \"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom\"\n", + " }\n", + " },\n", + " \"scan\": {\n", + " \"vulns\": [\n", + " {\n", + " \"vuln_id\": \"CVE-2024-21762\"\n", + " },\n", + " {\n", + " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\"\n", + " },\n", + " {\n", + " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\"\n", + " }\n", + " ]\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "print(json.dumps(input_message, indent=2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This command sends a request with the input message to the running HTTP server. You should be able to see the logs from the pipeline in the console. Please wait until the pipeline execution is finished before proceeding to the next section." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "requests.post(f\"http://localhost:{input_port}/scan\", json=input_message)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "An overview of each provided CVE's affected status and vulnerability label will be printed to the logs upon completion of the pipeline, for example:\n", + "\n", + "```\n", + "Vulnerability 'CVE-2024-21762' affected status: FALSE. Label: code_not_present\n", + "Vulnerability 'GHSA-hh8p-p8mp-gqhm' affected status: TRUE. Label: vulnerable\n", + "Vulnerability 'GHSA-3f63-hfp8-52jq' affected status: FALSE. Label: code_not_reachable\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Additional inputs can be sent to the HTTP server and the results will be appended to the output file until the server is shut down." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Processing Results\n", + "## Output schema\n", + "### 1. `input`\n", + "This section captures the details about the target of the vulnerability investigation. It includes subsections for the security scan and information on the scanned container image. It is structured as follows:\n", + "\n", + "- `scan`: Details about the security scan and the vulnerabilities discovered for the system to investigate. Many of the fields are null in illustrative example input. These fields can be populated with security scan metadata when integrating scan results as the HTTP input source.\n", + " - `id`: ID for the scan.\n", + " - `type`: Scan type.\n", + " - `started_at` and `completed_at`: Timestamps indicating scan start and end.\n", + " - `vulns`: Array of detected vulnerabilities, each described with details including `vuln_id`, severity, description, and package-specific data.\n", + "\n", + "- `image`: Specifies the container image analyzed in this scan. This information is directly parsed from the input message we posted.\n", + "\n", + "### 2. `info`\n", + "This section provides metadata and intel on vulnerabilities gathered from threat databases and advisories.\n", + "\n", + "- `vdb`: Paths to vulnerability vector databases used for code (`code_vdb_path`) and documentation (`doc_vdb_path`) relevant to the container.\n", + "\n", + "- `intel`: Vulnerability intel that groups vulnerabilities by their IDs.\n", + " - `vuln_id`: Identifier for each vulnerability.\n", + " - `ghsa`, `nvd`, `rhsa`, `ubuntu`: Subsections providing detailed advisories from different vulnerability databases:\n", + " - Details like `cve_id`, `severity`, `summary`, `description`, `cwes` explain the nature of the vulnerability and provide information on affected versions, available patches, publication dates, severity, and other relevant details.\n", + " - Depending on the intel source, information about specific configurations or environments where the vulnerabilities may occur could also be included.\n", + "\n", + "- `sbom`(Software Bill of Materials): Lists packages used in the container.\n", + " - Each package contains fields like `name`, `version`, `path`, and `system`.\n", + "\n", + "- `vulnerable_dependencies`: Lists dependencies in the container that match identified vulnerabilities.\n", + " - For each vulnerability (`vuln_id`), this section lists vulnerable SBOM packages, identifying any affected packages and their versions.\n", + "\n", + "### 3. `output`\n", + "The output section captures the results of the vulnerability investigation from the agentic system.\n", + "\n", + "- `vuln_id`: Identifier for each vulnerability.\n", + "- `checklist`: Detailed investigative steps and results for investigating each vulnerability's exploitability.\n", + " - `input`: The checklist item for the agent to investigate.\n", + " - `response`: The agent's finding and conclusion for the checklist item.\n", + " - `intermediate_steps`: (Optional) Additional information on the intermediate actions taken during the investigation by the agent. Useful for debugging and explainability.\n", + "- `summary`: A summary of the investigation, indicating whether the vulnerability is exploitable or not, based on checklist findings.\n", + "- `justification`: A detailed conclusion on the vulnerability's exploitability status, including:\n", + " - `label`: A VEX justification label for the vulnerability.\n", + " - `reason`: Explanation for the decision of the classification.\n", + " - `status`: Final boolean decision on whether the vulnerability is currently exploitable.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load results into the notebook" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "#Load first result in case multiple requests were sent\n", + "with open(output_file_path, 'r') as result_in:\n", + " full_input = result_in.read()\n", + " offsets = [res.start() for res in re.finditer('{\"input\":{\"scan\":', full_input)]\n", + " offsets.append(len(full_input))\n", + " first_result = json.loads(full_input[offsets[0]:offsets[1]])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"input\": {\n", + " \"scan\": {\n", + " \"id\": null,\n", + " \"type\": null,\n", + " \"started_at\": null,\n", + " \"completed_at\": null,\n", + " \"vulns\": [\n", + " {\n", + " \"vuln_id\": \"CVE-2024-21762\",\n", + " \"description\": null,\n", + " \"score\": null,\n", + " \"severity\": null,\n", + " \"published_date\": null,\n", + " \"last_modified_date\": null,\n", + " \"url\": null,\n", + " \"feed_group\": null,\n", + " \"package\": null,\n", + " \"package_version\": null,\n", + " \"package_name\": null,\n", + " \"package_type\": null\n", + " },\n", + " {\n", + " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", + " \"description\": null,\n", + " \"score\": null,\n", + " \"severity\": null,\n", + " \"published_date\": null,\n", + " \"last_modified_date\": null,\n", + " \"url\": null,\n", + " \"feed_group\": null,\n", + " \"package\": null,\n", + " \"package_version\": null,\n", + " \"package_name\": null,\n", + " \"package_type\": null\n", + " },\n", + " {\n", + " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\",\n", + " \"description\": null,\n", + " \"score\": null,\n", + " \"severity\": null,\n", + " \"published_date\": null,\n", + " \"last_modified_date\": null,\n", + " \"url\": null,\n", + " \"feed_group\": null,\n", + " \"package\": null,\n", + " \"package_version\": null,\n", + " \"package_name\": null,\n", + " \"package_type\": null\n", + " }\n", + " ]\n", + " },\n", + " \"image\": {\n", + " \"name\": \"nvcr.io/nvidia/morpheus/morpheus\",\n", + " \"tag\": \"23.11-runtime\",\n", + " \"digest\": null,\n", + " \"platform\": null,\n", + " \"feed_group\": null,\n", + " \"source_info\": [\n", + " {\n", + " \"type\": \"code\",\n", + " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", + " \"tag\": \"v23.11.01\",\n", + " \"include\": [\n", + " \"**/*.cpp\",\n", + " \"**/*.cu\",\n", + " \"**/*.cuh\",\n", + " \"**/*.h\",\n", + " \"**/*.hpp\",\n", + " \"**/*.ipynb\",\n", + " \"**/*.py\",\n", + " \"**/*Dockerfile\"\n", + " ],\n", + " \"exclude\": [\n", + " \"tests/**/*\"\n", + " ]\n", + " },\n", + " {\n", + " \"type\": \"doc\",\n", + " \"git_repo\": \"https://github.com/nv-morpheus/Morpheus.git\",\n", + " \"tag\": \"v23.11.01\",\n", + " \"include\": [\n", + " \"**/*.md\",\n", + " \"docs/**/*.rst\"\n", + " ],\n", + " \"exclude\": []\n", + " }\n", + " ],\n", + " \"sbom_info\": {\n", + " \"_type\": \"file\",\n", + " \"file_path\": \"data/sboms/nvcr.io/nvidia/morpheus/morpheus:v23.11.01-runtime.sbom\"\n", + " }\n", + " }\n", + " },\n", + " \"info\": {\n", + " \"vdb\": {\n", + " \"code_vdb_path\": \".cache/am_cache/vdb/code/ddb7e6a2171ab6bc\",\n", + " \"doc_vdb_path\": \".cache/am_cache/vdb/doc/46e24f8f781501d1\"\n", + " },\n", + " \"intel\": [\n", + " {\n", + " \"vuln_id\": \"CVE-2024-21762\",\n", + " \"ghsa\": {\n", + " \"ghsa_id\": \"GHSA-v4hq-m4wr-6pmj\",\n", + " \"cve_id\": \"CVE-2024-21762\",\n", + " \"summary\": \"A out-of-bounds write in Fortinet FortiOS versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.6, 7.0...\",\n", + " \"description\": \"A out-of-bounds write in Fortinet FortiOS versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.6, 7.0.0 through 7.0.13, 6.4.0 through 6.4.14, 6.2.0 through 6.2.15, 6.0.0 through 6.0.17, FortiProxy versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.8, 7.0.0 through 7.0.14, 2.0.0 through 2.0.13, 1.2.0 through 1.2.13, 1.1.0 through 1.1.6, 1.0.0 through 1.0.7 allows attacker to execute unauthorized code or commands via specifically crafted requests\",\n", + " \"severity\": \"critical\",\n", + " \"vulnerabilities\": [],\n", + " \"cvss\": {\n", + " \"score\": 9.8,\n", + " \"vector_string\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\"\n", + " },\n", + " \"cwes\": [\n", + " {\n", + " \"cwe_id\": \"CWE-787\",\n", + " \"name\": \"Out-of-bounds Write\"\n", + " }\n", + " ],\n", + " \"url\": \"https://api.github.com/advisories/GHSA-v4hq-m4wr-6pmj\",\n", + " \"html_url\": \"https://github.com/advisories/GHSA-v4hq-m4wr-6pmj\",\n", + " \"type\": \"unreviewed\",\n", + " \"repository_advisory_url\": null,\n", + " \"source_code_location\": \"\",\n", + " \"identifiers\": [\n", + " {\n", + " \"value\": \"GHSA-v4hq-m4wr-6pmj\",\n", + " \"type\": \"GHSA\"\n", + " },\n", + " {\n", + " \"value\": \"CVE-2024-21762\",\n", + " \"type\": \"CVE\"\n", + " }\n", + " ],\n", + " \"references\": [\n", + " \"https://nvd.nist.gov/vuln/detail/CVE-2024-21762\",\n", + " \"https://fortiguard.com/psirt/FG-IR-24-015\",\n", + " \"https://github.com/advisories/GHSA-v4hq-m4wr-6pmj\"\n", + " ],\n", + " \"published_at\": \"2024-02-09T09:31:31Z\",\n", + " \"updated_at\": \"2024-02-09T09:31:40Z\",\n", + " \"github_reviewed_at\": null,\n", + " \"nvd_published_at\": \"2024-02-09T09:15:08Z\",\n", + " \"withdrawn_at\": null,\n", + " \"credits\": [],\n", + " \"cvss_severities\": {\n", + " \"cvss_v3\": {\n", + " \"vector_string\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", + " \"score\": 9.8\n", + " },\n", + " \"cvss_v4\": {\n", + " \"vector_string\": null,\n", + " \"score\": 0.0\n", + " }\n", + " },\n", + " \"epss\": {\n", + " \"percentage\": 0.01842,\n", + " \"percentile\": 0.88574\n", + " }\n", + " },\n", + " \"nvd\": {\n", + " \"cve_id\": \"CVE-2024-21762\",\n", + " \"cve_description\": \"A out-of-bounds write in Fortinet FortiOS versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.6, 7.0.0 through 7.0.13, 6.4.0 through 6.4.14, 6.2.0 through 6.2.15, 6.0.0 through 6.0.17, FortiProxy versions 7.4.0 through 7.4.2, 7.2.0 through 7.2.8, 7.0.0 through 7.0.14, 2.0.0 through 2.0.13, 1.2.0 through 1.2.13, 1.1.0 through 1.1.6, 1.0.0 through 1.0.7 allows attacker to execute unauthorized code or commands via specifically crafted requests\",\n", + " \"cvss_vector\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", + " \"cwe_name\": \"CWE-787: Out-of-bounds Write (4.15)\",\n", + " \"cwe_description\": \"The product writes data past the end, or before the beginning, of the intended buffer.\",\n", + " \"cwe_extended_description\": null,\n", + " \"configurations\": [\n", + " {\n", + " \"package\": \"fortiproxy\",\n", + " \"system\": null,\n", + " \"versionStartExcluding\": null,\n", + " \"versionEndExcluding\": \"2.0.14\",\n", + " \"versionStartIncluding\": \"1.0.0\",\n", + " \"versionEndIncluding\": null\n", + " },\n", + " {\n", + " \"package\": \"fortiproxy\",\n", + " \"system\": null,\n", + " \"versionStartExcluding\": null,\n", + " \"versionEndExcluding\": \"7.0.15\",\n", + " \"versionStartIncluding\": \"7.0.0\",\n", + " \"versionEndIncluding\": null\n", + " },\n", + " {\n", + " \"package\": \"fortiproxy\",\n", + " \"system\": null,\n", + " \"versionStartExcluding\": null,\n", + " \"versionEndExcluding\": \"7.2.9\",\n", + " \"versionStartIncluding\": \"7.2.0\",\n", + " \"versionEndIncluding\": null\n", + " },\n", + " {\n", + " \"package\": \"fortiproxy\",\n", + " \"system\": null,\n", + " \"versionStartExcluding\": null,\n", + " \"versionEndExcluding\": \"7.4.3\",\n", + " \"versionStartIncluding\": \"7.4.0\",\n", + " \"versionEndIncluding\": null\n", + " },\n", + " {\n", + " \"package\": \"fortios\",\n", + " \"system\": null,\n", + " \"versionStartExcluding\": null,\n", + " \"versionEndExcluding\": \"6.2.16\",\n", + " \"versionStartIncluding\": \"6.0.0\",\n", + " \"versionEndIncluding\": null\n", + " },\n", + " {\n", + " \"package\": \"fortios\",\n", + " \"system\": null,\n", + " \"versionStartExcluding\": null,\n", + " \"versionEndExcluding\": \"6.4.15\",\n", + " \"versionStartIncluding\": \"6.4.0\",\n", + " \"versionEndIncluding\": null\n", + " },\n", + " {\n", + " \"package\": \"fortios\",\n", + " \"system\": null,\n", + " \"versionStartExcluding\": null,\n", + " \"versionEndExcluding\": \"7.0.14\",\n", + " \"versionStartIncluding\": \"7.0.0\",\n", + " \"versionEndIncluding\": null\n", + " },\n", + " {\n", + " \"package\": \"fortios\",\n", + " \"system\": null,\n", + " \"versionStartExcluding\": null,\n", + " \"versionEndExcluding\": \"7.2.7\",\n", + " \"versionStartIncluding\": \"7.2.0\",\n", + " \"versionEndIncluding\": null\n", + " },\n", + " {\n", + " \"package\": \"fortios\",\n", + " \"system\": null,\n", + " \"versionStartExcluding\": null,\n", + " \"versionEndExcluding\": \"7.4.3\",\n", + " \"versionStartIncluding\": \"7.4.0\",\n", + " \"versionEndIncluding\": null\n", + " }\n", + " ],\n", + " \"vendor_names\": [\n", + " \"Fortinet\"\n", + " ]\n", + " },\n", + " \"rhsa\": {\n", + " \"bugzilla\": {\n", + " \"description\": null,\n", + " \"id\": null,\n", + " \"url\": null\n", + " },\n", + " \"details\": null,\n", + " \"statement\": null,\n", + " \"package_state\": null,\n", + " \"upstream_fix\": null\n", + " },\n", + " \"ubuntu\": {\n", + " \"description\": null,\n", + " \"notes\": null,\n", + " \"priority\": null,\n", + " \"ubuntu_description\": null\n", + " },\n", + " \"epss\": {\n", + " \"epss\": 0.01842,\n", + " \"percentile\": 0.88672,\n", + " \"date\": \"2024-10-30\",\n", + " \"cve\": \"CVE-2024-21762\"\n", + " }\n", + " },\n", + " {\n", + " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", + " \"ghsa\": {\n", + " \"ghsa_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", + " \"cve_id\": \"CVE-2023-6975\",\n", + " \"summary\": \"MLFlow Path Traversal Vulnerability\",\n", + " \"description\": \"A malicious user could use this issue to get command execution on the vulnerable machine and get access to data & models information.\",\n", + " \"severity\": \"critical\",\n", + " \"vulnerabilities\": [\n", + " {\n", + " \"package\": {\n", + " \"ecosystem\": \"pip\",\n", + " \"name\": \"mlflow\"\n", + " },\n", + " \"vulnerable_version_range\": \"< 2.9.2\",\n", + " \"first_patched_version\": \"2.9.2\",\n", + " \"vulnerable_functions\": []\n", + " }\n", + " ],\n", + " \"cvss\": {\n", + " \"score\": 9.8,\n", + " \"vector_string\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\"\n", + " },\n", + " \"cwes\": [\n", + " {\n", + " \"cwe_id\": \"CWE-29\",\n", + " \"name\": \"Path Traversal: '..filename'\"\n", + " }\n", + " ],\n", + " \"url\": \"https://api.github.com/advisories/GHSA-hh8p-p8mp-gqhm\",\n", + " \"html_url\": \"https://github.com/advisories/GHSA-hh8p-p8mp-gqhm\",\n", + " \"type\": \"reviewed\",\n", + " \"repository_advisory_url\": null,\n", + " \"source_code_location\": \"https://github.com/mlflow/mlflow\",\n", + " \"identifiers\": [\n", + " {\n", + " \"value\": \"GHSA-hh8p-p8mp-gqhm\",\n", + " \"type\": \"GHSA\"\n", + " },\n", + " {\n", + " \"value\": \"CVE-2023-6975\",\n", + " \"type\": \"CVE\"\n", + " }\n", + " ],\n", + " \"references\": [\n", + " \"https://nvd.nist.gov/vuln/detail/CVE-2023-6975\",\n", + " \"https://github.com/mlflow/mlflow/commit/b9ab9ed77e1deda9697fe472fb1079fd428149ee\",\n", + " \"https://huntr.com/bounties/029a3824-cee3-4cf1-b260-7138aa539b85\",\n", + " \"https://github.com/advisories/GHSA-hh8p-p8mp-gqhm\"\n", + " ],\n", + " \"published_at\": \"2023-12-20T06:30:25Z\",\n", + " \"updated_at\": \"2024-01-02T15:14:39Z\",\n", + " \"github_reviewed_at\": \"2023-12-20T20:32:39Z\",\n", + " \"nvd_published_at\": \"2023-12-20T06:15:45Z\",\n", + " \"withdrawn_at\": null,\n", + " \"credits\": [],\n", + " \"cvss_severities\": {\n", + " \"cvss_v3\": {\n", + " \"vector_string\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", + " \"score\": 9.8\n", + " },\n", + " \"cvss_v4\": {\n", + " \"vector_string\": null,\n", + " \"score\": 0.0\n", + " }\n", + " },\n", + " \"epss\": {\n", + " \"percentage\": 0.00118,\n", + " \"percentile\": 0.46635\n", + " }\n", + " },\n", + " \"nvd\": {\n", + " \"cve_id\": \"CVE-2023-6975\",\n", + " \"cve_description\": \"A malicious user could use this issue to get command execution on the vulnerable machine and get access to data & models information.\",\n", + " \"cvss_vector\": \"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", + " \"cwe_name\": \"CWE-29: Path Traversal: '\\\\..\\\\filename' (4.15)\",\n", + " \"cwe_description\": \"The product uses external input to construct a pathname that should be within a restricted directory, but it does not properly neutralize '\\\\..\\\\filename' (leading backslash dot dot) sequences that can resolve to a location that is outside of that directory.\",\n", + " \"cwe_extended_description\": \"This allows attackers to traverse the file system to access files or directories that are outside of the restricted directory. This is similar to CWE-25, except using \\\"\\\\\\\" instead of \\\"/\\\". Sometimes a program checks for \\\"..\\\\\\\" at the beginning of the input, so a \\\"\\\\..\\\\\\\" can bypass that check. It is also useful for bypassing path traversal protection schemes that only assume that the \\\"/\\\" separator is valid.\",\n", + " \"configurations\": [\n", + " {\n", + " \"package\": \"mlflow\",\n", + " \"system\": null,\n", + " \"versionStartExcluding\": null,\n", + " \"versionEndExcluding\": \"2.9.2\",\n", + " \"versionStartIncluding\": null,\n", + " \"versionEndIncluding\": null\n", + " }\n", + " ],\n", + " \"vendor_names\": [\n", + " \"Lfprojects\"\n", + " ]\n", + " },\n", + " \"rhsa\": {\n", + " \"bugzilla\": {\n", + " \"description\": null,\n", + " \"id\": null,\n", + " \"url\": null\n", + " },\n", + " \"details\": null,\n", + " \"statement\": null,\n", + " \"package_state\": null,\n", + " \"upstream_fix\": null\n", + " },\n", + " \"ubuntu\": {\n", + " \"description\": null,\n", + " \"notes\": null,\n", + " \"priority\": null,\n", + " \"ubuntu_description\": null\n", + " },\n", + " \"epss\": {\n", + " \"epss\": 0.00118,\n", + " \"percentile\": 0.46869,\n", + " \"date\": \"2024-10-28\",\n", + " \"cve\": \"CVE-2023-6975\"\n", + " }\n", + " },\n", + " {\n", + " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\",\n", + " \"ghsa\": {\n", + " \"ghsa_id\": \"GHSA-3f63-hfp8-52jq\",\n", + " \"cve_id\": \"CVE-2023-50447\",\n", + " \"summary\": \"Arbitrary Code Execution in Pillow\",\n", + " \"description\": \"Pillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution via the environment parameter, a different vulnerability than CVE-2022-22817 (which was about the expression parameter).\",\n", + " \"severity\": \"high\",\n", + " \"vulnerabilities\": [\n", + " {\n", + " \"package\": {\n", + " \"ecosystem\": \"pip\",\n", + " \"name\": \"Pillow\"\n", + " },\n", + " \"vulnerable_version_range\": \"< 10.2.0\",\n", + " \"first_patched_version\": \"10.2.0\",\n", + " \"vulnerable_functions\": [\n", + " \"PIL.ImageMath.eval\"\n", + " ]\n", + " }\n", + " ],\n", + " \"cvss\": {\n", + " \"score\": 8.1,\n", + " \"vector_string\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\"\n", + " },\n", + " \"cwes\": [\n", + " {\n", + " \"cwe_id\": \"CWE-94\",\n", + " \"name\": \"Improper Control of Generation of Code ('Code Injection')\"\n", + " },\n", + " {\n", + " \"cwe_id\": \"CWE-95\",\n", + " \"name\": \"Improper Neutralization of Directives in Dynamically Evaluated Code ('Eval Injection')\"\n", + " }\n", + " ],\n", + " \"url\": \"https://api.github.com/advisories/GHSA-3f63-hfp8-52jq\",\n", + " \"html_url\": \"https://github.com/advisories/GHSA-3f63-hfp8-52jq\",\n", + " \"type\": \"reviewed\",\n", + " \"repository_advisory_url\": null,\n", + " \"source_code_location\": \"https://github.com/python-pillow/Pillow\",\n", + " \"identifiers\": [\n", + " {\n", + " \"value\": \"GHSA-3f63-hfp8-52jq\",\n", + " \"type\": \"GHSA\"\n", + " },\n", + " {\n", + " \"value\": \"CVE-2023-50447\",\n", + " \"type\": \"CVE\"\n", + " }\n", + " ],\n", + " \"references\": [\n", + " \"https://nvd.nist.gov/vuln/detail/CVE-2023-50447\",\n", + " \"https://github.com/python-pillow/Pillow/commit/45c726fd4daa63236a8f3653530f297dc87b160a\",\n", + " \"https://pillow.readthedocs.io/en/stable/releasenotes/10.2.0.html#security\",\n", + " \"http://www.openwall.com/lists/oss-security/2024/01/20/1\",\n", + " \"https://github.com/python-pillow/Pillow/releases\",\n", + " \"https://lists.debian.org/debian-lts-announce/2024/01/msg00019.html\",\n", + " \"https://devhub.checkmarx.com/cve-details/CVE-2023-50447\",\n", + " \"https://duartecsantos.github.io/2023-01-02-CVE-2023-50447\",\n", + " \"https://duartecsantos.github.io/2024-01-02-CVE-2023-50447\",\n", + " \"https://github.com/advisories/GHSA-3f63-hfp8-52jq\"\n", + " ],\n", + " \"published_at\": \"2024-01-19T21:30:35Z\",\n", + " \"updated_at\": \"2024-08-02T10:37:52Z\",\n", + " \"github_reviewed_at\": \"2024-01-22T21:28:18Z\",\n", + " \"nvd_published_at\": \"2024-01-19T20:15:11Z\",\n", + " \"withdrawn_at\": null,\n", + " \"credits\": [],\n", + " \"cvss_severities\": {\n", + " \"cvss_v3\": {\n", + " \"vector_string\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", + " \"score\": 8.1\n", + " },\n", + " \"cvss_v4\": {\n", + " \"vector_string\": \"CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:H/VA:H/SC:N/SI:N/SA:N\",\n", + " \"score\": 9.3\n", + " }\n", + " },\n", + " \"epss\": {\n", + " \"percentage\": 0.00074,\n", + " \"percentile\": 0.32886\n", + " }\n", + " },\n", + " \"nvd\": {\n", + " \"cve_id\": \"CVE-2023-50447\",\n", + " \"cve_description\": \"Pillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution via the environment parameter, a different vulnerability than CVE-2022-22817 (which was about the expression parameter).\",\n", + " \"cvss_vector\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", + " \"cwe_name\": \"CWE-94: Improper Control of Generation of Code ('Code Injection') (4.15)\",\n", + " \"cwe_description\": \"The product constructs all or part of a code segment using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the syntax or behavior of the intended code segment.\",\n", + " \"cwe_extended_description\": \"When a product allows a user's input to contain code syntax, it might be possible for an attacker to craft the code in such a way that it will alter the intended control flow of the product. Such an alteration could lead to arbitrary code execution. Injection problems encompass a wide variety of issues -- all mitigated in very different ways. For this reason, the most effective way to discuss these weaknesses is to note the distinct features which classify them as injection weaknesses. The most important issue to note is that all injection problems share one thing in common -- i.e., they allow for the injection of control plane data into the user-controlled data plane. This means that the execution of the process may be altered by sending code in through legitimate data channels, using no other mechanism. While buffer overflows, and many other flaws, involve the use of some further issue to gain execution, injection problems need only for the data to be parsed. The most classic instantiations of this category of weakness are SQL injection and format string vulnerabilities.\",\n", + " \"configurations\": [\n", + " {\n", + " \"package\": \"pillow\",\n", + " \"system\": null,\n", + " \"versionStartExcluding\": null,\n", + " \"versionEndExcluding\": null,\n", + " \"versionStartIncluding\": null,\n", + " \"versionEndIncluding\": \"10.1.0\"\n", + " },\n", + " {\n", + " \"package\": \"debian_linux\",\n", + " \"system\": null,\n", + " \"versionStartExcluding\": null,\n", + " \"versionEndExcluding\": null,\n", + " \"versionStartIncluding\": \"10.0\",\n", + " \"versionEndIncluding\": \"10.0\"\n", + " }\n", + " ],\n", + " \"vendor_names\": [\n", + " \"Debian\",\n", + " \"Python\"\n", + " ]\n", + " },\n", + " \"rhsa\": {\n", + " \"bugzilla\": {\n", + " \"description\": \"pillow: Arbitrary Code Execution via the environment parameter\",\n", + " \"id\": \"2259479\",\n", + " \"url\": \"https://bugzilla.redhat.com/show_bug.cgi?id=2259479\"\n", + " },\n", + " \"details\": [\n", + " \"Pillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution via the environment parameter, a different vulnerability than CVE-2022-22817 (which was about the expression parameter).\",\n", + " \"A vulnerability was found in Pillow, a popular Python imaging library. The flaw identified in the PIL.ImageMath.eval function enables arbitrary code execution by manipulating the environment parameter.\"\n", + " ],\n", + " \"statement\": \"The vulnerability in Pillow's PIL.ImageMath.eval function poses a significant threat due to its potential for arbitrary code execution. Pillow's widespread use in diverse domains makes this flaw particularly impactful, as it could lead to unauthorized access, data breaches, and compromise of entire systems. The complex exploitation method involving Python's dunder methods adds sophistication to potential attacks.\",\n", + " \"package_state\": null,\n", + " \"upstream_fix\": null,\n", + " \"threat_severity\": \"Important\",\n", + " \"public_date\": \"2024-01-19T00:00:00Z\",\n", + " \"cvss3\": {\n", + " \"cvss3_base_score\": \"8.1\",\n", + " \"cvss3_scoring_vector\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", + " \"status\": \"verified\"\n", + " },\n", + " \"cwe\": \"CWE-77\",\n", + " \"affected_release\": [\n", + " {\n", + " \"product_name\": \"Red Hat Ansible Automation Platform 2.4 for RHEL 8\",\n", + " \"release_date\": \"2024-06-10T00:00:00Z\",\n", + " \"advisory\": \"RHSA-2024:3781\",\n", + " \"cpe\": \"cpe:/a:redhat:ansible_automation_platform:2.4::el8\",\n", + " \"package\": \"python3x-pillow-0:10.3.0-1.el8ap\"\n", + " },\n", + " {\n", + " \"product_name\": \"Red Hat Ansible Automation Platform 2.4 for RHEL 9\",\n", + " \"release_date\": \"2024-06-10T00:00:00Z\",\n", + " \"advisory\": \"RHSA-2024:3781\",\n", + " \"cpe\": \"cpe:/a:redhat:ansible_automation_platform:2.4::el9\",\n", + " \"package\": \"python-pillow-0:10.3.0-1.el9ap\"\n", + " },\n", + " {\n", + " \"product_name\": \"Red Hat Enterprise Linux 7\",\n", + " \"release_date\": \"2024-02-19T00:00:00Z\",\n", + " \"advisory\": \"RHSA-2024:0857\",\n", + " \"cpe\": \"cpe:/o:redhat:enterprise_linux:7\",\n", + " \"package\": \"python-pillow-0:2.0.0-25.gitd1c6db8.el7_9\"\n", + " },\n", + " {\n", + " \"product_name\": \"Red Hat Enterprise Linux 8\",\n", + " \"release_date\": \"2024-02-20T00:00:00Z\",\n", + " \"advisory\": \"RHSA-2024:0893\",\n", + " \"cpe\": \"cpe:/a:redhat:enterprise_linux:8\",\n", + " \"package\": \"python-pillow-0:5.1.1-18.el8_9.1\"\n", + " },\n", + " {\n", + " \"product_name\": \"Red Hat Enterprise Linux 8.2 Advanced Update Support\",\n", + " \"release_date\": \"2024-02-29T00:00:00Z\",\n", + " \"advisory\": \"RHSA-2024:1059\",\n", + " \"cpe\": \"cpe:/a:redhat:rhel_aus:8.2\",\n", + " \"package\": \"python-pillow-0:5.1.1-15.el8_2\"\n", + " },\n", + " {\n", + " \"product_name\": \"Red Hat Enterprise Linux 8.4 Advanced Mission Critical Update Support\",\n", + " \"release_date\": \"2024-02-29T00:00:00Z\",\n", + " \"advisory\": \"RHSA-2024:1060\",\n", + " \"cpe\": \"cpe:/a:redhat:rhel_aus:8.4\",\n", + " \"package\": \"python-pillow-0:5.1.1-15.el8_4\"\n", + " },\n", + " {\n", + " \"product_name\": \"Red Hat Enterprise Linux 8.4 Telecommunications Update Service\",\n", + " \"release_date\": \"2024-02-29T00:00:00Z\",\n", + " \"advisory\": \"RHSA-2024:1060\",\n", + " \"cpe\": \"cpe:/a:redhat:rhel_tus:8.4\",\n", + " \"package\": \"python-pillow-0:5.1.1-15.el8_4\"\n", + " },\n", + " {\n", + " \"product_name\": \"Red Hat Enterprise Linux 8.4 Update Services for SAP Solutions\",\n", + " \"release_date\": \"2024-02-29T00:00:00Z\",\n", + " \"advisory\": \"RHSA-2024:1060\",\n", + " \"cpe\": \"cpe:/a:redhat:rhel_e4s:8.4\",\n", + " \"package\": \"python-pillow-0:5.1.1-15.el8_4\"\n", + " },\n", + " {\n", + " \"product_name\": \"Red Hat Enterprise Linux 8.6 Extended Update Support\",\n", + " \"release_date\": \"2024-02-29T00:00:00Z\",\n", + " \"advisory\": \"RHSA-2024:1058\",\n", + " \"cpe\": \"cpe:/a:redhat:rhel_eus:8.6\",\n", + " \"package\": \"python-pillow-0:5.1.1-19.el8_6\"\n", + " },\n", + " {\n", + " \"product_name\": \"Red Hat Enterprise Linux 8.8 Extended Update Support\",\n", + " \"release_date\": \"2024-02-08T00:00:00Z\",\n", + " \"advisory\": \"RHSA-2024:0754\",\n", + " \"cpe\": \"cpe:/a:redhat:rhel_eus:8.8\",\n", + " \"package\": \"python-pillow-0:5.1.1-19.el8_8\"\n", + " }\n", + " ],\n", + " \"references\": [\n", + " \"https://www.cve.org/CVERecord?id=CVE-2023-50447\\nhttps://nvd.nist.gov/vuln/detail/CVE-2023-50447\\nhttp://www.openwall.com/lists/oss-security/2024/01/20/1\\nhttps://devhub.checkmarx.com/cve-details/CVE-2023-50447/\\nhttps://duartecsantos.github.io/2023-01-02-CVE-2023-50447/\\nhttps://github.com/python-pillow/Pillow/releases\"\n", + " ],\n", + " \"name\": \"CVE-2023-50447\",\n", + " \"mitigation\": {\n", + " \"value\": \"Mitigation for this issue is either not available or the currently available options do not meet the Red Hat Product Security criteria comprising ease of use and deployment, applicability to widespread installation base or stability.\",\n", + " \"lang\": \"en:us\"\n", + " },\n", + " \"csaw\": false\n", + " },\n", + " \"ubuntu\": {\n", + " \"description\": \"\\nPillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution\\nvia the environment parameter, a different vulnerability than\\nCVE-2022-22817 (which was about the expression parameter).\",\n", + " \"notes\": [],\n", + " \"priority\": \"medium\",\n", + " \"ubuntu_description\": \"\",\n", + " \"bugs\": [\n", + " \"http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1061172\"\n", + " ],\n", + " \"codename\": null,\n", + " \"cvss3\": 8.1,\n", + " \"id\": \"CVE-2023-50447\",\n", + " \"impact\": {\n", + " \"baseMetricV3\": {\n", + " \"cvssV3\": {\n", + " \"attackComplexity\": \"HIGH\",\n", + " \"attackVector\": \"NETWORK\",\n", + " \"availabilityImpact\": \"HIGH\",\n", + " \"baseScore\": 8.1,\n", + " \"baseSeverity\": \"HIGH\",\n", + " \"confidentialityImpact\": \"HIGH\",\n", + " \"integrityImpact\": \"HIGH\",\n", + " \"privilegesRequired\": \"NONE\",\n", + " \"scope\": \"UNCHANGED\",\n", + " \"userInteraction\": \"NONE\",\n", + " \"vectorString\": \"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\",\n", + " \"version\": \"3.1\"\n", + " },\n", + " \"exploitabilityScore\": 2.2,\n", + " \"impactScore\": 5.9\n", + " }\n", + " },\n", + " \"mitigation\": \"\",\n", + " \"notices\": [\n", + " {\n", + " \"cves_ids\": [\n", + " \"CVE-2023-44271\",\n", + " \"CVE-2023-50447\"\n", + " ],\n", + " \"description\": \"It was discovered that Pillow incorrectly handled certain long text\\narguments. An attacker could possibly use this issue to cause Pillow to\\nconsume resources, leading to a denial of service. This issue only affected\\nUbuntu 20.04 LTS, and Ubuntu 22.04 LTS. (CVE-2023-44271)\\n\\nDuarte Santos discovered that Pillow incorrectly handled the environment\\nparameter to PIL.ImageMath.eval. An attacker could possibly use this issue\\nto execute arbitrary code. (CVE-2023-50447)\\n\",\n", + " \"id\": \"USN-6618-1\",\n", + " \"instructions\": \"In general, a standard system update will make all the necessary changes.\\n\",\n", + " \"is_hidden\": false,\n", + " \"published\": \"2024-01-30T15:17:54.445907\",\n", + " \"references\": [],\n", + " \"release_packages\": {\n", + " \"focal\": [\n", + " {\n", + " \"description\": \"Python Imaging Library\",\n", + " \"is_source\": true,\n", + " \"name\": \"pillow\",\n", + " \"version\": \"7.0.0-4ubuntu0.8\"\n", + " },\n", + " {\n", + " \"is_source\": false,\n", + " \"is_visible\": false,\n", + " \"name\": \"python-pil-doc\",\n", + " \"pocket\": \"security\",\n", + " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", + " \"version\": \"7.0.0-4ubuntu0.8\",\n", + " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/7.0.0-4ubuntu0.8\"\n", + " },\n", + " {\n", + " \"is_source\": false,\n", + " \"is_visible\": true,\n", + " \"name\": \"python3-pil\",\n", + " \"pocket\": \"security\",\n", + " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", + " \"version\": \"7.0.0-4ubuntu0.8\",\n", + " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/7.0.0-4ubuntu0.8\"\n", + " },\n", + " {\n", + " \"is_source\": false,\n", + " \"is_visible\": false,\n", + " \"name\": \"python3-pil.imagetk\",\n", + " \"pocket\": \"security\",\n", + " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", + " \"version\": \"7.0.0-4ubuntu0.8\",\n", + " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/7.0.0-4ubuntu0.8\"\n", + " }\n", + " ],\n", + " \"jammy\": [\n", + " {\n", + " \"description\": \"Python Imaging Library\",\n", + " \"is_source\": true,\n", + " \"name\": \"pillow\",\n", + " \"version\": \"9.0.1-1ubuntu0.2\"\n", + " },\n", + " {\n", + " \"is_source\": false,\n", + " \"is_visible\": false,\n", + " \"name\": \"python-pil-doc\",\n", + " \"pocket\": \"security\",\n", + " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", + " \"version\": \"9.0.1-1ubuntu0.2\",\n", + " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/9.0.1-1ubuntu0.2\"\n", + " },\n", + " {\n", + " \"is_source\": false,\n", + " \"is_visible\": true,\n", + " \"name\": \"python3-pil\",\n", + " \"pocket\": \"security\",\n", + " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", + " \"version\": \"9.0.1-1ubuntu0.2\",\n", + " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/9.0.1-1ubuntu0.2\"\n", + " },\n", + " {\n", + " \"is_source\": false,\n", + " \"is_visible\": false,\n", + " \"name\": \"python3-pil.imagetk\",\n", + " \"pocket\": \"security\",\n", + " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", + " \"version\": \"9.0.1-1ubuntu0.2\",\n", + " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/9.0.1-1ubuntu0.2\"\n", + " }\n", + " ],\n", + " \"mantic\": [\n", + " {\n", + " \"description\": \"Python Imaging Library\",\n", + " \"is_source\": true,\n", + " \"name\": \"pillow\",\n", + " \"version\": \"10.0.0-1ubuntu0.1\"\n", + " },\n", + " {\n", + " \"is_source\": false,\n", + " \"is_visible\": false,\n", + " \"name\": \"python-pil-doc\",\n", + " \"pocket\": \"security\",\n", + " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", + " \"version\": \"10.0.0-1ubuntu0.1\",\n", + " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/10.0.0-1ubuntu0.1\"\n", + " },\n", + " {\n", + " \"is_source\": false,\n", + " \"is_visible\": true,\n", + " \"name\": \"python3-pil\",\n", + " \"pocket\": \"security\",\n", + " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", + " \"version\": \"10.0.0-1ubuntu0.1\",\n", + " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/10.0.0-1ubuntu0.1\"\n", + " },\n", + " {\n", + " \"is_source\": false,\n", + " \"is_visible\": false,\n", + " \"name\": \"python3-pil.imagetk\",\n", + " \"pocket\": \"security\",\n", + " \"source_link\": \"https://launchpad.net/ubuntu/+source/pillow\",\n", + " \"version\": \"10.0.0-1ubuntu0.1\",\n", + " \"version_link\": \"https://launchpad.net/ubuntu/+source/pillow/10.0.0-1ubuntu0.1\"\n", + " }\n", + " ]\n", + " },\n", + " \"summary\": \"Several security issues were fixed in Pillow.\\n\",\n", + " \"title\": \"Pillow vulnerabilities\",\n", + " \"type\": \"USN\"\n", + " }\n", + " ],\n", + " \"notices_ids\": [\n", + " \"USN-6618-1\"\n", + " ],\n", + " \"packages\": [\n", + " {\n", + " \"debian\": \"https://tracker.debian.org/pkg/pillow\",\n", + " \"name\": \"pillow\",\n", + " \"source\": \"https://ubuntu.com/security/cve?package=pillow\",\n", + " \"statuses\": [\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"bionic\",\n", + " \"status\": \"needs-triage\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"7.0.0-4ubuntu0.8\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"focal\",\n", + " \"status\": \"released\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"9.0.1-1ubuntu0.2\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"jammy\",\n", + " \"status\": \"released\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"end of life, was needed\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"lunar\",\n", + " \"status\": \"ignored\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"10.0.0-1ubuntu0.1\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"mantic\",\n", + " \"status\": \"released\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"10.2.0-1\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"noble\",\n", + " \"status\": \"released\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"10.2.0-1\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"oracular\",\n", + " \"status\": \"released\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"trusty\",\n", + " \"status\": \"needs-triage\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"10.2.0-1\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"upstream\",\n", + " \"status\": \"released\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"xenial\",\n", + " \"status\": \"needs-triage\"\n", + " }\n", + " ],\n", + " \"ubuntu\": \"https://packages.ubuntu.com/search?suite=all§ion=all&arch=any&searchon=sourcenames&keywords=pillow\"\n", + " },\n", + " {\n", + " \"debian\": \"https://tracker.debian.org/pkg/pillow-python2\",\n", + " \"name\": \"pillow-python2\",\n", + " \"source\": \"https://ubuntu.com/security/cve?package=pillow-python2\",\n", + " \"statuses\": [\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"bionic\",\n", + " \"status\": \"DNE\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"focal\",\n", + " \"status\": \"needs-triage\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"jammy\",\n", + " \"status\": \"DNE\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"lunar\",\n", + " \"status\": \"DNE\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"mantic\",\n", + " \"status\": \"DNE\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"noble\",\n", + " \"status\": \"DNE\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"oracular\",\n", + " \"status\": \"DNE\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"trusty\",\n", + " \"status\": \"DNE\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"upstream\",\n", + " \"status\": \"needs-triage\"\n", + " },\n", + " {\n", + " \"component\": null,\n", + " \"description\": \"\",\n", + " \"pocket\": \"security\",\n", + " \"release_codename\": \"xenial\",\n", + " \"status\": \"DNE\"\n", + " }\n", + " ],\n", + " \"ubuntu\": \"https://packages.ubuntu.com/search?suite=all§ion=all&arch=any&searchon=sourcenames&keywords=pillow-python2\"\n", + " }\n", + " ],\n", + " \"patches\": {\n", + " \"pillow\": [\n", + " \"upstream: https://github.com/python-pillow/Pillow/commit/45c726fd4daa63236a8f3653530f297dc87b160a\",\n", + " \"upstream: https://github.com/python-pillow/Pillow/commit/0ca3c33c59927e1c7e0c14dbc1eea1dfb2431a80\",\n", + " \"upstream: https://github.com/python-pillow/Pillow/commit/557ba59d13de919d04b3fd4cdef8634f7d4b3348\"\n", + " ],\n", + " \"pillow-python2\": []\n", + " },\n", + " \"published\": \"2024-01-19T20:15:00\",\n", + " \"references\": [\n", + " \"https://duartecsantos.github.io/2023-01-02-CVE-2023-50447/\",\n", + " \"https://pillow.readthedocs.io/en/stable/releasenotes/10.2.0.html#imagemath-eval-restricted-environment-keys\",\n", + " \"https://devhub.checkmarx.com/cve-details/CVE-2023-50447/\",\n", + " \"http://www.openwall.com/lists/oss-security/2024/01/20/1\",\n", + " \"https://ubuntu.com/security/notices/USN-6618-1\",\n", + " \"https://www.cve.org/CVERecord?id=CVE-2023-50447\"\n", + " ],\n", + " \"status\": \"active\",\n", + " \"tags\": {},\n", + " \"updated_at\": \"2024-07-24T15:57:39.284958+00:00\"\n", + " },\n", + " \"epss\": {\n", + " \"epss\": 0.00074,\n", + " \"percentile\": 0.33095,\n", + " \"date\": \"2024-10-25\",\n", + " \"cve\": \"CVE-2023-50447\"\n", + " }\n", + " }\n", + " ],\n", + " \"sbom\": {\n", + " \"packages\": [\n", + " {\n", + " \"name\": \"Babel\",\n", + " \"version\": \"2.13.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"Brotli\",\n", + " \"version\": \"1.0.9\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"Brotli\",\n", + " \"version\": \"1.1.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"ConfigArgParse\",\n", + " \"version\": \"1.5.5\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"Flask\",\n", + " \"version\": \"3.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"GitPython\",\n", + " \"version\": \"3.1.40\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"Jinja2\",\n", + " \"version\": \"3.1.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"Mako\",\n", + " \"version\": \"1.3.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"Markdown\",\n", + " \"version\": \"3.5.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"MarkupSafe\",\n", + " \"version\": \"2.1.3\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"Pillow\",\n", + " \"version\": \"10.1.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"PyJWT\",\n", + " \"version\": \"2.8.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"PySocks\",\n", + " \"version\": \"1.7.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"PyYAML\",\n", + " \"version\": \"6.0.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"Pygments\",\n", + " \"version\": \"2.17.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"SQLAlchemy\",\n", + " \"version\": \"2.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"SciPy\",\n", + " \"version\": \"1.11.4\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"Simple\",\n", + " \"version\": \"Launcher\",\n", + " \"path\": null,\n", + " \"system\": \"1.1.0.14\"\n", + " },\n", + " {\n", + " \"name\": \"Sphinx\",\n", + " \"version\": \"7.2.6\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"Werkzeug\",\n", + " \"version\": \"3.0.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"absl-py\",\n", + " \"version\": \"1.4.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"adduser\",\n", + " \"version\": \"3.118ubuntu5\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"alabaster\",\n", + " \"version\": \"0.7.13\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"alembic\",\n", + " \"version\": \"1.13.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"anyio\",\n", + " \"version\": \"3.7.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"appdirs\",\n", + " \"version\": \"1.4.4\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"apt\",\n", + " \"version\": \"2.4.11\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"asn1crypto\",\n", + " \"version\": \"1.5.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"attrs\",\n", + " \"version\": \"23.1.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"base-files\",\n", + " \"version\": \"12ubuntu4.4\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"base-passwd\",\n", + " \"version\": \"3.5.52build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"bash\",\n", + " \"version\": \"5.1-6ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"bc\",\n", + " \"version\": \"1.07.1-3build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"beautifulsoup4\",\n", + " \"version\": \"4.12.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"betterproto\",\n", + " \"version\": \"1.2.5\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"binutils\",\n", + " \"version\": \"2.38-4ubuntu2.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"binutils-common\",\n", + " \"version\": \"2.38-4ubuntu2.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"binutils-x86-64-linux-gnu\",\n", + " \"version\": \"2.38-4ubuntu2.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"blinker\",\n", + " \"version\": \"1.7.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"boa\",\n", + " \"version\": \"0.16.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"bokeh\",\n", + " \"version\": \"2.4.3\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"boltons\",\n", + " \"version\": \"23.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"bsdutils\",\n", + " \"version\": \"1:2.37.2-4ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"build-essential\",\n", + " \"version\": \"12.9ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"bzip2\",\n", + " \"version\": \"1.0.8-5build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"ca-certificates\",\n", + " \"version\": \"20230311ubuntu0.22.04.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"ca-certificates-java\",\n", + " \"version\": \"java-archive\",\n", + " \"path\": null,\n", + " \"system\": \"\\u001b[38;2;119;119;119m\\u001b[0m\"\n", + " },\n", + " {\n", + " \"name\": \"ca-certificates-java\",\n", + " \"version\": \"20190909ubuntu1.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cachetools\",\n", + " \"version\": \"5.3.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"cattrs\",\n", + " \"version\": \"23.2.3\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"certifi\",\n", + " \"version\": \"2023.11.17\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"cffi\",\n", + " \"version\": \"1.15.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"cffi\",\n", + " \"version\": \"1.16.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"chardet\",\n", + " \"version\": \"5.2.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"charset-normalizer\",\n", + " \"version\": \"3.2.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"charset-normalizer\",\n", + " \"version\": \"3.3.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"click\",\n", + " \"version\": \"8.1.7\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"cloudpickle\",\n", + " \"version\": \"3.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"colorama\",\n", + " \"version\": \"0.4.6\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"conda\",\n", + " \"version\": \"23.3.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"conda-build\",\n", + " \"version\": \"3.28.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"conda-libmamba-solver\",\n", + " \"version\": \"23.3.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"conda-package-handling\",\n", + " \"version\": \"2.2.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"conda_index\",\n", + " \"version\": \"0.3.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"conda_package_streaming\",\n", + " \"version\": \"0.9.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"configparser\",\n", + " \"version\": \"5.3.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"confluent-kafka\",\n", + " \"version\": \"1.9.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"contourpy\",\n", + " \"version\": \"1.2.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"coreutils\",\n", + " \"version\": \"8.32-4.1ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cpp\",\n", + " \"version\": \"4:11.2.0-1ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cpp-11\",\n", + " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cryptography\",\n", + " \"version\": \"41.0.3\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"cryptography\",\n", + " \"version\": \"41.0.7\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"cubinlinker\",\n", + " \"version\": \"0.3.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-cccl-11-8\",\n", + " \"version\": \"11.8.89-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-compat-11-8\",\n", + " \"version\": \"520.61.05-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-compiler-11-8\",\n", + " \"version\": \"11.8.0-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-cudart-11-8\",\n", + " \"version\": \"11.8.89-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-cudart-dev-11-8\",\n", + " \"version\": \"11.8.89-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-cuobjdump-11-8\",\n", + " \"version\": \"11.8.86-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-cupti-11-8\",\n", + " \"version\": \"11.8.87-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-cuxxfilt-11-8\",\n", + " \"version\": \"11.8.86-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-driver-dev-11-8\",\n", + " \"version\": \"11.8.89-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-gdb-11-8\",\n", + " \"version\": \"11.8.86-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-keyring\",\n", + " \"version\": \"1.1-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-nvcc-11-8\",\n", + " \"version\": \"11.8.89-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-nvdisasm-11-8\",\n", + " \"version\": \"11.8.86-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-nvprune-11-8\",\n", + " \"version\": \"11.8.86-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-python\",\n", + " \"version\": \"11.8.3\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-toolkit-11-8-config-common\",\n", + " \"version\": \"11.8.89-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-toolkit-11-config-common\",\n", + " \"version\": \"11.8.89-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cuda-toolkit-config-common\",\n", + " \"version\": \"12.3.101-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cudf\",\n", + " \"version\": \"23.6.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"cudf-kafka\",\n", + " \"version\": \"23.6.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"cupy\",\n", + " \"version\": \"12.2.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"curl\",\n", + " \"version\": \"7.81.0-1ubuntu1.15\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"cycler\",\n", + " \"version\": \"0.12.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"cytoolz\",\n", + " \"version\": \"0.12.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"dash\",\n", + " \"version\": \"0.5.11+git20210903+057cd650a4ed-3build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"dask\",\n", + " \"version\": \"2023.3.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"dask-cuda\",\n", + " \"version\": \"23.6.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"dask-cudf\",\n", + " \"version\": \"23.6.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"databricks-cli\",\n", + " \"version\": \"0.18.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"dataclasses\",\n", + " \"version\": \"0.8\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"datacompy\",\n", + " \"version\": \"0.8.4\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"debconf\",\n", + " \"version\": \"1.5.79ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"debianutils\",\n", + " \"version\": \"5.5-1ubuntu2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"diffutils\",\n", + " \"version\": \"1:3.8-0ubuntu2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"dill\",\n", + " \"version\": \"0.3.7\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"dirmngr\",\n", + " \"version\": \"2.2.27-3ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"distributed\",\n", + " \"version\": \"2023.3.2.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"docker\",\n", + " \"version\": \"5.0.3\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"docker-pycreds\",\n", + " \"version\": \"0.4.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"docutils\",\n", + " \"version\": \"0.20.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"dpkg\",\n", + " \"version\": \"1.21.1ubuntu2.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"dpkg-dev\",\n", + " \"version\": \"1.21.1ubuntu2.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"e2fsprogs\",\n", + " \"version\": \"1.46.5-2ubuntu1.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"entrypoints\",\n", + " \"version\": \"0.4\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"exceptiongroup\",\n", + " \"version\": \"1.2.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"fastrlock\",\n", + " \"version\": \"0.8.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"filelock\",\n", + " \"version\": \"3.13.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"findutils\",\n", + " \"version\": \"4.8.0-1ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"fontconfig-config\",\n", + " \"version\": \"2.13.1-4.2ubuntu5\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"fonts-dejavu-core\",\n", + " \"version\": \"2.37-2build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"fonttools\",\n", + " \"version\": \"4.46.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"fsspec\",\n", + " \"version\": \"2023.12.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"g++\",\n", + " \"version\": \"4:11.2.0-1ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"g++-11\",\n", + " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gcc\",\n", + " \"version\": \"4:11.2.0-1ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gcc-11\",\n", + " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gcc-11-base\",\n", + " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gcc-12-base\",\n", + " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gitdb\",\n", + " \"version\": \"4.0.11\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"gmpy2\",\n", + " \"version\": \"2.1.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"gnupg\",\n", + " \"version\": \"2.2.27-3ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gnupg-l10n\",\n", + " \"version\": \"2.2.27-3ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gnupg-utils\",\n", + " \"version\": \"2.2.27-3ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gnupg2\",\n", + " \"version\": \"2.2.27-3ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"googleapis-common-protos\",\n", + " \"version\": \"1.61.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"gpg\",\n", + " \"version\": \"2.2.27-3ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gpg-agent\",\n", + " \"version\": \"2.2.27-3ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gpg-wks-client\",\n", + " \"version\": \"2.2.27-3ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gpg-wks-server\",\n", + " \"version\": \"2.2.27-3ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gpgconf\",\n", + " \"version\": \"2.2.27-3ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gpgsm\",\n", + " \"version\": \"2.2.27-3ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"gpgv\",\n", + " \"version\": \"2.2.27-3ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"greenlet\",\n", + " \"version\": \"3.0.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"grep\",\n", + " \"version\": \"3.7-1build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"grpcio\",\n", + " \"version\": \"1.54.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"grpclib\",\n", + " \"version\": \"0.4.6\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"gunicorn\",\n", + " \"version\": \"21.2.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"gzip\",\n", + " \"version\": \"1.10-4ubuntu4.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"h2\",\n", + " \"version\": \"4.1.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"hostname\",\n", + " \"version\": \"3.23ubuntu2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"hpack\",\n", + " \"version\": \"4.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"hyperframe\",\n", + " \"version\": \"6.0.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"idna\",\n", + " \"version\": \"3.4\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"idna\",\n", + " \"version\": \"3.6\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"imagesize\",\n", + " \"version\": \"1.4.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"importlib-metadata\",\n", + " \"version\": \"7.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"importlib-resources\",\n", + " \"version\": \"6.1.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"init-system-helpers\",\n", + " \"version\": \"1.62\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"itsdangerous\",\n", + " \"version\": \"2.1.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"java-common\",\n", + " \"version\": \"0.72build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"joblib\",\n", + " \"version\": \"1.3.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"jq\",\n", + " \"version\": \"1.6-2.1ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"jrt-fs\",\n", + " \"version\": \"11.0.21\",\n", + " \"path\": null,\n", + " \"system\": \"java-archive\"\n", + " },\n", + " {\n", + " \"name\": \"js\",\n", + " \"version\": \"1.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"npm\"\n", + " },\n", + " {\n", + " \"name\": \"json5\",\n", + " \"version\": \"0.9.14\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"jsonpatch\",\n", + " \"version\": \"1.32\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"jsonpointer\",\n", + " \"version\": \"2.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"jsonschema\",\n", + " \"version\": \"4.20.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"jsonschema-specifications\",\n", + " \"version\": \"2023.11.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"jupyter_client\",\n", + " \"version\": \"8.6.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"jupyter_core\",\n", + " \"version\": \"5.5.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"kiwisolver\",\n", + " \"version\": \"1.4.5\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"libacl1\",\n", + " \"version\": \"2.3.1-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libapt-pkg6.0\",\n", + " \"version\": \"2.4.11\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libarchive-c\",\n", + " \"version\": \"5.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"libasan6\",\n", + " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libasound2\",\n", + " \"version\": \"1.2.6.1-1ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libasound2-data\",\n", + " \"version\": \"1.2.6.1-1ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libassuan0\",\n", + " \"version\": \"2.5.5-1build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libatomic1\",\n", + " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libattr1\",\n", + " \"version\": \"1:2.5.1-1build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libaudit-common\",\n", + " \"version\": \"1:3.0.7-1build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libaudit1\",\n", + " \"version\": \"1:3.0.7-1build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libavahi-client3\",\n", + " \"version\": \"0.8-5ubuntu5.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libavahi-common-data\",\n", + " \"version\": \"0.8-5ubuntu5.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libavahi-common3\",\n", + " \"version\": \"0.8-5ubuntu5.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libbinutils\",\n", + " \"version\": \"2.38-4ubuntu2.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libblkid1\",\n", + " \"version\": \"2.37.2-4ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libbrotli1\",\n", + " \"version\": \"1.0.9-2build6\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libbsd0\",\n", + " \"version\": \"0.11.5-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libbz2-1.0\",\n", + " \"version\": \"1.0.8-5build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libc-bin\",\n", + " \"version\": \"2.35-0ubuntu3.5\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libc-dev-bin\",\n", + " \"version\": \"2.35-0ubuntu3.5\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libc6\",\n", + " \"version\": \"2.35-0ubuntu3.5\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libc6-dev\",\n", + " \"version\": \"2.35-0ubuntu3.5\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libcap-ng0\",\n", + " \"version\": \"0.7.9-2.2build3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libcap2\",\n", + " \"version\": \"1:2.44-1ubuntu0.22.04.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libcbor0.8\",\n", + " \"version\": \"0.8.0-2ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libcc1-0\",\n", + " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libcom-err2\",\n", + " \"version\": \"1.46.5-2ubuntu1.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libcrypt-dev\",\n", + " \"version\": \"1:4.4.27-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libcrypt1\",\n", + " \"version\": \"1:4.4.27-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libctf-nobfd0\",\n", + " \"version\": \"2.38-4ubuntu2.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libctf0\",\n", + " \"version\": \"2.38-4ubuntu2.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libcublas-11-8\",\n", + " \"version\": \"11.11.3.6-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libcufft-11-8\",\n", + " \"version\": \"10.9.0.58-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libcups2\",\n", + " \"version\": \"2.4.1op1-1ubuntu4.7\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libcurand-11-8\",\n", + " \"version\": \"10.3.0.86-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libcurl4\",\n", + " \"version\": \"7.81.0-1ubuntu1.15\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libcusolver-11-8\",\n", + " \"version\": \"11.4.1.48-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libdb5.3\",\n", + " \"version\": \"5.3.28+dfsg1-0.8ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libdbus-1-3\",\n", + " \"version\": \"1.12.20-2ubuntu4.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libdebconfclient0\",\n", + " \"version\": \"0.261ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libdpkg-perl\",\n", + " \"version\": \"1.21.1ubuntu2.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libedit2\",\n", + " \"version\": \"3.1-20210910-1build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libexpat1\",\n", + " \"version\": \"2.4.7-1ubuntu0.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libext2fs2\",\n", + " \"version\": \"1.46.5-2ubuntu1.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libffi8\",\n", + " \"version\": \"3.4.2-4\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libfido2-1\",\n", + " \"version\": \"1.10.0-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libfontconfig1\",\n", + " \"version\": \"2.13.1-4.2ubuntu5\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libfreetype6\",\n", + " \"version\": \"2.11.1+dfsg-1ubuntu0.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libgcc-11-dev\",\n", + " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libgcc-s1\",\n", + " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libgcrypt20\",\n", + " \"version\": \"1.9.4-3ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libgdbm-compat4\",\n", + " \"version\": \"1.23-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libgdbm6\",\n", + " \"version\": \"1.23-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libglib2.0-0\",\n", + " \"version\": \"2.72.4-0ubuntu2.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libgmp10\",\n", + " \"version\": \"2:6.2.1+dfsg-3ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libgnutls30\",\n", + " \"version\": \"3.7.3-4ubuntu1.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libgomp1\",\n", + " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libgpg-error0\",\n", + " \"version\": \"1.43-3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libgraphite2-3\",\n", + " \"version\": \"1.3.14-1build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libgssapi-krb5-2\",\n", + " \"version\": \"1.19.2-2ubuntu0.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libharfbuzz0b\",\n", + " \"version\": \"2.7.4-1ubuntu3.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libhogweed6\",\n", + " \"version\": \"3.7.3-1build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libidn2-0\",\n", + " \"version\": \"2.3.2-2build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libisl23\",\n", + " \"version\": \"0.24-2build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libitm1\",\n", + " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libjpeg-turbo8\",\n", + " \"version\": \"2.1.2-0ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libjpeg8\",\n", + " \"version\": \"8c-2ubuntu10\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libjq1\",\n", + " \"version\": \"1.6-2.1ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libk5crypto3\",\n", + " \"version\": \"1.19.2-2ubuntu0.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libkeyutils1\",\n", + " \"version\": \"1.6.1-2ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libkrb5-3\",\n", + " \"version\": \"1.19.2-2ubuntu0.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libkrb5support0\",\n", + " \"version\": \"1.19.2-2ubuntu0.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libksba8\",\n", + " \"version\": \"1.6.0-2ubuntu0.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"liblcms2-2\",\n", + " \"version\": \"2.12~rc1-2build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libldap-2.5-0\",\n", + " \"version\": \"2.5.16+dfsg-0ubuntu0.22.04.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"liblsan0\",\n", + " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"liblz4-1\",\n", + " \"version\": \"1.9.3-2build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"liblzma5\",\n", + " \"version\": \"5.2.5-2ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libmambapy\",\n", + " \"version\": \"1.5.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"libmd0\",\n", + " \"version\": \"1.0.4-1build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libmount1\",\n", + " \"version\": \"2.37.2-4ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libmpc3\",\n", + " \"version\": \"1.2.1-2build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libmpfr6\",\n", + " \"version\": \"4.1.0-3build3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libncurses6\",\n", + " \"version\": \"6.3-2ubuntu0.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libncursesw6\",\n", + " \"version\": \"6.3-2ubuntu0.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libnettle8\",\n", + " \"version\": \"3.7.3-1build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libnghttp2-14\",\n", + " \"version\": \"1.43.0-1ubuntu0.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libnpth0\",\n", + " \"version\": \"1.6-3build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libnsl-dev\",\n", + " \"version\": \"1.3.0-2build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libnsl2\",\n", + " \"version\": \"1.3.0-2build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libnspr4\",\n", + " \"version\": \"2:4.32-3build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libnss3\",\n", + " \"version\": \"2:3.68.2-0ubuntu1.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libnuma1\",\n", + " \"version\": \"2.0.14-3ubuntu2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libonig5\",\n", + " \"version\": \"6.9.7.1-2build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libp11-kit0\",\n", + " \"version\": \"0.24.0-6build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libpam-modules\",\n", + " \"version\": \"1.4.0-11ubuntu2.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libpam-modules-bin\",\n", + " \"version\": \"1.4.0-11ubuntu2.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libpam-runtime\",\n", + " \"version\": \"1.4.0-11ubuntu2.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libpam0g\",\n", + " \"version\": \"1.4.0-11ubuntu2.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libpcre2-8-0\",\n", + " \"version\": \"10.39-3ubuntu0.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libpcre3\",\n", + " \"version\": \"2:8.39-13ubuntu0.22.04.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libpcsclite1\",\n", + " \"version\": \"1.9.5-3ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libperl5.34\",\n", + " \"version\": \"5.34.0-3ubuntu1.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libpng16-16\",\n", + " \"version\": \"1.6.37-3build5\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libprocps8\",\n", + " \"version\": \"2:3.3.17-6ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libpsl5\",\n", + " \"version\": \"0.21.0-1.2build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libquadmath0\",\n", + " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libreadline8\",\n", + " \"version\": \"8.1.2-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"librtmp1\",\n", + " \"version\": \"2.4+20151223.gitfa8646d.1-2build4\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libsasl2-2\",\n", + " \"version\": \"2.1.27+dfsg2-3ubuntu1.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libsasl2-modules-db\",\n", + " \"version\": \"2.1.27+dfsg2-3ubuntu1.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libseccomp2\",\n", + " \"version\": \"2.5.3-2ubuntu2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libselinux1\",\n", + " \"version\": \"3.3-1build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libsemanage-common\",\n", + " \"version\": \"3.3-1build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libsemanage2\",\n", + " \"version\": \"3.3-1build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libsepol2\",\n", + " \"version\": \"3.3-1build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libsmartcols1\",\n", + " \"version\": \"2.37.2-4ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libsqlite3-0\",\n", + " \"version\": \"3.37.2-2ubuntu0.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libss2\",\n", + " \"version\": \"1.46.5-2ubuntu1.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libssh-4\",\n", + " \"version\": \"0.9.6-2ubuntu0.22.04.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libssl3\",\n", + " \"version\": \"3.0.2-0ubuntu1.12\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libstdc++-11-dev\",\n", + " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libstdc++6\",\n", + " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libsystemd0\",\n", + " \"version\": \"249.11-0ubuntu3.11\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libtasn1-6\",\n", + " \"version\": \"4.18.0-4build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libtinfo6\",\n", + " \"version\": \"6.3-2ubuntu0.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libtirpc-common\",\n", + " \"version\": \"1.3.2-2ubuntu0.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libtirpc-dev\",\n", + " \"version\": \"1.3.2-2ubuntu0.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libtirpc3\",\n", + " \"version\": \"1.3.2-2ubuntu0.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libtsan0\",\n", + " \"version\": \"11.4.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libubsan1\",\n", + " \"version\": \"12.3.0-1ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libudev1\",\n", + " \"version\": \"249.11-0ubuntu3.11\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libunistring2\",\n", + " \"version\": \"1.0-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libuuid1\",\n", + " \"version\": \"2.37.2-4ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libxxhash0\",\n", + " \"version\": \"0.8.1-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"libzstd1\",\n", + " \"version\": \"1.4.8+dfsg-3build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"linux-libc-dev\",\n", + " \"version\": \"5.15.0-89.99\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"llvmlite\",\n", + " \"version\": \"0.41.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"locket\",\n", + " \"version\": \"1.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"login\",\n", + " \"version\": \"1:4.8.1-2ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"logsave\",\n", + " \"version\": \"1.46.5-2ubuntu1.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"lsb-base\",\n", + " \"version\": \"11.1.0ubuntu4\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"lto-disabled-list\",\n", + " \"version\": \"24\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"lz4\",\n", + " \"version\": \"4.3.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"make\",\n", + " \"version\": \"4.3-4.1build1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"mamba\",\n", + " \"version\": \"1.5.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"markdown-it-py\",\n", + " \"version\": \"3.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"matplotlib\",\n", + " \"version\": \"3.8.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"mawk\",\n", + " \"version\": \"1.3.4.20200120-3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"mdurl\",\n", + " \"version\": \"0.1.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"menuinst\",\n", + " \"version\": \"2.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"merlin-core\",\n", + " \"version\": \"23.6.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"merlin-dataloader\",\n", + " \"version\": \"23.6.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"mlflow\",\n", + " \"version\": \"2.9.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"more-itertools\",\n", + " \"version\": \"10.1.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"morpheus\",\n", + " \"version\": \"23.11.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"mount\",\n", + " \"version\": \"2.37.2-4ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"mpmath\",\n", + " \"version\": \"1.3.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"mrc\",\n", + " \"version\": \"23.11.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"msgpack\",\n", + " \"version\": \"1.0.7\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"multidict\",\n", + " \"version\": \"6.0.4\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"munkres\",\n", + " \"version\": \"1.1.4\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"nb-conda-kernels\",\n", + " \"version\": \"2.3.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"ncurses-base\",\n", + " \"version\": \"6.3-2ubuntu0.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"ncurses-bin\",\n", + " \"version\": \"6.3-2ubuntu0.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"networkx\",\n", + " \"version\": \"3.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"npy-append-array\",\n", + " \"version\": \"0.9.16\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"numba\",\n", + " \"version\": \"0.58.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"numpy\",\n", + " \"version\": \"1.26.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"numpydoc\",\n", + " \"version\": \"1.4.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"nvtabular\",\n", + " \"version\": \"23.6.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"nvtx\",\n", + " \"version\": \"0.2.8\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"oauthlib\",\n", + " \"version\": \"3.2.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"openjdk-11-jre-headless\",\n", + " \"version\": \"11.0.21+9-0ubuntu1~22.04\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"openssh-client\",\n", + " \"version\": \"1:8.9p1-3ubuntu0.4\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"openssl\",\n", + " \"version\": \"3.0.2-0ubuntu1.12\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"openssl\",\n", + " \"version\": \"3.2.0\",\n", + " \"path\": null,\n", + " \"system\": \"binary\"\n", + " },\n", + " {\n", + " \"name\": \"ordered-set\",\n", + " \"version\": \"4.1.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"packaging\",\n", + " \"version\": \"23.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"packaging\",\n", + " \"version\": \"23.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pandas\",\n", + " \"version\": \"1.3.5\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"partd\",\n", + " \"version\": \"1.4.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"passwd\",\n", + " \"version\": \"1:4.8.1-2ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"patch\",\n", + " \"version\": \"2.7.6-7build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"perl\",\n", + " \"version\": \"5.34.0-3ubuntu1.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"perl-base\",\n", + " \"version\": \"5.34.0-3ubuntu1.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"perl-modules-5.34\",\n", + " \"version\": \"5.34.0-3ubuntu1.3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"pinentry-curses\",\n", + " \"version\": \"1.1.1-1build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"pip\",\n", + " \"version\": \"23.3.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pkg-config\",\n", + " \"version\": \"0.29.2-1ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"pkginfo\",\n", + " \"version\": \"1.9.6\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pkgutil_resolve_name\",\n", + " \"version\": \"1.3.10\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"platformdirs\",\n", + " \"version\": \"4.1.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pluggy\",\n", + " \"version\": \"1.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pluggy\",\n", + " \"version\": \"1.3.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"procps\",\n", + " \"version\": \"2:3.3.17-6ubuntu2.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"prometheus-client\",\n", + " \"version\": \"0.19.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"prometheus-flask-exporter\",\n", + " \"version\": \"0.23.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"prompt-toolkit\",\n", + " \"version\": \"3.0.41\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"protobuf\",\n", + " \"version\": \"4.21.12\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"psutil\",\n", + " \"version\": \"5.9.5\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"ptxcompiler\",\n", + " \"version\": \"0.8.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pyOpenSSL\",\n", + " \"version\": \"23.2.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pyOpenSSL\",\n", + " \"version\": \"23.3.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pyarrow\",\n", + " \"version\": \"11.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pyarrow-hotfix\",\n", + " \"version\": \"0.6\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pycosat\",\n", + " \"version\": \"0.6.4\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pycparser\",\n", + " \"version\": \"2.21\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pynvml\",\n", + " \"version\": \"11.4.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pyparsing\",\n", + " \"version\": \"3.1.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"python\",\n", + " \"version\": \"3.10.12\",\n", + " \"path\": null,\n", + " \"system\": \"binary\"\n", + " },\n", + " {\n", + " \"name\": \"python\",\n", + " \"version\": \"3.10.13\",\n", + " \"path\": null,\n", + " \"system\": \"binary\"\n", + " },\n", + " {\n", + " \"name\": \"python-dateutil\",\n", + " \"version\": \"2.8.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"python-rapidjson\",\n", + " \"version\": \"1.13\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pytz\",\n", + " \"version\": \"2023.3.post1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"pyzmq\",\n", + " \"version\": \"25.1.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"querystring-parser\",\n", + " \"version\": \"1.2.4\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"readline-common\",\n", + " \"version\": \"8.1.2-1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"referencing\",\n", + " \"version\": \"0.31.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"requests\",\n", + " \"version\": \"2.31.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"requests-cache\",\n", + " \"version\": \"1.1.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"rich\",\n", + " \"version\": \"13.7.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"rmm\",\n", + " \"version\": \"23.6.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"rpcsvc-proto\",\n", + " \"version\": \"1.4.2-0ubuntu6\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"rpds-py\",\n", + " \"version\": \"0.13.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"ruamel.yaml\",\n", + " \"version\": \"0.17.32\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"ruamel.yaml.clib\",\n", + " \"version\": \"0.2.7\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"scikit-learn\",\n", + " \"version\": \"1.2.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"sed\",\n", + " \"version\": \"4.8-1ubuntu2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"sensible-utils\",\n", + " \"version\": \"0.0.17\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"setuptools\",\n", + " \"version\": \"59.8.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"setuptools\",\n", + " \"version\": \"68.1.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"six\",\n", + " \"version\": \"1.16.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"smmap\",\n", + " \"version\": \"5.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"sniffio\",\n", + " \"version\": \"1.3.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"snowballstemmer\",\n", + " \"version\": \"2.2.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"sortedcontainers\",\n", + " \"version\": \"2.4.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"soupsieve\",\n", + " \"version\": \"2.5\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"sphinxcontrib-applehelp\",\n", + " \"version\": \"1.0.7\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"sphinxcontrib-devhelp\",\n", + " \"version\": \"1.0.5\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"sphinxcontrib-htmlhelp\",\n", + " \"version\": \"2.0.4\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"sphinxcontrib-jsmath\",\n", + " \"version\": \"1.0.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"sphinxcontrib-qthelp\",\n", + " \"version\": \"1.0.6\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"sphinxcontrib-serializinghtml\",\n", + " \"version\": \"1.1.9\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"sqlparse\",\n", + " \"version\": \"0.4.4\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"stringcase\",\n", + " \"version\": \"1.2.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"sympy\",\n", + " \"version\": \"1.12\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"sysvinit-utils\",\n", + " \"version\": \"3.01-1ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"tabulate\",\n", + " \"version\": \"0.9.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"tar\",\n", + " \"version\": \"1.34+dfsg-1ubuntu0.1.22.04.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"tblib\",\n", + " \"version\": \"2.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"tensorflow-metadata\",\n", + " \"version\": \"1.13.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"threadpoolctl\",\n", + " \"version\": \"3.2.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"tomli\",\n", + " \"version\": \"2.0.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"toolz\",\n", + " \"version\": \"0.12.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"torch\",\n", + " \"version\": \"2.0.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"tornado\",\n", + " \"version\": \"6.3.3\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"tqdm\",\n", + " \"version\": \"4.66.1\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"traitlets\",\n", + " \"version\": \"5.14.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"triton\",\n", + " \"version\": \"2.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"tritonclient\",\n", + " \"version\": \"2.26.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"typing-utils\",\n", + " \"version\": \"0.1.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"typing_extensions\",\n", + " \"version\": \"4.8.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"ubuntu-keyring\",\n", + " \"version\": \"2021.03.26\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"ucf\",\n", + " \"version\": \"3.0043\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"ujson\",\n", + " \"version\": \"5.8.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"unicodedata2\",\n", + " \"version\": \"15.1.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"unzip\",\n", + " \"version\": \"6.0-26ubuntu3.1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"url-normalize\",\n", + " \"version\": \"1.4.3\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"urllib3\",\n", + " \"version\": \"2.0.4\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"urllib3\",\n", + " \"version\": \"2.1.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"usrmerge\",\n", + " \"version\": \"25ubuntu2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"util-linux\",\n", + " \"version\": \"2.37.2-4ubuntu3\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"watchdog\",\n", + " \"version\": \"2.1.9\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"watchgod\",\n", + " \"version\": \"0.8.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"wcwidth\",\n", + " \"version\": \"0.2.12\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"websocket-client\",\n", + " \"version\": \"1.7.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"websockets\",\n", + " \"version\": \"12.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"wget\",\n", + " \"version\": \"1.21.2-2ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"wheel\",\n", + " \"version\": \"0.41.2\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"wheel\",\n", + " \"version\": \"0.42.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"xz-utils\",\n", + " \"version\": \"5.2.5-2ubuntu1\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"zict\",\n", + " \"version\": \"3.0.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"zip\",\n", + " \"version\": \"3.0-12build2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"zipp\",\n", + " \"version\": \"3.17.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " },\n", + " {\n", + " \"name\": \"zlib1g\",\n", + " \"version\": \"1:1.2.11.dfsg-2ubuntu9.2\",\n", + " \"path\": null,\n", + " \"system\": \"deb\"\n", + " },\n", + " {\n", + " \"name\": \"zstandard\",\n", + " \"version\": \"0.19.0\",\n", + " \"path\": null,\n", + " \"system\": \"python\"\n", + " }\n", + " ]\n", + " },\n", + " \"vulnerable_dependencies\": [\n", + " {\n", + " \"vuln_id\": \"CVE-2024-21762\",\n", + " \"vuln_package_intel_sources\": [\n", + " \"nvd\"\n", + " ],\n", + " \"vulnerable_sbom_packages\": []\n", + " },\n", + " {\n", + " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", + " \"vuln_package_intel_sources\": [\n", + " \"ghsa\",\n", + " \"nvd\"\n", + " ],\n", + " \"vulnerable_sbom_packages\": [\n", + " {\n", + " \"name\": \"mlflow\",\n", + " \"version\": \"2.9.1\",\n", + " \"vulnerable_dependency_package\": {\n", + " \"system\": \"python\",\n", + " \"name\": \"mlflow\",\n", + " \"version\": \"2.9.1\",\n", + " \"relation\": \"SELF\"\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"mlflow\",\n", + " \"version\": \"2.9.1\",\n", + " \"vulnerable_dependency_package\": {\n", + " \"system\": \"python\",\n", + " \"name\": \"mlflow\",\n", + " \"version\": \"2.9.1\",\n", + " \"relation\": \"SELF\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\",\n", + " \"vuln_package_intel_sources\": [\n", + " \"ghsa\",\n", + " \"nvd\",\n", + " \"ubuntu\",\n", + " \"rhsa\"\n", + " ],\n", + " \"vulnerable_sbom_packages\": [\n", + " {\n", + " \"name\": \"pillow\",\n", + " \"version\": \"10.1.0\",\n", + " \"vulnerable_dependency_package\": {\n", + " \"system\": \"python\",\n", + " \"name\": \"pillow\",\n", + " \"version\": \"10.1.0\",\n", + " \"relation\": \"SELF\"\n", + " }\n", + " },\n", + " {\n", + " \"name\": \"pillow\",\n", + " \"version\": \"10.1.0\",\n", + " \"vulnerable_dependency_package\": {\n", + " \"system\": \"python\",\n", + " \"name\": \"pillow\",\n", + " \"version\": \"10.1.0\",\n", + " \"relation\": \"SELF\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " ]\n", + " },\n", + " \"output\": [\n", + " {\n", + " \"vuln_id\": \"CVE-2024-21762\",\n", + " \"checklist\": [\n", + " {\n", + " \"input\": \"Check SBOM and dependencies for vulnerability.\",\n", + " \"response\": \"The VulnerableDependencyChecker did not find any vulnerable packages or dependencies in the SBOM.\",\n", + " \"intermediate_steps\": null\n", + " }\n", + " ],\n", + " \"summary\": \"The VulnerableDependencyChecker did not find any vulnerable packages or dependencies in the SBOM.\",\n", + " \"justification\": {\n", + " \"label\": \"code_not_present\",\n", + " \"reason\": \"No vulnerable packages or dependencies were detected in the SBOM.\",\n", + " \"status\": \"FALSE\"\n", + " }\n", + " },\n", + " {\n", + " \"vuln_id\": \"GHSA-hh8p-p8mp-gqhm\",\n", + " \"checklist\": [\n", + " {\n", + " \"input\": \"Review mlflow Usage: Is the mlflow package actively used by applications within the container? Check for scripts or applications that rely on mlflow, particularly those that might construct pathnames using user-input data.\",\n", + " \"response\": \"The mlflow package is actively used by applications within the container, specifically for logging metrics. While I couldn't find any direct references to constructing pathnames using user-input data, I did find evidence of potential path traversal vulnerabilities related to mlflow, which could be a significant concern. It's recommended to review the container's codebase and dependencies to ensure that mlflow is used securely and that any potential vulnerabilities are addressed.\",\n", + " \"intermediate_steps\": null\n", + " },\n", + " {\n", + " \"input\": \"Inspect Pathname Construction Logic: Does the codebase contain any usage of mlflow that constructs pathnames using external input? Focus on how pathnames are handled and whether there is a reliance on restricted directories to filter out potentially harmful path traversals.\",\n", + " \"response\": \"The Docker container's codebase uses mlflow, which constructs pathnames using external input. However, mlflow has had path traversal vulnerabilities in the past due to improper sanitization of user-supplied paths and insufficient input sanitization. Although these vulnerabilities may have been addressed in newer versions of mlflow, the version used in the Docker container's codebase (>=2.2.1 and <3) may still be vulnerable to path traversal attacks. Therefore, it is essential to ensure that the pathname construction logic in the Docker container's codebase properly sanitizes external input and has sufficient restrictions in place to prevent harmful path traversals.\",\n", + " \"intermediate_steps\": null\n", + " },\n", + " {\n", + " \"input\": \"Evaluate Input Validation Practices: Assess the robustness of input validation practices within the application. Since the CVE involves path traversal, are inputs, especially those used to construct pathnames, properly sanitized and validated against unexpected or malicious data, such as '..\\\\..' sequences?\",\n", + " \"response\": \"The input validation practices within the application are not robust enough to prevent path traversal attacks. While the `find_matching_input_for_placeholder` function performs some input validation and sanitization, it does not follow best practices in input validation and sanitization. The `process_input_names` function is not fully implemented and lacks proper input validation and sanitization. To improve the robustness of the input validation practices, the application should perform more comprehensive input validation on all function parameters, use more robust input sanitization techniques, and consider using input validation and sanitization libraries or frameworks.\",\n", + " \"intermediate_steps\": null\n", + " }\n", + " ],\n", + " \"summary\": \"Based on the provided Checklist and Findings, the CVE is exploitable. The following checklist items have definitive answers indicating exploitability: \\n\\nThe mlflow package is actively used by applications within the container, and its pathname construction logic uses external input, which may be vulnerable to path traversal attacks due to improper sanitization of user-supplied paths. Furthermore, the input validation practices within the application are not robust enough to prevent path traversal attacks, as they do not follow best practices in input validation and sanitization.\",\n", + " \"justification\": {\n", + " \"label\": \"vulnerable\",\n", + " \"reason\": \"The analysis indicates that the mlflow package is actively used and its pathname construction logic is vulnerable to path traversal attacks due to improper sanitization of user-supplied paths, making the package exploitable.\",\n", + " \"status\": \"TRUE\"\n", + " }\n", + " },\n", + " {\n", + " \"vuln_id\": \"GHSA-3f63-hfp8-52jq\",\n", + " \"checklist\": [\n", + " {\n", + " \"input\": \"Review Pillow Usage: Is the Pillow library actively used by applications within the container? Check for scripts or applications that rely on Pillow, particularly those that might use the `PIL.ImageMath.eval` function.\",\n", + " \"response\": \"Based on the investigation, it appears that the Pillow library is not actively used by applications within the container. Although Pillow is commonly used in Python applications for image processing, the code within the container does not reference Pillow or its functions, and the primary purpose of the container is for machine learning and inference, not image processing. However, it's possible that Pillow might be used as a dependency in some Python applications, but without more specific information, it cannot be confirmed.\",\n", + " \"intermediate_steps\": null\n", + " },\n", + " {\n", + " \"input\": \"Inspect Environment Parameter Handling: Does the codebase contain any usage of the `PIL.ImageMath.eval` function with the environment parameter? Focus on how the environment parameter is handled and whether it is properly sanitized and validated against malicious input.\",\n", + " \"response\": \"Based on the investigation, it appears that the Docker container's codebase does not contain any usage of the `PIL.ImageMath.eval` function with the environment parameter. Additionally, the container handles environment parameters by accepting environment variables that can be passed to the script, but there is no evidence of these variables being used in conjunction with the `PIL` library.\",\n", + " \"intermediate_steps\": null\n", + " },\n", + " {\n", + " \"input\": \"Evaluate Code Injection Risks: Assess the robustness of input validation practices within the application. Since the CVE involves improper control of code generation, are inputs, especially those related to the environment parameter, properly neutralized or sanitized to prevent arbitrary code execution?\",\n", + " \"response\": \"Based on the investigation, it appears that the application does handle environment parameters, but the validation of user inputs is not explicitly shown. There are hints that suggest input validation is being performed, such as checking the validity of input parameters and using regular expressions to validate node names. However, without more context or code, it is difficult to provide a comprehensive answer on how the application validates user inputs. Additionally, searching for external libraries or dependencies used for input validation did not yield specific results, but it highlighted the importance of input validation in securing Docker containers. Therefore, it is recommended to further investigate the application's input validation practices, especially regarding the environment parameter, to assess the robustness of input validation and prevent arbitrary code execution.\",\n", + " \"intermediate_steps\": null\n", + " }\n", + " ],\n", + " \"summary\": \"Based on the investigation results, the CVE is **not exploitable**. The definitive answers from the checklist items are: \\n\\n* The Pillow library is not actively used by applications within the container (Checklist Item 1).\\n* The codebase does not contain any usage of the `PIL.ImageMath.eval` function with the environment parameter (Checklist Item 2).\\n\\nThese findings suggest that the CVE is not exploitable in this specific context, as the vulnerable library and function are not being used.\",\n", + " \"justification\": {\n", + " \"label\": \"code_not_reachable\",\n", + " \"reason\": \"The CVE is not exploitable because the vulnerable library (Pillow) is not actively used by applications within the container, and the codebase does not contain any usage of the vulnerable function (`PIL.ImageMath.eval` with the environment parameter).\",\n", + " \"status\": \"FALSE\"\n", + " }\n", + " }\n", + " ]\n", + "}\n" + ] + } + ], + "source": [ + "print(json.dumps(first_result, indent=2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Process and display relevant results" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "def process_results(json_file):\n", + " with open(json_file, 'r') as result_in:\n", + " full_input = result_in.read()\n", + " offsets = [res.start() for res in re.finditer('{\"input\":{\"scan\":', full_input)]\n", + " offsets.append(len(full_input))\n", + " outputs = [json.loads(full_input[offsets[i]:offsets[i+1]])['output'] for i in range(len(offsets)-1)]\n", + " flattened = (vuln_line for output in outputs for vuln_line in output)\n", + " df = pd.DataFrame(flattened)\n", + " df[['label', 'reason', 'affected']] = pd.json_normalize(df['justification'])\n", + " df.drop('justification', axis='columns', inplace=True)\n", + " return df" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
vuln_idchecklistsummarylabelreasonaffected
0CVE-2024-21762[{'input': 'Check SBOM and dependencies for vu...The VulnerableDependencyChecker did not find a...code_not_presentNo vulnerable packages or dependencies were de...FALSE
1GHSA-hh8p-p8mp-gqhm[{'input': 'Review mlflow Usage: Is the mlflow...Based on the provided Checklist and Findings, ...vulnerableThe analysis indicates that the mlflow package...TRUE
2GHSA-3f63-hfp8-52jq[{'input': 'Review Pillow Usage: Is the Pillow...Based on the investigation results, the CVE is...code_not_reachableThe CVE is not exploitable because the vulnera...FALSE
\n", + "
" + ], + "text/plain": [ + " vuln_id checklist \\\n", + "0 CVE-2024-21762 [{'input': 'Check SBOM and dependencies for vu... \n", + "1 GHSA-hh8p-p8mp-gqhm [{'input': 'Review mlflow Usage: Is the mlflow... \n", + "2 GHSA-3f63-hfp8-52jq [{'input': 'Review Pillow Usage: Is the Pillow... \n", + "\n", + " summary label \\\n", + "0 The VulnerableDependencyChecker did not find a... code_not_present \n", + "1 Based on the provided Checklist and Findings, ... vulnerable \n", + "2 Based on the investigation results, the CVE is... code_not_reachable \n", + "\n", + " reason affected \n", + "0 No vulnerable packages or dependencies were de... FALSE \n", + "1 The analysis indicates that the mlflow package... TRUE \n", + "2 The CVE is not exploitable because the vulnera... FALSE " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "process_results(output_file_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## View Markdown Reports\n", + "\n", + "In addition to the raw JSON output object, you can also view a Markdown-formatted report for each CVE in the `.tmp/vulnerability_markdown_reports` directory. For example, the report for `GHSA-3f63-hfp8-52jq` can be viewed at `.tmp/vulnerability_markdown_reports/vulnerability_reports_morpheus_23_11-runtime/GHSA-3f63-hfp8-52jq.md`." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.15" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/requirements.yaml b/requirements.yaml new file mode 100644 index 0000000..a3035e8 --- /dev/null +++ b/requirements.yaml @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +channels: + - conda-forge + - nvidia + - nvidia/label/dev # Required for the morpheus-llm package alpha packages. Will be overridden when released + - rapidsai + - huggingface +dependencies: + - aiohttp-client-cache=0.11 + - aiohttp=3.9 + - aioresponses=0.7.6 + - beautifulsoup4=4.12 + - esprima-python=4.0 # Required to parse JavaScript code + # - faiss-gpu=1.7 # Uncomment this when the package supports CUDA 12. See: https://github.com/conda-forge/faiss-split-feedstock/pull/72 + - faiss=1.7 + - gitpython=3.1 + - json5=0.9 + - morpheus-llm=24.10 + # cuda-python=12.6.1 cannot be used because of - + # https://github.com/NVIDIA/cuda-python/issues/215 + # NOTE: cuda-python is being downloaded some times from the nvidia channel and + # sometimes from conda-forge. cuda-python can be dropped from this list of + # dependencies after 12.6.2 is available on both channels. + - cuda-python!=12.6.1 + - openai=1.13 + - packaging>=23.2,<24.0 # versions required by langchain-core + - pydantic=2.6 + - python>=3.10.15,<3.11 + - tenacity>=8.1.0,<9.0.0 # versions required by langchain + - tiktoken>=0.3.2,<0.6.0 + - transformers + - uvloop + + ####### Pip Transitive Dependencies (keep sorted!) ####### + # These are dependencies that are available on conda, but are required by the pip packages listed below. Its much + # better to install them with conda than pip to allow for better dependency resolution. + - huggingface_hub>=0.23.2,<1.0 # required by langchain-community > sentence-transformers > transformers + + ####### Pip Dependencies (keep sorted!) ####### + - pip + - pip: + - google-search-results==2.4 + # Some of the langchain packages are available in conda but cause the solver to fail + - langchain==0.2.16 + - langchain-community==0.2.17 + - langchain-core==0.2.41 + - langchain-nvidia-ai-endpoints==0.2.0 + - nemollm + - pydpkg==1.9.2 + - rank_bm25==0.2.2 + # The conda version of sentence-transformers installs CPU-only versions of torch packages + - sentence-transformers==2.7 + # The tree-sitter packages are langchain transitive dependencies for parsing other code languages. + # The conda version is missing library files https://github.com/conda-forge/tree_sitter_languages-feedstock/issues/9 + - tree-sitter==0.21.3 + - tree-sitter-languages==1.10.2 + - univers==30.12 # Required for dependency version comparison diff --git a/scripts/run_all_messages.sh b/scripts/run_all_messages.sh new file mode 100755 index 0000000..e9ca2b0 --- /dev/null +++ b/scripts/run_all_messages.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Get the repository root directory +export REPO_ROOT=$(git rev-parse --show-toplevel) + +# Find all of the input message JSON files and store in an array +input_files=($(find ${REPO_ROOT}/data/input_messages -name ${1:-"*.json"})) + +# Run the LLM script for each input message +for input_file in ${input_files[@]}; do + + base_name=$(basename ${input_file}) + + echo "Running pipeline for ${base_name}..." + + dotenv -f .env run -- \ + python ${REPO_ROOT}/src/main.py --log_level=DEBUG cve \ + pipeline --config_file=${REPO_ROOT}/configs/from_file.json \ + config \ + input-file --file=${input_file} \ + general --max_retries=3 \ + output-file --file_path=${REPO_ROOT}/.tmp/output_${base_name} --overwrite +done diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..28d3a3a --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/cli.py b/src/cli.py new file mode 100644 index 0000000..a41cb29 --- /dev/null +++ b/src/cli.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import time + +import click + +from morpheus.cli.utils import get_log_levels +from morpheus.cli.utils import parse_log_level +from morpheus.utils.logger import configure_logging + +from .cve import run as run_cve + + +@click.group(name="morpheus_llm", chain=False, invoke_without_command=True, no_args_is_help=True) +@click.option("--log_level", + default=logging.getLevelName(logging.INFO), + type=click.Choice(get_log_levels(), case_sensitive=False), + callback=parse_log_level, + help="Specify the logging level to use.") +@click.option('--use_cpp', + default=True, + type=bool, + help=("Whether or not to use C++ node and message types or to prefer python. " + "Only use as a last resort if bugs are encountered")) +@click.version_option() +@click.pass_context +def cli(ctx: click.Context, log_level: int, use_cpp: bool): + """Main entrypoint for the Vulnerability Analysis for Container Security""" + + from morpheus.config import CppConfig + + ctx_dict = ctx.ensure_object(dict) + + CppConfig.set_should_use_cpp(use_cpp) + + # Configure the logging + configure_logging(log_level=log_level) + + morpheus_logger = logging.getLogger("morpheus") + + logger = logging.getLogger(__package__) + + # Set the parent logger for all of the llm examples to use morpheus so we can take advantage of configure_logging + logger.parent = morpheus_logger + + ctx_dict["start_time"] = time.time() + + +cli.add_command(run_cve, name="cve") + + +@cli.result_callback() +@click.pass_context +def after_pipeline(ctx: click.Context, pipeline_start_time: float, *_, **__): + logger = logging.getLogger(__name__) + + end_time = time.time() + + ctx_dict = ctx.ensure_object(dict) + + start_time = ctx_dict["start_time"] + + logger.info("Total time: %.2f sec", end_time - start_time) + + if (pipeline_start_time is not None): + logger.info("Pipeline runtime: %.2f sec", end_time - pipeline_start_time) diff --git a/src/cve/__init__.py b/src/cve/__init__.py new file mode 100644 index 0000000..750bbbd --- /dev/null +++ b/src/cve/__init__.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .run import run + +__all__ = [ + "run", +] diff --git a/src/cve/data_models/common.py b/src/cve/data_models/common.py new file mode 100644 index 0000000..11ebf1f --- /dev/null +++ b/src/cve/data_models/common.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys +import typing +from hashlib import sha512 + +from pydantic import BaseModel +from pydantic import Field + +_LT = typing.TypeVar("_LT") + + +class HashableModel(BaseModel): + """ + Subclass of a Pydantic BaseModel that is hashable. Use in objects that need to be hashed for caching purposes. + """ + + def __hash__(self): + return int.from_bytes(bytes=sha512(f"{self.__class__.__qualname__}::{self.model_dump_json()}".encode( + 'utf-8', errors='ignore')).digest(), + byteorder=sys.byteorder) + + def __lt__(self, other): + return self.__hash__() < other.__hash__() + + def __eq__(self, other): + return self.__hash__() == other.__hash__() + + def __ne__(self, other): + return self.__hash__() != other.__hash__() + + def __gt__(self, other): + return self.__hash__() > other.__hash__() + + +class TypedBaseModel(BaseModel, typing.Generic[_LT]): + """ + Subclass of Pydantic BaseModel that allows for specifying the object type. Use in Pydantic discriminated unions. + """ + + @classmethod + def __pydantic_init_subclass__(cls, **kwargs): + super().__pydantic_init_subclass__(**kwargs) + + if (cls.__pydantic_generic_metadata__["origin"] is TypedBaseModel): + # Because we are using a generic type, we need to set the default value of the type field when instantiating + # a base class. This is the only way to get the value from a typing.Literal + cls.model_fields["type"].default = typing.get_args(cls.__pydantic_generic_metadata__["args"][0])[0] + + type: _LT = Field(init=False, alias="_type", description="The type of the object", title="Type") + + @classmethod + def static_type(cls): + return cls.model_fields["type"].default + + @staticmethod + def discriminator(v: typing.Any) -> str | None: + # If its serialized, then we use the alias + if isinstance(v, dict): + return v.get("_type", v.get("type")) + + # Otherwise we use the property + return getattr(v, "type") diff --git a/src/cve/data_models/config.py b/src/cve/data_models/config.py new file mode 100644 index 0000000..dc852d1 --- /dev/null +++ b/src/cve/data_models/config.py @@ -0,0 +1,313 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import tempfile +import typing + +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import Discriminator +from pydantic import Field +from pydantic import FilePath +from pydantic import NonNegativeInt +from pydantic import PositiveInt +from pydantic import Tag +from pydantic import field_validator + +from morpheus.utils.http_utils import HTTPMethod + +from .common import TypedBaseModel +from .input import AgentMorpheusInput + + +def _llm_discriminator(v: typing.Any) -> str | None: + + if isinstance(v, dict): + return v.get("service").get("_type") + return getattr(getattr(v, "service"), "_type") + + +class GeneralConfig(BaseModel): + + model_config = ConfigDict(protected_namespaces=()) + + base_vdb_dir: str = os.path.join(tempfile.gettempdir(), "am_cache", "vdb") + base_git_dir: str = os.path.join(tempfile.gettempdir(), "am_cache", "git") + enable_llm_list_parsing: bool = False + cache_dir: str | None = None + ignore_build_vdb_errors: bool = False + max_retries: NonNegativeInt = 10 + model_max_batch_size: PositiveInt = 64 + num_threads: PositiveInt = Field(default_factory=os.cpu_count) + pipeline_batch_size: PositiveInt = 1024 + use_uvloop: bool = True + """ + Whether to use uvloop for the event loop. This can provide a significant speedup in some cases. Disable to provide + better error messages when debugging. + """ + + code_search_tool: bool = False + + @field_validator("num_threads") + @classmethod + def get_num_threads(cls, v: int) -> int: + if v is None or v < 1: + return os.cpu_count() or 1 + else: + return v + + +########################### LLM Model Configs ########################### +class NeMoLLMServiceConfig(TypedBaseModel[typing.Literal["nemo"]]): + + api_key: str | None = None + org_id: str | None = None + retry_count: NonNegativeInt = 5 + + +class NeMoLLMModelConfig(BaseModel): + + service: NeMoLLMServiceConfig + + model_name: str + customization_id: str | None = None + temperature: typing.Annotated[float, Field(ge=0.0, le=1.0)] = 0.0 + top_k: NonNegativeInt = 0 + top_p: float = 1 + random_seed: int | None = None + tokens_to_generate: int = 300 + beam_search_diversity_rate: float = 0.0 + beam_width: int = 1 + repetition_penalty: float = 1.0 + length_penalty: float = 1.0 + logprobs: bool = True + + model_config = ConfigDict(protected_namespaces=()) + + +class NVFoundationLLMServiceConfig(TypedBaseModel[typing.Literal["nvfoundation"]]): + + api_key: str | None = None + base_url: str | None = None + + +class NVFoundationLLMModelConfig(TypedBaseModel[typing.Literal["nvfoundation"]]): + + service: NVFoundationLLMServiceConfig + + model_name: str + temperature: float = 0.0 + top_p: float | None = None + max_tokens: PositiveInt = 300 + seed: int | None = None + + model_config = ConfigDict(protected_namespaces=()) + + +class OpenAIServiceConfig(TypedBaseModel[typing.Literal["openai"]]): + + api_key: str | None = None + base_url: str | None = None + + +class OpenAIModelConfig(BaseModel): + + service: OpenAIServiceConfig + + model_name: str + temperature: float = 0.0 + top_p: float = 1.0 + seed: int | None = None + max_retries: int = 10 + json_output: bool = Field(False, alias='json') + + model_config = ConfigDict(protected_namespaces=(), populate_by_name=True) + + +LLMModelConfig = typing.Annotated[typing.Annotated[NeMoLLMModelConfig, Tag("nemo")] + | typing.Annotated[OpenAIModelConfig, Tag("openai")] + | typing.Annotated[NVFoundationLLMModelConfig, Tag("nvfoundation")], + Discriminator(_llm_discriminator)] + + +########################### Input Configs ########################### +class ManualInputConfig(TypedBaseModel[typing.Literal["manual"]]): + + message: AgentMorpheusInput + + repeat_count: int = 1 + + +class FileInputConfig(TypedBaseModel[typing.Literal["file"]]): + + file: str + + repeat_count: int = 1 + + +class HttpInputConfig(TypedBaseModel[typing.Literal["http"]]): + + address: str = "127.0.0.1" + + endpoint: str = "/scan" + + port: int = 8080 + + http_method: HTTPMethod = HTTPMethod.POST + + stop_after: int = 0 + + +class PluginInputConfig(TypedBaseModel[typing.Literal["plugin"]]): + + plugin_name: str + + plugin_config: dict[str, typing.Any] = {} + + +InputConfig = typing.Annotated[typing.Annotated[ManualInputConfig, Tag(ManualInputConfig.static_type())] + | typing.Annotated[FileInputConfig, Tag(FileInputConfig.static_type())] + | typing.Annotated[HttpInputConfig, Tag(HttpInputConfig.static_type())] + | typing.Annotated[PluginInputConfig, Tag(PluginInputConfig.static_type())], + Discriminator(TypedBaseModel.discriminator)] + + +class HuggingFaceEmbeddingModelConfig(TypedBaseModel[typing.Literal["huggingface"]]): + + model_config = ConfigDict(protected_namespaces=()) + + model_name: str + model_kwargs: dict[str, typing.Any] = {} + encode_kwargs: dict[str, typing.Any] = {} + + +class OpenAIEmbeddingModelConfig(TypedBaseModel[typing.Literal["openai"]]): + + openai_api_key: str | None = None + max_retries: int = 10 + chunk_size: int = 128 + + +class NIMEmbeddingModelConfig(TypedBaseModel[typing.Literal["nim"]]): + + api_key: str | None = None + model: str + truncate: typing.Literal["NONE", "START", "END"] = "END" + max_batch_size: int = 128 + + +EmbeddingModelConfig = typing.Annotated[ + typing.Annotated[HuggingFaceEmbeddingModelConfig, Tag(HuggingFaceEmbeddingModelConfig.static_type())] + | typing.Annotated[OpenAIEmbeddingModelConfig, Tag(OpenAIEmbeddingModelConfig.static_type())] + | typing.Annotated[NIMEmbeddingModelConfig, Tag(NIMEmbeddingModelConfig.static_type())], + Discriminator(TypedBaseModel.discriminator)] + + +class EngineAgentConfig(BaseModel): + + model: LLMModelConfig + version_compare_tool: bool = False + prompt_examples: bool = False + verbose: bool = False + return_intermediate_steps: bool = False + return_source_documents: bool = False + + +class EngineConfig(BaseModel): + + agent: EngineAgentConfig + checklist_model: LLMModelConfig + justification_model: LLMModelConfig + rag_embedding: EmbeddingModelConfig + sbom_faiss_dir: str | None = None + summary_model: LLMModelConfig + + +class OutputPrintConfig(TypedBaseModel[typing.Literal["print"]]): + + pass + + +class OutputFileConfig(TypedBaseModel[typing.Literal["file"]]): + + file_path: str | None = "./.tmp/agent_morpheus_output.json" + + markdown_dir: str | None = "./.tmp/vulnerability_markdown_reports" + + overwrite: bool = False + + +class OutputHttpConfig(TypedBaseModel[typing.Literal["http"]]): + + url: str + + endpoint: str + + +class OutputElasticsearchConfig(TypedBaseModel[typing.Literal["elasticsearch"]]): + + url: str + + index: str + + conf_file: FilePath + + raise_on_exception: bool = False + + +class OutputPluginConfig(TypedBaseModel[typing.Literal["plugin"]]): + + plugin_name: str + + plugin_config: dict[str, typing.Any] = {} + + +OutputConfig = typing.Annotated[typing.Annotated[OutputPrintConfig, Tag(OutputPrintConfig.static_type())] + | typing.Annotated[OutputFileConfig, Tag(OutputFileConfig.static_type())] + | typing.Annotated[OutputPluginConfig, Tag(OutputPluginConfig.static_type())], + Discriminator(TypedBaseModel.discriminator)] + + +class RunConfig(BaseModel): + + model_config = ConfigDict(protected_namespaces=()) + + # Global Options + general: GeneralConfig = GeneralConfig() + + # Input Configuration + input: InputConfig + + # Engine Configuration + engine: EngineConfig + + # Output Configuration + output: OutputConfig + + @staticmethod + def generate_json_schema() -> dict[str, typing.Any]: + return RunConfig.model_json_schema() + + @staticmethod + def write_json_schema(schema_path: str) -> None: + + import json + + schema = RunConfig.generate_json_schema() + + with open(schema_path, "w", encoding="utf-8") as f: + json.dump(schema, f, indent=2) diff --git a/src/cve/data_models/cve_intel.py b/src/cve/data_models/cve_intel.py new file mode 100644 index 0000000..cb62094 --- /dev/null +++ b/src/cve/data_models/cve_intel.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import typing + +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import Field + + +class CveIntelGhsa(BaseModel): + """ + Information about a GHSA (GitHub Security Advisory) entry. + """ + model_config = ConfigDict(extra="allow") + + class CVSS(BaseModel): + score: float | None = None + vector_string: str | None = None + + class CWE(BaseModel): + cwe_id: str + name: str | None = None + + ghsa_id: str + cve_id: str | None = None + summary: str | None = None + description: str | None = None + severity: str | None = None + vulnerabilities: list | None = None + cvss: CVSS | None = None + cwes: list[CWE] | None = None + + +class CveIntelNvd(BaseModel): + """ + Information about an NVD (National Vulnerability Database) entry. + """ + + model_config = ConfigDict(extra="allow") + + class Configuration(BaseModel): + package: str + system: str | None = None + versionStartExcluding: str | None = None + versionEndExcluding: str | None = None + versionStartIncluding: str | None = None + versionEndIncluding: str | None = None + + cve_id: str + cve_description: str | None = None + cvss_vector: str | None = None + cwe_name: str | None = None + cwe_description: str | None = None + cwe_extended_description: str | None = None + configurations: list[Configuration] | None = None + vendor_names: list[str] | None = None + + +class CveIntelRhsa(BaseModel): + """ + Information about a RHSA (Red Hat Security Advisory) entry. + """ + model_config = ConfigDict(extra="allow") + + class Bugzilla(BaseModel): + description: str | None = None + id: str | None = None + url: str | None = None + + class PackageState(BaseModel): + product_name: str | None = None + fix_state: str | None = None + package_name: str | None = None + cpe: str | None = None + + bugzilla: typing.Annotated[Bugzilla, Field(default_factory=Bugzilla)] + details: list[str] | None = None + statement: str | None = None + package_state: list[PackageState] | None = None + upstream_fix: str | None = None + + +class CveIntelUbuntu(BaseModel): + """ + Information about a Ubuntu CVE entry. + """ + model_config = ConfigDict(extra="allow") + + class Note(BaseModel): + author: str | None = None + note: str | None = None + + description: str | None = None + notes: list[Note] | None = None + priority: str | None = None + ubuntu_description: str | None = None + + +class CveIntelEpss(BaseModel): + """ + Information about an EPSS (Elastic Product Security Service) entry. + """ + model_config = ConfigDict(extra="allow") + + epss: float | None = None + percentile: float | None = None + date: str | None = None + + +class CveIntel(BaseModel): + """ + Information about a CVE (Common Vulnerabilities and Exposures) entry. + """ + + vuln_id: str + """ + The input indentifier. Can be either GHSA or CVE + """ + + ghsa: CveIntelGhsa | None = None + nvd: CveIntelNvd | None = None + rhsa: CveIntelRhsa | None = None + ubuntu: CveIntelUbuntu | None = None + epss: CveIntelEpss | None = None + + @property + def cve_id(self): + """ + The CVE identifier. + + Returns + ------- + str + The CVE identifier. + + Raises + ------ + ValueError + If the CVE ID is not found. An exception will be raised + """ + cve_id = self.get_cve_id() + + if (cve_id is not None): + return cve_id + + raise ValueError("CVE ID not found") + + @property + def ghsa_id(self): + """ + The GHSA identifier. + """ + ghsa_id = self.get_ghsa_id() + + if (ghsa_id is not None): + return ghsa_id + + raise ValueError("GHSA ID not found") + + def has_cve_id(self): + """ + Check if the object has a CVE ID. + """ + return self.get_cve_id() is not None + + def has_ghsa_id(self): + """ + Check if the object has a GHSA ID. + """ + return self.get_ghsa_id() is not None + + def get_cve_id(self): + """ + Get the CVE ID. + + Returns + ------- + str | None + The CVE ID or None if not found. + """ + if (self.nvd is not None): + return self.nvd.cve_id + + if (self.ghsa is not None and self.ghsa.cve_id is not None): + return self.ghsa.cve_id + + if (self.vuln_id.startswith("CVE-")): + return self.vuln_id + + return None + + def get_ghsa_id(self): + """ + Get the GHSA ID. + + Returns + ------- + str | None + The GHSA ID or None if not found. + """ + if (self.ghsa is not None): + return self.ghsa.ghsa_id + + if (self.vuln_id.startswith("GHSA-")): + return self.vuln_id + + return None diff --git a/src/cve/data_models/dependencies.py b/src/cve/data_models/dependencies.py new file mode 100644 index 0000000..1ac1aae --- /dev/null +++ b/src/cve/data_models/dependencies.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import typing + +from pydantic import BaseModel + + +class DependencyPackage(BaseModel): + """ + Information about a dependency package as obtained from deps.dev API for a given SBOM package. + """ + system: str | None = None + name: str | None = None + version: str | None = None + relation: typing.Literal["SELF", "DIRECT", "INDIRECT"] | None = None + + +class VulnerableSBOMPackage(BaseModel): + """ + Information about a vulnerable SBOM package and its related vulnerable dependency package. + + - name: SBOM package name + - version: SBOM package version + - vulnerable_dependency_package: DependencyPackage object with info about the vulnerable dependency package. + If an SBOM package itself is vulnerable, the vulnerable_dependency_package.relation will be "SELF". + Otherwise, if it is vulnerable due to its dependency, the vulnerable_dependency_package.relation will be either + "DIRECT" or "INDIRECT". + """ + name: str + version: str + vulnerable_dependency_package: DependencyPackage + + +class VulnerableDependencies(BaseModel): + """ + Information about the vulnerable SBOM packages associated with the vuln_id. + + - vuln_id: vulnerability ID (e.g. CVE ID, GHSA ID) associated with the vulnerable package list. + - vuln_package_intel_sources: list of sources (e.g. "ghsa", "nvd", "ubuntu", "rhsa") that provided + the vulnerable package/version intel for the vuln_id. + - vulnerable_sbom_packages: list of VulnerableSBOMPackage objects, representing the SBOM packages that are + vulnerable for a given vuln_id. + """ + vuln_id: str + vuln_package_intel_sources: list[str] + vulnerable_sbom_packages: list[VulnerableSBOMPackage] diff --git a/src/cve/data_models/info.py b/src/cve/data_models/info.py new file mode 100644 index 0000000..e47d5b1 --- /dev/null +++ b/src/cve/data_models/info.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pydantic import BaseModel + +from .cve_intel import CveIntel +from .dependencies import VulnerableDependencies + + +class SBOMPackage(BaseModel): + """ + Information about a single package in the container image's Software Bill of Materials (SBOM). + """ + name: str + version: str + path: str | None = None + system: str + + +class AgentMorpheusInfo(BaseModel): + """ + Information used for decisioning in the Agent Morpheus engine. These information can all be automatically + generated or retrieved by the pipeline from the input information. + + - vdb: paths to source code and documentation vector databases (VDBs) used to understand whether a vulnerability + is exploitable in the source code. + - intel: list of CveIntel objects representing intelligence for each vulnerability pulled from various vulnerability + databases and APIs. + - sbom: software bill of materials listing the packages and versions in the container image, used to understand + whether the vulnerable package exists in the image. + - vulnerable_dependencies: a list of VulnerableDependencies objects for each vuln_id, representing the SBOM packages + and transitive dependencies that are vulnerable for the vuln_id. + """ + + class VdbPaths(BaseModel): + """ + Paths to where the generated VDBs are stored. + """ + code_vdb_path: str | None = None + doc_vdb_path: str | None = None + + class SBOMInfo(BaseModel): + """ + List of SBOMPackage objects representing the packages found in the input image. + """ + packages: list[SBOMPackage] + + vdb: VdbPaths | None = None + intel: list[CveIntel] | None = None + sbom: SBOMInfo | None = None + vulnerable_dependencies: list[VulnerableDependencies] | None = None diff --git a/src/cve/data_models/input.py b/src/cve/data_models/input.py new file mode 100644 index 0000000..3b5398e --- /dev/null +++ b/src/cve/data_models/input.py @@ -0,0 +1,162 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import typing + +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import Discriminator +from pydantic import Field +from pydantic import Tag +from pydantic import field_validator + +from ..utils.string_utils import is_valid_cve_id +from ..utils.string_utils import is_valid_ghsa_id +from .common import HashableModel +from .common import TypedBaseModel +from .info import AgentMorpheusInfo +from .info import SBOMPackage + + +class SourceDocumentsInfo(HashableModel): + """ + Information about the source documents for the container image. + + - type: document type. + - git_repo: git repo URL where the source documents can be cloned. + - tag: git tag. + - include: file extensions to include when indexing the source documents. + - exclude: file extensions to exclude when indexing the source documents. + """ + + type: typing.Literal["code", "doc"] + + git_repo: typing.Annotated[str, Field(min_length=1)] + tag: typing.Annotated[str, Field(min_length=1)] + + include: list[str] = ["*.py", "*.ipynb"] + exclude: list[str] = [] + + @field_validator("include", "exclude") + @classmethod + def sort_lists(cls, v: list[str]) -> list[str]: + return list(sorted(v)) + + +class VulnInfo(HashableModel): + """ + Information about a vulnerability. + """ + + model_config = ConfigDict(extra="allow") + + vuln_id: str + description: str | None = None + score: float | None = None + severity: str | None = None + published_date: str | None = None + last_modified_date: str | None = None + url: str | None = None + feed_group: str | None = None + + package: str | None = None + """ + The full package name, including the version, that is affected by the vulnerability. + """ + package_version: str | None = None + """ + The version of the package that is affected by the vulnerability. + """ + package_name: str | None = None + """ + The name of the package that is affected by the vulnerability without the version. + """ + package_type: str | None = None + """ + The type of the package that is affected by the vulnerability. + """ + + @field_validator("vuln_id") + @classmethod + def is_valid_vuln_id(cls, v: str): + if not (is_valid_cve_id(v) or is_valid_ghsa_id(v)): + raise ValueError(f"{v} is not a valid CVE ID or GHSA ID.") + else: + return v + + +class ScanInfoInput(HashableModel): + """ + Information about a unique scan for a container image against a list of vulnerabilies. + """ + id: str | None = None + type: str | None = None + started_at: str | None = None + completed_at: str | None = None + + vulns: typing.Annotated[list[VulnInfo], Field(min_length=1)] + + +class ManualSBOMInfoInput(TypedBaseModel[typing.Literal["manual"]]): + """ + Manually provided Software Bill of Materials, consisting of a list of SBOMPackage objects. + """ + packages: list[SBOMPackage] + + +class FileSBOMInfoInput(TypedBaseModel[typing.Literal["file"]]): + """ + A file path pointing to a Software Bill of Materials file. + """ + file_path: str + + +SBOMInfoInput = typing.Annotated[typing.Annotated[ManualSBOMInfoInput, Tag(ManualSBOMInfoInput.static_type())] + | typing.Annotated[FileSBOMInfoInput, Tag(FileSBOMInfoInput.static_type())], + Discriminator(TypedBaseModel.discriminator)] + + +class ImageInfoInput(HashableModel): + """ + Information about a container image, including the source information and sbom information. + """ + name: str | None = None # The image name + tag: str | None = None # i.e. latest + digest: str | None = None # i.e. sha256:... + platform: str | None = None # i.e. linux/amd64 + feed_group: str | None = None # i.e. ubuntu:22.04 + + source_info: list[SourceDocumentsInfo] + sbom_info: SBOMInfoInput + + +class AgentMorpheusInput(HashableModel): + """ + Inputs required by the Agent Morpheus pipeline. + """ + scan: ScanInfoInput + image: ImageInfoInput + + +class AgentMorpheusEngineInput(BaseModel): + """ + Inputs required by the Agent Morpheus Engine. + + - input: AgentMorpheusInput object that must be provided to the pipeline. + - info: AgentMorpheusInfo object that is retrieved/generated by the pipeline. + """ + input: AgentMorpheusInput + info: AgentMorpheusInfo diff --git a/src/cve/data_models/output.py b/src/cve/data_models/output.py new file mode 100644 index 0000000..18d00d3 --- /dev/null +++ b/src/cve/data_models/output.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import typing + +from pydantic import BaseModel +from pydantic import model_validator + +from .input import AgentMorpheusEngineInput + + +class AgentIntermediateStep(BaseModel): + """ + Represents info for an intermediate step taken by an agent. + """ + tool_name: str + action_log: str + tool_input: str | dict + tool_output: typing.Any + + +class ChecklistItemOutput(BaseModel): + """ + Input, response, and intermediate steps for a single checklist item provided to the LLM agent. + """ + input: str + response: str + intermediate_steps: list[AgentIntermediateStep] | None = None + + +class JustificationOutput(BaseModel): + """ + Final justification for the vulnerability. + + - label: a categorical justification label classifying the status of an image against a given vulnerability, e.g. + code_not_present, code_not_reachable, false_positive. + - reason: a human-readable explanation for why justification label was selected. + - status: a ternary status (TRUE, FALSE, OR UNKNOWN) that indicates whether the image can be exploited for a given + vulnerability. Determined based on a mapping from the justification label. + """ + label: str # TODO: convert to typing.Literal after completing issue #115 + reason: str + status: typing.Literal["TRUE", "FALSE", "UNKNOWN"] + + +class AgentMorpheusEngineOutput(BaseModel): + """ + Contains all output generated by the main Agent Morpheus LLM Engine for a given vulnerability. + + - vuln_id: the ID of the vulnerability being processed by the LLM engine. + - checklist: a list of ChecklistItemOutput objects, each containing an input and a response from the LLM agent. + - summary: a short summary of the checklist inputs and responses, generated by an LLM. + - justification: a JustificationOutput object containing details of the model's justification decision. + """ + vuln_id: str + checklist: list[ChecklistItemOutput] + summary: str + justification: JustificationOutput + + +class AgentMorpheusOutput(AgentMorpheusEngineInput): + """" + The final output of the Agent Morpheus pipeline. + Contains all fields in the AgentMorpheusEngineInput, plus the AgentMorpheusEngineOuput for each input vulnerability. + """ + output: list[AgentMorpheusEngineOutput] + + @model_validator(mode="before") + @classmethod + def validate_not_none(cls, data: typing.Any) -> typing.Any: + """Validate that no fields are None in the output data""" + if isinstance(data, dict): + for k, v in data.items(): + if v is None: + raise ValueError(f"{k} must not be None") + return data diff --git a/src/cve/data_models/plugin.py b/src/cve/data_models/plugin.py new file mode 100644 index 0000000..79f3aa3 --- /dev/null +++ b/src/cve/data_models/plugin.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import typing +from abc import ABC +from abc import abstractmethod +from pydoc import locate + +from morpheus.config import Config +from morpheus.pipeline.linear_pipeline import LinearPipeline + +from ..data_models.config import RunConfig + +_T = typing.TypeVar('_T', bound='PluginSchema') + + +class PluginSchema(ABC): + + @classmethod + def locate(cls: type[_T], plugin_name: str) -> _T: + '''Locate input plugin''' + pluginClass: type | None = locate(plugin_name) + + if not pluginClass: + raise ValueError(f"Plugin not found: {plugin_name}") + if not issubclass(pluginClass, cls): + raise ValueError("Plugin object must be a subclass of {cls}") + + return pluginClass() + + +class InputPluginSchema(PluginSchema): + + @abstractmethod + def build_input(self, pipe: LinearPipeline, config: Config, run_config: RunConfig): + # add the plugin specific input building logic here + pass + + +class OutputPluginSchema(PluginSchema): + + @abstractmethod + def build_output(self, pipe: LinearPipeline, config: Config, run_config: RunConfig): + # add the plugin specific output building logic here + pass diff --git a/src/cve/data_models/vulnerability_level.py b/src/cve/data_models/vulnerability_level.py new file mode 100644 index 0000000..8e85400 --- /dev/null +++ b/src/cve/data_models/vulnerability_level.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import typing +from enum import Enum + + +class VulnerabilityLevel(Enum): + """Vulnerability levels.""" + CRITICAL = "critical" + HIGH = "high" + MEDIUM = "medium" + LOW = "low" + NEGLIGIBLE = "negligible" + + @staticmethod + def levels_to_str(vuln_levels: typing.Iterable["VulnerabilityLevel | str"]) -> typing.Generator[str, None, None]: + + for level in vuln_levels: + if isinstance(level, VulnerabilityLevel): + level_value = level.value + elif isinstance(level, str): + level_value = level.lower() + if level_value in VulnerabilityLevel.__members__: + yield level_value + else: + raise ValueError(f"Invalid vulnerability level: {level}") + else: + raise ValueError(f"Invalid vulnerability level: {level}") + + yield level_value diff --git a/src/cve/nodes/__init__.py b/src/cve/nodes/__init__.py new file mode 100644 index 0000000..28d3a3a --- /dev/null +++ b/src/cve/nodes/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/cve/nodes/cve_checklist_node.py b/src/cve/nodes/cve_checklist_node.py new file mode 100644 index 0000000..defa696 --- /dev/null +++ b/src/cve/nodes/cve_checklist_node.py @@ -0,0 +1,154 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import ast +import logging + +from morpheus_llm.llm import LLMLambdaNode +from morpheus_llm.llm import LLMNode +from morpheus_llm.llm.nodes.llm_generate_node import LLMGenerateNode +from morpheus_llm.llm.nodes.prompt_template_node import PromptTemplateNode +from morpheus_llm.llm.services.llm_service import LLMService + +from ..data_models.config import LLMModelConfig +from ..utils.prompting import MOD_FEW_SHOT +from ..utils.prompting import additional_intel_prompting +from ..utils.prompting import get_mod_examples +from ..utils.string_utils import attempt_fix_list_string + +logger = logging.getLogger(__name__) + +cve_prompt1 = ( + MOD_FEW_SHOT.format(examples=get_mod_examples()) + + additional_intel_prompting + + "\nThe vulnerable version of the vulnerable package is already verified to be installed within the container. Check only the other factors that affect exploitability, no need to verify version again." +) + +cve_prompt2 = """Parse the following numbered checklist into a python list in the format ["x", "y", "z"], a comma separated list surrounded by square braces: {{template}}""" + + +async def _parse_list(text: list[str]) -> list[list[str]]: + """ + Asynchronously parse a list of strings, each representing a list, into a list of lists. + + Parameters + ---------- + text : list of str + A list of strings, each intended to be parsed into a list. + + Returns + ------- + list of lists of str + A list of lists, parsed from the input strings. + + Raises + ------ + ValueError + If the string cannot be parsed into a list or if the parsed object is not a list. + + Notes + ----- + This function tries to fix strings that represent lists with unescaped quotes by calling + `attempt_fix_list_string` and then uses `ast.literal_eval` for safe parsing of the string into a list. + It ensures that each element of the parsed list is actually a list and will raise an error if not. + """ + return_val = [] + + for checklist_num, x in enumerate(text): + try: + # Try to cut out verbosity: + x = x[x.rfind('['):x.find(']') + 1] + + # Remove newline characters that can cause incorrect string escaping in the next step + x = x.replace("\n", "") + + # Try to do some very basic string cleanup to fix unescaped quotes + x = attempt_fix_list_string(x) + + # Only proceed if the input is a valid Python literal + # This isn't really dangerous, literal_eval only evaluates a small subset of python + current = ast.literal_eval(x) + + # Ensure that the parsed data is a list + if not isinstance(current, list): + raise ValueError(f"Input is not a list: {x}") + + # Process the list items + for i in range(len(current)): + if (isinstance(current[i], list) and len(current[i]) == 1): + current[i] = current[i][0] + + return_val.append(current) + except (ValueError, SyntaxError) as e: + # Handle the error, log it, or re-raise it with additional context + raise ValueError(f"Failed to parse input for checklist number {checklist_num}: {x}. Error: {e}") + + return return_val + + +class CVEChecklistNode(LLMNode): + """ + A node that orchestrates the process of generating a checklist for CVE (Common Vulnerabilities and Exposures) items. + It integrates various nodes that handle CVE lookup, prompting, generation, and parsing to produce an actionable checklist. + """ + + def __init__(self, *, checklist_model_config: LLMModelConfig, enable_llm_list_parsing: bool = False): + """ + Initialize the CVEChecklistNode with optional caching and a vulnerability endpoint retriever. + + Parameters + ---------- + model_name : str, optional + The name of the language model to be used for generating text, by default "gpt-3.5-turbo". + cache_dir : str, optional + The directory where the node's cache should be stored. If None, caching is not used. + vuln_endpoint_retriever : object, optional + An instance of a vulnerability endpoint retriever. If None, defaults to `NISTCVERetriever`. + """ + super().__init__() + + chat_service = LLMService.create(checklist_model_config.service.type, + **checklist_model_config.service.model_dump(exclude={"type"}, by_alias=True)) + + # Add a node to create a prompt for CVE checklist generation based on the CVE details obtained from the lookup + # node + self.add_node("checklist_prompt", + inputs=[("*", "*")], + node=PromptTemplateNode(template=cve_prompt1, template_format="jinja")) + + # Instantiate a chat service and configure a client for generating responses to the checklist prompt + llm_client_1 = chat_service.get_client( + **checklist_model_config.model_dump(exclude={"service", "type"}, by_alias=True) + ) + gen_node_1 = LLMGenerateNode(llm_client=llm_client_1) + self.add_node("chat1", inputs=["/checklist_prompt"], node=gen_node_1) + + if enable_llm_list_parsing: + # Add a node to parse the generated response into a format suitable for a secondary checklist prompt + self.add_node("parse_checklist_prompt", + inputs=["/chat1"], + node=PromptTemplateNode(template=cve_prompt2, template_format="jinja")) + + # Configure a second client for generating a follow-up response based on the parsed checklist prompt + llm_client_2 = chat_service.get_client( + **checklist_model_config.model_dump(exclude={"service", "type"}, by_alias=True) + ) + gen_node_2 = LLMGenerateNode(llm_client=llm_client_2) + self.add_node("chat2", inputs=[("/parse_checklist_prompt", "prompt")], node=gen_node_2) + + checklist_prompts = ["/chat2"] if enable_llm_list_parsing else ["/chat1"] + # Add an output parser node to process the final generated checklist into a structured list + self.add_node("output_parser", inputs=checklist_prompts, node=LLMLambdaNode(_parse_list), is_output=True) diff --git a/src/cve/nodes/cve_justification_node.py b/src/cve/nodes/cve_justification_node.py new file mode 100644 index 0000000..30bf8c8 --- /dev/null +++ b/src/cve/nodes/cve_justification_node.py @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +from textwrap import dedent + +from morpheus_llm.llm import LLMLambdaNode +from morpheus_llm.llm import LLMNode +from morpheus_llm.llm.nodes.llm_generate_node import LLMGenerateNode +from morpheus_llm.llm.nodes.prompt_template_node import PromptTemplateNode +from morpheus_llm.llm.services.llm_service import LLMClient + +logger = logging.getLogger(__name__) + + +class CVEJustifyNode(LLMNode): + """ + A node to classify the results of the summary node. + """ + + JUSTIFICATION_LABEL_COL_NAME = "justification_label" + JUSTIFICATION_REASON_COL_NAME = "justification" + AFFECTED_STATUS_COL_NAME = "affected_status" + + JUSTIFICATION_PROMPT = dedent(""" + The summary provided below (delimited with XML tags), generated by the software agent named Sherlock, evaluates + a specific CVE (Common Vulnerabilities and Exposures) against the backdrop of a software package or environment + information. This information may include a Software Bill of Materials (SBOM), source code, and dependency + documentation among others related to environments such as Docker containers. Sherlock's role is to analyze this + data to ascertain if the CVE impacts the software environment, determining the necessity for a patch. + + Your task is to review Sherlock's analysis and classify the situation into one of the following categories based + on the described scenario: + - false_positive: The association between the software package and the CVE is incorrect, either due to an + erroneous package identification or a mismatched CVE. + - code_not_present: The software product is unaffected as it does not contain the code or library that harbors + the vulnerability. + - code_not_reachable: During runtime, the vulnerable code is not executed. + - requires_configuration: The exploitability of this issue depends on whether a specific configuration option is + enabled or disabled. In this case, the configuration is set in a manner that prevents exploitability. + - requires_dependency: Exploitability is contingent upon a missing dependency. + - requires_environment: A specific environment, absent in this case, is required for exploitability. + - compiler_protected: Exploitability hinges on the setting or unsetting of a compiler flag. In this case, the + flag is set in a manner that prevents exploitability. + - runtime_protected: Mechanisms are in place that prevent exploits during runtime. + - perimeter_protected: Protective measures block attacks at the physical, logical, or network perimeters. + - mitigating_control_protected: Implemented controls mitigate the likelihood or impact of the vulnerability. + - uncertain: Not enough information to determine the package's exploitability status. + - vulnerable: the package is actually vulnerable to the CVE and needs to be patched. + + Response only with the category name on the first line, and reasoning on the second line. + {{summary}} + """).strip("\n") + + # Map raw justification labels optimized for LLM acccuracy to final labels expected by downstream systems + RAW_TO_FINAL_JUSTIFICATION_MAP = { + "false_positive": "false_positive", + "code_not_present": "code_not_present", + "code_not_reachable": "code_not_reachable", + "requires_configuration": "requires_configuration", + "requires_dependency": "requires_dependency", + "requires_environment": "requires_environment", + "compiler_protected": "protected_by_compiler", + "runtime_protected": "protected_at_runtime", + "perimeter_protected": "protected_at_perimeter", + "mitigating_control_protected": "protected_by_mitigating_control", + "uncertain": "uncertain", + "vulnerable": "vulnerable" + } + + JUSTIFICATION_TO_AFFECTED_STATUS_MAP = { + "false_positive": "FALSE", + "code_not_present": "FALSE", + "code_not_reachable": "FALSE", + "requires_configuration": "FALSE", + "requires_dependency": "FALSE", + "requires_environment": "FALSE", + "protected_by_compiler": "FALSE", + "protected_at_runtime": "FALSE", + "protected_at_perimeter": "FALSE", + "protected_by_mitigating_control": "FALSE", + "uncertain": "UNKNOWN", + "vulnerable": "TRUE" + } + + def __init__(self, *, llm_client: LLMClient): + """ + Initialize the CVEJustificationNode with a selected model. + + Parameters + ---------- + llm_client : LLMClient + The LLM client to use for generating the justification. + """ + super().__init__() + + async def _strip_summaries(summaries: list[str]) -> list[str]: + return [summary.strip() for summary in summaries] + + self.add_node('stripped_summaries', inputs=['summaries'], node=LLMLambdaNode(_strip_summaries)) + + self.add_node("justification_prompt", + inputs=[("/stripped_summaries", "summary")], + node=PromptTemplateNode(template=self.JUSTIFICATION_PROMPT, template_format='jinja')) + + self.add_node("justify", inputs=["/justification_prompt"], node=LLMGenerateNode(llm_client=llm_client)) + + async def _parse_justification(justifications: list[str]) -> dict[str, list[str]]: + labels: list[str] = [] + reasons: list[str] = [] + affected_status: list[str] = [] + + split_justifications = [j.split('\n') for j in justifications] + + for j in split_justifications: + + if len(j) < 2: + raise ValueError( + f"Invalid justification format: {j}. Must be the label and the reason separated by a newline") + + justification_label_raw = j[0].strip() + justification_label = self.RAW_TO_FINAL_JUSTIFICATION_MAP.get(justification_label_raw, + justification_label_raw) + labels.append(justification_label) + reasons.append('\n'.join(j[1:]).strip()) + try: + affected_status.append(self.JUSTIFICATION_TO_AFFECTED_STATUS_MAP[justification_label]) + except KeyError: + logger.error("Invalid justification label: '%s', setting affected_status='UNKNOWN'", + justification_label) + affected_status.append("UNKNOWN") + + return { + self.JUSTIFICATION_LABEL_COL_NAME: labels, + self.JUSTIFICATION_REASON_COL_NAME: reasons, + self.AFFECTED_STATUS_COL_NAME: affected_status + } + + self.add_node("parse_justification", + inputs=['/justify'], + node=LLMLambdaNode(_parse_justification), + is_output=True) diff --git a/src/cve/nodes/cve_langchain_agent_node.py b/src/cve/nodes/cve_langchain_agent_node.py new file mode 100644 index 0000000..f5f9f5a --- /dev/null +++ b/src/cve/nodes/cve_langchain_agent_node.py @@ -0,0 +1,184 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import asyncio +import logging +import typing + +from langchain_core.exceptions import OutputParserException + +from morpheus_llm.llm import LLMContext +from morpheus_llm.llm import LLMNodeBase + +from ..data_models.output import AgentIntermediateStep +from ..utils.data_utils import to_json + +logger = logging.getLogger(__name__) + +if typing.TYPE_CHECKING: + from langchain.agents import AgentExecutor + + +class CVELangChainAgentNode(LLMNodeBase): + """ + LangChainAgentNode which stores the VDB names in the metadata. + + Parameters + ---------- + agent_executor : AgentExecutor + The agent executor to use to execute. + + vdb_names : tuple[str, str] + Name of the VDBs to load from the input. + """ + + def __init__(self, + *, + create_agent_executor_fn: "typing.Callable[[LLMContext], AgentExecutor]", + replace_exceptions: bool = False, + replace_exceptions_value: typing.Optional[str] = None): + super().__init__() + + self._create_agent_executor_fn = create_agent_executor_fn + self._replace_exceptions = replace_exceptions + self._replace_exceptions_value = replace_exceptions_value + + self._input_names = ["input"] + + def get_input_names(self): + return self._input_names + + @staticmethod + def _is_all_lists(data: dict[str, typing.Any]) -> bool: + if (len(data) == 0): + return False + + return all(isinstance(v, list) for v in data.values()) + + @staticmethod + def _transform_dict_of_lists(data: dict[str, typing.Any]) -> list[dict[str, typing.Any]]: + return [dict(zip(data, t)) for t in zip(*data.values())] + + @staticmethod + def _parse_intermediate_step(step: tuple[typing.Any, typing.Any]) -> dict[str, typing.Any]: + """ + Parse an agent intermediate step into an AgentIntermediateStep object. Return the dictionary representation for + compatibility with cudf. + """ + if len(step) != 2: + raise ValueError(f"Expected 2 values in each intermediate step but got {len(step)}.") + else: + action, output = step + + # Flatten tool output to a JSON string for compatibility with cudf + output_json = to_json(output) + + return AgentIntermediateStep(tool_name=action.tool, + action_log=action.log, + tool_input=action.tool_input, + tool_output=output_json).model_dump() + + def _postprocess_results(self, results: list[list[dict]]) -> tuple[list[list[str]], list[list[list]]]: + """ + Post-process results into lists of outputs and intermediate steps. Replace exceptions with placholder values if + self._replace_exceptions = True. + """ + outputs = [[] for _ in range(len(results))] + intermediate_steps = [[] for _ in range(len(results))] + + for i, answer_list in enumerate(results): + for j, answer in enumerate(answer_list): + + # Handle exceptions returned by the agent + # OutputParserException is not a subclass of Exception, so we need to check for it separately + if isinstance(answer, (OutputParserException, Exception)): + if self._replace_exceptions: + # If the agent encounters a parsing error or a server error after retries, replace the error + # with default values to prevent the pipeline from crashing + outputs[i].append(self._replace_exceptions_value) + intermediate_steps[i].append(None) + logger.warning(f"Exception encountered in result[{i}][{j}]: {answer}. " + f"Replacing with default output: \"{self._replace_exceptions_value}\" " + "and intermediate_steps: None") + + # For successful agent responses, extract the output, and intermediate steps if available + else: + outputs[i].append(answer["output"]) + + # intermediate_steps availability depends on run_config.engine.agent.return_intermediate_steps + if "intermediate_steps" in answer: + intermediate_steps[i].append( + [self._parse_intermediate_step(step) for step in answer["intermediate_steps"]]) + else: + intermediate_steps[i].append(None) + + return outputs, intermediate_steps + + async def _run_single(self, + agent: "AgentExecutor", + metadata: dict[str, typing.Any] = None, + **kwargs) -> dict[str, typing.Any]: + + all_lists = self._is_all_lists(kwargs) + + # Check if all values are a list + if all_lists: + + # Transform from dict[str, list[Any]] to list[dict[str, Any]] + input_list = self._transform_dict_of_lists(kwargs) + + # If all metadata values are lists of the same length and the same length as the input list + # then transform them the same way as the input list + if self._is_all_lists(metadata) and all(len(v) == len(input_list) for v in metadata.values()): + metadata_list = self._transform_dict_of_lists(metadata) + else: + metadata_list = [metadata] * len(input_list) + + # Run multiple again + results_async = [ + self._run_single(agent=agent, metadata=metadata_list[i], **x) for (i, x) in enumerate(input_list) + ] + + results = await asyncio.gather(*results_async, return_exceptions=True) + + # # Transform from list[dict[str, Any]] to dict[str, list[Any]] + # results = {k: [x[k] for x in results] for k in results[0]} + + return results + + # We are not dealing with a list, so run single + try: + input_single = {"input": kwargs.pop("input")} + config = {"callbacks": agent.callbacks, "tags": agent.tags, "metadata": metadata} + return await agent.ainvoke(input=input_single, config=config, **kwargs) + except Exception as e: + logger.exception("Error running agent: %s", e) + return e + + async def execute(self, context: LLMContext) -> LLMContext: + + input_dict: dict = context.get_inputs() # type: ignore + metadata = {} + + agent = self._create_agent_executor_fn(context) + + results = await self._run_single(agent=agent, metadata=metadata, **input_dict) + + outputs, intermediate_steps = self._postprocess_results(results) + + context.set_output({"outputs": outputs, "intermediate_steps": intermediate_steps}) + + return context diff --git a/src/cve/nodes/cve_summary_node.py b/src/cve/nodes/cve_summary_node.py new file mode 100644 index 0000000..1dcedfd --- /dev/null +++ b/src/cve/nodes/cve_summary_node.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging + +from morpheus_llm.llm import LLMLambdaNode +from morpheus_llm.llm import LLMNode +from morpheus_llm.llm.nodes.llm_generate_node import LLMGenerateNode +from morpheus_llm.llm.nodes.prompt_template_node import PromptTemplateNode +from morpheus_llm.llm.services.llm_service import LLMClient + +from ..utils.string_utils import get_checklist_item_string + +logger = logging.getLogger(__name__) + +SUMMARY_PROMPT = """Summarize the exploitability investigation results of a Common Vulnerabilities and Exposures (CVE) \ +based on the provided Checklist and Findings. Write a concise paragraph focusing only on checklist items with \ +definitive answers. Begin your response by clearly stating whether the CVE is exploitable. Disregard any ambiguous \ +checklist items. +Checklist and Findings: +{{response}}""" + + +class CVESummaryNode(LLMNode): + """ + A node to summarize the results of the checklist responses. + """ + + def __init__(self, *, llm_client: LLMClient): + """ + Initialize the CVESummaryNode with a selected model. + + Parameters + ---------- + llm_client : LLMClient + The LLM client to use for generating the summary. + """ + super().__init__() + + async def build_summary_output(checklist_inputs: list[list[str]], + checklist_outputs: list[list[str]], + intermediate_steps: list[list[list]]) -> list[list[dict]]: + summary_output = [] + + for check_in, check_out, steps in zip(checklist_inputs, checklist_outputs, intermediate_steps): + summary_output.append([{ + "question": q, "response": r, "intermediate_steps": s + } for q, r, s in zip(check_in, check_out, steps)]) # yapf: disable + + return summary_output + + # Build the output for the checklist as a list of dictionaries + self.add_node("checklist", + inputs=["checklist_inputs", "checklist_outputs", "intermediate_steps"], + node=LLMLambdaNode(build_summary_output), + is_output=True) + + # Concatenate the output of the checklist responses into a single string + async def concat_checklist_responses(agent_q_and_a: list[list[dict]]) -> list[str]: + concatted_responses = [] + + for checklist in agent_q_and_a: + checklist_str = '\n'.join([get_checklist_item_string(i + 1, item) for i, item in enumerate(checklist)]) + concatted_responses.append(checklist_str) + + return concatted_responses + + self.add_node('results', inputs=['/checklist'], node=LLMLambdaNode(concat_checklist_responses)) + + self.add_node('summary_prompt', + inputs=[('/results', 'response')], + node=PromptTemplateNode(template=SUMMARY_PROMPT, template_format='jinja')) + + # Generate a summary from the combined checklist responses + self.add_node("summary", + inputs=["/summary_prompt"], + node=LLMGenerateNode(llm_client=llm_client), + is_output=True) diff --git a/src/cve/pipeline/__init__.py b/src/cve/pipeline/__init__.py new file mode 100644 index 0000000..f527735 --- /dev/null +++ b/src/cve/pipeline/__init__.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .pipeline import pipeline +from .pipeline import build_pipeline_without_sink + +__all__ = ["pipeline", "build_pipeline_without_sink"] diff --git a/src/cve/pipeline/engine.py b/src/cve/pipeline/engine.py new file mode 100644 index 0000000..33ee6ce --- /dev/null +++ b/src/cve/pipeline/engine.py @@ -0,0 +1,239 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging + +from langchain.agents import AgentType +from langchain.agents import Tool +from langchain.agents import initialize_agent +from langchain.chains.retrieval_qa.base import RetrievalQA +from langchain.vectorstores.faiss import FAISS +from langchain_core.embeddings import Embeddings + +from morpheus_llm.llm import LLMContext +from morpheus_llm.llm import LLMEngine +from morpheus_llm.llm.nodes.extracter_node import ManualExtracterNode +from morpheus_llm.llm.services.llm_service import LLMService +from morpheus_llm.llm.services.utils.langchain_llm_client_wrapper import LangchainLLMClientWrapper +from morpheus_llm.llm.task_handlers.simple_task_handler import SimpleTaskHandler + +from ..data_models.config import RunConfig +from ..data_models.info import AgentMorpheusInfo +from ..nodes.cve_checklist_node import CVEChecklistNode +from ..nodes.cve_justification_node import CVEJustifyNode +from ..nodes.cve_langchain_agent_node import CVELangChainAgentNode +from ..nodes.cve_summary_node import CVESummaryNode +from ..utils.code_searcher import LangchainCodeSearcher +from ..utils.document_embedding import DocumentEmbedding +from ..utils.prompting import agent_examples_for_prompt +from ..utils.serp_api_wrapper import MorpheusSerpAPIWrapper + +logger = logging.getLogger(__name__) + + +def _build_dynamic_agent_fn(run_config: RunConfig, embeddings: Embeddings): + + chat_service = LLMService.create(run_config.engine.agent.model.service.type, + **run_config.engine.agent.model.service.model_dump(exclude={"type"}, + by_alias=True)) + chat_client = chat_service.get_client(**run_config.engine.agent.model.model_dump(exclude={"service", "type"}, + by_alias=True)) + langchain_llm = LangchainLLMClientWrapper(client=chat_client) + + # Initialize a SerpAPIWrapper object to perform internet searches. + search = MorpheusSerpAPIWrapper(max_retries=run_config.general.max_retries) + + # Append new Tools to the tools list, which allows for internet searches and software version comparisons. + # The first tool can be especially useful for answering questions about external libraries while the second + # allows for more consistent and accurate comparisons of software versions. + + def inner_create_agent_fn(context: LLMContext): + + tools: list[Tool] = [ + Tool( + name="Internet Search", + func=search.run, # Synchronous function for running searches. + coroutine=search.arun, # Asynchronous coroutine for running searches. + description="useful for when you need to answer questions about external libraries", + ), + ] + + vdb_map: AgentMorpheusInfo.VdbPaths = context.message().get_metadata("info.vdb") # type: ignore + + def run_retrieval_qa_tool(retrieval_qa_tool: RetrievalQA, query: str) -> str | dict: + """ + Runs a given retrieval QA tool on the provided query. Returns a dict of the result string and source + documents if the `return_source_documents` config is true, otherwise it returns just the result string if + `return_source_documents` is false. + """ + output_dict = retrieval_qa_tool(query) + + # If returning source documents, include the result and source_documents keys in the output + if run_config.engine.agent.return_source_documents: + return {k: v for k, v in output_dict.items() if k in ["result", "source_documents"]} + + # If not returning source documents, return only the result as a string + else: + return output_dict["result"] + + if (vdb_map.code_vdb_path is not None): + # load code vector DB + code_vector_db = FAISS.load_local(vdb_map.code_vdb_path, embeddings, allow_dangerous_deserialization=True) + code_qa_tool = RetrievalQA.from_chain_type( + llm=langchain_llm, + chain_type="stuff", + retriever=code_vector_db.as_retriever(), + return_source_documents=run_config.engine.agent.return_source_documents) + tools.append( + Tool(name="Docker Container Code QA System", + func=lambda query: run_retrieval_qa_tool(code_qa_tool, query), + description=("useful for when you need to check if an application or any dependency within " + "the Docker container uses a function or a component of a library."))) + elif run_config.general.code_search_tool: + + logger.info("Preparing source code documents for the code search tool.") + + # Use existing document loader and chunker from DocumentEmbedding class, without embedding. + embedder = DocumentEmbedding(embedding=None, + vdb_directory=run_config.general.base_vdb_dir, + git_directory=run_config.general.base_git_dir) + + documents = [] + sources = context.message().get_metadata("input").image.source_info + for source_info in sources: + if source_info.type == 'code': + documents.extend(embedder.collect_documents(source_info)) + + if len(documents) > 0: + documents_index = embedder._chunk_documents(documents) + lexical_code_searcher = LangchainCodeSearcher(documents_index, rank_documents=True, k=5) + + tools.append( + Tool(name="Docker Container Code Search", + func=lexical_code_searcher.search, + description=("useful for when you need to search the Docker container's code for a given " + "function or component of a library. This requires exact function name or library" + "without no additional information"))) + else: + logger.warning("No code documents found for the code search tool.") + + if (vdb_map.doc_vdb_path is not None): + guide_vector_db = FAISS.load_local(vdb_map.doc_vdb_path, embeddings, allow_dangerous_deserialization=True) + guide_qa_tool = RetrievalQA.from_chain_type( + llm=langchain_llm, + chain_type="stuff", + retriever=guide_vector_db.as_retriever(), + return_source_documents=run_config.engine.agent.return_source_documents) + tools.append( + Tool(name="Docker Container Developer Guide QA System", + func=lambda query: run_retrieval_qa_tool(guide_qa_tool, query), + description=( + "Useful for when you need to ask questions about the purpose and functionality of the Docker " + "container."))) + + # Define a system prompt that sets the context for the language model's task. This prompt positions the assistant + # as a powerful entity capable of investigating CVE impacts on Docker containers. + sys_prompt = ( + "You are a very powerful assistant who helps investigate the impact of reported Common Vulnerabilities and " + "Exposures (CVE) on Docker containers. Information about the Docker container under investigation is stored in " + "vector databases available to you via tools.") + + # Initialize an agent with the tools and settings defined above. + # This agent is designed to handle zero-shot reaction descriptions and parse errors. + agent = initialize_agent( + tools, + langchain_llm, + agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, + verbose=run_config.engine.agent.verbose, + handle_parsing_errors="Check your output and make sure it conforms, use the Action/Action Input syntax", + max_iterations=10, + early_stopping_method="generate", + return_intermediate_steps=run_config.engine.agent.return_intermediate_steps) + + # Modify the language model chain's prompt template to adjust how the model should process inputs and structure + # responses. + prompt_template = agent.agent.llm_chain.prompt.template.replace( + "Answer the following questions as best you can.", + ("If the input is not a question, formulate it into a question first. Include intermediate thought in the " + "final answer."), + 1).replace( + "Use the following format:", + "Use the following format (start each response with one of the following prefixes: " + "[Question, Thought, Action, Action Input, Final Answer]):", + 1) + if run_config.engine.agent.prompt_examples: + prompt_template = prompt_template.replace("Begin!\n\n", agent_examples_for_prompt + "Begin!\n\n") + agent.agent.llm_chain.prompt.template = f'{sys_prompt} {prompt_template}' + + return agent + + return inner_create_agent_fn + + +def build_engine(*, run_config: RunConfig, embeddings: Embeddings): + + summary_service = LLMService.create(run_config.engine.summary_model.service.type, + **run_config.engine.summary_model.service.model_dump(exclude={"type"})) + justification_service = LLMService.create( + run_config.engine.justification_model.service.type, + **run_config.engine.justification_model.service.model_dump(exclude={"type"})) + + engine = LLMEngine() + + checklist_node = CVEChecklistNode(checklist_model_config=run_config.engine.checklist_model, + enable_llm_list_parsing=run_config.general.enable_llm_list_parsing) + + engine.add_node("extract_prompt", node=ManualExtracterNode(input_names=checklist_node.get_input_names())) + + engine.add_node("checklist", inputs=[("/extract_prompt/*", "*")], node=checklist_node) + + engine.add_node("agent", + inputs=[("/checklist", "input")], + node=CVELangChainAgentNode( + create_agent_executor_fn=_build_dynamic_agent_fn(run_config, embeddings), + replace_exceptions=True, + replace_exceptions_value="I do not have a definitive answer for this checklist item.")) + + engine.add_node('summary', + inputs=[("/checklist", "checklist_inputs"), ("/agent/outputs", "checklist_outputs"), + "/agent/intermediate_steps"], + node=CVESummaryNode(llm_client=summary_service.get_client( + **run_config.engine.summary_model.model_dump(exclude={"service", "type"})))) + + engine.add_node('justification', + inputs=[("/summary/summary", "summaries")], + node=CVEJustifyNode(llm_client=justification_service.get_client( + **run_config.engine.justification_model.model_dump(exclude={"service", "type"})))) + + handler_inputs = [ + "/summary/checklist", + "/summary/summary", + f"/justification/{CVEJustifyNode.JUSTIFICATION_LABEL_COL_NAME}", + f"/justification/{CVEJustifyNode.JUSTIFICATION_REASON_COL_NAME}", + f"/justification/{CVEJustifyNode.AFFECTED_STATUS_COL_NAME}", + ] + handler_outputs = [ + "checklist", + "summary", + CVEJustifyNode.JUSTIFICATION_LABEL_COL_NAME, + CVEJustifyNode.JUSTIFICATION_REASON_COL_NAME, + CVEJustifyNode.AFFECTED_STATUS_COL_NAME + ] + + # Add our task handler + engine.add_task_handler(inputs=handler_inputs, handler=SimpleTaskHandler(output_columns=handler_outputs)) + + return engine diff --git a/src/cve/pipeline/input.py b/src/cve/pipeline/input.py new file mode 100644 index 0000000..0848142 --- /dev/null +++ b/src/cve/pipeline/input.py @@ -0,0 +1,324 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import asyncio +import logging +import os +import typing + +import aiohttp +import appdirs +import json5 +import mrc +from pydantic import BaseModel + +from morpheus.config import Config +from morpheus.pipeline.linear_pipeline import LinearPipeline +from morpheus.pipeline.stage_decorator import source +from morpheus.pipeline.stage_decorator import stage + +from ..data_models.config import FileInputConfig +from ..data_models.config import HttpInputConfig +from ..data_models.config import ManualInputConfig +from ..data_models.config import PluginInputConfig +from ..data_models.config import RunConfig +from ..data_models.cve_intel import CveIntel +from ..data_models.dependencies import VulnerableDependencies +from ..data_models.dependencies import VulnerableSBOMPackage +from ..data_models.info import AgentMorpheusInfo +from ..data_models.input import AgentMorpheusEngineInput +from ..data_models.input import AgentMorpheusInput +from ..data_models.input import FileSBOMInfoInput +from ..data_models.input import ManualSBOMInfoInput +from ..data_models.input import SBOMPackage +from ..data_models.plugin import InputPluginSchema +from ..stages.build_vdb_stage import BuildSourceCodeVdbStage +from ..stages.pydantic_http_stage import PydanticHttpStage +from ..utils.document_embedding import DocumentEmbedding +from ..utils.embedding_loader import EmbeddingLoader +from ..utils.intel_retriever import IntelRetriever +from ..utils.vulnerable_dependency_checker import VulnerableDependencyChecker + +logger = logging.getLogger(__name__) + + +def build_http_input(pipe: LinearPipeline, config: Config, input: HttpInputConfig, + input_schema: type[BaseModel]) -> logging.Logger: + # Create a new logger handler for the HTTP server source. This allows saving the HTTP server logs to a file + # in addition to writing them to the console. + http_server_logger = logging.getLogger(logger.name + ".http_server") + + # Log to a file in the user's log directory next to morpheus logs + log_file = os.path.join(appdirs.user_log_dir(appauthor="NVIDIA", appname="morpheus"), "http_server.log") + + # Add a file handler + file_handler = logging.FileHandler(log_file) + + # Set the format + file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")) + + http_server_logger.addHandler(file_handler) + http_server_logger.setLevel(logging.INFO) + + http_server_logger.info(f"Http Server Started for {input.http_method.name} " + f"""at {input.address}:{input.port} + {input.endpoint}.""") + + pipe.set_source( + PydanticHttpStage(config, + bind_address=input.address, + port=input.port, + endpoint=input.endpoint, + method=input.http_method, + stop_after=input.stop_after, + input_schema=input_schema)) + + @stage + def print_payload(payload: typing.Any) -> typing.Any: + assert isinstance(payload, BaseModel) + + serialized_str = payload.model_dump_json(indent=2) + + http_server_logger.info("======= Got Request =======\n%s\n===========================", serialized_str) + + return payload + + pipe.add_stage(print_payload(config)) + + return http_server_logger + + +def build_input(pipe: LinearPipeline, config: Config, run_config: RunConfig): + + # Create the base directories for the VDB and source code if they don't already exist + for file_dir in (run_config.general.base_vdb_dir, run_config.general.base_git_dir): + os.makedirs(file_dir, exist_ok=True) + + if (run_config.input.type == ManualInputConfig.static_type()): + + @source + def emit_input_object(subscription: mrc.Subscription) -> typing.Generator[AgentMorpheusInput, None, None]: + assert isinstance(run_config.input, ManualInputConfig) + + for repead_idx in range(run_config.input.repeat_count): + if not subscription.is_subscribed(): + return + + yield run_config.input.message.model_copy(deep=True) + + pipe.set_source(emit_input_object(config)) + + elif (run_config.input.type == FileInputConfig.static_type()): + + assert isinstance(run_config.input, FileInputConfig) + + # Read the input data from the file + with open(run_config.input.file, "r") as f: + json_dict = json5.load(fp=f) + + # Create the input object + input_obj = AgentMorpheusInput.model_validate(json_dict) + repeat_count = run_config.input.repeat_count + + @source + def emit_input_object(subscription: mrc.Subscription) -> typing.Generator[AgentMorpheusInput, None, None]: + + for _ in range(repeat_count): + if not subscription.is_subscribed(): + return + + yield input_obj.model_copy(deep=True) + + pipe.set_source(emit_input_object(config)) + + elif (run_config.input.type == HttpInputConfig.static_type()): + + assert isinstance(run_config.input, HttpInputConfig) + + build_http_input(pipe, config, run_config.input, AgentMorpheusInput) + + elif (run_config.input.type == PluginInputConfig.static_type()): + # Set source based on plugins + plugin = InputPluginSchema.locate(run_config.input.plugin_name) + plugin.build_input(pipe, config, run_config) + else: + raise ValueError(f"Invalid source type: {run_config.input.type}") + + # Load embedding model to be used throughout the pipeline + embedding = EmbeddingLoader.create(run_config.engine.rag_embedding.type, + **run_config.engine.rag_embedding.model_dump(exclude={"type"})) + + embedder = DocumentEmbedding(embedding=embedding, + vdb_directory=run_config.general.base_vdb_dir, + git_directory=run_config.general.base_git_dir) + + build_vdb_stage = BuildSourceCodeVdbStage(config, + build_vdb_fn=embedder.build_vdbs, + ignore_errors=run_config.general.ignore_build_vdb_errors, + ignore_code_embedding=run_config.general.code_search_tool) + + pipe.add_stage(build_vdb_stage) + + @stage + def fetch_intel(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + + async def _inner(): + + async with aiohttp.ClientSession() as session: + + intel_retriever = IntelRetriever(session=session) + + intel_coros = [intel_retriever.retrieve(vuln_id=cve.vuln_id) for cve in message.input.scan.vulns] + + intel_objs = await asyncio.gather(*intel_coros) + + return intel_objs + + result = asyncio.run(_inner()) + + message.info.intel = result + + return message + + pipe.add_stage(fetch_intel(config)) + + @stage + def process_sbom(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + + if (message.input.image.sbom_info.type == ManualSBOMInfoInput.static_type()): + assert isinstance(message.input.image.sbom_info, ManualSBOMInfoInput) + + # Create the SBOM object + message.info.sbom = AgentMorpheusInfo.SBOMInfo(packages=message.input.image.sbom_info.packages) + + elif (message.input.image.sbom_info.type == FileSBOMInfoInput.static_type()): + assert isinstance(message.input.image.sbom_info, FileSBOMInfoInput) + + # Read the file to an object + with open(message.input.image.sbom_info.file_path, "r") as f: + sbom_lines = f.readlines() + + # Extract the packages + packages: list[SBOMPackage] = [] + + # Skip the first header line + for line in sbom_lines[1:]: + split = line.split() + if len(split) < 3: + continue + p_version = split[1] + p_name = split[0] + p_sys = split[2] + + packages.append(SBOMPackage(name=p_name, version=p_version, system=p_sys)) + + # Create the SBOM object + message.info.sbom = AgentMorpheusInfo.SBOMInfo(packages=packages) + + return message + + pipe.add_stage(process_sbom(config)) + + @stage + def check_vulnerable_dependencies(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + """Check for vulnerable packages in the dependency graph and update the message object.""" + + sbom = message.info.sbom.packages + image = f"{message.input.image.name}:{message.input.image.tag}" + + async def _inner(): + + async with aiohttp.ClientSession() as session: + + vuln_dep_checker = VulnerableDependencyChecker(session=session, image=image, sbom_list=sbom) + + await vuln_dep_checker.load_dependencies() + + async def _calc_dep(cve_intel: CveIntel): + vuln_id = cve_intel.vuln_id + + vuln_deps = [] + vuln_package_intel_sources = [] + + # Check vulnerabilities from GHSA + if (cve_intel.ghsa is not None and cve_intel.ghsa.vulnerabilities): + vuln_package_intel_sources.append("ghsa") + vuln_deps.extend(await vuln_dep_checker.run_ghsa(cve_intel.ghsa.vulnerabilities)) + + # Check vulnerabilities from NVD + if (cve_intel.nvd is not None and cve_intel.nvd.configurations): + vuln_package_intel_sources.append("nvd") + vuln_deps.extend(await vuln_dep_checker.run_nvd(cve_intel.nvd.configurations)) + + if not len(vuln_package_intel_sources): + logger.warning("No vulnerabilities were found in either GHSA or NVD intel for %s.", vuln_id) + + # Check vulnerabilities from Ubuntu notices + if (cve_intel.ubuntu is not None and hasattr(cve_intel.ubuntu, 'notices') + and cve_intel.ubuntu.notices): + vuln_package_intel_sources.append("ubuntu") + vuln_deps.extend(await vuln_dep_checker.run_ubuntu(cve_intel.ubuntu.notices)) + + # Check vulnerabilities from RHSA + if (cve_intel.rhsa is not None and hasattr(cve_intel.rhsa, 'affected_release') + and cve_intel.rhsa.affected_release): + vuln_package_intel_sources.append("rhsa") + vuln_deps.extend(await vuln_dep_checker.run_rhsa(cve_intel.rhsa.affected_release)) + + # Create list of vulnerable SBOM packages and the related vulnerable dependency for the CVE + vulnerable_sbom_packages: list[VulnerableSBOMPackage] = [] + for vuln_dep in vuln_deps: + for (sbom_pkg_name, sbom_pkg_version), vuln_dep_pkg in vuln_dep.items(): + vulnerable_sbom_packages.append( + VulnerableSBOMPackage(name=sbom_pkg_name, + version=sbom_pkg_version, + vulnerable_dependency_package=vuln_dep_pkg)) + + if len(vulnerable_sbom_packages) > 0: + logger.info("Found vulnerable dependencies for %s.", vuln_id) + else: + logger.info("No vulnerable dependencies found for %s.", vuln_id) + + # Add the vulnerable dependencies for this CVE to the overall list + return VulnerableDependencies(vuln_id=vuln_id, + vuln_package_intel_sources=vuln_package_intel_sources, + vulnerable_sbom_packages=vulnerable_sbom_packages) + + # Check vulnerable dependencies for each CVE + vulnerable_dependencies: list[VulnerableDependencies] = await asyncio.gather( + *[_calc_dep(cve_intel) for cve_intel in message.info.intel]) + + return vulnerable_dependencies + + # Update the message info with the vulnerable dependencies list + message.info.vulnerable_dependencies = asyncio.run(_inner()) + return message + + pipe.add_stage(check_vulnerable_dependencies(config)) + + @stage + def debug_input(message: AgentMorpheusEngineInput) -> AgentMorpheusEngineInput: + + # Save the input object to disk for debugging + with open("./.tmp/input_object.json", "w") as f: + f.write(message.model_dump_json(indent=2)) + + return message + + pipe.add_stage(debug_input(config)) + + # Return embedding model to be used later in the pipeline + return embedding diff --git a/src/cve/pipeline/output.py b/src/cve/pipeline/output.py new file mode 100644 index 0000000..8fc534e --- /dev/null +++ b/src/cve/pipeline/output.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +import logging +import os + +from morpheus.config import Config +from morpheus.pipeline.linear_pipeline import LinearPipeline +from morpheus.pipeline.stage_decorator import stage + +from ..data_models.config import RunConfig +from ..data_models.output import AgentMorpheusOutput +from ..data_models.plugin import OutputPluginSchema +from ..stages.write_pydantic_to_file import WriteFileMode +from ..stages.write_pydantic_to_file import WritePydanticToFile +from ..utils.output_formatter import generate_vulnerability_reports + +logger = logging.getLogger(__name__) + + +def build_output(pipe: LinearPipeline, config: Config, run_config: RunConfig): + + if (run_config.output.type == "print"): + + @stage + def print_output_stage(message: AgentMorpheusOutput) -> AgentMorpheusOutput: + logger.info("Got output message for image: %s", message.input.image.name) + + pod_output = [x.model_dump(mode="json") for x in message.output] + + logger.info(json.dumps(pod_output, indent=2)) + + return message + + pipe.add_stage(print_output_stage(config)) + + elif (run_config.output.type == "file"): + + pipe.add_stage( + WritePydanticToFile( + config, + filename=run_config.output.file_path, + mode=(WriteFileMode.OVERWRITE if run_config.output.overwrite else WriteFileMode.CREATE))) + + if run_config.output.markdown_dir is not None: + os.makedirs(os.path.realpath(run_config.output.markdown_dir), exist_ok=True) + + @stage + def write_output_to_markdown(message: AgentMorpheusOutput) -> AgentMorpheusOutput: + generate_vulnerability_reports(message, run_config.output.markdown_dir) + return message + + pipe.add_stage(write_output_to_markdown(config)) + + elif (run_config.output.type == "plugin"): + plugin = OutputPluginSchema.locate(run_config.output.plugin_name) + plugin.build_output(pipe, config, run_config) + else: + # TODO: re-enable other output types. Issue: #100 + raise NotImplementedError(f'Unsupported output type "{run_config.output.type}", use type "file" instead.') diff --git a/src/cve/pipeline/pipeline.py b/src/cve/pipeline/pipeline.py new file mode 100644 index 0000000..e5cf872 --- /dev/null +++ b/src/cve/pipeline/pipeline.py @@ -0,0 +1,227 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import datetime +import logging +import os +import time + +import langchain +import langchain.globals +import pandas as pd + +import cudf + +from morpheus.config import Config +from morpheus.config import CppConfig +from morpheus.config import PipelineModes +from morpheus.messages import ControlMessage +from morpheus.messages import MessageMeta +from morpheus.pipeline.linear_pipeline import LinearPipeline +from morpheus.pipeline.stage_decorator import stage +from morpheus.stages.general.monitor_stage import MonitorStage +from morpheus.utils.concat_df import concat_dataframes +from morpheus_llm.stages.llm.llm_engine_stage import LLMEngineStage + +from ..data_models.config import RunConfig +from ..data_models.input import AgentMorpheusEngineInput +from ..stages.convert_to_output_object import convert_to_output_object +from .engine import build_engine +from .input import build_input +from .output import build_output + +logger = logging.getLogger(__name__) + + +def build_pipeline_without_sink(run_config: RunConfig) -> tuple[Config, LinearPipeline]: + """ + Builds the entire pipeline with the exception of the sink. + """ + if (run_config.general.use_uvloop): + # If uvloop is installed, use that instead of asyncio for running the loop. Results in a significantly faster + # pipeline + try: + import uvloop + + uvloop.install() + except ImportError: + pass + + if (run_config.general.cache_dir is not None): + # This import is expensive, do it only if needed + from langchain.cache import SQLiteCache + + # Set up the cache to avoid repeated calls + os.makedirs(run_config.general.cache_dir, exist_ok=True) + + langchain.globals.set_llm_cache( + SQLiteCache(database_path=os.path.join(run_config.general.cache_dir, "langchain.sqlite"))) + + CppConfig.set_should_use_cpp(False) + + config = Config() + config.mode = PipelineModes.NLP + + # Below properties are specified by the command line + config.num_threads = run_config.general.num_threads + config.pipeline_batch_size = run_config.general.pipeline_batch_size + config.model_max_batch_size = run_config.general.model_max_batch_size + config.edge_buffer_size = 128 + + pipe = LinearPipeline(config) + + embeddings = build_input(pipe, config, run_config) + + engine = build_engine(run_config=run_config, embeddings=embeddings) + + @stage + def convert_input_to_df(message: AgentMorpheusEngineInput) -> ControlMessage: + + assert message.info.intel is not None, "The input message must have intel information" + + input_names = ['vuln_id', 'ghsa_id', 'cve_id'] + + # Determine the input columns that need to be set from the engine (based on checklist_prompt fields) + input_names.extend([name for name in engine.get_input_names() if name not in input_names]) + + # Convert intel object to DataFrame + # Using pandas since cudf doesn't support json_normalize() + full_df = pd.json_normalize([{ + "vuln_id": x.vuln_id, "ghsa_id": x.get_ghsa_id(), "cve_id": x.get_cve_id(), **x.model_dump(mode="json") + } for x in message.info.intel], + sep="_") + + # Ensure all columns for the input_names exist in the DataFrame + for name in input_names: + if name not in full_df.columns: + full_df[name] = [None] * len(full_df) + + # Add vulnerable dependencies to DataFrame + vulnerable_dependencies = [] + for v in message.info.vulnerable_dependencies: + vulnerable_sbom_packages = v.vulnerable_sbom_packages + vulnerable_dependencies.append([p.name for p in vulnerable_sbom_packages]) + + has_vuln_package_info_flags = [ + len(v.vuln_package_intel_sources) > 0 for v in message.info.vulnerable_dependencies + ] + + full_df["vulnerable_dependencies"] = vulnerable_dependencies + + # Filter full_df by whether the CVE has vulnerable dependencies or lacks vulnerable package info from intel + filtered_df = full_df[[ + len(vuln_deps) > 0 or not has_vuln_package_info for vuln_deps, + has_vuln_package_info in zip(vulnerable_dependencies, has_vuln_package_info_flags) + ]] + + # Convert pandas to cudf + # NOTE: keep this AFTER the filtering step above to avoid a bug when filtering cudfs with certain dtypes + if len(filtered_df) > 0: + filtered_df = cudf.from_pandas(filtered_df) + # To avoid a bug when converting empty dataframe to cudf, create empty cudf from scratch + else: + filtered_df = cudf.DataFrame(columns=filtered_df.columns) + + # Drop duplicate vuln_ids + if filtered_df.duplicated(subset="vuln_id").any(): + logger.warning( + "Input contains duplicate vuln_ids. Passing only the first instance of each vuln_id to the LLMEngine.") + filtered_df = filtered_df.drop_duplicates(subset="vuln_id") + + logger.info("Passing %d vuln_id(s) with vulnerable dependencies to the LLM Engine", len(filtered_df)) + + # Convert input columns to MessageMeta + mm = MessageMeta(df=filtered_df[input_names]) + + cm = ControlMessage() + + cm.payload(mm) + + cm.set_metadata("input", message.input) + cm.set_metadata("info.vdb", message.info.vdb) + cm.set_metadata("info.intel", message.info.intel) + cm.set_metadata("info.sbom", message.info.sbom) + cm.set_metadata("info.vulnerable_dependencies", message.info.vulnerable_dependencies) + + completion_task = { + "task_type": "completion", + "task_dict": { + "input_keys": ['code_vdb', 'cves', 'doc_vdb', 's_feed_group', 's_url', 's_vuln'], + }, + } + + cm.add_task(task_type="llm_engine", task=completion_task) + + return cm + + pipe.add_stage(convert_input_to_df(config)) + + @stage + def add_start_timestamp(message: ControlMessage) -> ControlMessage: + message.set_timestamp("start_time", datetime.datetime.now()) + + return message + + pipe.add_stage(add_start_timestamp(config=config)) + + pipe.add_stage(MonitorStage(config, description="Source")) + + pipe.add_stage(LLMEngineStage(config, engine=engine)) + + @stage + def add_end_timestamp(message: ControlMessage) -> ControlMessage: + current_epoch = datetime.datetime.now() + + message.set_timestamp("end_time", current_epoch) + + start_epoch: datetime.datetime = message.get_timestamp("start_time") + + # Print the message elapsed time in milliseconds + logger.info(f"Message elapsed time: {(current_epoch - start_epoch).total_seconds()} sec") + + return message + + pipe.add_stage(add_end_timestamp(config=config)) + + pipe.add_stage(MonitorStage(config, description="LLM")) + + pipe.add_stage(convert_to_output_object(config)) + + return (config, pipe) + + +def pipeline(run_config: RunConfig): + + (config, pipe) = build_pipeline_without_sink(run_config) + + in_mem_sink = build_output(pipe, config, run_config) + + start_time = time.time() + + pipe.run() + + if in_mem_sink is not None: + messages = in_mem_sink.get_messages() + responses = concat_dataframes(messages) + logger.info("Pipeline complete. Received %s responses", len(responses)) + + if logger.isEnabledFor(logging.DEBUG): + # The responses are quite long, when debug is enabled disable the truncation that pandas and cudf normally + # perform on the output + pd.set_option('display.max_colwidth', None) + logger.debug("Responses:\n%s", responses) + + return start_time diff --git a/src/cve/run.py b/src/cve/run.py new file mode 100644 index 0000000..b780acb --- /dev/null +++ b/src/cve/run.py @@ -0,0 +1,409 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os + +import click +import json5 +from click.core import ParameterSource + +from morpheus.utils.http_utils import HTTPMethod + +from .data_models.config import FileInputConfig +from .data_models.config import GeneralConfig +from .data_models.config import HttpInputConfig +from .data_models.config import ManualInputConfig +from .data_models.config import OutputFileConfig +from .data_models.config import OutputPrintConfig +from .data_models.config import RunConfig +from .data_models.input import AgentMorpheusInput +from .data_models.input import ImageInfoInput +from .data_models.input import ManualSBOMInfoInput +from .data_models.input import ScanInfoInput +from .data_models.input import VulnInfo +from .utils.string_utils import is_valid_cve_id + +logger = logging.getLogger(__name__) + + +def validate_cve_id(ctx, param, value): + for item in value: + if not is_valid_cve_id(item): + raise click.BadParameter(f"{item} is not a valid CVE ID format. Correct format: 'CVE-[YEAR]-[NUMBER]'.") + if not is_valid_cve_id(item): + raise click.BadParameter(f"{item} is not a valid CVE ID format. Correct format: 'CVE-[YEAR]-[NUMBER]'.") + return value + + +def _was_argument_defined(ctx: click.Context, arg_name: str): + + # Clean up the arg_name just in case + arg_name = arg_name.replace("-", "_").lower() + + arg_source = ctx.get_parameter_source(arg_name) + + assert arg_source is not None, f"Argument '{arg_name}' was not found in the context. Check the spelling." + + return arg_source != ParameterSource.DEFAULT + + +@click.group(name=__name__, help="Run the Vulnerability Analysis for Container Security pipeline") +def run(): + pass + + +@run.group(invoke_without_command=True) +@click.option( + "--config_file", + type=str, + default=None, + show_envvar=True, + help=("A json file that sets the parameters for the pipeline and the models " + "inside the pipeline."), +) +def pipeline(**kwargs): + pass + + +@pipeline.result_callback(replace=True) +def process_pipeline(processors, config_file: str): + from .pipeline import pipeline as _pipeline + + run_config = RunConfig.model_construct() + + if config_file is not None: + with open(config_file, 'r') as config_in: + + run_config_json = json5.load(config_in) + + run_config = RunConfig.model_validate(run_config_json) + + for processor in processors or []: + run_config = processor(run_config) + + # Make sure we still have a valid config object + run_config = RunConfig.model_validate(run_config.model_dump(by_alias=True)) + + return _pipeline(run_config) + + +@pipeline.group('config', chain=True, help="This command can be used to override specific configuration file options") +def config(): + pass + + +@config.command('input-manual', + help="Sets the input type to 'manual' and allows for setting options for the manual input.") +@click.option( + "--scan-vuln-id", + type=str, + multiple=True, + help=("The vulnerability ID to scan. Using this option overwrites any existing vulnerabilities."), +) +@click.option( + "--add-scan-vuln-id", + type=str, + multiple=True, + help=("The vulnerability ID to scan. Using this option appends to any existing vulnerabilities."), +) +@click.option( + "--repeat_count", + type=int, + default=ManualInputConfig.model_fields['repeat_count'].default, + help=("The number of times to repeat the input data."), +) +@click.pass_context +def config_input_manual(ctx: click.Context, **kwargs): + + def processor(run_config: RunConfig) -> RunConfig: + override_kwargs = ManualInputConfig( + message=AgentMorpheusInput(scan=ScanInfoInput(vulns=[]), + image=ImageInfoInput(source_info=[], sbom_info=ManualSBOMInfoInput( + packages=[])))).model_dump(by_alias=True) + + if (run_config.input.type == 'manual'): + override_kwargs = run_config.input.model_dump(by_alias=True) + + if (_was_argument_defined(ctx, 'scan-vuln-id')): + + override_kwargs['message']['scan']['vulns'] = [{'vuln_id': vuln_id} for vuln_id in kwargs['scan_vuln_id']] + + if (_was_argument_defined(ctx, 'add-scan-vuln-id')): + + override_kwargs['message']['scan']['vulns'].extend( + [VulnInfo(vuln_id=vuln_id).model_dump(by_alias=True) for vuln_id in kwargs['add_scan_vuln_id']]) + + if (_was_argument_defined(ctx, 'repeat_count')): + override_kwargs['repeat_count'] = kwargs['repeat_count'] + + run_config.input = ManualInputConfig.model_validate(override_kwargs) + + return run_config + + return processor + + +@config.command('input-file', help="Sets the input type to 'file' and allows for setting options for the file input.") +@click.option( + "--file", + type=str, + default=None, + help=("A JSON file which represents an instance of AgentMorpheusInput parameters."), +) +@click.option( + "--repeat_count", + type=int, + default=FileInputConfig.model_fields['repeat_count'].default, + help=("The number of times to repeat the input data."), +) +@click.pass_context +def config_input_file(ctx: click.Context, **kwargs): + + def processor(run_config: RunConfig) -> RunConfig: + + override_kwargs = {} + + if (run_config.input.type == 'file'): + override_kwargs = run_config.input.model_dump(by_alias=True) + + if (_was_argument_defined(ctx, 'file')): + override_kwargs['file'] = kwargs['file'] + if (_was_argument_defined(ctx, 'repeat_count')): + override_kwargs['repeat_count'] = kwargs['repeat_count'] + + run_config.input = FileInputConfig.model_validate(override_kwargs) + + return run_config + + return processor + + +@config.command('input-http', help="Sets the input type to 'http' and allows for setting options for the HTTP input.") +@click.option( + "--address", + type=str, + default=HttpInputConfig.model_fields['address'].default, + help=("The HTTP address to bind to."), +) +@click.option( + "--endpoint", + type=str, + default=HttpInputConfig.model_fields['endpoint'].default, + help=("The HTTP endpoint to listen on."), +) +@click.option( + "--port", + type=str, + default=HttpInputConfig.model_fields['port'].default, + help=("The HTTP port to listen on."), +) +@click.option( + "--http_method", + type=HTTPMethod, + default=HttpInputConfig.model_fields['http_method'].default, + help=("The HTTP method to listen for.Valid values: " + f"{', '.join([v.value for v in HTTPMethod])}"), +) +@click.pass_context +def config_input_http(ctx: click.Context, **kwargs): + + def processor(run_config: RunConfig) -> RunConfig: + override_kwargs = HttpInputConfig().model_dump(by_alias=True) + + if (run_config.input.type == 'http'): + override_kwargs = run_config.input.model_dump(by_alias=True) + + if (_was_argument_defined(ctx, 'address')): + override_kwargs['address'] = kwargs['address'] + + if (_was_argument_defined(ctx, 'endpoint')): + override_kwargs['endpoint'] = kwargs['endpoint'] + + if (_was_argument_defined(ctx, 'port')): + override_kwargs['port'] = kwargs['port'] + + if (_was_argument_defined(ctx, 'http_method')): + override_kwargs['http_method'] = kwargs['http_method'] + + run_config.input = HttpInputConfig.model_validate(override_kwargs) + + return run_config + + return processor + + +@config.command('general', help="Allows for setting general config options") +@click.option( + "--base_vdb_dir", + type=str, + default=GeneralConfig.model_fields['base_vdb_dir'].default, + help=("The base directory to store the VDB files."), +) +@click.option( + "--base_git_dir", + type=str, + default=GeneralConfig.model_fields['base_git_dir'].default, + help=("The base directory to clone Git repositories."), +) +@click.option( + "--enable_llm_list_parsing", + type=bool, + is_flag=True, + default=GeneralConfig.model_fields['enable_llm_list_parsing'].default, + help=("Adds an additional call to the LLM to parse the the returned checklist. " + "Enable if the LLM isn't able to return valid JSON."), +) +@click.option( + "--cache_dir", + type=str, + default=GeneralConfig.model_fields['cache_dir'].default, + help=("The directory to use for caching. If not specified, no caching will be used."), +) +@click.option( + "--ignore_build_vdb_errors", + type=bool, + is_flag=True, + default=GeneralConfig.model_fields['ignore_build_vdb_errors'].default, + help=("Whether or not to ignore errors when building the VDB."), +) +@click.option( + "--max_retries", + type=str, + default=GeneralConfig.model_fields['max_retries'].default, + help=("Maximum number of retries for a failed HTTP request."), +) +@click.option( + "--model_max_batch_size", + type=str, + default=GeneralConfig.model_fields['model_max_batch_size'].default, + help=("Maximum batch size to use for the model."), +) +@click.option( + "--num_threads", + type=str, + default=GeneralConfig.model_fields['num_threads'].default, + help=("The number of threads to use for the pipeline."), +) +@click.option( + "--pipeline_batch_size", + type=str, + default=GeneralConfig.model_fields['pipeline_batch_size'].default, + help=("The batch size to use for the pipeline."), +) +@click.option( + "--use_uvloop", + type=bool, + default=GeneralConfig.model_fields['use_uvloop'].default, + help=("Whether to use uvloop for the event loop. This can provide a significant speedup in some cases. " + "Disabling can give more helpful error messages"), +) +@click.pass_context +def config_general(ctx: click.Context, **kwargs): + + def processor(run_config: RunConfig) -> RunConfig: + override_kwargs = run_config.general.model_dump(by_alias=True) + + for arg in kwargs.keys(): + + if (_was_argument_defined(ctx, arg)): + override_kwargs[arg] = kwargs[arg] + + run_config.general = GeneralConfig.model_validate(override_kwargs) + + return run_config + + return processor + + +@config.command('output-print', + help="Sets the output type to 'print' and allows for setting options for the print output.") +@click.pass_context +def config_output_print(ctx: click.Context, **kwargs): + + def processor(run_config: RunConfig) -> RunConfig: + override_kwargs = OutputPrintConfig().model_dump(by_alias=True) + + if (run_config.output.type == 'print'): + override_kwargs = run_config.output.model_dump(by_alias=True) + + run_config.output = OutputPrintConfig.model_validate(override_kwargs) + + return run_config + + return processor + + +@config.command('output-file', + help="Sets the output type to 'file' and allows for setting options for the file output.") +@click.option("--file_path", + type=str, + default=OutputFileConfig.model_fields['file_path'].default, + help=("The path to the output file.")) +@click.option("--overwrite", + type=bool, + is_flag=True, + default=OutputFileConfig.model_fields['overwrite'].default, + help=("Whether or not to overwrite the output file.")) +@click.option("--markdown_dir", + type=str, + default=OutputFileConfig.model_fields['markdown_dir'].default, + help=("The path to the directory that will store the output markdown reports.")) +@click.pass_context +def config_output_file(ctx: click.Context, **kwargs): + + def processor(run_config: RunConfig) -> RunConfig: + override_kwargs = run_config.general.model_dump(by_alias=True) + + for arg in kwargs.keys(): + + if (_was_argument_defined(ctx, arg)): + override_kwargs[arg] = kwargs[arg] + + run_config.output = OutputFileConfig.model_validate(override_kwargs) + + return run_config + + return processor + + +def _relative_path(abs_path: str) -> str: + return os.path.relpath(abs_path, os.getcwd()) + + +@run.group(help="Set of tools for the LLM examples.") +def tools(): + pass + + +@tools.command(help="Generates the JSON schema for the configuration file.") +@click.option( + '--output_file', + type=str, + default=_relative_path(os.path.abspath(os.path.join(__file__, "../../configs/schemas/config.schema.json"))), + required=True, + help=("The output file to write the schema to."), +) +def gen_config_schema(output_file: str): + + # Ensure the directory exists + os.makedirs(os.path.dirname(output_file), exist_ok=True) + + with open(output_file, "w") as out_file: + + json5.dump(RunConfig.generate_json_schema(), fp=out_file, indent=2, quote_keys=True, trailing_commas=False) + + logger.info("Generated JSON schema for the configuration file at '%s'.", output_file) diff --git a/src/cve/stages/__init__.py b/src/cve/stages/__init__.py new file mode 100644 index 0000000..28d3a3a --- /dev/null +++ b/src/cve/stages/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/cve/stages/build_vdb_stage.py b/src/cve/stages/build_vdb_stage.py new file mode 100644 index 0000000..d2d305b --- /dev/null +++ b/src/cve/stages/build_vdb_stage.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import typing +from pathlib import Path + +import mrc +import mrc.core.operators as ops + +from morpheus.config import Config +from morpheus.pipeline.single_port_stage import SinglePortStage +from morpheus.pipeline.stage_schema import StageSchema + +from ..data_models.info import AgentMorpheusInfo +from ..data_models.input import AgentMorpheusEngineInput +from ..data_models.input import AgentMorpheusInput +from ..data_models.input import SourceDocumentsInfo + +logger = logging.getLogger(f"morpheus.{__name__}") + + +class BuildSourceCodeVdbStage(SinglePortStage): + + def __init__(self, + c: Config, + build_vdb_fn: typing.Callable[[list[SourceDocumentsInfo]], tuple[Path | None, Path | None]], + ignore_errors: bool = False, + ignore_code_embedding: bool = False): + super().__init__(c) + + self._build_vdb_fn = build_vdb_fn + self._ignore_errors = ignore_errors + self._ignore_code_embedding = ignore_code_embedding + + @property + def name(self) -> str: + return "build-source-code-vdb-stage" + + def accepted_types(self) -> typing.Tuple: + """ + Accepted input types for this stage are returned. + + Returns + ------- + typing.Tuple + Accepted input types. + + """ + return (AgentMorpheusInput, ) + + def compute_schema(self, schema: StageSchema): + for (port_idx, port_schema) in enumerate(schema.input_schemas): + schema.output_schemas[port_idx].set_type(AgentMorpheusEngineInput) + + def supports_cpp_node(self): + return False + + def _build_source_code_vdb_stage(self, message: AgentMorpheusInput) -> AgentMorpheusEngineInput | None: + """ + Stage that builds source code and documentation FAISS databases based upon the source repositories. + For now we are only storing a path to the FAISS databases in the message. + + In the future we will want to store the actual FAISS databases in the message. + + Parameters + ---------- + message : MessageMeta + The input message + build_vdb_fn : typing.Callable[[str, str, list[SourceCodeRepo]], dict[str, str]] + The function that builds the VDB database for a given vulnerability scan id, base image, and source code + repos + ignore_errors : bool + When True raised exceptions will be logged, when False raised exceptions will be re-raised + """ + + vdb_code_path = None + vdb_doc_path = None + + try: + base_image = message.image.name + + source_code_repos = message.image.source_info + + vdb_code_path, vdb_doc_path = self._build_vdb_fn(source_code_repos, self._ignore_code_embedding) + + if (vdb_code_path is None): + # Only log warning if we're not ignoring code embeddings + if (not self._ignore_code_embedding): + logger.warning(("Failed to generate code VDB for image '%s'. " + "Ensure the source repositories are setup correctly"), + base_image) + else: + vdb_code_path = str(vdb_code_path) + + if (vdb_doc_path is None): + logger.warning(("Failed to generate documentation VDB for image '%s'. " + "Ensure the source repositories are setup correctly"), + base_image) + else: + vdb_doc_path = str(vdb_doc_path) + + except Exception as e: + # For now just skip the row + logger.error("Failure to build VDB for image, '%s', with source code info: %s\nError: %s", + base_image, + source_code_repos, + e, + exc_info=True) + + if not self._ignore_errors: + raise + + # Return None to skip this message + return None + + return AgentMorpheusEngineInput( + input=message, + info=AgentMorpheusInfo( + vdb=AgentMorpheusInfo.VdbPaths(code_vdb_path=vdb_code_path, doc_vdb_path=vdb_doc_path))) + + def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: + + node = builder.make_node(self.unique_name, + ops.map(self._build_source_code_vdb_stage), + ops.filter(lambda x: x is not None)) + + builder.make_edge(input_node, node) + + return node diff --git a/src/cve/stages/convert_to_output_object.py b/src/cve/stages/convert_to_output_object.py new file mode 100644 index 0000000..6db39ec --- /dev/null +++ b/src/cve/stages/convert_to_output_object.py @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +import logging + +from morpheus.messages import ControlMessage +from morpheus.pipeline.stage_decorator import stage + +from ..data_models.cve_intel import CveIntel +from ..data_models.dependencies import VulnerableDependencies +from ..data_models.info import AgentMorpheusInfo +from ..data_models.input import AgentMorpheusInput +from ..data_models.output import AgentIntermediateStep +from ..data_models.output import AgentMorpheusEngineOutput +from ..data_models.output import AgentMorpheusOutput +from ..data_models.output import ChecklistItemOutput +from ..data_models.output import JustificationOutput + +logger = logging.getLogger(__name__) + + +def _parse_checklist_item(item: dict) -> ChecklistItemOutput: + """ + Parse checklist item dict into a ChecklistItemOutput object. + """ + + # If intermediate steps are available, parse them to a list of AgentIntermediateStep objects + if item["intermediate_steps"] is not None: + intermediate_steps = [] + for step in item["intermediate_steps"]: + + intermediate_step = AgentIntermediateStep.model_validate(step) + + # Try reading the JSON-serialized output string back to structured data + assert isinstance(intermediate_step.tool_output, str) == True, \ + f"Expecting tool output to be a string but got {type(intermediate_step.tool_output)}." + try: + intermediate_step.tool_output = json.loads(intermediate_step.tool_output) + except Exception: + logger.warning("Error deserializing tool output JSON, leaving as string: %s", + f"{intermediate_step.tool_output[:100]}...") + + intermediate_steps.append(intermediate_step) + else: + intermediate_steps = item["intermediate_steps"] + + return ChecklistItemOutput(input=item["question"], response=item["response"], intermediate_steps=intermediate_steps) + + +def _parse_agent_morpheus_engine_output(row: dict) -> AgentMorpheusEngineOutput: + """ + Parse the output row for a single vulnerability into an AgentMorpheusEngineOutput object. + """ + # Convert list of checklist item dicts to list of ChecklistItemOutput objects + checklist_output = [_parse_checklist_item(item) for item in row['checklist']] + + # Combine justification model outputs into a single JustificationOutput object + justification_output = JustificationOutput(label=row['justification_label'], + reason=row['justification'], + status=row['affected_status']) + + return AgentMorpheusEngineOutput(vuln_id=row['vuln_id'], + checklist=checklist_output, + summary=row['summary'], + justification=justification_output) + + +def _get_placeholder_output(vuln_id: str) -> AgentMorpheusEngineOutput: + SUMMARY = "The VulnerableDependencyChecker did not find any vulnerable packages or dependencies in the SBOM." + JUSTIFICATION = JustificationOutput(label="code_not_present", + reason="No vulnerable packages or dependencies were detected in the SBOM.", + status="FALSE") + return AgentMorpheusEngineOutput( + vuln_id=vuln_id, + checklist=[ + ChecklistItemOutput( + input="Check SBOM and dependencies for vulnerability.", + response= + "The VulnerableDependencyChecker did not find any vulnerable packages or dependencies in the SBOM.", + intermediate_steps=None) + ], + summary=SUMMARY, + justification=JUSTIFICATION) + + +@stage +def convert_to_output_object(message: ControlMessage) -> AgentMorpheusOutput: + """ + Takes a ControlMessage containing input, info, and output data and returns an AgentMorpheusOutput object. + """ + + # Pull input and info objects from control message metadata + input: AgentMorpheusInput = message.get_metadata("input") + intel: list[CveIntel] = message.get_metadata("info.intel") + vdb: AgentMorpheusInfo.VdbPaths = message.get_metadata("info.vdb") + sbom: AgentMorpheusInfo.SBOMInfo = message.get_metadata("info.sbom") + vulnerable_dependencies: list[VulnerableDependencies] = message.get_metadata("info.vulnerable_dependencies") + + filtered_vulns = [ + vuln_dep.vuln_id for vuln_dep in vulnerable_dependencies if len(vuln_dep.vulnerable_sbom_packages) == 0 + ] + + # Extract LLMEngine output from message df to dict of {vuln_id: row} + with message.payload().mutable_dataframe() as df: + df2 = df.set_index("vuln_id", drop=False) + llm_engine_output = df2.to_dict(orient="index") + + input_vuln_ids = [vuln.vuln_id for vuln in input.scan.vulns] + + # For each vuln_id, get LLMEngine output if it exists + # or create placeholder output if it had no vulnerable dependencies and skipped the LLMEngine + output: list[AgentMorpheusEngineOutput] = [] + for vuln_id in input_vuln_ids: + + if vuln_id in llm_engine_output: + output.append(_parse_agent_morpheus_engine_output(llm_engine_output[vuln_id])) + + elif vuln_id in filtered_vulns: + output.append(_get_placeholder_output(vuln_id)) + + else: + assert False, "CVE has vulnerable dependencies but there is no LLMEngine output." + + for out in output: + logger.info(f"Vulnerability '{out.vuln_id}' affected status: {out.justification.status}. " + f"Label: {out.justification.label}") + + # Make intermediate info object + info = AgentMorpheusInfo(vdb=vdb, intel=intel, sbom=sbom, vulnerable_dependencies=vulnerable_dependencies) + + # Return final output object + return AgentMorpheusOutput(input=input, info=info, output=output) diff --git a/src/cve/stages/pydantic_http_stage.py b/src/cve/stages/pydantic_http_stage.py new file mode 100644 index 0000000..a800b7b --- /dev/null +++ b/src/cve/stages/pydantic_http_stage.py @@ -0,0 +1,233 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os +import queue +import time +import typing +from http import HTTPStatus + +import json5 +import mrc +from pydantic import BaseModel + +from morpheus.common import FiberQueue +from morpheus.common import HttpEndpoint +from morpheus.config import Config +from morpheus.pipeline.preallocator_mixin import PreallocatorMixin +from morpheus.pipeline.single_output_source import SingleOutputSource +from morpheus.pipeline.stage_schema import StageSchema +from morpheus.stages.input.http_server_source_stage import SUPPORTED_METHODS +from morpheus.utils.http_utils import HTTPMethod +from morpheus.utils.http_utils import HttpParseResponse +from morpheus.utils.http_utils import MimeTypes +from morpheus.utils.producer_consumer_queue import Closed + +logger = logging.getLogger(f"morpheus.{__name__}") + + +class PydanticHttpStage(PreallocatorMixin, SingleOutputSource): + """ + Source stage that starts an HTTP server and listens for incoming requests on a specified endpoint. + + Parameters + ---------- + config : `morpheus.config.Config` + Pipeline configuration instance. + bind_address : str, default "127.0.0.1" + The address to bind the HTTP server to. + port : int, default 8080 + The port to bind the HTTP server to. + endpoint : str, default "/" + The endpoint to listen for requests on. + method : `morpheus.utils.http_utils.HTTPMethod`, optional, case_sensitive = False + HTTP method to listen for. Valid values are "POST" and "PUT". + accept_status: `http.HTTPStatus`, default 201, optional + The HTTP status code to return when a request is accepted. Valid values must be in the 2xx range. + sleep_time : float, default 0.1 + Amount of time in seconds to sleep if the request queue is empty. + queue_timeout : int, default 5 + Maximum amount of time in seconds to wait for a request to be added to the queue before rejecting requests. + max_queue_size : int, default None + Maximum number of requests to queue before rejecting requests. If `None` then `config.edge_buffer_size` will be + used. + num_server_threads : int, default None + Number of threads to use for the HTTP server. If `None` then `os.cpu_count()` will be used. + max_payload_size : int, default 10 + The maximum size in megabytes of the payload that the server will accept in a single request. + request_timeout_secs : int, default 30 + The maximum amount of time in seconds for any given request. + lines : bool, default False + If False, the HTTP server will expect each request to be a JSON array of objects. If True, the HTTP server will + expect each request to be a JSON object per line. + stop_after : int, default 0 + Stops ingesting after processing `stop_after` requests. Useful for testing. Disabled if `0` + payload_to_df_fn : callable, default None + A callable that takes the HTTP payload string as the first argument and the `lines` parameter is passed in as + the second argument and returns a cudf.DataFrame. When supplied, the C++ implementation of this stage is + disabled, and the Python impl is used. + """ + + def __init__(self, + config: Config, + *, + bind_address: str = "127.0.0.1", + port: int = 8080, + endpoint: str = "/message", + method: HTTPMethod = HTTPMethod.POST, + accept_status: HTTPStatus = HTTPStatus.CREATED, + sleep_time: float = 0.1, + queue_timeout: int = 5, + max_queue_size: int = None, + num_server_threads: int = None, + max_payload_size: int = 10, + request_timeout_secs: int = 30, + lines: bool = False, + stop_after: int = 0, + input_schema: type[BaseModel]): + super().__init__(config) + self._bind_address = bind_address + self._port = port + self._endpoint = endpoint + self._method = method + self._accept_status = accept_status + self._sleep_time = sleep_time + self._queue_timeout = queue_timeout + self._max_queue_size = max_queue_size or config.edge_buffer_size + self._num_server_threads = num_server_threads or os.cpu_count() + self._max_payload_size_bytes = max_payload_size * 1024 * 1024 + self._request_timeout_secs = request_timeout_secs + self._lines = lines + self._stop_after = stop_after + + # These are only used when C++ mode is disabled + self._queue: FiberQueue = None + self._processing = False + self._messages_processed = 0 + self._input_schema = input_schema + self._http_server = None + + if method not in SUPPORTED_METHODS: + raise ValueError(f"Unsupported method: {method}") + + if accept_status.value < 200 or accept_status.value > 299: + raise ValueError(f"Invalid accept_status: {accept_status}") + + @property + def name(self) -> str: + """Unique name of the stage.""" + return "from-cve-http" + + def supports_cpp_node(self) -> bool: + """Disabling the C++ impl allows for the `source_code_repos` keys to be preserved""" + return False + + def compute_schema(self, schema: StageSchema): + schema.output_schema.set_type(self._input_schema) + + def stop(self): + """ + Performs cleanup steps when pipeline is stopped. + """ + logger.debug("Stopping %s", self.name) + self._processing = False + # Indicate we need to stop + if self._http_server is not None: + self._http_server.stop() + + return super().stop() + + def _parse_payload(self, payload: str) -> HttpParseResponse: + try: + payload_dict = json5.loads(payload) + + message = self._input_schema.model_validate(payload_dict) + + except Exception as e: + err_msg = f"Error occurred converting HTTP payload to {self._input_schema}: {e}" + logger.error("%s", err_msg, exc_info=True) + return HttpParseResponse(status_code=HTTPStatus.BAD_REQUEST.value, + content_type=MimeTypes.TEXT.value, + body=err_msg) + + try: + self._queue.put(message, block=True, timeout=self._queue_timeout) + return HttpParseResponse(status_code=self._accept_status.value, content_type=MimeTypes.TEXT.value, body="") + + except (queue.Full, Closed) as e: + err_msg = "HTTP payload queue is " + if isinstance(e, queue.Full): + err_msg += "full" + else: + err_msg += "closed" + logger.error(err_msg) + return HttpParseResponse(status_code=HTTPStatus.SERVICE_UNAVAILABLE.value, + content_type=MimeTypes.TEXT.value, + body=err_msg) + + except Exception as e: + err_msg = "Error occurred while pushing payload to queue" + logger.error("%s: %s", err_msg, e) + return HttpParseResponse(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, + content_type=MimeTypes.TEXT.value, + body=err_msg) + + def _generate_frames(self, subscription: mrc.Subscription) -> typing.Iterator[BaseModel]: + from morpheus.common import FiberQueue + from morpheus.common import HttpServer + + endpoints = [HttpEndpoint(py_parse_fn=self._parse_payload, url=self._endpoint, method=self._method.value)] + with (FiberQueue(self._max_queue_size) as self._queue, + HttpServer(bind_address=self._bind_address, + port=self._port, + endpoints=endpoints, + num_threads=self._num_server_threads, + max_payload_size=self._max_payload_size_bytes, + request_timeout=self._request_timeout_secs) as self._http_server): + + self._processing = True + while self._processing: + # Read as many messages as we can from the queue if it's empty check to see if we should be shutting + # down. It is important that any messages we received that are in the queue are processed before we + # shutdown since we already returned an OK response to the client. + message: self._input_schema | None = None + try: + # Intentionally not using self._queue_timeout here since that value is rather high + message = self._queue.get(block=False, timeout=0.1) + except queue.Empty: + if (not self._http_server.is_running() or self.is_stop_requested() + or not subscription.is_subscribed()): + self._processing = False + else: + time.sleep(self._sleep_time) + except Closed: + logger.error("Queue closed unexpectedly, shutting down") + self._processing = False + + if message is not None: + yield message + self._messages_processed += 1 + + if self._stop_after > 0 and self._messages_processed >= self._stop_after: + logger.info("Processed %d messages; stop processing", self._messages_processed) + self._processing = False + + def _build_source(self, builder: mrc.Builder) -> mrc.SegmentObject: + + node = builder.make_source(self.unique_name, self._generate_frames) + + return node diff --git a/src/cve/stages/write_pydantic_to_file.py b/src/cve/stages/write_pydantic_to_file.py new file mode 100644 index 0000000..8deef01 --- /dev/null +++ b/src/cve/stages/write_pydantic_to_file.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os +import typing +from enum import Enum +from pathlib import Path + +import mrc +import mrc.core.operators as ops +from pydantic import BaseModel + +from morpheus.config import Config +from morpheus.pipeline.pass_thru_type_mixin import PassThruTypeMixin +from morpheus.pipeline.single_port_stage import SinglePortStage + +logger = logging.getLogger(__name__) + + +class WriteFileMode(Enum): + """ + Enum for the write file mode. + """ + CREATE = "x" # Create a new file and open it for writing. If the file already exists, the operation will fail. + APPEND = "a" # Open the file and append to it. If the file does not exist, it will be created. + OVERWRITE = "w" # Open the file and write to it. If the file exists, it will be overwritten. + + +class WritePydanticToFile(PassThruTypeMixin, SinglePortStage): + """ + Writes pydantic model to file, used in place of `WriteToFileStage` which requires MessageMeta. + + Parameters + ---------- + c : Config + The configuration object. + filename : str + The name of the file to write to. + mode : WriteFileMode, optional + The write file mode, by default WriteFileMode.CREATE. + """ + + def __init__(self, c: Config, filename: str, mode: WriteFileMode = WriteFileMode.CREATE): + + super().__init__(c) + + self._filename = filename + self._mode = mode + + if (os.path.exists(self._filename)): + if (self._mode == WriteFileMode.CREATE): + raise FileExistsError( + f"Cannot write model output to '{self._filename}'. File exists and overwrite = False") + elif (self._mode == WriteFileMode.APPEND): + logger.warning(f"Appending to existing file: {self._filename}") + elif (self._mode == WriteFileMode.OVERWRITE): + logger.warning(f"Overwriting existing file: {self._filename}") + os.remove(self._filename) + else: + assert False, f"Invalid mode: {self._mode}" + else: + # Ensure our directory exists + os.makedirs(os.path.realpath(os.path.dirname(self._filename)), exist_ok=True) + + @property + def name(self) -> str: + return "write-pydantic-to-file" + + def accepted_types(self) -> typing.Tuple: + """ + Accepted input types for this stage are returned. + + Returns + ------- + typing.Tuple + Accepted input types. + + """ + return (BaseModel, ) + + def supports_cpp_node(self): + return False + + def _on_data(self, model: BaseModel) -> BaseModel: + # Validate that model is a Pydantic object + if not isinstance(model, BaseModel): + raise TypeError(f"Expected output to be a Pydantic BaseModel but received: {type(model)}") + + file_path = Path(self._filename) + + # Write file depending on setting for overwrite + model_json = model.model_dump_json(by_alias=True) + + file_path.parent.mkdir(parents=True, exist_ok=True) + + with open(file_path, 'a') as f: + f.write(model_json) + + # Make stage a pass-through + return model + + def _build_single(self, builder: mrc.Builder, input_node: mrc.SegmentObject) -> mrc.SegmentObject: + + node = builder.make_node(self.unique_name, ops.map(self._on_data)) + + builder.make_edge(input_node, node) + + return node diff --git a/src/cve/utils/__init__.py b/src/cve/utils/__init__.py new file mode 100644 index 0000000..28d3a3a --- /dev/null +++ b/src/cve/utils/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/cve/utils/async_http_utils.py b/src/cve/utils/async_http_utils.py new file mode 100644 index 0000000..93fc349 --- /dev/null +++ b/src/cve/utils/async_http_utils.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import asyncio +import logging +import time +import typing +from contextlib import asynccontextmanager + +import aiohttp + +logger = logging.getLogger(f"morpheus.{__name__}") + + +@asynccontextmanager +async def request_with_retry(session: aiohttp.ClientSession, + request_kwargs: dict, + max_retries: int = 10, + sleep_time: float = 0.1, + respect_retry_after_header: bool = True, + log_on_error=True) -> typing.AsyncIterator[aiohttp.ClientResponse]: + """ + Async version of `morpheus.utils.http_utils.request_with_retry` + """ + assert not request_kwargs.get('raise_for_status'), "raise_for_status is cincompatible with `request_with_retry`" + try_count = 0 + done = False + while try_count <= max_retries and not done: + response = None + response_headers = {} + try: + async with session.request(**request_kwargs) as response: + response_headers = response.headers + response.raise_for_status() + yield response + done = True + except Exception as e: + try_count += 1 + + if try_count >= max_retries: + if (log_on_error): + logger.error("Failed requesting %s after %d retries: %s", request_kwargs['url'], max_retries, e) + raise e + + actual_sleep_time = (2**(try_count - 1)) * sleep_time + + if respect_retry_after_header and 'Retry-After' in response_headers: + actual_sleep_time = max(int(response_headers["Retry-After"]), actual_sleep_time) + elif respect_retry_after_header and 'X-RateLimit-Reset' in response_headers: + actual_sleep_time = max(int(response_headers["X-RateLimit-Reset"]) - time.time(), actual_sleep_time) + + logger.warning("Error requesting [%d/%d]: (Retry %.1f sec) %s: %s", + try_count, + max_retries, + actual_sleep_time, + request_kwargs['url'], + e) + + await asyncio.sleep(actual_sleep_time) + + +_T = typing.TypeVar('_T') +_P = typing.ParamSpec('_P') + + +def retry_async(exception_types: type[BaseException] | tuple[type[BaseException], ...] = Exception): + """ + Retries an async function with exponential backoff + + Parameters + ---------- + exception_types : type[BaseException] | tuple[type[BaseException], ...], optional + The types of exceptions to trigger a retry, by default Exception + """ + import tenacity + + def inner(func: typing.Callable[_P, typing.Awaitable[_T]]) -> typing.Callable[_P, typing.Awaitable[_T]]: + + @tenacity.retry(wait=tenacity.wait_exponential_jitter(0.1), + stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type(exception_types), + reraise=True) + async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T: + return await func(*args, **kwargs) + + return wrapper + + return inner diff --git a/src/cve/utils/clients/__init__.py b/src/cve/utils/clients/__init__.py new file mode 100644 index 0000000..28d3a3a --- /dev/null +++ b/src/cve/utils/clients/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/cve/utils/clients/first_client.py b/src/cve/utils/clients/first_client.py new file mode 100644 index 0000000..7e62560 --- /dev/null +++ b/src/cve/utils/clients/first_client.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os + +import aiohttp + +from ...data_models.cve_intel import CveIntelEpss +from ...utils.url_utils import url_join +from .intel_client import IntelClient + +logger = logging.getLogger(f"morpheus.{__name__}") + + +class FirstClient(IntelClient): + """ + Async client for First EPSS API + """ + + def __init__(self, + *, + base_url: str | None = None, + session: aiohttp.ClientSession | None = None, + retry_count: int = 10, + sleep_time: float = 0.1, + respect_retry_after_header: bool = True): + + super().__init__(session=session, + base_url=base_url or os.environ.get('FIRST_BASE_URL'), + retry_count=retry_count, + sleep_time=sleep_time, + respect_retry_after_header=respect_retry_after_header) + + self._headers = {'Accept': 'application/json'} + + @classmethod + def default_base_url(cls) -> str: + return "https://api.first.org/data/v1" + + async def get_intel_dict(self, cve_id: str, date: None | str) -> dict: + params = {'cve': cve_id, 'date': date} if date else {'cve': cve_id} + response_json = await self.request(method='GET', + url=url_join(self.base_url, "epss"), + params=params, + headers=self._headers) + + data = response_json.get('data', []) + + if (len(data) > 0): + return data[0] + + return {} + + async def get_intel(self, cve_id: str, date: None | str = None) -> CveIntelEpss: + intel_dict = await self.get_intel_dict(cve_id, date) + + return CveIntelEpss.model_validate(intel_dict) diff --git a/src/cve/utils/clients/ghsa_client.py b/src/cve/utils/clients/ghsa_client.py new file mode 100644 index 0000000..05997b0 --- /dev/null +++ b/src/cve/utils/clients/ghsa_client.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os + +import aiohttp + +from ...data_models.cve_intel import CveIntelGhsa +from ...utils.url_utils import url_join +from .intel_client import IntelClient + +logger = logging.getLogger(f"morpheus.{__name__}") + + +class GHSAClient(IntelClient): + """ + Async client for GitHub Security Advisory API + + While not strictly required, obtaining an API key is recommended, to obtain one refer: + https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens + """ + + def __init__(self, + *, + base_url: str | None = None, + api_key: str | None = None, + session: aiohttp.ClientSession | None = None, + retry_count: int = 10, + sleep_time: float = 0.1, + respect_retry_after_header: bool = True): + super().__init__(session=session, + base_url=base_url or os.environ.get('GHSA_BASE_URL'), + retry_count=retry_count, + sleep_time=sleep_time, + respect_retry_after_header=respect_retry_after_header) + + self._api_key = api_key or os.environ.get('GHSA_API_KEY', None) + + self._headers = {'Accept': 'application/vnd.github+json', "X-GitHub-Api-Version": "2022-11-28"} + + if self._api_key is not None: + self._headers['Authorization'] = f"Bearer {self._api_key}" + + @classmethod + def default_base_url(cls) -> str: + return "https://api.github.com" + + async def get_intel_dict(self, vuln_id: str) -> dict: + + params = {} + + if (vuln_id.startswith("GHSA-")): + params["ghsa_id"] = vuln_id + else: + params["cve_id"] = vuln_id + + response_json = await self.request(method="GET", + url=url_join(self.base_url, "advisories"), + params=params, + headers=self._headers) + + if (isinstance(response_json, list)): + + if (len(response_json) != 1): + raise ValueError(f"Expected 1 GHSA entry for {vuln_id}, got {len(response_json)}") + + return response_json[0] + + elif (isinstance(response_json, dict)): + + return response_json + + raise ValueError(f"Unexpected response type {type(response_json)}") + + async def get_intel(self, vuln_id: str) -> CveIntelGhsa: + + intel_dict = await self.get_intel_dict(vuln_id) + + return CveIntelGhsa.model_validate(intel_dict) diff --git a/src/cve/utils/clients/intel_client.py b/src/cve/utils/clients/intel_client.py new file mode 100644 index 0000000..5d52589 --- /dev/null +++ b/src/cve/utils/clients/intel_client.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import typing +from abc import abstractmethod + +import aiohttp + +from ...utils.async_http_utils import request_with_retry + + +class IntelClient: + + def __init__(self, + *, + session: aiohttp.ClientSession | None, + base_url: str | None = None, + retry_count: int = 10, + sleep_time: float = 0.1, + respect_retry_after_header: bool = True): + + if (session is None): + session = aiohttp.ClientSession() + + if (base_url is None): + base_url = self.default_base_url() + + self._session = session + + self._base_url = base_url + self._retry_count = retry_count + self._sleep_time = sleep_time + self._respect_retry_after_header = respect_retry_after_header + + @classmethod + @abstractmethod + def default_base_url(cls) -> str: + raise NotImplementedError + + @property + def base_url(self) -> str: + return self._base_url + + def is_default_base_url(self) -> bool: + """ + Check if the base URL is the default one + + Returns + ------- + bool + True if the base URL is the default one + """ + return self.base_url == self.default_base_url() + + async def request(self, + method: typing.Literal["GET", "POST", "PUT", "PATCH"], + url: str, + params: dict[str, str] | None = None, + headers: dict[str, str] | None = None, + log_on_error=True, + **kwargs): + + async with request_with_retry(session=self._session, + request_kwargs={ + 'method': method, 'url': url, "params": params, "headers": headers, **kwargs + }, + max_retries=self._retry_count, + sleep_time=self._sleep_time, + respect_retry_after_header=self._respect_retry_after_header, + log_on_error=log_on_error) as response: + + return await response.json() diff --git a/src/cve/utils/clients/nvd_client.py b/src/cve/utils/clients/nvd_client.py new file mode 100644 index 0000000..63409ab --- /dev/null +++ b/src/cve/utils/clients/nvd_client.py @@ -0,0 +1,290 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os +import re + +import aiohttp +from bs4 import BeautifulSoup + +from ...data_models.cve_intel import CveIntelNvd +from ..async_http_utils import request_with_retry +from ..intel_utils import parse +from ..url_utils import url_join +from .intel_client import IntelClient + +logger = logging.getLogger(f"morpheus.{__name__}") + + +class NVDClient(IntelClient): + """ + Async client for NIST's NVD API + + While not strictly required, obtaining an API key is recommended, to obtain one refer: + https://nvd.nist.gov/developers/start-here + """ + + CWE_NAME_RE = re.compile(r'^.*?-\s*') + VENDOR_HREF_RE = re.compile('/vendor/') + CVE_DETAILS_URL = "https://www.cvedetails.com" + CWE_DETAILS_URL = "http://cwe.mitre.org" + + def __init__(self, + *, + base_url: str | None = None, + api_key: str | None = None, + lang_code: str = 'en', + session: aiohttp.ClientSession | None = None, + retry_count: int = 10, + sleep_time: float = 0.1, + respect_retry_after_header: bool = True): + + super().__init__(session=session, + base_url=base_url or os.environ.get('NVD_BASE_URL'), + retry_count=retry_count, + sleep_time=sleep_time, + respect_retry_after_header=respect_retry_after_header) + + self._api_key = api_key or os.environ.get('NVD_API_KEY', None) + + self._cve_details_url_template = url_join(os.environ.get('CVE_DETAILS_BASE_URL', self.CVE_DETAILS_URL), + "cve", + "{CVE_ID}") + self._cwe_details_url_template = url_join(os.environ.get('CWE_DETAILS_BASE_URL', self.CWE_DETAILS_URL), + "data/definitions", + "{CWE_ID}.html") + + self._lang_code = lang_code + + self._headers = {'Content-Type': 'application/json'} + + if self._api_key is not None: + self._headers['apiKey'] = self._api_key + + @classmethod + def default_base_url(cls) -> str: + return "https://services.nvd.nist.gov/rest" + + async def _get_soup(self, url: str) -> BeautifulSoup: + + async with request_with_retry(session=self._session, + request_kwargs={ + 'method': 'GET', + 'url': url, + "skip_auto_headers": {"User-Agent"}, + }, + max_retries=self._retry_count) as response: + return BeautifulSoup(await response.text(), 'html.parser') + + def _get_cvss_vector_from_metric(self, metrics: dict, metric_version: str) -> str | None: + versioned_metrics = metrics[metric_version] + for metric_type in ('primary', 'secondary'): + for metric in versioned_metrics: + if metric['type'].lower() == metric_type: + return metric['cvssData']['vectorString'] + + def _get_cwe(self, cwes: list[dict]) -> str | None: + for cwe_type in ('primary', 'secondary'): + for cwe in cwes: + if cwe['type'].lower() == cwe_type: + cwe_descriptions = cwe['description'] + for cwe_description in cwe_descriptions: + if cwe_description['lang'] == self._lang_code: + return cwe_description['value'] + + async def _get_cwe_elements(self, cve_obj: dict) -> dict: + """ + Asynchronously extract CWE (Common Weakness Enumeration) elements from the CVE object, and retreive details for + those CWEs. + """ + # Get CWE name + cwe_id = None + weaknesses = cve_obj.get('weaknesses', []) + cwe_id = self._get_cwe(weaknesses) + cwe_link = None + cwe_name = None + cwe_description = None + cwe_extended_description = None + if cwe_id is not None: + if cwe_id.startswith('CWE-'): + cwe_id = cwe_id.replace('CWE-', '', 1) + + if cwe_id.isnumeric(): + cwe_link = self._cwe_details_url_template.format(CWE_ID=cwe_id) + + if cwe_link is not None: + soup = await self._get_soup(cwe_link) + + if soup is not None: + title_tag = soup.find('title') + if title_tag: + cwe_name = title_tag.string.strip() + cwe_name = self.CWE_NAME_RE.sub('', cwe_name).strip() + description_div = soup.find('div', id='Description') + if description_div: + cwe_description = description_div.find('div', class_='indent').text.strip() + + extended_description_div = soup.find('div', id='Extended_Description') + if extended_description_div: + cwe_extended_description = extended_description_div.find('div', class_='indent').text.strip() + + return { + "cwe_name": cwe_name, + "cwe_description": cwe_description, + "cwe_extended_description": cwe_extended_description, + } + + def _parse_nvd_cvss_vector(self, cve_obj: dict) -> str | None: + """ + Extract the CVSS vector from a CVE json object. + + Parameters + ---------- + cve_obj : dict + The cve sub-dictionary of the JSON document. + + Returns + ------- + str or None + The CVSS vector string, if found. Otherwise, None. + """ + # metrics is optional https://nvd.nist.gov/developers/vulnerabilities + metrics = cve_obj.get('metrics') + if metrics is not None: + # Attempt to find the CVSS vector in order of preference + for metric_version in ('cvssMetricV31', 'cvssMetricV30'): + try: + cvss_vector = self._get_cvss_vector_from_metric(metrics, metric_version) + if cvss_vector is not None: + return cvss_vector + except KeyError: + continue + + async def _get_vendor_names(self, cve_details_url: str) -> list[str] | None: + """ + Asynchronously retrieve vendor names associated with the CVE from the CVE Details page. + + Parameters + ---------- + session : aiohttp.ClientSession + The session used to make HTTP requests. + cve_details_url : str + The URL to the CVE details page. + + Returns + ------- + list of str or None + A list of vendor names associated with the CVE, if found. Otherwise, None. + """ + + try: + # This website has been blocking non-browser user agents with a 403, don't spend time retrying + # as Rachel has code to replace it + soup = await self._get_soup(cve_details_url) + + # Find all the vendor names within the tags + if soup is not None: + vendor_tags = soup.find_all('a', href=self.VENDOR_HREF_RE) + if vendor_tags: + # Extract the unique text from the vendor tags + # Make sure to return the sorted list for consistency + return sorted(set(tag.text.strip() for tag in vendor_tags)) + + except Exception as e: + logger.error("Error fetching vendor names for %s : %s", cve_details_url, e) + + return None + + # if vuln_feed is not None and 'ubuntu' in vuln_feed: + # try: + # soup = await self._get_soup(session, vuln_url, max_retries=max_retries) + + # security_team_header = soup.find('h2', string='From the Ubuntu Security Team') + # if security_team_header is not None: + # security_team_note = security_team_header.find_next_sibling('p') + # ubuntu_security_note = security_team_note.get_text(strip=True) if security_team_note else None + # else: + # ubuntu_security_note = None + + # # Extracting the priority reason + # priority_reason_pre = soup.find('pre') + # ubuntu_priority_reason = priority_reason_pre.text.strip() if priority_reason_pre else None + + # # Extracting the priority level + # priority_level_p = soup.find('p', class_='p-heading-icon__title u-no-margin--bottom p-heading--4') + # ubuntu_priority_level = priority_level_p.text.strip() if priority_level_p else None + + # ubuntu_intel.ubuntu_security_note = ubuntu_security_note + # ubuntu_intel.ubuntu_priority_reason = ubuntu_priority_reason + # ubuntu_intel.ubuntu_priority_level = ubuntu_priority_level + # except Exception as e: + # logger.error("Error fetching Ubuntu security information for %s : %s", vuln_url, e) + + # return ubuntu_intel + + async def get_intel_dict(self, cve_id: str) -> dict: + + response = await self.request(method="GET", + url=url_join(self.base_url, 'json/cves/2.0'), + params={'cveId': cve_id}, + headers=self._headers) + + # Get the vulnerabilities from the dict + vulns = response.get("vulnerabilities", []) + + if (len(vulns) == 0): + raise ValueError(f"Could not find CVE entry for {cve_id}") + + if (len(vulns) > 1): + logger.warning(f"Found multiple CVE entries for {cve_id}, using the first one") + + return vulns[0] + + async def get_intel(self, cve_id: str) -> CveIntelNvd: + """ + Get the CVE Intel object for the given CVE ID + + Args: + cve_id (str): The CVE ID to get the Intel for + + Returns: + CveIntelNvd: The CVE Intel object + """ + intel_dict = await self.get_intel_dict(cve_id) + + cve_vuln: dict = intel_dict.get("cve", {}) + + cve_description = "\n".join( + desc.get('value', '') for desc in cve_vuln.get('descriptions', []) if desc.get('lang') == self._lang_code) + + cve_configurations = parse(cve_vuln.get('configurations', [])) + cvss_vector = self._parse_nvd_cvss_vector(cve_vuln) + cwe_elements = await self._get_cwe_elements(cve_vuln) + + cve_details_url = self._cve_details_url_template.format(CVE_ID=cve_id) + vendor_names = await self._get_vendor_names(cve_details_url) + + intel = CveIntelNvd(cve_id=cve_id, + cve_description=cve_description, + cvss_vector=cvss_vector, + cwe_name=cwe_elements["cwe_name"], + cwe_description=cwe_elements["cwe_description"], + cwe_extended_description=cwe_elements["cwe_extended_description"], + configurations=cve_configurations, + vendor_names=vendor_names) + + return intel diff --git a/src/cve/utils/clients/rhsa_client.py b/src/cve/utils/clients/rhsa_client.py new file mode 100644 index 0000000..0a2f888 --- /dev/null +++ b/src/cve/utils/clients/rhsa_client.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os + +import aiohttp + +from ...data_models.cve_intel import CveIntelRhsa +from ..url_utils import url_join +from .intel_client import IntelClient + +logger = logging.getLogger(f"morpheus.{__name__}") + + +class RHSAClient(IntelClient): + """ + Async client for Red Hat Security Advisory API + """ + + def __init__(self, + *, + base_url: str | None = None, + session: aiohttp.ClientSession | None = None, + retry_count: int = 10, + sleep_time: float = 0.1, + respect_retry_after_header: bool = True): + + super().__init__(session=session, + base_url=base_url or os.environ.get('RHSA_BASE_URL'), + retry_count=retry_count, + sleep_time=sleep_time, + respect_retry_after_header=respect_retry_after_header) + + @classmethod + def default_base_url(cls) -> str: + return "https://access.redhat.com/hydra/rest" + + async def get_intel_dict(self, cve_id: str) -> dict: + response = await self.request(method='GET', + url=url_join(self.base_url, "securitydata", "cve.json"), + params={"ids": cve_id}) + + vulns = response + + if (len(vulns) == 0): + logger.info(f"No Red Hat CVE entry found for {cve_id}") + return {} + + if (len(vulns) > 1): + logger.warning(f"Found multiple Red Hat CVE entries for {cve_id}, using the first one") + + response_full = await self.request(method='GET', + url=url_join(self.base_url, "securitydata", f"cve/{cve_id}.json")) + + return response_full + + async def get_intel(self, cve_id: str) -> CveIntelRhsa: + intel_dict = await self.get_intel_dict(cve_id) + + return CveIntelRhsa.model_validate(intel_dict) diff --git a/src/cve/utils/clients/ubuntu_client.py b/src/cve/utils/clients/ubuntu_client.py new file mode 100644 index 0000000..4a9f319 --- /dev/null +++ b/src/cve/utils/clients/ubuntu_client.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import aiohttp + +from ...data_models.cve_intel import CveIntelUbuntu +from ..url_utils import url_join +from .intel_client import IntelClient +from .rhsa_client import logger + + +class UbuntuClient(IntelClient): + """ + Async client for Ubuntu Security Advisory API + """ + + def __init__(self, + *, + base_url: str | None = None, + session: aiohttp.ClientSession | None = None, + retry_count: int = 10, + sleep_time: float = 0.1, + respect_retry_after_header: bool = True): + + super().__init__(session=session, + base_url=base_url or os.environ.get('UBUNTU_BASE_URL'), + retry_count=retry_count, + sleep_time=sleep_time, + respect_retry_after_header=respect_retry_after_header) + + @classmethod + def default_base_url(cls) -> str: + return "https://ubuntu.com" + + async def get_intel_dict(self, cve_id: str) -> dict: + response = await self.request(method='GET', + url=url_join(self.base_url, "security", "cves.json"), + params={"q": cve_id}) + + # Get the vulnerabilities from the dict + vulns = response.get("cves", []) + + if (len(vulns) == 0): + logger.info(f"No Ubuntu CVE entry found for {cve_id}") + return {} + + if (len(vulns) > 1): + logger.warning(f"Found multiple Ubuntu CVE entries for {cve_id}, using the first one") + + return vulns[0] + + async def get_intel(self, cve_id: str) -> CveIntelUbuntu: + intel_dict = await self.get_intel_dict(cve_id) + + return CveIntelUbuntu.model_validate(intel_dict) diff --git a/src/cve/utils/code_searcher.py b/src/cve/utils/code_searcher.py new file mode 100644 index 0000000..1607377 --- /dev/null +++ b/src/cve/utils/code_searcher.py @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +from collections.abc import Iterable + +from langchain.docstore.document import Document +from langchain.document_loaders.text import TextLoader +from langchain_community.retrievers import BM25Retriever + + +class LangchainCodeSearcher: + """ + Searcher for code using langchain document loader/chunker + + Parameters + ---------- + documents : list[Document] + list of documents to search + k : int, default = 5 + Number of documents to retrieve when performing a search + rank_documents : bool, default = False + When True, and the number of retrieved documents is greater than k, search results are ranked using the bm25 + method. + """ + + def __init__(self, documents: list[Document], k: int = 5, rank_documents: bool = False): + self.documents = documents + self.k = k + self.rank_documents = rank_documents + + @classmethod + def load_from_repo(cls, + repo_path: str, + include_extensions: Iterable[str], + k: int = 5, + rank_documents: bool = False): + """ + Loads all files from a repo into a list of documents. + + Parameters + ---------- + repo_path : str + path to the repository + include_extensions : list[str] + list of file extensions to include + k : int, default = 5 + number of documents to retrieve when performing a search + rank_documents : bool, default = False + When True, and the number of retrieved documents is greater than k, search results are ranked using the bm25 + method. + """ + include_extensions = frozenset(include_extensions) + + documents = [] + for root, _, files in os.walk(repo_path): + for file in files: + ext = os.path.splitext(file)[1].lstrip('.') + if ext in include_extensions: + file_path = os.path.join(root, file) + loader = TextLoader(file_path, encoding="utf-8") + documents.extend(loader.load()) + + return cls(documents=documents, k=k, rank_documents=rank_documents) + + def search(self, query: str) -> list[Document]: + """search for code + + Parameters + ---------- + query : str + query to search + + Returns + ------- + list[Document] + list of retrieved documents + """ + + query = re.sub(r"[^\w_.]+", "", query) + pattern = re.compile(re.escape(query), re.IGNORECASE) + results = [] + for doc in self.documents: + if pattern.search(doc.page_content): + results.append(doc) + + # use optional bm25 to rank the retrieved documents + if self.rank_documents and len(results) > self.k: + return BM25Retriever.from_documents(results, k=self.k).invoke(query) + else: + return results[:self.k] diff --git a/src/cve/utils/data_utils.py b/src/cve/utils/data_utils.py new file mode 100644 index 0000000..32399b7 --- /dev/null +++ b/src/cve/utils/data_utils.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +import logging +import typing + +from pydantic import BaseModel as BaseModel +from pydantic.v1 import BaseModel as BaseModelV1 + +logger = logging.getLogger(__name__) + + +def to_serializable_object(obj: typing.Any): + """ + Utility function for converting an object to a serializable one, useful for passing as a default to json.dumps(). + """ + # Convert Pydantic V1 objects + if isinstance(obj, BaseModelV1): + return obj.dict() + + # Convert Pydantic V2 objects + elif isinstance(obj, BaseModel): + return obj.model_dump() + + # Convert all other objects to string + else: + logger.warning("Serializing object with unsupported data type to string. Object: %s", obj) + return str(obj) + + +def to_json(obj: typing.Any) -> str: + """ + Serialize object to JSON string with json.dumps(), using the to_serializable_object function as default. + """ + return json.dumps(obj, default=to_serializable_object) diff --git a/src/cve/utils/document_embedding.py b/src/cve/utils/document_embedding.py new file mode 100644 index 0000000..7008495 --- /dev/null +++ b/src/cve/utils/document_embedding.py @@ -0,0 +1,478 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import copy +import json +import logging +import os +import sys +import time +import typing +from hashlib import sha512 +from pathlib import Path +from pathlib import PurePath + +from langchain.docstore.document import Document +from langchain.document_loaders.generic import GenericLoader +from langchain.document_loaders.parsers.language.language_parser import LanguageParser +from langchain.text_splitter import Language +from langchain.text_splitter import RecursiveCharacterTextSplitter +from langchain.vectorstores.faiss import FAISS +from langchain_community.document_loaders.parsers.language.code_segmenter import CodeSegmenter +from langchain_community.document_loaders.parsers.language.language_parser import LANGUAGE_EXTENSIONS +from langchain_community.document_loaders.parsers.language.language_parser import LANGUAGE_SEGMENTERS +from langchain_core.document_loaders.blob_loaders import Blob + +from ..data_models.input import SourceDocumentsInfo +from .source_code_git_loader import SourceCodeGitLoader + +if typing.TYPE_CHECKING: + from langchain_core.embeddings import Embeddings # pragma: no cover + +PathLike = typing.Union[str, os.PathLike] + +logger = logging.getLogger(f"morpheus.{__name__}") + + +class MultiLanguageRecursiveCharacterTextSplitter(RecursiveCharacterTextSplitter): + """ + A version of langchain's RecursiveCharacterTextSplitter that supports multiple languages. + """ + + def __init__( + self, + keep_separator: bool = True, + **kwargs, + ) -> None: + """Create a new RecursiveCharacterTextSplitter.""" + super().__init__(is_separator_regex=True, keep_separator=keep_separator, **kwargs) + + def _get_separators(self, language: Language) -> list[str]: + try: + return RecursiveCharacterTextSplitter.get_separators_for_language(language) + except ValueError: + return self._separators + + def create_documents(self, texts: list[str], metadatas: list[dict] | None = None) -> list[Document]: + """Create documents from a list of texts.""" + _metadatas = metadatas or [{}] * len(texts) + documents = [] + for i, text in enumerate(texts): + index = 0 + previous_chunk_len = 0 + + # Determine the language of the text from the metadata + language = _metadatas[i].get("language", None) + + for chunk in self._split_text(text, separators=self._get_separators(language)): + metadata = copy.deepcopy(_metadatas[i]) + if self._add_start_index: + offset = index + previous_chunk_len - self._chunk_overlap + index = text.find(chunk, max(0, offset)) + metadata["start_index"] = index + previous_chunk_len = len(chunk) + new_doc = Document(page_content=chunk, metadata=metadata) + documents.append(new_doc) + return documents + + +class ExtendedLanguageParser(LanguageParser): + """ + A version of langchain's LanguageParser that supports extended file extension and language parsing. + """ + + LANGUAGE_EXTENSIONS: dict[str, str] = { + **LANGUAGE_EXTENSIONS, + "h": "c", + "hpp": "cpp", + } + + LANGUAGE_SEGMENTERS: dict[str, type[CodeSegmenter]] = { + **LANGUAGE_SEGMENTERS, + } + + def lazy_parse(self, blob: Blob) -> typing.Iterator[Document]: + try: + code = blob.as_string() + except Exception as e: + logger.warning("Failed to read code for '%s'. Ignoring this file. Error: %s", blob.source, e) + return + + language = self.language or (self.LANGUAGE_EXTENSIONS.get(blob.source.rsplit(".", 1)[-1]) if isinstance( + blob.source, str) else None) + + if language is None: + yield Document( + page_content=code, + metadata={ + "source": blob.source, + }, + ) + return + + if self.parser_threshold >= len(code.splitlines()): + yield Document( + page_content=code, + metadata={ + "source": blob.source, + "language": language, + }, + ) + return + + segmenter = self.LANGUAGE_SEGMENTERS[language](blob.as_string()) + + try: + extracted_functions_classes = segmenter.extract_functions_classes() + + except Exception as e: + + logger.warning("Failed to parse code for '%s'. Ignoring this file. Error: %s", + blob.source, + e, + exc_info=True) + extracted_functions_classes = [] + + # If the code didnt parse, and there are no functions or classes, return the original code + if not segmenter.is_valid() and len(extracted_functions_classes) == 0: + yield Document( + page_content=code, + metadata={ + "source": blob.source, + "language": language, + }, + ) + return + + for functions_classes in extracted_functions_classes: + yield Document( + page_content=functions_classes, + metadata={ + "source": blob.source, + "content_type": "functions_classes", + "language": language, + }, + ) + + try: + simplified_code = segmenter.simplify_code() + # If simplifying the code fails, return the original code + except Exception as e: + logger.warning("Failed to simplify code for '%s'. Returning original code. Error: %s", + blob.source, + e, + exc_info=True) + yield Document( + page_content=code, + metadata={ + "source": blob.source, + "language": language, + }, + ) + else: + yield Document( + page_content=simplified_code, + metadata={ + "source": blob.source, + "content_type": "simplified_code", + "language": language, + }, + ) + + +class DocumentEmbedding: + """ + A class to create a FAISS database from a list of source documents. The source documents are collected from git + repositories and chunked into smaller pieces for embedding. + """ + + def __init__(self, + *, + embedding: "Embeddings", + vdb_directory: PathLike = "./.cache/am_cache/vdb", + git_directory: PathLike = "./.cache/am_cache/git", + chunk_size: int = 800, + chunk_overlap: int = 160): + """ + Create a new DocumentEmbedding instance. + + Parameters + ---------- + embedding : Embeddings + The embedding to use for the FAISS database. + vdb_directory : PathLike + The directory to save the FAISS database. The database will be saved in a subdirectory based on the hash of + the source documents. + git_directory : PathLike, optional + The directory to use for the Git repository cloning, by default "./.tmp/git_cache" + chunk_size : int, optional + Maximum size of a single chunk, by default 1000 + chunk_overlap : int, optional + Overlap between chunks, by default 200 + """ + + self._embedding = embedding + self._vdb_directory = Path(vdb_directory) + self._git_directory = Path(git_directory) + self._chunk_size = chunk_size + self._chunk_overlap = chunk_overlap + + @property + def embedding(self): + return self._embedding + + @property + def vdb_directory(self): + return self._vdb_directory + + @property + def git_directory(self): + return self._git_directory + + @property + def chunk_size(self): + return self._chunk_size + + @property + def chunk_overlap(self): + return self._chunk_overlap + + def hash_source_documents_info(self, source_infos: list[SourceDocumentsInfo]): + """ + Hash the source documents info to create a unique hash for the source documents. This is used to determine if + the VDB already exists for the source documents. + + Parameters + ---------- + source_infos : list[SourceDocumentsInfo] + A list of source documents info to hash. + + Returns + ------- + str + Returns a unique hash for the source documents as a positive integer. Will always return the same hash for + the same source regardless of order. + """ + + obj_to_hash = { + "class": self.__class__.__qualname__, + "source_infos": [hash(x) for x in sorted(source_infos)], + "embedding": str(self._embedding), + "chunk_size": self._chunk_size, + "chunk_overlap": self._chunk_overlap, + } + + # Hash the source documents info + hash_val = int.from_bytes(bytes=sha512(f"{json.dumps(obj_to_hash)}".encode('utf-8', errors='ignore')).digest(), + byteorder=sys.byteorder) + + # Convert to a positive, consistent hash (Avoids negative hex values) + nbits = 64 + return hex((hash_val + (1 << nbits)) % (1 << nbits))[2:] # Skip the '0x' prefix + + def _chunk_documents(self, documents: list[Document]) -> list[Document]: + """ + Chunk the documents into smaller pieces for embedding. + """ + + splitter = MultiLanguageRecursiveCharacterTextSplitter(chunk_size=self._chunk_size, + chunk_overlap=self._chunk_overlap) + split_documents = splitter.split_documents(documents) + + return split_documents + + def get_repo_path(self, source_info: SourceDocumentsInfo): + """ + Returns the path to the git repository for a source document info. + + Parameters + ---------- + source_info : SourceDocumentsInfo + The source document info to get the git repository path. + + Returns + ------- + Path + Returns the path to the git repository. + """ + return self._git_directory / PurePath(source_info.git_repo) + + def collect_documents(self, source_info: SourceDocumentsInfo) -> list[Document]: + """ + Collect documents from a source document info. This will clone the git repository and collect files from the + repository based on the include and exclude patterns. Each file is then parsed and segmented based on its + language into Documents. + + Parameters + ---------- + source_info : SourceDocumentsInfo + The source document info to collect documents + + Returns + ------- + list[Document] + Returns a list of documents collected from the source document info. + """ + + repo_path = self.get_repo_path(source_info) + + blob_loader = SourceCodeGitLoader(repo_path=repo_path, + clone_url=source_info.git_repo, + branch=source_info.tag, + include=source_info.include, + exclude=source_info.exclude) + + blob_parser = ExtendedLanguageParser() + + loader = GenericLoader(blob_loader=blob_loader, blob_parser=blob_parser) + + documents = loader.load() + + logger.debug("Collected documents for '%s', Document count: %d", repo_path, len(documents)) + + return documents + + def create_vdb(self, source_infos: list[SourceDocumentsInfo], output_path: PathLike): + """ + Create a FAISS database from a list of input directories. + + Parameters + ---------- + source_infos : list[SourceDocumentsInfo] + A list of source documents to create the VDB from. All documents will be collected from the source + documents. + output_path : PathLike + The location to save the FAISS database. + + Returns + ------- + FAISS + Returns an instance of the FAISS database. + """ + + logger.debug("Collecting documents from git repos. Source Infos: %s", + json.dumps([x.model_dump(mode="json") for x in source_infos])) + + output_path = Path(output_path) + + # Warn if the output path already exists and we will overwrite it + if (output_path.exists()): + + logger.warning("Vector Database already exists and will be overwritten: %s", output_path) + + documents = [] + for input_dir in source_infos: + documents.extend(self.collect_documents(input_dir)) + + # Apply chunking on the source documents + chunked_documents = self._chunk_documents(documents) + + logger.debug("Creating FAISS database from source documents. Doc count: %d, Chunks: %s, Location: %s", + len(documents), + len(chunked_documents), + output_path) + + saved_show_progress = getattr(self._embedding, "show_progress", None) + saved_show_progress_bar = getattr(self._embedding, "show_progress_bar", None) + + # Attempt to enable progress bar for VDB generation if supported + if (hasattr(self._embedding, "show_progress")): + setattr(self._embedding, "show_progress", True) + if (hasattr(self._embedding, "show_progress_bar")): + setattr(self._embedding, "show_progress_bar", True) + + embedding_start_time = time.time() + + # Create the FAISS database + db = FAISS.from_documents(chunked_documents, self._embedding) + + logger.debug("Completed embedding in %.2f seconds for '%s'", time.time() - embedding_start_time, output_path) + + # Clear the CUDA cache if torch is available. This is to prevent the pytorch cache from growing when it will not be reused + try: + import torch + torch.cuda.empty_cache() + except ImportError: + pass + + # Revert progress bar setting + if (saved_show_progress is not None): + setattr(self._embedding, "show_progress", saved_show_progress) + if (saved_show_progress_bar is not None): + setattr(self._embedding, "show_progress_bar", saved_show_progress_bar) + + # Ensure the directory exists + output_path.mkdir(exist_ok=True, parents=True) + + # Save the database + db.save_local(str(output_path)) + + return db + + def build_vdbs(self, + input_sources: list[SourceDocumentsInfo], + ignore_code_embedding: bool = False) -> tuple[Path | None, Path | None]: + """ + Build the code and document VDB based on a list of source documents. + + Parameters + ---------- + input_sources : list[SourceDocumentsInfo] + A list of source documents to create the VDB from. + + Returns + ------- + tuple[Path | None, Path | None] + Returns a tuple of (code_vdb, doc_vdb). If the VDB was not created, the value will be None for each type. + """ + code_vdb: Path | None = None + doc_vdb: Path | None = None + + # Create embeddings for each source type + for source_type in ["code", "doc"]: + + if ignore_code_embedding and source_type == "code": + continue + + # Filter the source documents + source_infos = [source_info for source_info in input_sources if source_info.type == source_type] + + if source_infos: + + # Determine the output path by combining the vdb_directory with the hash of the source documents + vdb_output_dir = self.vdb_directory / source_type / str(self.hash_source_documents_info(source_infos)) + + if (not vdb_output_dir.exists() or os.environ.get("MORPHEUS_ALWAYS_REBUILD_VDB", "0") == "1"): + vdb = self.create_vdb(source_infos=source_infos, output_path=vdb_output_dir) + else: + logger.debug("Cache hit on VDB. Loading existing FAISS database: %s", vdb_output_dir) + + vdb = FAISS.load_local(str(vdb_output_dir), self._embedding, allow_dangerous_deserialization=True) + + else: + vdb_output_dir = None + vdb = None + + if (source_type == "code"): + if (vdb is not None): + code_vdb = vdb_output_dir + elif (source_type == "doc"): + if (vdb is not None): + doc_vdb = vdb_output_dir + else: + raise ValueError(f"Unknown source type: {source_type}") # pragma: no cover + + return code_vdb, doc_vdb diff --git a/src/cve/utils/embedding_loader.py b/src/cve/utils/embedding_loader.py new file mode 100644 index 0000000..6808f21 --- /dev/null +++ b/src/cve/utils/embedding_loader.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import importlib +import os +import typing + +if typing.TYPE_CHECKING: + from langchain.embeddings.huggingface import HuggingFaceEmbeddings # nocover + from langchain.embeddings.openai import OpenAIEmbeddings # nocover + from langchain_core.embeddings import Embeddings # nocover + from langchain_nvidia_ai_endpoints import NVIDIAEmbeddings # nocover + +EmbeddingTypes = typing.Literal["huggingface", "openai", "nim"] + + +class EmbeddingLoader: + + @typing.overload + @staticmethod + def create(type: typing.Literal["openai"], *embedding_args, **embedding_kwargs) -> "OpenAIEmbeddings": + pass + + @typing.overload + @staticmethod + def create(type: typing.Literal["huggingface"], *embedding_args, **embedding_kwargs) -> "HuggingFaceEmbeddings": + pass + + @typing.overload + @staticmethod + def create(type: typing.Literal["nim"], *embedding_args, **embedding_kwargs) -> "NVIDIAEmbeddings": + pass + + @typing.overload + @staticmethod + def create(type: str, *embedding_args, **embedding_kwargs) -> "Embeddings": + pass + + @staticmethod + def create(type: str | EmbeddingTypes, *embedding_args, **embedding_kwargs) -> "Embeddings": + """ + Returns a LangChaing embedding class from a properly defined config. + Parameters + ---------- + embedding_type : str + The embedding model to create + embedding_args: dict[str, typing.Any] + Additional keyword arguments to pass to the model + """ + match type.lower(): + case "openai": + module_name = "langchain.embeddings.openai" + module_class = "OpenAIEmbeddings" + case "huggingface": + module_name = "langchain.embeddings.huggingface" + module_class = "HuggingFaceEmbeddings" + case "nim": + module_name = "langchain_nvidia_ai_endpoints" + module_class = "NVIDIAEmbeddings" + if "api_key" not in embedding_kwargs or embedding_kwargs["api_key"] is None: + embedding_kwargs["api_key"] = os.environ.get("NVIDIA_API_KEY", None) + if "base_url" not in embedding_kwargs or embedding_kwargs["base_url"] is None: + # Use environment variable if available, otherwise don't set `base_url` to use LangChain default + if "NIM_EMBED_BASE_URL" in os.environ: + embedding_kwargs["base_url"] = os.environ.get("NIM_EMBED_BASE_URL") + + module = importlib.import_module(module_name) + + class_ = getattr(module, module_class) + + instance = class_(*embedding_args, **embedding_kwargs) + + return instance diff --git a/src/cve/utils/intel_retriever.py b/src/cve/utils/intel_retriever.py new file mode 100644 index 0000000..44e6cb5 --- /dev/null +++ b/src/cve/utils/intel_retriever.py @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import asyncio +import logging +import os +from contextlib import asynccontextmanager + +import aiohttp + +from ..data_models.cve_intel import CveIntel +from ..data_models.cve_intel import CveIntelEpss +from ..data_models.cve_intel import CveIntelNvd +from ..data_models.cve_intel import CveIntelRhsa +from ..data_models.cve_intel import CveIntelUbuntu +from .clients.first_client import FirstClient +from .clients.ghsa_client import GHSAClient +from .clients.nvd_client import NVDClient +from .clients.rhsa_client import RHSAClient +from .clients.ubuntu_client import UbuntuClient + +logger = logging.getLogger(f"morpheus.{__name__}") + + +class IntelRetriever: + """ + A class that retrieves details about CVEs (Common Vulnerabilities and Exposures) from NIST and CVE Details websites. + """ + + def __init__(self, + session: aiohttp.ClientSession, + nist_api_key: str | None = None, + ghsa_api_key: str | None = None, + lang_code: str = 'en', + max_retries: int = 10): + """ + Initialize the NISTCVERetriever with URL templates for vulnerability and CVE details. + """ + + # Create a shared session object for all requests + self._session = session + + self._nvd_client = NVDClient(api_key=os.environ.get('NVD_API_KEY', nist_api_key), + session=self._session, + lang_code=lang_code, + retry_count=max_retries) + self._first_client = FirstClient(session=self._session, retry_count=max_retries) + self._ghsa_client = GHSAClient(api_key=os.environ.get('GHSA_API_KEY', ghsa_api_key), + session=self._session, + retry_count=max_retries) + self._rhsa_client = RHSAClient(session=self._session, retry_count=max_retries) + self._ubuntu_client = UbuntuClient(session=self._session, retry_count=max_retries) + + @asynccontextmanager + async def _get_session(self, session: aiohttp.ClientSession | None = None): + + if (session is not None): + yield session + elif (self._session is not None): + yield self._session + else: + async with aiohttp.ClientSession() as session: + yield session + + async def _get_ghsa_intel(self, vuln_id: str) -> CveIntel: + """ + Gets the GHSA intel and returns an intel object. Must be called first!!!! + """ + + intel = CveIntel(vuln_id=vuln_id) + + try: + intel.ghsa = await self._ghsa_client.get_intel(vuln_id=intel.vuln_id) + + except Exception as e: + logger.error("Error fetching GitHub security advisory for %s : %s", vuln_id, e) + + return intel + + async def _get_nvd_intel(self, intel: CveIntel) -> CveIntelNvd | None: + + if (not intel.has_cve_id()): + logger.warning("Skipping NVD retrieval since '%s' does not have an associated CVE ID", intel.vuln_id) + return None + + try: + intel.nvd = await self._nvd_client.get_intel(cve_id=intel.cve_id) + + return intel.nvd + + except Exception as e: + logger.error("Error fetching NVD intel for %s : %s", intel.vuln_id, e) + + return None + + async def _get_ubuntu_intel(self, intel: CveIntel) -> CveIntelUbuntu | None: + + if (not intel.has_cve_id()): + logger.warning("Skipping Ubuntu retrieval since '%s' does not have an associated CVE ID", intel.vuln_id) + return None + + try: + intel.ubuntu = await self._ubuntu_client.get_intel(cve_id=intel.cve_id) + + return intel.ubuntu + + except Exception as e: + logger.error("Error fetching Ubuntu security advisory for %s : %s", intel.vuln_id, e) + + return None + + async def _get_rhsa_intel(self, intel: CveIntel) -> CveIntelRhsa | None: + + if (not intel.has_cve_id()): + logger.warning("Skipping RHSA retrieval since '%s' does not have an associated CVE ID", intel.vuln_id) + return None + + try: + intel.rhsa = await self._rhsa_client.get_intel(cve_id=intel.cve_id) + + return intel.rhsa + + except Exception as e: + logger.error("Error fetching Red Hat security advisory for %s : %s", intel.vuln_id, e) + + return None + + async def _get_epss_score(self, intel: CveIntel) -> CveIntelEpss | None: + + if (not intel.has_cve_id()): + logger.warning("Skipping EPSS Score retrieval since '%s' does not have an associated CVE ID", intel.vuln_id) + return None + + try: + intel.epss = await self._first_client.get_intel(cve_id=intel.cve_id) + + return intel.epss + except Exception as e: + logger.error("Error fetching EPSS score for %s : %s", intel.vuln_id, e) + + return None + + async def retrieve(self, vuln_id: str) -> CveIntel: + """ + Asynchronously retrieve all relevant details for a given CVE ID. + + Parameters + ---------- + vuln_id : str + The CVE identifier. Can be GHSA or NVD + + Returns + ------- + dict + A dictionary containing all the retrieved CVE details. + """ + + # Build the intel objects. Must start with GHSA to get the cve_id + intel = await self._get_ghsa_intel(vuln_id=vuln_id) + + if (not intel.has_cve_id()): + logger.warning( + "Skipping additional intel retrieval beyond GHSA since '%s' " + "does not have an associated CVE ID", + vuln_id) + return intel + + # Run all the coroutines concurrently + coros = [ + self._get_nvd_intel(intel=intel), + self._get_ubuntu_intel(intel=intel), + self._get_rhsa_intel(intel=intel), + self._get_epss_score(intel=intel) + ] + + await asyncio.gather(*coros) + + return intel diff --git a/src/cve/utils/intel_utils.py b/src/cve/utils/intel_utils.py new file mode 100644 index 0000000..4ca3647 --- /dev/null +++ b/src/cve/utils/intel_utils.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from packaging.version import InvalidVersion +from packaging.version import parse as parse_version +from pydpkg import Dpkg +from pydpkg.exceptions import DpkgVersionError + +from ..data_models.cve_intel import CveIntelNvd + + +def update_version(incoming_version, current_version, compare): + """ + Updates the version based on the comparison result. + + Args: + incoming_version (str): The incoming version from NIST. + current_version (str): The current version observed. + compare (str): The comparison operator, either "older" or "newer". + + Returns: + str: The updated version. + """ + + if compare == "older": + compare_func = lambda x, y: x < y + dpkg_compare = lambda x: x == -1 + elif compare == "newer": + compare_func = lambda x, y: x > y + dpkg_compare = lambda x: x == 1 + + if incoming_version is None: + return current_version + if current_version is None: + return incoming_version + try: + if compare_func(parse_version(incoming_version), parse_version(current_version)): + updated = incoming_version + else: + updated = current_version + except InvalidVersion: + # Failed PEP440 versioning; moving on to Debian + try: + compare_result = Dpkg.compare_versions(incoming_version, current_version) + updated = (incoming_version if dpkg_compare(compare_result) else current_version) + except DpkgVersionError: + # Debian versioning failed; moving on to alpha + updated = (incoming_version if compare_func(incoming_version, current_version) else current_version) + return updated + + +def parse_cpe(cpe): + """ + Parses a Common Platform Enumeration (CPE) string. + + Args: + cpe (str): The CPE string. + + Returns: + tuple: A tuple containing the package, version, and system information. + """ + package, version, system = None, None, None + split_cpe = cpe.split(":") + if len(split_cpe) > 4: + package = split_cpe[4] if split_cpe[4] != "*" and split_cpe[4] != "-" else None + if len(split_cpe) > 5: + version = split_cpe[5] if split_cpe[5] != "*" and split_cpe[5] != "-" else None + if len(split_cpe) > 10: + system = split_cpe[10] if split_cpe[10] != "*" and split_cpe[5] != "-" else None + return (package, version, system) + + +def parse(configurations: list): + """ + Parses a list of configurations into a list of Configuration + (version information) objects. + + Args: + configurations (list): A list of version information inputs. + + Returns: + list: A list of configurationn objects. + """ + version_info = [] + cache = set() + for config in configurations: + nodes = config.get("nodes", []) + for node in nodes: + for cpe_match in node.get("cpeMatch", []): + package, version, system = parse_cpe(cpe_match.get("criteria", "*:*:*:*:*:*:*:*:*:*:*:*:*")) + ver_start_exclude = cpe_match.get("versionStartExcluding") + ver_start_include = update_version(version, cpe_match.get("versionStartIncluding"), "older") + ver_end_include = update_version(version, cpe_match.get("versionEndIncluding"), "newer") + ver_end_exclude = cpe_match.get("versionEndExcluding") + info = [ + package, + system, + ver_start_exclude, + ver_end_exclude, + ver_start_include, + ver_end_include, + ] + + if tuple(info) in cache: + continue + if package: + if any(info[2:]): + cache.add(tuple(info)) + obj = CveIntelNvd.Configuration( + package=package, + system=system, + versionStartExcluding=ver_start_exclude, + versionEndExcluding=ver_end_exclude, + versionStartIncluding=ver_start_include, + versionEndIncluding=ver_end_include, + ) + version_info.append(obj) + + return version_info diff --git a/src/cve/utils/output_formatter.py b/src/cve/utils/output_formatter.py new file mode 100644 index 0000000..c8e27c8 --- /dev/null +++ b/src/cve/utils/output_formatter.py @@ -0,0 +1,280 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import html +from ..data_models.output import AgentMorpheusOutput + + +def generate_vulnerability_reports(model_dict: AgentMorpheusOutput, output_dir): + """Creates a markdown file for each CVE ID in the markdown content dictionary. + + Args: + markdown_content (dict): A dictionary where keys are CVE IDs and values are markdown content. + output_dir (str): The directory where the markdown files will be created. + + Returns: + None + """ + markdown_content = _transform_to_markdown(model_dict) + + # Get the container directory name based on the image name and tag + container_dir_name = model_dict.input.image.name.split("/")[-1].replace(".", "_") + "_" + model_dict.input.image.tag.replace(".","_") + + # Create a subdirectory named after the prefixed container ID inside the output_dir + container_dir = os.path.join(output_dir, f"vulnerability_reports_{container_dir_name}") + + if not os.path.exists(container_dir): + os.makedirs(container_dir, exist_ok=True) + + for cve_id, content in markdown_content.items(): + file_name = f"{cve_id}.md" + file_path = os.path.join(container_dir, file_name) + with open(file_path, "w") as f: + f.write("\n".join(content)) + + +def _transform_to_markdown(model_dict: AgentMorpheusOutput): + """Convert JSON data to Markdown content. + + Args: + model_dict (AgentMorpheusOutput): JSON data containing vulnerability information. + + Returns: + dict: Markdown content for each CVE ID. + """ + cve_ids = [vuln.vuln_id for vuln in model_dict.input.scan.vulns] + markdown_content = {} + for cve_id in cve_ids: + markdown_content[cve_id] = [] + _add_header(markdown_content[cve_id], cve_id, model_dict) + + _add_vulnerability_details(markdown_content, model_dict) + _add_table_of_contents(markdown_content, model_dict) + _add_checklist_info(markdown_content, model_dict) + _add_vulnerable_sboms(markdown_content, model_dict) + return markdown_content + + +def _add_header(markdown_content, cve_id, model_dict: AgentMorpheusOutput): + """Add header to Markdown content. + + Args: + markdown_content (list): Markdown content for a CVE ID. + cve_id (str): CVE ID. + model_dict (AgentMorpheusOutput): JSON data containing vulnerability information. + """ + input_image = model_dict.input.image + markdown_content.append(f"# Vulnerability Analysis Report for {cve_id}") + markdown_content.append( + f"> **Container Analyzed:** `{input_image.name} : {input_image.tag}`\n\n" + ) + markdown_content.append(f"> **SBOM Info:** `{input_image.sbom_info}`") + + +def _add_table_of_contents(markdown_content, model_dict: AgentMorpheusOutput): + """Add table of contents for checklists per CVE. + + Args: + markdown_content (dict): Markdown content for each CVE ID. + model_dict (AgentMorpheusOutput): JSON data containing vulnerability information. + """ + for entry in model_dict.output: + cve_id = entry.vuln_id + checklist = entry.checklist + markdown_content[cve_id].append( + "## Table of Contents \n" + ) + markdown_content[cve_id].append( + "#### [Checklist](#checklist) " + ) + if checklist: + steps = { + item.input.split(":")[0]: [ + intermediate_step.tool_name + for intermediate_step in item.intermediate_steps or [] + ] + for item in checklist + } + for i, (step, intermediate_steps) in enumerate(steps.items(), start=1): + markdown_content[cve_id].append(f"{i}. [{step}](#checklist-step-{i})\n") + for j, intermediate_step in enumerate(intermediate_steps, start=1): + markdown_content[cve_id].append( + f"\t {j}. [{intermediate_step}](#checklist-step-{i}.{j})" + ) + markdown_content[cve_id].append( + "#### [Vulnerable SBOMS](#sbom) " + ) + + +def _add_checklist_info(markdown_content, model_dict: AgentMorpheusOutput): + """Add info for checklists per CVE. + + Args: + markdown_content (dict): Markdown content for each CVE ID. + model_dict (AgentMorpheusOutput): JSON data containing vulnerability information. + """ + for entry in model_dict.output: + cve_id = entry.vuln_id + checklist = entry.checklist + if checklist: + markdown_content[cve_id].append( + f"\n## Checklist Details " + ) + for i, item in enumerate(checklist, start=1): + input_text = item.input + response = item.response + markdown_content[cve_id].append( + f"\n## Step {i} : {input_text.split(':')[0]}\n" + ) + markdown_content[cve_id].append(f"\n> **Input**: *{input_text}*") + markdown_content[cve_id].append(f"\n> **Response**: *{response}*") + intermediate_steps = item.intermediate_steps + if intermediate_steps: + for j, step in enumerate(intermediate_steps, start=1): + tool_name = step.tool_name + action_log = step.action_log + tool_input = step.tool_input + tool_output = step.tool_output + markdown_content[cve_id].append( + f"\n### Step {i}.{j} : *{tool_name}*" + ) + markdown_content[cve_id].append( + f"\n\n#### Action Log \n
{action_log}
" + ) + markdown_content[cve_id].append( + f"\n\n#### Tool Input \n
{tool_input}
" + ) + markdown_content[cve_id].append( + f"\n\n#### Tool Output \n{_process_tool_output(tool_output)}\n\n" + ) + if j != len(intermediate_steps): + markdown_content[cve_id].append( + "\n[back to top](#checklist-toc)" + ) + elif not intermediate_steps and i != len(checklist): + markdown_content[cve_id].append("\n[back to top](#checklist-toc)") + + markdown_content[cve_id].append("\n[back to top](#checklist-toc)") + + +def _process_tool_output(content): + """Process tool_output content and return a renderable markdown object + + Args: + content (dict, str): JSON or string object + """ + if isinstance(content, str): + return f"
{content}
" + result = content.get("result", "") + content_markdown = "" + if result: + content_markdown += f"> **Response:** {result}" + metadata_keys = set() + for item in content.get("source_documents", []): + if "metadata" in item: + metadata_keys.update(item["metadata"].keys()) + table = ( + "\n\n Source Documents \n\n | ID | Type | " + + " |... ".join(metadata_keys) + + " | Page Content |\n" + ) + table += "| --- " * (len(metadata_keys) + 3) + "|\n" + for item in content.get("source_documents", []): + row = [str(item.get("id", "")), str(item.get("type", ""))] + for key in metadata_keys: + row.append(str(item.get("metadata", {}).get(key, ""))) + # Retrieve and sanitize page content + page_content = item.get("page_content", "") + # Remove extra whitespace + page_content = " ".join(page_content.split()) + # Escape special characters + page_content = html.escape(page_content) + row.append( + f"
View Content
{page_content}
" + ) + table += "| " + " | ".join(row) + " |\n" + return content_markdown + table + + +def _add_vulnerability_details(markdown_content, model_dict: AgentMorpheusOutput): + """Add vulnerability details to Markdown content. + + Args: + markdown_content (dict): Markdown content for each CVE ID. + model_dict (AgentMorpheusOutput): JSON data containing vulnerability information. + """ + for entry in model_dict.output: + cve_id = entry.vuln_id + summary = entry.summary + justification = entry.justification + exploitability = justification.status + exploitability_text = _get_expoiltability_text(exploitability) + markdown_content[cve_id].append( + f"## Summary \n{summary}" + ) + markdown_content[cve_id].append( + f"\n## Justification ({exploitability_text}) \n" + ) + markdown_content[cve_id].append(f"\n>label: {justification.label}") + markdown_content[cve_id].append(f"\n{justification.reason}") + + +def _add_vulnerable_sboms(markdown_content, model_dict: AgentMorpheusOutput): + """Add vulnerable SBOMs to Markdown content. + + Args: + markdown_content (dict): Markdown content for each CVE ID. + model_dict (AgentMorpheusOutput): JSON data containing vulnerability information. + """ + vulnerable_sboms_per_cve_id = { + i.vuln_id: i.vulnerable_sbom_packages + for i in model_dict.info.vulnerable_dependencies + } + for cve_id, vulnerable_sboms in vulnerable_sboms_per_cve_id.items(): + markdown_content[cve_id].append( + "\n---\n## Vulnerable SBOM Dependencies \n" + ) + for sbom in vulnerable_sboms: + sbom_name = sbom.name + sbom_type = sbom.version + vulnerable_package_dependencies = sbom.vulnerable_dependency_package + markdown_content[cve_id].append("\n") + markdown_content[cve_id].append(f"\n### {sbom_name}") + markdown_content[cve_id].append(f"**Version:** {sbom_type}") + markdown_content[cve_id].append("\n**Vulnerable Dependencies:**") + for dependency, value in vulnerable_package_dependencies: + markdown_content[cve_id].append(f" - {dependency} : {value}") + markdown_content[cve_id].append("\n[back to top](#toc)") + + +def _get_expoiltability_text(exploitability): + """Get color based on exploitability status. + + Args: + exploitability (str): Exploitability status. + + Returns: + str: Color code. + """ + color = "#9E9E9E" # Gray + exploitability_text = "Exploitability Unknown" + if exploitability == "TRUE": + color = "#F44336" # Red + exploitability_text = "Exploitable" + elif exploitability == "FALSE": + color = "#4CAF50" # Green + exploitability_text = "Not Exploitable" + return f"{exploitability_text}" diff --git a/src/cve/utils/prompting.py b/src/cve/utils/prompting.py new file mode 100644 index 0000000..d3cceb6 --- /dev/null +++ b/src/cve/utils/prompting.py @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Some examples to be optionally passed along in the agent prompt. +from abc import ABC +from abc import abstractmethod +from textwrap import dedent + +agent_examples_for_prompt = """Example 1: + + Question: 1. Identify the version of Python: Check which version(s) of Python are installed in the Docker container. The vulnerability affects versions up to and including 3.11.3. + Thought: I should check if Python is installed. I will check the Software Bill of Materials. + Action: SBOM Package Checker + Action Input: Python + Observation: 3.10.0 + Thought: Python 3.10.0 is installed, I need to check if the installed version is vulnerable. To do this I'll compare the installed version to the vulnerable version. + Action: Docker Container software version comparator + Action Input: 3.10.0, 3.11.3 + Observation: True + Thought: The installed software is vulnerable. I now know the answer. + Final Answer: Python version 3.10.0 is installed and is vulnerable to the CVE. + + Example 2: + + Question: Assess the threat that CVE-20xx-xxxxx poses to the container. + Thought: I should search for more information on CVE-20xx-xxxxx. + Action: Internet Search + Action Input: What is CVE-20xx-xxxxx? + Observation: CVE-20xx-xxxxx causes memory leaks and possible denial of service attack vectors when using urllib.parse + Thought: I should check the code base of the container for instances of urllib.parse + Action: Docker Container Code QA System + Action Input: Is urllib.parse present in the code? + Observation: + Question: Is urllib.parse present in the code? + Helpful answer: No, that function is not called in the code. + Thought: Since the function is not called in the code, the container is not vulnerable. I know the final answer. + Final Answer: The function urllib.parse is not present in the code, so the container is not vulnerable. + + Example 3: + + Question: Check if the container is using Java Runtime Environment (JRE). If it is not using JRE, then it is not vulnerable to CVE-xxxx-xxxxx. + Thought: I should check if JRE is installed. I will check the Software Bill of Materials. + Action: SBOM Package Checker + Action Input: JRE + Observation: False + Thought: JRE is not present in the container. I now know the answer. + Final Answer: JRE is not installed in the container. Therefore, it is not vulnerable to CVE-20xx-xxxxx. + + Example 4: + + Question: Check if the container is using Apache. If it is not using Apache, then it is not vulnerable to CVE-xxxx-xxxxx. + Thought: I should check if Apache is installed. I will check the Software Bill of Materials. + Action: SBOM Package Checker + Action Input: Apache + Observation: 1.0.1 + Thought: Apache is present in the container. I now know the answer. + Final Answer: Apache is installed in the container. Therefore, it is potentially vulnerable to CVE-20xx-xxxxx. + + """ + + +class PromptBuilder(ABC): + + @abstractmethod + def build_prompt(self) -> str: + pass + + +class IfPromptBuilder(PromptBuilder): + + def __init__(self, prop_name: str, description: str) -> None: + + self.prop_name = prop_name + self.description = description + + def build_prompt(self) -> str: + + f_string = dedent(f""" + {{% if {self.prop_name} %}} + - {self.description}{{{{{self.prop_name} | string() | truncate(1024)}}}} + {{% endif %}} + """).strip() + + return f_string + + +class IfElsePromptBuilder(PromptBuilder): + + def __init__(self, first_prop_name: str, second_prop_name: str, description: str) -> None: + + self.first_prop_name = first_prop_name + self.second_prop_name = second_prop_name + self.description = description + + def build_prompt(self) -> str: + + f_string = dedent(f""" + {{% if {self.first_prop_name} %}} + - {self.description}{{{{{self.first_prop_name} | string() | truncate(1024)}}}} + {{% elif {self.second_prop_name} %}} + - {self.description}{{{{{self.second_prop_name} | string() | truncate(1024)}}}} + {{% endif %}} + """).strip() + + return f_string + + +# Add additional intel fields for the checklist prompt here (field name, description) +additional_intel_fields: list[PromptBuilder] = [ + # Keep these at the top + IfPromptBuilder('cve_id', 'CVE ID: '), + IfElsePromptBuilder('nvd_cve_description', "ghsa_description", 'CVE Description: '), + + # Sort these alphabetically + IfElsePromptBuilder('nvd_cvss_vector', "ghsa_cvss_vector_string", 'CVSS Vector: '), + IfElsePromptBuilder('nvd_cwe_name', "ghsa_cwes", 'CWE Name: '), + IfPromptBuilder('ghsa_vulnerabilities', 'GHSA Details: '), + IfPromptBuilder('nvd_configurations', 'Known Affected Software: '), + IfPromptBuilder('nvd_cwe_description', 'CWE Description: '), + IfPromptBuilder('nvd_cwe_extended_description', ''), + IfPromptBuilder('nvd_vendor_names', 'Notable Vulnerable Software Vendors: '), + IfPromptBuilder('rhsa_bugzilla_description', 'RHSA Description: '), + IfPromptBuilder('rhsa_details', 'RHSA Details: '), + IfPromptBuilder('rhsa_package_state', 'RHSA Affected Packages: '), + IfPromptBuilder('rhsa_statement', 'RHSA Statement: '), + # IfPromptBuilder('ubuntu_notices', 'Ubuntu Priority Reason: '), # Disabling for now since its very long + IfPromptBuilder('ubuntu_ubuntu_description', 'Ubuntu Security Note: '), + IfPromptBuilder('vulnerable_dependencies', 'Identified Vulnerable Dependencies: '), +] + +additional_intel_prompting = '\n'.join([pb.build_prompt() for pb in additional_intel_fields]) + +ex_questions = [ + 'Example {idx}: CVE Details:\n- CVE ID: CVE-2023-24329\n- CVE description: An issue in the urllib.parse component of Python before 3.11.4 allows attackers to bypass blocklisting methods by supplying a URL that starts with blank characters.\n- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:N\n- CWE Name: CWE-20: Improper Input Validation (4.14)\n- CWE Description: The product receives input or data, but it does not validate or incorrectly validates that the input has the properties that are required to process the data safely and correctly.\nInput validation is a frequently-used technique for checking potentially dangerous inputs in order to ensure that the inputs are safe for processing within the code, or when communicating with other components. When software does not validate input properly, an attacker is able to craft the input in a form that is not expected by the rest of the application. This will lead to parts of the system receiving unintended input, which may result in altered control flow, arbitrary control of a resource, or arbitrary code execution. Input validation is not the only technique for processing input, however. Other techniques attempt to transform potentially-dangerous input into something safe, such as filtering (CWE-790) - which attempts to remove dangerous inputs - or encoding/escaping (CWE-116), which attempts to ensure that the input is not misinterpreted when it is included in output to another component. Other techniques exist as well (see CWE-138 for more examples.) Input validation can be applied to: raw data - strings, numbers, parameters, file contents, etc. metadata - information about the raw data, such as headers or size Data can be simple or structured. Structured data can be composed of many nested layers, composed of combinations of metadata and raw data, with other simple or structured data. Many properties of raw data or metadata may need to be validated upon entry into the code, such as: specified quantities such as size, length, frequency, price, rate, number of operations, time, etc. implied or derived quantities, such as the actual size of a file instead of a specified size indexes, offsets, or positions into more complex data structures symbolic keys or other elements into hash tables, associative arrays, etc. well-formedness, i.e. syntactic correctness - compliance with expected syntax lexical token correctness - compliance with rules for what is treated as a token specified or derived type - the actual type of the input (or what the input appears to be) consistency - between individual data elements, between raw data and metadata, between references, etc. conformance to domain-specific rules, e.g. business logic equivalence - ensuring that equivalent inputs are treated the same authenticity, ownership, or other attestations about the input, e.g. a cryptographic signature to prove the source of the data Implied or derived properties of data must often be calculated or inferred by the code itself. Errors in deriving properties may be considered a contributing factor to improper input validation. Note that "input validation" has very different meanings to different people, or within different classification schemes. Caution must be used when referencing this CWE entry or mapping to it. For example, some weaknesses might involve inadvertently giving control to an attacker over an input when they should not be able to provide an input at all, but sometimes this is referred to as input validation. Finally, it is important to emphasize that the distinctions between input validation and output escaping are often blurred, and developers must be careful to understand the difference, including how input validation is not always sufficient to prevent vulnerabilities, especially when less stringent data types must be supported, such as free-form text. Consider a SQL injection scenario in which a person\'s last name is inserted into a query. The name "O\'Reilly" would likely pass the validation step since it is a common last name in the English language. However, this valid name cannot be directly inserted into the database because it contains the "\'" apostrophe character, which would need to be escaped or otherwise transformed. In this case, removing the apostrophe might reduce the risk of SQL injection, but it would produce incorrect behavior because the wrong name would be recorded.\n- Notable Vulnerable Software Vendors: [\'Fedoraproject\', \'Netapp\', \'Python\']\n\nExample {idx}: Checklist:\n[\n\t"Review Python Usage: Is the Python installation actively used by applications within the container? Check for scripts or applications that rely on Python, particularly those that might parse URLs using `urllib.parse`.",\n\t"Inspect URL Parsing Logic: Does the codebase contain any usage of `urllib.parse` or similar URL parsing mechanisms? Focus on how URLs are handled and whether there is a reliance on blocklisting methods to filter out potentially harmful URLs. This is crucial since the CVE describes a bypass of blocklisting methods via URLs starting with blank characters.",\n\t"Evaluate Input Validation Practices: Assess the robustness of input validation practices within the application. Since the CVE involves improper input validation, are inputs, especially URLs, properly sanitized and validated against unexpected or malicious data?"\n]', + 'Example {idx}: CVE Details:\n- CVE ID: CVE-2023-36632\n- CVE description: The legacy email.utils.parseaddr function in Python through 3.11.4 allows attackers to trigger "RecursionError: maximum recursion depth exceeded while calling a Python object" via a crafted argument. This argument is plausibly an untrusted value from an application\'s input data that was supposed to contain a name and an e-mail address. NOTE: email.utils.parseaddr is categorized as a Legacy API in the documentation of the Python email package. Applications should instead use the email.parser.BytesParser or email.parser.Parser class. NOTE: the vendor\'s perspective is that this is neither a vulnerability nor a bug. The email package is intended to have size limits and to throw an exception when limits are exceeded; they were exceeded by the example demonstration code.\n- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H\n- CWE Name: CWE-674: Uncontrolled Recursion (4.14)\n- CWE Description: The product does not properly control the amount of recursion that takes place, consuming excessive resources, such as allocated memory or the program stack.\n- Notable Vulnerable Software Vendors: [\'Python\']\n\nExample {idx}: Checklist:\n[\n\t"Identify Usage of `email.utils.parseaddr`: Review the application code within the Docker container to check if the `email.utils.parseaddr` function is being used. This function is the specific target of the vulnerability. Does the codebase contain the string \'email.utils.parseaddr\'?",\n\t"Assess Input Data Handling: Does the application using `email.utils.parseaddr` process potentially untrusted input data that could include crafted arguments designed to exploit this vulnerability? Evaluate how the application handles exceptions like `RecursionError`, as the exploit triggers this specific error.",\n\t"Review Alternative Implementations: Has the application already migrated to recommended alternatives such as `email.parser.BytesParser` or `email.parser.Parser` in the code base? If not, recommend transitioning away from the legacy `email.utils.parseaddr` to these safer alternatives."\n]', + "Example {idx}: CVE Details:\n- CVE ID: CVE-2023-50447\n- CVE description: Pillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution via the environment parameter, a different vulnerability than CVE-2022-22817 (which was about the expression parameter).\n- CVSS Vector: CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\n- CWE Name: CWE-94: Improper Control of Generation of Code ('Code Injection') (4.14)\n- CWE Description: The product constructs all or part of a code segment using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the syntax or behavior of the intended code segment.\nWhen a product allows a user's input to contain code syntax, it might be possible for an attacker to craft the code in such a way that it will alter the intended control flow of the product. Such an alteration could lead to arbitrary code execution. Injection problems encompass a wide variety of issues -- all mitigated in very different ways. For this reason, the most effective way to discuss these weaknesses is to note the distinct features which classify them as injection weaknesses. The most important issue to note is that all injection problems share one thing in common -- i.e., they allow for the injection of control plane data into the user-controlled data plane. This means that the execution of the process may be altered by sending code in through legitimate data channels, using no other mechanism. While buffer overflows, and many other flaws, involve the use of some further issue to gain execution, injection problems need only for the data to be parsed. The most classic instantiations of this category of weakness are SQL injection and format string vulnerabilities.\n- Notable Vulnerable Software Vendors: ['Debian', 'Python']\n\nExample {idx}: Checklist:\n[\n\t\"Evaluate Use of PIL.ImageMath.eval: Does any application within the container use the PIL.ImageMath.eval function with user-supplied input in the environment parameter? This is the method and parameter where the vulnerability exists.\",\n\t\"Assess Data Input Sources: Do the applications using Pillow receive input directly from untrusted sources (e.g., user uploads, external APIs)? This will help in understanding the risk exposure and potential for exploitation.\",\n\t\"Security Controls and Sanitization: Review the application's input validation and sanitization measures. Since the vulnerability allows for arbitrary code execution via code injection, is the input properly sanitized before being processed?\"\n]", + 'Example {idx}: CVE Details:\n- CVE ID: CVE-2023-5363\n- CVE description: Issue summary: A bug has been identified in the processing of key and initialisation vector (IV) lengths. This can lead to potential truncation or overruns during the initialisation of some symmetric ciphers. Impact summary: A truncation in the IV can result in non-uniqueness, which could result in loss of confidentiality for some cipher modes. When calling EVP_EncryptInit_ex2(), EVP_DecryptInit_ex2() or EVP_CipherInit_ex2() the provided OSSL_PARAM array is processed after the key and IV have been established. Any alterations to the key length, via the "keylen" parameter or the IV length, via the "ivlen" parameter, within the OSSL_PARAM array will not take effect as intended, potentially causing truncation or overreading of these values. The following ciphers and cipher modes are impacted: RC2, RC4, RC5, CCM, GCM and OCB. For the CCM, GCM and OCB cipher modes, truncation of the IV can result in loss of confidentiality. For example, when following NIST\'s SP 800-38D section 8.2.1 guidance for constructing a deterministic IV for AES in GCM mode, truncation of the counter portion could lead to IV reuse. Both truncations and overruns of the key and overruns of the IV will produce incorrect results and could, in some cases, trigger a memory exception. However, these issues are not currently assessed as security critical.\nChanging the key and/or IV lengths is not considered to be a common operation and the vulnerable API was recently introduced. Furthermore it is likely that application developers will have spotted this problem during testing since decryption would fail unless both peers in the communication were similarly vulnerable. For these reasons we expect the probability of an application being vulnerable to this to be quite low. However if an application is vulnerable then this issue is considered very serious. For these reasons we have assessed this issue as Moderate severity overall.\nThe OpenSSL SSL/TLS implementation is not affected by this issue. The OpenSSL 3.0 and 3.1 FIPS providers are not affected by this because the issue lies outside of the FIPS provider boundary. OpenSSL 3.1 and 3.0 are vulnerable to this issue.\n- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N\n- Notable Vulnerable Software Vendors: [\'Debian\', \'Netapp\', \'Openssl\']\n\nExample {idx}: Checklist:\n[\n\t"Check OpenSSL Version: What version of OpenSSL is running in the Docker container? The vulnerability specifically affects OpenSSL versions 3.0 and 3.1. Is the version running in the container within the vulnerability range? If the container is running these versions, it may be vulnerable.",\n\t"Identify Affected Cipher Modes: Does the application within the Docker container use any of the affected cipher modes: RC2, RC4, RC5, CCM, GCM, or OCB? Special attention should be given to applications using CCM, GCM, and OCB modes as these are particularly noted for potential loss of confidentiality due to IV truncation.",\n\t"Review Cryptographic Operations: Does the code or configuration of applications using OpenSSL have any instances where `EVP_EncryptInit_ex2()`, `EVP_DecryptInit_ex2()`, or `EVP_CipherInit_ex2()` are called? Are there any modifications to the `keylen` or `ivlen` parameters after initialization which might not be taking effect as intended?",\n\t"Check for Custom Cryptographic Implementations: Since changing the key and/or IV lengths is not a common operation and the issue is in a recently introduced API, it\'s crucial to identify if any custom cryptographic implementations might be performing such operations. This is less likely but should be checked especially in bespoke or highly customized applications. Are there any custom cryptographic implemenations changing the key and/or IV lengths?"\n]', + 'Example {idx}: CVE Details:\n- CVE ID: CVE-2024-2961\n- CVE description: The iconv() function in the GNU C Library versions 2.39 and older may overflow the output buffer passed to it by up to 4 bytes when converting strings to the ISO-2022-CN-EXT character set, which may be used to crash an application or overwrite a neighbouring variable.\n- CWE Name: CWE-787: Out-of-bounds Write (4.14)\n- CWE Description: The product writes data past the end, or before the beginning, of the intended buffer.\nTypically, this can result in corruption of data, a crash, or code execution. The product may modify an index or perform pointer arithmetic that references a memory location that is outside of the boundaries of the buffer. A subsequent write operation then produces undefined or unexpected results.\n- Notable Vulnerable Software Vendors: [\'GNU\']\n\nExample {idx}: Checklist:\n[\n\t"Identify Usage of `iconv()` Function: Review the application code or dependencies. Is the `iconv()` function used? Look particularly for conversions involving the ISO-2022-CN-EXT character set. This function is the specific target of the vulnerability.",\n\t"Assess Data Handling and Boundary Conditions: Since the vulnerability involves an out-of-bounds write, it\'s crucial to analyze how data boundaries are handled in the code. Are there any custom implementations or patches that might mitigate boundary issues around buffer sizes?",\n\t"Review Application\'s Character Encoding Needs: Does the application specifically need to handle the ISO-2022-CN-EXT character set? If not, consider disabling this character set or using alternative safe functions or libraries for character set conversions.",\n\t"Evaluate Network Exposure and Attack Surface: Are the affected services exposed to the network? If so, this could increase the risk of exploitation. Additionally, if the application using the `iconv()` function is accessible externally, the risk is higher."\n]', + "Example {idx}: CVE Details:\n- CVE ID: GHSA-8ghj-p4vj-mr35\n- CVE description: An issue was discovered in Pillow before 10.0.0. It is a Denial of Service that uncontrollably allocates memory to process a given task, potentially causing a service to crash by having it run out of memory. This occurs for truetype in ImageFont when textlength in an ImageDraw instance operates on a long text argument.\n- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H\n- CWE Name: CWE-770: Allocation of Resources Without Limits or Throttling (4.14)\n- CWE Description: The product allocates a reusable resource or group of resources on behalf of an actor without imposing any restrictions on the size or number of resources that can be allocated, in violation of the intended security policy for that actor.\n- Code frequently has to work with limited resources, so programmers must be careful to ensure that resources are not consumed too quickly, or too easily. Without use of quotas, resource limits, or other protection mechanisms, it can be easy for an attacker to consume many resources by rapidly making many requests, or causing larger resources to be used than is needed. When too many resources are allocated, or if a single resource is too large, then it can prevent the code from working correctly, possibly leading to a denial of service.\n- Notable Vulnerable Software Vendors: ['Fedoraproject', 'Python']\n- GHSA Summary: Pillow Denial of Service vulnerability\n- GHSA Details: [<'first_patched_version': '10.0.0', 'package': <'ecosystem': 'pip', 'name': 'pillow'>, 'vulnerable_functions': ['PIL.ImageFont'], 'vulnerable_version_range': '>= 0, < 10.0.0'>]\n- GHSA CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H\n\nExample {idx}: Checklist:\n[\n\t\"Assess Usage of Vulnerable Functions: Specifically, the vulnerability is related to the `PIL.ImageFont` module when processing long text arguments. Does the application code or dependencies use this module and functionality? If your applications use this module to process user-supplied or uncontrolled text inputs, they are likely at risk.\",\n\t\"Evaluate Resource Limits: The vulnerability leads to a denial of service through memory exhaustion. Are there any resource limits set at the container level (e.g., using Docker or Kubernetes settings) that might mitigate the impact of such an attack? Consider setting or reviewing memory limits to prevent a single container from consuming all available system resources.\"\n]", +] + +ex_statements = [ + 'Example {idx}: CVE Details:\n- CVE ID: CVE-2023-24329\n- CVE description: An issue in the urllib.parse component of Python before 3.11.4 allows attackers to bypass blocklisting methods by supplying a URL that starts with blank characters.\n- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:N\n- CWE Name: CWE-20: Improper Input Validation (4.14)\n- CWE Description: The product receives input or data, but it does not validate or incorrectly validates that the input has the properties that are required to process the data safely and correctly.\nInput validation is a frequently-used technique for checking potentially dangerous inputs in order to ensure that the inputs are safe for processing within the code, or when communicating with other components. When software does not validate input properly, an attacker is able to craft the input in a form that is not expected by the rest of the application. This will lead to parts of the system receiving unintended input, which may result in altered control flow, arbitrary control of a resource, or arbitrary code execution. Input validation is not the only technique for processing input, however. Other techniques attempt to transform potentially-dangerous input into something safe, such as filtering (CWE-790) - which attempts to remove dangerous inputs - or encoding/escaping (CWE-116), which attempts to ensure that the input is not misinterpreted when it is included in output to another component. Other techniques exist as well (see CWE-138 for more examples.) Input validation can be applied to: raw data - strings, numbers, parameters, file contents, etc. metadata - information about the raw data, such as headers or size Data can be simple or structured. Structured data can be composed of many nested layers, composed of combinations of metadata and raw data, with other simple or structured data. Many properties of raw data or metadata may need to be validated upon entry into the code, such as: specified quantities such as size, length, frequency, price, rate, number of operations, time, etc. implied or derived quantities, such as the actual size of a file instead of a specified size indexes, offsets, or positions into more complex data structures symbolic keys or other elements into hash tables, associative arrays, etc. well-formedness, i.e. syntactic correctness - compliance with expected syntax lexical token correctness - compliance with rules for what is treated as a token specified or derived type - the actual type of the input (or what the input appears to be) consistency - between individual data elements, between raw data and metadata, between references, etc. conformance to domain-specific rules, e.g. business logic equivalence - ensuring that equivalent inputs are treated the same authenticity, ownership, or other attestations about the input, e.g. a cryptographic signature to prove the source of the data Implied or derived properties of data must often be calculated or inferred by the code itself. Errors in deriving properties may be considered a contributing factor to improper input validation. Note that "input validation" has very different meanings to different people, or within different classification schemes. Caution must be used when referencing this CWE entry or mapping to it. For example, some weaknesses might involve inadvertently giving control to an attacker over an input when they should not be able to provide an input at all, but sometimes this is referred to as input validation. Finally, it is important to emphasize that the distinctions between input validation and output escaping are often blurred, and developers must be careful to understand the difference, including how input validation is not always sufficient to prevent vulnerabilities, especially when less stringent data types must be supported, such as free-form text. Consider a SQL injection scenario in which a person\'s last name is inserted into a query. The name "O\'Reilly" would likely pass the validation step since it is a common last name in the English language. However, this valid name cannot be directly inserted into the database because it contains the "\'" apostrophe character, which would need to be escaped or otherwise transformed. In this case, removing the apostrophe might reduce the risk of SQL injection, but it would produce incorrect behavior because the wrong name would be recorded.\n- Notable Vulnerable Software Vendors: [\'Fedoraproject\', \'Netapp\', \'Python\']\n\nExample {idx}: Checklist:\n[\n\t"Check Python Version: Determine the version of Python installed in the Docker container. The vulnerability affects Python versions before 3.11.4. Use commands like `python --version` or `python3 --version` to check the installed version. If the version is below 3.11.4, the container may be vulnerable.",\n\t"Review Python Usage: Identify if the Python installation is actively used by applications within the container. Check for scripts or applications that rely on Python, particularly those that might parse URLs using `urllib.parse`.",\n\t"Inspect URL Parsing Logic: Examine the codebase for any usage of `urllib.parse` or similar URL parsing mechanisms. Focus on how URLs are handled and whether there is a reliance on blocklisting methods to filter out potentially harmful URLs. This is crucial since the CVE describes a bypass of blocklisting methods via URLs starting with blank characters.",\n\t"Evaluate Input Validation Practices: Assess the robustness of input validation practices within the application. Since the CVE involves improper input validation, ensure that inputs, especially URLs, are properly sanitized and validated against unexpected or malicious data."\n]', + 'Example {idx}: CVE Details:\n- CVE ID: CVE-2023-36632\n- CVE description: The legacy email.utils.parseaddr function in Python through 3.11.4 allows attackers to trigger "RecursionError: maximum recursion depth exceeded while calling a Python object" via a crafted argument. This argument is plausibly an untrusted value from an application\'s input data that was supposed to contain a name and an e-mail address. NOTE: email.utils.parseaddr is categorized as a Legacy API in the documentation of the Python email package. Applications should instead use the email.parser.BytesParser or email.parser.Parser class. NOTE: the vendor\'s perspective is that this is neither a vulnerability nor a bug. The email package is intended to have size limits and to throw an exception when limits are exceeded; they were exceeded by the example demonstration code.\n- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H\n- CWE Name: CWE-674: Uncontrolled Recursion (4.14)\n- CWE Description: The product does not properly control the amount of recursion that takes place, consuming excessive resources, such as allocated memory or the program stack.\n- Notable Vulnerable Software Vendors: [\'Python\']\n\nExample {idx}: Checklist:\n[\n\t"Identify Usage of `email.utils.parseaddr`: Review the application code within the Docker container to check if the `email.utils.parseaddr` function is being used. This function is the specific target of the vulnerability. Consider searching the codebase for the string \'email.utils.parseaddr\' to find direct usages.",\n\t"Assess Input Data Handling: Determine if the application using `email.utils.parseaddr` processes potentially untrusted input data that could include crafted arguments designed to exploit this vulnerability. Evaluate how the application handles exceptions like `RecursionError`, as the exploit triggers this specific error.",\n\t"Review Alternative Implementations: Check if the application has already migrated to recommended alternatives such as `email.parser.BytesParser` or `email.parser.Parser`. If not, recommend transitioning away from the legacy `email.utils.parseaddr` to these safer alternatives."\n]', + "Example {idx}: CVE Details:\n- CVE ID: CVE-2023-50447\n- CVE description: Pillow through 10.1.0 allows PIL.ImageMath.eval Arbitrary Code Execution via the environment parameter, a different vulnerability than CVE-2022-22817 (which was about the expression parameter).\n- CVSS Vector: CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H\n- CWE Name: CWE-94: Improper Control of Generation of Code ('Code Injection') (4.14)\n- CWE Description: The product constructs all or part of a code segment using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the syntax or behavior of the intended code segment.\nWhen a product allows a user's input to contain code syntax, it might be possible for an attacker to craft the code in such a way that it will alter the intended control flow of the product. Such an alteration could lead to arbitrary code execution. Injection problems encompass a wide variety of issues -- all mitigated in very different ways. For this reason, the most effective way to discuss these weaknesses is to note the distinct features which classify them as injection weaknesses. The most important issue to note is that all injection problems share one thing in common -- i.e., they allow for the injection of control plane data into the user-controlled data plane. This means that the execution of the process may be altered by sending code in through legitimate data channels, using no other mechanism. While buffer overflows, and many other flaws, involve the use of some further issue to gain execution, injection problems need only for the data to be parsed. The most classic instantiations of this category of weakness are SQL injection and format string vulnerabilities.\n- Notable Vulnerable Software Vendors: ['Debian', 'Python']\n\nExample {idx}: Checklist:\n[\n\t\"Evaluate Use of PIL.ImageMath.eval: Specifically investigate whether any application within the container uses the PIL.ImageMath.eval function with user-supplied input in the environment parameter. This is the method and parameter where the vulnerability exists.\",\n\t\"Assess Data Input Sources: Determine if the applications using Pillow are receiving input directly from untrusted sources (e.g., user uploads, external APIs). This will help in understanding the risk exposure and potential for exploitation.\",\n\t\"Security Controls and Sanitization: Review the application's input validation and sanitization measures. Since the vulnerability allows for arbitrary code execution via code injection, ensuring that input is properly sanitized before being processed can mitigate potential exploitation.\"\n]", + 'Example {idx}: CVE Details:\n- CVE ID: CVE-2023-5363\n- CVE description: Issue summary: A bug has been identified in the processing of key and initialisation vector (IV) lengths. This can lead to potential truncation or overruns during the initialisation of some symmetric ciphers. Impact summary: A truncation in the IV can result in non-uniqueness, which could result in loss of confidentiality for some cipher modes. When calling EVP_EncryptInit_ex2(), EVP_DecryptInit_ex2() or EVP_CipherInit_ex2() the provided OSSL_PARAM array is processed after the key and IV have been established. Any alterations to the key length, via the "keylen" parameter or the IV length, via the "ivlen" parameter, within the OSSL_PARAM array will not take effect as intended, potentially causing truncation or overreading of these values. The following ciphers and cipher modes are impacted: RC2, RC4, RC5, CCM, GCM and OCB. For the CCM, GCM and OCB cipher modes, truncation of the IV can result in loss of confidentiality. For example, when following NIST\'s SP 800-38D section 8.2.1 guidance for constructing a deterministic IV for AES in GCM mode, truncation of the counter portion could lead to IV reuse. Both truncations and overruns of the key and overruns of the IV will produce incorrect results and could, in some cases, trigger a memory exception. However, these issues are not currently assessed as security critical.\nChanging the key and/or IV lengths is not considered to be a common operation and the vulnerable API was recently introduced. Furthermore it is likely that application developers will have spotted this problem during testing since decryption would fail unless both peers in the communication were similarly vulnerable. For these reasons we expect the probability of an application being vulnerable to this to be quite low. However if an application is vulnerable then this issue is considered very serious. For these reasons we have assessed this issue as Moderate severity overall.\nThe OpenSSL SSL/TLS implementation is not affected by this issue. The OpenSSL 3.0 and 3.1 FIPS providers are not affected by this because the issue lies outside of the FIPS provider boundary. OpenSSL 3.1 and 3.0 are vulnerable to this issue.\n- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N\n- Notable Vulnerable Software Vendors: [\'Debian\', \'Netapp\', \'Openssl\']\n\nExample {idx}: Checklist:\n[\n\t"Identify Affected Cipher Modes: Determine if the application within the Docker container uses any of the affected cipher modes: RC2, RC4, RC5, CCM, GCM, or OCB. Special attention should be given to applications using CCM, GCM, and OCB modes as these are particularly noted for potential loss of confidentiality due to IV truncation.",\n\t"Review Cryptographic Operations: Examine the code or configuration of applications using OpenSSL for any instances where `EVP_EncryptInit_ex2()`, `EVP_DecryptInit_ex2()`, or `EVP_CipherInit_ex2()` are called. Check if there are any modifications to the `keylen` or `ivlen` parameters after initialization which might not be taking effect as intended.",\n\t"Check for Custom Cryptographic Implementations: Since changing the key and/or IV lengths is not a common operation and the issue is in a recently introduced API, it\'s crucial to identify if any custom cryptographic implementations might be performing such operations. This is less likely but should be checked especially in bespoke or highly customized applications."\n]', + 'Example {idx}: CVE Details:\n- CVE ID: CVE-2024-2961\n- CVE description: The iconv() function in the GNU C Library versions 2.39 and older may overflow the output buffer passed to it by up to 4 bytes when converting strings to the ISO-2022-CN-EXT character set, which may be used to crash an application or overwrite a neighbouring variable.\n- CWE Name: CWE-787: Out-of-bounds Write (4.14)\n- CWE Description: The product writes data past the end, or before the beginning, of the intended buffer.\nTypically, this can result in corruption of data, a crash, or code execution. The product may modify an index or perform pointer arithmetic that references a memory location that is outside of the boundaries of the buffer. A subsequent write operation then produces undefined or unexpected results.\n- Notable Vulnerable Software Vendors: [\'GNU\']\n\nExample {idx}: Checklist:\n[\n\t"Identify Usage of `iconv()` Function: Review the application code or dependencies to check if the `iconv()` function is used, particularly for conversions involving the ISO-2022-CN-EXT character set. This function is the specific target of the vulnerability.",\n\t"Assess Data Handling and Boundary Conditions: Since the vulnerability involves an out-of-bounds write, it\'s crucial to analyze how data boundaries are handled in the code. Look for any custom implementations or patches that might mitigate boundary issues around buffer sizes.",\n\t"Review Application\'s Character Encoding Needs: Determine if the application specifically needs to handle the ISO-2022-CN-EXT character set. If not, consider disabling this character set or using alternative safe functions or libraries for character set conversions.",\n\t"Evaluate Network Exposure and Attack Surface: Consider whether the affected services are exposed to the network, which could increase the risk of exploitation. If the application using the `iconv()` function is accessible externally, the risk is higher."\n]', + "Example {idx}: CVE Details:\n- CVE ID: GHSA-8ghj-p4vj-mr35\n- CVE description: An issue was discovered in Pillow before 10.0.0. It is a Denial of Service that uncontrollably allocates memory to process a given task, potentially causing a service to crash by having it run out of memory. This occurs for truetype in ImageFont when textlength in an ImageDraw instance operates on a long text argument.\n- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H\n- CWE Name: CWE-770: Allocation of Resources Without Limits or Throttling (4.14)\n- CWE Description: The product allocates a reusable resource or group of resources on behalf of an actor without imposing any restrictions on the size or number of resources that can be allocated, in violation of the intended security policy for that actor.\n- Code frequently has to work with limited resources, so programmers must be careful to ensure that resources are not consumed too quickly, or too easily. Without use of quotas, resource limits, or other protection mechanisms, it can be easy for an attacker to consume many resources by rapidly making many requests, or causing larger resources to be used than is needed. When too many resources are allocated, or if a single resource is too large, then it can prevent the code from working correctly, possibly leading to a denial of service.\n- Notable Vulnerable Software Vendors: ['Fedoraproject', 'Python']\n- GHSA Summary: Pillow Denial of Service vulnerability\n- GHSA Details: [<'first_patched_version': '10.0.0', 'package': <'ecosystem': 'pip', 'name': 'pillow'>, 'vulnerable_functions': ['PIL.ImageFont'], 'vulnerable_version_range': '>= 0, < 10.0.0'>]\n- GHSA CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H\n\nExample {idx}: Checklist:\n[\n\t\"Assess Usage of Vulnerable Functions: Specifically, the vulnerability is related to the `PIL.ImageFont` module when processing long text arguments. Review the application code or dependencies to see if this module and functionality are used. If your applications use this module to process user-supplied or uncontrolled text inputs, they are likely at risk.\",\n\t\"Evaluate Resource Limits: Since the vulnerability leads to a denial of service through memory exhaustion, check if there are any resource limits set at the container level (e.g., using Docker or Kubernetes settings) that might mitigate the impact of such an attack. Consider setting or reviewing memory limits to prevent a single container from consuming all available system resources.\"\n]", +] + +FEW_SHOT = """Generate a checklist for a security analyst to use when assessing the exploitability of a specific CVE within a containerized environment. Use the provided examples as a guide to understand how to construct a checklist from a given set of CVE details, then apply this understanding to create a specific checklist for the CVE details provided below. All output should be a comma separated list enclosed in square brackets with each list item enclosed in quotes. + +Example 1: CVE Details: +- CVE ID: CVE-2024-23334 +- CVE Description: aiohttp is an asynchronous HTTP client/server framework for asyncio and Python. When using aiohttp as a web server and configuring static routes, it is necessary to specify the root path for static files. Additionally, the option 'follow_symlinks' can be used to determine whether to follow symbolic links outside the static root directory. When 'follow_symlinks' is set to True, there is no validation to check if reading a file is within the root directory. This can lead to directory traversal vulnerabilities, resulting in unauthorized access to arbitrary files on the system, even when symlinks are not present. Disabling follow_symlinks and using a reverse proxy are encouraged mitigations. Version 3.9.2 fixes this issue. +- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N +- CWE Name: CWE-22: Improper Limitation of a Pathname to a Restricted Directory ('Path Traversal') (4.14) +- CWE Description: The product uses external input to construct a pathname that is intended to identify a file or directory that is located underneath a restricted parent directory, but the product does not properly neutralize special elements within the pathname that can cause the pathname to resolve to a location that is outside of the restricted directory. +Many file operations are intended to take place within a restricted directory. By using special elements such as ".." and "/" separators, attackers can escape outside of the restricted location to access files or directories that are elsewhere on the system. One of the most common special elements is the "../" sequence, which in most modern operating systems is interpreted as the parent directory of the current location. This is referred to as relative path traversal. Path traversal also covers the use of absolute pathnames such as "/usr/local/bin", which may also be useful in accessing unexpected files. This is referred to as absolute path traversal. In many programming languages, the injection of a null byte (the 0 or NUL) may allow an attacker to truncate a generated filename to widen the scope of attack. For example, the product may add ".txt" to any pathname, thus limiting the attacker to text files, but a null injection may effectively remove this restriction. +- Notable Vulnerable Software Vendors: ['Aiohttp', 'Fedoraproject'] + +Example 1: Checklist: +[ + "Vulnerable package check. Does the project use the aiohttp library, which is the affected package? If aiohttp is not a dependency in your project, then your code is not vulnerable to this CVE.", + "Vulnerable version check. Is the version of aiohttp that the project depends on vulnerable? According to the vulnerability details, versions before 3.9.2 are vulnerable., + "Review code to check for vulnerability mitigation. Is the 'follow_symlinks' option set to False to mitigate the risk of directory traversal vulnerabilities?" +] + +Example 2: CVE Details: +- CVE ID: CVE-2022-2309 +- CVE description: NULL Pointer Dereference allows attackers to cause a denial of service (or application crash). This only applies when lxml (version 4.9.0 and earlier) is used together with libxml2 2.9.10 through 2.9.14. libxml2 2.9.9 and earlier are not affected. It allows triggering crashes through forged input data, given a vulnerable code sequence in the application. The vulnerability is caused by the iterwalk function (also used by the canonicalize function). Such code shouldn't be in wide-spread use, given that parsing + iterwalk would usually be replaced with the more efficient iterparse function. However, an XML converter that serialises to C14N would also be vulnerable, for example, and there are legitimate use cases for this code sequence. If untrusted input is received (also remotely) and processed via iterwalk function, a crash can be triggered. +- CVSS Vector: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H +- CWE Name: CWE-476: NULL Pointer Dereference (4.14) +- CWE Description: A NULL pointer dereference occurs when the application dereferences a pointer that it expects to be valid, but is NULL, typically causing a crash or exit. +NULL pointer dereference issues can occur through a number of flaws, including race conditions, and simple programming omissions. +- Notable Vulnerable Software Vendors: ['Fedoraproject', 'Lxml', 'Xmlsoft'] + +Example 2: Checklist: +[ + “Vulnerable package check. Does the project use the lxml library, which is the affected package? If lxml is not a dependency in your project, then your code is not vulnerable to this CVE.”, + “Vulnerable version check. Is the version of lxml that the project depends on vulnerable? According to the vulnerability details, versions 4.9.0 and earlier are vulnerable.”, + “Vulnerable version check of connected dependency. Is the version of libxml, the connected dependency, that the project depends on vulnerable? The package is only vulnerable if libxml 2.9.10 through 2.9.14 is also present.”, + “Review code for vulnerable functionality. The library is vulnerable through its `iterwalk` function, which is also utilized by the `canonicalize` function. Are either of these functions used in your code base?” +] + +Given CVE Details: +""" + additional_intel_prompting + +MOD_FEW_SHOT = """Generate a checklist for a security analyst to use when assessing the exploitability of a specific CVE within a containerized environment. Use the provided examples as a guide to understand how to construct a checklist from a given set of CVE details, then apply this understanding to create a specific checklist for the CVE details provided below. All output should be a comma separated list enclosed in square brackets with each list item enclosed in quotes. + +{examples} + +Given CVE Details: +""" + +investigation_guideline = """1. Is the flagged component in the product? Determine if the Docker container includes the flagged package version. Verify the presence of the library or software in question by checking the container's software bill of materials (SBOM). +2. Is the vulnerable code of the flagged component in the product? Check if an application or any dependency within the Docker container uses a function or a component of the library that contains the vulnerability. Check whether a patch has been applied or if the vulnerable code has been removed. +3. Is the vulnerable code of the flagged component executed by the product? Analyze the application's execution paths to confirm whether the vulnerable code is executed during normal operation. Examine the application's dependencies to ensure there are no indirect execution paths that could trigger the vulnerable code. +4. If the vulnerable code is executed, is it exploitable? Investigate whether the exploitability of the issue depends on a specific configuration option and if this configuration is enabled. +5. If the vulnerable code is executed, is it exploitable? Determine if the exploitability relies on a library dependency and verify the existence of this dependency in the product. +6. If the vulnerable code is executed, is it exploitable? Research if the exploitability of the issue depends on a specific environment and verify whether this environment is absent or present in the container. +7. Does other protection exist? Compiler Flags: Assess whether the exploitability hinges on the setting or unsetting of compiler flags. +8. Does other protection exist? Runtime Protections: Investigate the presence of mechanisms that prevent exploits during runtime. +9. Does other protection exist? Perimeter Defenses: Explore protective measures that block attacks at the physical, logical, or network perimeters. +10. Does other protection exist? Mitigating Controls: Identify any mitigating controls in place to prevent exploitability. +""" + +ZERO_SHOT = f"""Your task is to create a checklist for a security analyst to help determine if a given CVE is +exploitable in a containerized environment. Use the information provided below, structured within XML tags for both CVE +details and investigation guidelines. The checklist should convert the steps from the investigation guidelines into +actionable steps specific to the CVE. Each step should be self-contained, including details specific to the CVE, and +start with an action verb to clearly state the step the analyst needs to take. Avoid references to the CVE ID, focusing +on providing detailed investigative directions within the steps themselves. +All output should be a comma separated list enclosed in square brackets with each list item enclosed in quotes.\n +{additional_intel_prompting}\n{investigation_guideline}""" + +_ONE_SHOT = """This is an example of (1) CVE information, and (2) a checklist produced to determine if a given CVE is exploitable in a containerized environment: +(1) CVE Information: +CVE Description: DISPUTED: In Apache Batik 1.x before 1.10, when deserializing subclass of `AbstractDocument`, the class takes a string from the inputStream as the class name which then use it to call the no-arg constructor of the class. Fix was to check the class type before calling newInstance in deserialization. +CVSS Vector: CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H +CWE Name: CWE-502: Deserialization of Untrusted Data (4.11) +CWE Description: The product deserializes untrusted data without sufficiently verifying that the resulting data will be valid. It is often convenient to serialize objects for communication or to save them for later use. However, deserialized data or code can often be modified without using the provided accessor functions if it does not use cryptography to protect itself. Furthermore, any cryptography would still be client-side security -- which is a dangerous security assumption. Data that is untrusted can not be trusted to be well-formed. When developers place no restrictions on gadget chains, or series of instances and method invocations that can self-execute during the deserialization process (i.e., before the object is returned to the caller), it is sometimes possible for attackers to leverage them to perform unauthorized actions, like generating a shell. +Notable Vulnerable Software Vendors: ['Oracle', 'Apache', 'Canonical', 'Debian'] +(2) Checklist: +Based on the information available, CVE-2018-8013 is a critical vulnerability that involves the deserialization of untrusted data in Apache Batik before version 1.10. The Common Vulnerability Scoring System (CVSS) 3.x score is 9.8, indicating a critical severity. +All output should be a comma separated list enclosed in square brackets with each list item enclosed in quotes. +Here's an example of preliminary checklist similiar to what you should create to determine if a Docker container is vulnerable to this exploit: +["1. Check the Apache Batik version: The vulnerability affects Apache Batik versions 1.0 to 1.9.1. If your Docker container is running Apache Batik, verify the version. If it's between 1.0 and 1.9.1, it's vulnerable and you should update to version 1.10 or later.", +"2. Check for any software using vulnerable Apache Batik versions: It's important to note that other applications might use Apache Batik and thus be vulnerable. Check all applications within your Docker container for dependencies on vulnerable versions of Apache Batik. Some notable software includes certain versions of Ubuntu Linux, Debian Linux, and multiple Oracle applications.", +"3. Evaluate the deserialization risk: Since the vulnerability involves deserialization of untrusted data, you should consider whether your applications are performing such operations. If they are, consider if these deserialization operations are using untrusted, user-supplied data. If so, this is a potential attack vector.", +"4. Network exposure: The vulnerability has a network attack vector. Consider whether your Docker container has network exposure, especially if the Apache Batik service or any service using it is exposed to the internet.", +"5. Disputed status: CVE-2018-8013 is disputed and may not be a vulnerability."] + +Given the following information about {{cve_id}}, make preliminary checklist for a security analyst to follow to determine whether a Docker container is vulnerable to this exploit. +""" + additional_intel_prompting + + +def get_mod_examples(type='questions', choices=[0, 1]): + if type == 'questions': + ex_list = [q for idx, q in enumerate(ex_questions) if idx in choices] + else: + ex_list = [s for idx, s in enumerate(ex_statements) if idx in choices] + + examples = '\n'.join(q.format(idx=idx + 1) for idx, q in enumerate(ex_list)) + return examples diff --git a/src/cve/utils/sbom_parsers/__init__.py b/src/cve/utils/sbom_parsers/__init__.py new file mode 100644 index 0000000..28d3a3a --- /dev/null +++ b/src/cve/utils/sbom_parsers/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/cve/utils/sbom_parsers/sbom_parser.py b/src/cve/utils/sbom_parsers/sbom_parser.py new file mode 100644 index 0000000..44c35b5 --- /dev/null +++ b/src/cve/utils/sbom_parsers/sbom_parser.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import abc +import importlib +import typing + +if (typing.TYPE_CHECKING): + from .syft_parser import Syft + + +class SBOMParser(metaclass=abc.ABCMeta): + + @abc.abstractmethod + def __init__(self, file: str, *args, **kwargs): + self.file = file + + @classmethod + @abc.abstractmethod + def get_name(cls) -> str: + pass + + @abc.abstractmethod + def parse(self) -> dict: + """ + The parse method takes the SBOM input and creates a dictionary where keys are the names of the + installed packages and values are the versions of those packages. + + Returns + ------- + dict[str, str] + The installed packages and their versions from the provided SBOM. + """ + pass + + @typing.overload + @staticmethod + def create(parser_type: typing.Literal["syft"], *parser_args, **parser_kwargs) -> "Syft": + pass + + @typing.overload + @staticmethod + def create(parser_type: str, *parser_args, **parser_kwargs) -> "SBOMParser": + pass + + @staticmethod + def create(parser_type: str | typing.Literal["syft"], *parser_args, **parser_kwargs): + """ + Returns an SBOM parser for use in the SBOM Look-up tool. + Parameters + ---------- + parser_type : str + The type of SBOM parser to create + parser_args : dict[str, typing.Any] + Additional keyword arguments to pass to the parser + """ + module_name = f".{parser_type.lower()}_parser" + module = importlib.import_module(module_name, __package__) + + mod_classes = dict([(name, cls) for name, cls in module.__dict__.items() if isinstance(cls, type)]) + + matching_classes = [name for name in mod_classes if name.lower() == parser_type] + + assert len(matching_classes) == 1, (f"Expected to find exactly one class with name {parser_type} in module " + f"{module_name}, but found {matching_classes}") + + class_ = getattr(module, matching_classes[0]) + + instance = class_(*parser_args, **parser_kwargs) + + return instance diff --git a/src/cve/utils/sbom_parsers/syft_parser.py b/src/cve/utils/sbom_parsers/syft_parser.py new file mode 100644 index 0000000..3a67b7f --- /dev/null +++ b/src/cve/utils/sbom_parsers/syft_parser.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +from .sbom_parser import SBOMParser + + +class Syft(SBOMParser): + + def __init__(self, file: str, *args, **kwargs): + assert file is not None, 'File parameter required for Syft parser' + super().__init__(file=file) + + @classmethod + def get_name(cls) -> str: + return 'syft' + + def parse(self) -> dict[str, str]: + if os.path.isfile(self.file): + with open(self.file, 'r') as sbom_in: + sbom = sbom_in.read() + else: + sbom = self.file + sbom_split = sbom.split('\n') + + sbom_map = dict() + for idx, line in enumerate(sbom_split): + if line.startswith('['): + if sbom_split[idx + 1].startswith(' Version:'): + sbom_map[line[1:-1].lower()] = sbom_split[idx + 1].split('\t')[-1].strip() + return sbom_map diff --git a/src/cve/utils/serp_api_wrapper.py b/src/cve/utils/serp_api_wrapper.py new file mode 100644 index 0000000..4bcb432 --- /dev/null +++ b/src/cve/utils/serp_api_wrapper.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import aiohttp +from langchain.pydantic_v1 import root_validator +from langchain.utilities.serpapi import SerpAPIWrapper +from langchain.utils.env import get_from_env + +from ..utils.async_http_utils import retry_async +from ..utils.url_utils import url_join + + +class MorpheusSerpAPIWrapper(SerpAPIWrapper): + + base_url: str = "https://serpapi.com" + max_retries: int = 10 + + @root_validator() + def validate_environment(cls, values: dict) -> dict: + """Validate the base URL from the environment.""" + + values["base_url"] = get_from_env(key="base_url", env_key="SERPAPI_BASE_URL", default=values["base_url"]) + + # Build from the base class + values = super().validate_environment(values) + + # Update the base URL for search_engine + values["search_engine"].BACKEND = values["base_url"] + + return values + + @retry_async() + async def _session_get_with_retry(self, session: aiohttp.ClientSession, url: str, params: dict) -> dict: + + async with session.get(url, params=params) as response: + res = await response.json() + return res + + # Override the method with hardcoded URL + async def aresults(self, query: str) -> dict: + """Use aiohttp to run query through SerpAPI and return the results async.""" + + def construct_url_and_params() -> tuple[str, dict[str, str]]: + params = self.get_params(query) + params["source"] = "python" + if self.serpapi_api_key: + params["serp_api_key"] = self.serpapi_api_key + params["output"] = "json" + + # Use the base path for the URL (add a "/" to ensure they get joined) + url = url_join(self.base_url, "search") + return url, params + + url, params = construct_url_and_params() + if not self.aiosession: + async with aiohttp.ClientSession() as session: + res = await self._session_get_with_retry(session, url, params) + else: + res = await self._session_get_with_retry(self.aiosession, url, params) + + return res + + @staticmethod + def _process_response(res: dict) -> str: + """Catch the ValueError and return a message if no good search result found.""" + try: + return SerpAPIWrapper._process_response(res) + except ValueError: + return "No good search result found" diff --git a/src/cve/utils/source_code_git_loader.py b/src/cve/utils/source_code_git_loader.py new file mode 100644 index 0000000..46171f7 --- /dev/null +++ b/src/cve/utils/source_code_git_loader.py @@ -0,0 +1,194 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os +import typing +from pathlib import Path + +from git import Blob as GitBlob +from git import Repo +from langchain_community.document_loaders.blob_loaders.schema import BlobLoader +from langchain_core.document_loaders.blob_loaders import Blob +from tqdm import tqdm + +PathLike = typing.Union[str, os.PathLike] + +logger = logging.getLogger(__name__) + + +class SourceCodeGitLoader(BlobLoader): + """ + Load `Git` repository files. + + The Repository can be local on disk available at `repo_path`, + or remote at `clone_url` that will be cloned to `repo_path`. + Currently, supports only text files. + + Each document represents one file in the repository. The `path` points to + the local Git repository, and the `branch` specifies the branch to load + files from. By default, it loads from the `main` branch. + """ + + def __init__( + self, + repo_path: PathLike, + clone_url: str | None = None, + branch: typing.Optional[str] = "main", + include: typing.Optional[typing.Iterable[str]] = None, + exclude: typing.Optional[typing.Iterable[str]] = None, + ): + """ + Initialize the Git loader. + + Parameters + ---------- + repo_path : PathLike + Path to the local Git repository. + clone_url : str | None, optional + URL to the remote Git repository, by default None + branch : typing.Optional[str], optional + Branch to load files from, by default "main" + include : typing.Optional[typing.Iterable[str]], optional + A list of file patterns to include. Uses the glob syntax, by default None + exclude : typing.Optional[typing.Iterable[str]], optional + A list of file patterns to exclude. Uses the glob syntax, by default None + """ + + self.repo_path = Path(repo_path) + self.clone_url = clone_url + self.branch = branch + + self.include = include + self.exclude = exclude + + self._repo: Repo | None = None + + def load_repo(self): + """ + Load the Git repository and return the GitPython `Repo` object. + + Returns + ------- + Repo + GitPython `Repo` object representing the Git repository. + + Raises + ------ + ValueError + If the repository path does not exist. + ValueError + If a different repository is already cloned at the path. + """ + + if (self._repo is not None): + return self._repo + + if not os.path.exists(self.repo_path) and self.clone_url is None: + raise ValueError(f"Path {self.repo_path} does not exist") + elif self.clone_url: + # If the repo_path already contains a git repository, verify that it's the + # same repository as the one we're trying to clone. + if os.path.isdir(os.path.join(self.repo_path, ".git")): + repo = Repo(self.repo_path) + # If the existing repository is not the same as the one we're trying to + # clone, raise an error. + if repo.remotes.origin.url != self.clone_url: + raise ValueError("A different repository is already cloned at this path.") + + logger.debug("Updating existing Git repo for URL '%s' and branch '%s'", self.clone_url, self.branch) + + # Reliable way to checkout the branch of a shallow clone + repo.git.fetch("origin", self.branch, depth=1) + repo.git.checkout("FETCH_HEAD") + else: + logger.debug("Cloning repository from URL: '%s' for branch/tag '%s'", self.clone_url, self.branch) + + repo = Repo.clone_from(self.clone_url, self.repo_path, branch=self.branch, depth=1) + + # When creating the directory, ensure its a safe directory + with repo.config_writer(config_level="global") as config: + config.add_value("safe", "directory", str(self.repo_path.absolute())) + + else: + logger.debug("Using existing Git repo at path: '%s' for branch/tag '%s'", self.repo_path, self.branch) + + repo = Repo(self.repo_path) + repo.git.checkout(self.branch) + + logger.debug("Loaded Git repository at path: '%s' for branch/tag '%s'", self.repo_path, self.branch) + + self._repo = repo + + return repo + + def yield_blobs(self) -> typing.Iterator[Blob]: + """ + Yield the blobs from the Git repository. One blob will be generated for each file in the repo which passes the + include and exclude filters. + + Returns + ------- + typing.Iterator[Blob] + An iterator of `Blob` objects representing the files in the repository. + + Yields + ------ + Iterator[typing.Iterator[Blob]] + An iterator of `Blob` objects representing the files in the repository. + """ + + repo = self.load_repo() + + logger.debug("Scanning documents for Git repository at path: '%s'", self.repo_path) + + all_files_in_repo = [str(item.path) for item in repo.tree().traverse() if isinstance(item, GitBlob)] + + base_path = Path(self.repo_path) + + include_files: set[str] = set() + exclude_files: set[str] = set() + + for inc in self.include or ["**/*"]: + include_files = include_files.union(set(str(x.relative_to(base_path)) for x in base_path.glob(inc))) + + for exc in self.exclude or {}: + exclude_files = exclude_files.union(set(str(x.relative_to(base_path)) for x in base_path.glob(exc))) + + # Filter out files that are not in the repo + include_files = include_files.intersection(all_files_in_repo) + + # Take the include files and remove the exclude files. + final_files = include_files - exclude_files + + logger.debug("Processing %d files in the Git repository at path: '%s'", len(final_files), self.repo_path) + + for f in tqdm(final_files): + + file_path = Path(f) + + abs_file_path = base_path / file_path + + rel_file_path = str(file_path) + + metadata = { + "source": rel_file_path, + "file_path": rel_file_path, + "file_name": file_path.name, + "file_type": file_path.suffix, + } + + yield Blob.from_path(abs_file_path, metadata=metadata) diff --git a/src/cve/utils/string_utils.py b/src/cve/utils/string_utils.py new file mode 100644 index 0000000..61c8196 --- /dev/null +++ b/src/cve/utils/string_utils.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import re + +# Find all substrings that start and end with quotes, allowing for spaces before a comma or closing bracket +re_quote_capture = re.compile( + r""" + (['"]) # Opening quote + ( # Start capturing the quoted content + (?:\\.|[^\\])*? # Non-greedy match for any escaped character or non-backslash character + ) # End capturing the quoted content + \1 # Matching closing quote + (?=\s*,|\s*\]) # Lookahead for whitespace followed by a comma or closing bracket, without including it in the match + """, + re.VERBOSE) + +# Using the ASCII flag to stricly match numbers 0-9 and not on unicode numbers which include characters like ٤. +CVE_RE = re.compile(r'^CVE-\d{4}-\d{4,7}$', re.ASCII) +# https://docs.github.com/en/code-security/security-advisories/working-with-global-security-advisories-from-the-github-advisory-database/about-the-github-advisory-database#about-ghsa-ids +GHSA_RE = re.compile(r'^GHSA(-[23456789cfghjmpqrvwx]{4}){3}', re.ASCII) + + +def is_valid_ghsa_id(ghsa_id): + return False if ghsa_id is None else GHSA_RE.match(ghsa_id) is not None + + +def is_valid_cve_id(cve_id): + return False if cve_id is None else CVE_RE.match(cve_id) is not None + + +def attempt_fix_list_string(s: str) -> str: + """ + Attempt to fix unescaped quotes in a string that represents a list to make it parsable. + + Parameters + ---------- + s : str + A string representation of a list that potentially contains unescaped quotes. + + Returns + ------- + str + The corrected string where internal quotes are properly escaped, ensuring it can be parsed as a list. + + Notes + ----- + This function is useful for preparing strings to be parsed by `ast.literal_eval` by ensuring that quotes inside + the string elements of the list are properly escaped. It adds brackets at the beginning and end if they are missing. + """ + # Check if the input starts with '[' and ends with ']' + s = s.strip() + if (not s.startswith('[')): + s = "[" + s + if (not s.endswith(']')): + s = s + "]" + + def fix_quotes(match): + # Extract the captured groups + quote_char, content = match.group(1), match.group(2) + # Escape quotes inside the string content + fixed_content = re.sub(r"(? str: + """ + Removes number prefix, e.g. '1.' from a string and returns the modified string. + """ + # Regular expression pattern to match 'number.' at the beginning of the string + pattern = r'^\d+\.' + + # Strip any leading whitespace that could interfere with regex + text = text.lstrip() + # Remove the matching pattern (if found) + text = re.sub(pattern, '', text) + # Strip any remaining whitespace + text = text.strip() + return text + + +def get_checklist_item_string(item_num: int, item: dict) -> str: + """ + Formats and returns a string containing the checklist item number, question, and answer. + """ + question = remove_number_prefix(item['question']) + answer = remove_number_prefix(item['response']) + return f"- Checklist Item {item_num}: {question}\n - Answer: {answer}" + + +def package_names_match(pkg1: str, pkg2: str) -> bool: + """ + Compares two package names to determine if they represent the same package, + ignoring version numbers and the optional 'g' suffix at the end. + + Package names that follow the GNU versioning convention (e.g., 'zlib1g') + will have the version-like part (numbers and 'g') removed before comparison. + """ + # Regex pattern to match version numbers and optional 'g' at the end of the package name + version_pattern = r'([0-9]+(\.[0-9]+)*g?)$' + + # Convert both package names to lowercase and remove version and 'g' suffix + cleaned_pkg1 = re.sub(version_pattern, '', pkg1.lower()) + cleaned_pkg2 = re.sub(version_pattern, '', pkg2.lower()) + + return cleaned_pkg1 == cleaned_pkg2 diff --git a/src/cve/utils/tools.py b/src/cve/utils/tools.py new file mode 100644 index 0000000..dbce3fe --- /dev/null +++ b/src/cve/utils/tools.py @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import warnings + +from packaging.version import InvalidVersion +from packaging.version import parse as parse_version +from pydpkg import Dpkg +from pydpkg.exceptions import DpkgVersionError + +from .sbom_parsers.sbom_parser import SBOMParser +from .sbom_parsers.syft_parser import Syft + +logger = logging.getLogger(f"morpheus.{__name__}") + +SBOM_PICKER: dict[str, type[SBOMParser]] = {Syft.get_name(): Syft} + + +def range_version_comparator(software_version: str, vulnerability_lower_range: str, vulnerability_upper_range: str): + """ + Compare a software's version to a range of vulnerable versions to determine vulnerability. + + Parameters + ---------- + software_version : str + The version of the software currently in use. + vulnerability_lower_range : str + The lower bound of the vulnerable version range. + vulnerability_upper_range : str + The upper bound of the vulnerable version range. + + Returns + ------- + bool + Returns True if the software version is within the range of vulnerable versions, + indicating potential vulnerability. + + Raises + ------ + InvalidVersion + If the version strings are not in a valid format, a warning is issued and alphabetic + comparison is used instead. + + Notes + ----- + This function assumes that the software is vulnerable if its version falls inclusively + between the lower and upper bounds of the vulnerability range. It uses the `parse_version` + function to interpret the versions and compares them accordingly. If `parse_version` fails, + Debian version parsing is attempted. Finally, if both of these fail, it falls + back to a simple string comparison. + """ + try: + sv = parse_version(str(software_version)) + lvv = parse_version(str(vulnerability_lower_range)) + uvv = parse_version(str(vulnerability_upper_range)) + return sv <= uvv and sv >= lvv + except InvalidVersion: + # Failed PEP440 versioning; moving on to Debian + pass + + try: + return Dpkg.compare_versions(str(software_version), + str(vulnerability_lower_range)) != -1 and Dpkg.compare_versions( + str(software_version), str(vulnerability_upper_range)) != 1 + except DpkgVersionError: + warnings.warn('Unable to parse provided versions. Using alpha sorting.', stacklevel=2) + # Fallback to alphabetic comparison if version parsing fails + return str(software_version) <= str(vulnerability_upper_range) and str(software_version) >= str( + vulnerability_lower_range) + + +def single_version_comparator(software_version: str, vulnerability_version: str): + """ + Compare a software's version to a known vulnerable version. + + Parameters + ---------- + software_version : str + The version of the software currently in use. + vulnerability_version : str + The version of the software that is known to be vulnerable. + + Returns + ------- + bool + Returns True if the software version is less than or equal to the vulnerability version, + indicating potential vulnerability. + + Raises + ------ + InvalidVersion + If the version strings are not in a valid format, a warning is issued and alphabetic + comparison is used instead. + """ + try: + sv = parse_version(str(software_version)) + vv = parse_version(str(vulnerability_version)) + return sv <= vv + except InvalidVersion: + # Failed PEP440 versioning; moving on to Debian + pass + try: + return Dpkg.compare_versions(str(software_version), str(vulnerability_version)) != 1 + except DpkgVersionError: + warnings.warn('Unable to parse provided versions. Using alpha sorting.', stacklevel=2) + return str(software_version) <= str(vulnerability_version) + + +def version_comparison(software_version: str): + """ + Compare a software's version to multiple known vulnerable versions. + + Parameters + ---------- + software_version : str + A string containing the software version to compare, and the vulnerable versions, + separated by commas. A single vulnerable version, a vulnerable range (two versions), + or multiple specific vulnerable versions can be provided. + + Returns + ------- + bool or str + Returns True if the software version matches any of the vulnerable versions, + or is within the vulnerable range. Returns a string message if the input doesn't + contain enough information for a comparison. + + Notes + ----- + This function can compare against a single vulnerable version, a range of versions, + or a list of specific versions. It uses the `single_version_comparator` for single comparisons, + and `range_version_comparator` for range comparisons. + """ + v = software_version.split(',') + if len(v) == 2: + return single_version_comparator(v[0], v[1]) + elif len(v) == 3: + return range_version_comparator(v[0], v[1], v[2]) + elif len(v) > 3: + return any([v[0] == v_ for v_ in v[1:]]) + else: + return "Couldn't able compare the software version, not enough input" + + +class SBOMChecker: + + def __init__(self, sbom_map: dict[str, str]): + + # Lowercase all keys in the SBOM map and strip whitespace + self.sbom_map = {k.lower().strip(): v for k, v in sbom_map.items()} + + def sbom_checker(self, package_name: str): + "use this tool to check the version of the software package from the SBOM" + "returns the software version if the package is present in the SBOM" + "if the package is not in the SBOM returns False" + + try: + version = self.sbom_map.get(package_name.lower().strip(), False) + except Exception as e: + warnings.warn(str(e), stacklevel=2) + version = False + return version diff --git a/src/cve/utils/url_utils.py b/src/cve/utils/url_utils.py new file mode 100644 index 0000000..d9b2c39 --- /dev/null +++ b/src/cve/utils/url_utils.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def url_join(*parts): + """ + Functionally similar to `os.path.join` but for URLs. This function will join the parts of a URL together, ensuring + that the resulting URL is valid and all `/` have been deduped. + + Returns + ------- + str + The joined URL. + """ + return "/".join(part.strip("/") for part in parts) diff --git a/src/cve/utils/vulnerable_dependency_checker.py b/src/cve/utils/vulnerable_dependency_checker.py new file mode 100644 index 0000000..d375088 --- /dev/null +++ b/src/cve/utils/vulnerable_dependency_checker.py @@ -0,0 +1,679 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import asyncio +import logging +import os +import re +import urllib.parse + +import aiohttp +import aiohttp.client_exceptions +import aiohttp.http_exceptions +import requests +from tqdm import tqdm +from univers import versions + +from ..data_models.cve_intel import CveIntelNvd +from ..data_models.dependencies import DependencyPackage +from .async_http_utils import request_with_retry +from .clients.intel_client import IntelClient +from .string_utils import package_names_match +from .url_utils import url_join + +logger = logging.getLogger(f"morpheus.{__name__}") + +REQUESTS_TIMEOUT = 2 + +# Mapping of ecosystem strings to standard ecosystem names in deps.dev and univers +SYS_STANDARD_MAPPING = { + ".net": "nuget", + "cargo": "cargo", + "composer": "composer", + "conan": "conan", + "conda": "pypi", + "deb": "deb", + "dpkg": "deb", + "go": "go", + "go-module": "go", + "golang": "go", + "java": "maven", + "maven": "maven", + "node.js": "npm", + "npm": "npm", + "nuget": "nuget", + "php": "composer", + "pip": "pypi", + "pypi": "pypi", + "python": "pypi", + "rpm": "rpm", + "ruby": "rubygems", + "rubygems": "rubygems", + "rust": "cargo" +} + +# Mapping of standard ecosystem names to univers structured version functions +SYS_VERSION_FUNC_MAPPING = { + "pypi": versions.PypiVersion, + "npm": versions.SemverVersion, + "rubygems": versions.RubygemsVersion, + "deb": versions.DebianVersion, + "maven": versions.MavenVersion, + "nuget": versions.NugetVersion, + "rpm": versions.RpmVersion, + "composer": versions.ComposerVersion, + "go": versions.GolangVersion, + "conan": versions.ConanVersion, +} + +# Standard ecosystem names supported in deps.dev API https://docs.deps.dev/api/v3/#getpackage +DEPDEV_SUPPORTED_SYS = set(["go", "npm", "cargo", "maven", "pypi", "nuget"]) + + +def ubuntu_check(version: str = None): + version = str(version) + if version is None: + return False + if "ubuntu" in version: + return True + return False + + +def deb_check(version: str = None): + version = str(version) + if version is None: + return False + if "deb" in version: + return True + return False + + +def rhel_check(version: str = None): + version = str(version) + if version is None: + return False + if "el" in version: + return True + return False + + +def _is_valid_comparison(v, u): + return ubuntu_check(v) == ubuntu_check(u) and deb_check(u) == deb_check(v) and rhel_check(u) == rhel_check(v) + + +class VulnerableDependencyChecker(IntelClient): + """ + This tool will scan through the input SBOM file and identify all vulnerable packages and dependencies + indicated by the provided CVEs. + """ + + def __init__(self, + *, + base_url: str | None = None, + image: str, + sbom_list: list, + session: aiohttp.ClientSession | None = None, + respect_retry_after_header: bool = True): + + super().__init__( + session=session, + base_url=base_url or os.environ.get('DEPSDEV_BASE_URL'), + retry_count=1, # Service returns 404 if the package is not found. So dont retry + respect_retry_after_header=respect_retry_after_header) + + self._semaphore = asyncio.Semaphore(25) + + self.image = image + self.ref_packages = sbom_list + self.found_count = 0 + + @classmethod + def default_base_url(cls) -> str: + return "https://api.deps.dev/" + + @staticmethod + def _check_version_in_range(version_to_check: str, version_range: list[str], system: str) -> bool: + """Determine if a given version is within a specified range. + + Args: + version_to_check (str): The version of the target package to check, either str or None + version_range (list[str]): A set containing four elements: the start/end of the version range + (inclusive/exclusive). + system (str): The ecosystem of the package, either str or None + + Returns: + bool: True if the version is within the specified range, False otherwise. + """ + + if not version_to_check: + raise ValueError("Invalid version to check. version_to_check is empty.") + elif all([not v for v in version_range]): + raise ValueError("Invalid version range. version_range is empty.") + + all_versions = version_range + [version_to_check] + if max([ubuntu_check(v) or deb_check(v) for v in all_versions]): + system = 'deb' + if max([rhel_check(v) for v in all_versions]): + system = 'rpm' + + # Get structured version conversion function based on the package system. Use GenericVersion as default. + version_func = SYS_VERSION_FUNC_MAPPING.get(system, versions.GenericVersion) + + # Convert version strings to structured version objects + try: + version_to_check = version_func(version_to_check) + ver_start_excl, ver_end_excl, ver_start_incl, ver_end_incl = [ + version_func(v) if v is not None else v for v in version_range + ] + except versions.InvalidVersion as e: + raise e + + # Check the version range start + if ver_start_incl and _is_valid_comparison(version_to_check, ver_start_incl): + matches_start = ver_start_incl <= version_to_check + elif ver_start_excl and _is_valid_comparison(version_to_check, ver_start_excl): + matches_start = ver_start_excl < version_to_check + else: + matches_start = True + + # Check the version range end + if ver_end_incl and _is_valid_comparison(version_to_check, ver_end_incl): + matches_end = version_to_check <= ver_end_incl + elif ver_end_excl and _is_valid_comparison(version_to_check, ver_end_excl): + matches_end = version_to_check < ver_end_excl + else: + matches_end = True + + return matches_start and matches_end + + @staticmethod + def _check_package_vulnerability(package: DependencyPackage, system: str, vuln: CveIntelNvd.Configuration) -> bool: + """Determine if a package is vulnerable for a configuration based on its name and version. + + Args: + package (DependencyPackage): package to check against the vulnerable configuration. + system (str): standardized system name. + vuln (CveIntelNvd.Configuration): vulnerable configuration information. + + Returns: + bool: True if the package matches the vulnerable configuration or version check fails, False otherwise. + """ + + # First, ensure package names match + if not package_names_match(package.name, vuln.package): + return False + + if 'ubuntu' in package.version: + system = 'deb' + + # If package names match, check the version + try: + return VulnerableDependencyChecker._check_version_in_range(package.version, + [ + vuln.versionStartExcluding, + vuln.versionEndExcluding, + vuln.versionStartIncluding, + vuln.versionEndIncluding, + ], + system) + except Exception as e: + logger.warning("Version comparison failed for package %s, assuming package may be vulnerable. Error: %s. ", + package.name, + e) + return True + + async def _is_maven_package(self, package_name: str, base_url): + """Check if a package exists in Maven. + + Args: + package_name (str): Package name in the format "group_id:artifact_id" + base_url (str): Base URL for the Maven repository + + Returns: + bool: True if the package exists, False otherwise + """ + parts = package_name.split(":") + if len(parts) != 2: + return False + group_id, artifact_id = parts + + query = f"g:{group_id}+AND+a:{artifact_id}" + payload = {"q": query, "core": "gav", "rows": "20", "wt": "json"} + + payload_str = urllib.parse.urlencode(payload, safe=":+") + + try: + response = await self.request(method="GET", url=base_url, params=payload) + return response["response"]["numFound"] > 0 + except requests.exceptions.RequestException: + return False + + async def _infer_pkg_ecosystem(self, package_name: str, version: str = None) -> str: + """Attempt to infer the ecosystem given the package name and version (this might not be 100% accurate) + + Args: + package_name (str): Package name + + Returns: + str: Ecosystem name (e.g. pip, npm, maven, etc.) or "unknown" if not found + """ + # Try to first infer based on version name + if ubuntu_check(version) or deb_check(version): + return "deb" + elif rhel_check(version): + return "rpm" + + # Define the mapping of ecosystems to their registries + registry_mapping = { + "pip": "https://pypi.org/project/", + "npm": "https://registry.npmjs.org/", + "go": "https://pkg.go.dev/", + "maven": "https://search.maven.org/solrsearch/select", + "nuget": "https://www.nuget.org/packages/", + "rubygems": "https://rubygems.org/gems/", + "rust": "https://crates.io/api/v1/crates/", + "erlang": "https://hex.pm/api/packages/", + "composer": "https://packagist.org/packages/", + "conan": "https://cpp.libhunt.com/", + } + + # Check each registry to see if the package exists + for ecosystem, base_url in registry_mapping.items(): + if ecosystem == "maven": + if await self._is_maven_package(package_name, base_url): + return ecosystem + else: + url = f"{base_url}{package_name}" + if ecosystem == "php": + url = f"{base_url}{package_name}.json" + elif ecosystem == "erlang": + url = f"{base_url}{package_name.lower()}" + + try: + response = await self._session.get(url, timeout=REQUESTS_TIMEOUT) + if response.status == 200: + if ecosystem == "composer": + content = await response.text(encoding="utf-8") + if ("vendor_not_found" in content or "package_not_found" in content): + continue + return ecosystem + except ( + aiohttp.http_exceptions.HttpProcessingError, + aiohttp.client_exceptions.ClientError, + asyncio.TimeoutError, + ): + continue + + return "unknown" + + async def _get_dependency(self, system=None, package: str = "", version: str = ""): + """Get dependencies for a package from deps.dev API + + Args: + system (str, optional): Ecosystem of the package. Defaults to None. + package (str, optional): Package name. Defaults to "". + version (str, optional): Package version. Defaults to "". + + Returns: + list: List of dependencies for the package. + """ + # Attempt to infer missing package ecosystem if not provided in the SBOM + if system is None: + system = await self._infer_pkg_ecosystem(package, version) + + self_package = [DependencyPackage( + system=system, + name=package, + version=version, + relation="SELF", + )] + + system = SYS_STANDARD_MAPPING.get(system) + + if system not in DEPDEV_SUPPORTED_SYS: + return self_package + + api_url = (f"v3/systems/{system}/packages/{package}/versions/{version}:dependencies") + url = url_join(self.base_url, api_url) + package_info = {"package": package, "system": system, "version": version} + + try: + + # Limit the number of concurrent requests to avoid overloading the server + async with self._semaphore: + + data = await self.request(method="GET", url=url, log_on_error=False) + if not data: + raise ValueError(f"Response from {self.base_url} contains no dependencies for {package_info}") + + self.found_count += 1 + return [ + DependencyPackage( + system=node["versionKey"]["system"], + name=node["versionKey"]["name"], + version=node["versionKey"]["version"], + relation=node["relation"], + ) for node in data["nodes"] + ] + + # Handle packages with no dependencies found + except (ValueError, aiohttp.ClientResponseError) as e: + + # Only log warning for unexpected errors + if (isinstance(e, ValueError) or (isinstance(e, aiohttp.ClientResponseError) and e.status == 404)): + pass + else: + logger.warning("Error calling %s for %s: %s", self.base_url, package_info, e) + + return self_package + + async def _collect_all_dependencies(self): + """Collect all direct and indirect dependencies given the sbom package info + + Returns: + dict: A dictionary with the following structure: { + (sbom_pkg, pkg_version): [ + DependencyPackage( + system="unknown", + name=pkg_name, + version=version, + relation="SELF", + ), + ... + ] + } + """ + logger.info("Collecting SBOM package dependencies from %s for %s", self.base_url, self.image) + dependencies_info = {} + + with tqdm(total=len(self.ref_packages)) as pbar: + + async def _wrap_coro(pkg_info): + pkg_name, version, system = ( + pkg_info.name.lower(), + pkg_info.version, + pkg_info.system, + ) + + dep_out = await self._get_dependency(system, pkg_name, version) + + pbar.update(1) + + return pkg_name, version, dep_out + + dependency_coros = [_wrap_coro(pkg_info) for pkg_info in self.ref_packages] + + gathered_dep_info = await asyncio.gather(*dependency_coros) + + for pkg_name, version, dependencies in gathered_dep_info: + dependencies_info[(pkg_name, version)] = dependencies + + logger.info("Collected additional dependency info for %d of %d SBOM packages.", + self.found_count, + len(self.ref_packages)) + + return dependencies_info + + async def load_dependencies(self): + """ + Load all dependencies for the SBOM packages + """ + + self.dependencies = await self._collect_all_dependencies() + + async def _get_vuln_packages(self, vuln, dependencies_info): + """Retrieve packages that matches with CVE vulnerability, specific for ghsa, nvd inputs + + Args: + vuln (Configuration): see return format from _reconstruct_ghsa + dependencies_info (dict): see return format from _collect_all_dependencies + + Returns: + list: A list of dictionary with a structure as follows: + key (tuple): (pkg_name, version) => the affected package in SBOM + value (DependencyPackage): the matched package informaiton related to the vulnerability indicated by CVE + """ + if not vuln.package: + return [] + + system = vuln.system.lower() if vuln.system else await self._infer_pkg_ecosystem(vuln.package.lower()) + system = SYS_STANDARD_MAPPING.get(system, system) + + vulnerable_dep = [{ + root_pkg: dependency + } for root_pkg, + dependencies in dependencies_info.items() for dependency in dependencies + if VulnerableDependencyChecker._check_package_vulnerability(dependency, system, vuln)] + + return vulnerable_dep + + def _get_vuln_ubuntu_packages(self, notices, dependencies_info): + """Retrieve packages that match with CVE vulnerability, specific for ubuntu inputs + + Args: + notices (dict): example format: + { + "focal": [ + { + "description": "Simple Linux Utility for Resource Management", + "is_source": true, + "name": "slurm-llnl", + "version": "19.05.5-1ubuntu0.1~esm2" + }, ... + ], ... + } + dependencies_info (dict): see return format from _collect_all_dependencies + + Returns: + list: A list of dictionary with a structure as follows: + key: (pkg_name, version) => the affected package in SBOM + value: a list of matched vulnerable package information associated with CVE + """ + vulnerable_dep = [] + version_func = SYS_VERSION_FUNC_MAPPING["deb"] + + for patched_pkg_list in notices.values(): + for patched_pkg in patched_pkg_list: + name, version = patched_pkg.get("name"), patched_pkg.get("version") + + if not (name and version): + continue + + for root_pkg, dependencies in dependencies_info.items(): + for dependency in dependencies: + if package_names_match(dependency.name, name) and version_func( + dependency.version) < version_func(version): + vulnerable_dep.append({root_pkg: dependency}) + + return vulnerable_dep + + def _get_vuln_rhsa_packages(self, vuln_pkg_info, dependencies_info): + """Retrieve packages that match with CVE vulnerability, specific for ubuntu inputs + + Args: + vuln_package_info (list): example format: ["python39", "3.9-8100020240214182535.7044f6c1"] + dependencies_info (dict): see return format from _collect_all_dependencies + + Returns: + list: A list of dictionary with a structure as follows: + key: (pkg_name, version) => the affected package in SBOM + value: a list of matched vulnerable package information associated with CVE + """ + name, version = vuln_pkg_info + vulnerable_dep = [] + version_func = SYS_VERSION_FUNC_MAPPING["rpm"] + + if name and version: + for root_pkg, dependencies in dependencies_info.items(): + for dependency in dependencies: + if package_names_match(dependency.name, name) and version_func( + dependency.version) < version_func(version): + vulnerable_dep.append({root_pkg: dependency}) + + return vulnerable_dep + + @staticmethod + def _extract_range_structure(data, vuln_range): + """Turn the vulnerable_version_range string from ghsa to structure format + + Args: + data (Configuration): information before applying vuln_range + vuln_range (str): vulnerable_version_range string from ghsa. + - sample1: < 3.1.3 + - sample2: >= 0.14.0, < 14.0.1 + + Returns: + dict: structured output for updating data in function _reconstruct_ghsa + """ + vuln_list = vuln_range.split(",") + if len(vuln_list) > 1: + low, high = vuln_list[0].strip(), vuln_list[1].strip() + sign_low, low_v = low.split(" ") + sign_high, high_v = high.split(" ") + if sign_low == ">=": + data.versionStartIncluding = low_v + elif sign_low == ">": + data.versionStartExcluding = low_v + else: + high = vuln_list[0].strip() + sign_high, high_v = high.split(" ") + + if sign_high == "<=": + data.versionEndIncluding = high_v + elif sign_high == "<": + data.versionEndExcluding = high_v + + return data + + def _reconstruct_ghsa(self, vuln): + """Transform a ghsa vulnerability input to a defined structure + + Args: + vuln (dict): ghsa vulnerability input + + Returns: + Configuration: the defined versioning structure + """ + pkg_info = vuln.get("package", None) + if pkg_info: + data = CveIntelNvd.Configuration( + package=pkg_info.get("name", None), + system=pkg_info.get("ecosystem", None), + versionStartExcluding=None, + versionEndExcluding=vuln.get("first_patched_version", None), + versionStartIncluding=None, + versionEndIncluding=None, + ) + + vulnerable_version_range = vuln.get("vulnerable_version_range", None) + if vulnerable_version_range: + data = self._extract_range_structure(data, vulnerable_version_range) + + return data + + async def run_ghsa(self, vulnerabilities): + """Get vulnerable packages information from intel.ghsa.vulnerabilities + + Args: + vulnerabilities (list): list of vulnerabilities information from + intel.ghsa.vulnerabilities + sample input: [ + { + "package": { + "ecosystem": "pip", + "name": "mlflow" + }, + "vulnerable_version_range": "< 2.9.2", + "first_patched_version": "2.9.2", + "vulnerable_functions": [] + }, ... + ] + + list: A list of dictionary with a structure as follows: + - key (tuple): (pkg_name, version) => the affected package in SBOM + - value (DependencyPackage): the matched package informaiton related to the + vulnerability indicated by CVE + """ + vuln_pkgs = [] + for vuln in vulnerabilities: + vuln = self._reconstruct_ghsa(vuln) + vuln_pkg = await self._get_vuln_packages(vuln, self.dependencies) + if vuln_pkg: + vuln_pkgs.extend(vuln_pkg) + return vuln_pkgs + + async def run_nvd(self, vulnerabilities): + """Get vulnerable packages information from intel.nvd.configurations + + Args: + vulnerabilities (list): list of vulnerabilities information from + intel.nvd.configurations + + list: A list of dictionary with a structure as follows: + - key (tuple): (pkg_name, version) => the affected package in SBOM + - value (DependencyPackage): the matched package informaiton related to the + vulnerability indicated by CVE + """ + vuln_pkgs = [] + for vuln in vulnerabilities: + vuln_pkg = await self._get_vuln_packages(vuln, self.dependencies) + if vuln_pkg: + vuln_pkgs.extend(vuln_pkg) + return vuln_pkgs + + async def run_ubuntu(self, notices): + """Get vulnerable packages information from intel.ubuntu.notices + + Args: + notices (list): list of information from intel.ubuntu.notices + + Returns: + list: a list of package information from sbom that is related to the vulnerability + """ + vuln_pkgs = [] + for notice in notices: + ubuntu_patched_dict = notice.get("release_packages", {}) + if ubuntu_patched_dict: + vuln_pkg = self._get_vuln_ubuntu_packages(ubuntu_patched_dict, self.dependencies) + vuln_pkgs.extend(vuln_pkg) + + return vuln_pkgs + + async def run_rhsa(self, vulnerabilities): + """Get vulnerable packages information from intel.rhsa.affected_release + + Args: + vulnerabilities (list): list of vulnerabilities information from + intel.rhsa.affected_release. + example format: [{ + "product_name": "Red Hat Enterprise Linux 8", + "release_date": "2024-05-22T00:00:00Z", + "advisory": "RHSA-2024:2985", + "cpe": "cpe:/a:redhat:enterprise_linux:8", + "package": "python39:3.9-8100020240214182535.7044f6c1" + }, ...] + + Returns: + list: a list of package information from sbom that is related to the vulnerability + """ + vuln_pkgs = [] + for vuln in vulnerabilities: + cve_vuln_pkg = vuln.get("package", "").split(":") + if len(cve_vuln_pkg) == 2: + vuln_pkg = self._get_vuln_rhsa_packages(cve_vuln_pkg, self.dependencies) + vuln_pkgs.extend(vuln_pkg) + + return vuln_pkgs diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..e669949 --- /dev/null +++ b/src/main.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +# The purpose of this function is to allow loading the current directory as a module. This allows relative imports and +# more specifically `..common` to function correctly +def run_cli(): + import os + import sys + + main_parent_dir = os.path.dirname(os.path.dirname(__file__)) + + if (main_parent_dir not in sys.path): + sys.path.append(main_parent_dir) + + from src.cli import cli + + cli(obj={}, auto_envvar_prefix='MORPHEUS_LLM', show_default=True, prog_name="morpheus_llm") + + +if __name__ == '__main__': + run_cli()