Merge pull request #219 from BenediktMKuehne/SBOM

SBOM

VERSION.txt

@@ -1 +1 @@
-0.1-ebab62b9
+0.1-c58e97d5

dev-tools/debug-server-start.sh

@@ -91,10 +91,12 @@ if [[ "${WSL}" -eq 1 ]]; then
 fi
 
 # check emba
-echo -e "${BLUE}""${BOLD}""checking EMBA""${NC}"
-if ! (cd ./emba && ./emba -d 1) ; then
-  echo -e "${RED}""EMBA is not configured correctly""${NC}"
-  exit 1
+if ! grep -q "EMBA_INSTALL=no" ./.env ; then
+  echo -e "${BLUE}""${BOLD}""checking EMBA""${NC}"
+  if ! (cd ./emba && ./emba -d 1) ; then
+    echo -e "${RED}""EMBA is not configured correctly""${NC}"
+    exit 1
+  fi
 fi
 
 check_db

dev-tools/update_pipenv.sh

@@ -22,7 +22,7 @@ fi
 
 # 2. update pipfile
 if [[ "${INSTALL}" == "dev" ]]; then
-  MYSQLCLIENT_LDFLAGS='-L/usr/mysql/lib -lmysqlclient -lssl -lcrypto -lresolv' MYSQLCLIENT_CFLAGS='-I/usr/include/mysql/' PIPENV_VENV_IN_PROJECT=1 pipenv update --dev
+  MYSQLCLIENT_LDFLAGS='-L/usr/mysql/lib -lmysqlclient -lssl -lcrypto -lresolv' MYSQLCLIENT_CFLAGS='-I/usr/include/mysql/' PIPENV_VENV_IN_PROJECT=1 pipenv update --dev -v
 else
   echo "This is not a developer setup...still trying to update"
   MYSQLCLIENT_LDFLAGS='-L/usr/mysql/lib -lmysqlclient -lssl -lcrypto -lresolv' MYSQLCLIENT_CFLAGS='-I/usr/include/mysql/' PIPENV_VENV_IN_PROJECT=1 pipenv update

embark/dashboard/admin.py

@@ -4,6 +4,9 @@
 
 from django.contrib import admin
 
-from dashboard.models import Result
+from dashboard.models import Result, Vulnerability, SoftwareInfo, SoftwareBillOfMaterial
 
-admin.register(Result)
+admin.site.register(Result)
+admin.site.register(Vulnerability)
+admin.site.register(SoftwareInfo)
+admin.site.register(SoftwareBillOfMaterial)

embark/dashboard/models.py

@@ -2,8 +2,10 @@
 __author__ = 'Benedikt Kuehne'
 __license__ = 'MIT'
 
+import uuid
 from django.db import models
 from django.core.validators import MinLengthValidator
+from django.utils import timezone
 
 from uploader.models import FirmwareAnalysis
 
@@ -13,7 +15,33 @@ class Vulnerability(models.Model):
     Many-to-Many object for CVEs
     """
     cve = models.CharField(max_length=18, validators=[MinLengthValidator(13)], help_text='CVE-XXXX-XXXXXXX')
-    info = models.JSONField(null=True)
+    info = models.JSONField(null=True, editable=True)
+
+
+class SoftwareInfo(models.Model):
+    """
+    Many-to-one object for SBOM entries
+    """
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True)
+    type = models.CharField(verbose_name="type of blob", blank=False, editable=True, default="NA", max_length=256)
+    name = models.CharField(verbose_name="software name", blank=False, editable=True, default="NA", max_length=256)
+    group = models.CharField(verbose_name="grouping", blank=False, editable=True, default="NA", max_length=256)
+    version = models.CharField(verbose_name="software version", blank=False, editable=True, default="1.0", max_length=32)
+    hashes = models.CharField(verbose_name="identivication hash", blank=False, editable=True, default="NA", max_length=1024)
+    cpe = models.CharField(verbose_name="CPE identifier", blank=False, editable=True, default="NA", max_length=256)
+    type = models.CharField(verbose_name="software type", blank=False, editable=True, default="data", max_length=50)
+    purl = models.CharField(verbose_name="PUrl identifier", blank=False, editable=True, default="NA", max_length=256)
+    description = models.CharField(verbose_name="description", blank=False, editable=True, default="NA", max_length=1024)
+    properties = models.JSONField(verbose_name="Properties", null=True, editable=True, serialize=True)
+
+
+class SoftwareBillOfMaterial(models.Model):
+    """
+    1-to-1 object for result
+    """
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True)
+    meta = models.CharField(verbose_name="meta data sbom", blank=False, editable=True, default="NA", max_length=1024)
+    component = models.ManyToManyField(SoftwareInfo, help_text='Software Bill of Material', related_query_name='sbom', editable=True, blank=True)
 
 
 class Result(models.Model):
@@ -24,6 +52,7 @@ class Result(models.Model):
     firmware_analysis = models.OneToOneField(FirmwareAnalysis, on_delete=models.CASCADE, primary_key=True)
     emba_command = models.CharField(blank=True, null=True, max_length=(FirmwareAnalysis.MAX_LENGTH * 6), help_text='')
     restricted = models.BooleanField(default=False, help_text='')
+    date = models.DateTimeField(default=timezone.now, blank=True)
 
     # base identifier
     os_verified = models.CharField(blank=True, null=True, max_length=256, help_text='')
@@ -71,3 +100,4 @@ class Result(models.Model):
     system_bin = models.TextField(default='{}')
 
     vulnerability = models.ManyToManyField(Vulnerability, help_text='CVE/Vulnerability', related_query_name='CVE', editable=True, blank=True)
+    sbom = models.OneToOneField(SoftwareBillOfMaterial, help_text='Software Bill of Material', related_query_name='sbom', editable=True, blank=True, on_delete=models.CASCADE, null=True)

embark/dashboard/urls.py

@@ -9,6 +9,7 @@
 
 # view routing
 urlpatterns = [
+    path('', views.main_dashboard, name='embark-MainDashboard'),
     path('dashboard/main/', views.main_dashboard, name='embark-MainDashboard'),
     path('dashboard/service/', views.service_dashboard, name='embark-dashboard-service'),
     path('dashboard/report/', views.report_dashboard, name='embark-ReportDashboard'),

embark/embark/asgi.py

@@ -22,7 +22,7 @@
 
 from embark.routing import ws_urlpatterns
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.dev')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.deploy')
 
 
 application = ProtocolTypeRouter({

embark/embark/settings/deploy.py

@@ -23,17 +23,16 @@
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
 
-# HASHID_FIELD_SALT = os.environ.get('HASHID_SALT')
-# HASHID_FIELD_MIN_LENGTH = os.environ.get('HASHID_FIELD_MIN_LENGTH', 7)
-# HASHID_FIELD_ENABLE_HASHID_OBJECT = os.environ.get('HASHID_FIELD_ENABLE_HASHID_OBJECT', False)
-# HASHID_FIELD_ENABLE_DESCRIPTOR = os.environ.get('HASHID_FIELD_ENABLE_DESCRIPTOR', False)
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = False
 
 ALLOWED_HOSTS = ['*']
 
+DOMAIN = "embark.local"
+EMAIL_ACTIVE = False
+
 # EMBA stuff
 # EMBA location
 EMBA_ROOT = os.path.join(BASE_DIR.parent, 'emba')

embark/embark/settings/dev.py

@@ -21,6 +21,9 @@
 EMBA_LOG_URL = 'emba_logs/'
 
 DEBUG = True
+DOMAIN = "embark.local"
+EMAIL_ACTIVE = True
+
 
 INSTALLED_APPS = [
     'django.contrib.admin',
@@ -79,6 +82,7 @@
 
 SESSION_COOKIE_SAMESITE = 'Strict'
 SESSION_COOKIE_HTTPONLY = True
+SESSION_COOKIE_SECURE = False
 
 WSGI_APPLICATION = 'embark.wsgi.application'
 
@@ -184,16 +188,19 @@
 
 AUTH_PASSWORD_VALIDATORS = [
     {
-        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
+        "OPTIONS": {
+            "min_length": 8,
+        },
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
     },
 ]
 
@@ -223,10 +230,10 @@
 # STATICFILES_FINDERS
 
 # URL of Login-Page
-LOGIN_URL = ''
+LOGIN_URL = 'user/login/'
 
 # URL of Logout-Page
-LOGOUT_REDIRECT_URL = ''
+LOGOUT_REDIRECT_URL = 'user/logout'
 
 # Added for FIle storage to get the path to save Firmware images.
 MEDIA_ROOT = os.path.join(BASE_DIR.parent, 'media')
@@ -269,6 +276,7 @@
         },
     },
 }
+# TODO check this https://docs.djangoproject.com/en/5.1/topics/cache/
 TEMP_DIR = Path("/tmp/")
 
 try:

embark/embark/urls.py

@@ -17,12 +17,12 @@
 __author__ = 'm-1-k-3, RaviChandra, Garima Chauhan, Maximilian Wagner, diegiesskanne'
 __license__ = 'MIT'
 
-# from django.contrib import admin
+from django.contrib import admin
 from django.urls import path, include
 from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 
 urlpatterns = [
-    # path('admin/', admin.site.urls),
+    path('admin/', admin.site.urls),
     path('', include('updater.urls')),
     path('', include('uploader.urls')),
     path('', include('users.urls')),

embark/embark/wsgi.py

@@ -14,6 +14,6 @@
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.dev')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.deploy')
 
 application = get_wsgi_application()

embark/manage.py

@@ -7,7 +7,7 @@
 
 def main():
     """Run administrative tasks."""
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.dev')
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.deploy')
 
     try:
         # pylint: disable=import-outside-toplevel

embark/porter/forms.py

@@ -27,3 +27,7 @@ class FirmwareAnalysisExportForm(forms.Form):
 
 class DeleteZipForm(forms.Form):
     zip_file = forms.ModelChoiceField(queryset=LogZipFile.objects, empty_label='Select zip-file to delete')
+
+
+class RetryImportForm(forms.Form):
+    analysis = forms.ModelChoiceField(queryset=FirmwareAnalysis.objects.filter(failed=True))

embark/porter/importer.py

@@ -13,8 +13,9 @@
 import re
 
 from django.conf import settings
+from django.db import DatabaseError
 
-from dashboard.models import Vulnerability, Result
+from dashboard.models import SoftwareBillOfMaterial, SoftwareInfo, Vulnerability, Result
 from uploader.models import FirmwareAnalysis
 
 logger = logging.getLogger(__name__)
@@ -27,8 +28,8 @@ def result_read_in(analysis_id):
     """
     logger.debug("starting read-in of %s", analysis_id)
     res = None
-    directory = f"{settings.EMBA_LOG_ROOT}/{analysis_id}/emba_logs/csv_logs/"
-    csv_list = [os.path.join(directory, file_) for file_ in os.listdir(directory)]
+    csv_directory = f"{settings.EMBA_LOG_ROOT}/{analysis_id}/emba_logs/csv_logs/"
+    csv_list = [os.path.join(csv_directory, file_) for file_ in os.listdir(csv_directory)]
     for file_ in csv_list:
         logger.debug("trying to read: %s", file_)
         if os.path.isfile(file_):      # TODO change check. > if valid EMBA csv file
@@ -41,7 +42,23 @@ def result_read_in(analysis_id):
                 # FIXME f20 in emba is broken!
                 # res = f20_csv(file_, analysis_id)
                 # logger.debug("Result for %s created or updated", analysis_id)
-            # TODO license info etc
+    # json_directory = f"{settings.EMBA_LOG_ROOT}/{analysis_id}/emba_logs/json_logs/"
+    # json_list = [os.path.join(json_directory, file_) for file_ in os.listdir(json_directory)]
+    # for file_ in json_list:
+    #     logger.debug("trying to read: %s", file_)
+    #     if os.path.isfile(file_):      # TODO change check. > if valid EMBA json file
+    #         logger.debug("File %s found and attempting to read", file_)
+    #         if file_.endswith('f15_cyclonedx_sbom.json'):
+    #             logger.info("f15 readin for %s skipped", analysis_id)
+    #             # f21_cyclonedx_sbom_json.json move into db object
+    #             res = f15_json(file_, analysis_id)
+    sbom_file = f"{settings.EMBA_LOG_ROOT}/{analysis_id}/emba_logs/SBOM/EMBA_cyclonedx_sbom.json"
+    if os.path.isfile(sbom_file):
+        logger.debug("File %s found and attempting to read", sbom_file)
+        try:
+            res = f15_json(sbom_file, analysis_id)
+        except DatabaseError as error:
+            logger.error("DB error in f15_json: %s", error, exc_info=1)
     return res
 
 
@@ -193,13 +210,53 @@ def f10_csv(_file_path, _analysis_id):
     logger.debug("read f10 csv done")
 
 
+def f15_json(_file_path, _analysis_id):
+    """
+    return: result obj/ None
+    SBOM json
+    """
+    logger.debug("starting f15 json import")
+    with open(_file_path, 'r', encoding='utf-8') as f15_json_file:
+        f15_data = json.load(f15_json_file)
+        sbom_uuid = f15_data['serialNumber'].split(":")[2]
+        logger.debug("Reading sbom uuid=%s", sbom_uuid)
+        sbom_obj, add_sbom = SoftwareBillOfMaterial.objects.get_or_create(id=sbom_uuid)
+        if not add_sbom:
+            for component_ in f15_data['components']:
+                logger.debug("Component is %s", component_)
+                try:
+                    new_sitem, add_sitem = SoftwareInfo.objects.get_or_create(
+                        id=component_['bom-ref'],
+                        name=component_['name'],
+                        type=component_['type'],
+                        group=component_['group'] or 'NA',
+                        version=component_['version'] or 'NA',
+                        hashes=[f"{key}:{value}" for key, value in component_['hashes']],
+                        cpe=component_['cpe'] or 'NA',
+                        purl=component_['purl'] or 'NA',
+                        properties=component_['properties'] or 'NA'
+                    )
+                    logger.debug("Was new? %s", add_sitem)
+                    logger.debug("Adding SBOM item: %s to sbom %s", new_sitem, sbom_obj)
+                    sbom_obj.component.add(new_sitem)
+                except builtins.Exception as error_:
+                    logger.error("Error in f15 readin: %s", error_)
+        res, _ = Result.objects.get_or_create(
+            firmware_analysis=FirmwareAnalysis.objects.get(id=_analysis_id),
+        )
+        res.sbom = sbom_obj
+        res.save()
+    logger.debug("read f15 json done")
+    return res
+
+
 if __name__ == "__main__":
     BASE_DIR = Path(__file__).resolve().parent.parent.parent
     TEST_DIR = os.path.join(BASE_DIR, 'test/porter')
 
     # test print f50
-    with open(os.path.join(TEST_DIR, 'f50_test.json'), 'w', encoding='utf-8') as json_file:
-        json_file.write(json.dumps(read_csv(os.path.join(TEST_DIR, 'f50_test.csv')), indent=4))
+    # with open(os.path.join(TEST_DIR, 'f50_test.json'), 'w', encoding='utf-8') as json_file:
+    #     json_file.write(json.dumps(read_csv(os.path.join(TEST_DIR, 'f50_test.csv')), indent=4))
 
     # with open(os.path.join(TEST_DIR, 'f20_test.json'), 'w', encoding='utf-8') as json_file:
     #     json_file.write(json.dumps(

embark/porter/urls.py

@@ -16,7 +16,7 @@
 
     path('export/', views.export_menu, name='embark-export-menu'),
     path('export/download/', views.export_analysis, name='embark-export-analysis'),
-    path('export/zip/<uuid:analysis_id>/', views.make_zip, name='embark-make-zip')
-
+    path('export/zip/<uuid:analysis_id>/', views.make_zip, name='embark-make-zip'),
 
+    path('retry-import/', views.retry_import, name='embark-retry-import'),
 ]