diff --git a/.gitignore b/.gitignore index 216549b4..5695f955 100644 --- a/.gitignore +++ b/.gitignore @@ -142,3 +142,9 @@ dmypy.json # Pyre type checker .pyre/ + +# DABs Generated Template +dabs/dabs_template/template/tmp + +**/.terraform* +**/terraform.tfstate* \ No newline at end of file diff --git a/README.md b/README.md index bb89fa90..71b709cd 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,10 @@ # Security Analysis Tool (SAT) -Refer to [manual setup guide](./docs/setup.md) or [Terraform](./terraform) to setup and optional [video](https://www.youtube.com/watch?v=xAav6GslSd8) overview with follow along instruction. +Refer to specific use-case: +- [Standard setup guide](./docs/setup.md) +- [Terraform](./terraform/README.md) +- [Deprecated: Manual setup](./docs/deprecated_old_setup.md) ## Introduction @@ -85,7 +88,7 @@ For example, The diagram below shows the individual checks in various categories ## Configuration and Usage instructions -Refer to [manul setup guide](./docs/setup.md) or [Terraform](./terraform) to setup +Refer to [Standard setup guide](./docs/setup.md) or [Terraform](./terraform) to setup ## Project support diff --git a/configs/sat_dasf_mapping.csv b/configs/sat_dasf_mapping.csv new file mode 100644 index 00000000..f355deeb --- /dev/null +++ b/configs/sat_dasf_mapping.csv @@ -0,0 +1,36 @@ +sat_id,dasf_control_id,dasf_control_name +1,DASF 33,Manage credentials securely +2,DASF 8,Encrypt data at rest +2,DASF 30,Encrypt models +3,DASF 8,Encrypt data at rest +3,DASF 30,Encrypt models +4,DASF 8,Encrypt data at rest +8,DASF 14,Audit actions performed on datasets +8,DASF 55,Monitor Audit logs +9,DASF 38,Platform security — vulnerability management +10,DASF 38,Platform security — vulnerability management +18,DASF 1,SSO with IdP and MFA +19,DASF 2,Sync users and groups +29,DASF 43,Use access control lists +30,DASF 43,Use access control lists +31,DASF 43,Use access control lists +32,DASF 52,Source code control +35,DASF 4,Restrict access using private link +37,DASF 3,Restrict access using IP access lists +52,DASF 52,Source code control +53,DASF 5,Control access to data and other objects +53,DASF 16,Secure model features +53,DASF 24,Control access to models and model assets +53,DASF 43,Use access control lists +54,DASF 51,Share data and AI assets securely +55,DASF 51,Share data and AI assets securely +56,DASF 51,Share data and AI assets securely +89,DASF 31,Secure model serving endpoints +90,DASF 32,Streamline the usage and management of various large language model (LLM) providers +101,DASF 46,Store and retrieve embeddings securely +103,DASF 50,Platform compliance +104,DASF 53,Third-party library control +105,DASF 55,Monitor Audit logs +107,DASF 38,Platform security — vulnerability management +108,DASF 50,Platform compliance +109,DASF 50,Platform compliance \ No newline at end of file diff --git a/configs/security_best_practices.csv b/configs/security_best_practices.csv index 60394c29..5e8874fb 100644 --- a/configs/security_best_practices.csv +++ b/configs/security_best_practices.csv @@ -66,7 +66,7 @@ id,check_id,category,check,evaluation_value,severity,recommendation,aws,azure,gc 65,GOV-26,Governance,Legacy cluster-named init scripts,-1,High,Databricks recommends that you migrate legacy cluster-named init scripts to the new cluster-scoped init scripts framework and then disable legacy cluster named init scripts,1,1,0,1,0,Check workspace-conf for enableDeprecatedClusterNamedInitScripts setting,curl -n -X GET 'https:///api/2.0/preview/workspace-conf?keys= enableDeprecatedClusterNamedInitScripts',https://docs.databricks.com/clusters/init-scripts.html#cluster-scoped-init-scripts,https://learn.microsoft.com/en-us/azure/databricks/clusters/init-scripts,N/A 78,GOV-28,Governance,Govern model assets,-1,Medium,Manage model lifecycle in Unity Catalog,1,1,1,1,0, List the registered models and check if there are any models in UC,curl -n -X GET 'https:///api/2.1/unity-catalog/models',https://docs.databricks.com/en/machine-learning/manage-model-lifecycle/index.html,https://learn.microsoft.com/en-us/azure/databricks/machine-learning/manage-model-lifecycle/,https://docs.gcp.databricks.com/en/machine-learning/manage-model-lifecycle/index.html 89,NS-7,Network Security,Secure model serving endpoints,-1,High,Secure endpoints of the underlying models served as REST API endpoints using Private connectivity or IP Access Lists,1,1,0,1,0,Check if there are any model serving endpoints configured then check if IP Access lists or PL is deployed to prevent the endpoints from being accessed from the public internet,curl -n -X GET 'https:///api/2.0/serving-endpoints',https://docs.databricks.com/en/machine-learning/model-serving/manage-serving-endpoints.html,https://learn.microsoft.com/en-us/azure/databricks/machine-learning/model-serving/manage-serving-endpoints,N/A -90,INFO-29,Informational,Streamline the usage and management of various large language model(LLM) providers,-1, Medium,Secure third-party models hosted outside of Databricks with Model Serving to streamline the usage and management of various large language model (LLM) providers such as OpenAI and Anthropic within an organization by storing API keys in one secure location,1,1,0,1,0,Check if there is not at least one EXTERNAL_MODEL in model serving endpoints configured and alert incase of none,curl -n -X GET 'https:///api/2.0/serving-endpoints',https://docs.databricks.com/en/generative-ai/external-models/index.html,https://learn.microsoft.com/en-us/azure/databricks/generative-ai/external-models,N/A +90,INFO-29,Informational,Streamline the usage and management of various large language model(LLM) providers,-1,Medium,Secure third-party models hosted outside of Databricks with Model Serving to streamline the usage and management of various large language model (LLM) providers such as OpenAI and Anthropic within an organization by storing API keys in one secure location,1,1,0,1,0,Check if there is not at least one EXTERNAL_MODEL in model serving endpoints configured and alert incase of none,curl -n -X GET 'https:///api/2.0/serving-endpoints',https://docs.databricks.com/en/generative-ai/external-models/index.html,https://learn.microsoft.com/en-us/azure/databricks/generative-ai/external-models,N/A 101,DP-14,Data Protection,Store and retrieve embeddings securely,-1,Low,Store and retrieve embeddings securely using the Vector Search,1,1,0,1,0,List all the Vector Search endpoints and see if at least one endpoint is configured,curl -n -X GET 'https:///api/2.0/vector-search/endpoints',https://docs.databricks.com/en/generative-ai/vector-search.html,https://learn.microsoft.com/en-us/azure/databricks/generative-ai/vector-search,N/A 103,INFO-37,Informational,Compliance security profile for new workspaces,-1,Low,Validate and deploy on a platform that has put in place controls to meet the unique compliance needs of highly regulated industries,1,0,0,1,0,Check if compliance security profile for new workspaces is enabled,curl -n -X GET 'https://accounts.cloud.databricks.com/api/2.0/accounts///accounts/{accountid}/settings/types/shield_csp_enablement_ac/names/default',https://docs.databricks.com/en/security/privacy/security-profile.html,https://learn.microsoft.com/en-us/azure/databricks/security/privacy/security-profile,N/A 104,INFO-38,Informational,Third-party library control,-1,Low,Add libraries and init scripts to the allowlist in Unity Catalog,1,1,1,1,0,Get the artifact allowlist of and check if any allowed artifacts are configured,curl -n -X GET 'https:///api/2.1/unity-catalog/artifact-allowlists/{artifact_type},https://docs.databricks.com/en/data-governance/unity-catalog/manage-privileges/allowlist.html,https://learn.microsoft.com/en-us/azure/databricks/data-governance/unity-catalog/manage-privileges/allowlist,https://docs.gcp.databricks.com/en/data-governance/unity-catalog/manage-privileges/allowlist.html diff --git a/configs/self_assessment_checks.yaml b/configs/self_assessment_checks.yaml index baf38df7..5d6db687 100644 --- a/configs/self_assessment_checks.yaml +++ b/configs/self_assessment_checks.yaml @@ -3,15 +3,15 @@ check: Object storage encryption for data sources - id: 18 - enabled: true + enabled: false check: Enable single sign-on - id: 19 - enabled: true + enabled: false check: SCIM for user provisioning - id: 20 - enabled: true + enabled: false check: Table Access Control for clusters that don't use Unity Catalog - id: 28 diff --git a/dabs/dabs_template/databricks_template_schema.json b/dabs/dabs_template/databricks_template_schema.json new file mode 100644 index 00000000..45f8c690 --- /dev/null +++ b/dabs/dabs_template/databricks_template_schema.json @@ -0,0 +1,26 @@ +{ + "welcome_message": "", + "properties": { + "catalog": { + "type": "string", + "description": "The catalog for SAT" + }, + "cloud": { + "type": "string", + "description": "Cloud type" + }, + "google_service_account": { + "type": "string", + "description": "Google service account" + }, + "latest_lts": { + "type": "string", + "description": "Latest LTS version" + }, + "node_type": { + "type": "string", + "description": "Node Type" + } + }, + "success_message": "" +} \ No newline at end of file diff --git a/dabs/dabs_template/initialize.py.tmpl b/dabs/dabs_template/initialize.py.tmpl new file mode 100644 index 00000000..b3eb6cef --- /dev/null +++ b/dabs/dabs_template/initialize.py.tmpl @@ -0,0 +1,139 @@ +# Databricks notebook source +# MAGIC %md +# MAGIC **Notebook name:** initialize +# MAGIC **Functionality:** initializes the necessary configruation values for the rest of the process into a json + +# COMMAND ---------- + +# MAGIC %run ./common + +# COMMAND ---------- + +# replace values for accounts exec +hostname = ( + dbutils.notebook.entry_point.getDbutils() + .notebook() + .getContext() + .apiUrl() + .getOrElse(None) +) +cloud_type = getCloudType(hostname) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ##### Modify JSON values +# MAGIC * **account_id** Account ID. Can get this from the accounts console +# MAGIC * **sql_warehouse_id** SQL Warehouse ID to import dashboard +# MAGIC * **verbosity** (optional). debug, info, warning, error, critical +# MAGIC * **master_name_scope** Secret Scope for Account Name +# MAGIC * **master_name_key** Secret Key for Account Name +# MAGIC * **master_pwd_scope** Secret Scope for Account Password +# MAGIC * **master_pwd_key** Secret Key for Account Password +# MAGIC * **workspace_pat_scope** Secret Scope for Workspace PAT +# MAGIC * **workspace_pat_token_prefix** Secret Key prefix for Workspace PAT. Workspace ID will automatically be appended to this per workspace +# MAGIC * **use_mastercreds** (optional) Use master account credentials for all workspaces + +# COMMAND ---------- + +import json + +json_ = { + "account_id": dbutils.secrets.get(scope="sat_scope", key="account-console-id"), + "sql_warehouse_id": dbutils.secrets.get(scope="sat_scope", key="sql-warehouse-id"), + "analysis_schema_name": "{{.catalog}}.security_analysis", + "verbosity": "info", +} + +# COMMAND ---------- + +json_.update( + { + "master_name_scope": "sat_scope", + "master_name_key": "user", + "master_pwd_scope": "sat_scope", + "master_pwd_key": "pass", + "workspace_pat_scope": "sat_scope", + "workspace_pat_token_prefix": "sat-token", + "dashboard_id": "317f4809-8d9d-4956-a79a-6eee51412217", + "dashboard_folder": f"{basePath()}/dashboards/", + "dashboard_tag": "SAT", + "use_mastercreds": True, + "use_parallel_runs": True, + } +) + + +# COMMAND ---------- + +# DBTITLE 1,GCP configurations +if cloud_type == "gcp": + json_.update( + { + "service_account_key_file_path": dbutils.secrets.get( + scope="sat_scope", key="gs-path-to-json" + ), + "impersonate_service_account": dbutils.secrets.get( + scope="sat_scope", key="impersonate-service-account" + ), + "use_mastercreds": False, + } + ) + + +# COMMAND ---------- + +# DBTITLE 1,Azure configurations +if cloud_type == "azure": + json_.update( + { + "account_id": "azure", + "subscription_id": dbutils.secrets.get( + scope="sat_scope", key="subscription-id" + ), # Azure subscriptionId + "tenant_id": dbutils.secrets.get( + scope="sat_scope", key="tenant-id" + ), # The Directory (tenant) ID for the application registered in Azure AD. + "client_id": dbutils.secrets.get( + scope="sat_scope", key="client-id" + ), # The Application (client) ID for the application registered in Azure AD. + "client_secret_key": "client-secret", # The secret generated by AAD during your confidential app registration + "use_mastercreds": True, + } + ) + + +# COMMAND ---------- + +# DBTITLE 1,AWS configurations +if cloud_type == "aws": + sp_auth = { + "use_sp_auth": "False", + "client_id": "", + "client_secret_key": "client-secret", + } + try: + use_sp_auth = ( + dbutils.secrets.get(scope="sat_scope", key="use-sp-auth").lower() == "true" + ) + if use_sp_auth: + sp_auth["use_sp_auth"] = "True" + sp_auth["client_id"] = dbutils.secrets.get( + scope="sat_scope", key="client-id" + ) + except: + pass + json_.update(sp_auth) + +# COMMAND ---------- + +create_schema() +create_security_checks_table() +create_account_info_table() +create_account_workspaces_table() +create_workspace_run_complete_table() + +# COMMAND ---------- + +# Initialize best practices if not already loaded into database +readBestPracticesConfigsFile() \ No newline at end of file diff --git a/dabs/dabs_template/template/tmp/databricks.yml.tmpl b/dabs/dabs_template/template/tmp/databricks.yml.tmpl new file mode 100644 index 00000000..b201cad6 --- /dev/null +++ b/dabs/dabs_template/template/tmp/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: SAT + +include: + - resources/*.yml + +targets: + sat: + default: true + mode: production + workspace: + host: {{workspace_host}} + root_path: /Applications/${bundle.name}/ + run_as: + {{- if is_service_principal}} + service_principal_name: {{user_name}} + {{- else}} + user_name: {{user_name}} + {{- end}} \ No newline at end of file diff --git a/dabs/dabs_template/template/tmp/resources/sat_driver_job.yml.tmpl b/dabs/dabs_template/template/tmp/resources/sat_driver_job.yml.tmpl new file mode 100644 index 00000000..41b3a9c7 --- /dev/null +++ b/dabs/dabs_template/template/tmp/resources/sat_driver_job.yml.tmpl @@ -0,0 +1,27 @@ +resources: + jobs: + sat_driver: + name: "SAT Driver Notebook" + schedule: + quartz_cron_expression: "0 0 8 ? * Mon,Wed,Fri" + timezone_id: "America/New_York" + tasks: + - task_key: "sat_initializer" + job_cluster_key: job_cluster + libraries: + - pypi: + package: dbl-sat-sdk + notebook_task: + notebook_path: "../notebooks/security_analysis_driver.py" + + job_clusters: + - job_cluster_key: job_cluster + new_cluster: + num_workers: 5 + spark_version: {{.latest_lts}} + runtime_engine: "PHOTON" + node_type_id: {{.node_type}} + {{- if eq .cloud "gcp" }} + gcp_attributes: + google_service_account: {{.google_service_account}} + {{- end }} diff --git a/dabs/dabs_template/template/tmp/resources/sat_initiliazer_job.yml.tmpl b/dabs/dabs_template/template/tmp/resources/sat_initiliazer_job.yml.tmpl new file mode 100644 index 00000000..248949a7 --- /dev/null +++ b/dabs/dabs_template/template/tmp/resources/sat_initiliazer_job.yml.tmpl @@ -0,0 +1,25 @@ +resources: + jobs: + sat_initializer: + name: "SAT Initializer Notebook (one-time)" + + tasks: + - task_key: "sat_initializer" + job_cluster_key: job_cluster + libraries: + - pypi: + package: dbl-sat-sdk + notebook_task: + notebook_path: "../notebooks/security_analysis_initializer.py" + + job_clusters: + - job_cluster_key: job_cluster + new_cluster: + num_workers: 5 + spark_version: {{.latest_lts}} + runtime_engine: "PHOTON" + node_type_id: {{.node_type}} + {{- if eq .cloud "gcp" }} + gcp_attributes: + google_service_account: {{.google_service_account}} + {{- end }} \ No newline at end of file diff --git a/dabs/main.py b/dabs/main.py new file mode 100644 index 00000000..d006f731 --- /dev/null +++ b/dabs/main.py @@ -0,0 +1,52 @@ +import json +import os +import subprocess + +from databricks.sdk import WorkspaceClient +from sat.config import form, generate_secrets +from sat.utils import cloud_type + + +def install(client: WorkspaceClient, answers: dict, profile: str): + cloud = cloud_type(client) + generate_secrets(client, answers, cloud) + config = { + "catalog": answers.get("catalog", None), + "cloud": cloud, + "google_service_account": answers.get("gcp-impersonate-service-account", None), + "latest_lts": client.clusters.select_spark_version( + long_term_support=True, + latest=True, + ), + "node_type": client.clusters.select_node_type( + local_disk=True, + min_cores=4, + gb_per_core=8, + photon_driver_capable=True, + photon_worker_capable=True, + ), + } + + config_file = "tmp_config.json" + with open(config_file, "w") as fp: + json.dump(config, fp) + + os.system("clear") + subprocess.call(f"sh ./setup.sh tmp {profile} {config_file}".split(" ")) + print("Installation complete.") + print(f"Review workspace -> {client.config.host}") + + +def setup(): + try: + client, answers, profile = form() + install(client, answers, profile) + except KeyboardInterrupt: + print("Installation aborted.") + except Exception as e: + print(f"An error occurred: {e}") + + +if __name__ == "__main__": + os.system("clear") + setup() diff --git a/dabs/requirements.txt b/dabs/requirements.txt new file mode 100644 index 00000000..8bedc8ed --- /dev/null +++ b/dabs/requirements.txt @@ -0,0 +1,3 @@ +inquirer +databricks-sdk +rich \ No newline at end of file diff --git a/dabs/sat/__init__.py b/dabs/sat/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dabs/sat/config.py b/dabs/sat/config.py new file mode 100644 index 00000000..d5a2a069 --- /dev/null +++ b/dabs/sat/config.py @@ -0,0 +1,152 @@ +import json +import os +import re +import subprocess + +from databricks.sdk import WorkspaceClient +from inquirer import Confirm, List, Password, Text, list_input, prompt +from rich.progress import Progress, SpinnerColumn, TextColumn +from sat.utils import ( + cloud_validation, + get_catalogs, + get_profiles, + get_warehouses, + loading, + uc_enabled, +) + + +def form(): + profile = list_input( + message="Select profile", + choices=loading(get_profiles, "Loading profiles..."), + ) + client = WorkspaceClient(profile=profile) + questions = [ + Text( + name="account_id", + message="Databricks Account ID", + validate=lambda _, x: re.match( + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", x + ), + ), + Confirm( + name="enable_uc", + message="Use Unity Catalog?", + default=lambda x: uc_enabled(client), + ignore=lambda x: not uc_enabled(client), + ), + List( + name="catalog", + message="Select catalog", + choices=loading(get_catalogs, client=client), + ignore=lambda x: not x["enable_uc"], + default="hive_metastore", + ), + List( + name="warehouse", + message="Select warehouse", + choices=loading(get_warehouses, client=client), + ), + ] + questions = questions + cloud_specific_questions(client) + return client, prompt(questions), profile + + +def cloud_specific_questions(client: WorkspaceClient): + azure = [ + Text( + name="azure-tenant-id", + message="Azure Tenant ID", + ignore=cloud_validation(client, "azure"), + ), + Text( + name="azure-subscription-id", + message="Azure Subscription ID", + ignore=cloud_validation(client, "azure"), + ), + Text( + name="azure-client-id", + message="Client ID", + ignore=cloud_validation(client, "azure"), + ), + Password( + name="azure-client-secret", + message="Client Secret", + ignore=cloud_validation(client, "azure"), + echo="", + ), + ] + gcp = [ + Text( + name="gcp-gs-path-to-json", + message="Path to JSON key file", + ignore=cloud_validation(client, "gcp"), + ), + Text( + name="gcp-impersonate-service-account", + message="Impersonate Service Account", + ignore=cloud_validation(client, "gcp"), + default="", + ), + ] + aws = [ + Text( + name="aws-client-id", + message="Client ID", + ignore=cloud_validation(client, "aws"), + ), + Password( + name="aws-client-secret", + message="Client Secret", + ignore=cloud_validation(client, "aws"), + echo="", + ), + ] + return aws + azure + gcp + + +def generate_secrets(client: WorkspaceClient, answers: dict, cloud_type: str): + + scope_name = "sat_scope" + for scope in client.secrets.list_scopes(): + if scope.name == scope_name: + client.secrets.delete_scope(scope_name) + break + + client.secrets.create_scope(scope_name) + + token = client.tokens.create( + lifetime_seconds=86400 * 90, + comment="Security Analysis Tool", + ) + client.secrets.put_secret( + scope=scope_name, + key=f"sat-token-{client.get_workspace_id()}", + string_value=token.token_value, + ) + client.secrets.put_secret( + scope=scope_name, + key="account-console-id", + string_value=answers["account_id"], + ) + client.secrets.put_secret( + scope=scope_name, + key="sql-warehouse-id", + string_value=answers["warehouse"]["id"], + ) + + if cloud_type == "aws": + client.secrets.put_secret( + scope=scope_name, + key="use-sp-auth", + string_value=True, + ) + + for value in answers.keys(): + if cloud_type in value: + client.secrets.put_secret( + scope=scope_name, + key=value.replace(f"{cloud_type}-", ""), + string_value=answers[value], + ) diff --git a/dabs/sat/utils.py b/dabs/sat/utils.py new file mode 100644 index 00000000..f65be15b --- /dev/null +++ b/dabs/sat/utils.py @@ -0,0 +1,92 @@ +import json +import os +import re +import subprocess + +from databricks.sdk import WorkspaceClient +from inquirer import Confirm, List, Password, Text, list_input, prompt +from rich.progress import Progress, SpinnerColumn, TextColumn + + +def loading(func, message: str = "Loading...", client: WorkspaceClient = None): + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + transient=True, + ) as progress: + progress.start() + progress.add_task(description=message) + if client is None: + res = func() + else: + res = func(client) + progress.stop() + + return res + + +def cloud_validation(client, cloud): + cloud_type = None + if "azure" in client.config.host: + cloud_type = "azure" + elif "gcp" in client.config.host: + cloud_type = "gcp" + else: + cloud_type = "aws" + + if cloud_type == cloud: + return False + return True + + +def cloud_type(client: WorkspaceClient): + if "azure" in client.config.host: + return "azure" + elif "gcp" in client.config.host: + return "gcp" + else: + return "aws" + + +def databricks_command(commmand: str): + return json.loads( + subprocess.run( + commmand.split(" "), + capture_output=True, + text=True, + ).stdout.strip() + ) + + +def get_profiles(): + output = databricks_command("databricks auth profiles -o json")["profiles"] + valid_profiles = [] + for p in output: + if p["valid"] and "accounts" not in p["host"]: + valid_profiles.append(p["name"]) + return valid_profiles + + +def get_catalogs(client: WorkspaceClient): + if uc_enabled(client) is False: + return [] + valid_catalogs = [] + for c in client.catalogs.list(): + if c.catalog_type is not None and c.catalog_type.value != "SYSTEM_CATALOG": + valid_catalogs.append(c.name) + return valid_catalogs + + +def get_warehouses(client: WorkspaceClient): + valid_warehouses = [] + for w in client.warehouses.list(): + valid_warehouses.append({"name": w.name, "id": w.id}) + return valid_warehouses + + +def uc_enabled(client: WorkspaceClient): + try: + client.metastores.current() + return True + except: + return False diff --git a/dabs/setup.sh b/dabs/setup.sh new file mode 100644 index 00000000..80a29783 --- /dev/null +++ b/dabs/setup.sh @@ -0,0 +1,19 @@ +#/bin/bash + +project=$1 +profile=$2 +config_file=$3 + + +cp -r ../configs ../notebooks ../dashboards ./dabs_template/template/tmp +rm ./dabs_template/template/tmp/notebooks/Utils/initialize.py +cp ./dabs_template/initialize.py.tmpl ./dabs_template/template/tmp/notebooks/Utils/initialize.py.tmpl + +databricks bundle init ./dabs_template -p $profile --config-file $config_file +rm -rf $config_file +cd $project +databricks bundle deploy -p $profile --force-lock +cd ../ +rm -rf $project +rm -rf ./dabs_template/template/tmp/configs ./dabs_template/template/tmp/dashboards ./dabs_template/template/tmp/notebooks + diff --git a/dashboards/SAT_Dashboard_definition.json b/dashboards/SAT_Dashboard_definition.json new file mode 100644 index 00000000..d76c8f74 --- /dev/null +++ b/dashboards/SAT_Dashboard_definition.json @@ -0,0 +1,4863 @@ +{ + "datasets": [ + { + "name": "ae573d91", + "displayName": "category_pillar_dp", + "query": "SELECT DISTINCT\n BP.check_id,\n BP.check,\n BP.severity,\n BP.doc_url as Recommendation,\n BP.recommendation as Rec_text,\n BP.category,\n concat(COALESCE(dasf.dasf_control_id,' '),' ',COALESCE(dasf.dasf_control_name,' ')) as DASF_control,\n WRC.check_time,\n WRC.chk_date,\n WRC.workspace_id,\n concat(ac.workspace_name, \" (\", ac.workspace_id, \")\") as name,\n SC.score as Status,\n WRC.run_id\nFROM \n hive_metastore.security_analysis.security_checks SC\nJOIN \n hive_metastore.security_analysis.security_best_practices BP ON SC.id = BP.id\nJOIN \n hive_metastore.security_analysis.workspace_run_complete WRC ON WRC.run_id = SC.run_id\nLEFT JOIN\n hive_metastore.security_analysis.sat_dasf_mapping DASF ON DASF.sat_id = BP.id \nINNER JOIN hive_metastore.security_analysis.account_workspaces ac\n ON wrc.workspace_id = ac.workspace_id\nWHERE BP.category = 'Data Protection' AND SC.workspaceid = WRC.workspace_id\nORDER BY Status DESC;" + }, + { + "name": "6729e1a1", + "displayName": "category_counter_inf_m", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"Medium\"\n AND sc.score = 1\n AND sbp.category = \"Informational\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "61330f46", + "displayName": "category_counter_gov_l", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"Low\"\n AND sc.score = 1\n AND sbp.category = \"Governance\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "97f5b253", + "displayName": "category_counter_ns_m", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"Medium\"\n AND sc.score = 1\n AND sbp.category = \"Network Security\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "4d57d703", + "displayName": "category_counter_gov_h", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"High\"\n AND sc.score = 1\n AND sbp.category = \"Governance\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "07804eaa", + "displayName": "summary_stats", + "query": "Select\n results.workspaceid as workspaceid,\n concat(\n results.category,\n ' (Out of ',\n category_count,\n ')'\n ) as category,\n severity,\n severity_count,\n ord,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n (\n SELECT\n sbp.category,\n sbp.severity,\n sc.workspaceid,\n count(sbp.severity) severity_count,\n CASE\n WHEN sbp.severity = 'High' THEN 3\n WHEN sbp.severity = 'Medium' THEN 2\n ELSE 1\n END as ord\n FROM\n hive_metastore.security_analysis.security_checks sc,\n hive_metastore.security_analysis.security_best_practices sbp\n where\n score = 1\n and sc.id = sbp.id\n and run_id = coalesce (\n (\n select\n max(run_id)\n from\n hive_metastore.security_analysis.workspace_run_complete\n where\n completed = true\n ),\n (\n select\n max(run_id)\n from\n hive_metastore.security_analysis.workspace_run_complete\n where\n completed = true\n )\n )\n and category not IN ('Workspace Settings', 'Workspace Stats')\n group by\n sc.workspaceid,\n sbp.category,\n sbp.severity\n ) as results,\n (\n SELECT\n category,\n count(*) as category_count\n FROM\n hive_metastore.security_analysis.security_best_practices\n WHERE\n category not IN ('Workspace Settings', 'Workspace Stats')\n group by\n category\n ) as master\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON results.workspaceid = aw.workspace_id\nwhere\n results.category = master.category" + }, + { + "name": "fdcac69c", + "displayName": "category_counter_gov_m", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"Medium\"\n AND sc.score = 1\n AND sbp.category = \"Governance\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "338dcb50", + "displayName": "category_pillar_inf", + "query": "SELECT DISTINCT\n BP.check_id,\n BP.check,\n BP.severity,\n BP.doc_url as Recommendation,\n BP.recommendation as Rec_text,\n BP.category,\n concat(COALESCE(dasf.dasf_control_id,' '),' ',COALESCE(dasf.dasf_control_name,' ')) as DASF_control,\n WRC.check_time,\n WRC.chk_date,\n WRC.workspace_id,\n concat(ac.workspace_name, \" (\", ac.workspace_id, \")\") as name,\n SC.score as Status,\n WRC.run_id\nFROM \n hive_metastore.security_analysis.security_checks SC\nJOIN \n hive_metastore.security_analysis.security_best_practices BP ON SC.id = BP.id\nLEFT JOIN\n hive_metastore.security_analysis.sat_dasf_mapping DASF ON DASF.sat_id = BP.id \nJOIN \n hive_metastore.security_analysis.workspace_run_complete WRC ON WRC.run_id = SC.run_id\nINNER JOIN hive_metastore.security_analysis.account_workspaces ac\n ON wrc.workspace_id = ac.workspace_id\nWHERE BP.category = 'Informational' AND SC.workspaceid = WRC.workspace_id\nORDER BY Status DESC;" + }, + { + "name": "109df071", + "displayName": "category_counter_ns_h", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"High\"\n AND sc.score = 1\n AND sbp.category = \"Network Security\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "91cf6843", + "displayName": "category_counter_dp_m", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"Medium\"\n AND sc.score = 1\n AND sbp.category = \"Data Protection\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "3da334cd", + "displayName": "category_counter_i&a_l", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"Low\"\n AND sc.score = 1\n AND sbp.category = \"Identity & Access\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "322d185a", + "displayName": "category_pillar_ns", + "query": "SELECT DISTINCT\n BP.check_id,\n BP.check,\n BP.severity,\n BP.doc_url as Recommendation,\n BP.recommendation as Rec_text,\n BP.category,\n concat(COALESCE(dasf.dasf_control_id,' '),' ',COALESCE(dasf.dasf_control_name,' ')) as DASF_control,\n WRC.check_time,\n WRC.chk_date,\n WRC.workspace_id,\n concat(ac.workspace_name, \" (\", ac.workspace_id, \")\") as name,\n SC.score as Status,\n WRC.run_id\nFROM \n hive_metastore.security_analysis.security_checks SC\nJOIN \n hive_metastore.security_analysis.security_best_practices BP ON SC.id = BP.id\nJOIN \n hive_metastore.security_analysis.workspace_run_complete WRC ON WRC.run_id = SC.run_id\nLEFT JOIN\n hive_metastore.security_analysis.sat_dasf_mapping DASF ON DASF.sat_id = BP.id \nINNER JOIN hive_metastore.security_analysis.account_workspaces ac\n ON wrc.workspace_id = ac.workspace_id\nWHERE BP.category = 'Network Security' AND SC.workspaceid = WRC.workspace_id\nORDER BY Status DESC;" + }, + { + "name": "648b5c27", + "displayName": "account_stats", + "query": "SELECT\n ai.workspaceid,\n ai.name,\n ai.value.value,\n concat(ac.workspace_name, \" (\", ac.workspace_id, \")\") as ws_name\nFROM\n hive_metastore.security_analysis.account_info ai\nINNER JOIN hive_metastore.security_analysis.account_workspaces ac\n ON ai.workspaceid = ac.workspace_id\nwhere\n category = 'Account Stats'\n and run_id = coalesce (\n (\n select\n max(run_id)\n from\n hive_metastore.security_analysis.workspace_run_complete\n where\n completed = true\n ),\n (\n select\n max(run_id)\n from\n hive_metastore.security_analysis.workspace_run_complete\n where\n completed = true\n )\n )\nORDER BY\n name" + }, + { + "name": "a79323d9", + "displayName": "category_counter_inf_h", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"High\"\n AND sc.score = 1\n AND sbp.category = \"Informational\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "7fe66d3b", + "displayName": "analysis_date_time", + "query": "SELECT\n check_time date,\n wrc. workspace_id,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.workspace_run_complete wrc\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON wrc.workspace_id = aw.workspace_id\nwhere completed = true\nORDER BY date DESC" + }, + { + "name": "7778c9d9", + "displayName": "category_counter_dp_l", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"Low\"\n AND sc.score = 1\n AND sbp.category = \"Data Protection\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "8a55ae87", + "displayName": "category_pillar_i&a", + "query": "SELECT DISTINCT\n BP.check_id,\n BP.check,\n BP.severity,\n BP.doc_url as Recommendation,\n BP.recommendation as Rec_text,\n BP.category,\n concat(COALESCE(dasf.dasf_control_id,' '),' ',COALESCE(dasf.dasf_control_name,' ')) as DASF_control,\n WRC.check_time,\n WRC.chk_date,\n WRC.workspace_id,\n concat(ac.workspace_name, \" (\", ac.workspace_id, \")\") as name,\n SC.score as Status,\n WRC.run_id\nFROM \n hive_metastore.security_analysis.security_checks SC\nJOIN \n hive_metastore.security_analysis.security_best_practices BP ON SC.id = BP.id\nJOIN \n hive_metastore.security_analysis.workspace_run_complete WRC ON WRC.run_id = SC.run_id\nLEFT JOIN\n hive_metastore.security_analysis.sat_dasf_mapping DASF ON DASF.sat_id = BP.id \nINNER JOIN hive_metastore.security_analysis.account_workspaces ac\n ON wrc.workspace_id = ac.workspace_id\nWHERE BP.category = 'Identity & Access' AND SC.workspaceid = WRC.workspace_id\nORDER BY Status DESC;" + }, + { + "name": "d53104a9", + "displayName": "category_counter_i&a_h", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"High\"\n AND sc.score = 1\n AND sbp.category = \"Identity & Access\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "4c4caf0a", + "displayName": "category_counter_ns_l", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"Low\"\n AND sc.score = 1\n AND sbp.category = \"Network Security\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "a4e4c607", + "displayName": "category_counter_i&a_m", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"Medium\"\n AND sc.score = 1\n AND sbp.category = \"Identity & Access\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "5060eeb0", + "displayName": "category_counter_dp_h", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"High\"\n AND sc.score = 1\n AND sbp.category = \"Data Protection\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "623f0dd8", + "displayName": "category_pillar_gov", + "query": "SELECT DISTINCT\n BP.check_id,\n BP.check,\n BP.severity,\n BP.doc_url as Recommendation,\n BP.recommendation as Rec_text,\n BP.category,\n concat(COALESCE(dasf.dasf_control_id,' '),' ',COALESCE(dasf.dasf_control_name,' ')) as DASF_control,\n WRC.check_time,\n WRC.chk_date,\n WRC.workspace_id,\n concat(ac.workspace_name, \" (\", ac.workspace_id, \")\") as name,\n SC.score as Status,\n WRC.run_id\nFROM \n hive_metastore.security_analysis.security_checks SC\nJOIN \n hive_metastore.security_analysis.security_best_practices BP ON SC.id = BP.id\nJOIN \n hive_metastore.security_analysis.workspace_run_complete WRC ON WRC.run_id = SC.run_id\nLEFT JOIN\n hive_metastore.security_analysis.sat_dasf_mapping DASF ON DASF.sat_id = BP.id \nINNER JOIN hive_metastore.security_analysis.account_workspaces ac\n ON wrc.workspace_id = ac.workspace_id\nWHERE BP.category = 'Governance' AND SC.workspaceid = WRC.workspace_id\nORDER BY Status DESC;" + }, + { + "name": "7f52e401", + "displayName": "workspace_config_changes", + "query": "SELECT * from (\n SELECT\n sc1.score as current_score,\n sc2.score as previous_score,\n BP.check_id,\n check as Check,\n BP.severity as Severity,\n sc1.score as Run1Status,\n sc2.score as Run2Status,\n sc1.check_time as Run1Date,\n sc2.check_time as Run2Date,\n concat(\n '',\n BP.recommendation,\n ''\n ) as Recommendation,\n CASE\n WHEN BP.severity = 'High' THEN 3\n WHEN BP.severity = 'Medium' THEN 2\n ELSE 1\n END as ord,\n sc1.workspaceid,\n concat(ac.workspace_name, \" (\", ac.workspace_id, \")\") as name\n FROM\n hive_metastore.security_analysis.security_checks sc1,\n hive_metastore.security_analysis.security_checks sc2,\n hive_metastore.security_analysis.security_best_practices BP,\n hive_metastore.security_analysis.account_workspaces ac\n where\n sc2.workspaceid = sc1.workspaceid\n and sc1.id = sc2.id\n and sc1.id = BP.id\n and sc1.workspaceid = ac.workspace_id\n and sc1.run_id = (\n select\n max(run_id)\n from\n hive_metastore.security_analysis.workspace_run_complete\n where\n completed = true\n and chk_date = :date1 \n ) \n and sc2.run_id = (\n select\n max(run_id)\n from\n hive_metastore.security_analysis.workspace_run_complete\n where\n completed = true\n and run_id = (\n select\n max(run_id)\n from\n hive_metastore.security_analysis.workspace_run_complete\n where\n completed = true\n and chk_date = :date2 \n )\n ) \n ) where name = :workspace_name", + "parameters": [ + { + "displayName": "workspace_name", + "keyword": "workspace_name", + "dataType": "STRING", + "defaultSelection": { + "values": { + "dataType": "STRING", + "values": [ + { + "value": "Select a Workspace" + } + ] + } + } + }, + { + "displayName": "date1", + "keyword": "date1", + "dataType": "DATE", + "defaultSelection": { + "values": { + "dataType": "DATE", + "values": [ + { + "value": "2024-05-29T00:00:00.000" + } + ] + } + } + }, + { + "displayName": "date2", + "keyword": "date2", + "dataType": "DATE", + "defaultSelection": { + "values": { + "dataType": "DATE", + "values": [ + { + "value": "2024-06-03T00:00:00.000" + } + ] + } + } + } + ] + }, + { + "name": "d1808288", + "displayName": "category_counter_inf_l", + "query": "SELECT\n sc.workspaceid,\n COALESCE(COUNT(sbp.id), 0) AS counter,\n concat(aw.workspace_name, \" (\", aw.workspace_id, \")\") as name\nFROM\n hive_metastore.security_analysis.security_checks sc\nLEFT JOIN\n hive_metastore.security_analysis.security_best_practices sbp\nON\n sc.id = sbp.id\n AND sbp.severity = \"Low\"\n AND sc.score = 1\n AND sbp.category = \"Informational\"\nINNER JOIN hive_metastore.security_analysis.account_workspaces aw\n ON sc.workspaceid = aw.workspace_id\nWHERE\n sc.run_id = COALESCE (\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n ),\n (\n SELECT\n MAX(run_id)\n FROM\n hive_metastore.security_analysis.workspace_run_complete\n WHERE\n completed = true\n )\n )\nGROUP BY\n sc.workspaceid, name;" + }, + { + "name": "7004dfe6", + "displayName": "security_check_details", + "query": "select\n sbp.check_id,\n sbp.check,\n sc.score as Status,\n sbp.recommendation,\n concat(COALESCE(dasf.dasf_control_id,' '),' ',COALESCE(dasf.dasf_control_name,' ')) as DASF_control,\n sc.additional_details,\n sbp.logic,\n if(\n sbp.evaluation_value == \"-1\",\n \"N/A\",\n sbp.evaluation_value\n ) as evaluation_value,\n regexp_replace(sbp.api, \"\", aw.deployment_url) as api,\n sc.check_time\nfrom\n hive_metastore.security_analysis.security_checks sc,\n hive_metastore.security_analysis.security_best_practices sbp,\n hive_metastore.security_analysis.account_workspaces aw\n LEFT JOIN\n hive_metastore.security_analysis.sat_dasf_mapping DASF ON DASF.sat_id = sbp.id \nwhere\n sc.id = sbp.id\n and sc.run_id = coalesce (\n (\n select\n max(run_id)\n from\n hive_metastore.security_analysis.workspace_run_complete\n where\n completed = true\n ),\n (\n select\n max(run_id)\n from\n hive_metastore.security_analysis.workspace_run_complete\n where\n completed = true\n )\n )\n and aw.workspace_id = sc.workspaceid" + } + ], + "pages": [ + { + "name": "4f55a704", + "displayName": "New Page", + "layout": [ + { + "widget": { + "name": "b67302f8", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "4c4caf0a", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "displayName": "counter" + } + }, + "frame": { + "title": "Low", + "showTitle": true + } + } + }, + "position": { + "x": 5, + "y": 10, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "9ae91d5b", + "textbox_spec": "![alt text](https://databricks.com/wp-content/uploads/2022/02/icon-orange-encryption.svg)" + }, + "position": { + "x": 0, + "y": 29, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "d5f9d10a", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "07804eaa", + "fields": [ + { + "name": "severity", + "expression": "`severity`" + }, + { + "name": "sum(severity_count)", + "expression": "SUM(`severity_count`)" + }, + { + "name": "category", + "expression": "`category`" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 3, + "widgetType": "bar", + "encodings": { + "x": { + "fieldName": "sum(severity_count)", + "scale": { + "type": "quantitative" + }, + "axis": { + "title": "Count of deviations by severity" + }, + "displayName": "Count of deviations by severity" + }, + "y": { + "fieldName": "category", + "scale": { + "type": "categorical" + }, + "axis": { + "title": "Category" + }, + "displayName": "Category" + }, + "color": { + "fieldName": "severity", + "scale": { + "type": "categorical", + "mappings": [ + { + "value": "High", + "color": "#D60303" + }, + { + "value": "Low", + "color": "#E1D006" + }, + { + "value": "Medium", + "color": "#BF5102" + } + ] + }, + "displayName": "severity" + } + }, + "frame": { + "title": "Summary", + "showTitle": true + }, + "mark": { + "layout": "stack" + } + } + }, + "position": { + "x": 0, + "y": 2, + "width": 5, + "height": 5 + } + }, + { + "widget": { + "name": "fbec0c12", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "ae573d91", + "fields": [ + { + "name": "check_id", + "expression": "`check_id`" + }, + { + "name": "check", + "expression": "`check`" + }, + { + "name": "severity", + "expression": "`severity`" + }, + { + "name": "Status", + "expression": "`Status`" + }, + { + "name": "Recommendation", + "expression": "`Recommendation`" + }, + { + "name": "DASF_control", + "expression": "`DASF_control`" + }, + { + "name": "Rec_text", + "expression": "`Rec_text`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 1, + "widgetType": "table", + "encodings": { + "columns": [ + { + "fieldName": "check_id", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 0, + "title": "Check id", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "description": "", + "displayName": "check_id" + }, + { + "fieldName": "check", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 1, + "title": "Check", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "check" + }, + { + "fieldName": "severity", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 2, + "title": "Severity", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "severity" + }, + { + "fieldName": "Status", + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "https://hls-eng-data-public.s3.amazonaws.com/img/{{Status}}.png", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "20", + "imageHeight": "20", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "integer", + "displayAs": "image", + "visible": true, + "order": 3, + "title": "Status", + "allowSearch": false, + "alignContent": "center", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "description": "Your workspace configuration status", + "displayName": "Status" + }, + { + "fieldName": "Recommendation", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ Rec_text }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "link", + "visible": true, + "order": 5, + "title": "Recommendation", + "allowSearch": false, + "alignContent": "left", + "allowHTML": true, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "Recommendation" + }, + { + "fieldName": "DASF_control", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 11, + "title": "DASF Control", + "allowSearch": true, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "cellFormat": { + "default": { + "foregroundColor": null + }, + "rules": [] + }, + "displayName": "DASF_control" + } + ] + }, + "invisibleColumns": [ + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "category", + "type": "string", + "displayAs": "string", + "order": 4, + "title": "category", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "check_time", + "type": "datetime", + "displayAs": "datetime", + "order": 6, + "title": "check_time", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "Rec_text", + "type": "string", + "displayAs": "string", + "order": 7, + "title": "Rec_text", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "dateTimeFormat": "YYYY-MM-DD", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "chk_date", + "type": "date", + "displayAs": "datetime", + "order": 8, + "title": "chk_date", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "workspace_id", + "type": "string", + "displayAs": "string", + "order": 9, + "title": "workspace_id", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "run_id", + "type": "integer", + "displayAs": "number", + "order": 10, + "title": "run_id", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "name", + "type": "string", + "displayAs": "string", + "order": 12, + "title": "name", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + } + ], + "allowHTMLByDefault": false, + "itemsPerPage": 25, + "paginationSize": "default", + "condensed": false, + "withRowNumber": true, + "frame": { + "title": "Data Protection summary", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 32, + "width": 6, + "height": 6 + } + }, + { + "widget": { + "name": "992d2d0e", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "7778c9d9", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "rowNumber": 2, + "displayName": "counter" + } + }, + "frame": { + "title": "Low", + "showTitle": true + } + } + }, + "position": { + "x": 5, + "y": 29, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "dd5ad45d", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "623f0dd8", + "fields": [ + { + "name": "check_id", + "expression": "`check_id`" + }, + { + "name": "check", + "expression": "`check`" + }, + { + "name": "severity", + "expression": "`severity`" + }, + { + "name": "Status", + "expression": "`Status`" + }, + { + "name": "Recommendation", + "expression": "`Recommendation`" + }, + { + "name": "DASF_control", + "expression": "`DASF_control`" + }, + { + "name": "Rec_text", + "expression": "`Rec_text`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 1, + "widgetType": "table", + "encodings": { + "columns": [ + { + "fieldName": "check_id", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 0, + "title": "Check id", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "description": "", + "displayName": "check_id" + }, + { + "fieldName": "check", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 1, + "title": "Check", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "check" + }, + { + "fieldName": "severity", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 2, + "title": "Severity", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "severity" + }, + { + "fieldName": "Status", + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "https://hls-eng-data-public.s3.amazonaws.com/img/{{Status}}.png", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "20", + "imageHeight": "20", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "integer", + "displayAs": "image", + "visible": true, + "order": 3, + "title": "Status", + "allowSearch": false, + "alignContent": "center", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "description": "Your workspace configuration status", + "displayName": "Status" + }, + { + "fieldName": "Recommendation", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ Rec_text }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "link", + "visible": true, + "order": 6, + "title": "Recommendation", + "allowSearch": false, + "alignContent": "left", + "allowHTML": true, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "Recommendation" + }, + { + "fieldName": "DASF_control", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 11, + "title": "DASF control", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "DASF_control" + } + ] + }, + "invisibleColumns": [ + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "category", + "type": "string", + "displayAs": "string", + "order": 4, + "title": "category", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "Rec_text", + "type": "string", + "displayAs": "string", + "order": 5, + "title": "Rec_text", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "check_time", + "type": "datetime", + "displayAs": "datetime", + "order": 7, + "title": "check_time", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "dateTimeFormat": "YYYY-MM-DD", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "chk_date", + "type": "date", + "displayAs": "datetime", + "order": 8, + "title": "chk_date", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "workspace_id", + "type": "string", + "displayAs": "string", + "order": 9, + "title": "workspace_id", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "run_id", + "type": "integer", + "displayAs": "number", + "order": 10, + "title": "run_id", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "name", + "type": "string", + "displayAs": "string", + "order": 12, + "title": "name", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + } + ], + "allowHTMLByDefault": false, + "itemsPerPage": 25, + "paginationSize": "default", + "condensed": false, + "withRowNumber": true, + "frame": { + "title": "Governance summary", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 41, + "width": 6, + "height": 8 + } + }, + { + "widget": { + "name": "7736909d", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "648b5c27", + "fields": [ + { + "name": "value", + "expression": "`value`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "value", + "rowNumber": 3, + "displayName": "value" + } + }, + "frame": { + "title": "Workspace", + "showTitle": true, + "description": "workspace name", + "showDescription": true + } + } + }, + "position": { + "x": 2, + "y": 7, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "c09361ed", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "4d57d703", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "rowNumber": 1, + "displayName": "counter" + } + }, + "frame": { + "title": "High", + "showTitle": true + } + } + }, + "position": { + "x": 3, + "y": 38, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "8caefe16", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "97f5b253", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "displayName": "counter" + } + }, + "frame": { + "title": "Medium", + "showTitle": true + } + } + }, + "position": { + "x": 4, + "y": 10, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "053668e3", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "322d185a", + "fields": [ + { + "name": "check_id", + "expression": "`check_id`" + }, + { + "name": "check", + "expression": "`check`" + }, + { + "name": "severity", + "expression": "`severity`" + }, + { + "name": "Status", + "expression": "`Status`" + }, + { + "name": "Recommendation", + "expression": "`Recommendation`" + }, + { + "name": "DASF_control", + "expression": "`DASF_control`" + }, + { + "name": "Rec_text", + "expression": "`Rec_text`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 1, + "widgetType": "table", + "encodings": { + "columns": [ + { + "fieldName": "check_id", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 0, + "title": "Check id", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "description": "", + "displayName": "check_id" + }, + { + "fieldName": "check", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 1, + "title": "Check", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "check" + }, + { + "fieldName": "severity", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 2, + "title": "Severity", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "cellFormat": { + "default": { + "foregroundColor": "#ff0000" + }, + "rules": [] + }, + "displayName": "severity" + }, + { + "fieldName": "Status", + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "https://hls-eng-data-public.s3.amazonaws.com/img/{{Status}}.png", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "20", + "imageHeight": "20", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "integer", + "displayAs": "image", + "visible": true, + "order": 3, + "title": "Status", + "allowSearch": false, + "alignContent": "center", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "description": "Your workspace configuration status", + "displayName": "Status" + }, + { + "fieldName": "Recommendation", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ Rec_text }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "link", + "visible": true, + "order": 4, + "title": "Recommendation", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "Recommendation" + }, + { + "fieldName": "DASF_control", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 11, + "title": "DASF Control", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "DASF_control" + } + ] + }, + "invisibleColumns": [ + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "category", + "type": "string", + "displayAs": "string", + "order": 5, + "title": "category", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "check_time", + "type": "datetime", + "displayAs": "datetime", + "order": 6, + "title": "check_time", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "dateTimeFormat": "YYYY-MM-DD", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "chk_date", + "type": "date", + "displayAs": "datetime", + "order": 7, + "title": "chk_date", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "workspace_id", + "type": "string", + "displayAs": "string", + "order": 8, + "title": "workspace_id", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "run_id", + "type": "integer", + "displayAs": "number", + "order": 9, + "title": "run_id", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "Rec_text", + "type": "string", + "displayAs": "string", + "order": 10, + "title": "Rec_text", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "name", + "type": "string", + "displayAs": "string", + "order": 12, + "title": "name", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + } + ], + "allowHTMLByDefault": false, + "itemsPerPage": 25, + "paginationSize": "default", + "condensed": false, + "withRowNumber": true, + "frame": { + "title": "Network security summary", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 13, + "width": 6, + "height": 7 + } + }, + { + "widget": { + "name": "25acb87c", + "textbox_spec": "![alt text](https://www.databricks.com/wp-content/uploads/2021/04/fed-gov-icon-1.svg)" + }, + "position": { + "x": 0, + "y": 49, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "6056652e", + "textbox_spec": "# Governance\n* Cluster policy\n* Audit logs (diagnostic logs on Azure)\n* Init script\n* DBFS mounts" + }, + "position": { + "x": 1, + "y": 38, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "fc037494", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "d53104a9", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "rowNumber": 1, + "displayName": "counter" + } + }, + "frame": { + "title": "High", + "showTitle": true + } + } + }, + "position": { + "x": 3, + "y": 20, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "a533eed1", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "8a55ae87", + "fields": [ + { + "name": "check_id", + "expression": "`check_id`" + }, + { + "name": "check", + "expression": "`check`" + }, + { + "name": "severity", + "expression": "`severity`" + }, + { + "name": "Status", + "expression": "`Status`" + }, + { + "name": "Recommendation", + "expression": "`Recommendation`" + }, + { + "name": "DASF_control", + "expression": "`DASF_control`" + }, + { + "name": "Rec_text", + "expression": "`Rec_text`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 1, + "widgetType": "table", + "encodings": { + "columns": [ + { + "fieldName": "check_id", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 0, + "title": "Check id", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "description": "", + "displayName": "check_id" + }, + { + "fieldName": "check", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 1, + "title": "Check", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "check" + }, + { + "fieldName": "severity", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 2, + "title": "Severity", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "severity" + }, + { + "fieldName": "Status", + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "https://hls-eng-data-public.s3.amazonaws.com/img/{{Status}}.png", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "20", + "imageHeight": "20", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "integer", + "displayAs": "image", + "visible": true, + "order": 3, + "title": "Status", + "allowSearch": false, + "alignContent": "center", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "description": "Your workspace configuration status", + "displayName": "Status" + }, + { + "fieldName": "Recommendation", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ Rec_text }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "link", + "visible": true, + "order": 5, + "title": "Recommendation", + "allowSearch": false, + "alignContent": "left", + "allowHTML": true, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "Recommendation" + }, + { + "fieldName": "DASF_control", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 11, + "title": "DASF control", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "DASF_control" + } + ] + }, + "invisibleColumns": [ + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "category", + "type": "string", + "displayAs": "string", + "order": 4, + "title": "category", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "Rec_text", + "type": "string", + "displayAs": "string", + "order": 6, + "title": "Rec_text", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "check_time", + "type": "datetime", + "displayAs": "datetime", + "order": 7, + "title": "check_time", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "dateTimeFormat": "YYYY-MM-DD", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "chk_date", + "type": "date", + "displayAs": "datetime", + "order": 8, + "title": "chk_date", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "workspace_id", + "type": "string", + "displayAs": "string", + "order": 9, + "title": "workspace_id", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "run_id", + "type": "integer", + "displayAs": "number", + "order": 10, + "title": "run_id", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "name", + "type": "string", + "displayAs": "string", + "order": 12, + "title": "name", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + } + ], + "allowHTMLByDefault": false, + "itemsPerPage": 25, + "paginationSize": "default", + "condensed": false, + "withRowNumber": true, + "frame": { + "title": "Identity & Access summary", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 23, + "width": 6, + "height": 6 + } + }, + { + "widget": { + "name": "af46a0fc", + "textbox_spec": "![alt text](https://www.databricks.com/wp-content/uploads/2021/10/icon-orange-Risk-Fraud-Detection.svg)" + }, + "position": { + "x": 5, + "y": 2, + "width": 1, + "height": 5 + } + }, + { + "widget": { + "name": "82f47845", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "91cf6843", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "rowNumber": 3, + "displayName": "counter" + } + }, + "frame": { + "title": "Medium", + "showTitle": true + } + } + }, + "position": { + "x": 4, + "y": 29, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "f9d7bfc3", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "7004dfe6", + "fields": [ + { + "name": "check_id", + "expression": "`check_id`" + }, + { + "name": "check", + "expression": "`check`" + }, + { + "name": "recommendation", + "expression": "`recommendation`" + }, + { + "name": "Status", + "expression": "`Status`" + }, + { + "name": "additional_details", + "expression": "`additional_details`" + }, + { + "name": "logic", + "expression": "`logic`" + }, + { + "name": "DASF_control", + "expression": "`DASF_control`" + }, + { + "name": "evaluation_value", + "expression": "`evaluation_value`" + }, + { + "name": "api", + "expression": "`api`" + }, + { + "name": "check_time", + "expression": "`check_time`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 1, + "widgetType": "table", + "encodings": { + "columns": [ + { + "fieldName": "check_id", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 0, + "title": "Check Id", + "allowSearch": true, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "check_id" + }, + { + "fieldName": "check", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 1, + "title": "Check", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "check" + }, + { + "fieldName": "recommendation", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 2, + "title": "Recommendation", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "recommendation" + }, + { + "fieldName": "Status", + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "https://hls-eng-data-public.s3.amazonaws.com/img/{{Status}}.png", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "20", + "imageHeight": "", + "linkUrlTemplate": "https://hls-eng-data-public.s3.amazonaws.com/img/{{Status}}.png", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "integer", + "displayAs": "image", + "visible": true, + "order": 3, + "title": "Status", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "Status" + }, + { + "fieldName": "additional_details", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "complex", + "displayAs": "json", + "visible": true, + "order": 4, + "title": "Additional details", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "additional_details" + }, + { + "fieldName": "logic", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 5, + "title": "Logic", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "logic" + }, + { + "fieldName": "DASF_control", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 6, + "title": "DASF control", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "DASF_control" + }, + { + "fieldName": "evaluation_value", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 7, + "title": "Evaluation value", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "evaluation_value" + }, + { + "fieldName": "api", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 8, + "title": "api", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "api" + }, + { + "fieldName": "check_time", + "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "datetime", + "displayAs": "datetime", + "visible": true, + "order": 9, + "title": "Check time", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "check_time" + } + ] + }, + "invisibleColumns": [], + "allowHTMLByDefault": false, + "itemsPerPage": 25, + "paginationSize": "default", + "condensed": false, + "withRowNumber": false, + "frame": { + "title": "Additional details", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 65, + "width": 6, + "height": 10 + } + }, + { + "widget": { + "name": "04445746", + "textbox_spec": "![alt text](https://databricks.com/wp-content/uploads/2022/02/icon-orange-gdpr.svg)" + }, + "position": { + "x": 0, + "y": 38, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "efbfd00c", + "queries": [ + { + "name": "d8abbb6f55d64e41a92c5eafd84c7b82", + "query": { + "datasetName": "648b5c27", + "disaggregated": true + } + } + ], + "spec": { + "version": 0, + "viz_spec": { + "display_name": "Pricing tier", + "description": "", + "viz_type": "COUNTER", + "serialized_options": "{\"counterColName\": \"value\", \"counterLabel\": \"\", \"rowNumber\": 4, \"stringDecChar\": \".\", \"stringDecimal\": 0, \"stringThouSep\": \",\", \"targetRowNumber\": 1, \"tooltipFormat\": \"0,0.000\"}", + "query_name": "d8abbb6f55d64e41a92c5eafd84c7b82" + } + } + }, + "position": { + "x": 4, + "y": 7, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "52efacdd", + "queries": [ + { + "name": "6c8ce3c8148946789a188d175d545509", + "query": { + "datasetName": "7fe66d3b", + "disaggregated": true + } + } + ], + "spec": { + "version": 0, + "viz_spec": { + "display_name": "Analysis date", + "description": "", + "viz_type": "COUNTER", + "serialized_options": "{\"condensed\": true, \"counterColName\": \"date\", \"counterLabel\": \"\", \"formatTargetValue\": false, \"rowNumber\": 1, \"stringDecChar\": \".\", \"stringDecimal\": 0, \"stringThouSep\": \",\", \"targetColName\": \"\", \"targetRowNumber\": 1, \"tooltipFormat\": \"0,0.000\", \"withRowNumber\": true}", + "query_name": "6c8ce3c8148946789a188d175d545509" + } + } + }, + "position": { + "x": 0, + "y": 7, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "3e5c628d", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "5060eeb0", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "rowNumber": 1, + "displayName": "counter" + } + }, + "frame": { + "title": "High", + "showTitle": true + } + } + }, + "position": { + "x": 3, + "y": 29, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "708e66b0", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "338dcb50", + "fields": [ + { + "name": "check_id", + "expression": "`check_id`" + }, + { + "name": "check", + "expression": "`check`" + }, + { + "name": "severity", + "expression": "`severity`" + }, + { + "name": "Status", + "expression": "`Status`" + }, + { + "name": "Recommendation", + "expression": "`Recommendation`" + }, + { + "name": "DASF_control", + "expression": "`DASF_control`" + }, + { + "name": "Rec_text", + "expression": "`Rec_text`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 1, + "widgetType": "table", + "encodings": { + "columns": [ + { + "fieldName": "check_id", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 0, + "title": "Check id", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "description": "", + "displayName": "check_id" + }, + { + "fieldName": "check", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 1, + "title": "Check", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "check" + }, + { + "fieldName": "severity", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 2, + "title": "Severity", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "severity" + }, + { + "fieldName": "Status", + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "https://hls-eng-data-public.s3.amazonaws.com/img/{{Status}}.png", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "20", + "imageHeight": "20", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "integer", + "displayAs": "image", + "visible": true, + "order": 3, + "title": "Status", + "allowSearch": false, + "alignContent": "center", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "description": "Your workspace configuration status", + "displayName": "Status" + }, + { + "fieldName": "Recommendation", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ Rec_text }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "link", + "visible": true, + "order": 6, + "title": "Recommendation", + "allowSearch": false, + "alignContent": "left", + "allowHTML": true, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "Recommendation" + }, + { + "fieldName": "DASF_control", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 11, + "title": "DASF control", + "allowSearch": true, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "DASF_control" + } + ] + }, + "invisibleColumns": [ + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "category", + "type": "string", + "displayAs": "string", + "order": 4, + "title": "category", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "Rec_text", + "type": "string", + "displayAs": "string", + "order": 5, + "title": "Rec_text", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "dateTimeFormat": "YYYY-MM-DD HH:mm:ss.SSS", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "check_time", + "type": "datetime", + "displayAs": "datetime", + "order": 7, + "title": "check_time", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "dateTimeFormat": "YYYY-MM-DD", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "chk_date", + "type": "date", + "displayAs": "datetime", + "order": 8, + "title": "chk_date", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "workspace_id", + "type": "string", + "displayAs": "string", + "order": 9, + "title": "workspace_id", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "run_id", + "type": "integer", + "displayAs": "number", + "order": 10, + "title": "run_id", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "name", + "type": "string", + "displayAs": "string", + "order": 12, + "title": "name", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + } + ], + "allowHTMLByDefault": false, + "itemsPerPage": 25, + "paginationSize": "default", + "condensed": false, + "withRowNumber": true, + "frame": { + "title": "Informational summary", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 52, + "width": 6, + "height": 13 + } + }, + { + "widget": { + "name": "8fcb1324", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "a79323d9", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "rowNumber": 1, + "displayName": "counter" + } + }, + "frame": { + "title": "High", + "showTitle": true + } + } + }, + "position": { + "x": 3, + "y": 49, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "cedaebb7", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "3da334cd", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "rowNumber": 1, + "displayName": "counter" + } + }, + "frame": { + "title": "Low", + "showTitle": true + } + } + }, + "position": { + "x": 5, + "y": 20, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "2327c56d", + "textbox_spec": "![alt text](https://databricks.com/wp-content/uploads/2022/02/icon-orange-enterprise-security.svg)" + }, + "position": { + "x": 0, + "y": 20, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "8ec38759", + "textbox_spec": "# Data protection\n* Secrets management\n* Encryption\n* Data exfiltration protections\n" + }, + "position": { + "x": 1, + "y": 29, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "1e953334", + "queries": [ + { + "name": "dashboards/01ef16d35b091bb38cda8eaa2a3214dd/datasets/01ef16d35b3713e7aab6898685339d15_date", + "query": { + "datasetName": "7fe66d3b", + "fields": [ + { + "name": "date", + "expression": "`date`" + }, + { + "name": "date_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef16d35b091bb38cda8eaa2a3214dd/datasets/01ef16d35b2e1255af714930c209a40b_check_time", + "query": { + "datasetName": "322d185a", + "fields": [ + { + "name": "check_time", + "expression": "`check_time`" + }, + { + "name": "check_time_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef16d35b091bb38cda8eaa2a3214dd/datasets/01ef16d35b3d15d9bca5f141491d6e3d_check_time", + "query": { + "datasetName": "8a55ae87", + "fields": [ + { + "name": "check_time", + "expression": "`check_time`" + }, + { + "name": "check_time_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef16d35b091bb38cda8eaa2a3214dd/datasets/01ef16d35b0d138c8a133515db6190fa_check_time", + "query": { + "datasetName": "ae573d91", + "fields": [ + { + "name": "check_time", + "expression": "`check_time`" + }, + { + "name": "check_time_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef16d35b091bb38cda8eaa2a3214dd/datasets/01ef16d35b221298926440838631b988_check_time", + "query": { + "datasetName": "338dcb50", + "fields": [ + { + "name": "check_time", + "expression": "`check_time`" + }, + { + "name": "check_time_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef16d35b091bb38cda8eaa2a3214dd/datasets/01ef16d35b541294aecb0b42e50be874_check_time", + "query": { + "datasetName": "623f0dd8", + "fields": [ + { + "name": "check_time", + "expression": "`check_time`" + }, + { + "name": "check_time_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-single-select", + "encodings": { + "fields": [ + { + "fieldName": "date", + "displayName": "date", + "queryName": "dashboards/01ef16d35b091bb38cda8eaa2a3214dd/datasets/01ef16d35b3713e7aab6898685339d15_date" + }, + { + "fieldName": "check_time", + "displayName": "check_time", + "queryName": "dashboards/01ef16d35b091bb38cda8eaa2a3214dd/datasets/01ef16d35b2e1255af714930c209a40b_check_time" + }, + { + "fieldName": "check_time", + "displayName": "check_time", + "queryName": "dashboards/01ef16d35b091bb38cda8eaa2a3214dd/datasets/01ef16d35b3d15d9bca5f141491d6e3d_check_time" + }, + { + "fieldName": "check_time", + "displayName": "check_time", + "queryName": "dashboards/01ef16d35b091bb38cda8eaa2a3214dd/datasets/01ef16d35b0d138c8a133515db6190fa_check_time" + }, + { + "fieldName": "check_time", + "displayName": "check_time", + "queryName": "dashboards/01ef16d35b091bb38cda8eaa2a3214dd/datasets/01ef16d35b221298926440838631b988_check_time" + }, + { + "fieldName": "check_time", + "displayName": "check_time", + "queryName": "dashboards/01ef16d35b091bb38cda8eaa2a3214dd/datasets/01ef16d35b541294aecb0b42e50be874_check_time" + } + ] + }, + "disallowAll": true, + "frame": { + "showTitle": true, + "title": "Analysis date" + } + } + }, + "position": { + "x": 3, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "widget": { + "name": "e8ebdfee", + "textbox_spec": "# Informational\n* Admin counts\n* ACLs\n* User privileges\n* Library and repo usage \n" + }, + "position": { + "x": 1, + "y": 49, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "fdcb8ba7", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "109df071", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "displayName": "counter" + } + }, + "frame": { + "title": "High", + "showTitle": true + } + } + }, + "position": { + "x": 3, + "y": 10, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "9e72f63e", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "7f52e401", + "fields": [ + { + "name": "check_id", + "expression": "`check_id`" + }, + { + "name": "Check", + "expression": "`Check`" + }, + { + "name": "Severity", + "expression": "`Severity`" + }, + { + "name": "Run1Status", + "expression": "`Run1Status`" + }, + { + "name": "Run2Status", + "expression": "`Run2Status`" + }, + { + "name": "Run1Date", + "expression": "`Run1Date`" + }, + { + "name": "Run2Date", + "expression": "`Run2Date`" + }, + { + "name": "Recommendation", + "expression": "`Recommendation`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 1, + "widgetType": "table", + "encodings": { + "columns": [ + { + "fieldName": "check_id", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 100002, + "title": "Check id", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "check_id" + }, + { + "fieldName": "Check", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 100003, + "title": "Check", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "Check" + }, + { + "fieldName": "Severity", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 100004, + "title": "Severity", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "cellFormat": { + "default": { + "foregroundColor": null + }, + "rules": [ + { + "if": { + "column": "Severity", + "fn": "=", + "literal": "High" + }, + "value": { + "foregroundColor": "#D60303" + } + }, + { + "if": { + "column": "Severity", + "fn": "=", + "literal": "Medium" + }, + "value": { + "foregroundColor": "#BF5102" + } + }, + { + "if": { + "column": "Severity", + "fn": "=", + "literal": "Low" + }, + "value": { + "foregroundColor": "#E1D006" + } + } + ] + }, + "displayName": "Severity" + }, + { + "fieldName": "Run1Status", + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "https://hls-eng-data-public.s3.amazonaws.com/img/{{Run1Status}}.png", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "20", + "imageHeight": "20", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "integer", + "displayAs": "image", + "visible": true, + "order": 100005, + "title": "Run 1 status", + "allowSearch": false, + "alignContent": "center", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "cellFormat": { + "default": { + "foregroundColor": null + }, + "rules": [ + { + "if": { + "column": "Run1Status", + "fn": "=", + "literal": "0" + }, + "value": { + "foregroundColor": "#049235" + } + }, + { + "if": { + "column": "Run1Status", + "fn": "=", + "literal": "1" + }, + "value": { + "foregroundColor": "#E92828" + } + } + ] + }, + "description": "Your workspace configuration status", + "displayName": "Run1Status" + }, + { + "fieldName": "Run2Status", + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "https://hls-eng-data-public.s3.amazonaws.com/img/{{Run2Status}}.png", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "20", + "imageHeight": "20", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "integer", + "displayAs": "image", + "visible": true, + "order": 100006, + "title": "Run 2 status", + "allowSearch": false, + "alignContent": "center", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "description": "Your workspace configuration status", + "displayName": "Run2Status" + }, + { + "fieldName": "Run1Date", + "dateTimeFormat": "YYYY-MM-DD HH:mm:ss", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "datetime", + "displayAs": "datetime", + "visible": true, + "order": 100007, + "title": "Run1 date", + "allowSearch": false, + "alignContent": "center", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "Run1Date" + }, + { + "fieldName": "Run2Date", + "dateTimeFormat": "YYYY-MM-DD HH:mm:ss", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "datetime", + "displayAs": "datetime", + "visible": true, + "order": 100008, + "title": "Run 2 date", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "Run2Date" + }, + { + "fieldName": "Recommendation", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "type": "string", + "displayAs": "string", + "visible": true, + "order": 100009, + "title": "Recommendation", + "allowSearch": false, + "alignContent": "left", + "allowHTML": true, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false, + "displayName": "Recommendation" + } + ] + }, + "invisibleColumns": [ + { + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "current_score", + "type": "integer", + "displayAs": "number", + "order": 100000, + "title": "current_score", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "previous_score", + "type": "integer", + "displayAs": "number", + "order": 100001, + "title": "previous_score", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "numberFormat": "0", + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "ord", + "type": "integer", + "displayAs": "number", + "order": 100010, + "title": "ord", + "allowSearch": false, + "alignContent": "right", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "workspaceid", + "type": "string", + "displayAs": "string", + "order": 100011, + "title": "workspaceid", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + }, + { + "booleanValues": [ + "false", + "true" + ], + "imageUrlTemplate": "{{ @ }}", + "imageTitleTemplate": "{{ @ }}", + "imageWidth": "", + "imageHeight": "", + "linkUrlTemplate": "{{ @ }}", + "linkTextTemplate": "{{ @ }}", + "linkTitleTemplate": "{{ @ }}", + "linkOpenInNewTab": true, + "name": "name", + "type": "string", + "displayAs": "string", + "order": 100012, + "title": "name", + "allowSearch": false, + "alignContent": "left", + "allowHTML": false, + "highlightLinks": false, + "useMonospaceFont": false, + "preserveWhitespace": false + } + ], + "allowHTMLByDefault": false, + "itemsPerPage": 25, + "paginationSize": "default", + "condensed": true, + "withRowNumber": true, + "frame": { + "title": "Security configuration comparison", + "showTitle": true + } + } + }, + "position": { + "x": 0, + "y": 77, + "width": 6, + "height": 14 + } + }, + { + "widget": { + "name": "0ea932c3", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "6729e1a1", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "rowNumber": 1, + "displayName": "counter" + } + }, + "frame": { + "title": "Medium", + "showTitle": true + } + } + }, + "position": { + "x": 4, + "y": 49, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "f32c4099", + "queries": [ + { + "name": "d8abbb6f55d64e41a92c5eafd84c7b82", + "query": { + "datasetName": "648b5c27", + "disaggregated": true + } + } + ], + "spec": { + "version": 0, + "viz_spec": { + "display_name": "Region", + "description": "", + "viz_type": "COUNTER", + "serialized_options": "{\"counterColName\": \"value\", \"counterLabel\": \"\", \"rowNumber\": 2, \"stringDecChar\": \".\", \"stringDecimal\": 0, \"stringThouSep\": \",\", \"targetRowNumber\": 1, \"tooltipFormat\": \"0,0.000\"}", + "query_name": "d8abbb6f55d64e41a92c5eafd84c7b82" + } + } + }, + "position": { + "x": 5, + "y": 7, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "a4b10ce6", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "d1808288", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "rowNumber": 1, + "displayName": "counter" + } + }, + "frame": { + "title": "Low", + "showTitle": true + } + } + }, + "position": { + "x": 5, + "y": 49, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "9b08894d", + "textbox_spec": "# Network security\n* Secure cluster connectivity (on Azure, NoPublicIP)\n* Customer-managed VPC (or VNet injection)\n* VPC (or VNet) peering \n* IP access lists" + }, + "position": { + "x": 1, + "y": 10, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "6770c9d3", + "textbox_spec": "# Identity & Access\n* Single sign-on (AWS only, Azure and GCP always enabled)\n* SCIM for user provisioning\n* Role-based access control\n* Token management" + }, + "position": { + "x": 1, + "y": 20, + "width": 2, + "height": 3 + } + }, + { + "widget": { + "name": "2723b5ad", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "fdcac69c", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "rowNumber": 1, + "displayName": "counter" + } + }, + "frame": { + "title": "Medium", + "showTitle": true + } + } + }, + "position": { + "x": 4, + "y": 38, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "6ad9e41f", + "textbox_spec": "![alt text](https://databricks.com/wp-content/uploads/2022/02/icon-orange-cloud-security.svg)" + }, + "position": { + "x": 0, + "y": 10, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "d8573deb", + "queries": [ + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d6910cd8185366682e56a6c_name", + "query": { + "datasetName": "7fe66d3b", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d0110f5b7e54d08655c4190_name", + "query": { + "datasetName": "ae573d91", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d0f140cbc39c0a8a7f014f9_name", + "query": { + "datasetName": "6729e1a1", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d1b129a94e5ef4e8922d36e_name", + "query": { + "datasetName": "61330f46", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d1e1d35b3995b91589dd50b_name", + "query": { + "datasetName": "97f5b253", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d2c1556a8feb9d9996a5a70_name", + "query": { + "datasetName": "4d57d703", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d391d90b6fcf841cdaa8a87_name", + "query": { + "datasetName": "07804eaa", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d3d1e11a3b232c222aacbb3_name", + "query": { + "datasetName": "fdcac69c", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d49151089ea9466c6c16bda_name", + "query": { + "datasetName": "338dcb50", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d4c1845b8370df3d409c43b_name", + "query": { + "datasetName": "109df071", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d5013a5ac58fb764128a16c_name", + "query": { + "datasetName": "91cf6843", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d5b119494e225a8d5f5210d_name", + "query": { + "datasetName": "3da334cd", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d5e1a819e535fa2d0cf7c85_name", + "query": { + "datasetName": "322d185a", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d611d85bd933c6ea5965c30_ws_name", + "query": { + "datasetName": "648b5c27", + "fields": [ + { + "name": "ws_name", + "expression": "`ws_name`" + }, + { + "name": "ws_name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d6516c6b523d0c46e3ac427_name", + "query": { + "datasetName": "a79323d9", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d6c1bcc8404e1d08fff1239_name", + "query": { + "datasetName": "7778c9d9", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d7014779a91a95d6c5b3251_name", + "query": { + "datasetName": "8a55ae87", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d731f6484b65a4d2d1b607c_name", + "query": { + "datasetName": "d53104a9", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d8017ba80c06aa6b0e3f3ea_name", + "query": { + "datasetName": "4c4caf0a", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d88165c9b7e254543fdd4e2_name", + "query": { + "datasetName": "a4e4c607", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d8c160ca45ab07188596931_name", + "query": { + "datasetName": "5060eeb0", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7da31dbd81432b539f301aca_name", + "query": { + "datasetName": "623f0dd8", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7db419caa0ffd7b90d246d07_name", + "query": { + "datasetName": "d1808288", + "fields": [ + { + "name": "name", + "expression": "`name`" + }, + { + "name": "name_associativity", + "expression": "COUNT_IF(`associative_filter_predicate_group`)" + } + ], + "disaggregated": false + } + }, + { + "name": "parameter_dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7da81668b0c904e4b3dcd771_workspace_name", + "query": { + "datasetName": "7f52e401", + "parameters": [ + { + "name": "workspace_name", + "keyword": "workspace_name" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-single-select", + "encodings": { + "fields": [ + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d6910cd8185366682e56a6c_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d0110f5b7e54d08655c4190_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d0f140cbc39c0a8a7f014f9_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d1b129a94e5ef4e8922d36e_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d1e1d35b3995b91589dd50b_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d2c1556a8feb9d9996a5a70_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d391d90b6fcf841cdaa8a87_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d3d1e11a3b232c222aacbb3_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d49151089ea9466c6c16bda_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d4c1845b8370df3d409c43b_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d5013a5ac58fb764128a16c_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d5b119494e225a8d5f5210d_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d5e1a819e535fa2d0cf7c85_name" + }, + { + "fieldName": "ws_name", + "displayName": "ws_name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d611d85bd933c6ea5965c30_ws_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d6516c6b523d0c46e3ac427_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d6c1bcc8404e1d08fff1239_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d7014779a91a95d6c5b3251_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d731f6484b65a4d2d1b607c_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d8017ba80c06aa6b0e3f3ea_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d88165c9b7e254543fdd4e2_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7d8c160ca45ab07188596931_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7da31dbd81432b539f301aca_name" + }, + { + "fieldName": "name", + "displayName": "name", + "queryName": "dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7db419caa0ffd7b90d246d07_name" + }, + { + "parameterName": "workspace_name", + "queryName": "parameter_dashboards/01ef21fc7cfc123a8dac8f284edd132b/datasets/01ef21fc7da81668b0c904e4b3dcd771_workspace_name" + } + ] + }, + "disallowAll": true, + "frame": { + "showTitle": true, + "title": "Workspace" + } + } + }, + "position": { + "x": 0, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "widget": { + "name": "1f7e2290", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "a4e4c607", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "rowNumber": 1, + "displayName": "counter" + } + }, + "frame": { + "title": "Medium", + "showTitle": true + } + } + }, + "position": { + "x": 4, + "y": 20, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "29c090cb", + "queries": [ + { + "name": "main_query", + "query": { + "datasetName": "61330f46", + "fields": [ + { + "name": "counter", + "expression": "`counter`" + } + ], + "disaggregated": true + } + } + ], + "spec": { + "version": 2, + "widgetType": "counter", + "encodings": { + "value": { + "fieldName": "counter", + "rowNumber": 1, + "displayName": "counter" + } + }, + "frame": { + "title": "Low", + "showTitle": true + } + } + }, + "position": { + "x": 5, + "y": 38, + "width": 1, + "height": 3 + } + }, + { + "widget": { + "name": "4d5db327", + "queries": [ + { + "name": "parameter_dashboards/01ef1defa099176d99b646c9c1f7c41f/datasets/01ef1defa0ff1302863973c0b0ba5199_date1", + "query": { + "datasetName": "7f52e401", + "parameters": [ + { + "name": "date1", + "keyword": "date1" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-date-picker", + "encodings": { + "fields": [ + { + "parameterName": "date1", + "queryName": "parameter_dashboards/01ef1defa099176d99b646c9c1f7c41f/datasets/01ef1defa0ff1302863973c0b0ba5199_date1" + } + ] + }, + "frame": { + "showTitle": true, + "title": "Run 1 date" + } + } + }, + "position": { + "x": 0, + "y": 75, + "width": 2, + "height": 2 + } + }, + { + "widget": { + "name": "2e3db0ce", + "queries": [ + { + "name": "parameter_dashboards/01ef1defa099176d99b646c9c1f7c41f/datasets/01ef1defa0ff1302863973c0b0ba5199_date2", + "query": { + "datasetName": "7f52e401", + "parameters": [ + { + "name": "date2", + "keyword": "date2" + } + ], + "disaggregated": false + } + } + ], + "spec": { + "version": 2, + "widgetType": "filter-date-picker", + "encodings": { + "fields": [ + { + "parameterName": "date2", + "queryName": "parameter_dashboards/01ef1defa099176d99b646c9c1f7c41f/datasets/01ef1defa0ff1302863973c0b0ba5199_date2" + } + ] + }, + "frame": { + "showTitle": true, + "title": "Run 2 date" + } + } + }, + "position": { + "x": 4, + "y": 75, + "width": 2, + "height": 2 + } + }, + { + "widget": { + "name": "4699ecc4", + "textbox_spec": "# Compare runs" + }, + "position": { + "x": 2, + "y": 75, + "width": 2, + "height": 2 + } + } + ] + } + ] +} \ No newline at end of file diff --git a/docs/deprecated_old_setup.md b/docs/deprecated_old_setup.md new file mode 100644 index 00000000..bbddb382 --- /dev/null +++ b/docs/deprecated_old_setup.md @@ -0,0 +1,616 @@ +## Checklist to prepare for SAT setup + +**Note**: SAT creates a new **security_analysis** databses and Delta tables. + +If you are an existing SAT user please run the following command to reset your Database in your DBSQL SQL Editor. + +for Hive metastore based SAT schema: + ``` + drop database security_analysis cascade + ``` + or for Unity Catalog based SAT schema: + ``` + drop database .security_analysis cascade + ``` +Please make sure you are using char - in all secret key names as opposed to char _ . + +**Note**: SAT can be setup as Terraform based deployment, if you use Terrafrom in your organization please please prefer Terrafrom instructions: +* [SAT AWS Terraform deployment](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/terraform/aws/TERRAFORM_AWS.md) +* [SAT Azure Terraform deployment](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/terraform/azure/TERRAFORM_Azure.md) +* [SAT GCP Terraform deployment](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/terraform/gcp/TERRAFORM_GCP.md) + +**Note**: SAT is a productivity tool to help verify security configurations of Databricks deployments, it's not meant to be used as certification or attestation of your deployments. SAT project is regularly updated to improve the correctness of checks, add new checks, and fix bugs. Please send your feedback and comments to sat@databricks.com. + + +You will need the following information to set up SAT, we will show you how to gather them in the next section. + +We created a companion Security Analysis Tool (SAT) [Setup primer video for AWS](https://www.youtube.com/watch?v=kLSc3UHKL40), Step by step follow along instructions [video for Azure Databricks](https://youtu.be/xAav6GslSd8) and a [Deployment checklist sheet](./) to help prepare for the SAT setup. + +Note: SAT is beneficial to customers on [Databrics Premium or higher](https://www.databricks.com/product/pricing/platform-addons) as most of the checks and recommendations involve security features available in tiers higher than the Standard. + + 1. Databricks Account ID + 2. A Single user cluster (To run the SAT checks) + 3. Databricks SQL Warehouse (To run the SQL dashboard) + 4. Ensure that Databricks Repos is enabled (To access the SAT git) and [Files in Repo](https://docs.databricks.com/files/workspace.html#configure-support-for-workspace-files) set to DBR 8.4+ or DBR 11.0+ (To allow arbitrary files in Repo operations) + 5. Pypi access from your workspace (To install the SAT utility library) + 6. Create secrets scopes (To store configuration values) + 7. Authentication information: + * **AWS:** Administrative user id and password or Service Principal Client ID and Secret, to call the account REST APIs + * **Azure:** Azure subscriptionId, The Directory Tenant ID, The Application Client ID and The Client secret generated by AAD (To call the account REST APIs) + * **GCP:** Service account key and impersonate-service-account (To call the account REST APIs and read service account key file from GCS) + 8. Setup configuration in secrets and access configuration via secrets + 9. (Optional but recommended) A [Unity Catalog catalog name](https://docs.databricks.com/en/data-governance/unity-catalog/index.html#catalogs) to store SAT database, by default SAT uses Hive metastore to store SAT analysis as tables. You can use the Unity catalog instead of using Hive metastore. Make sure the clusters have the necessary [permissions](https://docs.databricks.com/en/data-governance/unity-catalog/manage-privileges/privileges.html) when using the Unity Catalog. The CREATE SCHEMA and USE CATALOG are required so that SAT can create the Schema in the catalog. + +## Prerequisites + + (Estimated time to complete these steps: 15 - 30 mins) + + +Please gather the following information before you start setting up: + + 1. Databricks Account ID + * Please test your administrator account and password to make sure this is a working account:
+ **AWS:** [https://accounts.cloud.databricks.com](https://accounts.cloud.databricks.com)
+ **Azure:** [https://accounts.azuredatabricks.net](https://accounts.azuredatabricks.net)
+ **GCP:** [https://accounts.gcp.databricks.com](https://accounts.gcp.databricks.com)
+ + * Copy the account id as shown below + + + + 2. A Single user cluster + * Databricks Runtime Version 14.3 LTS + * Node type i3.xlarge (please start with a max of a two node cluster and adjust to 5 nodes if you have many workspaces that will be analyzed with SAT) + + + **Note:** In our tests we found that the full run of SAT takes about 10 mins per workspace. + + 3. Databricks SQL Warehouse + * Goto SQL (pane) -> SQL Warehouse -> and pick the SQL Warehouse for your dashboard and note down the ID as shown below + * This Warehouse needs to be in a running state when you run steps in the Setup section. + + + + 4. Databricks Repos to access SAT git + Import git repo into Databricks repo + + ``` + https://github.com/databricks-industry-solutions/security-analysis-tool + ``` + + + + +5. Please confirm that PyPI access is available + + * Open the \/notebooks/Includes/install_sat_sdk and run on the cluster that was created in the Step 2 above. + Please make sure there are no errors. + If your deployment does not allow PyPI access please see the FAQ below at the end of this doc to see alternative options. + +6. Create secrets scopes + + * Download and setup Databricks CLI (version 0.205 and above) by following the instructions [here](https://docs.databricks.com/dev-tools/cli/index.html) on your work laptop or your virtual workstation. + * Note: if you have multiple Databricks profiles you will need to use --profile switch to access the correct workspace, + follow the instructions [here](https://docs.databricks.com/dev-tools/cli/index.html#connection-profiles) . Throughout the documentation below we use an example profile **e2-sat**, please adjust your commands as per your workspace profile or exclude --profile if you are using the default profile. + * Setup authentication to your Databricks workspace by following the instructions [here](https://docs.databricks.com/dev-tools/cli/index.html#set-up-authentication) + + ``` + databricks --profile e2-sat configure + ``` + + + + You should see a listing of folders in your workspace : + ``` + databricks --profile e2-sat workspace list /Users + ``` + + + + + * Set up the secret scope with the scope name you prefer and note it down: + + Note: The values you place below are case sensitive and need to be exact. + + ``` + databricks --profile e2-sat secrets create-scope sat_scope + ``` + + For more details refer [here](https://docs.databricks.com/dev-tools/cli/secrets-cli.html) + +7. #### Authentication information: + + +
+ AWS instructions + + You can either authenticate with username and password or using Service Principals credentials. + + **Authenticate with Username and Password (default)** + + Create username secret and password secret of administrative user id and password as "user" and "pass" under the above "sat_scope" scope using Databricks Secrets CLI + + * Input your Databricks account console admin username to store it in a the secret store + ``` + databricks --profile e2-sat secrets put-secret sat_scope user + ``` + + * Input your Databricks account console admin account password to store it in a the secret store + + ``` + databricks --profile e2-sat secrets put-secret sat_scope pass + ``` + + **Authenticate using a Service Principal** + + Create a Service Principal and generate a secret for it. Follow steps 1 to 3 in [this documentation](https://docs.databricks.com/dev-tools/authentication-oauth.html#:~:text=To%20create%20an%20OAuth%20secret%20for%20a%20service,the%20same%20as%20the%20service%20principal%E2%80%99s%20application%20ID) + Follow the instructions and add a service principal to a workspace using the admin console as detailed in the document for each workspace you would like to analyze as an admin (Note: the Admin role for the Service Principle is required to analyze many of the APIs). + + 1. Set the use_sp_auth to `true` in order to use the Service Principal Authentication Flow + + ``` + databricks --profile e2-sat secrets put-secret sat_scope use-sp-auth --string-value true + ``` + + 2. Store your Databricks Service Principal Client ID in the secret store: + + ``` + databricks --profile e2-sat secrets put-secret sat_scope client-id --string-value + ``` + + 3. Store your Databricks Service Principal Secret in the secret store: + + ``` + databricks --profile e2-sat secrets put-secret sat_scope client-secret --string-value + ``` +
+ +
+ Azure instructions + + We will be using the instructions in [Get Azure AD tokens for service principals](https://learn.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/aad/service-prin-aad-token). + * Follow the document above and complete all steps in the "Provision a service principal in Azure portal" only as detailed in the document. + * On the application page’s Overview page, in the Essentials section, copy the following values: (You will need this in the step below) + * Application (client) ID as client_id + * Directory (tenant) ID tenant_id + * client_secret (The secret generated by AAD during your confidential app registration) client_credential + + * Notedown the "Display name" as Service Principle name. (You will need this in the step below) + * Notedown the Subscription ID as subscription_id from the Subscriptions section of the Azure portal + * Please add the service principle with "Reader" role into the subscription level via Access control (IAM) using Role assignments under your [subscription, Access control (IAM) section](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal#step-2-open-the-add-role-assignment-page) + + +
+ +
+ GCP instructions + + We will be using the instructions in [Authenticate to workspace or account APIs with a Google ID token](https://docs.gcp.databricks.com/dev-tools/api/latest/authentication-google-id.html). + + * Follow the document above and complete all steps in the [Step 1](https://docs.gcp.databricks.com/dev-tools/api/latest/authentication-google-id.html#step-1-create-two-service-accounts) as detailed in the document. + * Notedown the name and location of service account key json file. (You will need this in the steps below) + * Notedown the impersonate-service-account email address. (You will need this in the step below) + * Upload the service account key json file to your [GCS bucket](https://docs.gcp.databricks.com/storage/gcs.html) from the [Authentication information](#authentication-information) above. Make sure to use the impersonate-service-account email address that you used above for the service account on the bucket, copy the "gsutil URI" ("File path to this resource in Cloud Storage") path. (You will need this in the steps below) + + + +
+ +9. Setup configuration in secrets and access configuration via secrets + * Create a secret for the workspace PAT token + + **Note**: Replace \ with your SAT deployment workspace id. + You can find your workspace id by following the instructions [here](https://docs.databricks.com/workspace/workspace-details.html) + + You can create a PAT token by following the instructions [here](https://docs.databricks.com/dev-tools/api/latest/authentication.html#generate-a-personal-access-token). Please pay attention to _ and - , scopes use _ and keys must use - . + * Set the PAT token value for the workspace_id + * Set the value for the account_id + * Set the value for the sql_warehouse_id + + + ``` + databricks --profile e2-sat secrets put-secret sat_scope sat-token- + ``` + + ``` + databricks --profile e2-sat secrets put-secret sat_scope account-console-id + ``` + + ``` + databricks --profile e2-sat secrets put-secret sat_scope sql-warehouse-id + ``` + + + * In your environment where you imported SAT project from git (Refer to Step 4 in Prerequisites) Open the \/notebooks/Utils/initialize notebook and modify the JSON string with : + * Set the value for the account_id + * Set the value for the sql_warehouse_id + * Databricks secrets scope/key names to pick the secrets from the steps above. + + * Your config in \/notebooks/Utils/initialize CMD 4 should look like this if you are using the secrets (Required for TF deployments), no need to edit the cell: + ``` + { + "account_id": dbutils.secrets.get(scope="sat_scope", key="account-console-id"), + "sql_warehouse_id": dbutils.secrets.get(scope="sat_scope", key="sql-warehouse-id") + "verbosity":"info" + } + + ``` + * Your config in \/notebooks/Utils/initialize CMD 4 should look like this if you are NOT using Terraform deployment and the secrets are not configured (backward compatibility). Change the analysis_schema_name value from security_analysis to a different name for SAT database to store its internal tables, change it to "catalog.schamaname" if you want to use Unity Catalog for storing SAT internal tables: + + ``` + { + "account_id":"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", <- replace with the actual account_id value + "sql_warehouse_id":"4d9fef7de2b9995c", <- replace with the actual sql_warehouse_id value + "analysis_schema_name":"security_analysis", <- database for SAT, use "catalog.database" for Unity Catalog or use "database" for Hive metastore. + "verbosity":"info" + } + + ``` + + * Azure additional configurations: +
+ Azure instructions + + * Setup the Subscription ID in a secret as subscription-id + + ``` + databricks --profile e2-sat secrets put-secret sat_scope subscription-id + ``` + + * Set the Directory (tenant) ID as tenant-id + + ``` + databricks --profile e2-sat secrets put-secret sat_scope tenant-id + ``` + + * Setup the Application (client) ID as client-id + + ``` + databricks --profile e2-sat secrets put-secret sat_scope client-id + ``` + + * Setup the Client secret in a secret + ``` + databricks --profile e2-sat secrets put-secret sat_scope client-secret + ``` + + * Your config in \/notebooks/Utils/initialize CMD 7 should look like this if you are using the secrets (Required for TF deployments), no need to edit the cell: + + ``` + if cloud_type == 'azure': + json_.update({ + "account_id":"azure", + "subscription_id": dbutils.secrets.get(scope="sat_scope", key="subscription-id"), # Azure subscriptionId + "tenant_id": dbutils.secrets.get(scope="sat_scope", key="tenant-id"), #The Directory (tenant) ID for the application registered in Azure AD. + "client_id": dbutils.secrets.get(scope="sat_scope", key="client-id"), # The Application (client) ID for the application registered in Azure AD. + "client_secret_key":"client-secret", #The secret generated by AAD during your confidential app registration + "use_mastercreds":True + }) + + ``` + + * Your config in \/notebooks/Utils/initialize CMD 7 should look like this if you are NOT using Terrafrom deployment and the secrets are not configured (backward compatibility): + + ``` + json_.update({ + "account_id":"azure", + "subscription_id":"xxxxxxxx-fake-46d6-82bd-5cc8d962326b", # Azure subscriptionId + "tenant_id":"xxxxxxxx-fake-4280-9796-b1864a10effd", #The Directory (tenant) ID for the application registered in Azure AD. + "client_id":"xxxxxxxx-fake-4q1a-bb68-6ear3b26btbd", # The Application (client) ID for the application registered in Azure AD. + "client_secret_key":"client-secret", #The secret generated by AAD during your confidential app registration + "use_mastercreds":True + }) + + ``` + * Follow the instructions "Add a service principal to a workspace" [Add a service principal to a workspace using the admin console](https://learn.microsoft.com/en-us/azure/databricks/administration-guide/users-groups/service-principals#--add-a-service-principal-to-a-workspace) as detailed in the document for each workspace you would like to analyze as an admin (Note: the Admin role for the Service Principle is required to analyze many of the APIs). + + + +
+ + * GCP additional configurations: +
+ GCP instructions + + + + * Setup the service account key json file in a secret as gs-path-to-json with the the "gsutil URI" ("File path to this resource in Cloud Storage") path : + + ``` + databricks --profile e2-sat secrets put-secret sat_scope gs-path-to-json + ``` + * Setup the impersonate-service-account email address in a secret as impersonate-service-account + + ``` + databricks --profile e2-sat secrets put-secret sat_scope impersonate-service-account + ``` + + * Your config in \/notebooks/Utils/initialize CMD 6 should look like this if you are using the secrets (Required for TF deployments), no need to edit the cell: + + + ``` + #GCP configurations + json_.update({ + "service_account_key_file_path": dbutils.secrets.get(scope="sat_scope_arun", key="gs-path-to-json"), + "impersonate_service_account": dbutils.secrets.get(scope="sat_scope_arun", key="impersonate-service-account"), + "use_mastercreds":False + }) + + ``` + * Your config in \/notebooks/Utils/initialize CMD 7 should look like this if you are NOT using Terrafrom deployment and the secrets are not configured (backward compatibility): + ``` + #GCP configurations + json_.update({ + "service_account_key_file_path":"gs://sat_dev/key/SA-1-key.json", <- update this value + "impersonate_service_account":"xyz-sa-2@project.iam.gserviceaccount.com", <- update this value + "use_mastercreds":False <- don't update this value + }) + ``` + + * Follow the instructions in Step 4 of [Authenticate to workspace or account APIs with a Google ID token]([https://docs.gcp.databricks.com/dev-tools/api/latest/authentication-google-id-account-private-preview.html#step-1-create-two-service-accounts](https://docs.gcp.databricks.com/dev-tools/api/latest/authentication-google-id-account-private-preview.html#step-4-add-the-service-account-as-a-workspace-or-account-user)) as detailed in the document for each workspace you would like to analyze and the account to add your main service account (SA-2) as an admin (Note: the Admin role for the Service account is required to analyze many of the APIs). + + + + + + * Make sure the cluster you configured to run the analysis has ability to read the "service account key json file" by adding the Google service account under the "Advanced Options" of the cluster to the "Google Service Account" value you noted. + + + +
+ + + +## Setup option 1 (Simple and recommended method) + + (Estimated time to complete these steps: 15 - 30 mins, varies by number of workspaces in the account) + This method uses admin/service principle credentials (configured in the Step 6 of Prerequisites section) to call workspace APIs. + + Make sure both SAT job cluster (Refer to Prerequisites Step 2 ) and Warehouse (Refer to Prerequisites Step 3) are running. +
+ Setup instructions + Following is the one time easy setup to get your workspaces setup with the SAT: + +* Attach \/notebooks/security_analysis_initializer to the SAT cluster you created above and Run -> Run all + + + + + + + +
+ +## Setup option 2 (Most flexible for the power users) + + (Estimated time to complete these steps: 30 mins) + This method uses admin credentials (configured in the Step 6 of Prerequisites section) by default to call workspace APIs. But can be changed to use workspace PAT tokens instead. +
+ Setup instructions + Following are the one time easy steps to get your workspaces setup with the SAT: + + +1. List account workspaces to analyze with SAT + * Goto \/notebooks/Setup/1.list_account_workspaces_to_conf_file and Run -> Run all + * This creates a configuration file as noted at the bottom of the notebook. + + + + +2. Generate secrets setup file (AWS only. Not recommended for Azure and GCP) + Deprecated as AWS Service principles are encouraged inplace of workspace PATs. + + +3. Test API Connections + * Test connections from your workspace to accounts API calls and all workspace API calls by running \/notebooks/Setup/3. test_connections. The workspaces that didn't pass the connection test are marked in workspace_configs.csv with connection_test as False and are not analyzed. + + + +4. Enable workspaces for SAT analysis + * Enable workspaces by running \/notebooks/Setup/4. enable_workspaces_for_sat. This makes the registered workspaces ready for SAT to monitor + + + +5. Import SAT dashboard template + * We built a ready to go DBSQL dashboard for SAT. Import the dashboard by running \/notebooks/Setup/5. import_dashboard_template + + + +6. Configure Alerts + SAT can deliver alerts via email via Databricks SQL Alerts. Import the alerts template by running \/notebooks/Setup/6. configure_alerts_template (optional) + + + +
+ +## Update configuration files + +1. Modify security_best_practices (Optional) + * Go to \/notebooks/Setup/7. update_sat_check_configuration and use this utility to enable/disable a Check, modify Evaluation Value and Alert configuration value for each check. You can update this file any time and any analysis from there on will take these values into consideration. + * [Configure widget settings](https://docs.databricks.com/notebooks/widgets.html#configure-widget-settings-1) behavior "On Widget Change" for this notebooks to "Do Nothing" + + + +2. Modify workspace_configs file (Required for manual checks values) + * **Note**: Limit number of workspaces to be analyzed by SAT to 100. + * **Tip**: You can use this utility to turn on a specific workspace and turn off other workspaces for a specific run. + * **Tip**: You can use this utility to apply your edits to multiple workspaces settings by using "Apply Setting to all workspaces" option. + + * Go to\/notebooks/Setup/8. update_workspace_configuration and You will need to set analysis_enabled as True or False based on if you would like to enroll a workspace to analyze by the SAT. + * [Configure widget settings](https://docs.databricks.com/notebooks/widgets.html#configure-widget-settings-1) behavior "On Widget Change" for this notebooks to "Do Nothing" + + + Update values for each workspace for the manual checks:( sso_enabled,scim_enabled,vpc_peering_done,object_storage_encypted,table_access_control_enabled) + + * sso_enabled : True if you enabled Single Singn-on for the workspace + * scim_enabled: True if you integrated with SCIM for the workspace + * vpc_peering_done: False if you have not peered with another VPC + * object_storage_encypted: True if you encrypted your data buckets + * table_access_control_enabled : True if you enabled ACLs so that you can utilize Table ACL clusters that enforce user isolation + + +## Usage + + (Estimated time to complete these steps: 5 - 10 mins per workspace) + **Note**: Limit number of workspaces to be analyzed by SAT to 100. +1. Attach and run the notebook \/notebooks/security_analysis_driver + Note: This process takes upto 10 mins per workspace + + + + + At this point you should see **SAT** database and tables in your SQL Warehouses: + + + + + +2. Access Databricks SQL Dashboards section and find "SAT - Security Analysis Tool" dashboard to see the report. You can filter the dashboard by **SAT** tag. + + + + **Note:** You need to select the workspace and date and click "Apply Changes" to get the report. + **Note:** The dashbord shows last valid run for the selected date if there is one, if not it shows the latest report for that workspace. + + You can share SAT dashboard with other members of your team by using the "Share" functionality on the top right corner of the dashboard. + + + Here is what your SAT Dashboard should look like: + + + +3. Activate Alerts + * Goto Alerts and find the alert(s) created by SAT tag and adjust the schedule to your needs. You can add more recpients to alerts by configuring [notification destinations](https://docs.databricks.com/sql/admin/notification-destinations.html). + + + + + + + + + +## Configure Workflow (Optional) + + (Estimated time to complete these steps: 5 mins) + + * Databricks Workflows is the fully-managed orchestration service. You can configure SAT to automate when and how you would like to schedule it by using by taking advantage of Workflows. + + * Goto Workflows - > click on create jobs -> setup as following: + + Task Name : security_analysis_driver + + Type: Notebook + + Source: Workspace (or your git clone of SAT) + + Path : \/SAT/SecurityAnalysisTool-BranchV2Root/notebooks/security_analysis_driver + + Cluster: Make sure to pick the Single user mode job compute cluster you created before. + + + + Add a schedule as per your needs. That’s it. Now you are continuously monitoring the health of your account workspaces. + + +## FAQs + 1. How can SAT be configured if access to github is not possible due to firewall restrictions to git or other organization policies? + + You can still setup SAT by downloading the [release zip](https://github.com/databricks-industry-solutions/security-analysis-tool/releases) file and by using Git repo to load SAT project into your workspace. + * Add Repo by going to Repos in your workspace: + + + + * Type SAT as your "Repository name" and uncheck "Create repo by cloning a Git repository" + + + + * Click on the pulldown menu and click on Import + + + + * Drag and drop the release zip file and click Import + + + + + + You should see the SAT project in your workspace. +2. Can SAT make modifications to my workspaces and account? + + + No. SAT is meant to be a readonly analysis tool, it does not make changes to your workspace or account configurations. + +3. I added a new workspace for analysis, re-ran steps under initilaize and driver, ... the dashboard is not updated with the new workspace in the pulldown even though I see new data generated by the analysis scan for the new workspace in SAT database. What should I do? + + It is likely that the Dashboard cached the workspaces in the pulldown. You can go to SQL view of your workspace -> Queries -> find workspace_ids query and run it, that should refresh and have the new workspaces in the pull-down. + + +## Troubleshooting +We created diagnosis notebooks for respective clouds to help troubleshoot your SAT setup. Please review notebooks/diagnosis/ folder. + +1. Incorrectly configured secrets + * Error: + + Secret does not exist with scope: sat_scope and key: sat_tokens + + * Resolution: + Check if the tokens are configured with the correct names by listing and comparing with the configuration. + databricks --profile e2-sat secrets list-secrets sat_scope + +2. Invalid access token + + * Error: + + Error 403 Invalid access token. + + * Resolution: + + Check your PAT token configuration for “workspace_pat_token” key + +3. Firewall blocking databricks accounts console + + * Error: +

+ Traceback (most recent call last): File "/databricks/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 670, in urlopen httplib_response = self._make_request( File "/databricks/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 381, in _make_request self._validate_conn(conn) File "/databricks/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 978, in _validate_conn conn.connect() File "/databricks/python/lib/python3.8/site-packages/urllib3/connection.py", line 362, in connect self.sock = ssl_wrap_socket( File "/databricks/python/lib/python3.8/site-packages/urllib3/util/ssl_.py", line 386, in ssl_wrap_socket return context.wrap_socket(sock, server_hostname=server_hostname) File "/usr/lib/python3.8/ssl.py", line 500, in wrap_socket return self.sslsocket_class._create( File "/usr/lib/python3.8/ssl.py", line 1040, in _create self.do_handshake() File "/usr/lib/python3.8/ssl.py", line 1309, in do_handshake self._sslobj.do_handshake() ConnectionResetError: [Errno 104] Connection reset by peer During handling of the above exception, another exception occurred: + + * Resolution: + + Run this following command in your notebook %sh + curl -X GET -H "Authorization: Basic /" -H "Content-Type: application/json" https://accounts.cloud.databricks.com/api/2.0/accounts//workspaces + + or + + %sh curl -u 'user:password' -X GET  “Content-Type: application/json” https://accounts.cloud.databricks.com/api/2.0/accounts//workspaces + + If you don’t see a JSON with a clean listing of workspaces you are likely having a firewall issue that is blocking calls to the accounts console. Please have your infrastructure team add Databricks accounts.cloud.databricks.com to the allow-list. Please see that the private IPv4 address from NAT gateway to the IP allow list. + +4. Offline install of libraries incase of no PyPI access + + Download the dbl_sat_sdk version specified in the notebook notebooks/utils/initialize from PyPi + https://pypi.org/project/dbl-sat-sdk/ + Upload the dbl_sat_sdk-w.x.y-py3-none-any.whl to a dbfs location. You can use the databricks-cli as one mechanism to upload. + for e.g. + + ``` + databricks --profile e2-satfs cp /localdrive/whlfile/dbl_sat_sdk-w.x.y-py3-none-any.whl dbfs:/FileStore/wheels/ + ``` + Additionally download the following wheel files and upload them to the dbfs location as above. + https://github.com/databricks-industry-solutions/security-analysis-tool/tree/main/docs/wheels + + Upload all wheel files to /FileStore/wheels in your workspace + Verify all files are there by doing a %fs ls /FileStore/wheels from your notebook. + Then change the cell in your notebook install_sat_sdk to this + + ``` + %pip install cachetools --find-links /dbfs/FileStore/wheels/cachetools-5.3.1-py3-none-any.whl + %pip install pyasn1 --find-links /dbfs/FileStore/wheels/pyasn1-0.5.0-py2.py3-none-any.whl + %pip install pyasn1-modules --find-links /dbfs/FileStore/wheels/pyasn1_modules-0.3.0-py2.py3-none-any.whl + %pip install rsa --find-links /dbfs/FileStore/wheels/rsa-4.9-py3-none-any.whl + %pip install google-auth --find-links /dbfs/FileStore/wheels/google_auth-2.22.0-py2.py3-none-any.whl + %pip install PyJWT[crypto] --find-links /dbfs/FileStore/wheels/PyJWT-2.8.0-py3-none-any.whl + %pip install msal --find-links /dbfs/FileStore/wheels/msal-1.22.0-py2.py3-none-any.whl + %pip install dbl-sat-sdk==0.1.32 --find-links /dbfs/FileStore/wheels/dbl_sat_sdk-0.1.32-py3-none-any.whl + ``` + + Make sure the versions for the above libraries match. diff --git a/docs/gif/terminal-aws.gif b/docs/gif/terminal-aws.gif new file mode 100644 index 00000000..6aa2b06d Binary files /dev/null and b/docs/gif/terminal-aws.gif differ diff --git a/docs/gif/terminal-azure.gif b/docs/gif/terminal-azure.gif new file mode 100644 index 00000000..4f21e20d Binary files /dev/null and b/docs/gif/terminal-azure.gif differ diff --git a/docs/gif/terminal-gcp.gif b/docs/gif/terminal-gcp.gif new file mode 100644 index 00000000..fe538154 Binary files /dev/null and b/docs/gif/terminal-gcp.gif differ diff --git a/docs/images/aws_ws.png b/docs/images/aws_ws.png new file mode 100644 index 00000000..31086261 Binary files /dev/null and b/docs/images/aws_ws.png differ diff --git a/docs/images/azure_app_reg.png b/docs/images/azure_app_reg.png new file mode 100644 index 00000000..9c290951 Binary files /dev/null and b/docs/images/azure_app_reg.png differ diff --git a/docs/images/azure_ws.png b/docs/images/azure_ws.png new file mode 100644 index 00000000..e6515984 Binary files /dev/null and b/docs/images/azure_ws.png differ diff --git a/docs/setup.md b/docs/setup.md index bbddb382..26c831c8 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -1,616 +1,141 @@ -## Checklist to prepare for SAT setup +# Setup Guide -**Note**: SAT creates a new **security_analysis** databses and Delta tables. +Follow this guide to setup the Security Analysis Tool (SAT) on your Databricks workspace. -If you are an existing SAT user please run the following command to reset your Database in your DBSQL SQL Editor. +## Prerequisites -for Hive metastore based SAT schema: - ``` - drop database security_analysis cascade - ``` - or for Unity Catalog based SAT schema: - ``` - drop database .security_analysis cascade - ``` -Please make sure you are using char - in all secret key names as opposed to char _ . +Before proceeding with the installation, make sure you have the following prerequisites: -**Note**: SAT can be setup as Terraform based deployment, if you use Terrafrom in your organization please please prefer Terrafrom instructions: -* [SAT AWS Terraform deployment](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/terraform/aws/TERRAFORM_AWS.md) -* [SAT Azure Terraform deployment](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/terraform/azure/TERRAFORM_Azure.md) -* [SAT GCP Terraform deployment](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/terraform/gcp/TERRAFORM_GCP.md) +- Python 3.9 or higher +- Databricks CLI installed with a profile logged (See [here](https://docs.databricks.com/en/dev-tools/cli/install.html).) +- Databricks Account ID +- Databricks SQL Warehouse (To run the SQL dashboard) +- Pypi access from your workspace (To install the SAT utility library) + +> SAT is beneficial to customers on **Databrics Premium or Enterprise** as most of the checks and recommendations involve security features available in tiers higher than the Standard. -**Note**: SAT is a productivity tool to help verify security configurations of Databricks deployments, it's not meant to be used as certification or attestation of your deployments. SAT project is regularly updated to improve the correctness of checks, add new checks, and fix bugs. Please send your feedback and comments to sat@databricks.com. +### Considerations +SAT creates a new security_analysis database and Delta tables. If you are an existing SAT user please run the following command: -You will need the following information to set up SAT, we will show you how to gather them in the next section. +### Hive metastore based schema -We created a companion Security Analysis Tool (SAT) [Setup primer video for AWS](https://www.youtube.com/watch?v=kLSc3UHKL40), Step by step follow along instructions [video for Azure Databricks](https://youtu.be/xAav6GslSd8) and a [Deployment checklist sheet](./) to help prepare for the SAT setup. +```sql + drop database security_analysis cascade; +``` -Note: SAT is beneficial to customers on [Databrics Premium or higher](https://www.databricks.com/product/pricing/platform-addons) as most of the checks and recommendations involve security features available in tiers higher than the Standard. +### Unity Catalog based schema - 1. Databricks Account ID - 2. A Single user cluster (To run the SAT checks) - 3. Databricks SQL Warehouse (To run the SQL dashboard) - 4. Ensure that Databricks Repos is enabled (To access the SAT git) and [Files in Repo](https://docs.databricks.com/files/workspace.html#configure-support-for-workspace-files) set to DBR 8.4+ or DBR 11.0+ (To allow arbitrary files in Repo operations) - 5. Pypi access from your workspace (To install the SAT utility library) - 6. Create secrets scopes (To store configuration values) - 7. Authentication information: - * **AWS:** Administrative user id and password or Service Principal Client ID and Secret, to call the account REST APIs - * **Azure:** Azure subscriptionId, The Directory Tenant ID, The Application Client ID and The Client secret generated by AAD (To call the account REST APIs) - * **GCP:** Service account key and impersonate-service-account (To call the account REST APIs and read service account key file from GCS) - 8. Setup configuration in secrets and access configuration via secrets - 9. (Optional but recommended) A [Unity Catalog catalog name](https://docs.databricks.com/en/data-governance/unity-catalog/index.html#catalogs) to store SAT database, by default SAT uses Hive metastore to store SAT analysis as tables. You can use the Unity catalog instead of using Hive metastore. Make sure the clusters have the necessary [permissions](https://docs.databricks.com/en/data-governance/unity-catalog/manage-privileges/privileges.html) when using the Unity Catalog. The CREATE SCHEMA and USE CATALOG are required so that SAT can create the Schema in the catalog. - -## Prerequisites +```sql + drop database .security_analysis cascade; +``` - (Estimated time to complete these steps: 15 - 30 mins) +## Setup +> SAT is a productivity tool to help verify security configurations of Databricks deployments, it's not meant to be used as certification or attestation of your deployments. SAT project is regularly updated to improve the correctness of checks, add new checks, and fix bugs. Please send your feedback and comments to sat@databricks.com. -Please gather the following information before you start setting up: - - 1. Databricks Account ID - * Please test your administrator account and password to make sure this is a working account:
- **AWS:** [https://accounts.cloud.databricks.com](https://accounts.cloud.databricks.com)
- **Azure:** [https://accounts.azuredatabricks.net](https://accounts.azuredatabricks.net)
- **GCP:** [https://accounts.gcp.databricks.com](https://accounts.gcp.databricks.com)
- - * Copy the account id as shown below - - - - 2. A Single user cluster - * Databricks Runtime Version 14.3 LTS - * Node type i3.xlarge (please start with a max of a two node cluster and adjust to 5 nodes if you have many workspaces that will be analyzed with SAT) - - - **Note:** In our tests we found that the full run of SAT takes about 10 mins per workspace. - - 3. Databricks SQL Warehouse - * Goto SQL (pane) -> SQL Warehouse -> and pick the SQL Warehouse for your dashboard and note down the ID as shown below - * This Warehouse needs to be in a running state when you run steps in the Setup section. - - - - 4. Databricks Repos to access SAT git - Import git repo into Databricks repo +SAT can be setup on any of the cloud providers where Databricks is hosted. Follow the setup guide for the cloud provider you are using: - ``` - https://github.com/databricks-industry-solutions/security-analysis-tool - ``` +- [AWS Setup Guide](./setup/aws.md) +- [Azure Setup Guide](./setup/azure.md) +- [GCP Setup Guide](./setup/gcp.md) +**Note**: SAT can be setup as Terraform based deployment, if you use Terraform in your organization please prefer Terraform instructions: - +* [SAT AWS Terraform deployment](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/terraform/aws/TERRAFORM_AWS.md) +* [SAT Azure Terraform deployment](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/terraform/azure/TERRAFORM_Azure.md) +* [SAT GCP Terraform deployment](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/terraform/gcp/TERRAFORM_GCP.md) -5. Please confirm that PyPI access is available +## FAQs and Troubleshooting - * Open the \/notebooks/Includes/install_sat_sdk and run on the cluster that was created in the Step 2 above. - Please make sure there are no errors. - If your deployment does not allow PyPI access please see the FAQ below at the end of this doc to see alternative options. +[Find answers to frequently asked questions, troubleshoot SAT issues, or diagnose SAT setup](./setup/faqs_and_troubleshooting.md) -6. Create secrets scopes - * Download and setup Databricks CLI (version 0.205 and above) by following the instructions [here](https://docs.databricks.com/dev-tools/cli/index.html) on your work laptop or your virtual workstation. - * Note: if you have multiple Databricks profiles you will need to use --profile switch to access the correct workspace, - follow the instructions [here](https://docs.databricks.com/dev-tools/cli/index.html#connection-profiles) . Throughout the documentation below we use an example profile **e2-sat**, please adjust your commands as per your workspace profile or exclude --profile if you are using the default profile. - * Setup authentication to your Databricks workspace by following the instructions [here](https://docs.databricks.com/dev-tools/cli/index.html#set-up-authentication) +## Usage + - ``` - databricks --profile e2-sat configure - ``` + > **Note**: Go to Workspace -> Applications -> SAT -> files -> self_assessment_checks.yaml and make sure the "enabled" values reflect your environment for the listed manual checks with either true or false. SAT will automatically check the rest of the configurations. - - You should see a listing of folders in your workspace : - ``` - databricks --profile e2-sat workspace list /Users - ``` +### 1. Run jobs - + > **Note**: This process takes up to 10 mins per workspace + +You now have two jobs (SAT Initializer Notebook & SAT Driver Notebook). Run SAT Initializer Notebook and when it completes run SAT Driver Notebook; SAT Initializer Notebook should only be run once (although you can run it multiple times, it only needs to be run successfully one time), and SAT Driver Notebook can be run periodically (it's scheduled to run once every Monday, Wednesday, and Friday) + At this point you should see **SAT** database and tables in your SQL Warehouses: - * Set up the secret scope with the scope name you prefer and note it down: - - Note: The values you place below are case sensitive and need to be exact. - - ``` - databricks --profile e2-sat secrets create-scope sat_scope - ``` + - For more details refer [here](https://docs.databricks.com/dev-tools/cli/secrets-cli.html) -7. #### Authentication information: - - -

- AWS instructions - - You can either authenticate with username and password or using Service Principals credentials. - - **Authenticate with Username and Password (default)** - - Create username secret and password secret of administrative user id and password as "user" and "pass" under the above "sat_scope" scope using Databricks Secrets CLI - - * Input your Databricks account console admin username to store it in a the secret store - ``` - databricks --profile e2-sat secrets put-secret sat_scope user - ``` - - * Input your Databricks account console admin account password to store it in a the secret store - - ``` - databricks --profile e2-sat secrets put-secret sat_scope pass - ``` - - **Authenticate using a Service Principal** - - Create a Service Principal and generate a secret for it. Follow steps 1 to 3 in [this documentation](https://docs.databricks.com/dev-tools/authentication-oauth.html#:~:text=To%20create%20an%20OAuth%20secret%20for%20a%20service,the%20same%20as%20the%20service%20principal%E2%80%99s%20application%20ID) - Follow the instructions and add a service principal to a workspace using the admin console as detailed in the document for each workspace you would like to analyze as an admin (Note: the Admin role for the Service Principle is required to analyze many of the APIs). - - 1. Set the use_sp_auth to `true` in order to use the Service Principal Authentication Flow - - ``` - databricks --profile e2-sat secrets put-secret sat_scope use-sp-auth --string-value true - ``` - - 2. Store your Databricks Service Principal Client ID in the secret store: - - ``` - databricks --profile e2-sat secrets put-secret sat_scope client-id --string-value - ``` - - 3. Store your Databricks Service Principal Secret in the secret store: - - ``` - databricks --profile e2-sat secrets put-secret sat_scope client-secret --string-value - ``` -
- -
- Azure instructions - - We will be using the instructions in [Get Azure AD tokens for service principals](https://learn.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/aad/service-prin-aad-token). - * Follow the document above and complete all steps in the "Provision a service principal in Azure portal" only as detailed in the document. - * On the application page’s Overview page, in the Essentials section, copy the following values: (You will need this in the step below) - * Application (client) ID as client_id - * Directory (tenant) ID tenant_id - * client_secret (The secret generated by AAD during your confidential app registration) client_credential - - * Notedown the "Display name" as Service Principle name. (You will need this in the step below) - * Notedown the Subscription ID as subscription_id from the Subscriptions section of the Azure portal - * Please add the service principle with "Reader" role into the subscription level via Access control (IAM) using Role assignments under your [subscription, Access control (IAM) section](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal#step-2-open-the-add-role-assignment-page) - - -
- -
- GCP instructions - - We will be using the instructions in [Authenticate to workspace or account APIs with a Google ID token](https://docs.gcp.databricks.com/dev-tools/api/latest/authentication-google-id.html). - - * Follow the document above and complete all steps in the [Step 1](https://docs.gcp.databricks.com/dev-tools/api/latest/authentication-google-id.html#step-1-create-two-service-accounts) as detailed in the document. - * Notedown the name and location of service account key json file. (You will need this in the steps below) - * Notedown the impersonate-service-account email address. (You will need this in the step below) - * Upload the service account key json file to your [GCS bucket](https://docs.gcp.databricks.com/storage/gcs.html) from the [Authentication information](#authentication-information) above. Make sure to use the impersonate-service-account email address that you used above for the service account on the bucket, copy the "gsutil URI" ("File path to this resource in Cloud Storage") path. (You will need this in the steps below) - - - -
- -9. Setup configuration in secrets and access configuration via secrets - * Create a secret for the workspace PAT token - - **Note**: Replace \ with your SAT deployment workspace id. - You can find your workspace id by following the instructions [here](https://docs.databricks.com/workspace/workspace-details.html) - - You can create a PAT token by following the instructions [here](https://docs.databricks.com/dev-tools/api/latest/authentication.html#generate-a-personal-access-token). Please pay attention to _ and - , scopes use _ and keys must use - . - * Set the PAT token value for the workspace_id - * Set the value for the account_id - * Set the value for the sql_warehouse_id - +### 2. Access Databricks SQL Dashboards - ``` - databricks --profile e2-sat secrets put-secret sat_scope sat-token- - ``` - - ``` - databricks --profile e2-sat secrets put-secret sat_scope account-console-id - ``` - - ``` - databricks --profile e2-sat secrets put-secret sat_scope sql-warehouse-id - ``` - + > **Note:** You can also use Lakeview Dashboards to view the results, instead of classic Dashboards. - * In your environment where you imported SAT project from git (Refer to Step 4 in Prerequisites) Open the \/notebooks/Utils/initialize notebook and modify the JSON string with : - * Set the value for the account_id - * Set the value for the sql_warehouse_id - * Databricks secrets scope/key names to pick the secrets from the steps above. - - * Your config in \/notebooks/Utils/initialize CMD 4 should look like this if you are using the secrets (Required for TF deployments), no need to edit the cell: - ``` - { - "account_id": dbutils.secrets.get(scope="sat_scope", key="account-console-id"), - "sql_warehouse_id": dbutils.secrets.get(scope="sat_scope", key="sql-warehouse-id") - "verbosity":"info" - } - - ``` - * Your config in \/notebooks/Utils/initialize CMD 4 should look like this if you are NOT using Terraform deployment and the secrets are not configured (backward compatibility). Change the analysis_schema_name value from security_analysis to a different name for SAT database to store its internal tables, change it to "catalog.schamaname" if you want to use Unity Catalog for storing SAT internal tables: - - ``` - { - "account_id":"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", <- replace with the actual account_id value - "sql_warehouse_id":"4d9fef7de2b9995c", <- replace with the actual sql_warehouse_id value - "analysis_schema_name":"security_analysis", <- database for SAT, use "catalog.database" for Unity Catalog or use "database" for Hive metastore. - "verbosity":"info" - } - - ``` - - * Azure additional configurations: -
- Azure instructions - - * Setup the Subscription ID in a secret as subscription-id - - ``` - databricks --profile e2-sat secrets put-secret sat_scope subscription-id - ``` - - * Set the Directory (tenant) ID as tenant-id - - ``` - databricks --profile e2-sat secrets put-secret sat_scope tenant-id - ``` - - * Setup the Application (client) ID as client-id - - ``` - databricks --profile e2-sat secrets put-secret sat_scope client-id - ``` - - * Setup the Client secret in a secret - ``` - databricks --profile e2-sat secrets put-secret sat_scope client-secret - ``` - - * Your config in \/notebooks/Utils/initialize CMD 7 should look like this if you are using the secrets (Required for TF deployments), no need to edit the cell: - - ``` - if cloud_type == 'azure': - json_.update({ - "account_id":"azure", - "subscription_id": dbutils.secrets.get(scope="sat_scope", key="subscription-id"), # Azure subscriptionId - "tenant_id": dbutils.secrets.get(scope="sat_scope", key="tenant-id"), #The Directory (tenant) ID for the application registered in Azure AD. - "client_id": dbutils.secrets.get(scope="sat_scope", key="client-id"), # The Application (client) ID for the application registered in Azure AD. - "client_secret_key":"client-secret", #The secret generated by AAD during your confidential app registration - "use_mastercreds":True - }) - - ``` - - * Your config in \/notebooks/Utils/initialize CMD 7 should look like this if you are NOT using Terrafrom deployment and the secrets are not configured (backward compatibility): - - ``` - json_.update({ - "account_id":"azure", - "subscription_id":"xxxxxxxx-fake-46d6-82bd-5cc8d962326b", # Azure subscriptionId - "tenant_id":"xxxxxxxx-fake-4280-9796-b1864a10effd", #The Directory (tenant) ID for the application registered in Azure AD. - "client_id":"xxxxxxxx-fake-4q1a-bb68-6ear3b26btbd", # The Application (client) ID for the application registered in Azure AD. - "client_secret_key":"client-secret", #The secret generated by AAD during your confidential app registration - "use_mastercreds":True - }) - - ``` - * Follow the instructions "Add a service principal to a workspace" [Add a service principal to a workspace using the admin console](https://learn.microsoft.com/en-us/azure/databricks/administration-guide/users-groups/service-principals#--add-a-service-principal-to-a-workspace) as detailed in the document for each workspace you would like to analyze as an admin (Note: the Admin role for the Service Principle is required to analyze many of the APIs). - - - -
- - * GCP additional configurations: -
- GCP instructions - - - - * Setup the service account key json file in a secret as gs-path-to-json with the the "gsutil URI" ("File path to this resource in Cloud Storage") path : - - ``` - databricks --profile e2-sat secrets put-secret sat_scope gs-path-to-json - ``` - * Setup the impersonate-service-account email address in a secret as impersonate-service-account - - ``` - databricks --profile e2-sat secrets put-secret sat_scope impersonate-service-account - ``` - - * Your config in \/notebooks/Utils/initialize CMD 6 should look like this if you are using the secrets (Required for TF deployments), no need to edit the cell: - - ``` - #GCP configurations - json_.update({ - "service_account_key_file_path": dbutils.secrets.get(scope="sat_scope_arun", key="gs-path-to-json"), - "impersonate_service_account": dbutils.secrets.get(scope="sat_scope_arun", key="impersonate-service-account"), - "use_mastercreds":False - }) - - ``` - * Your config in \/notebooks/Utils/initialize CMD 7 should look like this if you are NOT using Terrafrom deployment and the secrets are not configured (backward compatibility): - ``` - #GCP configurations - json_.update({ - "service_account_key_file_path":"gs://sat_dev/key/SA-1-key.json", <- update this value - "impersonate_service_account":"xyz-sa-2@project.iam.gserviceaccount.com", <- update this value - "use_mastercreds":False <- don't update this value - }) - ``` - - * Follow the instructions in Step 4 of [Authenticate to workspace or account APIs with a Google ID token]([https://docs.gcp.databricks.com/dev-tools/api/latest/authentication-google-id-account-private-preview.html#step-1-create-two-service-accounts](https://docs.gcp.databricks.com/dev-tools/api/latest/authentication-google-id-account-private-preview.html#step-4-add-the-service-account-as-a-workspace-or-account-user)) as detailed in the document for each workspace you would like to analyze and the account to add your main service account (SA-2) as an admin (Note: the Admin role for the Service account is required to analyze many of the APIs). - - - - - - * Make sure the cluster you configured to run the analysis has ability to read the "service account key json file" by adding the Google service account under the "Advanced Options" of the cluster to the "Google Service Account" value you noted. - - - -
- - - -## Setup option 1 (Simple and recommended method) - - (Estimated time to complete these steps: 15 - 30 mins, varies by number of workspaces in the account) - This method uses admin/service principle credentials (configured in the Step 6 of Prerequisites section) to call workspace APIs. - - Make sure both SAT job cluster (Refer to Prerequisites Step 2 ) and Warehouse (Refer to Prerequisites Step 3) are running. -
- Setup instructions - Following is the one time easy setup to get your workspaces setup with the SAT: - -* Attach \/notebooks/security_analysis_initializer to the SAT cluster you created above and Run -> Run all - - - - - - - -
- -## Setup option 2 (Most flexible for the power users) - - (Estimated time to complete these steps: 30 mins) - This method uses admin credentials (configured in the Step 6 of Prerequisites section) by default to call workspace APIs. But can be changed to use workspace PAT tokens instead. -
- Setup instructions - Following are the one time easy steps to get your workspaces setup with the SAT: - - -1. List account workspaces to analyze with SAT - * Goto \/notebooks/Setup/1.list_account_workspaces_to_conf_file and Run -> Run all - * This creates a configuration file as noted at the bottom of the notebook. - - - - -2. Generate secrets setup file (AWS only. Not recommended for Azure and GCP) - Deprecated as AWS Service principles are encouraged inplace of workspace PATs. - - -3. Test API Connections - * Test connections from your workspace to accounts API calls and all workspace API calls by running \/notebooks/Setup/3. test_connections. The workspaces that didn't pass the connection test are marked in workspace_configs.csv with connection_test as False and are not analyzed. - - - -4. Enable workspaces for SAT analysis - * Enable workspaces by running \/notebooks/Setup/4. enable_workspaces_for_sat. This makes the registered workspaces ready for SAT to monitor - - - -5. Import SAT dashboard template - * We built a ready to go DBSQL dashboard for SAT. Import the dashboard by running \/notebooks/Setup/5. import_dashboard_template - - - -6. Configure Alerts - SAT can deliver alerts via email via Databricks SQL Alerts. Import the alerts template by running \/notebooks/Setup/6. configure_alerts_template (optional) - - - -
- -## Update configuration files - -1. Modify security_best_practices (Optional) - * Go to \/notebooks/Setup/7. update_sat_check_configuration and use this utility to enable/disable a Check, modify Evaluation Value and Alert configuration value for each check. You can update this file any time and any analysis from there on will take these values into consideration. - * [Configure widget settings](https://docs.databricks.com/notebooks/widgets.html#configure-widget-settings-1) behavior "On Widget Change" for this notebooks to "Do Nothing" - - - -2. Modify workspace_configs file (Required for manual checks values) - * **Note**: Limit number of workspaces to be analyzed by SAT to 100. - * **Tip**: You can use this utility to turn on a specific workspace and turn off other workspaces for a specific run. - * **Tip**: You can use this utility to apply your edits to multiple workspaces settings by using "Apply Setting to all workspaces" option. - - * Go to\/notebooks/Setup/8. update_workspace_configuration and You will need to set analysis_enabled as True or False based on if you would like to enroll a workspace to analyze by the SAT. - * [Configure widget settings](https://docs.databricks.com/notebooks/widgets.html#configure-widget-settings-1) behavior "On Widget Change" for this notebooks to "Do Nothing" - - - Update values for each workspace for the manual checks:( sso_enabled,scim_enabled,vpc_peering_done,object_storage_encypted,table_access_control_enabled) - - * sso_enabled : True if you enabled Single Singn-on for the workspace - * scim_enabled: True if you integrated with SCIM for the workspace - * vpc_peering_done: False if you have not peered with another VPC - * object_storage_encypted: True if you encrypted your data buckets - * table_access_control_enabled : True if you enabled ACLs so that you can utilize Table ACL clusters that enforce user isolation - - -## Usage - - (Estimated time to complete these steps: 5 - 10 mins per workspace) - **Note**: Limit number of workspaces to be analyzed by SAT to 100. -1. Attach and run the notebook \/notebooks/security_analysis_driver - Note: This process takes upto 10 mins per workspace - - - - - At this point you should see **SAT** database and tables in your SQL Warehouses: +In DBSQL find "SAT - Security Analysis Tool" dashboard to see the report. You can filter the dashboard by **SAT** tag. - - - - -2. Access Databricks SQL Dashboards section and find "SAT - Security Analysis Tool" dashboard to see the report. You can filter the dashboard by **SAT** tag. - - **Note:** You need to select the workspace and date and click "Apply Changes" to get the report. - **Note:** The dashbord shows last valid run for the selected date if there is one, if not it shows the latest report for that workspace. + > **Note:** You need to select the workspace and date and click "Apply Changes" to get the report. + + > **Note:** The dashbord shows last valid run for the selected date if there is one, if not it shows the latest report for that workspace. - You can share SAT dashboard with other members of your team by using the "Share" functionality on the top right corner of the dashboard. - - - Here is what your SAT Dashboard should look like: +You can share SAT dashboard with other members of your team by using the "Share" functionality on the top right corner of the dashboard. + +Here is what your SAT Dashboard should look like: -3. Activate Alerts - * Goto Alerts and find the alert(s) created by SAT tag and adjust the schedule to your needs. You can add more recpients to alerts by configuring [notification destinations](https://docs.databricks.com/sql/admin/notification-destinations.html). +### 3. Activate Alerts +Go to Alerts and find the alert(s) created by SAT tag and adjust the schedule to your needs. You can add more recipients to alerts by configuring [notification destinations](https://docs.databricks.com/sql/admin/notification-destinations.html). - - - - - - - -## Configure Workflow (Optional) - - (Estimated time to complete these steps: 5 mins) + - * Databricks Workflows is the fully-managed orchestration service. You can configure SAT to automate when and how you would like to schedule it by using by taking advantage of Workflows. - * Goto Workflows - > click on create jobs -> setup as following: + - Task Name : security_analysis_driver +### 4. Update configuration files (Optional) - Type: Notebook +#### 1. Modify security_best_practices (Optional) - Source: Workspace (or your git clone of SAT) +- Go to Workspace -> Applications -> SAT -> files -> notebooks /Setup/7. update_sat_check_configuration and use this utility to enable/disable a Check, modify Evaluation Value and Alert configuration value for each check. You can update this file any time and any analysis from there on will take these values into consideration. - Path : \/SAT/SecurityAnalysisTool-BranchV2Root/notebooks/security_analysis_driver - - Cluster: Make sure to pick the Single user mode job compute cluster you created before. - - - - Add a schedule as per your needs. That’s it. Now you are continuously monitoring the health of your account workspaces. - - -## FAQs - 1. How can SAT be configured if access to github is not possible due to firewall restrictions to git or other organization policies? - - You can still setup SAT by downloading the [release zip](https://github.com/databricks-industry-solutions/security-analysis-tool/releases) file and by using Git repo to load SAT project into your workspace. - * Add Repo by going to Repos in your workspace: - - - - * Type SAT as your "Repository name" and uncheck "Create repo by cloning a Git repository" - - +- [Configure widget settings](https://docs.databricks.com/notebooks/widgets.html#configure-widget-settings-1) behavior "On Widget Change" for this notebooks to "Do Nothing" + + - * Click on the pulldown menu and click on Import - - +#### 2. Modify workspace_configs file (Required for manual checks values) - * Drag and drop the release zip file and click Import - - - - - - You should see the SAT project in your workspace. -2. Can SAT make modifications to my workspaces and account? - - - No. SAT is meant to be a readonly analysis tool, it does not make changes to your workspace or account configurations. - -3. I added a new workspace for analysis, re-ran steps under initilaize and driver, ... the dashboard is not updated with the new workspace in the pulldown even though I see new data generated by the analysis scan for the new workspace in SAT database. What should I do? - - It is likely that the Dashboard cached the workspaces in the pulldown. You can go to SQL view of your workspace -> Queries -> find workspace_ids query and run it, that should refresh and have the new workspaces in the pull-down. - - -## Troubleshooting -We created diagnosis notebooks for respective clouds to help troubleshoot your SAT setup. Please review notebooks/diagnosis/ folder. +- **Tip**: You can use this utility to turn on a specific workspace and turn off other workspaces for a specific run. -1. Incorrectly configured secrets - * Error: - - Secret does not exist with scope: sat_scope and key: sat_tokens +- **Tip**: You can use this utility to apply your edits to multiple workspaces settings by using "Apply Setting to all workspaces" option. - * Resolution: - Check if the tokens are configured with the correct names by listing and comparing with the configuration. - databricks --profile e2-sat secrets list-secrets sat_scope +- Go to Workspace -> Applications -> SAT -> files -> notebooks/Setup/8. update_workspace_configuration and You will need to set analysis_enabled as True or False based on if you would like to enroll a workspace to analyze by the SAT. -2. Invalid access token - - * Error: - - Error 403 Invalid access token. +- [Configure widget settings](https://docs.databricks.com/notebooks/widgets.html#configure-widget-settings-1) behavior "On Widget Change" for this notebooks to "Do Nothing" - * Resolution: - - Check your PAT token configuration for “workspace_pat_token” key +Update values for each workspace for the manual checks -3. Firewall blocking databricks accounts console +- sso_enabled : True if you enabled Single Singn-on for the workspace +- scim_enabled: True if you integrated with SCIM for the workspace +- vpc_peering_done: False if you have not peered with another VPC +- object_storage_encypted: True if you encrypted your data buckets +- table_access_control_enabled : True if you enabled ACLs so that you can utilize Table ACL clusters that enforce user isolation - * Error: -

- Traceback (most recent call last): File "/databricks/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 670, in urlopen httplib_response = self._make_request( File "/databricks/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 381, in _make_request self._validate_conn(conn) File "/databricks/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 978, in _validate_conn conn.connect() File "/databricks/python/lib/python3.8/site-packages/urllib3/connection.py", line 362, in connect self.sock = ssl_wrap_socket( File "/databricks/python/lib/python3.8/site-packages/urllib3/util/ssl_.py", line 386, in ssl_wrap_socket return context.wrap_socket(sock, server_hostname=server_hostname) File "/usr/lib/python3.8/ssl.py", line 500, in wrap_socket return self.sslsocket_class._create( File "/usr/lib/python3.8/ssl.py", line 1040, in _create self.do_handshake() File "/usr/lib/python3.8/ssl.py", line 1309, in do_handshake self._sslobj.do_handshake() ConnectionResetError: [Errno 104] Connection reset by peer During handling of the above exception, another exception occurred: + - * Resolution: - - Run this following command in your notebook %sh - curl -X GET -H "Authorization: Basic /" -H "Content-Type: application/json" https://accounts.cloud.databricks.com/api/2.0/accounts//workspaces - or - - %sh curl -u 'user:password' -X GET  “Content-Type: application/json” https://accounts.cloud.databricks.com/api/2.0/accounts//workspaces - - If you don’t see a JSON with a clean listing of workspaces you are likely having a firewall issue that is blocking calls to the accounts console. Please have your infrastructure team add Databricks accounts.cloud.databricks.com to the allow-list. Please see that the private IPv4 address from NAT gateway to the IP allow list. +## Uninstall SAT -4. Offline install of libraries incase of no PyPI access +### Standard Setup - Download the dbl_sat_sdk version specified in the notebook notebooks/utils/initialize from PyPi - https://pypi.org/project/dbl-sat-sdk/ - Upload the dbl_sat_sdk-w.x.y-py3-none-any.whl to a dbfs location. You can use the databricks-cli as one mechanism to upload. - for e.g. +Steps: +- Delete `/Workspaces/SAT` folder +- Delete Workflows `SAT Initializer Notebook` and `SAT Driver Notebook` +- Delete the Dashboards - ``` - databricks --profile e2-satfs cp /localdrive/whlfile/dbl_sat_sdk-w.x.y-py3-none-any.whl dbfs:/FileStore/wheels/ - ``` - Additionally download the following wheel files and upload them to the dbfs location as above. - https://github.com/databricks-industry-solutions/security-analysis-tool/tree/main/docs/wheels - - Upload all wheel files to /FileStore/wheels in your workspace - Verify all files are there by doing a %fs ls /FileStore/wheels from your notebook. - Then change the cell in your notebook install_sat_sdk to this - - ``` - %pip install cachetools --find-links /dbfs/FileStore/wheels/cachetools-5.3.1-py3-none-any.whl - %pip install pyasn1 --find-links /dbfs/FileStore/wheels/pyasn1-0.5.0-py2.py3-none-any.whl - %pip install pyasn1-modules --find-links /dbfs/FileStore/wheels/pyasn1_modules-0.3.0-py2.py3-none-any.whl - %pip install rsa --find-links /dbfs/FileStore/wheels/rsa-4.9-py3-none-any.whl - %pip install google-auth --find-links /dbfs/FileStore/wheels/google_auth-2.22.0-py2.py3-none-any.whl - %pip install PyJWT[crypto] --find-links /dbfs/FileStore/wheels/PyJWT-2.8.0-py3-none-any.whl - %pip install msal --find-links /dbfs/FileStore/wheels/msal-1.22.0-py2.py3-none-any.whl - %pip install dbl-sat-sdk==0.1.32 --find-links /dbfs/FileStore/wheels/dbl_sat_sdk-0.1.32-py3-none-any.whl - ``` - - Make sure the versions for the above libraries match. +### Terraform + +Uninstalling using `terraform destroy` diff --git a/docs/setup/aws.md b/docs/setup/aws.md new file mode 100644 index 00000000..d841a231 --- /dev/null +++ b/docs/setup/aws.md @@ -0,0 +1,71 @@ +# AWS Setup Guide + +This guide will help you setup the Security Analysis Tool (SAT) on AWS Databricks. + +- [AWS Setup Guide](#aws-setup-guide) + - [Prerequisites](#prerequisites) + - [Service Principal](#service-principal) + - [Installation](#installation) + - [Credentials Needed](#credentials-needed) + - [Troubleshooting](#troubleshooting) + +## Prerequisites + +There are some prerequisites that need to be met before you can set up SAT on AWS. Make sure you have the appropriate permissions in your Databricks Account Console to create the resources mentioned below. + +> SAT is beneficial to customers on **Databrics Premium or Enterprise** as most of the checks and recommendations involve security features available in tiers higher than the Standard. + +### Service Principal + +The first step is to create a Service Principal in Databricks. This will allow SAT to authenticate with the other workspaces. Follow the steps: + +- Go to the [Account Console](https://accounts.cloud.databricks.com) +- On the left side bar menu, click on `User management` +- Select `Service Principal` and then `Add service principal` +- The Service Principal must be granted the `Account Admin` role. This role provides the ability to manage account-level settings and permissions. +- Assign the Workspace Admin Role: The Service Principal must be assigned the `Workspace Admin` role for each workspace it will manage. This role provides the ability to manage workspace-level settings and permissions. +- Add to the Metastore Admin Group: The Service Principal must be added to the `Metastore Admin` group or role. This role provides the ability to manage metastore-level settings and permissions. +- Type a new name for the service principal and then create a new OAuth Secret. +- Save the `Secret` and `Client ID` +- To deploy SAT in a workspace, you must add the Service Principal to the workspace. + +![AWS_SP_Workspace](../images/aws_ws.png) + +> The Service Principle requires an [Accounts Admin role](https://docs.databricks.com/en/admin/users-groups/service-principals.html#assign-account-admin-roles-to-a-service-principal), [Admin role](https://docs.databricks.com/en/admin/users-groups/service-principals.html#assign-a-service-principal-to-a-workspace-using-the-account-console) for **each workspace** and needs to be a member of the [metastore admin group](https://docs.databricks.com/en/data-governance/unity-catalog/manage-privileges/admin-privileges.html#who-has-metastore-admin-privileges) is required to analyze many of the APIs + +## Installation + +### Credentials Needed + +To setup SAT on AWS, you will need the following credentials: +- Databricks Account ID +- Databricks Service Principal ID +- Databricks Service Principal Secret + +To execute SAT follow this steps: + +- Clone the SAT repository locally + + ```sh + git clone https://github.com/databricks-industry-solutions/security-analysis-tool.git + ``` + +- Run the `install.sh` script on your terminal. + +> To ensure that the install.sh script is executable, you need to modify its permissions using the chmod command. + ```sh + chmod +x install.sh + ./install.sh + ``` + + +![](../gif/terminal-aws.gif) + +> Remember that the target workspace should have a [profile](https://docs.databricks.com/en/dev-tools/cli/profiles.html) in [Databricks CLI](https://docs.databricks.com/en/dev-tools/cli/index.html) + +Congratulations! 🎉 You are now ready to start using SAT. Please click [here](../setup.md#usage) for a detailed description on how to run and use it. + +## Troubleshooting + +Please review the FAQs and Troubleshooting resources documented [here](./faqs_and_troubleshooting.md) including a notebook to help diagnose your SAT setup. +If any issues arise during the installation process, please check your credentials and ensure that you have the appropriate configurations and permissions for your Databricks. If you are still facing issues, please send your feedback and comments to . diff --git a/docs/setup/azure.md b/docs/setup/azure.md new file mode 100644 index 00000000..2f6e198c --- /dev/null +++ b/docs/setup/azure.md @@ -0,0 +1,103 @@ +# Azure Setup Guide + +This guide will help you setup the Security Analysis Tool (SAT) on Azure Databricks. + + +- [Azure Setup Guide](#azure-setup-guide) + - [Prerequisites](#prerequisites) + - [App Registration](#app-registration) + - [App Client Secrets](#app-client-secrets) + - [Add Service Principle to Databricks](#add-service-principle-to-databricks) + - [Installation](#installation) + - [Credentials Needed](#credentials-needed) + - [Troubleshooting](#troubleshooting) + - [References](#references) + +## Prerequisites + +There are some pre-requisites that need to be met before you can setup SAT on Azure. Make sure you have the appropriate permissions in your Azure cloud account to create the resources mentioned below. + +> SAT is beneficial to customers on **Databrics Premium or Enterprise** as most of the checks and recommendations involve security features available in tiers higher than the Standard. + + +### App Registration + +The first step is to create an App Registration in Azure. This will allow SAT to authenticate with Azure services. Follow the steps below to create an App Registration: + +- Open the Azure portal and navigate to Microsoft Entra ID. +- Click on `App registrations` and then click on `New registration`. +- Enter a name for the App Registration and select the appropriate permissions. The minimum requirement is to have access in a single tenant. + +![alt text](../images/azure_app_reg.png) + +### App Client Secrets + +After creating the App Registration, you will need to create a client secret. This secret will be used to authenticate with Azure services. Follow the steps below to create a client secret: + +- Open the App Registration you created in the previous step. +- Click on `Certificates & secrets` and then click on `New client secret`. +- Enter a description for the client secret and select the expiry date. Click on `Add`. +- Copy the value of the client secret and save it in a secure location. + +### Add Service Principle to Databricks + +After creating the App Registration and client secret, you will need to add the App Registration as a service principal in Databricks. Follow the steps below to add the service principal: + +- Go to the [Account Console](https://accounts.azuredatabricks.net/) +- On the left side bar menu, click on `User management` +- Select `Service Principal` and then `Add service principal` +- Paste the App Client ID and App Client Secret in the respective fields. +- Click Add. +- The Service Principal must be granted the `Account Admin` role. This role provides the ability to manage account-level settings and permissions. +- Assign the Workspace Admin Role: The Service Principal must be assigned the `Workspace Admin` role for each workspace it will manage. This role provides the ability to manage workspace-level settings and permissions. +- Add to the Metastore Admin Group: The Service Principal must be added to the `Metastore Admin` group or role. This role provides the ability to manage metastore-level settings and permissions. + +![Azure_SP_Workspace](../images/azure_ws.png) + +See the [Databricks documentation](https://learn.microsoft.com/en-us/azure/databricks/admin/users-groups/service-principals#--databricks-and-microsoft-entra-id-formerly-azure-active-directory-service-principals) for more information on adding service principals. + +> The Service Principle requires an [Accounts Admin role](https://learn.microsoft.com/en-us/azure/databricks/admin/users-groups/service-principals#--assign-account-admin-roles-to-a-service-principal), [Admin role](https://learn.microsoft.com/en-us/azure/databricks/admin/users-groups/service-principals#assign-a-service-principal-to-a-workspace-using-the-account-console) for **each workspace** and needs to be a member of the [metastore admin group](https://learn.microsoft.com/en-us/azure/databricks/data-governance/unity-catalog/manage-privileges/admin-privileges#who-has-metastore-admin-privileges) is required to analyze many of the APIs. + +## Installation + +### Credentials Needed + +To setup SAT on Azure, you will need the following credentials: + +- Databricks Account ID +- Azure Tenant ID +- Azure Subscription ID +- Azure App Client ID (Obtained from App Registration) +- Azure App Client Secret (Obtained from App Client Secrets) + +To execute SAT follow this steps: + +- Clone the SAT repository locally + + ```sh + git clone https://github.com/databricks-industry-solutions/security-analysis-tool.git + ``` + +- Run the `install.sh` script on your terminal. + +> To ensure that the install.sh script is executable, you need to modify its permissions using the chmod command. + ```sh + chmod +x install.sh + ./install.sh + ``` + +![](../gif/terminal-azure.gif) + +> Remember the target workspace should have a [profile](https://learn.microsoft.com/en-us/azure/databricks/dev-tools/cli/profiles) in [Databricks CLI](https://learn.microsoft.com/en-us/azure/databricks/dev-tools/cli/) + +Congratulations! 🎉 You are now ready to start using SAT. Please click [here](../setup.md#usage) for a detailed description on how to run and use it. + + +## Troubleshooting +Please review the FAQs and Troubleshooting resources documented [here](./faqs_and_troubleshooting.md) including a notebook to help diagnose your SAT setup. +If any issues arise during the installation process, please check your credentials and ensure that you have the appropriate permissions in your Azure cloud account. If you are still facing issues, please send your feedback and comments to sat@databricks.com. + +## References + +- [Azure App Registration](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app) +- [Databricks Service Principals](https://learn.microsoft.com/en-us/azure/databricks/admin/users-groups/service-principals#--databricks-and-microsoft-entra-id-formerly-azure-active-directory-service-principals) diff --git a/docs/setup/faqs_and_troubleshooting.md b/docs/setup/faqs_and_troubleshooting.md new file mode 100644 index 00000000..833bffdf --- /dev/null +++ b/docs/setup/faqs_and_troubleshooting.md @@ -0,0 +1,136 @@ +# FAQs + +1. **Can SAT make modifications to my workspaces and account?** + + No. SAT is meant to be a readonly analysis tool, it does not make changes to your workspace or account configurations. + +2. **I added a new workspace for analysis, re-ran steps under initialize and driver, but the dashboard is not updated with the new workspace in the pulldown even though I see new data generated by the analysis scan for the new workspace in SAT database. What should I do?** + + It is likely that the Dashboard cached the workspaces in the pulldown. You can go to SQL view of your workspace -> Queries -> find `workspace_ids` query and run it, that should refresh and have the new workspaces in the pull-down. + +## Troubleshooting + +We created diagnosis notebooks for respective clouds to help troubleshoot your SAT setup. Please go to Workspace -> Applications -> SAT -> files -> notebooks -> diagnosis/ folder and run the respective cloud `sat_diagnosis_` notebook. + +* [SAT AWS troubleshooting notebook](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/notebooks/diagnosis/sat_diagnosis_aws.py) +* [SAT Azure troubleshooting notebook](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/notebooks/diagnosis/sat_diagnosis_azure.py) +* [SAT GCP troubleshooting notebook](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/notebooks/diagnosis/sat_diagnosis_gcp.py) + +### 1. Incorrectly configured secrets + +* **Error:** + + ``` + Secret does not exist with scope: sat_scope and key: sat_tokens + ``` + +* **Resolution:** + + Check if the tokens are configured with the correct names by listing and comparing with the configuration. + ``` + databricks --profile e2-sat secrets list-secrets sat_scope + ``` + +### 2. Invalid access token + +* **Error:** + + ``` + Error 403 Invalid access token. + ``` + +* **Resolution:** + + Check your PAT token configuration for the `workspace_pat_token` key. + +### 3. Firewall blocking Databricks accounts console + +* **Error:** + + ``` + Traceback (most recent call last): + File "/databricks/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 670, in urlopen + httplib_response = self._make_request( + File "/databricks/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 381, in _make_request + self._validate_conn(conn) + File "/databricks/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 978, in _validate_conn + conn.connect() + File "/databricks/python/lib/python3.8/site-packages/urllib3/connection.py", line 362, in connect + self.sock = ssl_wrap_socket( + File "/databricks/python/lib/python3.8/site-packages/urllib3/util/ssl_.py", line 386, in ssl_wrap_socket + return context.wrap_socket(sock, server_hostname=server_hostname) + File "/usr/lib/python3.8/ssl.py", line 500, in wrap_socket + return self.sslsocket_class._create( + File "/usr/lib/python3.8/ssl.py", line 1040, in _create + self.do_handshake() + File "/usr/lib/python3.8/ssl.py", line 1309, in do_handshake + self._sslobj.do_handshake() + ConnectionResetError: [Errno 104] Connection reset by peer + During handling of the above exception, another exception occurred: + ``` + +* **Resolution:** + + Run this following command in your notebook: + ``` + %sh + curl -X GET -H "Authorization: Basic /" -H "Content-Type: application/json" https://accounts.cloud.databricks.com/api/2.0/accounts//workspaces + ``` + + or + + ``` + %sh + curl -u 'user:password' -X GET "Content-Type: application/json" https://accounts.cloud.databricks.com/api/2.0/accounts//workspaces + ``` + + If you don’t see a JSON with a clean listing of workspaces, you are likely having a firewall issue that is blocking calls to the accounts console. Please have your infrastructure team add `accounts.cloud.databricks.com` to the allow-list. Ensure that the private IPv4 address from the NAT gateway is added to the IP allow list. + +### 4. Offline install of libraries in case of no PyPI access + +* **Steps:** + + 1. Download the `dbl_sat_sdk` version specified in the notebook `notebooks/utils/initialize` from PyPi: + ``` + https://pypi.org/project/dbl-sat-sdk/ + ``` + + 2. Upload the `dbl_sat_sdk-w.x.y-py3-none-any.whl` to a `dbfs` location. You can use the `databricks-cli` as one mechanism to upload. For example: + ``` + databricks --profile e2-satfs cp /localdrive/whlfile/dbl_sat_sdk-w.x.y-py3-none-any.whl dbfs:/FileStore/wheels/ + ``` + + 3. Additionally, download the following wheel files and upload them to the `dbfs` location as above: + ``` + https://github.com/databricks-industry-solutions/security-analysis-tool/tree/main/docs/wheels + ``` + + 4. Upload all wheel files to `/FileStore/wheels` in your workspace. + + 5. Verify all files are there by running: + + ``` + %fs ls /FileStore/wheels + ``` + + 6. Then change the cell in your notebook `install_sat_sdk` to this: + + ```python + %pip install cachetools --find-links /dbfs/FileStore/wheels/cachetools-5.3.1-py3-none-any.whl + + %pip install pyasn1 --find-links /dbfs/FileStore/wheels/pyasn1-0.5.0-py2.py3-none-any.whl + + %pip install pyasn1-modules --find-links /dbfs/FileStore/wheels/pyasn1_modules-0.3.0-py2.py3-none-any.whl + + %pip install rsa --find-links /dbfs/FileStore/wheels/rsa-4.9-py3-none-any.whl + + %pip install google-auth --find-links /dbfs/FileStore/wheels/google_auth-2.22.0-py2.py3-none-any.whl + + %pip install PyJWT[crypto] --find-links /dbfs/FileStore/wheels/PyJWT-2.8.0-py3-none-any.whl + + %pip install msal --find-links /dbfs/FileStore/wheels/msal-1.22.0-py2.py3-none-any.whl + + %pip install dbl-sat-sdk==0.1.32 --find-links /dbfs/FileStore/wheels/dbl_sat_sdk-0.1.32-py3-none-any.whl + ``` + + 7. Make sure the versions for the above libraries match. diff --git a/docs/setup/gcp.md b/docs/setup/gcp.md new file mode 100644 index 00000000..91c29aac --- /dev/null +++ b/docs/setup/gcp.md @@ -0,0 +1,69 @@ +# GCP Setup Guide + +This guide will help you setup the Security Analysis Tool (SAT) on GCP Databricks. + +- [GCP Setup Guide](#gcp-setup-guide) + - [Prerequisites](#prerequisites) + - [Service Accounts](#service-accounts) + - [Installation](#installation) + - [Credentials Needed](#credentials-needed) + - [Troubleshooting](#troubleshooting) + +## Prerequisites + +There are some pre-requisites that need to be met before you can setup SAT on GCP. Make sure you have the appropriate permissions in your GCP Cloud account to create the resources mentioned below. + +> SAT is beneficial to customers on **Databrics Premium or Enterprise** as most of the checks and recommendations involve security features available in tiers higher than the Standard. + +### Service Accounts + +The first step is to create a Service Principal in GCP. This will allow SAT to authenticate with GCP services. Follow the steps below to create a Service Principal: + +- Please follow [this](https://docs.gcp.databricks.com/en/dev-tools/authentication-google-id.html) guide to create the required service accounts. +- Now upload the SA-1.json file into a GCS bucket. +- To add the service account to the Account Console: + - You will need to create a new user and add the service account email as the user email. +- The Service Principal must be granted the `Account Admin` role. This role provides the ability to manage account-level settings and permissions. +- Assign the Workspace Admin Role: The Service Principal must be assigned the `Workspace Admin` role for each workspace it will manage. This role provides the ability to manage workspace-level settings and permissions. +- Add to the Metastore Admin Group: The Service Principal must be added to the `Metastore Admin` group or role. This role provides the ability to manage metastore-level settings and permissions. + +## Installation + +### Credentials Needed + +To setup SAT on Azure, you will need the following credentials: +- Databricks Account ID +- Service Account email +- gsutil URI from GCS Bucket +- ![alt text](../images/gs_path_to_json.png) + +To execute SAT follow this steps: + +- Clone the SAT repository locally + + ```sh + git clone https://github.com/databricks-industry-solutions/security-analysis-tool.git + ``` + +> To ensure that the install.sh script is executable, you need to modify its permissions using the chmod command. + +> For linux or mac users: +> ```sh +> chmod +x install.sh +>``` + + +- Run the `install.sh` script on your terminal. + + + +![](../gif/terminal-gcp.gif) + +> Remember that the target workspace should have a [profile](https://docs.gcp.databricks.com/en/dev-tools/cli/profiles.html) in [Databricks CLI](https://docs.gcp.databricks.com/en/dev-tools/cli/tutorial.html) + +Congratulations! 🎉 You are now ready to start using SAT. Please click [here](../setup.md#usage) for a detailed description on how to run and use it. + + +## Troubleshooting +Please review the FAQs and Troubleshooting resources documented [here](./faqs_and_troubleshooting.md) including a notebook to help diagnose your SAT setup. +If any issues arise during the installation process, please check your credentials and ensure that you have the appropriate permissions in your Azure cloud account. If you are still facing issues, please send your feedback and comments to . diff --git a/install.sh b/install.sh new file mode 100755 index 00000000..7f08f02b --- /dev/null +++ b/install.sh @@ -0,0 +1,21 @@ +#/bin/bash + +folder="dabs" + +version=$(python -c "import sys; print(sys.version_info[:])" 2>&1) +if [[ -z "$version" ]]; then + echo "Python not found" + exit 1 +fi + +major=$(echo $version | cut -d ',' -f 1 | tr -d '(') +minor=$(echo $version | cut -d ',' -f 2) + +if [[ $major -lt 3 || $minor -lt 9 ]]; then + echo "Python 3.9 or higher is required" + exit 1 +fi + +cd $folder +pip install -r requirements.txt +python main.py \ No newline at end of file diff --git a/notebooks/Setup/1. list_account_workspaces_to_conf_file.py b/notebooks/Setup/1. list_account_workspaces_to_conf_file.py index dcb75475..96c09a50 100644 --- a/notebooks/Setup/1. list_account_workspaces_to_conf_file.py +++ b/notebooks/Setup/1. list_account_workspaces_to_conf_file.py @@ -1,6 +1,6 @@ # Databricks notebook source # MAGIC %md -# MAGIC **Notebook name:** 1. list_account_workspaces_to_conf_file +# MAGIC **Notebook name:** 1. list_account_workspaces_to_conf_file # MAGIC **Functionality:** generates all the workspaces in the account (subscription in case of azure) and writes into a config file # COMMAND ---------- @@ -17,82 +17,119 @@ # COMMAND ---------- -#replace values for accounts exec -hostname = dbutils.notebook.entry_point.getDbutils().notebook().getContext().apiUrl().getOrElse(None) +# replace values for accounts exec +hostname = ( + dbutils.notebook.entry_point.getDbutils() + .notebook() + .getContext() + .apiUrl() + .getOrElse(None) +) cloud_type = getCloudType(hostname) clusterid = spark.conf.get("spark.databricks.clusterUsageTags.clusterId") -#dont know workspace token yet. -json_.update({'url':hostname, 'workspace_id': 'accounts', 'cloud_type': cloud_type, 'clusterid':clusterid}) +# dont know workspace token yet. +json_.update( + { + "url": hostname, + "workspace_id": "accounts", + "cloud_type": cloud_type, + "clusterid": clusterid, + } +) # COMMAND ---------- from core.logging_utils import LoggingUtils -LoggingUtils.set_logger_level(LoggingUtils.get_log_level(json_['verbosity'])) + +LoggingUtils.set_logger_level(LoggingUtils.get_log_level(json_["verbosity"])) loggr = LoggingUtils.get_logger() # COMMAND ---------- import json -dbutils.notebook.run('../Utils/accounts_bootstrap', 300, {"json_":json.dumps(json_)}) + +dbutils.notebook.run( + f"{basePath()}/notebooks/Utils/accounts_bootstrap", + 300, + {"json_": json.dumps(json_)}, +) # COMMAND ---------- -#this logic does not overwrite the previous config file. It just appends new lines so users can -#easily modify the new lines for new workspaces. + +# this logic does not overwrite the previous config file. It just appends new lines so users can +# easily modify the new lines for new workspaces. def generateWorkspaceConfigFile(workspace_prefix): - from pyspark.sql.functions import lit,concat,col - dfexist = readWorkspaceConfigFile() - excluded_configured_workspace = '' - header_value = True - if dfexist is not None: - dfexist.createOrReplaceTempView('configured_workspaces') - excluded_configured_workspace = ' AND workspace_id not in (select workspace_id from `configured_workspaces`)' - #don't append header to an existing file - header_value = False - else: - excluded_configured_workspace = '' #running first time - #get current workspaces that are not yet configured for analysis - spsql = f'''select workspace_id, deployment_name as deployment_url, workspace_name, workspace_status from `global_temp`.`acctworkspaces` - where workspace_status = "RUNNING" {excluded_configured_workspace}''' - df = spark.sql(spsql) - if(not df.rdd.isEmpty()): - if(cloud_type == 'azure'): - df = df.withColumn("deployment_url", concat(col('deployment_url'), lit('.azuredatabricks.net'))) #Azure - elif (cloud_type =='aws'): - df = df.withColumn("deployment_url", concat(col('deployment_url'), lit('.cloud.databricks.com'))) #AWS + from pyspark.sql.functions import col, concat, lit + + dfexist = readWorkspaceConfigFile() + excluded_configured_workspace = "" + header_value = True + if dfexist is not None: + dfexist.createOrReplaceTempView("configured_workspaces") + excluded_configured_workspace = " AND workspace_id not in (select workspace_id from `configured_workspaces`)" + # don't append header to an existing file + header_value = False else: - df = df.withColumn("deployment_url", concat(col('deployment_url'), lit('.gcp.databricks.com'))) #GCP - - df = df.withColumn("ws_token", concat(lit(workspace_prefix), lit('-'), col('workspace_id'))) #added with workspace prfeix - - if(cloud_type == 'azure'): - df = df.withColumn("sso_enabled", lit(True)) + excluded_configured_workspace = "" # running first time + # get current workspaces that are not yet configured for analysis + spsql = f"""select workspace_id, deployment_name as deployment_url, workspace_name, workspace_status from `global_temp`.`acctworkspaces` + where workspace_status = "RUNNING" {excluded_configured_workspace}""" + df = spark.sql(spsql) + if not df.rdd.isEmpty(): + if cloud_type == "azure": + df = df.withColumn( + "deployment_url", + concat(col("deployment_url"), lit(".azuredatabricks.net")), + ) # Azure + elif cloud_type == "aws": + df = df.withColumn( + "deployment_url", + concat(col("deployment_url"), lit(".cloud.databricks.com")), + ) # AWS + else: + df = df.withColumn( + "deployment_url", + concat(col("deployment_url"), lit(".gcp.databricks.com")), + ) # GCP + + df = df.withColumn( + "ws_token", concat(lit(workspace_prefix), lit("-"), col("workspace_id")) + ) # added with workspace prfeix + #both azure and gcp require sso + if cloud_type == "azure" or cloud_type == "gcp" : + df = df.withColumn("sso_enabled", lit(True)) + else: + df = df.withColumn("sso_enabled", lit(False)) + df = df.withColumn("scim_enabled", lit(False)) + df = df.withColumn("vpc_peering_done", lit(False)) + df = df.withColumn("object_storage_encrypted", lit(True)) + df = df.withColumn("table_access_control_enabled", lit(False)) + df = df.withColumn("connection_test", lit(False)) + df = df.withColumn("analysis_enabled", lit(True)) + + loggr.info("Appending following workspaces to configurations ...") + display(df) + prefix = getConfigPath() + df.toPandas().to_csv( + f"{prefix}/workspace_configs.csv", + mode="a+", + index=False, + header=header_value, + ) # Databricks Runtime 11.2 or above. else: - df = df.withColumn("sso_enabled", lit(False)) - df = df.withColumn("scim_enabled", lit(False)) - df = df.withColumn("vpc_peering_done", lit(False)) - df = df.withColumn("object_storage_encrypted", lit(True)) - df = df.withColumn("table_access_control_enabled", lit(False)) - df = df.withColumn("connection_test", lit(False)) - df = df.withColumn("analysis_enabled", lit(True)) - - loggr.info('Appending following workspaces to configurations ...') - display(df) - prefix = getConfigPath() - df.toPandas().to_csv(f'{prefix}/workspace_configs.csv', mode='a+', index=False, header=header_value) #Databricks Runtime 11.2 or above. - else: - loggr.info('No new workspaces found for appending into configurations') + loggr.info("No new workspaces found for appending into configurations") # COMMAND ---------- -generateWorkspaceConfigFile(json_['workspace_pat_token_prefix']) -dbutils.notebook.exit('OK') +generateWorkspaceConfigFile(json_["workspace_pat_token_prefix"]) +dbutils.notebook.exit("OK") # COMMAND ---------- -# MAGIC %md +# MAGIC %md # MAGIC #### Look in the Configs folder for generated Files # MAGIC * ##### Modify workspace_configs.csv. Update the analysis_enabled flag and verify sso_enabled,scim_enabled,vpc_peering_done,object_storage_encrypted,table_access_control_enabled for each workspace. # MAGIC * ##### New workspaces will be added to end of the file diff --git a/notebooks/Setup/5. import_dashboard_template_lakeview.py b/notebooks/Setup/5. import_dashboard_template_lakeview.py new file mode 100644 index 00000000..b22dde88 --- /dev/null +++ b/notebooks/Setup/5. import_dashboard_template_lakeview.py @@ -0,0 +1,216 @@ +# Databricks notebook source +# MAGIC %md +# MAGIC **Notebook name:** 5. import_dashboard_template_lakeview. +# MAGIC **Functionality:** Imports dashboard template from code repo into Lakeview Dashboards section for SAT report. + +# COMMAND ---------- + +# MAGIC %run ../Includes/install_sat_sdk + +# COMMAND ---------- + +# MAGIC %run ../Utils/initialize + +# COMMAND ---------- + +# MAGIC %run ../Utils/common + +# COMMAND ---------- + +from core.logging_utils import LoggingUtils +LoggingUtils.set_logger_level(LoggingUtils.get_log_level(json_['verbosity'])) +loggr = LoggingUtils.get_logger() + +# COMMAND ---------- + +dfexist = readWorkspaceConfigFile() +dfexist.filter((dfexist.analysis_enabled==True) & (dfexist.connection_test==True)).createOrReplaceGlobalTempView('all_workspaces') + +# COMMAND ---------- + +import json +context = json.loads(dbutils.notebook.entry_point.getDbutils().notebook().getContext().toJson()) +current_workspace = context['tags']['orgId'] + +# COMMAND ---------- + +workspacedf = spark.sql("select * from `global_temp`.`all_workspaces` where workspace_id='" + current_workspace + "'" ) +if (workspacedf.rdd.isEmpty()): + dbutils.notebook.exit("The current workspace is not found in configured list of workspaces for analysis.") +display(workspacedf) +ws = (workspacedf.collect())[0] + +# COMMAND ---------- + +import requests +DOMAIN = ws.deployment_url +TOKEN = dbutils.secrets.get(json_['workspace_pat_scope'], ws.ws_token) +loggr.info(f"Looking for data_source_id for : {json_['sql_warehouse_id']}!") +response = requests.get( + 'https://%s/api/2.0/preview/sql/data_sources' % (DOMAIN), + headers={'Authorization': 'Bearer %s' % TOKEN}, + json=None, + timeout=60 + ) +if '\"error_code\":\"403\"' not in response.text: + resources = json.loads(response.text) + found = False + for resource in resources: + if resource['endpoint_id'] == json_['sql_warehouse_id']: + data_source_id = resource['id'] + loggr.info(f"Found data_source_id for : {json_['sql_warehouse_id']}!") + found = True + break + if (found == False): + dbutils.notebook.exit("The configured SQL Warehouse Endpoint is not found.") +else: + dbutils.notebook.exit("Invalid access token, check PAT configuration value for this workspace.") + + +# COMMAND ---------- + +# MAGIC %md +# MAGIC # Modify json file with the selected catalog + +# COMMAND ---------- + + +# Path to the JSON file +file_path = f'{basePath()}/dashboards/SAT_Dashboard_definition.json' + +# String to search and replace +old_string = 'hive_metastore.security_analysis' +new_string = json_['analysis_schema_name'] + +# Read the JSON file +with open(file_path, 'r') as file: + data = json.load(file) + +# Modify the JSON by replacing the string +# Traverse the JSON object and replace the string when found +def replace_string(obj, old_str, new_str): + if isinstance(obj, dict): + for key in obj: + if isinstance(obj[key], dict) or isinstance(obj[key], list): + replace_string(obj[key], old_str, new_str) + elif isinstance(obj[key], str): + obj[key] = obj[key].replace(old_str, new_str) + elif isinstance(obj, list): + for item in obj: + replace_string(item, old_str, new_str) + +if json_['analysis_schema_name'] != 'hive_metastore.security_analysis': + replace_string(data, old_string, new_string) + + # Write the updated JSON back to the file + with open(file_path, 'w') as file: + json.dump(data, file, indent=4) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC # Delete previously created Dashboard + +# COMMAND ---------- + +import requests + +BODY = {'path': f'{basePath()}/dashboards/SAT - Security Analysis Tool (Lakeview).lvdash.json'} + +loggr.info(f"Getting Dashboard") +response = requests.get( + 'https://%s/api/2.0/workspace/get-status' % (DOMAIN), + headers={'Authorization': 'Bearer %s' % TOKEN}, + json=BODY, + timeout=60 + ) + +exists = True + +if 'RESOURCE_DOES_NOT_EXIST' not in response.text: + json_response = response.json() + dashboard_id = json_response['resource_id'] +else: + exists = False + print("Dashboard doesn't exist yet") + + +# COMMAND ---------- + +#Delete using the API DELETE /api/2.0/lakeview/dashboards/ + +if exists != False: + + loggr.info(f"Deleting Dashboard") + response = requests.delete( + 'https://%s/api/2.0/lakeview/dashboards/%s' % (DOMAIN, dashboard_id), + headers={'Authorization': 'Bearer %s' % TOKEN}, + json=BODY, + timeout=60 + ) + + +# COMMAND ---------- + +# MAGIC %md +# MAGIC # Create Dashboard from json definition + +# COMMAND ---------- + +import requests +json_file_path = f"{basePath()}/dashboards/SAT_Dashboard_definition.json" + +# Read the JSON file as a string +with open(json_file_path) as json_file: + json_data = json.load(json_file) + +json_string = json_string = json.dumps(json_data) + +BODY = {'display_name': 'SAT - Security Analysis Tool (Lakeview - Experimental)','warehouse_id': json_['sql_warehouse_id'], 'serialized_dashboard': json_string, 'parent_path': f"{basePath()}/dashboards"} + +loggr.info(f"Creating Dashboard") +response = requests.post( + 'https://%s/api/2.0/lakeview/dashboards' % (DOMAIN), + headers={'Authorization': 'Bearer %s' % TOKEN}, + json=BODY, + timeout=60 + ) + +exists = False + +if 'RESOURCE_ALREADY_EXISTS' not in response.text: + json_response = response.json() + dashboard_id = json_response['dashboard_id'] +else: + exists = True + print("Lakeview Dashboard already exists") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC # Publish the Dashboard using the SAT warehouse + +# COMMAND ---------- + +import requests +import json + +if exists != True: + + URL = "https://"+DOMAIN+"/api/2.0/lakeview/dashboards/"+dashboard_id+"/published" + BODY = {'embed_credentials': 'true', 'warehouse_id': json_['sql_warehouse_id']} + + loggr.info(f"Publishing the Dashboard using the SAT SQL Warehouse") + response = requests.post( + URL, + headers={'Authorization': 'Bearer %s' % TOKEN}, + json=BODY, + timeout=60 + ) + +else: + print("Dashboard already exists") + +# COMMAND ---------- + +dbutils.notebook.exit('OK') diff --git a/notebooks/Setup/9. self_assess_workspace_configuration.py b/notebooks/Setup/9. self_assess_workspace_configuration.py index 90a3f120..16f2c211 100644 --- a/notebooks/Setup/9. self_assess_workspace_configuration.py +++ b/notebooks/Setup/9. self_assess_workspace_configuration.py @@ -20,7 +20,7 @@ # COMMAND ---------- -# MAGIC %run ./../Utils/sat_checks_config +# MAGIC %run ../Utils/sat_checks_config # COMMAND ---------- diff --git a/notebooks/Utils/common.py b/notebooks/Utils/common.py index 8357264b..566c34f4 100644 --- a/notebooks/Utils/common.py +++ b/notebooks/Utils/common.py @@ -1,61 +1,78 @@ # Databricks notebook source # MAGIC %md -# MAGIC **Notebook name:** common. +# MAGIC **Notebook name:** common. # MAGIC **Functionality:** routines used across the project # COMMAND ---------- -def bootstrap(viewname, func, **kwargs ): - """bootstrap with function and store resulting dataframe as a global temp view - if the function doesnt return a value, creates an empty dataframe and corresponding view - :param str viewname - name of the view - :param func - Name of the function to call - :**kwargs - named args to pass to the function - - """ - from pyspark.sql.types import StructType - import json - apiDF=None - try: - lst = func(**kwargs) - if lst: - lstjson = [json.dumps(ifld) for ifld in lst] - apiDF = spark.read.json(sc.parallelize(lstjson)) - else: - apiDF = spark.createDataFrame([], StructType([])) - loggr.info('No Results!') - spark.catalog.dropGlobalTempView(viewname) - apiDF.createGlobalTempView(viewname) - loggr.info(f'View created. `global_temp`.`{viewname}`') - except Exception: - loggr.exception("Exception encountered") + +def bootstrap(viewname, func, **kwargs): + """bootstrap with function and store resulting dataframe as a global temp view + if the function doesnt return a value, creates an empty dataframe and corresponding view + :param str viewname - name of the view + :param func - Name of the function to call + :**kwargs - named args to pass to the function + + """ + import json + + from pyspark.sql.types import StructType + + apiDF = None + try: + lst = func(**kwargs) + if lst: + lstjson = [json.dumps(ifld) for ifld in lst] + apiDF = spark.read.json(sc.parallelize(lstjson)) + else: + apiDF = spark.createDataFrame([], StructType([])) + loggr.info("No Results!") + spark.catalog.dropGlobalTempView(viewname) + apiDF.createGlobalTempView(viewname) + loggr.info(f"View created. `global_temp`.`{viewname}`") + except Exception: + loggr.exception("Exception encountered") + # COMMAND ---------- + def handleAnalysisErrors(e): - """ - Handle AnalysisException when sql is run. This is raised when fields in sql are not found. - """ - v= e.getMessage() - vlst = v.lower().split(" ") - strField='' - if len(vlst)>2 and vlst[0]=='cannot' and vlst[1]=='resolve': - strField='cannot find field ' + vlst[2] + ' in SQL' - elif len(vlst)>8 and vlst[0]=='[unresolved_column.with_suggestion]' and vlst[5]=='function' and vlst[6]=='parameter': - strField='cannot find field ' + vlst[9] + ' in SQL' - elif len(vlst)>8 and vlst[0]=='[unresolved_column.without_suggestion]' and vlst[5]=='function' and vlst[6]=='parameter': - strField='cannot find field ' + vlst[9] + ' in SQL' - elif len(vlst)>3 and vlst[1]=='such' and vlst[2]=='struct': - strField='cannot find struct field `' + vlst[4] + '` in SQL' - elif len(vlst)>2 and 'Did you mean' in v: - strField='field ' + vlst[1] + ' not found' - else: - strField=v - return strField + """ + Handle AnalysisException when sql is run. This is raised when fields in sql are not found. + """ + v = e.getMessage() + vlst = v.lower().split(" ") + strField = "" + if len(vlst) > 2 and vlst[0] == "cannot" and vlst[1] == "resolve": + strField = "cannot find field " + vlst[2] + " in SQL" + elif ( + len(vlst) > 8 + and vlst[0] == "[unresolved_column.with_suggestion]" + and vlst[5] == "function" + and vlst[6] == "parameter" + ): + strField = "cannot find field " + vlst[9] + " in SQL" + elif ( + len(vlst) > 8 + and vlst[0] == "[unresolved_column.without_suggestion]" + and vlst[5] == "function" + and vlst[6] == "parameter" + ): + strField = "cannot find field " + vlst[9] + " in SQL" + elif len(vlst) > 3 and vlst[1] == "such" and vlst[2] == "struct": + strField = "cannot find struct field `" + vlst[4] + "` in SQL" + elif len(vlst) > 2 and "Did you mean" in v: + strField = "field " + vlst[1] + " not found" + else: + strField = v + return strField + # COMMAND ---------- -def sqlctrl(workspace_id, sqlstr, funcrule, info=False): #lambda + +def sqlctrl(workspace_id, sqlstr, funcrule, info=False): # lambda """Executes sql, tests the result with the function and write results to control table :param sqlstr sql to execute :param funcrule rule to execute to check if violation passed or failed @@ -63,68 +80,85 @@ def sqlctrl(workspace_id, sqlstr, funcrule, info=False): #lambda """ import pyspark.sql.utils from pyspark.sql.types import StructType + try: df = spark.sql(sqlstr) - except pyspark.sql.utils.AnalysisException as e: + except pyspark.sql.utils.AnalysisException as e: s = handleAnalysisErrors(e) df = spark.createDataFrame([], StructType([])) - loggr.info(s) + loggr.info(s) try: if funcrule: display(df) if info: - name,value,category = funcrule(df) - insertIntoInfoTable(workspace_id, name, value,category) - else: - ctrlname,ctrlscore,additional_details = funcrule(df) - if len(additional_details) ==0 and ctrlscore ==0: - additional_details = {'message':'No deviations from the security best practices found for this check'} - - insertIntoControlTable(workspace_id, ctrlname, ctrlscore, additional_details) + name, value, category = funcrule(df) + insertIntoInfoTable(workspace_id, name, value, category) + else: + ctrlname, ctrlscore, additional_details = funcrule(df) + if len(additional_details) == 0 and ctrlscore == 0: + additional_details = { + "message": "No deviations from the security best practices found for this check" + } + + insertIntoControlTable( + workspace_id, ctrlname, ctrlscore, additional_details + ) except Exception as e: loggr.exception(e) # COMMAND ---------- + def sqldisplay(sqlstr): - """ - execute a sql and display the dataframe. - :param str sqlstr SQL to execute - """ - import pyspark.sql.utils - try: - df = spark.sql(sqlstr) - display(df) - except pyspark.sql.utils.AnalysisException as e: - s = handleAnalysisErrors(e) - loggr.info(s) - except Exception as e: - loggr.exception(e) + """ + execute a sql and display the dataframe. + :param str sqlstr SQL to execute + """ + import pyspark.sql.utils + + try: + df = spark.sql(sqlstr) + display(df) + except pyspark.sql.utils.AnalysisException as e: + s = handleAnalysisErrors(e) + loggr.info(s) + except Exception as e: + loggr.exception(e) + # COMMAND ---------- + def insertIntoControlTable(workspace_id, id, score, additional_details): """ Insert results into a control table - :workspace_id workspace id for this check + :workspace_id workspace id for this check :param str id id mapping to best practices config of the check :param int score integer score based on violation :param dictionary additional_details additional details of the check """ - import time,json + import json + import time + ts = time.time() - #change this. Has to come via function. - #orgId = dbutils.notebook.entry_point.getDbutils().notebook().getContext().tags().get('orgId').getOrElse(None) - run_id = spark.sql(f'select max(runID) from {json_["analysis_schema_name"]}.run_number_table').collect()[0][0] + # change this. Has to come via function. + # orgId = dbutils.notebook.entry_point.getDbutils().notebook().getContext().tags().get('orgId').getOrElse(None) + run_id = spark.sql( + f'select max(runID) from {json_["analysis_schema_name"]}.run_number_table' + ).collect()[0][0] jsonstr = json.dumps(additional_details) - sql = '''INSERT INTO {}.`security_checks` (`workspaceid`, `id`, `score`, `additional_details`, `run_id`, `check_time`) - VALUES ('{}', '{}', cast({} as int), from_json('{}', 'MAP'), {}, cast({} as timestamp))'''.format(json_["analysis_schema_name"],workspace_id, id, score, jsonstr, run_id, ts) + sql = """INSERT INTO {}.`security_checks` (`workspaceid`, `id`, `score`, `additional_details`, `run_id`, `check_time`) + VALUES ('{}', '{}', cast({} as int), from_json('{}', 'MAP'), {}, cast({} as timestamp))""".format( + json_["analysis_schema_name"], workspace_id, id, score, jsonstr, run_id, ts + ) ###print(sql) spark.sql(sql) + # COMMAND ---------- + def insertIntoInfoTable(workspace_id, name, value, category): """ Insert values into an information table @@ -132,120 +166,214 @@ def insertIntoInfoTable(workspace_id, name, value, category): :param value additional_details additional details of the value :param str category of info for filtering """ - import time,json + import json + import time + ts = time.time() - #change this. Has to come via function. - #orgId = dbutils.notebook.entry_point.getDbutils().notebook().getContext().tags().get('orgId').getOrElse(None) - run_id = spark.sql(f'select max(runID) from {json_["analysis_schema_name"]}.run_number_table').collect()[0][0] + # change this. Has to come via function. + # orgId = dbutils.notebook.entry_point.getDbutils().notebook().getContext().tags().get('orgId').getOrElse(None) + run_id = spark.sql( + f'select max(runID) from {json_["analysis_schema_name"]}.run_number_table' + ).collect()[0][0] jsonstr = json.dumps(value) - sql = '''INSERT INTO {}.`account_info` (`workspaceid`,`name`, `value`, `category`, `run_id`, `check_time`) - VALUES ('{}','{}', from_json('{}', 'MAP'), '{}', '{}', cast({} as timestamp))'''.format(json_["analysis_schema_name"],workspace_id, name, jsonstr, category, run_id, ts) + sql = """INSERT INTO {}.`account_info` (`workspaceid`,`name`, `value`, `category`, `run_id`, `check_time`) + VALUES ('{}','{}', from_json('{}', 'MAP'), '{}', '{}', cast({} as timestamp))""".format( + json_["analysis_schema_name"], workspace_id, name, jsonstr, category, run_id, ts + ) ### print(sql) spark.sql(sql) + # COMMAND ---------- + def getCloudType(url): - if '.cloud.' in url: - return 'aws' - elif '.azuredatabricks.' in url: - return 'azure' - elif '.gcp.' in url: - return 'gcp' - return '' + if ".cloud." in url: + return "aws" + elif ".azuredatabricks." in url: + return "azure" + elif ".gcp." in url: + return "gcp" + return "" + # COMMAND ---------- + def readWorkspaceConfigFile(): - import pandas as pd - prefix = getConfigPath() - - dfa=pd.DataFrame() - schema = 'workspace_id string, deployment_url string, workspace_name string,workspace_status string, ws_token string, sso_enabled boolean, scim_enabled boolean, vpc_peering_done boolean, object_storage_encrypted boolean, table_access_control_enabled boolean, connection_test boolean, analysis_enabled boolean' - dfexist = spark.createDataFrame([], schema) - try: - dict = {'workspace_id': 'str', 'connection_test': 'bool', 'analysis_enabled': 'bool'} - dfa = pd.read_csv(f'{prefix}/workspace_configs.csv', header=0, dtype=dict) - if len(dfa) > 0: - dfexist = spark.createDataFrame(dfa, schema) - except FileNotFoundError: - print('Missing workspace Config file') - return - except pd.errors.EmptyDataError as e: - pass - return dfexist + import pandas as pd + + prefix = getConfigPath() + + dfa = pd.DataFrame() + schema = "workspace_id string, deployment_url string, workspace_name string,workspace_status string, ws_token string, sso_enabled boolean, scim_enabled boolean, vpc_peering_done boolean, object_storage_encrypted boolean, table_access_control_enabled boolean, connection_test boolean, analysis_enabled boolean" + dfexist = spark.createDataFrame([], schema) + try: + dict = { + "workspace_id": "str", + "connection_test": "bool", + "analysis_enabled": "bool", + } + dfa = pd.read_csv(f"{prefix}/workspace_configs.csv", header=0, dtype=dict) + if len(dfa) > 0: + dfexist = spark.createDataFrame(dfa, schema) + except FileNotFoundError: + print("Missing workspace Config file") + return + except pd.errors.EmptyDataError as e: + pass + return dfexist + # COMMAND ---------- + def getWorkspaceConfig(): - df = spark.sql(f'''select * from {json_["analysis_schema_name"]}.account_workspaces''') - return df + df = spark.sql( + f"""select * from {json_["analysis_schema_name"]}.account_workspaces""" + ) + return df + # COMMAND ---------- + # Read the best practices file. (security_best_practices.csv) # Sice User configs are present in this file, the file is renamed (to security_best_practices_user) # This is needed only on bootstrap, subsequetly the database is the master copy of the user configuration # Every time the values are altered, the _user file can be regenerated - but it is more as FYI def readBestPracticesConfigsFile(): + import shutil + from os.path import exists + + import pandas as pd + + hostname = ( + dbutils.notebook.entry_point.getDbutils() + .notebook() + .getContext() + .apiUrl() + .getOrElse(None) + ) + cloud_type = getCloudType(hostname) + doc_url = cloud_type + "_doc_url" + + prefix = getConfigPath() + origfile = f"{prefix}/security_best_practices.csv" + + schema_list = [ + "id", + "check_id", + "category", + "check", + "evaluation_value", + "severity", + "recommendation", + "aws", + "azure", + "gcp", + "enable", + "alert", + "logic", + "api", + doc_url, + ] + + schema = """id int, check_id string,category string,check string, evaluation_value string,severity string, + recommendation string,aws int,azure int,gcp int,enable int,alert int, logic string, api string, doc_url string""" + + security_best_practices_pd = pd.read_csv( + origfile, header=0, usecols=schema_list + ).rename(columns={doc_url: "doc_url"}) + + security_best_practices = spark.createDataFrame( + security_best_practices_pd, schema + ).select( + "id", + "check_id", + "category", + "check", + "evaluation_value", + "severity", + "recommendation", + "doc_url", + "aws", + "azure", + "gcp", + "enable", + "alert", + "logic", + "api", + ) + + security_best_practices.write.format("delta").mode("overwrite").saveAsTable( + json_["analysis_schema_name"] + ".security_best_practices" + ) + display(security_best_practices) + + +# COMMAND ---------- + +# Read and load the SAT and DASF mapping file. (SAT_DASF_mapping.csv) +def load_sat_dasf_mapping(): import pandas as pd from os.path import exists import shutil - hostname = dbutils.notebook.entry_point.getDbutils().notebook().getContext().apiUrl().getOrElse(None) - cloud_type = getCloudType(hostname) - doc_url = cloud_type + '_doc_url' - + prefix = getConfigPath() - origfile = f'{prefix}/security_best_practices.csv' - userfile = f'{prefix}/security_best_practices_user.csv' #delete this file to get latest - file_exists = exists(userfile) + origfile = f'{prefix}/sat_dasf_mapping.csv' - if(file_exists): #bootstrap has already been done, the DB is the master, do not overwrite - return - - schema_list = ['id', 'check_id', 'category', 'check', 'evaluation_value', 'severity', - 'recommendation', 'aws', 'azure', 'gcp', 'enable', 'alert', 'logic', 'api', doc_url] + schema_list = ['sat_id', 'dasf_control_id','dasf_control_name'] - schema = '''id int, check_id string,category string,check string, evaluation_value string,severity string, - recommendation string,aws int,azure int,gcp int,enable int,alert int, logic string, api string, doc_url string''' + schema = '''sat_id int, dasf_control_id string,dasf_control_name string''' - security_best_practices_pd = pd.read_csv(origfile, header=0, usecols=schema_list).rename(columns = {doc_url:'doc_url'}) - security_best_practices_pd.to_csv(userfile, encoding='utf-8', index=False) + sat_dasf_mapping_pd = pd.read_csv(origfile, header=0, usecols=schema_list) - security_best_practices = (spark.createDataFrame(security_best_practices_pd, schema) - .select('id', 'check_id', 'category', 'check', 'evaluation_value', - 'severity', 'recommendation', 'doc_url', 'aws', 'azure', 'gcp', 'enable', 'alert', 'logic', 'api')) + sat_dasf_mapping = (spark.createDataFrame(sat_dasf_mapping_pd, schema) + .select('sat_id', 'dasf_control_id','dasf_control_name')) - security_best_practices.write.format('delta').mode('overwrite').saveAsTable(json_["analysis_schema_name"]+'.security_best_practices') - display(security_best_practices) + sat_dasf_mapping.write.format('delta').mode('overwrite').saveAsTable(json_["analysis_schema_name"]+'.sat_dasf_mapping') + display(sat_dasf_mapping) + # COMMAND ---------- + def getSecurityBestPracticeRecord(id, cloud_type): - df = spark.sql(f'''select * from {json_["analysis_schema_name"]}.security_best_practices where id = '{id}' ''') - dict_elems = {} - enable=0 - if 'none' not in cloud_type and df is not None and df.count()>0: - dict_elems = df.collect()[0] - if dict_elems[cloud_type]==1 and dict_elems['enable']==1: - enable=1 - - return (enable, dict_elems) + df = spark.sql( + f"""select * from {json_["analysis_schema_name"]}.security_best_practices where id = '{id}' """ + ) + dict_elems = {} + enable = 0 + if "none" not in cloud_type and df is not None and df.count() > 0: + dict_elems = df.collect()[0] + if dict_elems[cloud_type] == 1 and dict_elems["enable"] == 1: + enable = 1 + + return (enable, dict_elems) + # COMMAND ---------- + def getConfigPath(): - import os - cwd = os.getcwd().lower() - if (cwd.rfind('/azure') != -1) or (cwd.rfind('/gcp') != -1): - return '../../../configs' - elif (cwd.rfind('/includes') != -1) or (cwd.rfind('/setup') != -1) or (cwd.rfind('/utils') != -1 or (cwd.rfind('/diagnosis') != -1)): - return '../../configs' - elif (cwd.rfind('/notebooks') != -1): - return '../configs' - else: - return 'configs' + return f"{basePath()}/configs" + + +# COMMAND ---------- + +def basePath(): + path = ( + dbutils.notebook.entry_point.getDbutils() + .notebook() + .getContext() + .notebookPath() + .get() + ) + path = path[: path.find("/notebooks")] + return f"/Workspace{path}" + # COMMAND ---------- @@ -259,36 +387,51 @@ def getConfigPath(): # COMMAND ---------- + def create_schema(): - df = spark.sql(f'CREATE DATABASE IF NOT EXISTS {json_["analysis_schema_name"]}' ) - df = spark.sql(f"""CREATE TABLE IF NOT EXISTS {json_["analysis_schema_name"]}.run_number_table ( + df = spark.sql(f'CREATE DATABASE IF NOT EXISTS {json_["analysis_schema_name"]}') + df = spark.sql( + f"""CREATE TABLE IF NOT EXISTS {json_["analysis_schema_name"]}.run_number_table ( runID BIGINT GENERATED ALWAYS AS IDENTITY, check_time TIMESTAMP ) - USING DELTA""") + USING DELTA""" + ) # COMMAND ---------- + def insertNewBatchRun(): - import time - ts = time.time() - df = spark.sql(f'insert into {json_["analysis_schema_name"]}.run_number_table (check_time) values ({ts})') + import time + + ts = time.time() + df = spark.sql( + f'insert into {json_["analysis_schema_name"]}.run_number_table (check_time) values ({ts})' + ) # COMMAND ---------- + def notifyworkspaceCompleted(workspaceID, completed): - import time - ts = time.time() - runID = spark.sql(f'select max(runID) from {json_["analysis_schema_name"]}.run_number_table').collect()[0][0] - spark.sql(f'''INSERT INTO {json_["analysis_schema_name"]}.workspace_run_complete (`workspace_id`,`run_id`, `completed`, `check_time`) VALUES ({workspaceID}, {runID}, {completed}, cast({ts} as timestamp))''') + import time + + ts = time.time() + runID = spark.sql( + f'select max(runID) from {json_["analysis_schema_name"]}.run_number_table' + ).collect()[0][0] + spark.sql( + f"""INSERT INTO {json_["analysis_schema_name"]}.workspace_run_complete (`workspace_id`,`run_id`, `completed`, `check_time`) VALUES ({workspaceID}, {runID}, {completed}, cast({ts} as timestamp))""" + ) # COMMAND ---------- + def create_security_checks_table(): - df = spark.sql(f"""CREATE TABLE IF NOT EXISTS {json_["analysis_schema_name"]}.security_checks ( + df = spark.sql( + f"""CREATE TABLE IF NOT EXISTS {json_["analysis_schema_name"]}.security_checks ( workspaceid string, id int, score integer, @@ -299,13 +442,16 @@ def create_security_checks_table(): chk_hhmm integer GENERATED ALWAYS AS (CAST(CAST(hour(check_time) as STRING) || CAST(minute(check_time) as STRING) as INTEGER)) ) USING DELTA - PARTITIONED BY (chk_date)""") + PARTITIONED BY (chk_date)""" + ) # COMMAND ---------- + def create_account_info_table(): - df = spark.sql(f"""CREATE TABLE IF NOT EXISTS {json_["analysis_schema_name"]}.account_info ( + df = spark.sql( + f"""CREATE TABLE IF NOT EXISTS {json_["analysis_schema_name"]}.account_info ( workspaceid string, name string, value map, @@ -316,13 +462,16 @@ def create_account_info_table(): chk_hhmm integer GENERATED ALWAYS AS (CAST(CAST(hour(check_time) as STRING) || CAST(minute(check_time) as STRING) as INTEGER)) ) USING DELTA - PARTITIONED BY (chk_date)""") + PARTITIONED BY (chk_date)""" + ) # COMMAND ---------- + def create_account_workspaces_table(): - df = spark.sql(f"""CREATE TABLE IF NOT EXISTS {json_["analysis_schema_name"]}.account_workspaces ( + df = spark.sql( + f"""CREATE TABLE IF NOT EXISTS {json_["analysis_schema_name"]}.account_workspaces ( workspace_id string, deployment_url string, workspace_name string, @@ -335,31 +484,31 @@ def create_account_workspaces_table(): object_storage_encrypted boolean, table_access_control_enabled boolean ) - USING DELTA""") + USING DELTA""" + ) # COMMAND ---------- + def create_workspace_run_complete_table(): - df = spark.sql(f"""CREATE TABLE IF NOT EXISTS {json_["analysis_schema_name"]}.workspace_run_complete( + df = spark.sql( + f"""CREATE TABLE IF NOT EXISTS {json_["analysis_schema_name"]}.workspace_run_complete( workspace_id string, run_id bigint, completed boolean, check_time timestamp, chk_date date GENERATED ALWAYS AS (CAST(check_time AS DATE)) ) - USING DELTA""") + USING DELTA""" + ) # COMMAND ---------- -#For testing -JSONLOCALTESTA='{"account_id": "", "sql_warehouse_id": "", "verbosity": "info", "master_name_scope": "sat_scope", "master_name_key": "user", "master_pwd_scope": "sat_scope", "master_pwd_key": "pass", "workspace_pat_scope": "sat_scope", "workspace_pat_token_prefix": "sat_token", "dashboard_id": "317f4809-8d9d-4956-a79a-6eee51412217", "dashboard_folder": "../../dashboards/", "dashboard_tag": "SAT", "use_mastercreds": true, "url": "https://satanalysis.cloud.databricks.com", "workspace_id": "2657683783405196", "cloud_type": "aws", "clusterid": "1115-184042-ntswg7ll", "sso": false, "scim": false, "object_storage_encryption": false, "vpc_peering": false, "table_access_control_enabled": false}' +# For testing +JSONLOCALTESTA = '{"account_id": "", "sql_warehouse_id": "", "verbosity": "info", "master_name_scope": "sat_scope", "master_name_key": "user", "master_pwd_scope": "sat_scope", "master_pwd_key": "pass", "workspace_pat_scope": "sat_scope", "workspace_pat_token_prefix": "sat_token", "dashboard_id": "317f4809-8d9d-4956-a79a-6eee51412217", "dashboard_folder": "../../dashboards/", "dashboard_tag": "SAT", "use_mastercreds": true, "url": "https://satanalysis.cloud.databricks.com", "workspace_id": "2657683783405196", "cloud_type": "aws", "clusterid": "1115-184042-ntswg7ll", "sso": false, "scim": false, "object_storage_encryption": false, "vpc_peering": false, "table_access_control_enabled": false}' # COMMAND ---------- JSONLOCALTESTB = '{"account_id": "", "sql_warehouse_id": "4a936419ee9b9d68", "verbosity": "info", "master_name_scope": "sat_scope", "master_name_key": "user", "master_pwd_scope": "sat_scope", "master_pwd_key": "pass", "workspace_pat_scope": "sat_scope", "workspace_pat_token_prefix": "sat_token", "dashboard_id": "317f4809-8d9d-4956-a79a-6eee51412217", "dashboard_folder": "../../dashboards/", "dashboard_tag": "SAT", "use_mastercreds": true, "subscription_id": "", "tenant_id": "", "client_id": "", "client_secret": "", "generate_pat_tokens": false, "url": "https://adb-83xxx7.17.azuredatabricks.net", "workspace_id": "83xxxx7", "clusterid": "0105-242242-ir40aiai", "sso": true, "scim": false, "object_storage_encryption": false, "vpc_peering": false, "table_access_control_enabled": false, "cloud_type":"azure"}' - -# COMMAND ---------- - - diff --git a/notebooks/Utils/initialize.py b/notebooks/Utils/initialize.py index f685faea..b3d63eac 100644 --- a/notebooks/Utils/initialize.py +++ b/notebooks/Utils/initialize.py @@ -1,6 +1,6 @@ # Databricks notebook source # MAGIC %md -# MAGIC **Notebook name:** initialize +# MAGIC **Notebook name:** initialize # MAGIC **Functionality:** initializes the necessary configruation values for the rest of the process into a json # COMMAND ---------- @@ -9,8 +9,14 @@ # COMMAND ---------- -#replace values for accounts exec -hostname = dbutils.notebook.entry_point.getDbutils().notebook().getContext().apiUrl().getOrElse(None) +# replace values for accounts exec +hostname = ( + dbutils.notebook.entry_point.getDbutils() + .notebook() + .getContext() + .apiUrl() + .getOrElse(None) +) cloud_type = getCloudType(hostname) # COMMAND ---------- @@ -33,70 +39,90 @@ import json json_ = { - "account_id": dbutils.secrets.get(scope="sat_scope", key="account-console-id"), - "sql_warehouse_id": dbutils.secrets.get(scope="sat_scope", key="sql-warehouse-id"), - "analysis_schema_name":"security_analysis", - "verbosity":"info" + "account_id": dbutils.secrets.get(scope="sat_scope", key="account-console-id"), + "sql_warehouse_id": dbutils.secrets.get(scope="sat_scope", key="sql-warehouse-id"), + "analysis_schema_name": "security_analysis", + "verbosity": "info", } # COMMAND ---------- -json_.update({ - "master_name_scope":"sat_scope", - "master_name_key":"user", - "master_pwd_scope":"sat_scope", - "master_pwd_key":"pass", - "workspace_pat_scope":"sat_scope", - "workspace_pat_token_prefix":"sat-token", - "dashboard_id":"317f4809-8d9d-4956-a79a-6eee51412217", - "dashboard_folder":"../../dashboards/", - "dashboard_tag":"SAT", - "use_mastercreds":True, - "use_parallel_runs":True -}) +json_.update( + { + "master_name_scope": "sat_scope", + "master_name_key": "user", + "master_pwd_scope": "sat_scope", + "master_pwd_key": "pass", + "workspace_pat_scope": "sat_scope", + "workspace_pat_token_prefix": "sat-token", + "dashboard_id": "317f4809-8d9d-4956-a79a-6eee51412217", + "dashboard_folder": f"{basePath()}/dashboards/", + "dashboard_tag": "SAT", + "use_mastercreds": True, + "use_parallel_runs": True, + } +) # COMMAND ---------- -# DBTITLE 1,GCP configurations -if cloud_type == 'gcp': - json_.update({ - "service_account_key_file_path": dbutils.secrets.get(scope="sat_scope", key="gs-path-to-json"), - "impersonate_service_account": dbutils.secrets.get(scope="sat_scope", key="impersonate-service-account"), - "use_mastercreds":False - }) +# DBTITLE 1,GCP configurations +if cloud_type == "gcp": + json_.update( + { + "service_account_key_file_path": dbutils.secrets.get( + scope="sat_scope", key="gs-path-to-json" + ), + "impersonate_service_account": dbutils.secrets.get( + scope="sat_scope", key="impersonate-service-account" + ), + "use_mastercreds": False, + } + ) # COMMAND ---------- -# DBTITLE 1,Azure configurations -if cloud_type == 'azure': - json_.update({ - "account_id":"azure", - "subscription_id": dbutils.secrets.get(scope="sat_scope", key="subscription-id"), # Azure subscriptionId - "tenant_id": dbutils.secrets.get(scope="sat_scope", key="tenant-id"), #The Directory (tenant) ID for the application registered in Azure AD. - "client_id": dbutils.secrets.get(scope="sat_scope", key="client-id"), # The Application (client) ID for the application registered in Azure AD. - "client_secret_key":"client-secret", #The secret generated by AAD during your confidential app registration - "use_mastercreds":True - }) +# DBTITLE 1,Azure configurations +if cloud_type == "azure": + json_.update( + { + "account_id": "azure", + "subscription_id": dbutils.secrets.get( + scope="sat_scope", key="subscription-id" + ), # Azure subscriptionId + "tenant_id": dbutils.secrets.get( + scope="sat_scope", key="tenant-id" + ), # The Directory (tenant) ID for the application registered in Azure AD. + "client_id": dbutils.secrets.get( + scope="sat_scope", key="client-id" + ), # The Application (client) ID for the application registered in Azure AD. + "client_secret_key": "client-secret", # The secret generated by AAD during your confidential app registration + "use_mastercreds": True, + } + ) # COMMAND ---------- # DBTITLE 1,AWS configurations -if cloud_type == 'aws': +if cloud_type == "aws": sp_auth = { - "use_sp_auth": 'False', - "client_id": '', - "client_secret_key": "client-secret" + "use_sp_auth": "False", + "client_id": "", + "client_secret_key": "client-secret", } try: - use_sp_auth = dbutils.secrets.get(scope="sat_scope", key="use-sp-auth").lower() == 'true' - if use_sp_auth: - sp_auth['use_sp_auth'] = 'True' - sp_auth['client_id'] = dbutils.secrets.get(scope="sat_scope", key="client-id") + use_sp_auth = ( + dbutils.secrets.get(scope="sat_scope", key="use-sp-auth").lower() == "true" + ) + if use_sp_auth: + sp_auth["use_sp_auth"] = "True" + sp_auth["client_id"] = dbutils.secrets.get( + scope="sat_scope", key="client-id" + ) except: - pass + pass json_.update(sp_auth) # COMMAND ---------- @@ -109,5 +135,10 @@ # COMMAND ---------- -#Initialize best practices if not already loaded into database +# Initialize best practices readBestPracticesConfigsFile() + +# COMMAND ---------- + +#Initialize sat dasf mapping +load_sat_dasf_mapping() diff --git a/notebooks/Utils/sat_checks_config.py b/notebooks/Utils/sat_checks_config.py index 7b8a9d5b..f8a821bc 100644 --- a/notebooks/Utils/sat_checks_config.py +++ b/notebooks/Utils/sat_checks_config.py @@ -1,159 +1,214 @@ # Databricks notebook source # MAGIC %md -# MAGIC **Notebook name:** sat_checks_config +# MAGIC **Notebook name:** sat_checks_config # MAGIC **Functionality:** initializes the necessary configruation values for the rest of the process into a json # COMMAND ---------- -#Determine cloud type -def getCloudType(url): - if '.cloud.' in url: - return 'aws' - elif '.azuredatabricks.' in url: - return 'azure' - elif '.gcp.' in url: - return 'gcp' - return '' - -def getConfigPath(): - import os - cwd = os.getcwd().lower() - if (cwd.rfind('/includes') != -1) or (cwd.rfind('/setup') != -1) or (cwd.rfind('/utils') != -1): - return '../../configs' - elif (cwd.rfind('/notebooks') != -1): - return '../configs' - else: - return 'configs' +# MAGIC %run ./common +# MAGIC # COMMAND ---------- -#Account Level SAT Check Configuration +# Account Level SAT Check Configuration -#SET & GET SAT check configurations for the organization + +# SET & GET SAT check configurations for the organization def get_sat_check_config(): sat_check = dbutils.widgets.get("sat_check") - check_id = sat_check.split('_')[0] + check_id = sat_check.split("_")[0] - s_sql = ''' + s_sql = """ SELECT enable, evaluation_value, alert FROM {analysis_schema_name}.security_best_practices WHERE check_id= '{check_id}' - '''.format(check_id = check_id, analysis_schema_name= json_["analysis_schema_name"]) + """.format( + check_id=check_id, analysis_schema_name=json_["analysis_schema_name"] + ) get_check = spark.sql(s_sql) - check = get_check.toPandas().to_dict(orient = 'list') - - enable = check['enable'][0] - evaluate = check['evaluation_value'][0] - alert = check['alert'][0] + check = get_check.toPandas().to_dict(orient="list") + + enable = check["enable"][0] + evaluate = check["evaluation_value"][0] + alert = check["alert"][0] - dbutils.widgets.dropdown("check_enabled", str(enable), ['0','1'], "b. Check Enabled") + dbutils.widgets.dropdown( + "check_enabled", str(enable), ["0", "1"], "b. Check Enabled" + ) dbutils.widgets.text("evaluation_value", str(evaluate), "c. Evaluation Value") dbutils.widgets.text("alert", str(alert), "d. Alert") - + + def set_sat_check_config(): - #Retrieve widget values + # Retrieve widget values sat_check = dbutils.widgets.get("sat_check") enable = dbutils.widgets.get("check_enabled") evaluation_value = dbutils.widgets.get("evaluation_value") alert = dbutils.widgets.get("alert") - - check_id = sat_check.split('_')[0] - s_sql = ''' + check_id = sat_check.split("_")[0] + + s_sql = """ UPDATE {analysis_schema_name}.security_best_practices SET enable = {enable}, evaluation_value={evaluation_value}, alert = {alert} WHERE check_id= '{check_id}' - '''.format(enable=enable, evaluation_value=evaluation_value, alert=alert, check_id = check_id, analysis_schema_name= json_["analysis_schema_name"]) + """.format( + enable=enable, + evaluation_value=evaluation_value, + alert=alert, + check_id=check_id, + analysis_schema_name=json_["analysis_schema_name"], + ) print(s_sql) spark.sql(s_sql) - - #update user config file (security_best_practices_user.csv) + + # update user config file (security_best_practices_user.csv) prefix = getConfigPath() - userfile = f'{prefix}/security_best_practices_user.csv' - security_best_practices_pd = spark.table(f'{json_["analysis_schema_name"]}.security_best_practices').toPandas() - security_best_practices_pd.to_csv(userfile, encoding='utf-8', index=False) - -#Reset SAT check widgets + userfile = f"{prefix}/security_best_practices_user.csv" + security_best_practices_pd = spark.table( + f'{json_["analysis_schema_name"]}.security_best_practices' + ).toPandas() + security_best_practices_pd.to_csv(userfile, encoding="utf-8", index=False) + + +# Reset SAT check widgets def get_all_sat_checks(): dbutils.widgets.removeAll() - hostname = dbutils.notebook.entry_point.getDbutils().notebook().getContext().apiUrl().getOrElse(None) + hostname = ( + dbutils.notebook.entry_point.getDbutils() + .notebook() + .getContext() + .apiUrl() + .getOrElse(None) + ) cloud_type = getCloudType(hostname) - s_sql = ''' + s_sql = """ SELECT CONCAT_WS('_',check_id, category, check) AS check FROM {analysis_schema_name}.security_best_practices WHERE {cloud_type}=1 - '''.format(cloud_type = cloud_type, analysis_schema_name= json_["analysis_schema_name"]) + """.format( + cloud_type=cloud_type, analysis_schema_name=json_["analysis_schema_name"] + ) all_checks = spark.sql(s_sql) - checks = all_checks.rdd.map(lambda row : row[0]).collect() + checks = all_checks.rdd.map(lambda row: row[0]).collect() checks.sort() first_check = str(checks[0]) - #Define Driver Widgets - dbutils.widgets.dropdown("sat_check", first_check, [str(x) for x in checks], "a. SAT Check") + # Define Driver Widgets + dbutils.widgets.dropdown( + "sat_check", first_check, [str(x) for x in checks], "a. SAT Check" + ) + # COMMAND ---------- -#Workspace Level SAT Check Configuration -params = {'Analysis Enabled': 'analysis_enabled', - 'SSO Enabled':'sso_enabled', - 'SCIM Enabled': 'scim_enabled', - 'ANY VPC PEERING': 'vpc_peering_done', - 'Object Storage Encrypted': 'object_storage_encrypted', - 'Table Access Control Enabled':'table_access_control_enabled'} -#SET & GET SAT check configurations for the workspace +# Workspace Level SAT Check Configuration +params = { + "Analysis Enabled": "analysis_enabled", + "SSO Enabled": "sso_enabled", + "SCIM Enabled": "scim_enabled", + "ANY VPC PEERING": "vpc_peering_done", + "Object Storage Encrypted": "object_storage_encrypted", + "Table Access Control Enabled": "table_access_control_enabled", +} + + +# SET & GET SAT check configurations for the workspace def get_workspace_check_config(): workspace = dbutils.widgets.get("workspaces") - ws_id = workspace.split('_')[-1] + ws_id = workspace.split("_")[-1] - s_sql = ''' + s_sql = """ SELECT analysis_enabled, sso_enabled, scim_enabled, vpc_peering_done, object_storage_encrypted, table_access_control_enabled FROM {analysis_schema_name}.account_workspaces WHERE workspace_id= '{ws_id}' - '''.format(ws_id = ws_id, analysis_schema_name= json_["analysis_schema_name"]) + """.format( + ws_id=ws_id, analysis_schema_name=json_["analysis_schema_name"] + ) get_workspace = spark.sql(s_sql) - check = get_workspace.toPandas().to_dict(orient = 'list') - - analysis_enabled = check['analysis_enabled'][0] - sso_enabled = check['sso_enabled'][0] - scim_enabled = check['scim_enabled'][0] - vpc_peering_done = check['vpc_peering_done'][0] - object_storage_encrypted = check['object_storage_encrypted'][0] - table_access_control_enabled = check['table_access_control_enabled'][0] - - - dbutils.widgets.dropdown("analysis_enabled", str(analysis_enabled), ['False','True'], "b. Analysis Enabled") - dbutils.widgets.dropdown("sso_enabled", str(sso_enabled), ['False','True'], "c. SSO Enabled") - dbutils.widgets.dropdown("scim_enabled", str(scim_enabled), ['False','True'], "d. SCIM Enabled") - dbutils.widgets.dropdown("vpc_peering_done", str(vpc_peering_done), ['False','True'], "e. ANY VPC PEERING") - dbutils.widgets.dropdown("object_storage_encrypted", str(object_storage_encrypted), ['False','True'], "f. Object Storage Encrypted") - dbutils.widgets.dropdown("table_access_control_enabled", str(table_access_control_enabled), ['False','True'], "g. Table Access Control Enabled") - dbutils.widgets.multiselect("apply_setting_to_all_ws_enabled", "", ['','Analysis Enabled', 'SSO Enabled', 'SCIM Enabled', 'ANY VPC PEERING', 'Object Storage Encrypted', 'Table Access Control Enabled'], "i. Apply Setting to all workspaces") - - + check = get_workspace.toPandas().to_dict(orient="list") + + analysis_enabled = check["analysis_enabled"][0] + sso_enabled = check["sso_enabled"][0] + scim_enabled = check["scim_enabled"][0] + vpc_peering_done = check["vpc_peering_done"][0] + object_storage_encrypted = check["object_storage_encrypted"][0] + table_access_control_enabled = check["table_access_control_enabled"][0] + + dbutils.widgets.dropdown( + "analysis_enabled", + str(analysis_enabled), + ["False", "True"], + "b. Analysis Enabled", + ) + dbutils.widgets.dropdown( + "sso_enabled", str(sso_enabled), ["False", "True"], "c. SSO Enabled" + ) + dbutils.widgets.dropdown( + "scim_enabled", str(scim_enabled), ["False", "True"], "d. SCIM Enabled" + ) + dbutils.widgets.dropdown( + "vpc_peering_done", + str(vpc_peering_done), + ["False", "True"], + "e. ANY VPC PEERING", + ) + dbutils.widgets.dropdown( + "object_storage_encrypted", + str(object_storage_encrypted), + ["False", "True"], + "f. Object Storage Encrypted", + ) + dbutils.widgets.dropdown( + "table_access_control_enabled", + str(table_access_control_enabled), + ["False", "True"], + "g. Table Access Control Enabled", + ) + dbutils.widgets.multiselect( + "apply_setting_to_all_ws_enabled", + "", + [ + "", + "Analysis Enabled", + "SSO Enabled", + "SCIM Enabled", + "ANY VPC PEERING", + "Object Storage Encrypted", + "Table Access Control Enabled", + ], + "i. Apply Setting to all workspaces", + ) + + def set_workspace_check_config(): - #Retrieve widget values + # Retrieve widget values workspace = dbutils.widgets.get("workspaces") analysis_enabled = dbutils.widgets.get("analysis_enabled").lower() sso_enabled = dbutils.widgets.get("sso_enabled").lower() scim_enabled = dbutils.widgets.get("scim_enabled").lower() vpc_peering_done = dbutils.widgets.get("vpc_peering_done").lower() object_storage_encrypted = dbutils.widgets.get("object_storage_encrypted").lower() - table_access_control_enabled = dbutils.widgets.get("table_access_control_enabled").lower() - apply_setting_to_all_ws_enabled = dbutils.widgets.get("apply_setting_to_all_ws_enabled") - - ws_id = workspace.split('_')[-1] - - if apply_setting_to_all_ws_enabled == '': - s_sql = ''' + table_access_control_enabled = dbutils.widgets.get( + "table_access_control_enabled" + ).lower() + apply_setting_to_all_ws_enabled = dbutils.widgets.get( + "apply_setting_to_all_ws_enabled" + ) + + ws_id = workspace.split("_")[-1] + + if apply_setting_to_all_ws_enabled == "": + s_sql = """ UPDATE {analysis_schema_name}.account_workspaces SET analysis_enabled = {analysis_enabled}, sso_enabled={sso_enabled}, @@ -162,89 +217,117 @@ def set_workspace_check_config(): object_storage_encrypted = {object_storage_encrypted}, table_access_control_enabled = {table_access_control_enabled} WHERE workspace_id= '{ws_id}' - '''.format(analysis_enabled=analysis_enabled, sso_enabled=sso_enabled, scim_enabled=scim_enabled, - vpc_peering_done=vpc_peering_done, object_storage_encrypted=object_storage_encrypted, - table_access_control_enabled=table_access_control_enabled, ws_id = ws_id, analysis_schema_name= json_["analysis_schema_name"]) + """.format( + analysis_enabled=analysis_enabled, + sso_enabled=sso_enabled, + scim_enabled=scim_enabled, + vpc_peering_done=vpc_peering_done, + object_storage_encrypted=object_storage_encrypted, + table_access_control_enabled=table_access_control_enabled, + ws_id=ws_id, + analysis_schema_name=json_["analysis_schema_name"], + ) print(s_sql) spark.sql(s_sql) else: s_sql = 'UPDATE "{analysis_schema_name}".account_workspaces SET ' - first_param=False + first_param = False for param in params: if param in apply_setting_to_all_ws_enabled: val = params[param] if first_param == True: s_sql = s_sql + "," s_sql = s_sql + val + "=" + eval(params[param]) - first_param=True + first_param = True print(s_sql) - spark.sql(s_sql.format(analysis_schema_name= json_["analysis_schema_name"])) - -#Reset widget values to empty + spark.sql(s_sql.format(analysis_schema_name=json_["analysis_schema_name"])) + + +# Reset widget values to empty def get_all_workspaces(): dbutils.widgets.removeAll() - s_sql = ''' + s_sql = """ SELECT CONCAT_WS('_',workspace_name, workspace_id) AS ws FROM {analysis_schema_name}.account_workspaces - '''.format(analysis_schema_name= json_["analysis_schema_name"]) + """.format( + analysis_schema_name=json_["analysis_schema_name"] + ) all_workspaces = spark.sql(s_sql) - workspaces = all_workspaces.rdd.map(lambda row : row[0]).collect() + workspaces = all_workspaces.rdd.map(lambda row: row[0]).collect() first_ws = str(workspaces[0]) - #Define Driver Widgets - dbutils.widgets.dropdown("workspaces", first_ws, [str(x) for x in workspaces], "a. Workspaces") + # Define Driver Widgets + dbutils.widgets.dropdown( + "workspaces", first_ws, [str(x) for x in workspaces], "a. Workspaces" + ) + # COMMAND ---------- -#Workspace Level SAT Check Configuration -params = {'Analysis Enabled': 'analysis_enabled', - 'SSO Enabled':'sso_enabled', - 'SCIM Enabled': 'scim_enabled', - 'ANY VPC PEERING': 'vpc_peering_done', - 'Object Storage Encrypted': 'object_storage_encrypted', - 'Table Access Control Enabled':'table_access_control_enabled'} -#SET & GET SAT check configurations for the workspace +# Workspace Level SAT Check Configuration +params = { + "Analysis Enabled": "analysis_enabled", + "SSO Enabled": "sso_enabled", + "SCIM Enabled": "scim_enabled", + "ANY VPC PEERING": "vpc_peering_done", + "Object Storage Encrypted": "object_storage_encrypted", + "Table Access Control Enabled": "table_access_control_enabled", +} +# SET & GET SAT check configurations for the workspace import yaml + + def get_workspace_self_assessment_check_config(): prefix = getConfigPath() - userfile = f'{prefix}/self_assessment_checks.yaml' + userfile = f"{prefix}/self_assessment_checks.yaml" # Load the YAML configuration - with open(userfile, 'r') as file: + with open(userfile, "r") as file: all_checks = yaml.safe_load(file) - + # Prepare a list to hold the extracted data checks_data = {} # Iterate over each entry to extract required fields for check in all_checks: - checks_data[check['id']] = { - 'check': check['check'], - 'enabled': check['enabled'] + checks_data[check["id"]] = { + "check": check["check"], + "enabled": check["enabled"], } return checks_data - + + def set_workspace_self_assessment_check_config(checks_data): - #Retrieve widget values - sso_enabled = checks_data.get(18)['enabled'] - scim_enabled = checks_data.get(19)['enabled'] - vpc_peering_done = checks_data.get(28)['enabled'] - object_storage_encrypted = checks_data.get(4)['enabled'] - table_access_control_enabled = checks_data.get(20)['enabled'] - #apply_setting_to_all_ws_enabled = dbutils.widgets.get("apply_setting_to_all_ws_enabled") - - #ws_id = workspace.split('_')[-1] - - - s_sql = ''' + # Retrieve widget values + + cloud_type = getCloudType(hostname) + if cloud_type == "azure" or cloud_type == "gcp" : + sso_enabled = 'true' + else: + sso_enabled = checks_data.get(18)["enabled"] + + scim_enabled = checks_data.get(19)["enabled"] + vpc_peering_done = checks_data.get(28)["enabled"] + object_storage_encrypted = checks_data.get(4)["enabled"] + table_access_control_enabled = checks_data.get(20)["enabled"] + # apply_setting_to_all_ws_enabled = dbutils.widgets.get("apply_setting_to_all_ws_enabled") + + # ws_id = workspace.split('_')[-1] + + s_sql = """ UPDATE {analysis_schema_name}.account_workspaces SET sso_enabled={sso_enabled}, scim_enabled = {scim_enabled}, vpc_peering_done = {vpc_peering_done}, object_storage_encrypted = {object_storage_encrypted}, table_access_control_enabled = {table_access_control_enabled} - '''.format(sso_enabled=sso_enabled, scim_enabled=scim_enabled, - vpc_peering_done=vpc_peering_done, object_storage_encrypted=object_storage_encrypted, - table_access_control_enabled=table_access_control_enabled, analysis_schema_name= json_["analysis_schema_name"]) + """.format( + sso_enabled=sso_enabled, + scim_enabled=scim_enabled, + vpc_peering_done=vpc_peering_done, + object_storage_encrypted=object_storage_encrypted, + table_access_control_enabled=table_access_control_enabled, + analysis_schema_name=json_["analysis_schema_name"], + ) print(s_sql) spark.sql(s_sql) diff --git a/notebooks/export/export_sat_report.py b/notebooks/export/export_sat_report.py index 2b790ffc..e855c687 100644 --- a/notebooks/export/export_sat_report.py +++ b/notebooks/export/export_sat_report.py @@ -5,39 +5,11 @@ # COMMAND ---------- -# DBTITLE 0,Use this notebook to export your SAT findings. Run the notebook and downlod the cell content as a CSV from the Table below -# MAGIC %sql -# MAGIC select -# MAGIC workspace_id, -# MAGIC BP.check_id, -# MAGIC BP.severity, -# MAGIC CASE -# MAGIC WHEN score = 1 THEN 'FAIL' -# MAGIC WHEN score = 0 THEN 'PASS' -# MAGIC END as status, -# MAGIC chk_date as RunDate, -# MAGIC BP.recommendation, -# MAGIC BP.doc_url, -# MAGIC BP.logic, -# MAGIC BP.api, -# MAGIC SC.additional_details -# MAGIC from -# MAGIC ( -# MAGIC select -# MAGIC workspace_id, -# MAGIC max(run_id) as run_id -# MAGIC from -# MAGIC security_analysis.workspace_run_complete -# MAGIC where -# MAGIC completed = true -# MAGIC group by -# MAGIC workspace_id -# MAGIC ) as lastest_run, -# MAGIC security_analysis.security_checks SC, -# MAGIC security_analysis.security_best_practices BP -# MAGIC -# MAGIC Where -# MAGIC sc.run_id = lastest_run.run_id -# MAGIC and sc.workspaceid = lastest_run.workspace_id -# MAGIC and SC.id = BP.id -# MAGIC order by workspace_id +# MAGIC %run ../Utils/initialize + +# COMMAND ---------- + +display( + spark.sql( + f"select workspace_id, BP.check_id, BP.severity, CASE WHEN score = 1 THEN 'FAIL' WHEN score = 0 THEN 'PASS' END as status,chk_date as RunDate, BP.recommendation, BP.doc_url, BP.logic, BP.api,SC.additional_details from (select workspace_id, max(run_id) as run_id from {json_['analysis_schema_name']}.workspace_run_complete where completed = true group by workspace_id ) as lastest_run, {json_['analysis_schema_name']}.security_checks SC, {json_ ['analysis_schema_name']}.security_best_practices BP Where sc.run_id = lastest_run.run_id and sc.workspaceid = lastest_run.workspace_id and SC.id = BP.id order by workspace_id" ) +) diff --git a/notebooks/security_analysis_driver.py b/notebooks/security_analysis_driver.py index 4aae4ea3..5c0c01ae 100644 --- a/notebooks/security_analysis_driver.py +++ b/notebooks/security_analysis_driver.py @@ -1,6 +1,6 @@ # Databricks notebook source # MAGIC %md -# MAGIC **Notebook name:** security_analysis_driver. +# MAGIC **Notebook name:** security_analysis_driver. # MAGIC **Functionality:** Main notebook to analyze and generate report of configured workspaces # MAGIC @@ -18,29 +18,46 @@ # COMMAND ---------- -#replace values for accounts exec -hostname = dbutils.notebook.entry_point.getDbutils().notebook().getContext().apiUrl().getOrElse(None) +# replace values for accounts exec +hostname = ( + dbutils.notebook.entry_point.getDbutils() + .notebook() + .getContext() + .apiUrl() + .getOrElse(None) +) cloud_type = getCloudType(hostname) clusterid = spark.conf.get("spark.databricks.clusterUsageTags.clusterId") -#dont know workspace token yet. -json_.update({'url':hostname, 'workspace_id': 'accounts', 'cloud_type': cloud_type, 'clusterid':clusterid}) +# dont know workspace token yet. +json_.update( + { + "url": hostname, + "workspace_id": "accounts", + "cloud_type": cloud_type, + "clusterid": clusterid, + } +) # COMMAND ---------- -from core.logging_utils import LoggingUtils import logging -LoggingUtils.set_logger_level(LoggingUtils.get_log_level(json_['verbosity'])) + +from core.logging_utils import LoggingUtils + +LoggingUtils.set_logger_level(LoggingUtils.get_log_level(json_["verbosity"])) loggr = LoggingUtils.get_logger() use_parallel_runs = json_.get("use_parallel_runs", False) # COMMAND ---------- -if cloud_type=='gcp': - #refresh account level tokens - gcp_status1 = dbutils.notebook.run('./Setup/gcp/configure_sa_auth_tokens', 3000) - if (gcp_status1 != 'OK'): - loggr.exception('Error Encountered in GCP Step#1', gcp_status1) +if cloud_type == "gcp": + # refresh account level tokens + gcp_status1 = dbutils.notebook.run( + f"{basePath()}/notebooks/Setup/gcp/configure_sa_auth_tokens", 3000 + ) + if gcp_status1 != "OK": + loggr.exception("Error Encountered in GCP Step#1", gcp_status1) dbutils.notebook.exit() @@ -48,7 +65,11 @@ import json -out = dbutils.notebook.run('./Utils/accounts_bootstrap', 300, {"json_":json.dumps(json_)}) +out = dbutils.notebook.run( + f"{basePath()}/notebooks/Utils/accounts_bootstrap", + 300, + {"json_": json.dumps(json_)}, +) loggr.info(out) # COMMAND ---------- @@ -57,8 +78,14 @@ # COMMAND ---------- +load_sat_dasf_mapping() + +# COMMAND ---------- + dfexist = getWorkspaceConfig() -dfexist.filter(dfexist.analysis_enabled==True).createOrReplaceGlobalTempView('all_workspaces') +dfexist.filter(dfexist.analysis_enabled == True).createOrReplaceGlobalTempView( + "all_workspaces" +) # COMMAND ---------- @@ -68,59 +95,102 @@ # COMMAND ---------- -workspacesdf = spark.sql('select * from `global_temp`.`all_workspaces`') +workspacesdf = spark.sql("select * from `global_temp`.`all_workspaces`") display(workspacesdf) workspaces = workspacesdf.collect() if workspaces is None or len(workspaces) == 0: - loggr.info('Workspaes are not configured for analyis, check the workspace_configs.csv and '+json_["analysis_schema_name"]+'.account_workspaces if analysis_enabled flag is enabled to True. Use security_analysis_initializer to auto configure workspaces for analysis. ') - #dbutils.notebook.exit("Unsuccessful analysis.") + loggr.info( + "Workspaes are not configured for analyis, check the workspace_configs.csv and " + + json_["analysis_schema_name"] + + ".account_workspaces if analysis_enabled flag is enabled to True. Use security_analysis_initializer to auto configure workspaces for analysis. " + ) + # dbutils.notebook.exit("Unsuccessful analysis.") # COMMAND ---------- + def renewWorkspaceTokens(workspace_id): - if cloud_type=='gcp': - #refesh workspace level tokens if PAT tokens are not used as the temp tokens expire in 10 hours - gcp_status2 = dbutils.notebook.run('./Setup/gcp/configure_tokens_for_worksaces', 3000, {"workspace_id":workspace_id}) - if (gcp_status2 != 'OK'): - loggr.exception('Error Encountered in GCP Step#2', gcp_status2) - dbutils.notebook.exit() + if cloud_type == "gcp": + # refesh workspace level tokens if PAT tokens are not used as the temp tokens expire in 10 hours + gcp_status2 = dbutils.notebook.run( + f"{basePath()}/notebooks/Setup/gcp/configure_tokens_for_worksaces", + 3000, + {"workspace_id": workspace_id}, + ) + if gcp_status2 != "OK": + loggr.exception("Error Encountered in GCP Step#2", gcp_status2) + dbutils.notebook.exit() # COMMAND ---------- -insertNewBatchRun() #common batch number for each run +insertNewBatchRun() # common batch number for each run + + def processWorkspace(wsrow): import json - hostname = 'https://' + wsrow.deployment_url + + hostname = "https://" + wsrow.deployment_url cloud_type = getCloudType(hostname) workspace_id = wsrow.workspace_id sso = wsrow.sso_enabled scim = wsrow.scim_enabled vpc_peering_done = wsrow.vpc_peering_done - object_storage_encrypted = wsrow.object_storage_encrypted + object_storage_encrypted = wsrow.object_storage_encrypted table_access_control_enabled = wsrow.table_access_control_enabled clusterid = spark.conf.get("spark.databricks.clusterUsageTags.clusterId") ws_json = dict(json_) - ws_json.update({"sso":sso, "scim":scim,"object_storage_encryption":object_storage_encrypted, "vpc_peering":vpc_peering_done,"table_access_control_enabled":table_access_control_enabled, 'url':hostname, 'workspace_id': workspace_id, 'cloud_type': cloud_type, 'clusterid':clusterid}) + ws_json.update( + { + "sso": sso, + "scim": scim, + "object_storage_encryption": object_storage_encrypted, + "vpc_peering": vpc_peering_done, + "table_access_control_enabled": table_access_control_enabled, + "url": hostname, + "workspace_id": workspace_id, + "cloud_type": cloud_type, + "clusterid": clusterid, + } + ) loggr.info(ws_json) - retstr = dbutils.notebook.run('./Utils/workspace_bootstrap', 3000, {"json_":json.dumps(ws_json)}) + retstr = dbutils.notebook.run( + f"{basePath()}/notebooks/Utils/workspace_bootstrap", + 3000, + {"json_": json.dumps(ws_json)}, + ) if "Completed SAT" not in retstr: - raise Exception('Workspace Bootstrap failed. Skipping workspace analysis') + raise Exception("Workspace Bootstrap failed. Skipping workspace analysis") else: - dbutils.notebook.run('./Includes/workspace_analysis', 3000, {"json_":json.dumps(ws_json)}) - dbutils.notebook.run('./Includes/workspace_stats', 1000, {"json_":json.dumps(ws_json)}) - dbutils.notebook.run('./Includes/workspace_settings', 3000, {"json_":json.dumps(ws_json)}) + dbutils.notebook.run( + f"{basePath()}/notebooks/Includes/workspace_analysis", + 3000, + {"json_": json.dumps(ws_json)}, + ) + dbutils.notebook.run( + f"{basePath()}/notebooks/Includes/workspace_stats", + 1000, + {"json_": json.dumps(ws_json)}, + ) + dbutils.notebook.run( + f"{basePath()}/notebooks/Includes/workspace_settings", + 3000, + {"json_": json.dumps(ws_json)}, + ) + # COMMAND ---------- from concurrent.futures import ThreadPoolExecutor + def combine(ws): renewWorkspaceTokens(ws.workspace_id) processWorkspace(ws) notifyworkspaceCompleted(ws.workspace_id, True) + if use_parallel_runs == True: loggr.info("Running in parallel") with ThreadPoolExecutor(max_workers=4) as executor: @@ -130,7 +200,7 @@ def combine(ws): print(r) except Exception as e: loggr.info(e) -else: +else: loggr.info("Running in sequence") for ws in workspaces: try: @@ -144,9 +214,17 @@ def combine(ws): # COMMAND ---------- -display(spark.sql(f'select * from {json_["analysis_schema_name"]}.security_checks order by run_id desc, workspaceid asc, check_time asc')) +display( + spark.sql( + f'select * from {json_["analysis_schema_name"]}.security_checks order by run_id desc, workspaceid asc, check_time asc' + ) +) # COMMAND ---------- -display(spark.sql(f'select * from {json_["analysis_schema_name"]}.workspace_run_complete order by run_id desc')) +display( + spark.sql( + f'select * from {json_["analysis_schema_name"]}.workspace_run_complete order by run_id desc' + ) +) diff --git a/notebooks/security_analysis_initializer.py b/notebooks/security_analysis_initializer.py index 3dcef57e..1e1d4863 100644 --- a/notebooks/security_analysis_initializer.py +++ b/notebooks/security_analysis_initializer.py @@ -1,6 +1,6 @@ # Databricks notebook source # MAGIC %md -# MAGIC **Notebook name:** security_analysis_initializer. +# MAGIC **Notebook name:** security_analysis_initializer. # MAGIC **Functionality:** Main notebook to initialize setup of SAT # MAGIC @@ -19,50 +19,75 @@ # COMMAND ---------- from core.logging_utils import LoggingUtils -LoggingUtils.set_logger_level(LoggingUtils.get_log_level(json_['verbosity'])) + +LoggingUtils.set_logger_level(LoggingUtils.get_log_level(json_["verbosity"])) loggr = LoggingUtils.get_logger() # COMMAND ---------- -hostname = dbutils.notebook.entry_point.getDbutils().notebook().getContext().apiUrl().getOrElse(None) +hostname = ( + dbutils.notebook.entry_point.getDbutils() + .notebook() + .getContext() + .apiUrl() + .getOrElse(None) +) cloud_type = getCloudType(hostname) # COMMAND ---------- -if cloud_type=='gcp': - #generate account level tokens for GCP for connection - gcp_status1 = dbutils.notebook.run('./Setup/gcp/configure_sa_auth_tokens', 3000) - if (gcp_status1 != 'OK'): - loggr.exception('Error Encountered in GCP Step#1', gcp_status1) +if cloud_type == "gcp": + # generate account level tokens for GCP for connection + gcp_status1 = dbutils.notebook.run( + f"{basePath()}/notebooks/Setup/gcp/configure_sa_auth_tokens", 3000 + ) + if gcp_status1 != "OK": + loggr.exception("Error Encountered in GCP Step#1", gcp_status1) dbutils.notebook.exit() - # COMMAND ---------- -status1 = dbutils.notebook.run('./Setup/1. list_account_workspaces_to_conf_file', 3000) -if (status1 != 'OK'): - loggr.exception('Error Encountered in Step#1', status1) +status1 = dbutils.notebook.run( + f"{basePath()}/notebooks/Setup/1. list_account_workspaces_to_conf_file", 3000 +) +if status1 != "OK": + loggr.exception("Error Encountered in Step#1", status1) dbutils.notebook.exit() -status3 = dbutils.notebook.run('./Setup/3. test_connections', 12000) -if (status3 != 'OK'): - loggr.exception('Error Encountered in Step#3', status3) +status3 = dbutils.notebook.run( + f"{basePath()}/notebooks/Setup/3. test_connections", 12000 +) +if status3 != "OK": + loggr.exception("Error Encountered in Step#3", status3) dbutils.notebook.exit() - -status4 = dbutils.notebook.run('./Setup/4. enable_workspaces_for_sat', 3000) -if (status4 != 'OK'): - loggr.exception('Error Encountered in Step#4', status4) + +status4 = dbutils.notebook.run( + f"{basePath()}/notebooks/Setup/4. enable_workspaces_for_sat", 3000 +) +if status4 != "OK": + loggr.exception("Error Encountered in Step#4", status4) dbutils.notebook.exit() - -status5 = dbutils.notebook.run('./Setup/5. import_dashboard_template', 3000) -if (status5 != 'OK'): - loggr.exception('Error Encountered in Step#5', status5) + +status5 = dbutils.notebook.run( + f"{basePath()}/notebooks/Setup/5. import_dashboard_template", 3000 +) +if status5 != "OK": + loggr.exception("Error Encountered in Step#5", status5) dbutils.notebook.exit() - -status6 = dbutils.notebook.run('./Setup/6. configure_alerts_template', 3000) -if (status6 != 'OK'): - loggr.exception('Error Encountered in Step#6', status6) + +status5_1 = dbutils.notebook.run( + f"{basePath()}/notebooks/Setup/5. import_dashboard_template_lakeview", 3000 +) +if status5_1 != "OK": + loggr.exception("Error Encountered in Step#5_1", status5_1) + dbutils.notebook.exit() + +status6 = dbutils.notebook.run( + f"{basePath()}/notebooks/Setup/6. configure_alerts_template", 3000 +) +if status6 != "OK": + loggr.exception("Error Encountered in Step#6", status6) dbutils.notebook.exit() # COMMAND ---------- @@ -74,12 +99,9 @@ # DBTITLE 0,%md Running Manual SAT Checks here (/notebooks/setup/update_workspace_configurations) -status9 = dbutils.notebook.run('./Setup/9. self_assess_workspace_configuration', 3000) +status9 = dbutils.notebook.run( + f"{basePath()}/notebooks/Setup/9. self_assess_workspace_configuration", 3000 + ) if (status9 != 'OK'): loggr.exception('Error Encountered in Step#9', status9) dbutils.notebook.exit() - -# COMMAND ---------- - -# MAGIC %md -# MAGIC Running Manual SAT Checks here (/notebooks/setup/update_workspace_configurations) diff --git a/terraform/aws/TERRAFORM_AWS.md b/terraform/aws/TERRAFORM_AWS.md index de16ea85..9653625e 100644 --- a/terraform/aws/TERRAFORM_AWS.md +++ b/terraform/aws/TERRAFORM_AWS.md @@ -15,9 +15,9 @@ Step 4: Change Directories cd security-analysis-tool/terraform// ``` -Step 5: Set values in `teraform.tfvars` file +Step 5: Set values in `terraform.tfvars` file -Using any editor set the values in the `teraform.tfvars` file. The descriptions of all the variables are located in the variables.tf file. Once the variables are set you are ready to run Terraform. +Using any editor set the values in the `terraform.tfvars` file. The descriptions of all the variables are located in the variables.tf file. Once the variables are set you are ready to run Terraform. Further Documentation for some of the variables: @@ -73,6 +73,6 @@ Supplemental Documentation: Additional Considerations: -Your jobs may fail if there was a pre-existing secret scope named sat_scope when you run terraform apply. To remedy this, you will need to change the name of your secret scope in secrets.tf, re-run terraform apply, and then navigate to /Repos//security-analysis-tool.git/notebooks/Utils/initialize and change the secret scope name in 6 places (3 times in CMD 4 and 3 times in CMD 5). You then can re-run your failed jobs. +Your jobs may fail if there was a pre-existing secret scope named sat_scope when you run terraform apply. To remedy this, you will need to change the name of your secret scope in secrets.tf, re-run terraform apply, and then navigate to Workspace -> Applications -> SAT-TF ->/notebooks/Utils/initialize and change the secret scope name in 6 places (3 times in CMD 4 and 3 times in CMD 5). You then can re-run your failed jobs. Congratulations!!! [Please review the setup documentation for the instructions on usage, FAQs and general understanding of SAT setup](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/docs/setup.md) diff --git a/terraform/aws/terraform.tfvars b/terraform/aws/terraform.tfvars index 28d468df..bcdf0def 100644 --- a/terraform/aws/terraform.tfvars +++ b/terraform/aws/terraform.tfvars @@ -5,6 +5,6 @@ account_console_id = "" ### AWS Specific Variables -account_user = "" -account_pass = "" +client_id = "" +client_secret = "" diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index dc1744c5..d1a4ee3d 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -33,27 +33,29 @@ variable "sqlw_id" { variable "account_user" { description = "Account Console Username" type = string + default = " " } variable "account_pass" { description = "Account Console Password" type = string + default = " " } variable "use_sp_auth" { description = "Authenticate with Service Principal OAuth tokens instead of user and password" - type = bool - default = false + type = bool + default = true } variable "client_id" { description = "Service Principal Application (client) ID" - type = string - default = "value" + type = string + default = "value" } variable "client_secret" { description = "SP Secret" - type = string - default = "value" + type = string + default = "value" } diff --git a/terraform/azure/TERRAFORM_Azure.md b/terraform/azure/TERRAFORM_Azure.md index 28c2b478..08ee7e13 100644 --- a/terraform/azure/TERRAFORM_Azure.md +++ b/terraform/azure/TERRAFORM_Azure.md @@ -86,6 +86,6 @@ Supplemental Documentation: Additional Considerations: -Your jobs may fail if there was a pre-existing secret scope named sat_scope when you run terraform apply. To remedy this, you will need to change the name of your secret scope in secrets.tf, re-run terraform apply, and then navigate to /Repos//security-analysis-tool.git/notebooks/Utils/initialize and change the secret scope name in 6 places (3 times in CMD 4 and 3 times in CMD 5). You then can re-run your failed jobs. +Your jobs may fail if there was a pre-existing secret scope named sat_scope when you run terraform apply. To remedy this, you will need to change the name of your secret scope in secrets.tf, re-run terraform apply, and then navigate to Workspace -> Applications -> SAT-TF /notebooks/Utils/initialize and change the secret scope name in 6 places (3 times in CMD 4 and 3 times in CMD 5). You then can re-run your failed jobs. Congratulations!!! [Please review the setup documentation for the instructions on usage, FAQs and general understanding of SAT setup](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/docs/setup.md) diff --git a/terraform/common/jobs.tf b/terraform/common/jobs.tf index 047a8108..d4a67b9b 100644 --- a/terraform/common/jobs.tf +++ b/terraform/common/jobs.tf @@ -1,53 +1,66 @@ resource "databricks_job" "initializer" { name = "SAT Initializer Notebook (one-time)" - new_cluster { - num_workers = 5 - spark_version = data.databricks_spark_version.latest_lts.id - node_type_id = data.databricks_node_type.smallest.id - runtime_engine = "PHOTON" - dynamic "gcp_attributes" { - for_each = var.gcp_impersonate_service_account == "" ? [] : [var.gcp_impersonate_service_account] - content { - google_service_account = var.gcp_impersonate_service_account + job_cluster { + job_cluster_key = "job_cluster" + new_cluster { + num_workers = 5 + spark_version = data.databricks_spark_version.latest_lts.id + node_type_id = data.databricks_node_type.smallest.id + runtime_engine = "PHOTON" + dynamic "gcp_attributes" { + for_each = var.gcp_impersonate_service_account == "" ? [] : [var.gcp_impersonate_service_account] + content { + google_service_account = var.gcp_impersonate_service_account + } } } } - library { - pypi { - package = "dbl-sat-sdk" + task { + task_key = "Initializer" + job_cluster_key = "job_cluster" + library { + pypi { + package = "dbl-sat-sdk" + } + } + notebook_task { + notebook_path = "${databricks_repo.security_analysis_tool.workspace_path}/notebooks/security_analysis_initializer" } - } - - notebook_task { - notebook_path = "${databricks_repo.security_analysis_tool.path}/notebooks/security_analysis_initializer" } } resource "databricks_job" "driver" { name = "SAT Driver Notebook" - new_cluster { - num_workers = 5 - spark_version = data.databricks_spark_version.latest_lts.id - node_type_id = data.databricks_node_type.smallest.id - runtime_engine = "PHOTON" - dynamic "gcp_attributes" { - for_each = var.gcp_impersonate_service_account == "" ? [] : [var.gcp_impersonate_service_account] - content { - google_service_account = var.gcp_impersonate_service_account + job_cluster { + job_cluster_key = "job_cluster" + new_cluster { + num_workers = 5 + spark_version = data.databricks_spark_version.latest_lts.id + node_type_id = data.databricks_node_type.smallest.id + runtime_engine = "PHOTON" + dynamic "gcp_attributes" { + for_each = var.gcp_impersonate_service_account == "" ? [] : [var.gcp_impersonate_service_account] + content { + google_service_account = var.gcp_impersonate_service_account + } } } } - library { - pypi { - package = "dbl-sat-sdk" - } - } - notebook_task { - notebook_path = "${databricks_repo.security_analysis_tool.path}/notebooks/security_analysis_driver" + task { + task_key = "Driver" + job_cluster_key = "job_cluster" + library { + pypi { + package = "dbl-sat-sdk" + } + } + notebook_task { + notebook_path = "${databricks_repo.security_analysis_tool.workspace_path}/notebooks/security_analysis_driver" + } } schedule { diff --git a/terraform/common/repo.tf b/terraform/common/repo.tf index 87c1dc59..7b211492 100644 --- a/terraform/common/repo.tf +++ b/terraform/common/repo.tf @@ -1,5 +1,7 @@ #Make sure Files in Repos option is enabled in Workspace Admin Console > Workspace Settings resource "databricks_repo" "security_analysis_tool" { - url = "https://github.com/databricks-industry-solutions/security-analysis-tool.git" + url = "https://github.com/databricks-industry-solutions/security-analysis-tool.git" + branch = "main" + path = "/Workspace/Applications/SAT_TF" } diff --git a/terraform/common/secrets.tf b/terraform/common/secrets.tf index 40ef89d5..a27fbd71 100644 --- a/terraform/common/secrets.tf +++ b/terraform/common/secrets.tf @@ -9,7 +9,7 @@ resource "databricks_secret" "user_email" { } resource "databricks_token" "pat" { - lifetime_seconds = 86400 * 365 + lifetime_seconds = 86400 * 90 comment = "Security Analysis Tool" } diff --git a/terraform/gcp/TERRAFORM_GCP.md b/terraform/gcp/TERRAFORM_GCP.md index 9b9d2183..4afe5e5d 100644 --- a/terraform/gcp/TERRAFORM_GCP.md +++ b/terraform/gcp/TERRAFORM_GCP.md @@ -76,6 +76,6 @@ Supplemental Documentation: Additional Considerations: -Your jobs may fail if there was a pre-existing secret scope named `sat_scope` when you run `terraform apply`. To remedy this, you will need to change the name of your secret scope in `secrets.tf`, re-run terraform apply, and then navigate to `/Repos//security-analysis-tool.git/notebooks/Utils/initialize` and change the secret scope name in 6 places (3 times in CMD 4 and 3 times in CMD 5). You then can re-run your failed jobs. +Your jobs may fail if there was a pre-existing secret scope named `sat_scope` when you run `terraform apply`. To remedy this, you will need to change the name of your secret scope in `secrets.tf`, re-run terraform apply, and then navigate to `Workspace -> Applications -> SAT-TF /notebooks/Utils/initialize` and change the secret scope name in 6 places (3 times in CMD 4 and 3 times in CMD 5). You then can re-run your failed jobs. Congratulations!!! [Please review the setup documentation for the instructions on usage, FAQs and general understanding of SAT setup](https://github.com/databricks-industry-solutions/security-analysis-tool/blob/main/docs/setup.md)