generated from databricks-industry-solutions/industry-solutions-blueprints
-
Notifications
You must be signed in to change notification settings - Fork 42
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #115 from databricks-industry-solutions/new_sat_setup
New SAT Setup
- Loading branch information
Showing
47 changed files
with
7,724 additions
and
1,116 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| sat_id,dasf_control_id,dasf_control_name | ||
| 1,DASF 33,Manage credentials securely | ||
| 2,DASF 8,Encrypt data at rest | ||
| 2,DASF 30,Encrypt models | ||
| 3,DASF 8,Encrypt data at rest | ||
| 3,DASF 30,Encrypt models | ||
| 4,DASF 8,Encrypt data at rest | ||
| 8,DASF 14,Audit actions performed on datasets | ||
| 8,DASF 55,Monitor Audit logs | ||
| 9,DASF 38,Platform security — vulnerability management | ||
| 10,DASF 38,Platform security — vulnerability management | ||
| 18,DASF 1,SSO with IdP and MFA | ||
| 19,DASF 2,Sync users and groups | ||
| 29,DASF 43,Use access control lists | ||
| 30,DASF 43,Use access control lists | ||
| 31,DASF 43,Use access control lists | ||
| 32,DASF 52,Source code control | ||
| 35,DASF 4,Restrict access using private link | ||
| 37,DASF 3,Restrict access using IP access lists | ||
| 52,DASF 52,Source code control | ||
| 53,DASF 5,Control access to data and other objects | ||
| 53,DASF 16,Secure model features | ||
| 53,DASF 24,Control access to models and model assets | ||
| 53,DASF 43,Use access control lists | ||
| 54,DASF 51,Share data and AI assets securely | ||
| 55,DASF 51,Share data and AI assets securely | ||
| 56,DASF 51,Share data and AI assets securely | ||
| 89,DASF 31,Secure model serving endpoints | ||
| 90,DASF 32,Streamline the usage and management of various large language model (LLM) providers | ||
| 101,DASF 46,Store and retrieve embeddings securely | ||
| 103,DASF 50,Platform compliance | ||
| 104,DASF 53,Third-party library control | ||
| 105,DASF 55,Monitor Audit logs | ||
| 107,DASF 38,Platform security — vulnerability management | ||
| 108,DASF 50,Platform compliance | ||
| 109,DASF 50,Platform compliance |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| { | ||
| "welcome_message": "", | ||
| "properties": { | ||
| "catalog": { | ||
| "type": "string", | ||
| "description": "The catalog for SAT" | ||
| }, | ||
| "cloud": { | ||
| "type": "string", | ||
| "description": "Cloud type" | ||
| }, | ||
| "google_service_account": { | ||
| "type": "string", | ||
| "description": "Google service account" | ||
| }, | ||
| "latest_lts": { | ||
| "type": "string", | ||
| "description": "Latest LTS version" | ||
| }, | ||
| "node_type": { | ||
| "type": "string", | ||
| "description": "Node Type" | ||
| } | ||
| }, | ||
| "success_message": "" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,139 @@ | ||
| # Databricks notebook source | ||
| # MAGIC %md | ||
| # MAGIC **Notebook name:** initialize | ||
| # MAGIC **Functionality:** initializes the necessary configruation values for the rest of the process into a json | ||
|
|
||
| # COMMAND ---------- | ||
|
|
||
| # MAGIC %run ./common | ||
|
|
||
| # COMMAND ---------- | ||
|
|
||
| # replace values for accounts exec | ||
| hostname = ( | ||
| dbutils.notebook.entry_point.getDbutils() | ||
| .notebook() | ||
| .getContext() | ||
| .apiUrl() | ||
| .getOrElse(None) | ||
| ) | ||
| cloud_type = getCloudType(hostname) | ||
|
|
||
| # COMMAND ---------- | ||
|
|
||
| # MAGIC %md | ||
| # MAGIC ##### Modify JSON values | ||
| # MAGIC * **account_id** Account ID. Can get this from the accounts console | ||
| # MAGIC * **sql_warehouse_id** SQL Warehouse ID to import dashboard | ||
| # MAGIC * **verbosity** (optional). debug, info, warning, error, critical | ||
| # MAGIC * **master_name_scope** Secret Scope for Account Name | ||
| # MAGIC * **master_name_key** Secret Key for Account Name | ||
| # MAGIC * **master_pwd_scope** Secret Scope for Account Password | ||
| # MAGIC * **master_pwd_key** Secret Key for Account Password | ||
| # MAGIC * **workspace_pat_scope** Secret Scope for Workspace PAT | ||
| # MAGIC * **workspace_pat_token_prefix** Secret Key prefix for Workspace PAT. Workspace ID will automatically be appended to this per workspace | ||
| # MAGIC * **use_mastercreds** (optional) Use master account credentials for all workspaces | ||
|
|
||
| # COMMAND ---------- | ||
|
|
||
| import json | ||
|
|
||
| json_ = { | ||
| "account_id": dbutils.secrets.get(scope="sat_scope", key="account-console-id"), | ||
| "sql_warehouse_id": dbutils.secrets.get(scope="sat_scope", key="sql-warehouse-id"), | ||
| "analysis_schema_name": "{{.catalog}}.security_analysis", | ||
| "verbosity": "info", | ||
| } | ||
|
|
||
| # COMMAND ---------- | ||
|
|
||
| json_.update( | ||
| { | ||
| "master_name_scope": "sat_scope", | ||
| "master_name_key": "user", | ||
| "master_pwd_scope": "sat_scope", | ||
| "master_pwd_key": "pass", | ||
| "workspace_pat_scope": "sat_scope", | ||
| "workspace_pat_token_prefix": "sat-token", | ||
| "dashboard_id": "317f4809-8d9d-4956-a79a-6eee51412217", | ||
| "dashboard_folder": f"{basePath()}/dashboards/", | ||
| "dashboard_tag": "SAT", | ||
| "use_mastercreds": True, | ||
| "use_parallel_runs": True, | ||
| } | ||
| ) | ||
|
|
||
|
|
||
| # COMMAND ---------- | ||
|
|
||
| # DBTITLE 1,GCP configurations | ||
| if cloud_type == "gcp": | ||
| json_.update( | ||
| { | ||
| "service_account_key_file_path": dbutils.secrets.get( | ||
| scope="sat_scope", key="gs-path-to-json" | ||
| ), | ||
| "impersonate_service_account": dbutils.secrets.get( | ||
| scope="sat_scope", key="impersonate-service-account" | ||
| ), | ||
| "use_mastercreds": False, | ||
| } | ||
| ) | ||
|
|
||
|
|
||
| # COMMAND ---------- | ||
|
|
||
| # DBTITLE 1,Azure configurations | ||
| if cloud_type == "azure": | ||
| json_.update( | ||
| { | ||
| "account_id": "azure", | ||
| "subscription_id": dbutils.secrets.get( | ||
| scope="sat_scope", key="subscription-id" | ||
| ), # Azure subscriptionId | ||
| "tenant_id": dbutils.secrets.get( | ||
| scope="sat_scope", key="tenant-id" | ||
| ), # The Directory (tenant) ID for the application registered in Azure AD. | ||
| "client_id": dbutils.secrets.get( | ||
| scope="sat_scope", key="client-id" | ||
| ), # The Application (client) ID for the application registered in Azure AD. | ||
| "client_secret_key": "client-secret", # The secret generated by AAD during your confidential app registration | ||
| "use_mastercreds": True, | ||
| } | ||
| ) | ||
|
|
||
|
|
||
| # COMMAND ---------- | ||
|
|
||
| # DBTITLE 1,AWS configurations | ||
| if cloud_type == "aws": | ||
| sp_auth = { | ||
| "use_sp_auth": "False", | ||
| "client_id": "", | ||
| "client_secret_key": "client-secret", | ||
| } | ||
| try: | ||
| use_sp_auth = ( | ||
| dbutils.secrets.get(scope="sat_scope", key="use-sp-auth").lower() == "true" | ||
| ) | ||
| if use_sp_auth: | ||
| sp_auth["use_sp_auth"] = "True" | ||
| sp_auth["client_id"] = dbutils.secrets.get( | ||
| scope="sat_scope", key="client-id" | ||
| ) | ||
| except: | ||
| pass | ||
| json_.update(sp_auth) | ||
|
|
||
| # COMMAND ---------- | ||
|
|
||
| create_schema() | ||
| create_security_checks_table() | ||
| create_account_info_table() | ||
| create_account_workspaces_table() | ||
| create_workspace_run_complete_table() | ||
|
|
||
| # COMMAND ---------- | ||
|
|
||
| # Initialize best practices if not already loaded into database | ||
| readBestPracticesConfigsFile() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| bundle: | ||
| name: SAT | ||
|
|
||
| include: | ||
| - resources/*.yml | ||
|
|
||
| targets: | ||
| sat: | ||
| default: true | ||
| mode: production | ||
| workspace: | ||
| host: {{workspace_host}} | ||
| root_path: /Applications/${bundle.name}/ | ||
| run_as: | ||
| {{- if is_service_principal}} | ||
| service_principal_name: {{user_name}} | ||
| {{- else}} | ||
| user_name: {{user_name}} | ||
| {{- end}} |
27 changes: 27 additions & 0 deletions
27
dabs/dabs_template/template/tmp/resources/sat_driver_job.yml.tmpl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| resources: | ||
| jobs: | ||
| sat_driver: | ||
| name: "SAT Driver Notebook" | ||
| schedule: | ||
| quartz_cron_expression: "0 0 8 ? * Mon,Wed,Fri" | ||
| timezone_id: "America/New_York" | ||
| tasks: | ||
| - task_key: "sat_initializer" | ||
| job_cluster_key: job_cluster | ||
| libraries: | ||
| - pypi: | ||
| package: dbl-sat-sdk | ||
| notebook_task: | ||
| notebook_path: "../notebooks/security_analysis_driver.py" | ||
|
|
||
| job_clusters: | ||
| - job_cluster_key: job_cluster | ||
| new_cluster: | ||
| num_workers: 5 | ||
| spark_version: {{.latest_lts}} | ||
| runtime_engine: "PHOTON" | ||
| node_type_id: {{.node_type}} | ||
| {{- if eq .cloud "gcp" }} | ||
| gcp_attributes: | ||
| google_service_account: {{.google_service_account}} | ||
| {{- end }} |
25 changes: 25 additions & 0 deletions
25
dabs/dabs_template/template/tmp/resources/sat_initiliazer_job.yml.tmpl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| resources: | ||
| jobs: | ||
| sat_initializer: | ||
| name: "SAT Initializer Notebook (one-time)" | ||
|
|
||
| tasks: | ||
| - task_key: "sat_initializer" | ||
| job_cluster_key: job_cluster | ||
| libraries: | ||
| - pypi: | ||
| package: dbl-sat-sdk | ||
| notebook_task: | ||
| notebook_path: "../notebooks/security_analysis_initializer.py" | ||
|
|
||
| job_clusters: | ||
| - job_cluster_key: job_cluster | ||
| new_cluster: | ||
| num_workers: 5 | ||
| spark_version: {{.latest_lts}} | ||
| runtime_engine: "PHOTON" | ||
| node_type_id: {{.node_type}} | ||
| {{- if eq .cloud "gcp" }} | ||
| gcp_attributes: | ||
| google_service_account: {{.google_service_account}} | ||
| {{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| import json | ||
| import os | ||
| import subprocess | ||
|
|
||
| from databricks.sdk import WorkspaceClient | ||
| from sat.config import form, generate_secrets | ||
| from sat.utils import cloud_type | ||
|
|
||
|
|
||
| def install(client: WorkspaceClient, answers: dict, profile: str): | ||
| cloud = cloud_type(client) | ||
| generate_secrets(client, answers, cloud) | ||
| config = { | ||
| "catalog": answers.get("catalog", None), | ||
| "cloud": cloud, | ||
| "google_service_account": answers.get("gcp-impersonate-service-account", None), | ||
| "latest_lts": client.clusters.select_spark_version( | ||
| long_term_support=True, | ||
| latest=True, | ||
| ), | ||
| "node_type": client.clusters.select_node_type( | ||
| local_disk=True, | ||
| min_cores=4, | ||
| gb_per_core=8, | ||
| photon_driver_capable=True, | ||
| photon_worker_capable=True, | ||
| ), | ||
| } | ||
|
|
||
| config_file = "tmp_config.json" | ||
| with open(config_file, "w") as fp: | ||
| json.dump(config, fp) | ||
|
|
||
| os.system("clear") | ||
| subprocess.call(f"sh ./setup.sh tmp {profile} {config_file}".split(" ")) | ||
| print("Installation complete.") | ||
| print(f"Review workspace -> {client.config.host}") | ||
|
|
||
|
|
||
| def setup(): | ||
| try: | ||
| client, answers, profile = form() | ||
| install(client, answers, profile) | ||
| except KeyboardInterrupt: | ||
| print("Installation aborted.") | ||
| except Exception as e: | ||
| print(f"An error occurred: {e}") | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| os.system("clear") | ||
| setup() |
Oops, something went wrong.