diff --git a/.envrc b/.envrc index 98e4371065..86bedd8e86 100644 --- a/.envrc +++ b/.envrc @@ -11,7 +11,7 @@ export DEV_ENV_NAME_PREFIX="${TF_VAR_name_prefix}" export TARGET_BIGQUERY_CREDENTIALS_JSON="$(echo $TF_VAR_sa_creds | base64 -d)" export TARGET_BIGQUERY_DATASET="${USER}_dataset" -export TARGET_BIGQUERY_LOCATION="EU" +export TARGET_BIGQUERY_LOCATION="US" export DBT_BIGQUERY_DATASET="dataform_${USER}" export DBT_BIGQUERY_KEYFILE="/lana/keyfile.json" echo $TARGET_BIGQUERY_CREDENTIALS_JSON > meltano/keyfile.json diff --git a/docker-compose.yml b/docker-compose.yml index 160b5dca43..370a46cb7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -109,3 +109,5 @@ services: - TARGET_BIGQUERY_CREDENTIALS_JSON=${TARGET_BIGQUERY_CREDENTIALS_JSON} - TARGET_BIGQUERY_DATASET=${TARGET_BIGQUERY_DATASET} - TARGET_BIGQUERY_LOCATION=${TARGET_BIGQUERY_LOCATION} + - DBT_BIGQUERY_DATASET=${DBT_BIGQUERY_DATASET} + - DBT_BIGQUERY_KEYFILE=${DBT_BIGQUERY_KEYFILE} diff --git a/tf/bq-setup/bq.tf b/tf/bq-setup/bq.tf index c82d456d41..12e86d76f4 100644 --- a/tf/bq-setup/bq.tf +++ b/tf/bq-setup/bq.tf @@ -67,3 +67,26 @@ resource "google_bigquery_dataset_iam_member" "dataform_assertions_additional_ow role = "roles/bigquery.dataOwner" member = "user:${each.value}" } + +resource "google_bigquery_dataset" "dbt" { + project = local.gcp_project + dataset_id = local.dbt_dataset_name + friendly_name = "${local.name_prefix} dbt" + description = "dbt for ${local.name_prefix}" + location = local.dbt_location +} + +resource "google_bigquery_dataset_iam_member" "dbt_owner" { + project = local.gcp_project + dataset_id = google_bigquery_dataset.dbt.dataset_id + role = "roles/bigquery.dataOwner" + member = "serviceAccount:${google_service_account.bq_access_sa.email}" +} + +resource "google_bigquery_dataset_iam_member" "dbt_additional_owners" { + for_each = toset(local.additional_owners) + project = local.gcp_project + dataset_id = google_bigquery_dataset.dbt.dataset_id + role = "roles/bigquery.dataOwner" + member = "user:${each.value}" +} diff --git a/tf/bq-setup/main.tf b/tf/bq-setup/main.tf index 94f157e33e..35b8b9ebf9 100644 --- a/tf/bq-setup/main.tf +++ b/tf/bq-setup/main.tf @@ -41,6 +41,8 @@ locals { dataform_git_commitish = var.dataform_git_commitish != "" ? var.dataform_git_commitish : "${var.name_prefix}-dataform" + dbt_dataset_name = replace("dbt_${local.name_prefix}", "-", "_") + dbt_location = "US" dataform_dataset_name = replace("dataform_${local.name_prefix}", "-", "_") dataform_assertions_dataset_name = replace("dataform_assertions_${local.name_prefix}", "-", "_") dataform_repo_name = "${local.name_prefix}-repo"