diff --git a/Cargo.lock b/Cargo.lock index 14354aea50d0b..17bf3b4d4f7cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4211,6 +4211,7 @@ dependencies = [ "mz-build-info", "mz-catalog", "mz-ccsr", + "mz-cloud-provider", "mz-cloud-resources", "mz-cluster-client", "mz-compute-client", @@ -4642,6 +4643,13 @@ dependencies = [ "workspace-hack", ] +[[package]] +name = "mz-cloud-provider" +version = "0.1.0" +dependencies = [ + "workspace-hack", +] + [[package]] name = "mz-cloud-resources" version = "0.0.0" @@ -5067,6 +5075,7 @@ dependencies = [ "mz-orchestrator-kubernetes", "mz-orchestrator-process", "mz-orchestrator-tracing", + "mz-orchestratord", "mz-ore", "mz-persist-client", "mz-pgrepr", @@ -5676,13 +5685,12 @@ dependencies = [ "mz-alloc", "mz-alloc-default", "mz-build-info", + "mz-cloud-provider", "mz-cloud-resources", - "mz-environmentd", "mz-orchestrator-kubernetes", "mz-orchestrator-tracing", "mz-ore", "mz-prof-http", - "mz-sql", "mz-tls-util", "prometheus", "rand", @@ -6431,6 +6439,7 @@ dependencies = [ "mz-audit-log", "mz-build-info", "mz-ccsr", + "mz-cloud-provider", "mz-cloud-resources", "mz-controller-types", "mz-dyncfg", diff --git a/Cargo.toml b/Cargo.toml index c88ccc274304d..3d57eacfd780d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "src/catalog-debug", "src/catalog-protos", "src/cloud-api", + "src/cloud-provider", "src/cluster", "src/clusterd", "src/cluster-client", @@ -134,6 +135,7 @@ default-members = [ "src/catalog-protos", "src/ccsr", "src/cloud-api", + "src/cloud-provider", "src/cloud-resources", "src/cluster", "src/cluster-client", diff --git a/WORKSPACE b/WORKSPACE index d7870c2e72867..87ecf3ee77cac 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -566,6 +566,7 @@ crates_repository( "//:src/catalog/Cargo.toml", "//:src/ccsr/Cargo.toml", "//:src/cloud-api/Cargo.toml", + "//:src/cloud-provider/Cargo.toml", "//:src/cloud-resources/Cargo.toml", "//:src/cluster-client/Cargo.toml", "//:src/cluster/Cargo.toml", diff --git a/src/adapter/BUILD.bazel b/src/adapter/BUILD.bazel index 83efca8759f02..4dc9c7a17a549 100644 --- a/src/adapter/BUILD.bazel +++ b/src/adapter/BUILD.bazel @@ -35,6 +35,7 @@ rust_library( "//src/build-info:mz_build_info", "//src/catalog:mz_catalog", "//src/ccsr:mz_ccsr", + "//src/cloud-provider:mz_cloud_provider", "//src/cloud-resources:mz_cloud_resources", "//src/cluster-client:mz_cluster_client", "//src/compute-client:mz_compute_client", @@ -104,6 +105,7 @@ rust_test( "//src/build-info:mz_build_info", "//src/catalog:mz_catalog", "//src/ccsr:mz_ccsr", + "//src/cloud-provider:mz_cloud_provider", "//src/cloud-resources:mz_cloud_resources", "//src/cluster-client:mz_cluster_client", "//src/compute-client:mz_compute_client", @@ -153,6 +155,7 @@ rust_doc_test( "//src/build-info:mz_build_info", "//src/catalog:mz_catalog", "//src/ccsr:mz_ccsr", + "//src/cloud-provider:mz_cloud_provider", "//src/cloud-resources:mz_cloud_resources", "//src/cluster-client:mz_cluster_client", "//src/compute-client:mz_compute_client", @@ -222,6 +225,7 @@ rust_test( "//src/build-info:mz_build_info", "//src/catalog:mz_catalog", "//src/ccsr:mz_ccsr", + "//src/cloud-provider:mz_cloud_provider", "//src/cloud-resources:mz_cloud_resources", "//src/cluster-client:mz_cluster_client", "//src/compute-client:mz_compute_client", @@ -291,6 +295,7 @@ rust_test( "//src/build-info:mz_build_info", "//src/catalog:mz_catalog", "//src/ccsr:mz_ccsr", + "//src/cloud-provider:mz_cloud_provider", "//src/cloud-resources:mz_cloud_resources", "//src/cluster-client:mz_cluster_client", "//src/compute-client:mz_compute_client", @@ -360,6 +365,7 @@ rust_test( "//src/build-info:mz_build_info", "//src/catalog:mz_catalog", "//src/ccsr:mz_ccsr", + "//src/cloud-provider:mz_cloud_provider", "//src/cloud-resources:mz_cloud_resources", "//src/cluster-client:mz_cluster_client", "//src/compute-client:mz_compute_client", diff --git a/src/adapter/Cargo.toml b/src/adapter/Cargo.toml index 2329330a336f4..3d6815bd1029c 100644 --- a/src/adapter/Cargo.toml +++ b/src/adapter/Cargo.toml @@ -37,6 +37,7 @@ mz-audit-log = { path = "../audit-log" } mz-build-info = { path = "../build-info" } mz-catalog = { path = "../catalog" } mz-ccsr = { path = "../ccsr" } +mz-cloud-provider = { path = "../cloud-provider", default-features = false } mz-cloud-resources = { path = "../cloud-resources" } mz-cluster-client = { path = "../cluster-client" } mz-compute-client = { path = "../compute-client" } diff --git a/src/adapter/src/config/frontend.rs b/src/adapter/src/config/frontend.rs index a4a9ecaab915e..5c5f3c8dc0451 100644 --- a/src/adapter/src/config/frontend.rs +++ b/src/adapter/src/config/frontend.rs @@ -14,8 +14,9 @@ use std::time::Duration; use derivative::Derivative; use launchdarkly_server_sdk as ld; use mz_build_info::BuildInfo; +use mz_cloud_provider::CloudProvider; use mz_ore::now::NowFn; -use mz_sql::catalog::{CloudProvider, EnvironmentId}; +use mz_sql::catalog::EnvironmentId; use tokio::time; use crate::config::{Metrics, SynchronizedParameters, SystemParameterSyncConfig}; diff --git a/src/cloud-provider/BUILD.bazel b/src/cloud-provider/BUILD.bazel new file mode 100644 index 0000000000000..25b649cb00f1a --- /dev/null +++ b/src/cloud-provider/BUILD.bazel @@ -0,0 +1,73 @@ +# Code generated by cargo-gazelle DO NOT EDIT + +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +load("@crates_io//:defs.bzl", "aliases", "all_crate_deps") +load("@rules_rust//rust:defs.bzl", "rust_doc_test", "rust_library", "rust_test") + +package(default_visibility = ["//visibility:public"]) + +rust_library( + name = "mz_cloud_provider", + srcs = glob(["src/**/*.rs"]), + aliases = aliases( + normal = True, + proc_macro = True, + ), + compile_data = [], + crate_features = ["default"], + data = [], + proc_macro_deps = [] + all_crate_deps(proc_macro = True), + rustc_env = {}, + rustc_flags = [], + version = "0.1.0", + deps = [] + all_crate_deps(normal = True), +) + +alias( + name = "cloud-provider", + actual = "mz_cloud_provider", +) + +rust_test( + name = "mz_cloud_provider_lib_tests", + size = "medium", + aliases = aliases( + normal = True, + normal_dev = True, + proc_macro = True, + proc_macro_dev = True, + ), + compile_data = [], + crate = ":mz_cloud_provider", + crate_features = ["default"], + data = [], + env = {}, + proc_macro_deps = [] + all_crate_deps( + proc_macro = True, + proc_macro_dev = True, + ), + rustc_env = {}, + rustc_flags = [], + version = "0.1.0", + deps = [] + all_crate_deps( + normal = True, + normal_dev = True, + ), +) + +rust_doc_test( + name = "mz_cloud_provider_doc_test", + crate = ":mz_cloud_provider", + deps = [] + all_crate_deps( + normal = True, + normal_dev = True, + ), +) diff --git a/src/cloud-provider/Cargo.toml b/src/cloud-provider/Cargo.toml new file mode 100644 index 0000000000000..74c80c582f16f --- /dev/null +++ b/src/cloud-provider/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "mz-cloud-provider" +description = "Representation of a cloud provider" +version = "0.1.0" +authors = ["Materialize, Inc."] +license = "proprietary" +edition.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true } + +[features] +default = ["workspace-hack"] + +[lints] +workspace = true diff --git a/src/cloud-provider/src/lib.rs b/src/cloud-provider/src/lib.rs new file mode 100644 index 0000000000000..d9f889a611a3a --- /dev/null +++ b/src/cloud-provider/src/lib.rs @@ -0,0 +1,85 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::fmt; +use std::str::FromStr; + +/// Identifies a supported cloud provider. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CloudProvider { + /// A pseudo-provider value used by local development environments. + Local, + /// A pseudo-provider value used by Docker. + Docker, + /// A deprecated psuedo-provider value used by mzcompose. + // TODO(benesch): remove once v0.39 ships. + MzCompose, + /// A pseudo-provider value used by cloudtest. + Cloudtest, + /// Amazon Web Services. + Aws, + /// Google Cloud Platform + Gcp, + /// Microsoft Azure + Azure, + /// Other generic cloud provider + Generic, +} + +impl CloudProvider { + /// Returns true if this provider actually runs in the cloud + pub fn is_cloud(&self) -> bool { + matches!(self, Self::Aws | Self::Gcp | Self::Azure | Self::Generic) + } +} + +impl fmt::Display for CloudProvider { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + CloudProvider::Local => f.write_str("local"), + CloudProvider::Docker => f.write_str("docker"), + CloudProvider::MzCompose => f.write_str("mzcompose"), + CloudProvider::Cloudtest => f.write_str("cloudtest"), + CloudProvider::Aws => f.write_str("aws"), + CloudProvider::Gcp => f.write_str("gcp"), + CloudProvider::Azure => f.write_str("azure"), + CloudProvider::Generic => f.write_str("generic"), + } + } +} + +impl FromStr for CloudProvider { + type Err = InvalidCloudProviderError; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_ref() { + "local" => Ok(CloudProvider::Local), + "docker" => Ok(CloudProvider::Docker), + "mzcompose" => Ok(CloudProvider::MzCompose), + "cloudtest" => Ok(CloudProvider::Cloudtest), + "aws" => Ok(CloudProvider::Aws), + "gcp" => Ok(CloudProvider::Gcp), + "azure" => Ok(CloudProvider::Azure), + "generic" => Ok(CloudProvider::Generic), + _ => Err(InvalidCloudProviderError), + } + } +} + +/// The error type for [`CloudProvider::from_str`]. +#[derive(Debug, Clone, PartialEq)] +pub struct InvalidCloudProviderError; + +impl fmt::Display for InvalidCloudProviderError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("invalid cloud provider") + } +} + +impl std::error::Error for InvalidCloudProviderError {} diff --git a/src/environmentd/BUILD.bazel b/src/environmentd/BUILD.bazel index adaf9e4f2852a..71f9f5119e560 100644 --- a/src/environmentd/BUILD.bazel +++ b/src/environmentd/BUILD.bazel @@ -66,6 +66,7 @@ rust_library( "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-process:mz_orchestrator_process", "//src/orchestrator-tracing:mz_orchestrator_tracing", + "//src/orchestratord:mz_orchestratord", "//src/ore:mz_ore", "//src/persist-client:mz_persist_client", "//src/pgrepr:mz_pgrepr", @@ -139,6 +140,7 @@ rust_test( "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-process:mz_orchestrator_process", "//src/orchestrator-tracing:mz_orchestrator_tracing", + "//src/orchestratord:mz_orchestratord", "//src/ore:mz_ore", "//src/persist-client:mz_persist_client", "//src/pgrepr:mz_pgrepr", @@ -188,6 +190,7 @@ rust_doc_test( "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-process:mz_orchestrator_process", "//src/orchestrator-tracing:mz_orchestrator_tracing", + "//src/orchestratord:mz_orchestratord", "//src/ore:mz_ore", "//src/persist-client:mz_persist_client", "//src/pgrepr:mz_pgrepr", @@ -274,6 +277,7 @@ rust_test( "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-process:mz_orchestrator_process", "//src/orchestrator-tracing:mz_orchestrator_tracing", + "//src/orchestratord:mz_orchestratord", "//src/ore:mz_ore", "//src/persist-client:mz_persist_client", "//src/pgrepr:mz_pgrepr", @@ -342,6 +346,7 @@ rust_test( "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-process:mz_orchestrator_process", "//src/orchestrator-tracing:mz_orchestrator_tracing", + "//src/orchestratord:mz_orchestratord", "//src/ore:mz_ore", "//src/persist-client:mz_persist_client", "//src/pgrepr:mz_pgrepr", @@ -410,6 +415,7 @@ rust_test( "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-process:mz_orchestrator_process", "//src/orchestrator-tracing:mz_orchestrator_tracing", + "//src/orchestratord:mz_orchestratord", "//src/ore:mz_ore", "//src/persist-client:mz_persist_client", "//src/pgrepr:mz_pgrepr", @@ -478,6 +484,7 @@ rust_test( "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-process:mz_orchestrator_process", "//src/orchestrator-tracing:mz_orchestrator_tracing", + "//src/orchestratord:mz_orchestratord", "//src/ore:mz_ore", "//src/persist-client:mz_persist_client", "//src/pgrepr:mz_pgrepr", @@ -546,6 +553,7 @@ rust_test( "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-process:mz_orchestrator_process", "//src/orchestrator-tracing:mz_orchestrator_tracing", + "//src/orchestratord:mz_orchestratord", "//src/ore:mz_ore", "//src/persist-client:mz_persist_client", "//src/pgrepr:mz_pgrepr", @@ -614,6 +622,7 @@ rust_test( "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-process:mz_orchestrator_process", "//src/orchestrator-tracing:mz_orchestrator_tracing", + "//src/orchestratord:mz_orchestratord", "//src/ore:mz_ore", "//src/persist-client:mz_persist_client", "//src/pgrepr:mz_pgrepr", @@ -682,6 +691,7 @@ rust_test( "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-process:mz_orchestrator_process", "//src/orchestrator-tracing:mz_orchestrator_tracing", + "//src/orchestratord:mz_orchestratord", "//src/ore:mz_ore", "//src/persist-client:mz_persist_client", "//src/pgrepr:mz_pgrepr", @@ -744,6 +754,7 @@ rust_binary( "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-process:mz_orchestrator_process", "//src/orchestrator-tracing:mz_orchestrator_tracing", + "//src/orchestratord:mz_orchestratord", "//src/ore:mz_ore", "//src/persist-client:mz_persist_client", "//src/pgrepr:mz_pgrepr", diff --git a/src/environmentd/Cargo.toml b/src/environmentd/Cargo.toml index 6cc0fa66ea1aa..036f0073dfe2c 100644 --- a/src/environmentd/Cargo.toml +++ b/src/environmentd/Cargo.toml @@ -58,6 +58,7 @@ mz-orchestrator = { path = "../orchestrator" } mz-orchestrator-kubernetes = { path = "../orchestrator-kubernetes" } mz-orchestrator-process = { path = "../orchestrator-process" } mz-orchestrator-tracing = { path = "../orchestrator-tracing" } +mz-orchestratord = { path = "../orchestratord", default-features = false } mz-ore = { path = "../ore", features = ["async", "process", "tracing_"] } mz-persist-client = { path = "../persist-client" } mz-pgrepr = { path = "../pgrepr" } diff --git a/src/environmentd/src/deployment/state.rs b/src/environmentd/src/deployment/state.rs index dc650abcf4e55..eea1249ae7cde 100644 --- a/src/environmentd/src/deployment/state.rs +++ b/src/environmentd/src/deployment/state.rs @@ -12,8 +12,8 @@ use std::future::Future; use std::sync::{Arc, Mutex}; +use mz_orchestratord::controller::materialize::environmentd::DeploymentStatus; use mz_ore::channel::trigger::{self, Trigger}; -use serde::{Deserialize, Serialize}; enum DeploymentStateInner { Initializing, @@ -192,20 +192,3 @@ impl DeploymentStateHandle { } } } - -/// Describes the status of a deployment. -/// -/// This is a simplified representation of `DeploymentState`, suitable for -/// announcement to the external orchestrator. -#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum DeploymentStatus { - /// This deployment is not the leader. It is initializing and is not yet - /// ready to become the leader. - Initializing, - /// This deployment is not the leader, but it is ready to become the leader. - ReadyToPromote, - /// This deployment is in the process of becoming the leader. - Promoting, - /// This deployment is the leader. - IsLeader, -} diff --git a/src/environmentd/src/lib.rs b/src/environmentd/src/lib.rs index f27314840dff4..af0f775306faf 100644 --- a/src/environmentd/src/lib.rs +++ b/src/environmentd/src/lib.rs @@ -58,7 +58,6 @@ use crate::deployment::preflight::{PreflightInput, PreflightOutput}; use crate::deployment::state::DeploymentState; use crate::http::{HttpConfig, HttpServer, InternalHttpConfig, InternalHttpServer}; -pub use crate::deployment::state::DeploymentStatus; pub use crate::http::{SqlResponse, WebSocketAuth, WebSocketResponse}; mod deployment; diff --git a/src/orchestrator-kubernetes/BUILD.bazel b/src/orchestrator-kubernetes/BUILD.bazel index 9fe599597bc23..2bf00de127f55 100644 --- a/src/orchestrator-kubernetes/BUILD.bazel +++ b/src/orchestrator-kubernetes/BUILD.bazel @@ -22,7 +22,7 @@ rust_library( proc_macro = True, ), compile_data = [], - crate_features = [], + crate_features = ["default"], data = [], proc_macro_deps = [] + all_crate_deps(proc_macro = True), rustc_env = {}, @@ -53,7 +53,7 @@ rust_test( ), compile_data = [], crate = ":mz_orchestrator_kubernetes", - crate_features = [], + crate_features = ["default"], data = [], env = {}, proc_macro_deps = [] + all_crate_deps( diff --git a/src/orchestrator-kubernetes/Cargo.toml b/src/orchestrator-kubernetes/Cargo.toml index e21e9852e67e2..b91201823a123 100644 --- a/src/orchestrator-kubernetes/Cargo.toml +++ b/src/orchestrator-kubernetes/Cargo.toml @@ -29,7 +29,10 @@ serde_json = "1.0.125" sha2 = "0.10.6" tokio = "1.32.0" tracing = "0.1.37" -workspace-hack = { version = "0.0.0", path = "../workspace-hack" } +workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true } + +[features] +default = ["workspace-hack"] [package.metadata.cargo-udeps.ignore] normal = ["workspace-hack"] diff --git a/src/orchestrator-process/BUILD.bazel b/src/orchestrator-process/BUILD.bazel index 943b04c7492fb..3402475acecb1 100644 --- a/src/orchestrator-process/BUILD.bazel +++ b/src/orchestrator-process/BUILD.bazel @@ -22,7 +22,7 @@ rust_library( proc_macro = True, ), compile_data = [], - crate_features = [], + crate_features = ["default"], data = [], proc_macro_deps = [] + all_crate_deps(proc_macro = True), rustc_env = {}, @@ -52,7 +52,7 @@ rust_test( ), compile_data = [], crate = ":mz_orchestrator_process", - crate_features = [], + crate_features = ["default"], data = [], env = {}, proc_macro_deps = [] + all_crate_deps( diff --git a/src/orchestrator-process/Cargo.toml b/src/orchestrator-process/Cargo.toml index af4d5b07fb71e..8a8ec1a027ad5 100644 --- a/src/orchestrator-process/Cargo.toml +++ b/src/orchestrator-process/Cargo.toml @@ -31,7 +31,10 @@ sha1 = "0.10.5" sysinfo = "0.27.2" tokio = { version = "1.38.0", features = [ "fs", "process", "time" ] } tracing = "0.1.37" -workspace-hack = { version = "0.0.0", path = "../workspace-hack" } +workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true } + +[features] +default = ["workspace-hack"] [package.metadata.cargo-udeps.ignore] normal = ["workspace-hack"] diff --git a/src/orchestrator-tracing/Cargo.toml b/src/orchestrator-tracing/Cargo.toml index 1de12ceb1c7bb..90dd69407f74c 100644 --- a/src/orchestrator-tracing/Cargo.toml +++ b/src/orchestrator-tracing/Cargo.toml @@ -29,13 +29,13 @@ tracing-capture = { version = "0.1.0", optional = true } tracing-subscriber = { version = "0.3.16", default-features = false } opentelemetry = { version = "0.24.0", features = ["trace"] } opentelemetry_sdk = { version = "0.24.1", features = ["rt-tokio"] } -workspace-hack = { version = "0.0.0", path = "../workspace-hack" } +workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true } [dev-dependencies] mz-ore = { path = "../ore", features = ["network", "test"] } [features] -default = ["tokio-console"] +default = ["tokio-console", "workspace-hack"] tokio-console = ["mz-ore/tokio-console", "mz-repr", "humantime"] capture = ["tracing-capture", "mz-ore/capture"] diff --git a/src/orchestrator/Cargo.toml b/src/orchestrator/Cargo.toml index 3e4cb4c6d5c25..2dafa7579804d 100644 --- a/src/orchestrator/Cargo.toml +++ b/src/orchestrator/Cargo.toml @@ -19,14 +19,14 @@ futures-core = "0.3.21" mz-ore = { path = "../ore"} prost = { version = "0.13.2", features = ["no-recursion-limit"] } serde = "1.0" -workspace-hack = { version = "0.0.0", path = "../workspace-hack" } +workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true } [build-dependencies] mz-build-tools = { path = "../build-tools", default-features = false } tonic-build = "0.12.1" [features] -default = ["mz-build-tools/default"] +default = ["mz-build-tools/default", "workspace-hack"] [package.metadata.cargo-udeps.ignore] normal = ["workspace-hack"] diff --git a/src/orchestratord/BUILD.bazel b/src/orchestratord/BUILD.bazel index 66343450b7737..f49ca9304f269 100644 --- a/src/orchestratord/BUILD.bazel +++ b/src/orchestratord/BUILD.bazel @@ -35,13 +35,12 @@ rust_library( "//src/alloc:mz_alloc", "//src/alloc-default:mz_alloc_default", "//src/build-info:mz_build_info", + "//src/cloud-provider:mz_cloud_provider", "//src/cloud-resources:mz_cloud_resources", - "//src/environmentd:mz_environmentd", "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-tracing:mz_orchestrator_tracing", "//src/ore:mz_ore", "//src/prof-http:mz_prof_http", - "//src/sql:mz_sql", "//src/tls-util:mz_tls_util", ] + all_crate_deps(normal = True), ) @@ -74,13 +73,12 @@ rust_test( "//src/alloc:mz_alloc", "//src/alloc-default:mz_alloc_default", "//src/build-info:mz_build_info", + "//src/cloud-provider:mz_cloud_provider", "//src/cloud-resources:mz_cloud_resources", - "//src/environmentd:mz_environmentd", "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-tracing:mz_orchestrator_tracing", "//src/ore:mz_ore", "//src/prof-http:mz_prof_http", - "//src/sql:mz_sql", "//src/tls-util:mz_tls_util", ] + all_crate_deps( normal = True, @@ -95,13 +93,12 @@ rust_doc_test( "//src/alloc:mz_alloc", "//src/alloc-default:mz_alloc_default", "//src/build-info:mz_build_info", + "//src/cloud-provider:mz_cloud_provider", "//src/cloud-resources:mz_cloud_resources", - "//src/environmentd:mz_environmentd", "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-tracing:mz_orchestrator_tracing", "//src/ore:mz_ore", "//src/prof-http:mz_prof_http", - "//src/sql:mz_sql", "//src/tls-util:mz_tls_util", ] + all_crate_deps( normal = True, @@ -130,13 +127,12 @@ rust_binary( "//src/alloc:mz_alloc", "//src/alloc-default:mz_alloc_default", "//src/build-info:mz_build_info", + "//src/cloud-provider:mz_cloud_provider", "//src/cloud-resources:mz_cloud_resources", - "//src/environmentd:mz_environmentd", "//src/orchestrator-kubernetes:mz_orchestrator_kubernetes", "//src/orchestrator-tracing:mz_orchestrator_tracing", "//src/ore:mz_ore", "//src/prof-http:mz_prof_http", - "//src/sql:mz_sql", "//src/tls-util:mz_tls_util", ] + all_crate_deps(normal = True), ) diff --git a/src/orchestratord/Cargo.toml b/src/orchestratord/Cargo.toml index bf2df2ea0ee33..8a5f858d3b9e8 100644 --- a/src/orchestratord/Cargo.toml +++ b/src/orchestratord/Cargo.toml @@ -21,17 +21,16 @@ k8s-controller = "0.3.3" k8s-openapi = { version = "0.22.0", features = ["v1_29"] } kube = { version = "0.92.1", default-features = false, features = ["client", "runtime", "ws"] } maplit = "1.0.2" -mz-alloc = { path = "../alloc" } -mz-alloc-default = { path = "../alloc-default", optional = true } -mz-build-info = { path = "../build-info" } -mz-cloud-resources = { path = "../cloud-resources" } -mz-environmentd = { path = "../environmentd" } -mz-ore = { path = "../ore" } -mz-orchestrator-kubernetes = { path = "../orchestrator-kubernetes" } -mz-orchestrator-tracing = { path = "../orchestrator-tracing" } -mz-prof-http = { path = "../prof-http" } -mz-sql = { path = "../sql" } -mz-tls-util = { path = "../tls-util" } +mz-alloc = { path = "../alloc", default-features = false } +mz-alloc-default = { path = "../alloc-default", optional = true, default-features = false } +mz-build-info = { path = "../build-info", default-features = false } +mz-cloud-provider = { path = "../cloud-provider", default-features = false } +mz-cloud-resources = { path = "../cloud-resources", default-features = false } +mz-ore = { path = "../ore", default-features = false } +mz-orchestrator-kubernetes = { path = "../orchestrator-kubernetes", default-features = false } +mz-orchestrator-tracing = { path = "../orchestrator-tracing", default-features = false } +mz-prof-http = { path = "../prof-http", default-features = false } +mz-tls-util = { path = "../tls-util", default-features = false } prometheus = { version = "0.13.3", default-features = false } rand = "0.8.5" reqwest = { version = "0.11.13", features = ["json"] } @@ -44,10 +43,10 @@ tokio-postgres = "0.7.8" tower-http = "0.5.2" tracing = "0.1.37" urlencoding = "2.1.2" -workspace-hack = { version = "0.0.0", path = "../workspace-hack" } +workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true } [features] -default = ["mz-alloc-default"] +default = ["mz-alloc-default", "workspace-hack"] jemalloc = ["mz-alloc/jemalloc"] [package.metadata.cargo-udeps.ignore] diff --git a/src/orchestratord/src/controller/materialize.rs b/src/orchestratord/src/controller/materialize.rs index 954e31e0200d8..0c01911126f98 100644 --- a/src/orchestratord/src/controller/materialize.rs +++ b/src/orchestratord/src/controller/materialize.rs @@ -21,18 +21,18 @@ use serde::Deserialize; use tracing::{debug, trace}; use crate::metrics::Metrics; +use mz_cloud_provider::CloudProvider; use mz_cloud_resources::crd::materialize::v1alpha1::{ Materialize, MaterializeCertSpec, MaterializeStatus, }; use mz_orchestrator_kubernetes::KubernetesImagePullPolicy; use mz_orchestrator_tracing::TracingCliArgs; use mz_ore::{cast::CastFrom, cli::KeyValueArg, instrument}; -use mz_sql::catalog::CloudProvider; -mod balancer; -mod console; -mod environmentd; -mod tls; +pub mod balancer; +pub mod console; +pub mod environmentd; +pub mod tls; #[derive(clap::Parser)] pub struct Args { diff --git a/src/orchestratord/src/controller/materialize/environmentd.rs b/src/orchestratord/src/controller/materialize/environmentd.rs index 77ebe0c14d699..6054943f92a61 100644 --- a/src/orchestratord/src/controller/materialize/environmentd.rs +++ b/src/orchestratord/src/controller/materialize/environmentd.rs @@ -38,12 +38,28 @@ use tracing::trace; use super::matching_image_from_environmentd_image_ref; use crate::controller::materialize::tls::{create_certificate, issuer_ref_defined}; use crate::k8s::{apply_resource, delete_resource, get_resource}; +use mz_cloud_provider::CloudProvider; use mz_cloud_resources::crd::gen::cert_manager::certificates::Certificate; use mz_cloud_resources::crd::materialize::v1alpha1::Materialize; -use mz_environmentd::DeploymentStatus; use mz_orchestrator_tracing::TracingCliArgs; use mz_ore::instrument; -use mz_sql::catalog::CloudProvider; + +/// Describes the status of a deployment. +/// +/// This is a simplified representation of `DeploymentState`, suitable for +/// announcement to the external orchestrator. +#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum DeploymentStatus { + /// This deployment is not the leader. It is initializing and is not yet + /// ready to become the leader. + Initializing, + /// This deployment is not the leader, but it is ready to become the leader. + ReadyToPromote, + /// This deployment is in the process of becoming the leader. + Promoting, + /// This deployment is the leader. + IsLeader, +} #[derive(Debug, Serialize)] pub struct Resources { diff --git a/src/secrets/BUILD.bazel b/src/secrets/BUILD.bazel index ceb94cb2c94bf..1253733fea5a9 100644 --- a/src/secrets/BUILD.bazel +++ b/src/secrets/BUILD.bazel @@ -22,7 +22,7 @@ rust_library( proc_macro = True, ), compile_data = [], - crate_features = [], + crate_features = ["default"], data = [], proc_macro_deps = [] + all_crate_deps(proc_macro = True), rustc_env = {}, @@ -47,7 +47,7 @@ rust_test( ), compile_data = [], crate = ":mz_secrets", - crate_features = [], + crate_features = ["default"], data = [], env = {}, proc_macro_deps = [] + all_crate_deps( diff --git a/src/secrets/Cargo.toml b/src/secrets/Cargo.toml index 57a5304b91987..92fd224a869e6 100644 --- a/src/secrets/Cargo.toml +++ b/src/secrets/Cargo.toml @@ -14,11 +14,14 @@ anyhow = "1.0.66" async-trait = "0.1.68" mz-repr = { path = "../repr" } tracing = "0.1.37" -workspace-hack = { version = "0.0.0", path = "../workspace-hack" } +workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true } [dev-dependencies] mz-ore = { path = "../ore" } tokio = { version = "1.38.0", features = ["macros", "rt"] } +[features] +default = ["workspace-hack"] + [package.metadata.cargo-udeps.ignore] normal = ["workspace-hack"] diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 76a605301d83d..aa619fdf2aaaa 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -42,7 +42,7 @@ tonic = "0.12.1" tower = "0.4.13" tracing = "0.1.37" sentry-tracing = "0.29.1" -workspace-hack = { version = "0.0.0", path = "../workspace-hack" } +workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true } [build-dependencies] mz-build-tools = { path = "../build-tools", default-features = false } @@ -50,7 +50,7 @@ prost-build = "0.13.2" tonic-build = "0.12.1" [features] -default = ["mz-build-tools/default"] +default = ["mz-build-tools/default", "workspace-hack"] [package.metadata.cargo-udeps.ignore] normal = ["workspace-hack"] diff --git a/src/sql/BUILD.bazel b/src/sql/BUILD.bazel index e158b43c3e9b2..33028efdc5563 100644 --- a/src/sql/BUILD.bazel +++ b/src/sql/BUILD.bazel @@ -34,6 +34,7 @@ rust_library( "//src/audit-log:mz_audit_log", "//src/build-info:mz_build_info", "//src/ccsr:mz_ccsr", + "//src/cloud-provider:mz_cloud_provider", "//src/cloud-resources:mz_cloud_resources", "//src/controller-types:mz_controller_types", "//src/dyncfg:mz_dyncfg", @@ -94,6 +95,7 @@ rust_test( "//src/audit-log:mz_audit_log", "//src/build-info:mz_build_info", "//src/ccsr:mz_ccsr", + "//src/cloud-provider:mz_cloud_provider", "//src/cloud-resources:mz_cloud_resources", "//src/controller-types:mz_controller_types", "//src/dyncfg:mz_dyncfg", @@ -135,6 +137,7 @@ rust_doc_test( "//src/audit-log:mz_audit_log", "//src/build-info:mz_build_info", "//src/ccsr:mz_ccsr", + "//src/cloud-provider:mz_cloud_provider", "//src/cloud-resources:mz_cloud_resources", "//src/controller-types:mz_controller_types", "//src/dyncfg:mz_dyncfg", diff --git a/src/sql/Cargo.toml b/src/sql/Cargo.toml index 2de54ecd6e4f5..7e761c6b2bb0e 100644 --- a/src/sql/Cargo.toml +++ b/src/sql/Cargo.toml @@ -37,6 +37,7 @@ mz-adapter-types = { path = "../adapter-types" } mz-audit-log = { path = "../audit-log" } mz-build-info = { path = "../build-info" } mz-ccsr = { path = "../ccsr" } +mz-cloud-provider = { path = "../cloud-provider", default-features = false } mz-cloud-resources = { path = "../cloud-resources" } mz-controller-types = { path = "../controller-types" } mz-dyncfg = { path = "../dyncfg" } diff --git a/src/sql/src/catalog.rs b/src/sql/src/catalog.rs index b5e1f9018783b..fcc90b5de17e8 100644 --- a/src/sql/src/catalog.rs +++ b/src/sql/src/catalog.rs @@ -22,6 +22,7 @@ use std::time::{Duration, Instant}; use chrono::{DateTime, Utc}; use mz_build_info::BuildInfo; +use mz_cloud_provider::{CloudProvider, InvalidCloudProviderError}; use mz_controller_types::{ClusterId, ReplicaId}; use mz_expr::MirScalarExpr; use mz_ore::now::{EpochMillis, NowFn}; @@ -1186,80 +1187,6 @@ impl From for InvalidEnvironmentIdError { } } -/// Identifies a supported cloud provider. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum CloudProvider { - /// A pseudo-provider value used by local development environments. - Local, - /// A pseudo-provider value used by Docker. - Docker, - /// A deprecated psuedo-provider value used by mzcompose. - // TODO(benesch): remove once v0.39 ships. - MzCompose, - /// A pseudo-provider value used by cloudtest. - Cloudtest, - /// Amazon Web Services. - Aws, - /// Google Cloud Platform - Gcp, - /// Microsoft Azure - Azure, - /// Other generic cloud provider - Generic, -} - -impl CloudProvider { - /// Returns true if this provider actually runs in the cloud - pub fn is_cloud(&self) -> bool { - matches!(self, Self::Aws | Self::Gcp | Self::Azure | Self::Generic) - } -} - -impl fmt::Display for CloudProvider { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - CloudProvider::Local => f.write_str("local"), - CloudProvider::Docker => f.write_str("docker"), - CloudProvider::MzCompose => f.write_str("mzcompose"), - CloudProvider::Cloudtest => f.write_str("cloudtest"), - CloudProvider::Aws => f.write_str("aws"), - CloudProvider::Gcp => f.write_str("gcp"), - CloudProvider::Azure => f.write_str("azure"), - CloudProvider::Generic => f.write_str("generic"), - } - } -} - -impl FromStr for CloudProvider { - type Err = InvalidCloudProviderError; - - fn from_str(s: &str) -> Result { - match s.to_lowercase().as_ref() { - "local" => Ok(CloudProvider::Local), - "docker" => Ok(CloudProvider::Docker), - "mzcompose" => Ok(CloudProvider::MzCompose), - "cloudtest" => Ok(CloudProvider::Cloudtest), - "aws" => Ok(CloudProvider::Aws), - "gcp" => Ok(CloudProvider::Gcp), - "azure" => Ok(CloudProvider::Azure), - "generic" => Ok(CloudProvider::Generic), - _ => Err(InvalidCloudProviderError), - } - } -} - -/// The error type for [`CloudProvider::from_str`]. -#[derive(Debug, Clone, PartialEq)] -pub struct InvalidCloudProviderError; - -impl fmt::Display for InvalidCloudProviderError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("invalid cloud provider") - } -} - -impl Error for InvalidCloudProviderError {} - /// An error returned by the catalog. #[derive(Clone, Debug, Eq, PartialEq)] pub enum CatalogError { diff --git a/src/tracing/Cargo.toml b/src/tracing/Cargo.toml index b4b817fe9a22c..c70744c43efaa 100644 --- a/src/tracing/Cargo.toml +++ b/src/tracing/Cargo.toml @@ -18,7 +18,7 @@ proptest = { version = "1.0.0", default-features = false, features = ["std"]} serde = { version = "1.0.152", features = ["derive"] } tracing = "0.1.37" tracing-subscriber = { version = "0.3.16", default-features = false, features = ["env-filter", "fmt"] } -workspace-hack = { version = "0.0.0", path = "../workspace-hack" } +workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true } [build-dependencies] mz-build-tools = { path = "../build-tools", default-features = false } @@ -26,7 +26,7 @@ prost-build = "0.13.2" tonic-build = "0.12.1" [features] -default = ["mz-build-tools/default"] +default = ["mz-build-tools/default", "workspace-hack"] [package.metadata.cargo-udeps.ignore] normal = ["workspace-hack"]