Skip to content

Commit

Permalink
[sled-agent] Nexus self assembling zone
Browse files Browse the repository at this point in the history
  • Loading branch information
@karencfv
karencfv committed Mar 7, 2024
1 parent c3f385e commit 6b47f0a
Show file tree
Hide file tree
Showing 9 changed files with 198 additions and 127 deletions.
2 changes: 1 addition & 1 deletion .github/buildomat/jobs/deploy.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ cd /opt/oxide/work

ptime -m tar xvzf /input/package/work/package.tar.gz
cp /input/package/work/zones/* out/
mv out/omicron-nexus-single-sled.tar.gz out/omicron-nexus.tar.gz
mv out/nexus-single-sled.tar.gz out/nexus.tar.gz
mkdir tests
for p in /input/ci-tools/work/end-to-end-tests/*.gz; do
ptime -m gunzip < "$p" > "tests/$(basename "${p%.gz}")"
Expand Down
6 changes: 3 additions & 3 deletions .github/buildomat/jobs/package.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ stamp_packages() {

# Keep the single-sled Nexus zone around for the deploy job. (The global zone
# build below overwrites the file.)
mv out/omicron-nexus.tar.gz out/omicron-nexus-single-sled.tar.gz
mv out/nexus.tar.gz out/nexus-single-sled.tar.gz

# Build necessary for the global zone
ptime -m cargo run --locked --release --bin omicron-package -- \
Expand Down Expand Up @@ -115,8 +115,8 @@ zones=(
out/crucible-zone.tar.gz
out/external-dns.tar.gz
out/internal-dns.tar.gz
out/omicron-nexus.tar.gz
out/omicron-nexus-single-sled.tar.gz
out/nexus.tar.gz
out/nexus-single-sled.tar.gz
out/oximeter.tar.gz
out/propolis-server.tar.gz
out/switch-*.tar.gz
Expand Down
2 changes: 1 addition & 1 deletion .github/buildomat/jobs/tuf-repo.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ target/release/omicron-package -t default target create -i standard -m gimlet -s
ln -s /input/package/work/zones/* out/
rm out/switch-softnpu.tar.gz # not used when target switch=asic
rm out/omicron-gateway-softnpu.tar.gz # not used when target switch=asic
rm out/omicron-nexus-single-sled.tar.gz # only used for deploy tests
rm out/nexus-single-sled.tar.gz # only used for deploy tests
for zone in out/*.tar.gz; do
target/release/omicron-package stamp "$(basename "${zone%.tar.gz}")" "$VERSION"
done
Expand Down
10 changes: 9 additions & 1 deletion illumos-utils/src/running_zone.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ impl RunningZone {

/// Returns the filesystem path to the zone's root in the GZ.
pub fn root(&self) -> Utf8PathBuf {
self.inner.zonepath.join(Self::ROOT_FS_PATH)
self.inner.root()
}

pub fn control_interface(&self) -> AddrObject {
Expand Down Expand Up @@ -1094,6 +1094,9 @@ pub struct InstalledZone {
}

impl InstalledZone {
/// The path to the zone's root filesystem (i.e., `/`), within zonepath.
pub const ROOT_FS_PATH: &'static str = "root";

/// Returns the name of a zone, based on the base zone name plus any unique
/// identifying info.
///
Expand Down Expand Up @@ -1135,6 +1138,11 @@ impl InstalledZone {
pub fn opte_ports(&self) -> impl Iterator<Item = &Port> {
self.opte_ports.iter().map(|(port, _)| port)
}

/// Returns the filesystem path to the zone's root in the GZ.
pub fn root(&self) -> Utf8PathBuf {
self.zonepath.join(Self::ROOT_FS_PATH)
}
}

#[derive(Clone)]
Expand Down
15 changes: 14 additions & 1 deletion package-manifest.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,21 @@ source.paths = [
output.type = "zone"
output.intermediate_only = true

[package.omicron-nexus]
[package.nexus]
service_name = "nexus"
only_for_targets.image = "standard"
source.type = "composite"
source.packages = [
"omicron-nexus.tar.gz",
"zone-network-setup.tar.gz",
"zone-network-install.tar.gz",
"opte-interface-setup.tar.gz"
]
output.type = "zone"

[package.omicron-nexus]
service_name = "omicron-nexus"
only_for_targets.image = "standard"
source.type = "local"
source.rust.binary_names = ["nexus", "schema-updater"]
source.rust.release = true
Expand All @@ -115,6 +127,7 @@ setup_hint = """
- Run `./tools/ci_download_console` to download the web console assets
- Run `pkg install library/postgresql-13` to download Postgres libraries
"""
output.intermediate_only = true

[package.oximeter]
service_name = "oximeter"
Expand Down
263 changes: 150 additions & 113 deletions sled-agent/src/services.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2038,6 +2038,155 @@ impl ServiceManager {

return Ok(RunningZone::boot(installed_zone).await?);
}
ZoneArgs::Omicron(OmicronZoneConfigLocal {
zone:
OmicronZoneConfig {
zone_type:
OmicronZoneType::Nexus {
internal_address,
external_tls,
external_dns_servers,
..
},
underlay_address,
id,
},
..
}) => {
let Some(info) = self.inner.sled_info.get() else {
return Err(Error::SledAgentNotReady);
};

let static_addr = underlay_address.to_string();

let nw_setup_service = Self::zone_network_setup_install(
&info,
&installed_zone,
&static_addr.clone(),
)?;

// While Nexus will be reachable via `external_ip`, it
// communicates atop an OPTE port which operates on a
// VPC private IP. OPTE will map the private IP to the
// external IP automatically.
let opte_interface_setup =
Self::opte_interface_set_up_install(&installed_zone)?;

let port_idx = 0;
let port = installed_zone
.opte_ports()
.nth(port_idx)
.ok_or_else(|| {
Error::ZoneEnsureAddress(
EnsureAddressError::MissingOptePort {
zone: String::from(installed_zone.name()),
port_idx,
},
)
})?;
let opte_ip = port.ip();

// Nexus takes a separate config file for parameters
// which cannot be known at packaging time.
let nexus_port = if *external_tls { 443 } else { 80 };
let deployment_config = DeploymentConfig {
id: *id,
rack_id: info.rack_id,
techport_external_server_port: NEXUS_TECHPORT_EXTERNAL_PORT,

dropshot_external: ConfigDropshotWithTls {
tls: *external_tls,
dropshot: dropshot::ConfigDropshot {
bind_address: SocketAddr::new(*opte_ip, nexus_port),
// This has to be large enough to support:
// - bulk writes to disks
request_body_max_bytes: 8192 * 1024,
default_handler_task_mode:
HandlerTaskMode::Detached,
},
},
dropshot_internal: dropshot::ConfigDropshot {
bind_address: (*internal_address).into(),
// This has to be large enough to support, among
// other things, the initial list of TLS
// certificates provided by the customer during
// rack setup.
request_body_max_bytes: 10 * 1024 * 1024,
default_handler_task_mode: HandlerTaskMode::Detached,
},
internal_dns: nexus_config::InternalDns::FromSubnet {
subnet: Ipv6Subnet::<RACK_PREFIX>::new(
info.underlay_address,
),
},
database: nexus_config::Database::FromDns,
external_dns_servers: external_dns_servers.clone(),
};

// Copy the partial config file to the expected
// location.
let config_dir = Utf8PathBuf::from(format!(
"{}/var/svc/manifest/site/nexus",
installed_zone.root()
));
// The filename of a half-completed config, in need of
// parameters supplied at runtime.
const PARTIAL_LEDGER_FILENAME: &str = "config-partial.toml";
// The filename of a completed config, merging the
// partial config with additional appended parameters
// known at runtime.
const COMPLETE_LEDGER_FILENAME: &str = "config.toml";
let partial_config_path =
config_dir.join(PARTIAL_LEDGER_FILENAME);
let config_path = config_dir.join(COMPLETE_LEDGER_FILENAME);
tokio::fs::copy(partial_config_path, &config_path)
.await
.map_err(|err| Error::io_path(&config_path, err))?;

// Serialize the configuration and append it into the
// file.
let serialized_cfg = toml::Value::try_from(&deployment_config)
.expect("Cannot serialize config");
let mut map = toml::map::Map::new();
map.insert("deployment".to_string(), serialized_cfg);
let config_str = toml::to_string(&map).map_err(|err| {
Error::TomlSerialize { path: config_path.clone(), err }
})?;
let mut file = tokio::fs::OpenOptions::new()
.append(true)
.open(&config_path)
.await
.map_err(|err| Error::io_path(&config_path, err))?;
file.write_all(b"\n\n")
.await
.map_err(|err| Error::io_path(&config_path, err))?;
file.write_all(config_str.as_bytes())
.await
.map_err(|err| Error::io_path(&config_path, err))?;

let nexus_config = PropertyGroupBuilder::new("config");
// .add_property("http_address", "astring", &http_addr)
// .add_property("dns_address", "astring", &dns_addr);
let nexus_service = ServiceBuilder::new("oxide/nexus")
.add_instance(
ServiceInstanceBuilder::new("default")
.add_property_group(nexus_config),
);

let profile = ProfileBuilder::new("omicron")
.add_service(nw_setup_service)
.add_service(opte_interface_setup)
.add_service(disabled_ssh_service)
.add_service(nexus_service)
.add_service(disabled_dns_client_service);
profile
.add_to_zone(&self.inner.log, &installed_zone)
.await
.map_err(|err| {
Error::io("Failed to setup Nexus profile", err)
})?;
return Ok(RunningZone::boot(installed_zone).await?);
}
_ => {}
}

Expand Down Expand Up @@ -2165,119 +2314,6 @@ impl ServiceManager {
smfh.import_manifest()?;

match &zone_config.zone.zone_type {
OmicronZoneType::Nexus {
internal_address,
external_tls,
external_dns_servers,
..
} => {
info!(self.inner.log, "Setting up Nexus service");

let sled_info = self
.inner
.sled_info
.get()
.ok_or(Error::SledAgentNotReady)?;

// While Nexus will be reachable via `external_ip`, it
// communicates atop an OPTE port which operates on a
// VPC private IP. OPTE will map the private IP to the
// external IP automatically.
let port_ip = running_zone
.ensure_address_for_port("public", 0)
.await?
.ip();

// Nexus takes a separate config file for parameters
// which cannot be known at packaging time.
let nexus_port = if *external_tls { 443 } else { 80 };
let deployment_config = DeploymentConfig {
id: zone_config.zone.id,
rack_id: sled_info.rack_id,
techport_external_server_port:
NEXUS_TECHPORT_EXTERNAL_PORT,

dropshot_external: ConfigDropshotWithTls {
tls: *external_tls,
dropshot: dropshot::ConfigDropshot {
bind_address: SocketAddr::new(
port_ip, nexus_port,
),
// This has to be large enough to support:
// - bulk writes to disks
request_body_max_bytes: 8192 * 1024,
default_handler_task_mode:
HandlerTaskMode::Detached,
},
},
dropshot_internal: dropshot::ConfigDropshot {
bind_address: (*internal_address).into(),
// This has to be large enough to support, among
// other things, the initial list of TLS
// certificates provided by the customer during
// rack setup.
request_body_max_bytes: 10 * 1024 * 1024,
default_handler_task_mode:
HandlerTaskMode::Detached,
},
internal_dns:
nexus_config::InternalDns::FromSubnet {
subnet: Ipv6Subnet::<RACK_PREFIX>::new(
sled_info.underlay_address,
),
},
database: nexus_config::Database::FromDns,
external_dns_servers: external_dns_servers.clone(),
};

// Copy the partial config file to the expected
// location.
let config_dir = Utf8PathBuf::from(format!(
"{}/var/svc/manifest/site/nexus",
running_zone.root()
));
// The filename of a half-completed config, in need of
// parameters supplied at runtime.
const PARTIAL_LEDGER_FILENAME: &str =
"config-partial.toml";
// The filename of a completed config, merging the
// partial config with additional appended parameters
// known at runtime.
const COMPLETE_LEDGER_FILENAME: &str = "config.toml";
let partial_config_path =
config_dir.join(PARTIAL_LEDGER_FILENAME);
let config_path =
config_dir.join(COMPLETE_LEDGER_FILENAME);
tokio::fs::copy(partial_config_path, &config_path)
.await
.map_err(|err| Error::io_path(&config_path, err))?;

// Serialize the configuration and append it into the
// file.
let serialized_cfg =
toml::Value::try_from(&deployment_config)
.expect("Cannot serialize config");
let mut map = toml::map::Map::new();
map.insert("deployment".to_string(), serialized_cfg);
let config_str =
toml::to_string(&map).map_err(|err| {
Error::TomlSerialize {
path: config_path.clone(),
err,
}
})?;
let mut file = tokio::fs::OpenOptions::new()
.append(true)
.open(&config_path)
.await
.map_err(|err| Error::io_path(&config_path, err))?;
file.write_all(b"\n\n")
.await
.map_err(|err| Error::io_path(&config_path, err))?;
file.write_all(config_str.as_bytes())
.await
.map_err(|err| Error::io_path(&config_path, err))?;
}
OmicronZoneType::InternalDns {
http_address,
dns_address,
Expand Down Expand Up @@ -2360,6 +2396,7 @@ impl ServiceManager {
| OmicronZoneType::CruciblePantry { .. }
| OmicronZoneType::ExternalDns { .. }
| OmicronZoneType::InternalNtp { .. }
| OmicronZoneType::Nexus { .. }
| OmicronZoneType::Oximeter { .. } => {
panic!(
"{} is a service which exists as part of a \
Expand Down
Loading

0 comments on commit 6b47f0a

Please sign in to comment.