Skip to content

Commit

Permalink
testing with OPTE interface and gateway
Browse files Browse the repository at this point in the history
  • Loading branch information
@karencfv
karencfv committed Feb 14, 2024
1 parent e1c3dd7 commit 1ecff77
Show file tree
Hide file tree
Showing 5 changed files with 186 additions and 46 deletions.
7 changes: 6 additions & 1 deletion illumos-utils/src/running_zone.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -888,7 +888,7 @@ impl RunningZone {

/// Return references to the OPTE ports for this zone.
pub fn opte_ports(&self) -> impl Iterator<Item = &Port> {
self.inner.opte_ports.iter().map(|(port, _)| port)
self.inner.opte_ports()
}

/// Remove the OPTE ports on this zone from the port manager.
Expand Down Expand Up @@ -1130,6 +1130,11 @@ impl InstalledZone {
path.push("root/var/svc/profile/site.xml");
path
}

/// Returns references to the OPTE ports for this zone.
pub fn opte_ports(&self) -> impl Iterator<Item = &Port> {
self.opte_ports.iter().map(|(port, _)| port)
}
}

#[derive(Clone)]
Expand Down
8 changes: 6 additions & 2 deletions package-manifest.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ output.type = "zone"
service_name = "external_dns"
only_for_targets.image = "standard"
source.type = "composite"
source.packages = [ "dns-server.tar.gz", "external-dns-customizations.tar.gz" ]
source.packages = [ "dns-server.tar.gz", "external-dns-customizations.tar.gz", "zone-network-setup.tar.gz" ]
output.type = "zone"

[package.dns-server]
Expand All @@ -245,7 +245,11 @@ output.type = "zone"
service_name = "external-dns-customizations"
only_for_targets.image = "standard"
source.type = "local"
source.paths = [ { from = "smf/external-dns", to = "/var/svc/manifest/site/external_dns" } ]
source.paths = [
{ from = "smf/external-dns", to = "/var/svc/manifest/site/external_dns" },
# TODO: Removeme as we'll have a separate service for setting up OPTE interfaces
{ from = "smf/external-dns/method_script.sh", to = "/opt/oxide/lib/svc/manifest/external_dns.sh" },
]
output.intermediate_only = true
output.type = "zone"

Expand Down
168 changes: 127 additions & 41 deletions sled-agent/src/services.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ use illumos_utils::dladm::{
use illumos_utils::link::{Link, VnicAllocator};
use illumos_utils::opte::{DhcpCfg, Port, PortManager, PortTicket};
use illumos_utils::running_zone::{
InstalledZone, RunCommandError, RunningZone, ZoneBuilderFactory,
EnsureAddressError, InstalledZone, RunCommandError, RunningZone,
ZoneBuilderFactory,
};
use illumos_utils::zfs::ZONE_ZFS_RAMDISK_DATASET_MOUNTPOINT;
use illumos_utils::zone::AddressRequest;
Expand Down Expand Up @@ -1835,6 +1836,91 @@ impl ServiceManager {
})?;
return Ok(RunningZone::boot(installed_zone).await?);
}
ZoneArgs::Omicron(OmicronZoneConfigLocal {
zone:
OmicronZoneConfig {
zone_type: OmicronZoneType::ExternalDns { .. },
underlay_address,
..
},
..
}) => {
let Some(info) = self.inner.sled_info.get() else {
return Err(Error::SledAgentNotReady);
};

// Like Nexus, we need to be reachable externally via
// `dns_address` but we don't listen on that address
// directly but instead on a VPC private IP. OPTE will
// en/decapsulate as appropriate.
let port_idx = 0;
let port = installed_zone
.opte_ports()
.nth(port_idx)
.ok_or_else(|| {
Error::ZoneEnsureAddress(
EnsureAddressError::MissingOptePort {
zone: String::from(installed_zone.name()),
port_idx,
},
)
})?;

// TODO: These should be set up as part of the networking service perhaps?
// Nexus will likely need this as well
//
// OPTE_INTERFACE="$(svcprop -c -p config/opte_interface "${SMF_FMRI}")"
// OPTE_GATEWAY="$(svcprop -c -p config/opte_gateway "${SMF_FMRI}")"
//
// # Set up OPTE interface
// if [[ "$OPTE_GATEWAY" =~ .*:.* ]]; then
// # IPv6 gateway
// echo "IPv6 OPTE gateways are not yet supported"
// exit 1
// else
// # IPv4 gateway
// ipadm show-addr "$OPTE_INTERFACE/public" || ipadm create-addr -t -T dhcp "$OPTE_INTERFACE/public"
// OPTE_IP=$(ipadm show-addr -p -o ADDR "$OPTE_INTERFACE/public" | cut -d'/' -f 1)
// route get -host "$OPTE_GATEWAY" "$OPTE_IP" -interface -ifp "$OPTE_INTERFACE" || route add -host "$OPTE_GATEWAY" "$OPTE_IP" -interface -ifp "$OPTE_INTERFACE"
// route get -inet default "$OPTE_GATEWAY" || route add -inet default "$OPTE_GATEWAY"
// fi
//
let opte_interface = port.vnic_name();
let opte_gateway = &port.gateway().ip().to_string();

let listen_addr = underlay_address.to_string();

let nw_setup_service = Self::zone_network_setup_install(
info,
&installed_zone,
&listen_addr.clone(),
)?;

let external_dns_config = PropertyGroupBuilder::new("config")
// TODO: Removeme and move to new opte interface service
.add_property("opte_gateway", "astring", opte_gateway)
.add_property("opte_interface", "astring", opte_interface)
// TODO: Keep these two
.add_property("http_address", "astring", opte_gateway)
.add_property("dns_address", "astring", &listen_addr);
let external_dns_service =
ServiceBuilder::new("oxide/external_dns").add_instance(
ServiceInstanceBuilder::new("default")
.add_property_group(external_dns_config),
);

let profile = ProfileBuilder::new("omicron")
.add_service(nw_setup_service)
.add_service(disabled_ssh_service)
.add_service(external_dns_service);
profile
.add_to_zone(&self.inner.log, &installed_zone)
.await
.map_err(|err| {
Error::io("Failed to setup External DNS profile", err)
})?;
return Ok(RunningZone::boot(installed_zone).await?);
}
_ => {}
}

Expand Down Expand Up @@ -2076,46 +2162,45 @@ impl ServiceManager {
.map_err(|err| Error::io_path(&config_path, err))?;
}

OmicronZoneType::ExternalDns {
http_address,
dns_address,
..
} => {
info!(
self.inner.log,
"Setting up external-dns service"
);

// Like Nexus, we need to be reachable externally via
// `dns_address` but we don't listen on that address
// directly but instead on a VPC private IP. OPTE will
// en/decapsulate as appropriate.
let port_ip = running_zone
.ensure_address_for_port("public", 0)
.await?
.ip();
let dns_address =
SocketAddr::new(port_ip, dns_address.port());

smfh.setprop(
"config/http_address",
format!(
"[{}]:{}",
http_address.ip(),
http_address.port(),
),
)?;
smfh.setprop(
"config/dns_address",
dns_address.to_string(),
)?;

// Refresh the manifest with the new properties we set,
// so they become "effective" properties when the
// service is enabled.
smfh.refresh()?;
}

// OmicronZoneType::ExternalDns {
// http_address,
// dns_address,
// ..
// } => {
// info!(
// self.inner.log,
// "Setting up external-dns service"
// );
//
// // Like Nexus, we need to be reachable externally via
// // `dns_address` but we don't listen on that address
// // directly but instead on a VPC private IP. OPTE will
// // en/decapsulate as appropriate.
// let port_ip = running_zone
// .ensure_address_for_port("public", 0)
// .await?
// .ip();
// let dns_address =
// SocketAddr::new(port_ip, dns_address.port());
//
// smfh.setprop(
// "config/http_address",
// format!(
// "[{}]:{}",
// http_address.ip(),
// http_address.port(),
// ),
// )?;
// smfh.setprop(
// "config/dns_address",
// dns_address.to_string(),
// )?;
//
// // Refresh the manifest with the new properties we set,
// // so they become "effective" properties when the
// // service is enabled.
// smfh.refresh()?;
// }
OmicronZoneType::InternalDns {
http_address,
dns_address,
Expand Down Expand Up @@ -2256,6 +2341,7 @@ impl ServiceManager {
| OmicronZoneType::CockroachDb { .. }
| OmicronZoneType::Crucible { .. }
| OmicronZoneType::CruciblePantry { .. }
| OmicronZoneType::ExternalDns { .. }
| OmicronZoneType::Oximeter { .. } => {
panic!(
"{} is a service which exists as part of a \
Expand Down
12 changes: 10 additions & 2 deletions smf/external-dns/manifest.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,29 @@
<service_bundle type='manifest' name='external_dns'>

<service name='oxide/external_dns' type='service' version='1'>
<create_default_instance enabled='false' />
<create_default_instance enabled='true' />

<dependency name='multi_user' grouping='require_all' restart_on='none'
type='service'>
<service_fmri value='svc:/milestone/multi-user:default' />
</dependency>

<dependency name='zone_network_setup' grouping='require_all' restart_on='none'
type='service'>
<service_fmri value='svc:/oxide/zone-network-setup:default' />
</dependency>

<exec_method type='method' name='start'
exec='ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/external_dns/config.toml --http-address %{config/http_address} --dns-address %{config/dns_address} &amp;'
exec='/opt/oxide/lib/svc/manifest/external_dns.sh'
timeout_seconds='0' />
<exec_method type='method' name='stop' exec=':kill' timeout_seconds='0' />

<property_group name='config' type='application'>
<propval name='http_address' type='astring' value='unknown' />
<propval name='dns_address' type='astring' value='unknown' />
<!-- TODO: Removeme when new opte interface service exists -->
<propval name='opte_interface' type='astring' value='unknown' />
<propval name='opte_gateway' type='astring' value='unknown' />
</property_group>

<property_group name='startd' type='framework'>
Expand Down
37 changes: 37 additions & 0 deletions smf/external-dns/method_script.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#!/bin/bash

set -x
set -o errexit
set -o pipefail

. /lib/svc/share/smf_include.sh

# TODO: OPTE variables to be moved to opte-nw-service
OPTE_INTERFACE="$(svcprop -c -p config/opte_interface "${SMF_FMRI}")"
OPTE_GATEWAY="$(svcprop -c -p config/opte_gateway "${SMF_FMRI}")"

HTTP_ADDR="$(svcprop -c -p config/http_address "${SMF_FMRI}")"
DNS_ADDR="$(svcprop -c -p config/dns_address "${SMF_FMRI}")"

# TODO: This should be its own service
# Set up OPTE interface
if [[ "$OPTE_GATEWAY" =~ .*:.* ]]; then
# IPv6 gateway
echo "IPv6 OPTE gateways are not yet supported"
exit 1
else
# IPv4 gateway
ipadm show-addr "$OPTE_INTERFACE/public" || ipadm create-addr -t -T dhcp "$OPTE_INTERFACE/public"
OPTE_IP=$(ipadm show-addr -p -o ADDR "$OPTE_INTERFACE/public" | cut -d'/' -f 1)
route get -host "$OPTE_GATEWAY" "$OPTE_IP" -interface -ifp "$OPTE_INTERFACE" || route add -host "$OPTE_GATEWAY" "$OPTE_IP" -interface -ifp "$OPTE_INTERFACE"
route get -inet default "$OPTE_GATEWAY" || route add -inet default "$OPTE_GATEWAY"
fi

# TODO: This command and args will be moved back to the start method on the manifest.xml file
args=(
"--config-file" "/var/svc/manifest/site/external_dns/config.toml"
"--http-address" "$HTTP_ADDR"
"--dns-address" "$DNS_ADDR"
)

ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server "${args[@]}" &

0 comments on commit 1ecff77

Please sign in to comment.