Use zone-network CLI to set up OPTE interface and package everything

karencfv · Feb 19, 2024 · dfca7ce · dfca7ce
1 parent 8150d2e
commit dfca7ce
Show file tree

Hide file tree

Showing 5 changed files with 89 additions and 81 deletions.
diff --git a/package-manifest.toml b/package-manifest.toml
@@ -120,7 +120,7 @@ setup_hint = """
 service_name = "oximeter"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "oximeter-collector.tar.gz", "zone-network-setup.tar.gz" ]
+source.packages = [ "oximeter-collector.tar.gz", "zone-network-setup.tar.gz", "zone-network-install.tar.gz" ]
 output.type = "zone"
 
 [package.oximeter-collector]
@@ -140,7 +140,12 @@ output.intermediate_only = true
 service_name = "clickhouse"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "clickhouse_svc.tar.gz", "internal-dns-cli.tar.gz", "zone-network-setup.tar.gz" ]
+source.packages = [
+  "clickhouse_svc.tar.gz",
+  "internal-dns-cli.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz"
+]
 output.type = "zone"
 
 [package.clickhouse_svc]
@@ -161,7 +166,12 @@ setup_hint = "Run `./tools/ci_download_clickhouse` to download the necessary bin
 service_name = "clickhouse_keeper"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "clickhouse_keeper_svc.tar.gz", "internal-dns-cli.tar.gz", "zone-network-setup.tar.gz" ]
+source.packages = [
+  "clickhouse_keeper_svc.tar.gz",
+  "internal-dns-cli.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz"
+]
 output.type = "zone"
 
 [package.clickhouse_keeper_svc]
@@ -182,7 +192,12 @@ setup_hint = "Run `./tools/ci_download_clickhouse` to download the necessary bin
 service_name = "cockroachdb"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "cockroachdb-service.tar.gz", "internal-dns-cli.tar.gz", "zone-network-setup.tar.gz" ]
+source.packages = [ 
+  "cockroachdb-service.tar.gz",
+  "internal-dns-cli.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz"
+]
 output.type = "zone"
 
 [package.cockroachdb-service]
@@ -220,7 +235,13 @@ output.type = "zone"
 service_name = "external_dns"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "dns-server.tar.gz", "external-dns-customizations.tar.gz", "zone-network-setup.tar.gz" ]
+source.packages = [ 
+  "dns-server.tar.gz",
+  "external-dns-customizations.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz",
+  "opte-interface-setup.tar.gz"
+]
 output.type = "zone"
 
 [package.dns-server]
@@ -247,7 +268,6 @@ only_for_targets.image = "standard"
 source.type = "local"
 source.paths = [ 
   { from = "smf/external-dns", to = "/var/svc/manifest/site/external_dns" },
-  # TODO: Removeme as we'll have a separate service for setting up OPTE interfaces
   { from = "smf/external-dns/method_script.sh", to = "/opt/oxide/lib/svc/manifest/external_dns.sh" },
 ]
 output.intermediate_only = true
@@ -630,14 +650,31 @@ source.packages = [
 ]
 output.type = "zone"
 
-[package.zone-network-setup]
+[package.zone-network-install]
 service_name = "zone-network-setup"
 only_for_targets.image = "standard"
 source.type = "local"
+source.paths = [
+    { from = "smf/zone-network-setup/manifest.xml", to = "/var/svc/manifest/site/zone-network-setup/manifest.xml" },
+]
+output.type = "zone"
+output.intermediate_only = true
+
+[package.zone-network-setup]
+service_name = "zone-network-cli"
+only_for_targets.image = "standard"
+source.type = "local"
 source.rust.binary_names = ["zone-networking"]
 source.rust.release = true
+output.type = "zone"
+output.intermediate_only = true
+
+[package.opte-interface-setup]
+service_name = "opte-interface-setup"
+only_for_targets.image = "standard"
+source.type = "local"
 source.paths = [
-    { from = "smf/zone-network-setup/manifest.xml", to = "/var/svc/manifest/site/zone-network-setup/manifest.xml" },
+    { from = "smf/opte-interface-setup/manifest.xml", to = "/var/svc/manifest/site/opte-interface-setup/manifest.xml" },
 ]
 output.type = "zone"
 output.intermediate_only = true

diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
@@ -1447,6 +1447,30 @@ impl ServiceManager {
             .add_instance(ServiceInstanceBuilder::new("default")))
     }
 
+    fn opte_interface_set_up_install(
+        zone: &InstalledZone,
+    ) -> Result<ServiceBuilder, Error> {
+        let port_idx = 0;
+        let port = zone.opte_ports().nth(port_idx).ok_or_else(|| {
+            Error::ZoneEnsureAddress(EnsureAddressError::MissingOptePort {
+                zone: String::from(zone.name()),
+                port_idx,
+            })
+        })?;
+
+        let opte_interface = port.vnic_name();
+        let opte_gateway = &port.gateway().ip().to_string();
+
+        let mut config_builder = PropertyGroupBuilder::new("config");
+        config_builder = config_builder
+            .add_property("interface", "astring", opte_interface)
+            .add_property("gateway", "astring", opte_gateway);
+
+        Ok(ServiceBuilder::new("oxide/opte-interface-setup")
+            .add_property_group(config_builder)
+            .add_instance(ServiceInstanceBuilder::new("default")))
+    }
+
     async fn initialize_zone(
         &self,
         request: ZoneArgs<'_>,
@@ -1851,45 +1875,6 @@ impl ServiceManager {
                     return Err(Error::SledAgentNotReady);
                 };
 
-                // Like Nexus, we need to be reachable externally via
-                // `dns_address` but we don't listen on that address
-                // directly but instead on a VPC private IP. OPTE will
-                // en/decapsulate as appropriate.
-                let port_idx = 0;
-                let port = installed_zone
-                    .opte_ports()
-                    .nth(port_idx)
-                    .ok_or_else(|| {
-                        Error::ZoneEnsureAddress(
-                            EnsureAddressError::MissingOptePort {
-                                zone: String::from(installed_zone.name()),
-                                port_idx,
-                            },
-                        )
-                    })?;
-
-                // TODO: These should be set up as part of the networking service perhaps?
-                // Nexus will likely need this as well
-                //
-                // OPTE_INTERFACE="$(svcprop -c -p config/opte_interface "${SMF_FMRI}")"
-                // OPTE_GATEWAY="$(svcprop -c -p config/opte_gateway "${SMF_FMRI}")"
-                //
-                // # Set up OPTE interface
-                // if [[ "$OPTE_GATEWAY" =~ .*:.* ]]; then
-                //     # IPv6 gateway
-                //     echo "IPv6 OPTE gateways are not yet supported"
-                //     exit 1
-                // else
-                //     # IPv4 gateway
-                //     ipadm show-addr "$OPTE_INTERFACE/public"  || ipadm create-addr -t -T dhcp "$OPTE_INTERFACE/public"
-                //     OPTE_IP=$(ipadm show-addr -p -o ADDR "$OPTE_INTERFACE/public" | cut -d'/' -f 1)
-                //     route get -host "$OPTE_GATEWAY" "$OPTE_IP" -interface -ifp "$OPTE_INTERFACE" || route add -host "$OPTE_GATEWAY" "$OPTE_IP" -interface -ifp "$OPTE_INTERFACE"
-                //     route get -inet default "$OPTE_GATEWAY" || route add -inet default "$OPTE_GATEWAY"
-                // fi
-                //
-                let opte_interface = port.vnic_name();
-                let opte_gateway = &port.gateway().ip().to_string();
-
                 let static_addr = underlay_address.to_string();
 
                 let nw_setup_service = Self::zone_network_setup_install(
@@ -1898,16 +1883,18 @@ impl ServiceManager {
                     &static_addr.clone(),
                 )?;
 
+                // Like Nexus, we need to be reachable externally via
+                // `dns_address` but we don't listen on that address
+                // directly but instead on a VPC private IP. OPTE will
+                // en/decapsulate as appropriate.
+                let opte_interface_setup = Self::opte_interface_set_up_install(&installed_zone)?;
+
                 let http_addr = format!("[{}]:{}", static_addr, DNS_HTTP_PORT);
-                let dns_addr = format!("{}", DNS_PORT);
+                let dns_port = format!("{}", DNS_PORT);
 
                 let external_dns_config = PropertyGroupBuilder::new("config")
-                    // TODO: Removeme and move to new opte interface service
-                    .add_property("opte_gateway", "astring", opte_gateway)
-                    .add_property("opte_interface", "astring", opte_interface)
-                    // TODO: Keep these two
                     .add_property("http_address", "astring", &http_addr)
-                    .add_property("dns_address", "astring", &dns_addr);
+                    .add_property("dns_port", "astring", &dns_port);
                 let external_dns_service =
                     ServiceBuilder::new("oxide/external_dns").add_instance(
                         ServiceInstanceBuilder::new("default")
@@ -1916,6 +1903,7 @@ impl ServiceManager {
 
                 let profile = ProfileBuilder::new("omicron")
                     .add_service(nw_setup_service)
+                    .add_service(opte_interface_setup)
                     .add_service(disabled_ssh_service)
                     .add_service(external_dns_service);
                 profile

diff --git a/smf/external-dns/manifest.xml b/smf/external-dns/manifest.xml
@@ -16,17 +16,19 @@
   <service_fmri value='svc:/oxide/zone-network-setup:default' />
   </dependency>
 
+  <dependency name='opte_interface_setup' grouping='require_all' restart_on='none'
+    type='service'>
+  <service_fmri value='svc:/oxide/opte-interface-setup:default' />
+  </dependency>
+
   <exec_method type='method' name='start'
       exec='/opt/oxide/lib/svc/manifest/external_dns.sh'
     timeout_seconds='0' />
   <exec_method type='method' name='stop' exec=':kill' timeout_seconds='0' />
 
   <property_group name='config' type='application'>
     <propval name='http_address' type='astring' value='unknown' />
-    <propval name='dns_address' type='astring' value='unknown' />
-    <!-- TODO: Removeme when new opte interface service exists -->
-    <propval name='opte_interface' type='astring' value='unknown' />
-    <propval name='opte_gateway' type='astring' value='unknown' />
+    <propval name='dns_port' type='astring' value='unknown' />
   </property_group>
 
   <property_group name='startd' type='framework'>

diff --git a/smf/external-dns/method_script.sh b/smf/external-dns/method_script.sh
@@ -6,34 +6,15 @@ set -o pipefail
 
 . /lib/svc/share/smf_include.sh
 
-# TODO: OPTE variables to be moved to opte-nw-service
-OPTE_INTERFACE="$(svcprop -c -p config/opte_interface "${SMF_FMRI}")"
-OPTE_GATEWAY="$(svcprop -c -p config/opte_gateway "${SMF_FMRI}")"
-
 HTTP_ADDR="$(svcprop -c -p config/http_address "${SMF_FMRI}")"
-# TODO: Rename DNS_ADDR to DNS_PORT if I don't find a way to retrieve OPTE_IP from the installed zone somehow
-DNS_ADDR="$(svcprop -c -p config/dns_address "${SMF_FMRI}")"
-
-# TODO: This should be its own service
-# Set up OPTE interface
-if [[ "$OPTE_GATEWAY" =~ .*:.* ]]; then
-    # IPv6 gateway
-    echo "IPv6 OPTE gateways are not yet supported"
-    exit 1
-else
-    # IPv4 gateway
-    ipadm show-addr "$OPTE_INTERFACE/public"  || ipadm create-addr -t -T dhcp "$OPTE_INTERFACE/public"
-    # TODO: Perhaps only keep this line to retrieve the IP which will be used by --dns-address?
-    OPTE_IP=$(ipadm show-addr -p -o ADDR "$OPTE_INTERFACE/public" | cut -d'/' -f 1)
-    route get -host "$OPTE_GATEWAY" "$OPTE_IP" -interface -ifp "$OPTE_INTERFACE" || route add -host "$OPTE_GATEWAY" "$OPTE_IP" -interface -ifp "$OPTE_INTERFACE"
-    route get -inet default "$OPTE_GATEWAY" || route add -inet default "$OPTE_GATEWAY"
-fi
+DNS_PORT="$(svcprop -c -p config/dns_port "${SMF_FMRI}")"
+# TODO: Replace for zonw nw CLI command?
+OPTE_IP=$(ipadm show-addr -p -o ADDR "$OPTE_INTERFACE/public" | cut -d'/' -f 1)
 
-# TODO: This command and args will be moved back to the start method on the manifest.xml file
 args=(
   "--config-file" "/var/svc/manifest/site/external_dns/config.toml"
   "--http-address" "$HTTP_ADDR"
-  "--dns-address" "$OPTE_IP:$DNS_ADDR"
+  "--dns-address" "$OPTE_IP:$DNS_PORT"
 )
 
 exec /opt/oxide/dns-server/bin/dns-server "${args[@]}" &
diff --git a/smf/zone-network-setup/manifest.xml b/smf/zone-network-setup/manifest.xml
@@ -18,7 +18,7 @@
   </dependency>
 
   <exec_method type='method' name='start'
-    exec='/opt/oxide/zone-network-setup/bin/zone-networking set-up -d %{config/datalink} -s %{config/static_addr} -g %{config/gateway}'
+    exec='/opt/oxide/zone-network-cli/bin/zone-networking set-up -d %{config/datalink} -s %{config/static_addr} -g %{config/gateway}'
     timeout_seconds='0' />
 
   <property_group name='startd' type='framework'>