integ: use retries when applying toxics

The forked noxious-server takes a little longer to start on MacOS. Using
retries ensures we keep trying until the server is ready.

integ/failure-server/Cargo.toml

@@ -14,6 +14,7 @@ rand = "0.8"
 serde_json = "1.0"
 tempfile = "3.8"
 tokio = "1.0"
+tokio-retry = "0.3"
 tower = { version = "0.4", features = ["util"] }
 tower-fault = "0.0.5"
 tower-http = { version = "0.4", features = ["fs"] }

integ/failure-server/src/toxic/tcp_proxy.rs

@@ -8,6 +8,7 @@ use std::net::ToSocketAddrs;
 use std::process::Command;
 use std::{fmt::Debug, net::SocketAddr};
 use tempfile::NamedTempFile;
+use tokio_retry::{strategy::ExponentialBackoff, Retry};
 
 /// A TCP proxy server that introduces artificial faults at the TCP layer.
 #[derive(Debug)]
@@ -26,6 +27,10 @@ pub(crate) struct ToxicTcpProxy {
     toxics: Vec<Toxic>,
 }
 
+fn retry_strategy() -> impl Iterator<Item = std::time::Duration> {
+    ExponentialBackoff::from_millis(500).take(10)
+}
+
 impl ToxicTcpProxy {
     pub(crate) fn new<T1, T2, T3>(
         name: String,
@@ -92,15 +97,21 @@ impl ToxicTcpProxy {
 
         // Configure toxics
         let client = Client::new(&self.api_listen.to_string());
-        let proxy = client.proxy(&self.name).await.context(format!(
-            "Failed to find our configured proxy '{}'",
-            self.name
-        ))?;
+        let proxy = Retry::spawn(retry_strategy(), || async {
+            client.proxy(&self.name).await.context(format!(
+                "Failed to find our configured proxy '{}'",
+                self.name
+            ))
+        })
+        .await?;
         for toxic in &self.toxics {
-            proxy.add_toxic(toxic).await.context(format!(
-                "Failed to apply toxic {:?} to proxy '{}'",
-                toxic, self.name
-            ))?;
+            Retry::spawn(retry_strategy(), || async {
+                proxy.add_toxic(toxic).await.context(format!(
+                    "Failed to apply toxic {:?} to proxy '{}'",
+                    toxic, self.name
+                ))
+            })
+            .await?;
         }
 
         Ok(())