Configure permissionless builder in native and docker demo

.env

@@ -26,6 +26,7 @@ ESPRESSO_ORCHESTRATOR_MAX_PROPOSE_TIME=2s
 ESPRESSO_SEQUENCER_CDN_ENDPOINT=marshal-0:${ESPRESSO_CDN_SERVER_PORT}
 ESPRESSO_SEQUENCER_ORCHESTRATOR_URL=http://orchestrator:${ESPRESSO_ORCHESTRATOR_PORT}
 ESPRESSO_SEQUENCER_API_PORT=50000
+ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT=42000
 ESPRESSO_SEQUENCER1_API_PORT=50001
 ESPRESSO_SEQUENCER2_API_PORT=50002
 ESPRESSO_SEQUENCER3_API_PORT=50003
@@ -45,6 +46,7 @@ ESPRESSO_STATE_RELAY_SERVER_URL=http://state-relay-server:${ESPRESSO_STATE_RELAY
 # Ethereum accounts (note 11-15 are used by the sequencer nodes)
 ESPRESSO_SEQUENCER_HOTSHOT_ACCOUNT_INDEX=5
 ESPRESSO_SEQUENCER_STATE_PROVER_ACCOUNT_INDEX=7
+ESPRESSO_BUILDER_ETH_ACCOUNT_INDEX=8
 ESPRESSO_DEPLOYER_ACCOUNT_INDEX=9
 
 # Contracts
@@ -66,11 +68,11 @@ ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_3=SCHNORR_SIGNING_KEY~bUrtzmLjEVW9ggG7
 ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_4=SCHNORR_SIGNING_KEY~4YXyoT7ZxwXcU0_J6NH9ziGLrG0jShNCtGJIZ7a0yQBr
 
 # Sequencer demo: Libp2p (advertise and bind) ports
-ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0 = 7000
-ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_1 = 7001
-ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_2 = 7002
-ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_3 = 7003
-ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_4 = 7004
+ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0=7000
+ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_1=7001
+ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_2=7002
+ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_3=7003
+ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_4=7004
 
 # Foundry
 # The mnemonic used by foundry to deploy contracts.
@@ -89,8 +91,13 @@ ESPRESSO_PROVER_SERVICE_PORT=60001
 ESPRESSO_STATE_PROVER_UPDATE_INTERVAL=10m
 
 # Builder service
-ESPRESSO_BUILDER_SERVER_PORT=60003
-BUILDER_CHANNEL_CAPACITY=1024
+ESPRESSO_BUILDER_L1_PROVIDER=${ESPRESSO_SEQUENCER_L1_PROVIDER}
+ESPRESSO_BUILDER_ETH_MNEMONIC=${ESPRESSO_SEQUENCER_ETH_MNEMONIC}
+ESPRESSO_BUILDER_SERVER_PORT=41003
+ESPRESSO_BUILDER_PRIVATE_STAKING_KEY=BLS_SIGNING_KEY~tI9He_sCnEbfEajycUXz9Scfy6ocLr0yL9ceD53s8QPa
+ESPRESSO_BUILDER_PRIVATE_STATE_KEY=SCHNORR_SIGNING_KEY~IftHINvgzqcd9agX13HHY3Uhz8vsH46i8soKgV7ZUQV-
+ESPRESSO_BUILDER_CHANNEL_CAPACITY=1024
+ESPRESSO_BUILDER_BOOTSTRAPPED_VIEW=0
 
 # Load generator
 ESPRESSO_SUBMIT_TRANSACTIONS_DELAY=1s
@@ -109,4 +116,4 @@ FEE_CONTRACT_SALT=17
 LIGHT_CLIENT_SALT=12
 FEE_CONTRACT_UPGRADE_NAME="FeeContract.sol"
 LIGHT_CLIENT_UPGRADE_NAME="LightClientV2.sol"
-FOUNDRY_OUT=contracts/out
+FOUNDRY_OUT=contracts/out

builder/src/bin/permissionless-builder.rs

@@ -4,7 +4,6 @@ use builder::non_permissioned::{build_instance_state, BuilderConfig};
 use clap::Parser;
 use cld::ClDuration;
 use es_version::SEQUENCER_VERSION;
-use ethers::types::Address;
 use hotshot::types::{BLSPubKey, SignatureKey};
 use hotshot_types::data::ViewNumber;
 use hotshot_types::light_client::StateSignKey;
@@ -22,15 +21,14 @@ pub struct NonPermissionedBuilderOptions {
     #[clap(long, env = "ESPRESSO_SEQUENCER_CHAIN_ID", default_value = "0")]
     pub chain_id: u16,
 
-    /// URL of the HotShot DA web server.
+    /// URL of hotshot events API running on Espresso Sequencer DA committee node
     /// The builder will subscribe to this server to receive hotshot events
     #[clap(
-        short,
         long,
-        env = "ESPRESSO_SEQUENCER_DA_SERVER_URL",
+        env = "ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_URL",
         default_value = "http://localhost:8081"
     )]
-    pub da_server_url: Url,
+    pub sequencer_url: Url,
 
     /// URL of the Light Client State Relay Server
     #[clap(
@@ -102,19 +100,15 @@ pub struct NonPermissionedBuilderOptions {
     pub state_peers: Vec<Url>,
 
     /// Port to run the builder server on.
-    #[clap(short, long, env = "BUILDER_SERVER_PORT")]
+    #[clap(short, long, env = "ESPRESSO_BUILDER_SERVER_PORT")]
     pub port: u16,
 
-    /// Port to run the builder server on.
-    #[clap(short, long, env = "BUILDER_ADDRESS")]
-    pub address: Address,
-
     /// Bootstrapping View number
-    #[clap(short, long, env = "BUILDER_BOOTSTRAPPED_VIEW")]
+    #[clap(short, long, env = "ESPRESSO_BUILDER_BOOTSTRAPPED_VIEW")]
     pub view_number: u64,
 
     /// BUILDER CHANNEL CAPACITY
-    #[clap(short, long, env = "BUILDER_CHANNEL_CAPACITY")]
+    #[clap(short, long, env = "ESPRESSO_BUILDER_CHANNEL_CAPACITY")]
     pub channel_capacity: NonZeroUsize,
 }
 
@@ -196,10 +190,13 @@ async fn main() -> anyhow::Result<()> {
         bootstrapped_view,
         opt.channel_capacity,
         instance_state,
-        opt.da_server_url,
+        opt.sequencer_url,
         builder_server_url,
     )
     .await;
 
+    // Sleep forever
+    async_std::future::pending::<()>().await;
+
     Ok(())
 }

builder/src/lib.rs

@@ -447,7 +447,7 @@ pub mod testing {
 
             let mut app = App::<_, EventStreamApiError, Version01>::with_state(source);
 
-            app.register_module("hotshot_events", hotshot_events_api)
+            app.register_module("hotshot-events", hotshot_events_api)
                 .expect("Failed to register hotshot events API");
 
             async_spawn(app.serve(url, STATIC_VER_0_1));

builder/src/non_permissioned.rs

@@ -136,6 +136,10 @@ impl BuilderConfig {
 
         // create a client for it
         // Start Client for the event streaming api
+        tracing::info!(
+            "Builder client connecting to hotshot events API at {}",
+            hotshot_events_api_url.to_string()
+        );
         let client = Client::<EventStreamApiError, Version01>::new(hotshot_events_api_url.clone());
 
         assert!(client.connect(Some(Duration::from_secs(60))).await);
@@ -144,8 +148,7 @@ impl BuilderConfig {
 
         // client subscrive to hotshot events
         let subscribed_events = client
-            .socket("hotshot_events/events")
-            .header(ACCEPT, "application/octet-stream")
+            .socket("hotshot-events/events")
             .subscribe::<BuilderEvent<SeqTypes>>()
             .await
             .unwrap();

docker-compose.yaml

@@ -158,12 +158,14 @@ services:
     image: ghcr.io/espressosystems/espresso-sequencer/sequencer:main
     ports:
       - "$ESPRESSO_SEQUENCER_API_PORT:$ESPRESSO_SEQUENCER_API_PORT"
+      - "$ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT:$ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT"
     # Run the full API server with all modules, default storage
-    command: sequencer -- http -- query -- catchup -- status -- submit
+    command: sequencer -- http -- query -- catchup -- status -- submit -- hotshot-events
     environment:
       - ESPRESSO_SEQUENCER_ORCHESTRATOR_URL
       - ESPRESSO_SEQUENCER_CDN_ENDPOINT
       - ESPRESSO_SEQUENCER_API_PORT
+      - ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT
       - ESPRESSO_SEQUENCER_STATE_PEERS=http://sequencer1:$ESPRESSO_SEQUENCER_API_PORT
       - ESPRESSO_SEQUENCER_STORAGE_PATH
       - ESPRESSO_SEQUENCER_L1_PROVIDER
@@ -381,19 +383,20 @@ services:
         condition: service_healthy
 
   permissionless-builder:
-    image: ghcr.io/espressosystems/espresso-sequencer/builder:main
+    image: ghcr.io/espressosystems/espresso-sequencer/permissionless-builder:main
     ports:
       - "$ESPRESSO_BUILDER_SERVER_PORT:$ESPRESSO_BUILDER_SERVER_PORT"
     environment:
-      - ESPRESSO_SEQUENCER_URL
-      - ESPRESSO_SEQUENCER_DA_SERVER_URL
-      - ESPRESSO_BUILDER_KEY_FILE
+      - ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_URL=http://sequencer0:$ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT
+      - ESPRESSO_SEQUENCER_STATE_PEERS=http://sequencer0:$ESPRESSO_SEQUENCER_API_PORT
+      - ESPRESSO_BUILDER_PRIVATE_STAKING_KEY
+      - ESPRESSO_BUILDER_PRIVATE_STATE_KEY
       - ESPRESSO_BUILDER_ETH_MNEMONIC
       - ESPRESSO_BUILDER_ETH_ACCOUNT_INDEX
       - ESPRESSO_BUILDER_L1_PROVIDER
-      - BUILDER_SERVER_PORT=$ESPRESSO_BUILDER_SERVER_PORT
-      - BUILDER_ADDRESS
-      - BUILDER_CHANNEL_CAPACITY=$BUILDER_CHANNEL_CAPACITY
+      - ESPRESSO_BUILDER_SERVER_PORT
+      - ESPRESSO_BUILDER_CHANNEL_CAPACITY
+      - ESPRESSO_BUILDER_BOOTSTRAPPED_VIEW
       - RUST_LOG
       - RUST_LOG_FORMAT
       - ASYNC_STD_THREAD_COUNT

docker/permissionless-builder.Dockerfile

@@ -12,4 +12,4 @@ RUN chmod +x /bin/permissionless-builder
 
 ENV BUILDER_SERVER_PORT=60003
 
-CMD [ "/bin/permisisonless-builder"]
+CMD [ "/bin/permissionless-builder"]

process-compose.yaml

@@ -71,7 +71,7 @@ processes:
         condition: process_completed
 
   sequencer0:
-    command: sequencer -- http -- query -- catchup -- status -- submit
+    command: sequencer -- http -- query -- catchup -- status -- submit -- hotshot-events
     environment:
       - ESPRESSO_SEQUENCER_API_PORT=$ESPRESSO_SEQUENCER_API_PORT
       - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0
@@ -333,10 +333,12 @@ processes:
 
   permissionless-builder:
     command: permissionless-builder
+    environment:
+      - ESPRESSO_SEQUENCER_STATE_PEERS=http://localhost:$ESPRESSO_SEQUENCER_API_PORT
+      - ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_URL=http://localhost:$ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT
     depends_on:
       sequencer0:
         condition: process_healthy
-      da_server:
 
   nasty-client:
     command: nasty-client

scripts/build-docker-images-native

@@ -90,5 +90,5 @@ docker build --platform $PLATFORM -t ghcr.io/espressosystems/espresso-sequencer/
 docker build --platform $PLATFORM -t ghcr.io/espressosystems/espresso-sequencer/commitment-task:main -f docker/commitment-task.Dockerfile ${WORKDIR}
 docker build --platform $PLATFORM -t ghcr.io/espressosystems/espresso-sequencer/submit-transactions:main -f docker/submit-transactions.Dockerfile ${WORKDIR}
 docker build --platform $PLATFORM -t ghcr.io/espressosystems/espresso-sequencer/deploy:main -f docker/deploy.Dockerfile ${WORKDIR}
-docker build -t ghcr.io/espressosystems/espresso-sequencer/permissionless-builder:main -f docker/permissionless-builder.Dockerfile ${WORKDIR}
+docker build --platform $PLATFORM -t ghcr.io/espressosystems/espresso-sequencer/permissionless-builder:main -f docker/permissionless-builder.Dockerfile ${WORKDIR}
 docker build --platform $PLATFORM -t ghcr.io/espressosystems/espresso-sequencer/nasty-client:main -f docker/nasty-client.Dockerfile ${WORKDIR}

scripts/smoke-test-demo

@@ -2,8 +2,11 @@
 
 set -e
 
-SERVER=http://localhost:50000
-LOAD_GENERATOR=http://localhost:44010
+# load env vars in .env
+set -a; source .env; set +a;
+
+SEQUENCER_API=http://localhost:$ESPRESSO_SEQUENCER_API_PORT
+LOAD_GENERATOR=http://localhost:$ESPRESSO_SUBMIT_TRANSACTIONS_PORT
 TIMEOUT=60
 
 # Wait for the load generator to start.
@@ -14,8 +17,8 @@ done
 
 # Get the block height and number of transactions, wait some time, and check that these numbers have
 # both increased.
-block_height=`curl -s $SERVER/node/block-height`
-num_tx=`curl -s $SERVER/node/transactions/count`
+block_height=`curl -s $SEQUENCER_API/node/block-height`
+num_tx=`curl -s $SEQUENCER_API/node/transactions/count`
 
 echo "Initial state:"
 echo "  block_height: $block_height"
@@ -27,8 +30,8 @@ START=$SECONDS
 # Check blocks and transactions are increasing
 # Every second until timeout after 60 seconds
 while true; do
-	new_block_height=`curl -s $SERVER/node/block-height`
-	new_num_tx=`curl -s $SERVER/node/transactions/count`
+	new_block_height=`curl -s $SEQUENCER_API/node/block-height`
+	new_num_tx=`curl -s $SEQUENCER_API/node/transactions/count`
 	if [[ $new_block_height -gt $block_height && $new_num_tx -gt $num_tx ]]; then
 		echo "Final state:"
 		echo "  block_height: $new_block_height"
@@ -53,3 +56,11 @@ while true; do
 		exit 1
 	fi
 done
+
+echo "Checking if builder is healthy at http://localhost:$ESPRESSO_BUILDER_SERVER_PORT"
+curl -s --fail http://localhost:$ESPRESSO_BUILDER_SERVER_PORT > /dev/null
+
+# TODO, uncomment: currently the healthcheck may hang for a long time if the prover
+# is generating a proof. See https://github.com/EspressoSystems/espresso-sequencer/issues/1325
+# echo "Checking if prover is healthy at http://localhost:$ESPRESSO_PROVER_SERVICE_PORT"
+# curl -s --fail "http://localhost:$ESPRESSO_PROVER_SERVICE_PORT" > /dev/null