Add Node Validator API (#1771)

Closes #1778
<!-- These comments should help create a useful PR message, please
delete any remaining comments before opening the PR. -->
<!-- If there is no issue number make sure to describe clearly *why*
this PR is necessary. -->
<!-- Mention open questions, remaining TODOs, if any -->

### This PR:
Provides a versioned implementation of the Node Validator API. This API
is meant to be used in conjunction with the Node Validator UI that has
been designed on the Block Explorer.

The create `node-metrics` was chosen over `node-validator` based on a
recommendation from @clu8, as the crate and systems may be utilized
without the node-validator aspects.
<!-- Describe what this PR adds to this repo and why -->
<!-- E.g. -->
<!-- * Implements feature 1 -->
<!-- * Fixes bug 3 -->

<!-- ### This PR does not: -->
<!-- Describe what is out of scope for this PR, if applicable. Leave
this section blank if it's not applicable -->
<!-- This section helps avoid the reviewer having to needlessly point
out missing parts -->
<!-- * Implement feature 3 because that feature is blocked by Issue 4
-->
<!-- * Implement xyz because that is tracked in issue #123. -->
<!-- * Address xzy for which I opened issue #456 -->

### Key places to review:
<!-- Describe key places for reviewers to pay close attention to -->
<!-- * file.rs, `add_integers` function -->
<!-- Or directly comment on those files/lines to make it easier for the
reviewers -->
node-metrics/src/service/data_state/mod.rs
node-metrics/src/service/client_state/mod.rs

Usage example test is located in
node-metrics/src/api/node_validator/v0/create_node_validator_api.rs

<!-- ### How to test this PR:  -->
<!-- Optional, uncomment the above line if this is relevant to your PR
-->
<!-- If your PR is fully tested through CI there is no need to add this
section -->
<!-- * E.g. `just test` -->

<!-- ### Things tested -->
<!-- Anything that was manually tested (that is not tested in CI). -->
<!-- E.g. building/running of docker containers. Changes to docker demo,
... -->
<!-- Especially mention anything untested, with reasoning and link an
issue to resolve this. -->

<!-- Complete the following items before creating this PR -->
<!-- [ ] Issue linked or PR description mentions why this change is
necessary. -->
<!-- [ ] PR description is clear enough for reviewers. -->
<!-- [ ] Documentation for changes (additions) has been updated (added).
-->
<!-- [ ] If this is a draft it is marked as "draft".  -->

<!-- To make changes to this template edit
https://github.com/EspressoSystems/.github/blob/main/PULL_REQUEST_TEMPLATE.md
-->

.env

@@ -37,6 +37,8 @@ ESPRESSO_SEQUENCER_GENESIS_FILE=/genesis/demo.toml
 ESPRESSO_SEQUENCER_L1_PORT=8545
 ESPRESSO_SEQUENCER_L1_WS_PORT=8546
 ESPRESSO_SEQUENCER_L1_PROVIDER=http://demo-l1-network:${ESPRESSO_SEQUENCER_L1_PORT}
+ESPRESSO_NODE_VALIDATOR_PORT=9000
+
 # Only allow 1 block to be processed for events at a time, simulating a very bad L1 provider.
 ESPRESSO_SEQUENCER_L1_EVENTS_MAX_BLOCK_RANGE=1
 ESPRESSO_SEQUENCER_ETH_MNEMONIC="test test test test test test test test test test test junk"

.github/workflows/build.yml

@@ -81,6 +81,7 @@ jobs:
             target/release/pub-key
             target/release/espresso-bridge
             target/release/marketplace-solver
+            target/release/node-metrics
 
   build-arm:
     runs-on: buildjet-4vcpu-ubuntu-2204-arm
@@ -133,6 +134,7 @@ jobs:
             target/release/pub-key
             target/release/espresso-bridge
             target/release/marketplace-solver
+            target/release/node-metrics
 
   build-dockers:
     runs-on: ubuntu-latest
@@ -153,6 +155,7 @@ jobs:
       espresso-dev-node-tag: ${{ steps.espresso-dev-node.outputs.tags }}
       bridge-tag: ${{ steps.bridge.outputs.tags }}
       marketplace-solver-tag: ${{ steps.marketplace-solver.outputs.tags }}
+      node-validator-tag: ${{ steps.node-validator.outputs.tags }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -272,6 +275,12 @@ jobs:
         with:
           images: ghcr.io/espressosystems/espresso-sequencer/marketplace-solver
 
+      - name: Generate node-validator metadata
+        uses: docker/metadata-action@v5
+        id: node-validator
+        with:
+          images: ghcr.io/espressosystems/espresso-sequencer/node-validator
+
       - name: Build and push sequencer docker
         uses: docker/build-push-action@v6
         with:
@@ -421,6 +430,16 @@ jobs:
           tags: ${{ steps.marketplace-solver.outputs.tags }}
           labels: ${{ steps.marketplace-solver.outputs.labels }}
 
+      - name: Build and push node-validator docker
+        uses: docker/build-push-action@v6
+        with:
+          context: ./
+          file: ./docker/node-validator.Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.node-validator.outputs.tags }}
+          labels: ${{ steps.node-validator.outputs.labels }}
+
   test-demo:
     if: ${{ github.event_name != 'pull_request' }}
     runs-on: ubuntu-latest
@@ -450,6 +469,7 @@ jobs:
           docker pull ${{ needs.build-dockers.outputs.nasty-client-tag }}
           docker pull ${{ needs.build-dockers.outputs.bridge-tag }}
           docker pull ${{ needs.build-dockers.outputs.marketplace-solver-tag }}
+          docker pull ${{ needs.build-dockers.outputs.node-validator-tag }}
           
       - name: Tag new docker images
         run: |
@@ -467,6 +487,7 @@ jobs:
           docker tag ${{ needs.build-dockers.outputs.nasty-client-tag }} ghcr.io/espressosystems/espresso-sequencer/nasty-client:main
           docker tag ${{ needs.build-dockers.outputs.bridge-tag }} ghcr.io/espressosystems/espresso-sequencer/bridge:main
           docker tag ${{ needs.build-dockers.outputs.marketplace-solver-tag }} ghcr.io/espressosystems/espresso-sequencer/marketplace-solver:main
+          docker tag ${{ needs.build-dockers.outputs.node-validator-tag }} ghcr.io/espressosystems/espresso-sequencer/node-validator:main
 
       - name: Test docker demo
         run: |

.github/workflows/build_static.yml

@@ -92,6 +92,7 @@ jobs:
             ${{ env.CARGO_TARGET_DIR }}/${{ env.TARGET_TRIPLET }}/release/espresso-bridge
             ${{ env.CARGO_TARGET_DIR }}/${{ env.TARGET_TRIPLET }}/release/espresso-dev-node
              ${{ env.CARGO_TARGET_DIR }}/${{ env.TARGET_TRIPLET }}/release/marketplace-solver
+            ${{ env.CARGO_TARGET_DIR }}/${{ env.TARGET_TRIPLET }}/release/node-metrics
 
   static-dockers:
     runs-on: ubuntu-latest
@@ -216,6 +217,13 @@ jobs:
           images: ghcr.io/espressosystems/espresso-sequencer/marketplace-solver
           flavor: suffix=musl
 
+      - name: Generate node-validator metadata
+        uses: docker/metadata-action@v5
+        id: node-validator
+        with:
+          images: ghcr.io/espressosystems/espresso-sequencer/node-validator
+          flavor: suffix=musl
+
       - name: Build and push sequencer docker
         uses: docker/build-push-action@v6
         with:
@@ -344,4 +352,14 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.marketplace-solver.outputs.tags }}
-          labels: ${{ steps.marketplace-solver.outputs.labels }}
+          labels: ${{ steps.marketplace-solver.outputs.labels }}
+
+      - name: Build and push node-validator docker
+        uses: docker/build-push-action@v6
+        with:
+          context: ./
+          file: ./docker/node-validator.Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.node-validator.outputs.tags }}
+          labels: ${{ steps.node-validator.outputs.labels }}

.typos.toml

@@ -4,4 +4,5 @@ extend-exclude = [
   "doc/*.svg",
   "contracts/lib",
   "contract-bindings",
+  "node-metrics/src/api/node_validator/v0/example_prometheus_metrics_output.txt",
 ]

Cargo.toml

@@ -15,6 +15,7 @@ members = [
     "contracts/rust/gen-vk-contract",
     "hotshot-state-prover",
     "marketplace-solver",
+    "node-metrics",
     "sequencer",
     "types",
     "utils",
@@ -31,16 +32,18 @@ ark-ff = "0.4"
 ark-poly = "0.4"
 ark-serialize = "0.4"
 ark-srs = "0.3.1"
-async-compatibility-layer = { version = "1.1", default-features = false, features = [
-    "logging-utils",
+async-compatibility-layer = { version = "1.2.1", default-features = false, features = [
+  "logging-utils",
 ] }
 async-once-cell = "0.5"
 async-std = { version = "1.12.0", features = ["attributes", "tokio1"] }
 async-trait = "0.1"
 base64 = "0.22"
 base64-bytes = "0.1"
 bincode = "1.3.3"
+bitvec = "1.0.1"
 blake3 = "1.5"
+circular-buffer = "0.1.7"
 clap = { version = "4.4", features = ["derive", "env", "string"] }
 cld = "0.5"
 derive_more = "0.99.17"
@@ -133,3 +136,4 @@ paste = "1.0"
 rand = "0.8.5"
 time = "0.3"
 trait-set = "0.3.0"
+

builder/src/bin/permissioned-builder.rs

@@ -256,6 +256,7 @@ async fn main() -> anyhow::Result<()> {
         private_staking_key: private_staking_key.clone(),
         private_state_key,
         state_peers: opt.state_peers,
+        public_api_url: None,
         config_peers: None,
         catchup_backoff: Default::default(),
     };

docker-compose.yaml

@@ -236,6 +236,16 @@ services:
       - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0
       - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=sequencer0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_0
       - ESPRESSO_SEQUENCER_IS_DA=true
+      - ESPRESSO_SEQUENCER_IDENTITY_NODE_NAME=sequencer0
+      - ESPRESSO_SEQUENCER_IDENTITY_WALLET_ADDRESS=0x0000000000000000000000000000000000000000
+      - ESPRESSO_SEQUENCER_IDENTITY_COMPANY_NAME=Espresso Systems
+      - ESPRESSO_SEQUENCER_IDENTITY_COMPANY_WEBSITE=https://www.espressosys.com/
+      - ESPRESSO_SEQUENCER_IDENTITY_OPERATING_SYSTEM=Linux 5.15.153.1
+      - ESPRESSO_SEQUENCER_IDENTITY_NETWORK_TYPE=local
+      - ESPRESSO_SEQUENCER_IDENTITY_COUNTRY_CODE=US
+      - ESPRESSO_SEQUENCER_IDENTITY_LATITUDE=40.7128
+      - ESPRESSO_SEQUENCER_IDENTITY_LONGITUDE=-74.0060
+      - ESPRESSO_SEQUENCER_PUBLIC_API_URL=http://sequencer0:$ESPRESSO_SEQUENCER_API_PORT/
       - RUST_LOG
       - RUST_LOG_FORMAT
       - ASYNC_STD_THREAD_COUNT
@@ -282,6 +292,16 @@ services:
       - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_1
       - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=sequencer1:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_1
       - ESPRESSO_SEQUENCER_IS_DA=true
+      - ESPRESSO_SEQUENCER_IDENTITY_NODE_NAME=sequencer1
+      - ESPRESSO_SEQUENCER_IDENTITY_WALLET_ADDRESS=0x0000000000000000000000000000000000000001
+      - ESPRESSO_SEQUENCER_IDENTITY_COMPANY_NAME=Espresso Systems
+      - ESPRESSO_SEQUENCER_IDENTITY_COMPANY_WEBSITE=https://www.espressosys.com/
+      - ESPRESSO_SEQUENCER_IDENTITY_OPERATING_SYSTEM=Darwin 23.5.0
+      - ESPRESSO_SEQUENCER_IDENTITY_NETWORK_TYPE=local
+      - ESPRESSO_SEQUENCER_IDENTITY_COUNTRY_CODE=GR
+      - ESPRESSO_SEQUENCER_IDENTITY_LATITUDE=39.0742
+      - ESPRESSO_SEQUENCER_IDENTITY_LONGITUDE=21.8243
+      - ESPRESSO_SEQUENCER_PUBLIC_API_URL=http://sequencer1:$ESPRESSO_SEQUENCER_API_PORT/
       - RUST_LOG
       - RUST_LOG_FORMAT
       - ASYNC_STD_THREAD_COUNT
@@ -324,6 +344,16 @@ services:
       - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_2
       - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=sequencer2:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_2
       - ESPRESSO_SEQUENCER_IS_DA=true
+      - ESPRESSO_SEQUENCER_IDENTITY_NODE_NAME=sequencer2
+      - ESPRESSO_SEQUENCER_IDENTITY_WALLET_ADDRESS=0x0000000000000000000000000000000000000002
+      - ESPRESSO_SEQUENCER_IDENTITY_COMPANY_NAME=Espresso Systems
+      - ESPRESSO_SEQUENCER_IDENTITY_COMPANY_WEBSITE=https://www.espressosys.com/
+      - ESPRESSO_SEQUENCER_IDENTITY_OPERATING_SYSTEM=Darwin 23.5.0
+      - ESPRESSO_SEQUENCER_IDENTITY_NETWORK_TYPE=local
+      - ESPRESSO_SEQUENCER_IDENTITY_COUNTRY_CODE=CN
+      - ESPRESSO_SEQUENCER_IDENTITY_LATITUDE=35.8617
+      - ESPRESSO_SEQUENCER_IDENTITY_LONGITUDE=104.1954
+      - ESPRESSO_SEQUENCER_PUBLIC_API_URL=http://sequencer2:$ESPRESSO_SEQUENCER_API_PORT/
       - RUST_LOG
       - RUST_LOG_FORMAT
       - ASYNC_STD_THREAD_COUNT
@@ -362,6 +392,17 @@ services:
       - ESPRESSO_SEQUENCER_PRIVATE_STATE_KEY=$ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_3
       - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_3
       - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=sequencer3:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_3
+      - ESPRESSO_SEQUENCER_IDENTITY_NODE_NAME=sequencer3
+      - ESPRESSO_SEQUENCER_IDENTITY_WALLET_ADDRESS=0x0000000000000000000000000000000000000003
+      - ESPRESSO_SEQUENCER_IDENTITY_COMPANY_NAME=Espresso Systems
+      - ESPRESSO_SEQUENCER_IDENTITY_COMPANY_WEBSITE=https://www.espressosys.com/
+      - ESPRESSO_SEQUENCER_IDENTITY_OPERATING_SYSTEM=Microsoft Windows NT 10.0.22621.0
+      - [email protected]
+      - ESPRESSO_SEQUENCER_IDENTITY_NETWORK_TYPE=local
+      - ESPRESSO_SEQUENCER_IDENTITY_COUNTRY_CODE=CN
+      - ESPRESSO_SEQUENCER_IDENTITY_LATITUDE=35.8617
+      - ESPRESSO_SEQUENCER_IDENTITY_LONGITUDE=104.1954
+      - ESPRESSO_SEQUENCER_PUBLIC_API_URL=http://sequencer3:$ESPRESSO_SEQUENCER_API_PORT/
       - RUST_LOG
       - RUST_LOG_FORMAT
       - ASYNC_STD_THREAD_COUNT
@@ -400,6 +441,16 @@ services:
       - ESPRESSO_SEQUENCER_PRIVATE_STATE_KEY=$ESPRESSO_DEMO_SEQUENCER_STATE_PRIVATE_KEY_4
       - ESPRESSO_SEQUENCER_LIBP2P_BIND_ADDRESS=0.0.0.0:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_4
       - ESPRESSO_SEQUENCER_LIBP2P_ADVERTISE_ADDRESS=sequencer4:$ESPRESSO_DEMO_SEQUENCER_LIBP2P_PORT_4
+      - ESPRESSO_SEQUENCER_IDENTITY_NODE_NAME=sequencer4
+      - ESPRESSO_SEQUENCER_IDENTITY_WALLET_ADDRESS=0x0000000000000000000000000000000000000004
+      - ESPRESSO_SEQUENCER_IDENTITY_COMPANY_NAME=Espresso Systems
+      - ESPRESSO_SEQUENCER_IDENTITY_COMPANY_WEBSITE=https://www.espressosys.com/
+      - ESPRESSO_SEQUENCER_IDENTITY_OPERATING_SYSTEM=TempleOS 5.03
+      - ESPRESSO_SEQUENCER_IDENTITY_NETWORK_TYPE=local
+      - ESPRESSO_SEQUENCER_IDENTITY_COUNTRY_CODE=AU
+      - ESPRESSO_SEQUENCER_IDENTITY_LATITUDE=-25.2744
+      - ESPRESSO_SEQUENCER_IDENTITY_LONGITUDE=133.7751
+      - ESPRESSO_SEQUENCER_PUBLIC_API_URL=http://sequencer4:$ESPRESSO_SEQUENCER_API_PORT/
       - RUST_LOG
       - RUST_LOG_FORMAT
       - ASYNC_STD_THREAD_COUNT
@@ -542,6 +593,26 @@ services:
       solver-db:
         condition: service_healthy
 
+  node-validator:
+    image: ghcr.io/espressosystems/espresso-sequencer/node-validator:main
+    ports:
+      - "$ESPRESSO_NODE_VALIDATOR_PORT:$ESPRESSO_NODE_VALIDATOR_PORT"
+    environment:
+      - ESPRESSO_NODE_VALIDATOR_STAKE_TABLE_SOURCE_BASE_URL=http://sequencer0:$ESPRESSO_SEQUENCER_API_PORT/v0/
+      - ESPRESSO_NODE_VALIDATOR_LEAF_STREAM_SOURCE_BASE_URL=http://sequencer0:$ESPRESSO_SEQUENCER_API_PORT/v0/
+      - ESPRESSO_NODE_VALIDATOR_INITIAL_NODE_PUBLIC_BASE_URLS=http://sequencer0:$ESPRESSO_SEQUENCER_API_PORT,http://sequencer1:$ESPRESSO_SEQUENCER1_API_PORT,http://sequencer2:$ESPRESSO_SEQUENCER2_API_PORT,http://sequencer3:$ESPRESSO_SEQUENCER3_API_PORT,http://sequencer4:$ESPRESSO_SEQUENCER4_API_PORT
+    depends_on:
+      sequencer0:
+        condition: service_healthy
+      sequencer1:
+        condition: service_healthy
+      sequencer2:
+        condition: service_healthy
+      sequencer3:
+        condition: service_healthy
+      sequencer4:
+        condition: service_healthy
+
   sequencer-db-0:
     image: postgres
     user: root

docker/node-validator.Dockerfile

@@ -0,0 +1,18 @@
+FROM ubuntu:jammy
+
+ARG TARGETARCH
+
+RUN apt-get update \
+    &&  apt-get install -y curl libcurl4 wait-for-it tini \
+    &&  rm -rf /var/lib/apt/lists/*
+ENTRYPOINT ["tini", "--"]
+
+COPY target/$TARGETARCH/release/node-metrics /bin/node-metrics
+RUN chmod +x /bin/node-metrics
+
+# Run a web server on this port by default. Port can be overridden by the container orchestrator.
+ENV ESPRESSO_NODE_VALIDATOR_PORT=80
+
+CMD [ "/bin/node-metrics"]
+HEALTHCHECK --interval=1s --timeout=1s --retries=100 CMD curl --fail http://localhost:${ESPRESSO_NODE_VALIDATOR_PORT}/healthcheck  || exit 1
+EXPOSE ${ESPRESSO_NODE_VALIDATOR_PORT}