From c0675720b6a412dfad1ca58e77af7d72e23aff69 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Tue, 28 May 2024 10:05:39 +0500 Subject: [PATCH 01/48] add ResolvableChainConfig to ValidatedState --- Cargo.lock | 730 +++++++++++++--------- Cargo.toml | 20 +- builder/src/bin/permissionless-builder.rs | 6 + builder/src/lib.rs | 6 +- builder/src/non_permissioned.rs | 11 +- builder/src/permissioned.rs | 17 +- sequencer/src/block.rs | 20 +- sequencer/src/block/payload.rs | 33 +- sequencer/src/block/queryable.rs | 4 +- sequencer/src/header.rs | 15 +- sequencer/src/hotshot_commitment.rs | 20 +- sequencer/src/lib.rs | 6 +- sequencer/src/message_compat_tests.rs | 24 +- sequencer/src/persistence.rs | 15 +- sequencer/src/reference_tests.rs | 2 +- sequencer/src/state.rs | 34 +- 16 files changed, 580 insertions(+), 383 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58a9c12f22..a061cb018a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -221,9 +221,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.83" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "arbitrary" @@ -541,8 +541,24 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" dependencies = [ - "asn1-rs-derive", - "asn1-rs-impl", + "asn1-rs-derive 0.4.0", + "asn1-rs-impl 0.1.0", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror", + "time 0.3.36", +] + +[[package]] +name = "asn1-rs" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ad1373757efa0f70ec53939aabc7152e1591cb485208052993070ac8d2429d" +dependencies = [ + "asn1-rs-derive 0.5.0", + "asn1-rs-impl 0.2.0", "displaydoc", "nom", "num-traits", @@ -563,6 +579,18 @@ dependencies = [ "synstructure 0.12.6", ] +[[package]] +name = "asn1-rs-derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7378575ff571966e99a744addeff0bff98b8ada0dedf1956d59e634db95eaac1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "synstructure 0.13.1", +] + [[package]] name = "asn1-rs-impl" version = "0.1.0" @@ -574,6 +602,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "asn1_der" version = "0.7.6" @@ -615,12 +654,11 @@ dependencies = [ [[package]] name = "async-channel" -version = "2.2.1" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d4d23bcc79e27423727b36823d86233aad06dfea531837b038394d11e9928" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" dependencies = [ "concurrent-queue", - "event-listener 5.3.0", "event-listener-strategy 0.5.2", "futures-core", "pin-project-lite 0.2.14", @@ -658,9 +696,9 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b10202063978b3351199d68f8b22c4e47e4b1b822f8d43fd862d5ea8c006b29a" +checksum = "c8828ec6e544c02b0d6691d21ed9f9218d0384a82542855073c2a3f58304aaf0" dependencies = [ "async-task", "concurrent-queue", @@ -687,7 +725,7 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ - "async-channel 2.2.1", + "async-channel 2.3.1", "async-executor", "async-io 2.3.2", "async-lock 3.3.0", @@ -913,7 +951,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -943,7 +981,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1046,7 +1084,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1063,7 +1101,7 @@ checksum = "edf3ee19dbc0a46d740f6f0926bde8c50f02bdbc7b536842da28f6ac56513a8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1281,12 +1319,11 @@ dependencies = [ [[package]] name = "blocking" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "495f7104e962b7356f0aeb34247aca1fe7d2e783b346582db7f2904cb5717e88" +checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" dependencies = [ - "async-channel 2.2.1", - "async-lock 3.3.0", + "async-channel 2.3.1", "async-task", "futures-io", "futures-lite 2.3.0", @@ -1346,7 +1383,7 @@ dependencies = [ "rand 0.8.5", "sequencer", "serde", - "snafu 0.8.2", + "snafu 0.8.3", "surf", "surf-disco", "tagged-base64", @@ -1440,18 +1477,18 @@ dependencies = [ [[package]] name = "camino" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239" dependencies = [ "serde", ] [[package]] name = "capnp" -version = "0.19.3" +version = "0.19.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b11832e6fb7a695c4a63cc42bd97bd2cda7165cd850caf5aff9a3d0e617720ed" +checksum = "3aed85272154b3c0bfda873c40395f13adcfbc89696bf639a512291077f8cd17" dependencies = [ "embedded-io", ] @@ -1482,7 +1519,7 @@ checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" dependencies = [ "camino", "cargo-platform", - "semver 1.0.22", + "semver 1.0.23", "serde", "serde_json", "thiserror", @@ -1499,9 +1536,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" dependencies = [ "jobserver", "libc", @@ -1579,7 +1616,7 @@ dependencies = [ "num_enum", "pem 3.0.4", "prometheus", - "quinn", + "quinn 0.10.2", "rand 0.8.5", "rcgen 0.12.1", "redis", @@ -1677,7 +1714,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.1", + "strsim", ] [[package]] @@ -1689,7 +1726,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1889,9 +1926,9 @@ dependencies = [ [[package]] name = "const-hex" -version = "1.11.3" +version = "1.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ba00838774b4ab0233e355d26710fbfc8327a05c017f6dc4873f876d1f79f78" +checksum = "70ff96486ccc291d36a958107caf2c0af8c78c0af7d31ae2f35ce055130de1a6" dependencies = [ "cfg-if", "cpufeatures", @@ -2047,18 +2084,18 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ "crossbeam-utils", ] @@ -2093,9 +2130,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crunchy" @@ -2252,7 +2289,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -2294,15 +2331,15 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", "synstructure 0.13.1", ] [[package]] name = "darling" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" +checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" dependencies = [ "darling_core", "darling_macro", @@ -2310,27 +2347,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" +checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", - "syn 2.0.60", + "strsim", + "syn 2.0.66", ] [[package]] name = "darling_macro" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" +checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" dependencies = [ "darling_core", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -2389,7 +2426,21 @@ version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" dependencies = [ - "asn1-rs", + "asn1-rs 0.5.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs 0.6.1", "displaydoc", "nom", "num-bigint", @@ -2426,7 +2477,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -2447,7 +2498,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -2457,7 +2508,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ "derive_builder_core", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -2607,7 +2658,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -2707,9 +2758,9 @@ checksum = "bbbaaaf38131deb9ca518a274a45bfdb8771f139517b073b16c2d3d32ae5037b" [[package]] name = "either" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" dependencies = [ "serde", ] @@ -2784,7 +2835,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -2795,18 +2846,19 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "erased-serde" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b73807008a3c7f171cc40312f37d95ef0396e048b5848d775f54b1a4dd4a0d3" +checksum = "24e2389d65ab4fab27dc2a5de7b191e1f6617d1f1c8855c0dc569c94a4cbb18d" dependencies = [ "serde", + "typeid", ] [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2987,7 +3039,7 @@ dependencies = [ "reqwest 0.11.27", "serde", "serde_json", - "syn 2.0.60", + "syn 2.0.66", "toml", "walkdir", ] @@ -3005,7 +3057,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -3031,7 +3083,7 @@ dependencies = [ "serde", "serde_json", "strum", - "syn 2.0.60", + "syn 2.0.66", "tempfile", "thiserror", "tiny-keccak", @@ -3048,7 +3100,7 @@ dependencies = [ "ethers-core", "ethers-solc", "reqwest 0.11.27", - "semver 1.0.22", + "semver 1.0.23", "serde", "serde_json", "thiserror", @@ -3157,7 +3209,7 @@ dependencies = [ "path-slash", "rayon", "regex", - "semver 1.0.22", + "semver 1.0.23", "serde", "serde_json", "solang-parser", @@ -3272,15 +3324,9 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38793c55593b33412e3ae40c2c9781ffaa6f438f6f8c10f24e71846fbd7ae01e" - -[[package]] -name = "finl_unicode" -version = "1.2.0" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "fixed-hash" @@ -3495,7 +3541,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -3508,6 +3554,17 @@ dependencies = [ "rustls 0.21.12", ] +[[package]] +name = "futures-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" +dependencies = [ + "futures-io", + "rustls 0.23.8", + "rustls-pki-types", +] + [[package]] name = "futures-sink" version = "0.3.30" @@ -3691,15 +3748,15 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" dependencies = [ + "atomic-waker", "bytes 1.6.0", "fnv", "futures-core", "futures-sink", - "futures-util", "http 1.1.0", "indexmap 2.2.6", "slab", @@ -3958,7 +4015,7 @@ dependencies = [ [[package]] name = "hotshot" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.55#81842b54a699a255645ac195354a0feb08fdc9d6" +source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" dependencies = [ "anyhow", "async-broadcast", @@ -3992,7 +4049,7 @@ dependencies = [ "rand 0.8.5", "serde", "sha2 0.10.8", - "snafu 0.8.2", + "snafu 0.8.3", "surf-disco", "time 0.3.36", "tokio", @@ -4003,7 +4060,7 @@ dependencies = [ [[package]] name = "hotshot-builder-api" version = "0.1.7" -source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.55#81842b54a699a255645ac195354a0feb08fdc9d6" +source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" dependencies = [ "async-trait", "clap", @@ -4012,7 +4069,7 @@ dependencies = [ "futures", "hotshot-types", "serde", - "snafu 0.8.2", + "snafu 0.8.3", "tagged-base64", "tide-disco", "toml", @@ -4022,7 +4079,7 @@ dependencies = [ [[package]] name = "hotshot-builder-core" version = "0.1.21" -source = "git+https://github.com/EspressoSystems/hotshot-builder-core?tag=0.1.23#978778b3852d57f0239cdb448a097845e67024a6" +source = "git+https://github.com/EspressoSystems/hotshot-builder-core?branch=abdul/update-hotshot#86db8fad09399489b002c3023d741fa09314f9b3" dependencies = [ "anyhow", "async-broadcast", @@ -4040,7 +4097,7 @@ dependencies = [ "hotshot-types", "serde", "sha2 0.10.8", - "snafu 0.8.2", + "snafu 0.8.3", "surf-disco", "tagged-base64", "tide-disco", @@ -4075,7 +4132,7 @@ dependencies = [ [[package]] name = "hotshot-events-service" version = "0.1.22" -source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?tag=0.1.23#7049c81d8e697894803ea76e7128d0aca7fa47c9" +source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?branch=abdul/update-hotshot#ef524944a9ef7c1b037d907ad9db70411e33936a" dependencies = [ "async-broadcast", "async-compatibility-layer", @@ -4089,7 +4146,7 @@ dependencies = [ "futures", "hotshot-types", "serde", - "snafu 0.8.2", + "snafu 0.8.3", "tagged-base64", "tide-disco", "toml", @@ -4100,7 +4157,7 @@ dependencies = [ [[package]] name = "hotshot-example-types" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.55#81842b54a699a255645ac195354a0feb08fdc9d6" +source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" dependencies = [ "anyhow", "async-broadcast", @@ -4121,7 +4178,7 @@ dependencies = [ "serde", "sha2 0.10.8", "sha3", - "snafu 0.8.2", + "snafu 0.8.3", "time 0.3.36", "tokio", "tracing", @@ -4131,18 +4188,18 @@ dependencies = [ [[package]] name = "hotshot-macros" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.55#81842b54a699a255645ac195354a0feb08fdc9d6" +source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" dependencies = [ "derive_builder", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] name = "hotshot-orchestrator" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.55#81842b54a699a255645ac195354a0feb08fdc9d6" +source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" dependencies = [ "anyhow", "async-compatibility-layer", @@ -4171,7 +4228,7 @@ dependencies = [ [[package]] name = "hotshot-query-service" version = "0.1.23" -source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=0.1.25#337df31f577acdc5ecf9e628d1bfa1644b8ce6aa" +source = "git+https://github.com/EspressoSystems/hotshot-query-service?branch=abdul/update-hotshot#9e392475b3efaf50c8e6a0f76b98db19acbc583d" dependencies = [ "anyhow", "ark-serialize", @@ -4205,7 +4262,7 @@ dependencies = [ "refinery", "serde", "serde_json", - "snafu 0.8.2", + "snafu 0.8.3", "spin_sleep", "surf-disco", "tagged-base64", @@ -4224,7 +4281,7 @@ dependencies = [ [[package]] name = "hotshot-stake-table" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.55#81842b54a699a255645ac195354a0feb08fdc9d6" +source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" dependencies = [ "ark-bn254", "ark-ed-on-bn254", @@ -4281,7 +4338,7 @@ dependencies = [ "rand_chacha 0.3.1", "sequencer-utils", "serde", - "snafu 0.8.2", + "snafu 0.8.3", "surf-disco", "tagged-base64", "tide-disco", @@ -4295,7 +4352,7 @@ dependencies = [ [[package]] name = "hotshot-task" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.55#81842b54a699a255645ac195354a0feb08fdc9d6" +source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" dependencies = [ "async-broadcast", "async-compatibility-layer", @@ -4308,7 +4365,7 @@ dependencies = [ [[package]] name = "hotshot-task-impls" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.55#81842b54a699a255645ac195354a0feb08fdc9d6" +source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" dependencies = [ "anyhow", "async-broadcast", @@ -4331,7 +4388,7 @@ dependencies = [ "rand 0.8.5", "serde", "sha2 0.10.8", - "snafu 0.8.2", + "snafu 0.8.3", "surf-disco", "tagged-base64", "time 0.3.36", @@ -4343,7 +4400,7 @@ dependencies = [ [[package]] name = "hotshot-testing" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.55#81842b54a699a255645ac195354a0feb08fdc9d6" +source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" dependencies = [ "async-broadcast", "async-compatibility-layer", @@ -4372,7 +4429,7 @@ dependencies = [ "serde", "sha2 0.10.8", "sha3", - "snafu 0.8.2", + "snafu 0.8.3", "tagged-base64", "tide-disco", "tokio", @@ -4383,7 +4440,7 @@ dependencies = [ [[package]] name = "hotshot-types" version = "0.1.11" -source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.55#81842b54a699a255645ac195354a0feb08fdc9d6" +source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" dependencies = [ "anyhow", "ark-bls12-381", @@ -4423,7 +4480,7 @@ dependencies = [ "rand_chacha 0.3.1", "serde", "sha2 0.10.8", - "snafu 0.8.2", + "snafu 0.8.3", "tagged-base64", "time 0.3.36", "tokio", @@ -4576,7 +4633,7 @@ dependencies = [ "bytes 1.6.0", "futures-channel", "futures-util", - "h2 0.4.4", + "h2 0.4.5", "http 1.1.0", "http-body 1.0.0", "httparse", @@ -4631,9 +4688,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +checksum = "3d8d52be92d09acc2e01dddb7fde3ad983fc6489c7db4837e605bc3fca4cb63e" dependencies = [ "bytes 1.6.0", "futures-channel", @@ -4660,7 +4717,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -4858,9 +4915,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", ] @@ -5317,9 +5374,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.154" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libm" @@ -5646,7 +5703,7 @@ dependencies = [ [[package]] name = "libp2p-networking" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.55#81842b54a699a255645ac195354a0feb08fdc9d6" +source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" dependencies = [ "anyhow", "async-compatibility-layer", @@ -5667,7 +5724,7 @@ dependencies = [ "serde", "serde_bytes", "serde_json", - "snafu 0.8.2", + "snafu 0.8.3", "tokio", "tokio-stream", "tracing", @@ -5751,9 +5808,9 @@ dependencies = [ [[package]] name = "libp2p-quic" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0375cdfee57b47b313ef1f0fdb625b78aed770d33a40cf1c294a371ff5e6666" +checksum = "c67296ad4e092e23f92aea3d2bdb6f24eab79c0929ed816dfb460ea2f4567d2b" dependencies = [ "async-std", "bytes 1.6.0", @@ -5764,10 +5821,10 @@ dependencies = [ "libp2p-identity", "libp2p-tls", "parking_lot", - "quinn", + "quinn 0.11.1", "rand 0.8.5", - "ring 0.16.20", - "rustls 0.21.12", + "ring 0.17.8", + "rustls 0.23.8", "socket2 0.5.7", "thiserror", "tokio", @@ -5879,7 +5936,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -5902,20 +5959,20 @@ dependencies = [ [[package]] name = "libp2p-tls" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ce7e3c2e7569d685d08ec795157981722ff96e9e9f9eae75df3c29d02b07a5" +checksum = "251b17aebdd29df7e8f80e4d94b782fae42e934c49086e1a81ba23b60a8314f2" dependencies = [ "futures", - "futures-rustls", + "futures-rustls 0.26.0", "libp2p-core", "libp2p-identity", "rcgen 0.11.3", - "ring 0.16.20", - "rustls 0.21.12", + "ring 0.17.8", + "rustls 0.23.8", "rustls-webpki 0.101.7", "thiserror", - "x509-parser", + "x509-parser 0.16.0", "yasna", ] @@ -5954,7 +6011,7 @@ checksum = "f4846d51afd08180e164291c3754ba30dd4fbac6fac65571be56403c16431a5e" dependencies = [ "either", "futures", - "futures-rustls", + "futures-rustls 0.24.0", "libp2p-core", "libp2p-identity", "parking_lot", @@ -6052,9 +6109,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.16" +version = "1.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9" +checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" dependencies = [ "cc", "libc", @@ -6076,9 +6133,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "local-ip-address" @@ -6190,7 +6247,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -6268,9 +6325,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" dependencies = [ "adler", ] @@ -6358,11 +6415,10 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" dependencies = [ - "lazy_static", "libc", "log", "openssl", @@ -6511,11 +6567,10 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -6551,7 +6606,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -6609,10 +6664,10 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ - "proc-macro-crate 3.1.0", + "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -6639,7 +6694,16 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" dependencies = [ - "asn1-rs", + "asn1-rs 0.5.2", +] + +[[package]] +name = "oid-registry" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c958dd45046245b9c3c2547369bb634eb461670b2e7e0de552905801a648d1d" +dependencies = [ + "asn1-rs 0.6.1", ] [[package]] @@ -6702,7 +6766,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -6753,9 +6817,9 @@ checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" [[package]] name = "parity-scale-codec" -version = "3.6.9" +version = "3.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "881331e34fa842a2fb61cc2db9643a8fedc615e47cfcc52597d1af0db9a7e8fe" +checksum = "306800abfa29c7f16596b5970a588435e3d5b3149683d00c12b699cc19f895ee" dependencies = [ "arrayvec", "bitvec", @@ -6767,11 +6831,11 @@ dependencies = [ [[package]] name = "parity-scale-codec-derive" -version = "3.6.9" +version = "3.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be30eaf4b0a9fba5336683b38de57bb86d179a35862ba6bfcf57625d006bde5b" +checksum = "d830939c76d294956402033aee57a6da7b438f2294eb94864c37b0569053a42c" dependencies = [ - "proc-macro-crate 2.0.0", + "proc-macro-crate", "proc-macro2", "quote", "syn 1.0.109", @@ -6785,9 +6849,9 @@ checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -6819,9 +6883,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "path-slash" @@ -6922,7 +6986,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -6938,9 +7002,9 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", "indexmap 2.2.6", @@ -6986,7 +7050,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -7024,7 +7088,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -7053,9 +7117,9 @@ checksum = "d15b6607fa632996eb8a17c9041cb6071cb75ac057abd45dece578723ea8c7c0" [[package]] name = "piper" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +checksum = "464db0c665917b13ebb5d453ccdec4add5658ee1adc7affc7677615356a8afaf" dependencies = [ "atomic-waker", "fastrand 2.1.0", @@ -7245,12 +7309,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.19" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ac2cf0f2e4f42b49f5ffd07dae8d746508ef7526c13940e5f524012ae6c6550" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -7267,25 +7331,6 @@ dependencies = [ "uint", ] -[[package]] -name = "proc-macro-crate" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" -dependencies = [ - "once_cell", - "toml_edit 0.19.15", -] - -[[package]] -name = "proc-macro-crate" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8" -dependencies = [ - "toml_edit 0.20.7", -] - [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -7326,9 +7371,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.81" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" +checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" dependencies = [ "unicode-ident", ] @@ -7368,7 +7413,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -7389,9 +7434,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes 1.6.0", "prost-derive", @@ -7399,22 +7444,22 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] name = "prost-types" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ "prost", ] @@ -7492,15 +7537,32 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cc2c5017e4b43d5995dcea317bc46c1e09404c0a9664d2908f7f02dfe943d75" dependencies = [ - "async-io 1.13.0", + "bytes 1.6.0", + "pin-project-lite 0.2.14", + "quinn-proto 0.10.6", + "quinn-udp 0.4.1", + "rustc-hash", + "rustls 0.21.12", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904e3d3ba178131798c6d9375db2b13b34337d489b089fc5ba0825a2ff1bee73" +dependencies = [ + "async-io 2.3.2", "async-std", "bytes 1.6.0", "futures-io", "pin-project-lite 0.2.14", - "quinn-proto", - "quinn-udp", + "quinn-proto 0.11.2", + "quinn-udp 0.5.1", "rustc-hash", - "rustls 0.21.12", + "rustls 0.23.8", "thiserror", "tokio", "tracing", @@ -7523,6 +7585,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "quinn-proto" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e974563a4b1c2206bbc61191ca4da9c22e4308b4c455e8906751cc7828393f08" +dependencies = [ + "bytes 1.6.0", + "rand 0.8.5", + "ring 0.17.8", + "rustc-hash", + "rustls 0.23.8", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + [[package]] name = "quinn-udp" version = "0.4.1" @@ -7536,6 +7615,19 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "quinn-udp" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4f0def2590301f4f667db5a77f9694fb004f82796dc1a8b1508fafa3d0e8b72" +dependencies = [ + "libc", + "once_cell", + "socket2 0.5.7", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.36" @@ -7682,7 +7774,7 @@ dependencies = [ "pem 3.0.4", "ring 0.17.8", "time 0.3.36", - "x509-parser", + "x509-parser 0.15.1", "yasna", ] @@ -7779,7 +7871,7 @@ dependencies = [ "quote", "refinery-core", "regex", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -7887,7 +7979,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.4", + "h2 0.4.5", "http 1.1.0", "http-body 1.0.0", "http-body-util", @@ -8104,9 +8196,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" @@ -8135,7 +8227,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.22", + "semver 1.0.23", ] [[package]] @@ -8170,7 +8262,7 @@ dependencies = [ "bitflags 2.5.0", "errno", "libc", - "linux-raw-sys 0.4.13", + "linux-raw-sys 0.4.14", "windows-sys 0.52.0", ] @@ -8208,7 +8300,21 @@ dependencies = [ "log", "ring 0.17.8", "rustls-pki-types", - "rustls-webpki 0.102.3", + "rustls-webpki 0.102.4", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls" +version = "0.23.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79adb16721f56eb2d843e67676896a61ce7a0fa622dc18d3e372477a029d2740" +dependencies = [ + "once_cell", + "ring 0.17.8", + "rustls-pki-types", + "rustls-webpki 0.102.4", "subtle", "zeroize", ] @@ -8234,9 +8340,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.5.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beb461507cee2c2ff151784c52762cf4d9ff6a61f3e80968600ed24fa837fa54" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" @@ -8250,9 +8356,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.3" +version = "0.102.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf" +checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" dependencies = [ "ring 0.17.8", "rustls-pki-types", @@ -8261,9 +8367,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "rw-stream-sink" @@ -8278,9 +8384,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "salsa20" @@ -8302,9 +8408,9 @@ dependencies = [ [[package]] name = "scale-info" -version = "2.11.2" +version = "2.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c453e59a955f81fb62ee5d596b450383d699f152d350e9d23a0db2adb78e4c0" +checksum = "eca070c12893629e2cc820a9761bedf6ce1dcddc9852984d1dc734b8bd9bd024" dependencies = [ "cfg-if", "derive_more", @@ -8314,11 +8420,11 @@ dependencies = [ [[package]] name = "scale-info-derive" -version = "2.11.2" +version = "2.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18cf6c6447f813ef19eb450e985bcce6705f9ce7660db221b59093d15c79c4b7" +checksum = "2d35494501194174bda522a32605929eefc9ecf7e0a326c26db1fdd85881eb62" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate", "proc-macro2", "quote", "syn 1.0.109", @@ -8431,9 +8537,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" dependencies = [ "serde", ] @@ -8518,7 +8624,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.8", - "snafu 0.8.2", + "snafu 0.8.3", "strum", "surf-disco", "tagged-base64", @@ -8561,9 +8667,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.200" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc6f9cc94d67c0e21aaf7eda3a010fd3af78ebf6e096aa6e2e13c79749cce4f" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] @@ -8590,13 +8696,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.200" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "856f046b9400cee3c8c94ed572ecdb752444c24528c035cd35882aad6f492bcb" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -8610,9 +8716,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "itoa", "ryu", @@ -8632,9 +8738,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" dependencies = [ "serde", ] @@ -8678,7 +8784,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -8919,11 +9025,11 @@ dependencies = [ [[package]] name = "snafu" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75976f4748ab44f6e5332102be424e7c2dc18daeaf7e725f2040c3ebb133512e" +checksum = "418b8136fec49956eba89be7da2847ec1909df92a9ae4178b5ff0ff092c8d95e" dependencies = [ - "snafu-derive 0.8.2", + "snafu-derive 0.8.3", ] [[package]] @@ -8940,14 +9046,14 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b19911debfb8c2fb1107bc6cb2d61868aaf53a988449213959bb1b5b1ed95f" +checksum = "1a4812a669da00d17d8266a0439eddcacbc88b17f732f927e52eeb9d196f7fb5" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -9347,21 +9453,15 @@ dependencies = [ [[package]] name = "stringprep" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ - "finl_unicode", "unicode-bidi", "unicode-normalization", + "unicode-properties", ] -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - [[package]] name = "strsim" version = "0.11.1" @@ -9387,7 +9487,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -9527,7 +9627,7 @@ dependencies = [ "hex", "once_cell", "reqwest 0.11.27", - "semver 1.0.22", + "semver 1.0.23", "serde", "serde_json", "sha2 0.10.8", @@ -9549,9 +9649,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.60" +version = "2.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" dependencies = [ "proc-macro2", "quote", @@ -9584,7 +9684,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -9619,7 +9719,7 @@ dependencies = [ "base64 0.22.1", "crc-any", "serde", - "snafu 0.8.2", + "snafu 0.8.3", "tagged-base64-macros", "wasm-bindgen", ] @@ -9665,22 +9765,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.59" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.59" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -9753,14 +9853,14 @@ dependencies = [ "prometheus", "reqwest 0.12.4", "routefinder", - "semver 1.0.22", + "semver 1.0.23", "serde", "serde_json", "serde_with", "shellexpand", "signal-hook", "signal-hook-async-std", - "snafu 0.8.2", + "snafu 0.8.3", "strum", "strum_macros", "tagged-base64", @@ -9927,7 +10027,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -10028,47 +10128,25 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.12" +version = "0.8.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" +checksum = "a4e43f8cc456c9704c851ae29c67e17ef65d2c30017c17a9765b89c382dc8bba" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.12", + "toml_edit 0.22.13", ] [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" dependencies = [ "serde", ] -[[package]] -name = "toml_edit" -version = "0.19.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" -dependencies = [ - "indexmap 2.2.6", - "toml_datetime", - "winnow 0.5.40", -] - -[[package]] -name = "toml_edit" -version = "0.20.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" -dependencies = [ - "indexmap 2.2.6", - "toml_datetime", - "winnow 0.5.40", -] - [[package]] name = "toml_edit" version = "0.21.1" @@ -10082,9 +10160,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.12" +version = "0.22.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3328d4f68a705b2a4498da1d580585d39a6510f98318a2cec3018a7ec61ddef" +checksum = "c127785850e8c20836d49732ae6abfa47616e60bf9d9f57c43c250361a9db96c" dependencies = [ "indexmap 2.2.6", "serde", @@ -10172,7 +10250,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -10287,7 +10365,7 @@ checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -10337,6 +10415,12 @@ dependencies = [ "utf-8", ] +[[package]] +name = "typeid" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "059d83cc991e7a42fc37bd50941885db0888e34209f8cfd9aab07ddec03bc9cf" + [[package]] name = "typenum" version = "1.17.0" @@ -10397,6 +10481,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" + [[package]] name = "unicode-segmentation" version = "1.11.0" @@ -10475,7 +10565,7 @@ dependencies = [ "once_cell", "rustls 0.22.4", "rustls-pki-types", - "rustls-webpki 0.102.3", + "rustls-webpki 0.102.4", "url", "webpki-roots 0.26.1", ] @@ -10613,9 +10703,9 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "waker-fn" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" [[package]] name = "walkdir" @@ -10704,7 +10794,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", "wasm-bindgen-shared", ] @@ -10738,7 +10828,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -10857,7 +10947,7 @@ version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" dependencies = [ - "windows-core", + "windows-core 0.51.1", "windows-targets 0.48.5", ] @@ -10870,6 +10960,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.5", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -11093,18 +11192,35 @@ version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" dependencies = [ - "asn1-rs", + "asn1-rs 0.5.2", "data-encoding", - "der-parser", + "der-parser 8.2.0", "lazy_static", "nom", - "oid-registry", + "oid-registry 0.6.1", "ring 0.16.20", "rusticata-macros", "thiserror", "time 0.3.36", ] +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs 0.6.1", + "data-encoding", + "der-parser 9.0.0", + "lazy_static", + "nom", + "oid-registry 0.7.0", + "rusticata-macros", + "thiserror", + "time 0.3.36", +] + [[package]] name = "xml-rs" version = "0.8.20" @@ -11177,29 +11293,29 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.33" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "087eca3c1eaf8c47b94d02790dd086cd594b912d2043d4de4bfdd466b3befb7c" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.33" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f4b6c273f496d8fd4eaf18853e6b448760225dc030ff2c485a786859aea6393" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] name = "zeroize" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" dependencies = [ "zeroize_derive", ] @@ -11212,7 +11328,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 51d92963ed..8393963142 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,18 +46,18 @@ dotenvy = "0.15" ethers = { version = "2.0", features = ["solc"] } futures = "0.3" -hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.55" } +hotshot = { git = "https://github.com/EspressoSystems/hotshot", branch = "abdul/pass-validated-state" } # Hotshot imports -hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.55" } -hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.23" } -hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.23" } -hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.55" } -hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.25" } -hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.55" } +hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", branch = "abdul/pass-validated-state" } +hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", branch = "abdul/update-hotshot" } +hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", branch = "abdul/update-hotshot" } +hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", branch = "abdul/pass-validated-state" } +hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", branch = "abdul/update-hotshot" } +hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", branch = "abdul/pass-validated-state" } hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" } -hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.55" } -hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.55" } -hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.55" } +hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", branch = "abdul/pass-validated-state" } +hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", branch = "abdul/pass-validated-state" } +hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", branch = "abdul/pass-validated-state" } # Push CDN imports cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", features = [ diff --git a/builder/src/bin/permissionless-builder.rs b/builder/src/bin/permissionless-builder.rs index 19c4a1e30b..45448f71af 100644 --- a/builder/src/bin/permissionless-builder.rs +++ b/builder/src/bin/permissionless-builder.rs @@ -4,6 +4,7 @@ use clap::Parser; use cld::ClDuration; use es_version::SEQUENCER_VERSION; use ethers::types::U256; +use hotshot::traits::ValidatedState; use hotshot_types::data::ViewNumber; use hotshot_types::traits::node_implementation::ConsensusTime; use sequencer::{eth_signature_key::EthKeyPair, options::parse_size, ChainConfig, L1Params}; @@ -131,6 +132,10 @@ async fn main() -> anyhow::Result<()> { let instance_state = build_instance_state(l1_params, opt.state_peers, chain_config, sequencer_version).unwrap(); + // ?? + + let validated_state = ValidatedState::genesis(&instance_state).0; + let api_response_timeout_duration = opt.max_api_timeout_duration; // make the txn timeout as 1/4 of the api_response_timeout_duration @@ -144,6 +149,7 @@ async fn main() -> anyhow::Result<()> { opt.channel_capacity, opt.node_count, instance_state, + validated_state, opt.hotshot_event_streaming_url, builder_server_url, api_response_timeout_duration, diff --git a/builder/src/lib.rs b/builder/src/lib.rs index 6263e4e736..c61a635b2d 100644 --- a/builder/src/lib.rs +++ b/builder/src/lib.rs @@ -555,6 +555,7 @@ pub mod testing { channel_capacity, node_count, node_state, + ValidatedState::default(), hotshot_events_streaming_api_url, hotshot_builder_api_url, Duration::from_millis(2000), @@ -619,6 +620,7 @@ pub mod testing { bootstrapped_view, channel_capacity, node_state, + ValidatedState::default(), hotshot_builder_api_url, Duration::from_millis(2000), 15, @@ -701,10 +703,12 @@ mod test { } let genesis_state = NodeState::mock(); + let validated_state = ValidatedState::default(); let mut parent = { // TODO refactor repeated code from other tests let (genesis_payload, genesis_ns_table) = - Payload::from_transactions([], &genesis_state) + Payload::from_transactions([], &validated_state, &genesis_state) + .await .expect("unable to create genesis payload"); let builder_commitment = genesis_payload.builder_commitment(&genesis_ns_table); let genesis_commitment = { diff --git a/builder/src/non_permissioned.rs b/builder/src/non_permissioned.rs index 86b3877f70..05bbea8cef 100644 --- a/builder/src/non_permissioned.rs +++ b/builder/src/non_permissioned.rs @@ -37,6 +37,7 @@ use hotshot_types::{ use sequencer::{ catchup::StatePeers, eth_signature_key::EthKeyPair, l1_client::L1Client, BuilderParams, ChainConfig, L1Params, NetworkParams, NodeState, Payload, PrivKey, PubKey, SeqTypes, + ValidatedState, }; use hotshot_events_service::{ @@ -82,6 +83,7 @@ impl BuilderConfig { channel_capacity: NonZeroUsize, node_count: NonZeroUsize, instance_state: NodeState, + validated_state: ValidatedState, hotshot_events_api_url: Url, hotshot_builder_apis_url: Url, max_api_timeout_duration: Duration, @@ -114,8 +116,10 @@ impl BuilderConfig { // builder api request channel let (req_sender, req_receiver) = broadcast::>(channel_capacity.get()); - let (genesis_payload, genesis_ns_table) = Payload::from_transactions([], &instance_state) - .expect("genesis payload construction failed"); + let (genesis_payload, genesis_ns_table) = + Payload::from_transactions([], &validated_state, &instance_state) + .await + .expect("genesis payload construction failed"); let builder_commitment = genesis_payload.builder_commitment(&genesis_ns_table); @@ -159,6 +163,9 @@ impl BuilderConfig { .as_u64() .context("the base fee exceeds the maximum amount that a builder can pay (defined by u64::MAX)")?, Arc::new(instance_state), + Arc::new(validated_state), + //???? + Duration::from_secs(60), ); // spawn the builder event loop diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index 5bcd7717a3..dc044508c5 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -187,7 +187,7 @@ pub async fn init_node>(channel_capacity.get()); - let (genesis_payload, genesis_ns_table) = Payload::from_transactions([], &instance_state) - .expect("genesis payload construction failed"); + let (genesis_payload, genesis_ns_table) = + Payload::from_transactions([], &validated_state, &instance_state) + .await + .expect("genesis payload construction failed"); let builder_commitment = genesis_payload.builder_commitment(&genesis_ns_table); @@ -446,6 +452,9 @@ impl { Arc::from(self.raw_payload.clone()) } } -impl BlockPayload for Payload { + +#[async_trait] +impl BlockPayload for Payload { type Error = crate::Error; type Transaction = Transaction; type Instance = NodeState; + type ValidatedState = ValidatedState; type Metadata = NsTable; /// Returns (Self, metadata). @@ -50,8 +54,9 @@ impl BlockPayload for Payload { /// https://github.com/EspressoSystems/espresso-sequencer/issues/757 /// /// TODO(746) refactor and make pretty "table" code for tx, namespace tables? - fn from_transactions( - txs: impl IntoIterator, + async fn from_transactions( + txs: impl IntoIterator + Send, + _validated_state: &Self::ValidatedState, instance_state: &Self::Instance, ) -> Result<(Self, Self::Metadata), Self::Error> { let payload = Payload::from_txs(txs, &instance_state.chain_config)?; @@ -66,14 +71,15 @@ impl BlockPayload for Payload { } } - // TODO remove - fn genesis() -> (Self, Self::Metadata) { + fn empty() -> (Self, Self::Metadata) { // this is only called from `Leaf::genesis`. Since we are // passing empty list, max_block_size is irrelevant so we can // use the mock NodeState. A future update to HotShot should // make a change there to remove the need for this workaround. - Self::from_transactions([], &NodeState::mock()).unwrap() + let payload = Payload::from_txs(vec![], &ChainConfig::default()).unwrap(); + let ns_table = payload.get_ns_table().clone(); + (payload, ns_table) } fn transaction_commitments(&self, meta: &Self::Metadata) -> Vec> { diff --git a/sequencer/src/block/payload.rs b/sequencer/src/block/payload.rs index 023c9399c4..aabfd62709 100644 --- a/sequencer/src/block/payload.rs +++ b/sequencer/src/block/payload.rs @@ -2,6 +2,7 @@ use crate::block::entry::{TxTableEntry, TxTableEntryWord}; use crate::block::payload; use crate::block::tables::NameSpaceTable; use crate::block::tables::TxTable; +use crate::SeqTypes; use crate::{BlockBuildingSnafu, ChainConfig, Error, NamespaceId, Transaction}; use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; use derivative::Derivative; @@ -152,7 +153,9 @@ impl Payload { } pub fn from_txs( - txs: impl IntoIterator as BlockPayload>::Transaction>, + txs: impl IntoIterator< + Item = as BlockPayload>::Transaction, + >, chain_config: &ChainConfig, ) -> Result { let mut namespaces: HashMap = Default::default(); @@ -183,7 +186,7 @@ impl Payload { fn update_namespace_with_tx( namespaces: &mut HashMap, - tx: as BlockPayload>::Transaction, + tx: as BlockPayload>::Transaction, ) { let tx_bytes_len: TxTableEntry = tx.payload().len().try_into().unwrap(); // TODO (Philippe) error handling @@ -317,13 +320,11 @@ pub fn parse_ns_payload(ns_bytes: &[u8], ns_id: NamespaceId) -> Vec } #[cfg(any(test, feature = "testing"))] -impl hotshot_types::traits::block_contents::TestableBlock +impl hotshot_types::traits::block_contents::TestableBlock for Payload { fn genesis() -> Self { - BlockPayload::from_transactions([], &Default::default()) - .unwrap() - .0 + BlockPayload::empty().0 } fn txn_count(&self) -> u64 { @@ -344,8 +345,9 @@ mod test { tx_iterator::TxIndex, }, transaction::NamespaceId, - ChainConfig, NodeState, Transaction, + ChainConfig, NodeState, Transaction, ValidatedState, }; + use async_compatibility_layer::art::async_test; use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use helpers::*; use hotshot_query_service::availability::QueryablePayload; @@ -396,12 +398,12 @@ mod test { assert_eq!(payload.txn_count(), txs.len() as u64); } - #[test] - fn basic_correctness() { - check_basic_correctness::() + #[async_test] + async fn basic_correctness() { + check_basic_correctness::().await } - fn check_basic_correctness() { + async fn check_basic_correctness() { // play with this let test_cases = [ // 1 namespace only @@ -515,8 +517,13 @@ mod test { let all_txs_iter = derived_nss .iter() .flat_map(|(_ns_id, ns)| ns.txs.iter().cloned()); - let (block, actual_ns_table) = - Payload::from_transactions(all_txs_iter, &NodeState::mock()).unwrap(); + let (block, actual_ns_table) = Payload::from_transactions( + all_txs_iter, + &ValidatedState::default(), + &NodeState::mock(), + ) + .await + .unwrap(); let disperse_data = vid.disperse(&block.raw_payload).unwrap(); // TEST ACTUAL STUFF AGAINST DERIVED STUFF diff --git a/sequencer/src/block/queryable.rs b/sequencer/src/block/queryable.rs index d8f1025b04..c555596d5e 100644 --- a/sequencer/src/block/queryable.rs +++ b/sequencer/src/block/queryable.rs @@ -1,4 +1,4 @@ -use crate::block::entry::TxTableEntryWord; +use crate::{block::entry::TxTableEntryWord, SeqTypes}; use crate::block::payload::Payload; use crate::block::tables::TxTable; use hotshot_query_service::availability::QueryablePayload; @@ -15,7 +15,7 @@ use super::{ }; // TODO don't hard-code TxTableEntryWord generic param -impl QueryablePayload for Payload { +impl QueryablePayload for Payload { type TransactionIndex = TxIndex; type Iter<'a> = TxIterator<'a, TxTableEntryWord>; type InclusionProof = TxInclusionProof; diff --git a/sequencer/src/header.rs b/sequencer/src/header.rs index cba5423360..7996752741 100644 --- a/sequencer/src/header.rs +++ b/sequencer/src/header.rs @@ -283,7 +283,7 @@ impl BlockHeader for Header { parent_leaf: &Leaf, payload_commitment: VidCommitment, builder_commitment: BuilderCommitment, - metadata: <::BlockPayload as BlockPayload>::Metadata, + metadata: <::BlockPayload as BlockPayload>::Metadata, builder_fee: BuilderFee, _vid_common: VidCommon, _version: Version, @@ -380,11 +380,12 @@ impl BlockHeader for Header { instance_state: &NodeState, payload_commitment: VidCommitment, builder_commitment: BuilderCommitment, - ns_table: <::BlockPayload as BlockPayload>::Metadata, + ns_table: <::BlockPayload as BlockPayload>::Metadata, ) -> Self { let ValidatedState { fee_merkle_tree, block_merkle_tree, + .. } = ValidatedState::genesis(instance_state).0; let block_merkle_tree_root = block_merkle_tree.commitment(); let fee_merkle_tree_root = fee_merkle_tree.commitment(); @@ -415,7 +416,9 @@ impl BlockHeader for Header { self.payload_commitment } - fn metadata(&self) -> &<::BlockPayload as BlockPayload>::Metadata { + fn metadata( + &self, + ) -> &<::BlockPayload as BlockPayload>::Metadata { &self.ns_table } @@ -541,6 +544,7 @@ mod test_headers { let mut validated_state = ValidatedState { block_merkle_tree: block_merkle_tree.clone(), fee_merkle_tree, + chain_config: genesis.instance_state.chain_config.into(), }; let (fee_account, fee_key) = FeeAccount::generated_from_seed_indexed([0; 32], 0); @@ -750,7 +754,10 @@ mod test_headers { fn default() -> Self { let instance_state = NodeState::mock(); let validated_state = ValidatedState::genesis(&instance_state).0; - let leaf = Leaf::genesis(&instance_state); + + //TODO(abdul): Change this + let leaf = async_std::task::block_on(Leaf::genesis(&validated_state, &instance_state)); + let header = leaf.block_header().clone(); let ns_table = leaf.block_payload().unwrap().get_ns_table().clone(); Self { diff --git a/sequencer/src/hotshot_commitment.rs b/sequencer/src/hotshot_commitment.rs index ad205ad64a..1aa8cb93c6 100644 --- a/sequencer/src/hotshot_commitment.rs +++ b/sequencer/src/hotshot_commitment.rs @@ -275,7 +275,7 @@ fn build_sequence_batches_txn( #[cfg(test)] mod test { use super::*; - use crate::{l1_client::L1Client, Leaf, NodeState}; + use crate::{l1_client::L1Client, Leaf, NodeState, ValidatedState}; use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use async_std::task::spawn; use committable::Committable; @@ -327,9 +327,10 @@ mod test { } } - fn mock_leaf(height: u64, node_state: &NodeState) -> LeafQueryData { - let mut leaf = Leaf::genesis(node_state); - let mut qc = QuorumCertificate::genesis(node_state); + async fn mock_leaf(height: u64, node_state: &NodeState) -> LeafQueryData { + let validated_state = &ValidatedState::default(); + let mut leaf = Leaf::genesis(validated_state, node_state).await; + let mut qc = QuorumCertificate::genesis(validated_state, node_state).await; leaf.block_header_mut().height = height; qc.data.leaf_commit = leaf.commit(); LeafQueryData::new(leaf, qc).unwrap() @@ -379,7 +380,8 @@ mod test { NodeState::mock().with_l1(L1Client::new(anvil.provider().url().clone(), 1)); for i in 0..num_batches { - data.leaves.push(Some(mock_leaf(i as u64, &node_state))); + data.leaves + .push(Some(mock_leaf(i as u64, &node_state).await)); } tracing::info!("sequencing batches: {:?}", data.leaves); @@ -447,7 +449,7 @@ mod test { let node_state = NodeState::mock().with_l1(L1Client::new(anvil.provider().url().clone(), 1)); - data.leaves.push(Some(mock_leaf(0, &node_state))); + data.leaves.push(Some(mock_leaf(0, &node_state).await)); // Connect to the HotShot contract with the expected L1 client. let hotshot = HotShot::new(l1.hotshot.address(), adaptor_l1_signer); @@ -472,7 +474,7 @@ mod test { assert_eq!(l1.hotshot.block_height().call().await.unwrap().as_u64(), 1); // Once a new batch is available, we can sequence it. - data.leaves.push(Some(mock_leaf(1, &node_state))); + data.leaves.push(Some(mock_leaf(1, &node_state).await)); sync_with_l1(1, &data, &hotshot).await.unwrap(); let (event, _) = wait_for_new_batches(&l1, from_block.as_u64()).await; assert_eq!(event.first_block_number.as_u64(), 1); @@ -513,7 +515,7 @@ mod test { // Create a sequence of leaves, some of which are missing. let mut data = MockDataSource::default(); data.leaves - .extend([None, Some(mock_leaf(1, &node_state)), None]); + .extend([None, Some(mock_leaf(1, &node_state).await), None]); // Connect to the HotShot contract with the expected L1 client. let hotshot = HotShot::new(l1.hotshot.address(), adaptor_l1_signer); @@ -524,7 +526,7 @@ mod test { // If the first leaf is present but subsequent leaves are missing, we should sequence the // leaves that are available. - data.leaves[0] = Some(mock_leaf(0, &node_state)); + data.leaves[0] = Some(mock_leaf(0, &node_state).await); sync_with_l1(3, &data, &hotshot).await.unwrap(); // Check the NewBatches event. diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 5a23a99bad..5523d2edc0 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -387,7 +387,7 @@ pub async fn init_node( let topics = { let mut topics = vec![Topic::Global]; if is_da { - topics.push(Topic::DA); + topics.push(Topic::Da); } topics }; @@ -843,7 +843,9 @@ mod test { let mut parent = { // TODO refactor repeated code from other tests let (genesis_payload, genesis_ns_table) = - Payload::from_transactions([], &NodeState::mock()).unwrap(); + Payload::from_transactions([], &ValidatedState::default(), &NodeState::mock()) + .await + .unwrap(); let genesis_commitment = { // TODO we should not need to collect payload bytes just to compute vid_commitment let payload_bytes = genesis_payload.encode(); diff --git a/sequencer/src/message_compat_tests.rs b/sequencer/src/message_compat_tests.rs index d7899bb95e..167fac6a9d 100644 --- a/sequencer/src/message_compat_tests.rs +++ b/sequencer/src/message_compat_tests.rs @@ -14,7 +14,8 @@ //! If this test is failing and you did not intend to change the consensus API, figure out what //! code changed caused the serialization change and revert it. -use crate::{Leaf, NodeState, Payload, PubKey, SeqTypes, Transaction}; +use crate::{Leaf, NodeState, Payload, PubKey, SeqTypes, Transaction, ValidatedState}; +use async_compatibility_layer::art::async_test; use committable::Committable; use es_version::SequencerVersion; use hotshot::traits::election::static_committee::GeneralStaticCommittee; @@ -49,8 +50,8 @@ use vbs::{version::Version, BinarySerializer}; type Serializer = vbs::Serializer; -#[test] -fn test_message_compat() { +#[async_test] +async fn test_message_compat() { let (sender, priv_key) = PubKey::generated_from_seed_indexed(Default::default(), 0); let signature = PubKey::sign(&priv_key, &[]).unwrap(); let membership = GeneralStaticCommittee::new(&[], vec![sender.stake_table_entry(1)], vec![], 0); @@ -62,11 +63,16 @@ fn test_message_compat() { old_version_last_view: ViewNumber::genesis(), new_version_first_view: ViewNumber::genesis(), }; - let leaf = Leaf::genesis(&NodeState::mock()); + let leaf = Leaf::genesis(&ValidatedState::default(), &NodeState::mock()).await; let block_header = leaf.block_header().clone(); let transaction = Transaction::new(1.into(), vec![1, 2, 3]); - let (payload, metadata) = - Payload::from_transactions([transaction.clone()], &NodeState::mock()).unwrap(); + let (payload, metadata) = Payload::from_transactions( + [transaction.clone()], + &ValidatedState::default(), + &NodeState::mock(), + ) + .await + .unwrap(); let view_sync_pre_commit_data = ViewSyncPreCommitData { relay: 0, round: ViewNumber::genesis(), @@ -91,7 +97,11 @@ fn test_message_compat() { data: QuorumProposal { block_header, view_number: ViewNumber::genesis(), - justify_qc: QuorumCertificate::genesis(&NodeState::mock()), + justify_qc: QuorumCertificate::genesis( + &ValidatedState::default(), + &NodeState::mock(), + ) + .await, upgrade_certificate: Some(UpgradeCertificate { data: upgrade_data.clone(), vote_commitment: upgrade_data.commit(), diff --git a/sequencer/src/persistence.rs b/sequencer/src/persistence.rs index b545e1a755..a116bf9777 100644 --- a/sequencer/src/persistence.rs +++ b/sequencer/src/persistence.rs @@ -104,6 +104,8 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { &self, state: NodeState, ) -> anyhow::Result> { + let genesis_validated_state = ValidatedState::genesis(&state).0; + let highest_voted_view = match self .load_latest_acted_view() .await @@ -137,12 +139,15 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { } None => { tracing::info!("no saved leaf, starting from genesis leaf"); - (Leaf::genesis(&state), QuorumCertificate::genesis(&state)) + ( + Leaf::genesis(&genesis_validated_state, &state).await, + QuorumCertificate::genesis(&genesis_validated_state, &state).await, + ) } }; let validated_state = if leaf.block_header().height == 0 { // If we are starting from genesis, we can provide the full state. - Some(Arc::new(ValidatedState::genesis(&state).0)) + Some(Arc::new(genesis_validated_state)) } else { // Otherwise, we will have to construct a sparse state and fetch missing data during // catchup. @@ -273,8 +278,8 @@ mod persistence_tests { assert_eq!(storage.load_anchor_leaf().await.unwrap(), None); // Store a leaf. - let leaf1 = Leaf::genesis(&NodeState::mock()); - let qc1 = QuorumCertificate::genesis(&NodeState::mock()); + let leaf1 = Leaf::genesis(&ValidatedState::default(), &NodeState::mock()).await; + let qc1 = QuorumCertificate::genesis(&ValidatedState::default(), &NodeState::mock()).await; storage.save_anchor_leaf(&leaf1, &qc1).await.unwrap(); assert_eq!( storage.load_anchor_leaf().await.unwrap().unwrap(), @@ -359,7 +364,7 @@ mod persistence_tests { None ); - let leaf = Leaf::genesis(&NodeState::mock()); + let leaf = Leaf::genesis(&ValidatedState::default(), &NodeState::mock()).await; let payload = leaf.block_payload().unwrap(); let bytes = payload.encode().to_vec(); let disperse = vid_scheme(2).disperse(bytes).unwrap(); diff --git a/sequencer/src/reference_tests.rs b/sequencer/src/reference_tests.rs index f29af5ce5a..3345171a48 100644 --- a/sequencer/src/reference_tests.rs +++ b/sequencer/src/reference_tests.rs @@ -86,7 +86,7 @@ fn reference_header() -> Header { let builder_key = FeeAccount::generated_from_seed_indexed(Default::default(), 0).1; let fee_info = reference_fee_info(); let ns_table = reference_ns_table(); - let payload = as TestableBlock>::genesis(); + let payload = as TestableBlock<_>>::genesis(); let payload_commitment = vid_commitment(&payload.encode(), 1); let builder_commitment = payload.builder_commitment(&ns_table); let builder_signature = FeeAccount::sign_fee( diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 378023ee95..44a3df6cae 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -1,6 +1,7 @@ use crate::{ - api::data_source::CatchupDataSource, catchup::SqlStateCatchup, eth_signature_key::EthKeyPair, - ChainConfig, Header, Leaf, NodeState, SeqTypes, + api::data_source::CatchupDataSource, catchup::SqlStateCatchup, + chain_config::ResolvableChainConfig, eth_signature_key::EthKeyPair, ChainConfig, Header, Leaf, + NodeState, SeqTypes, }; use anyhow::{anyhow, bail, ensure, Context}; use ark_serialize::{ @@ -58,9 +59,10 @@ pub struct ValidatedState { pub block_merkle_tree: BlockMerkleTree, /// Fee Merkle Tree pub fee_merkle_tree: FeeMerkleTree, + pub chain_config: ResolvableChainConfig, } -#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] pub struct Delta { pub fees_delta: HashSet, } @@ -83,9 +85,13 @@ impl Default for ValidatedState { Vec::<(FeeAccount, FeeAmount)>::new(), ) .unwrap(); + + let chain_config = ResolvableChainConfig::from(ChainConfig::default()); + Self { block_merkle_tree, fee_merkle_tree, + chain_config, } } } @@ -200,6 +206,7 @@ impl ValidatedState { block_merkle_tree: BlockMerkleTree::from_commitment( self.block_merkle_tree.commitment(), ), + chain_config: ResolvableChainConfig::from(self.chain_config.commit()), } } } @@ -255,6 +262,7 @@ pub fn validate_proposal( let ValidatedState { block_merkle_tree, fee_merkle_tree, + .. } = state; let block_merkle_tree_root = block_merkle_tree.commitment(); @@ -351,6 +359,7 @@ async fn store_state_update( let ValidatedState { fee_merkle_tree, block_merkle_tree, + .. } = state; let Delta { fees_delta } = delta; @@ -771,6 +780,7 @@ impl HotShotState for ValidatedState { Self { fee_merkle_tree, block_merkle_tree, + chain_config: block_header.chain_config, } } /// Construct a genesis validated state. @@ -1238,6 +1248,7 @@ impl FeeAccountProof { #[cfg(test)] mod test { use super::*; + use async_compatibility_layer::art::async_test; use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use hotshot_types::vid::vid_scheme; use jf_vid::VidScheme; @@ -1281,8 +1292,8 @@ mod test { FeeAccountProof::prove(&tree, account2).unwrap(); } - #[test] - fn test_validation_max_block_size() { + #[async_test] + async fn test_validation_max_block_size() { setup_logging(); setup_backtrace(); @@ -1296,7 +1307,9 @@ mod test { base_fee: 0.into(), ..Default::default() }); - let parent = Leaf::genesis(&instance); + + let validated_state = ValidatedState::genesis(&instance).0; + let parent = Leaf::genesis(&validated_state, &instance).await; let header = parent.block_header(); // Validation fails because the proposed block exceeds the maximum block size. @@ -1305,8 +1318,8 @@ mod test { tracing::info!(%err, "task failed successfully"); } - #[test] - fn test_validation_base_fee() { + #[async_test] + async fn test_validation_base_fee() { setup_logging(); setup_backtrace(); @@ -1320,7 +1333,10 @@ mod test { max_block_size, ..Default::default() }); - let parent = Leaf::genesis(&instance); + + let validated_state = ValidatedState::genesis(&instance).0; + + let parent = Leaf::genesis(&validated_state, &instance).await; let header = parent.block_header(); // Validation fails because the genesis fee (0) is too low. From cfd26627d87234fb494813e6c0ae1b1abd1d07be Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Tue, 28 May 2024 17:28:41 +0500 Subject: [PATCH 02/48] get_chain_config --- sequencer/src/block.rs | 8 ++++++-- sequencer/src/block/queryable.rs | 2 +- sequencer/src/state.rs | 31 ++++++++++++++++++++++++------- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/sequencer/src/block.rs b/sequencer/src/block.rs index 987a2bf675..5ad007e78a 100644 --- a/sequencer/src/block.rs +++ b/sequencer/src/block.rs @@ -56,10 +56,14 @@ impl BlockPayload for Payload { /// TODO(746) refactor and make pretty "table" code for tx, namespace tables? async fn from_transactions( txs: impl IntoIterator + Send, - _validated_state: &Self::ValidatedState, + validated_state: &Self::ValidatedState, instance_state: &Self::Instance, ) -> Result<(Self, Self::Metadata), Self::Error> { - let payload = Payload::from_txs(txs, &instance_state.chain_config)?; + let cf = validated_state + .get_chain_config(&instance_state, &validated_state.chain_config) + .await; + + let payload = Payload::from_txs(txs, &cf)?; let ns_table = payload.get_ns_table().clone(); // TODO don't clone ns_table Some((payload, ns_table)).context(BlockBuildingSnafu) } diff --git a/sequencer/src/block/queryable.rs b/sequencer/src/block/queryable.rs index c555596d5e..114f0a8269 100644 --- a/sequencer/src/block/queryable.rs +++ b/sequencer/src/block/queryable.rs @@ -1,6 +1,6 @@ -use crate::{block::entry::TxTableEntryWord, SeqTypes}; use crate::block::payload::Payload; use crate::block::tables::TxTable; +use crate::{block::entry::TxTableEntryWord, SeqTypes}; use hotshot_query_service::availability::QueryablePayload; use hotshot_types::vid::{vid_scheme, SmallRangeProofType}; use jf_vid::payload_prover::{PayloadProver, Statement}; diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 44a3df6cae..9567b65768 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -559,6 +559,21 @@ impl SequencerStateDataSource for T where } impl ValidatedState { + pub(crate) async fn get_chain_config( + &self, + instance: &NodeState, + cf: &ResolvableChainConfig, + ) -> ChainConfig { + if cf.commit() == instance.chain_config.commit() { + instance.chain_config + } else { + match cf.resolve() { + Some(cf) => cf, + None => ChainConfig::default(), // TODO: DO CATCHUP + } + } + } + pub(crate) async fn apply_header( &self, instance: &NodeState, @@ -571,17 +586,19 @@ impl ValidatedState { let l1_deposits = get_l1_deposits(instance, proposed_header, parent_leaf).await; let mut validated_state = self.clone(); + let chain_config = validated_state + .get_chain_config(instance, &proposed_header.chain_config) + .await; + + validated_state.chain_config = chain_config.into(); // Find missing fee state entries. We will need to use the builder account which is paying a // fee and the recipient account which is receiving it, plus any counts receiving deposits // in this block. let missing_accounts = self.forgotten_accounts( - [ - proposed_header.fee_info.account, - instance.chain_config().fee_recipient, - ] - .into_iter() - .chain(l1_deposits.iter().map(|fee_info| fee_info.account)), + [proposed_header.fee_info.account, chain_config.fee_recipient] + .into_iter() + .chain(l1_deposits.iter().map(|fee_info| fee_info.account)), ); let parent_height = parent_leaf.height(); @@ -643,7 +660,7 @@ impl ValidatedState { &mut validated_state, &mut delta, proposed_header.fee_info, - instance.chain_config().fee_recipient, + chain_config.fee_recipient, )?; Ok((validated_state, delta)) From 6242d2792ce2c2a9a8f944d040e913a7a5ab7621 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 29 May 2024 01:12:57 +0500 Subject: [PATCH 03/48] chain config route --- sequencer/api/catchup.toml | 7 +++ sequencer/src/api.rs | 28 ++++++++- sequencer/src/api/data_source.rs | 12 +++- sequencer/src/api/endpoints.rs | 13 ++++ sequencer/src/block.rs | 19 +++++- sequencer/src/catchup.rs | 103 +++++++++++++++++++++++++++++++ sequencer/src/lib.rs | 2 + sequencer/src/state.rs | 33 +++++++--- 8 files changed, 202 insertions(+), 15 deletions(-) diff --git a/sequencer/api/catchup.toml b/sequencer/api/catchup.toml index 416d034dd5..6eedd2a0b2 100644 --- a/sequencer/api/catchup.toml +++ b/sequencer/api/catchup.toml @@ -44,3 +44,10 @@ decided view. Returns the blocks Merkle tree frontier -- the path to the most recently appended leaf, relative to root node at the requested block height and view. """ + +[route.chainconfig] +PATH = ["/chain-config/:commitment"] +":commitment" = "TaggedBase64" +DOC = """ +TODO: +""" \ No newline at end of file diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 9188f4595f..6d9e866b37 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -4,12 +4,13 @@ use crate::{ persistence::SequencerPersistence, state::{BlockMerkleTree, FeeAccountProof}, state_signature::StateSigner, - Node, NodeState, PubKey, SeqTypes, SequencerContext, Transaction, + ChainConfig, Node, NodeState, PubKey, SeqTypes, SequencerContext, Transaction, }; -use anyhow::Context; +use anyhow::{bail, Context}; use async_once_cell::Lazy; use async_std::sync::{Arc, RwLock}; use async_trait::async_trait; +use committable::{Commitment, Committable}; use data_source::{CatchupDataSource, SubmitDataSource}; use derivative::Derivative; use ethers::prelude::{Address, U256}; @@ -239,6 +240,29 @@ impl, + ) -> anyhow::Result { + let node_state = self.node_state().await; + let node_state_cf = node_state.chain_config(); + if node_state_cf.commit() == commitment { + return Ok(*node_state_cf); + } + + let state = self.consensus().await.decided_state().await; + let state_cf = state.chain_config; + + if state_cf.commit() != commitment { + bail!("chain config for commitment={commitment} not found in decided state"); + } + + state_cf.resolve().context(format!( + "cannot resolve to a full chain config for commitment={commitment}" + )) + } } impl diff --git a/sequencer/src/api/data_source.rs b/sequencer/src/api/data_source.rs index 82e1dbc06f..f973be1c1c 100644 --- a/sequencer/src/api/data_source.rs +++ b/sequencer/src/api/data_source.rs @@ -8,10 +8,11 @@ use super::{ use crate::{ network, persistence::{self, SequencerPersistence}, - PubKey, SeqTypes, Transaction, + ChainConfig, PubKey, SeqTypes, Transaction, }; use anyhow::bail; use async_trait::async_trait; +use committable::Commitment; use ethers::prelude::Address; use futures::future::Future; use hotshot_query_service::{ @@ -141,6 +142,15 @@ pub(crate) trait CatchupDataSource { bail!("merklized state catchup is not supported for this data source"); } } + + fn get_chain_config( + &self, + _commitment: Commitment, + ) -> impl Send + Future> { + async { + bail!("chain config catchup is not supported for this data source"); + } + } } impl CatchupDataSource for MetricsDataSource {} diff --git a/sequencer/src/api/endpoints.rs b/sequencer/src/api/endpoints.rs index 78af833208..0a490aa283 100644 --- a/sequencer/src/api/endpoints.rs +++ b/sequencer/src/api/endpoints.rs @@ -279,6 +279,19 @@ where .map_err(|err| Error::catch_all(StatusCode::NotFound, format!("{err:#}"))) } .boxed() + })? + .get("chainconfig", |req, state| { + async move { + let commitment = req + .blob_param("commitment") + .map_err(Error::from_request_error)?; + + state + .get_chain_config(commitment) + .await + .map_err(|err| Error::catch_all(StatusCode::NotFound, format!("{err:#}"))) + } + .boxed() })?; Ok(api) diff --git a/sequencer/src/block.rs b/sequencer/src/block.rs index 5ad007e78a..3e4fc26abd 100644 --- a/sequencer/src/block.rs +++ b/sequencer/src/block.rs @@ -59,9 +59,22 @@ impl BlockPayload for Payload { validated_state: &Self::ValidatedState, instance_state: &Self::Instance, ) -> Result<(Self, Self::Metadata), Self::Error> { - let cf = validated_state - .get_chain_config(&instance_state, &validated_state.chain_config) - .await; + let validated_state_cf = validated_state.chain_config; + let instance_state_cf = instance_state.chain_config; + + let cf = if validated_state_cf.commit() == instance_state_cf.commit() { + instance_state_cf + } else { + match validated_state_cf.resolve() { + Some(cf) => cf, + None => instance_state + .peers + .as_ref() + .fetch_chain_config(validated_state_cf.commit()) + .await + .unwrap(), // TODO: + } + }; let payload = Payload::from_txs(txs, &cf)?; let ns_table = payload.get_ns_table().clone(); // TODO don't clone ns_table diff --git a/sequencer/src/catchup.rs b/sequencer/src/catchup.rs index a310289807..482575e6ac 100644 --- a/sequencer/src/catchup.rs +++ b/sequencer/src/catchup.rs @@ -2,10 +2,12 @@ use crate::{ api::{data_source::CatchupDataSource, AccountQueryData, BlocksFrontier}, persistence::PersistenceOptions, state::{BlockMerkleTree, FeeAccount, FeeMerkleCommitment}, + ChainConfig, }; use anyhow::{bail, Context}; use async_std::sync::RwLock; use async_trait::async_trait; +use committable::Commitment; use derive_more::From; use hotshot_types::{data::ViewNumber, traits::node_implementation::ConsensusTime as _}; use jf_merkle_tree::{prelude::MerkleNode, ForgetableMerkleTreeScheme, MerkleTreeScheme}; @@ -104,6 +106,26 @@ pub trait StateCatchup: Send + Sync + std::fmt::Debug { Ok(()) } + async fn try_fetch_chain_config( + &self, + commitment: Commitment, + ) -> anyhow::Result; + + async fn fetch_chain_config( + &self, + commitment: Commitment, + ) -> anyhow::Result { + loop { + match self.try_fetch_chain_config(commitment).await { + Ok(cf) => return Ok(cf), + Err(err) => { + tracing::warn!("Could not fetch chain config from any peer, retrying: {err:#}"); + async_std::task::sleep(self.retry_interval()).await; + } + } + } + } + fn retry_interval(&self) -> Duration { Duration::from_millis(100) } @@ -210,6 +232,28 @@ impl StateCatchup for StatePeers { bail!("Could not fetch frontier from any peer"); } + async fn try_fetch_chain_config( + &self, + commitment: Commitment, + ) -> anyhow::Result { + for client in self.clients.iter() { + tracing::info!("Fetching chain config from {}", client.url); + match client + .get::(&format!("catchup/chain-config/{}", commitment)) + .send() + .await + { + Ok(cf) => { + return Ok(cf); + } + Err(err) => { + tracing::warn!("Error fetching chain config from peer: {}", err); + } + } + } + bail!("Could not fetch chain config from any peer"); + } + fn retry_interval(&self) -> Duration { self.interval } @@ -263,6 +307,13 @@ where _ => bail!("invalid proof"), } } + + async fn try_fetch_chain_config( + &self, + _commitment: Commitment, + ) -> anyhow::Result { + bail!("not supported"); + } } #[async_trait] @@ -310,6 +361,20 @@ impl StateCatchup for Box { ) -> anyhow::Result<()> { (**self).remember_blocks_merkle_tree(height, view, mt).await } + + async fn try_fetch_chain_config( + &self, + commitment: Commitment, + ) -> anyhow::Result { + (**self).try_fetch_chain_config(commitment).await + } + + async fn fetch_chain_config( + &self, + commitment: Commitment, + ) -> anyhow::Result { + (**self).fetch_chain_config(commitment).await + } } #[async_trait] @@ -357,6 +422,20 @@ impl StateCatchup for Arc { ) -> anyhow::Result<()> { (**self).remember_blocks_merkle_tree(height, view, mt).await } + + async fn try_fetch_chain_config( + &self, + commitment: Commitment, + ) -> anyhow::Result { + (**self).try_fetch_chain_config(commitment).await + } + + async fn fetch_chain_config( + &self, + commitment: Commitment, + ) -> anyhow::Result { + (**self).fetch_chain_config(commitment).await + } } /// Catchup from multiple providers tries each provider in a round robin fashion until it succeeds. @@ -406,6 +485,22 @@ impl StateCatchup for Vec { bail!("could not fetch account from any provider"); } + + async fn try_fetch_chain_config( + &self, + commitment: Commitment, + ) -> anyhow::Result { + for provider in self { + match provider.try_fetch_chain_config(commitment).await { + Ok(cf) => return Ok(cf), + Err(err) => { + tracing::warn!(?provider, "failed to fetch chain config: {err:#}"); + } + } + } + + bail!("could not fetch chain config from any provider"); + } } #[cfg(any(test, feature = "testing"))] @@ -468,5 +563,13 @@ pub mod mock { Ok(()) } + + //TODO : ??? + async fn try_fetch_chain_config( + &self, + _commitment: Commitment, + ) -> anyhow::Result { + Ok(ChainConfig::default()) + } } } diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 5523d2edc0..05d5616ea9 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -270,6 +270,8 @@ pub enum Error { MerkleTreeError { error: String }, BlockBuilding, + + ChainConfigNotFound, } #[derive(Clone, Debug)] diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index c7a0bb07e9..c9072bfd2c 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -562,14 +562,29 @@ impl ValidatedState { pub(crate) async fn get_chain_config( &self, instance: &NodeState, - cf: &ResolvableChainConfig, - ) -> ChainConfig { - if cf.commit() == instance.chain_config.commit() { - instance.chain_config - } else { - match cf.resolve() { - Some(cf) => cf, - None => ChainConfig::default(), // TODO: DO CATCHUP + header_cf: &ResolvableChainConfig, + ) -> anyhow::Result { + if header_cf.commit() != self.chain_config.commit() { + bail!( + "Proposed header chain config commit={} expected={}", + header_cf.commit(), + self.chain_config.commit() + ); + } + + if self.chain_config.commit() == instance.chain_config.commit() { + return Ok(instance.chain_config); + } + + match (self.chain_config.resolve(), header_cf.resolve()) { + (Some(cf), _) => Ok(cf), + (_, Some(cf)) => Ok(cf), + (None, None) => { + instance + .peers + .as_ref() + .fetch_chain_config(self.chain_config.commit()) + .await } } } @@ -588,7 +603,7 @@ impl ValidatedState { let mut validated_state = self.clone(); let chain_config = validated_state .get_chain_config(instance, &proposed_header.chain_config) - .await; + .await?; validated_state.chain_config = chain_config.into(); From e9bde5a726d6209694d0ff4f533c9d0d3dd50cc8 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 29 May 2024 04:34:57 +0500 Subject: [PATCH 04/48] add test --- sequencer/api/catchup.toml | 7 +++++- sequencer/src/api.rs | 50 ++++++++++++++++++++++++++++++++++++++ sequencer/src/block.rs | 3 +++ sequencer/src/catchup.rs | 3 +-- sequencer/src/state.rs | 7 ++++++ 5 files changed, 67 insertions(+), 3 deletions(-) diff --git a/sequencer/api/catchup.toml b/sequencer/api/catchup.toml index 6eedd2a0b2..af7d942bae 100644 --- a/sequencer/api/catchup.toml +++ b/sequencer/api/catchup.toml @@ -49,5 +49,10 @@ root node at the requested block height and view. PATH = ["/chain-config/:commitment"] ":commitment" = "TaggedBase64" DOC = """ -TODO: + +This endpoint retrieves the chain config from a peer that matches the specified `:commitment`. +This is only called when the state does not have full chain config which is different from the genesis one. +This can happen if the node missed a protocol upgrade. + +Returns the chain config -- this includes parameters such as `max_block_size`, `chain_id`, `base_fee`, and `fee_recipient`. """ \ No newline at end of file diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index bb28ee0615..119b5476c2 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -828,6 +828,7 @@ mod test { use super::*; use crate::{ catchup::{mock::MockStateCatchup, StatePeers}, + chain_config::ResolvableChainConfig, persistence::no_storage, state::{FeeAccount, FeeAmount, ValidatedState}, testing::TestConfig, @@ -1056,6 +1057,55 @@ mod test { } } + #[async_std::test] + async fn test_validated_state_chain_config() { + setup_logging(); + setup_backtrace(); + + let mut state = ValidatedState::default(); + // ValidatedState only has the default chain config commitment. + // NodeState also has the default ChainConfig. + // Therefore, both commitments would match. + // This would update the ValidatedState's chain config to NodeState's ChainConfig. + state.chain_config = ChainConfig::default().commit().into(); + + // Start a sequencer network without catchup module + let port = pick_unused_port().expect("No ports free"); + let mut network = TestNetwork::with_state( + Options::from(options::Http { port }), + std::array::from_fn(|_| state.clone()), + [no_storage::Options; TestConfig::NUM_NODES], + std::array::from_fn(|_| MockStateCatchup::default()), + ) + .await; + + network + .server + .event_stream() + .filter(|event| future::ready(matches!(event.event, EventType::Decide { .. }))) + .take(4) + .collect::>() + .await; + + let decided_state = network.server.consensus().decided_state().await; + assert_eq!( + decided_state.chain_config, + ResolvableChainConfig::from(ChainConfig::default()) + ); + + for peer in &network.peers { + let state = peer.consensus().decided_state().await; + assert_eq!( + state.chain_config, + ResolvableChainConfig::from(ChainConfig::default()) + ); + } + + network.stop_consensus().await; + + drop(network); + } + #[async_std::test] pub(crate) async fn test_restart() { setup_logging(); diff --git a/sequencer/src/block.rs b/sequencer/src/block.rs index 3e4fc26abd..0c14ddd371 100644 --- a/sequencer/src/block.rs +++ b/sequencer/src/block.rs @@ -54,6 +54,9 @@ impl BlockPayload for Payload { /// https://github.com/EspressoSystems/espresso-sequencer/issues/757 /// /// TODO(746) refactor and make pretty "table" code for tx, namespace tables? + /// + /// This function also performs catchup from peers + /// when the ValidatedState's chain config commitment differs from the NodeState's chain config, and the ValidatedState does not have full chain config. async fn from_transactions( txs: impl IntoIterator + Send, validated_state: &Self::ValidatedState, diff --git a/sequencer/src/catchup.rs b/sequencer/src/catchup.rs index 482575e6ac..f740cf0745 100644 --- a/sequencer/src/catchup.rs +++ b/sequencer/src/catchup.rs @@ -312,7 +312,7 @@ where &self, _commitment: Commitment, ) -> anyhow::Result { - bail!("not supported"); + bail!("chain config catchup is not supported for SQL data source"); } } @@ -564,7 +564,6 @@ pub mod mock { Ok(()) } - //TODO : ??? async fn try_fetch_chain_config( &self, _commitment: Commitment, diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 4ead582e66..b60f046466 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -565,6 +565,13 @@ impl SequencerStateDataSource for T where } impl ValidatedState { + /// Retrieves the `ChainConfig`. + /// + /// Returns the `NodeState` `ChainConfig` if the `ValidatedState` `ChainConfig` commitment matches the `NodeState` `ChainConfig`` commitment. + /// If the commitments do not match, it returns the `ChainConfig` available in either `ValidatedState` or proposed header. + /// If neither has the `ChainConfig`, it fetches the config from the peers. + /// + /// Returns an error if the proposed header's `ChainConfig` commitment does not match the `ValidatedState` `ChainConfig` commitment, pub(crate) async fn get_chain_config( &self, instance: &NodeState, From 1f8d73fb202cc085205903add7481b72cd4aca65 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 29 May 2024 08:09:29 +0500 Subject: [PATCH 05/48] clippy --- sequencer/src/api.rs | 6 ++++-- sequencer/src/state.rs | 16 +++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 119b5476c2..dccdd7f18c 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -1062,12 +1062,14 @@ mod test { setup_logging(); setup_backtrace(); - let mut state = ValidatedState::default(); // ValidatedState only has the default chain config commitment. // NodeState also has the default ChainConfig. // Therefore, both commitments would match. // This would update the ValidatedState's chain config to NodeState's ChainConfig. - state.chain_config = ChainConfig::default().commit().into(); + let state = ValidatedState { + chain_config: ChainConfig::default().commit().into(), + ..Default::default() + }; // Start a sequencer network without catchup module let port = pick_unused_port().expect("No ports free"); diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index b60f046466..885d9562dc 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -577,13 +577,15 @@ impl ValidatedState { instance: &NodeState, header_cf: &ResolvableChainConfig, ) -> anyhow::Result { - if header_cf.commit() != self.chain_config.commit() { - bail!( - "Proposed header chain config commit={} expected={}", - header_cf.commit(), - self.chain_config.commit() - ); - } + // TODO(abdul) : to be uncommented after we have upgrade mechanism + + // if header_cf.commit() != self.chain_config.commit() { + // bail!( + // "Proposed header chain config commit={} expected={}", + // header_cf.commit(), + // self.chain_config.commit() + // ); + // } if self.chain_config.commit() == instance.chain_config.commit() { return Ok(instance.chain_config); From 35680985a99b81c88759129cdee46f03f726e608 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 31 May 2024 00:39:19 +0500 Subject: [PATCH 06/48] point hotshot to main and others to daily-build --- .vscode/settings.json | 1 + Cargo.lock | 45 +++++----- Cargo.toml | 25 +++--- builder/src/permissioned.rs | 8 +- sequencer/src/api.rs | 169 +++++++++++++++--------------------- sequencer/src/catchup.rs | 19 ++-- sequencer/src/context.rs | 23 ++--- 7 files changed, 131 insertions(+), 159 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 00c38dbdec..1e2618a81e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,4 +3,5 @@ "CARGO_TARGET_DIR": "target/vscode_rust_analyzer", "RUSTFLAGS": " --cfg async_executor_impl=\"async-std\" --cfg async_channel_impl=\"async-std\"" }, + "rust-analyzer.cargo.features": "all", } \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 84713ad34a..fe67f522ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4027,7 +4027,7 @@ dependencies = [ [[package]] name = "hotshot" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" dependencies = [ "anyhow", "async-broadcast", @@ -4072,7 +4072,7 @@ dependencies = [ [[package]] name = "hotshot-builder-api" version = "0.1.7" -source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" dependencies = [ "async-trait", "clap", @@ -4091,7 +4091,7 @@ dependencies = [ [[package]] name = "hotshot-builder-core" version = "0.1.21" -source = "git+https://github.com/EspressoSystems/hotshot-builder-core?branch=abdul/update-hotshot#86db8fad09399489b002c3023d741fa09314f9b3" +source = "git+https://github.com/EspressoSystems/hotshot-builder-core?branch=daily-build#6b1f25f3660229fb220fd23c6ab1a6a73a3b8348" dependencies = [ "anyhow", "async-broadcast", @@ -4143,8 +4143,8 @@ dependencies = [ [[package]] name = "hotshot-events-service" -version = "0.1.22" -source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?branch=abdul/update-hotshot#ef524944a9ef7c1b037d907ad9db70411e33936a" +version = "0.1.23" +source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?branch=daily-build#9b5891dd85b98baf42058131219d425514447325" dependencies = [ "async-broadcast", "async-compatibility-layer", @@ -4169,7 +4169,7 @@ dependencies = [ [[package]] name = "hotshot-example-types" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" dependencies = [ "anyhow", "async-broadcast", @@ -4200,7 +4200,7 @@ dependencies = [ [[package]] name = "hotshot-macros" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" dependencies = [ "derive_builder", "proc-macro2", @@ -4211,7 +4211,7 @@ dependencies = [ [[package]] name = "hotshot-orchestrator" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" dependencies = [ "anyhow", "async-compatibility-layer", @@ -4239,8 +4239,8 @@ dependencies = [ [[package]] name = "hotshot-query-service" -version = "0.1.23" -source = "git+https://github.com/EspressoSystems/hotshot-query-service?branch=abdul/update-hotshot#9e392475b3efaf50c8e6a0f76b98db19acbc583d" +version = "0.1.27" +source = "git+https://github.com/EspressoSystems/hotshot-query-service?branch=daily-build#2f2aa3adf38c344b63e380d3375086341f4be3c9" dependencies = [ "anyhow", "ark-serialize", @@ -4294,7 +4294,7 @@ dependencies = [ [[package]] name = "hotshot-stake-table" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" dependencies = [ "ark-bn254", "ark-ed-on-bn254", @@ -4365,11 +4365,13 @@ dependencies = [ [[package]] name = "hotshot-task" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" dependencies = [ + "anyhow", "async-broadcast", "async-compatibility-layer", "async-std", + "async-trait", "futures", "tokio", "tracing", @@ -4378,7 +4380,7 @@ dependencies = [ [[package]] name = "hotshot-task-impls" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" dependencies = [ "anyhow", "async-broadcast", @@ -4413,8 +4415,9 @@ dependencies = [ [[package]] name = "hotshot-testing" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" dependencies = [ + "anyhow", "async-broadcast", "async-compatibility-layer", "async-lock 2.8.0", @@ -4453,7 +4456,7 @@ dependencies = [ [[package]] name = "hotshot-types" version = "0.1.11" -source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" dependencies = [ "anyhow", "ark-bls12-381", @@ -5716,7 +5719,7 @@ dependencies = [ [[package]] name = "libp2p-networking" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=abdul/pass-validated-state#e15ab38bf381a9687c09699a946a8583fe30c9ca" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" dependencies = [ "anyhow", "async-compatibility-layer", @@ -9537,9 +9540,9 @@ dependencies = [ [[package]] name = "surf-disco" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b649b5919e809b105a8ae833154430f7069b0e3a392fff26a672211c6b8e30d" +checksum = "2e4318abbbbc4d640c6cb45d29cd3c70cf517c751fae9b68f2f76a6ff1950da7" dependencies = [ "async-std", "async-tungstenite", @@ -9841,11 +9844,13 @@ dependencies = [ [[package]] name = "tide-disco" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf3579d6f1362fffc7d1686ed061537afe8a187fd0b18e0f5694ad17e977ff3" +checksum = "44c9d803bc734a5cced767b36bb1caab87e9eb7eb144cffcab38160b084a8cef" dependencies = [ "anyhow", + "async-h1", + "async-lock 3.3.0", "async-std", "async-trait", "clap", diff --git a/Cargo.toml b/Cargo.toml index 5a99050d4c..d22f52f267 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,18 +47,19 @@ dotenvy = "0.15" ethers = { version = "2.0", features = ["solc"] } futures = "0.3" -hotshot = { git = "https://github.com/EspressoSystems/hotshot", branch = "abdul/pass-validated-state" } +hotshot = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } # Hotshot imports -hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", branch = "abdul/pass-validated-state" } -hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", branch = "abdul/update-hotshot" } -hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", branch = "abdul/update-hotshot" } -hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", branch = "abdul/pass-validated-state" } -hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", branch = "abdul/update-hotshot" } -hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", branch = "abdul/pass-validated-state" } +hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", branch = "main" } +hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", branch = "daily-build" } +hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" } +hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", branch = "daily-build" } +hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } +hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", branch = "daily-build" } +hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" } -hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", branch = "abdul/pass-validated-state" } -hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", branch = "abdul/pass-validated-state" } -hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", branch = "abdul/pass-validated-state" } +hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } +hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } +hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } # Push CDN imports cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", features = [ @@ -98,9 +99,9 @@ jf-relation = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4 jf-utils = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.5" } snafu = "0.8" strum = { version = "0.26", features = ["derive"] } -surf-disco = "0.6" +surf-disco = "0.7" tagged-base64 = "0.4" -tide-disco = "0.6" +tide-disco = "0.7" time = "0.3" tracing = "0.1" bytesize = "1.3" diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index 1add88b1d4..21459ac1e6 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -108,7 +108,7 @@ pub struct BuilderContext< Ver: StaticVersionType + 'static, > { /// The consensus handle - pub hotshot_handle: Consensus, + pub hotshot_handle: Arc>, /// Index of this sequencer node pub node_index: u64, @@ -283,7 +283,7 @@ pub async fn init_node>, + handle: Arc>>>, } impl @@ -72,7 +71,7 @@ impl impl Stream> + Unpin { let state = self.clone(); - async move { state.consensus().await.event_stream() } + async move { state.consensus().await.read().await.event_stream() } .boxed() .flatten_stream() } @@ -112,8 +111,8 @@ impl &SystemContextHandle> { - &self.consensus.as_ref().get().await.get_ref().handle + async fn consensus(&self) -> Arc>>> { + Arc::clone(&self.consensus.as_ref().get().await.get_ref().handle) } async fn node_state(&self) -> &NodeState { @@ -127,6 +126,8 @@ impl for ApiState { async fn submit(&self, tx: Transaction) -> anyhow::Result<()> { - self.consensus().await.submit_transaction(tx).await?; + self.consensus() + .await + .read() + .await + .submit_transaction(tx) + .await?; Ok(()) } } @@ -223,9 +229,16 @@ impl anyhow::Result { - let state = self.consensus().await.state(view).await.context(format!( - "state not available for height {height}, view {view:?}" - ))?; + let state = self + .consensus() + .await + .read() + .await + .state(view) + .await + .context(format!( + "state not available for height {height}, view {view:?}" + ))?; let (proof, balance) = FeeAccountProof::prove(&state.fee_merkle_tree, account).context( format!("account {account} not available for height {height}, view {view:?}"), )?; @@ -234,36 +247,20 @@ impl anyhow::Result { - let state = self.consensus().await.state(view).await.context(format!( - "state not available for height {height}, view {view:?}" - ))?; + let state = self + .consensus() + .await + .read() + .await + .state(view) + .await + .context(format!( + "state not available for height {height}, view {view:?}" + ))?; let tree = &state.block_merkle_tree; let frontier = tree.lookup(tree.num_leaves() - 1).expect_ok()?.1; Ok(frontier) } - - #[tracing::instrument(skip(self))] - async fn get_chain_config( - &self, - commitment: Commitment, - ) -> anyhow::Result { - let node_state = self.node_state().await; - let node_state_cf = node_state.chain_config; - if node_state_cf.commit() == commitment { - return Ok(node_state_cf); - } - - let state = self.consensus().await.decided_state().await; - let state_cf = state.chain_config; - - if state_cf.commit() != commitment { - bail!("chain config for commitment={commitment} not found in decided state"); - } - - state_cf.resolve().context(format!( - "cannot resolve to a full chain config for commitment={commitment}" - )) - } } impl @@ -403,7 +400,7 @@ pub mod test_helpers { // Hook the builder up to the event stream from the first node if let Some(builder_task) = builder_task { - builder_task.start(Box::new(handle_0.event_stream())); + builder_task.start(Box::new(handle_0.event_stream().await)); } for ctx in &nodes { @@ -437,9 +434,15 @@ pub mod test_helpers { } pub async fn stop_consensus(&mut self) { - self.server.consensus_mut().shut_down().await; + let consensus = self.server.consensus(); + let mut consensus_writer = consensus.write().await; + consensus_writer.shut_down().await; + drop(consensus_writer); + for ctx in &mut self.peers { - ctx.consensus_mut().shut_down().await; + let consensus = ctx.consensus(); + let mut consensus_writer = consensus.write().await; + consensus_writer.shut_down().await; } } } @@ -513,7 +516,7 @@ pub mod test_helpers { let l1 = anvil.endpoint().parse().unwrap(); let network = TestNetwork::new(options, [no_storage::Options; TestConfig::NUM_NODES], l1).await; - let mut events = network.server.event_stream(); + let mut events = network.server.event_stream().await; client.connect(None).await; @@ -550,7 +553,14 @@ pub mod test_helpers { // Wait for block >=2 appears // It's waiting for an extra second to make sure that the signature is generated loop { - height = network.server.consensus().decided_leaf().await.height(); + height = network + .server + .consensus() + .read() + .await + .decided_leaf() + .await + .height(); sleep(std::time::Duration::from_secs(1)).await; if height >= 2 { break; @@ -582,12 +592,12 @@ pub mod test_helpers { let options = opt(Options::from(options::Http { port }).catchup(Default::default())); let anvil = Anvil::new().spawn(); let l1 = anvil.endpoint().parse().unwrap(); - let mut network = + let network = TestNetwork::new(options, [no_storage::Options; TestConfig::NUM_NODES], l1).await; client.connect(None).await; // Wait for a few blocks to be decided. - let mut events = network.server.event_stream(); + let mut events = network.server.event_stream().await; loop { if let Event { event: EventType::Decide { leaf_chain, .. }, @@ -604,10 +614,19 @@ pub mod test_helpers { } // Stop consensus running on the node so we freeze the decided and undecided states. - network.server.consensus_mut().shut_down().await; + // We'll let it go out of scope here since it's a write lock. + { + let consensus = network.server.consensus(); + let mut consensus_writer = consensus.write().await; + consensus_writer.shut_down().await; + } + + // Re-acquire a read lock to the consensus state. + let consensus = network.server.consensus(); + let consensus_reader = consensus.read().await; // Undecided fee state: absent account. - let leaf = network.server.consensus().decided_leaf().await; + let leaf = consensus_reader.decided_leaf().await; let height = leaf.height() + 1; let view = leaf.view_number() + 1; let res = client @@ -626,6 +645,8 @@ pub mod test_helpers { &network .server .consensus() + .read() + .await .state(view) .await .unwrap() @@ -645,6 +666,8 @@ pub mod test_helpers { let root = &network .server .consensus() + .read() + .await .state(view) .await .unwrap() @@ -855,7 +878,6 @@ mod test { use super::*; use crate::{ catchup::{mock::MockStateCatchup, StatePeers}, - chain_config::ResolvableChainConfig, persistence::no_storage, state::{FeeAccount, FeeAmount, ValidatedState}, testing::TestConfig, @@ -1090,57 +1112,6 @@ mod test { } } - #[async_std::test] - async fn test_validated_state_chain_config() { - setup_logging(); - setup_backtrace(); - - // ValidatedState only has the default chain config commitment. - // NodeState also has the default ChainConfig. - // Therefore, both commitments would match. - // This would update the ValidatedState's chain config to NodeState's ChainConfig. - let state = ValidatedState { - chain_config: ChainConfig::default().commit().into(), - ..Default::default() - }; - - // Start a sequencer network without catchup module - let port = pick_unused_port().expect("No ports free"); - let mut network = TestNetwork::with_state( - Options::from(options::Http { port }), - std::array::from_fn(|_| state.clone()), - [no_storage::Options; TestConfig::NUM_NODES], - std::array::from_fn(|_| MockStateCatchup::default()), - ) - .await; - - network - .server - .event_stream() - .filter(|event| future::ready(matches!(event.event, EventType::Decide { .. }))) - .take(4) - .collect::>() - .await; - - let decided_state = network.server.consensus().decided_state().await; - assert_eq!( - decided_state.chain_config, - ResolvableChainConfig::from(ChainConfig::default()) - ); - - for peer in &network.peers { - let state = peer.consensus().decided_state().await; - assert_eq!( - state.chain_config, - ResolvableChainConfig::from(ChainConfig::default()) - ); - } - - network.stop_consensus().await; - - drop(network); - } - #[async_std::test] pub(crate) async fn test_restart() { setup_logging(); diff --git a/sequencer/src/catchup.rs b/sequencer/src/catchup.rs index 189bf101be..50f3ce1e2f 100644 --- a/sequencer/src/catchup.rs +++ b/sequencer/src/catchup.rs @@ -153,20 +153,23 @@ pub trait StateCatchup: Send + Sync + std::fmt::Debug { &self, commitment: Commitment, ) -> anyhow::Result { + // Retry until we succeed. + let mut delay = MIN_RETRY_DELAY; + loop { match self.try_fetch_chain_config(commitment).await { Ok(cf) => return Ok(cf), Err(err) => { - tracing::warn!("Could not fetch chain config from any peer, retrying: {err:#}"); - async_std::task::sleep(self.retry_interval()).await; + tracing::warn!( + ?delay, + "Could not fetch chain config from any peer, retrying: {err:#}" + ); + sleep(delay).await; + delay = backoff(delay); } } } } - - fn retry_interval(&self) -> Duration { - Duration::from_millis(100) - } } /// A catchup implementation that falls back to a remote provider, but prefers a local provider when @@ -289,10 +292,6 @@ impl StateCatchup for StatePeers { } bail!("Could not fetch chain config from any peer"); } - - fn retry_interval(&self) -> Duration { - self.interval - } } #[derive(Debug, From)] diff --git a/sequencer/src/context.rs b/sequencer/src/context.rs index b0fbefc7b8..781f0b15ce 100644 --- a/sequencer/src/context.rs +++ b/sequencer/src/context.rs @@ -41,7 +41,7 @@ pub struct SequencerContext< > { /// The consensus handle #[derivative(Debug = "ignore")] - handle: Consensus, + handle: Arc>>, /// Context for generating state signatures. state_signer: Arc>, @@ -162,7 +162,7 @@ impl impl Stream> { - self.handle.event_stream() + pub async fn event_stream(&self) -> impl Stream> { + self.handle.read().await.event_stream() } pub async fn submit_transaction(&self, tx: Transaction) -> anyhow::Result<()> { - self.handle.submit_transaction(tx).await?; + self.handle.read().await.submit_transaction(tx).await?; Ok(()) } @@ -216,19 +216,14 @@ impl &Consensus { - &self.handle + pub fn consensus(&self) -> Arc>> { + Arc::clone(&self.handle) } pub fn node_state(&self) -> NodeState { self.node_state.clone() } - /// Return a mutable reference to the underlying consensus handle. - pub fn consensus_mut(&mut self) -> &mut Consensus { - &mut self.handle - } - /// Start participating in consensus. pub async fn start_consensus(&self) { if let Some(orchestrator_client) = &self.wait_for_orchestrator { @@ -238,7 +233,7 @@ impl Date: Fri, 31 May 2024 00:50:05 +0500 Subject: [PATCH 07/48] pass simple builder config --- builder/src/non_permissioned.rs | 5 +++-- builder/src/permissioned.rs | 10 +++++----- sequencer/src/api.rs | 2 +- sequencer/src/lib.rs | 11 +++++++---- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/builder/src/non_permissioned.rs b/builder/src/non_permissioned.rs index d2269553a4..2cfabe6cb8 100644 --- a/builder/src/non_permissioned.rs +++ b/builder/src/non_permissioned.rs @@ -162,9 +162,10 @@ impl BuilderConfig { .as_u64() .context("the base fee exceeds the maximum amount that a builder can pay (defined by u64::MAX)")?, Arc::new(instance_state), + //???? + Duration::from_secs(60), Arc::new(validated_state), - //???? - Duration::from_secs(60), + ); // spawn the builder event loop diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index 21459ac1e6..ef2781e5da 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -381,7 +381,7 @@ impl, + hotshot_handle: Arc>, state_signer: StateSigner, node_index: u64, eth_key_pair: EthKeyPair, @@ -457,10 +457,10 @@ impl Self { let mut cfg = TestConfig::default_with_l1(l1); - let (builder_task, builder_url) = run_test_builder().await; + let (builder_task, builder_url) = run_test_builder(None).await; cfg.set_builder_url(builder_url); diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index e9cc116959..0c85ae3ab4 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -514,6 +514,7 @@ pub mod testing { }; use hotshot::types::{EventType::Decide, Message}; use hotshot_stake_table::vec_based::StakeTable; + use hotshot_testing::block_builder::SimpleBuilderConfig; use hotshot_testing::block_builder::{ BuilderTask, SimpleBuilderImplementation, TestBuilderImplementation, }; @@ -528,10 +529,12 @@ pub mod testing { const STAKE_TABLE_CAPACITY_FOR_TEST: u64 = 10; - pub async fn run_test_builder() -> (Option>>, Url) { + pub async fn run_test_builder( + _port: Option, + ) -> (Option>>, Url) { >::start( TestConfig::NUM_NODES, - (), + SimpleBuilderConfig::default(), ) .await } @@ -806,7 +809,7 @@ mod test { let url = anvil.url(); let mut config = TestConfig::default_with_l1(url); - let (builder_task, builder_url) = run_test_builder().await; + let (builder_task, builder_url) = run_test_builder(None).await; config.set_builder_url(builder_url); @@ -848,7 +851,7 @@ mod test { let url = anvil.url(); let mut config = TestConfig::default_with_l1(url); - let (builder_task, builder_url) = run_test_builder().await; + let (builder_task, builder_url) = run_test_builder(None).await; config.set_builder_url(builder_url); let handles = config.init_nodes(ver).await; From 48ce9bd2e20560640ca3fd5355e671f3ad506a24 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 31 May 2024 01:34:27 +0500 Subject: [PATCH 08/48] merge abdul/move-chain-config, hotshot ss/fee-upgrade others abdul/update-hotshot --- Cargo.lock | 66 ++++++++++++++++---------------- Cargo.toml | 20 +++++----- builder/src/lib.rs | 4 ++ sequencer/src/api/data_source.rs | 12 ++++++ sequencer/src/lib.rs | 4 ++ 5 files changed, 63 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe67f522ae..6461b92895 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -635,7 +635,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "258b52a1aa741b9f09783b2d86cf0aeeb617bbf847f6933340a39644227acbdb" dependencies = [ - "event-listener 5.3.0", + "event-listener 5.3.1", "event-listener-strategy 0.5.2", "futures-core", "pin-project-lite 0.2.14", @@ -1332,9 +1332,9 @@ dependencies = [ [[package]] name = "blst" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c94087b935a822949d3291a9989ad2b2051ea141eda0fd4e478a75f6aa3e604b" +checksum = "62dc83a094a71d43eeadd254b1ec2d24cb6a0bb6cadce00df51f0db594711a32" dependencies = [ "cc", "glob", @@ -1486,9 +1486,9 @@ dependencies = [ [[package]] name = "capnp" -version = "0.19.5" +version = "0.19.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aed85272154b3c0bfda873c40395f13adcfbc89696bf639a512291077f8cd17" +checksum = "de71387912cac7dd3cb7c219e09628411620a18061bba58c71453c26ae7bf66a" dependencies = [ "embedded-io", ] @@ -1926,9 +1926,9 @@ dependencies = [ [[package]] name = "const-hex" -version = "1.11.4" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ff96486ccc291d36a958107caf2c0af8c78c0af7d31ae2f35ce055130de1a6" +checksum = "94fb8a24a26d37e1ffd45343323dc9fe6654ceea44c12f2fcb3d7ac29e610bc6" dependencies = [ "cfg-if", "cpufeatures", @@ -3264,9 +3264,9 @@ dependencies = [ [[package]] name = "event-listener" -version = "5.3.0" +version = "5.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" dependencies = [ "concurrent-queue", "parking", @@ -3289,7 +3289,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" dependencies = [ - "event-listener 5.3.0", + "event-listener 5.3.1", "pin-project-lite 0.2.14", ] @@ -4027,7 +4027,7 @@ dependencies = [ [[package]] name = "hotshot" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=ss/fee-upgrade#53669fb01cdda37395f8421c568e22b9b6d0c8b3" dependencies = [ "anyhow", "async-broadcast", @@ -4072,7 +4072,7 @@ dependencies = [ [[package]] name = "hotshot-builder-api" version = "0.1.7" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=ss/fee-upgrade#53669fb01cdda37395f8421c568e22b9b6d0c8b3" dependencies = [ "async-trait", "clap", @@ -4091,7 +4091,7 @@ dependencies = [ [[package]] name = "hotshot-builder-core" version = "0.1.21" -source = "git+https://github.com/EspressoSystems/hotshot-builder-core?branch=daily-build#6b1f25f3660229fb220fd23c6ab1a6a73a3b8348" +source = "git+https://github.com/EspressoSystems/hotshot-builder-core?branch=abdul/update-hotshot#af1cca8d7ee1ea14cea36fd98b97c4ff6575df10" dependencies = [ "anyhow", "async-broadcast", @@ -4143,8 +4143,8 @@ dependencies = [ [[package]] name = "hotshot-events-service" -version = "0.1.23" -source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?branch=daily-build#9b5891dd85b98baf42058131219d425514447325" +version = "0.1.22" +source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?branch=abdul/update-hotshot#b89e29d44881fec8f1fb21db853dfee26aeaa79d" dependencies = [ "async-broadcast", "async-compatibility-layer", @@ -4169,7 +4169,7 @@ dependencies = [ [[package]] name = "hotshot-example-types" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=ss/fee-upgrade#53669fb01cdda37395f8421c568e22b9b6d0c8b3" dependencies = [ "anyhow", "async-broadcast", @@ -4200,7 +4200,7 @@ dependencies = [ [[package]] name = "hotshot-macros" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=ss/fee-upgrade#53669fb01cdda37395f8421c568e22b9b6d0c8b3" dependencies = [ "derive_builder", "proc-macro2", @@ -4211,7 +4211,7 @@ dependencies = [ [[package]] name = "hotshot-orchestrator" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=ss/fee-upgrade#53669fb01cdda37395f8421c568e22b9b6d0c8b3" dependencies = [ "anyhow", "async-compatibility-layer", @@ -4240,7 +4240,7 @@ dependencies = [ [[package]] name = "hotshot-query-service" version = "0.1.27" -source = "git+https://github.com/EspressoSystems/hotshot-query-service?branch=daily-build#2f2aa3adf38c344b63e380d3375086341f4be3c9" +source = "git+https://github.com/EspressoSystems/hotshot-query-service?branch=abdul/update-hotshot#0fd5941387b2f293bc8effd4bea7739b50e16260" dependencies = [ "anyhow", "ark-serialize", @@ -4294,7 +4294,7 @@ dependencies = [ [[package]] name = "hotshot-stake-table" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=ss/fee-upgrade#53669fb01cdda37395f8421c568e22b9b6d0c8b3" dependencies = [ "ark-bn254", "ark-ed-on-bn254", @@ -4365,7 +4365,7 @@ dependencies = [ [[package]] name = "hotshot-task" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=ss/fee-upgrade#53669fb01cdda37395f8421c568e22b9b6d0c8b3" dependencies = [ "anyhow", "async-broadcast", @@ -4380,7 +4380,7 @@ dependencies = [ [[package]] name = "hotshot-task-impls" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=ss/fee-upgrade#53669fb01cdda37395f8421c568e22b9b6d0c8b3" dependencies = [ "anyhow", "async-broadcast", @@ -4415,7 +4415,7 @@ dependencies = [ [[package]] name = "hotshot-testing" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=ss/fee-upgrade#53669fb01cdda37395f8421c568e22b9b6d0c8b3" dependencies = [ "anyhow", "async-broadcast", @@ -4456,7 +4456,7 @@ dependencies = [ [[package]] name = "hotshot-types" version = "0.1.11" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=ss/fee-upgrade#53669fb01cdda37395f8421c568e22b9b6d0c8b3" dependencies = [ "anyhow", "ark-bls12-381", @@ -4704,9 +4704,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d8d52be92d09acc2e01dddb7fde3ad983fc6489c7db4837e605bc3fca4cb63e" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" dependencies = [ "bytes 1.6.0", "futures-channel", @@ -5719,7 +5719,7 @@ dependencies = [ [[package]] name = "libp2p-networking" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=ss/fee-upgrade#53669fb01cdda37395f8421c568e22b9b6d0c8b3" dependencies = [ "anyhow", "async-compatibility-layer", @@ -7465,7 +7465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.66", @@ -10189,7 +10189,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.8", + "winnow 0.6.9", ] [[package]] @@ -10681,9 +10681,9 @@ dependencies = [ [[package]] name = "vbs" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "971bcbd6c41e4e53fa9eab95b2a8ddaf1826be2f3e86b4101efebc8c3a17fde0" +checksum = "1da07d9ca0c69892851ff70256392d1b0c6408a0007cc87409b1ca3064821e9b" dependencies = [ "anyhow", "bincode", @@ -11140,9 +11140,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c52e9c97a68071b23e836c9380edae937f17b9c4667bd021973efc689f618d" +checksum = "86c949fede1d13936a99f14fafd3e76fd642b556dd2ce96287fbe2e0151bfac6" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index d22f52f267..dfdcf0a5d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,19 +47,19 @@ dotenvy = "0.15" ethers = { version = "2.0", features = ["solc"] } futures = "0.3" -hotshot = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } +hotshot = { git = "https://github.com/EspressoSystems/hotshot", branch = "ss/fee-upgrade" } # Hotshot imports -hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", branch = "main" } -hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", branch = "daily-build" } +hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", branch = "ss/fee-upgrade" } +hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", branch = "abdul/update-hotshot" } hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" } -hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", branch = "daily-build" } -hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } -hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", branch = "daily-build" } -hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } +hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", branch = "abdul/update-hotshot" } +hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", branch = "ss/fee-upgrade" } +hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", branch = "abdul/update-hotshot" } +hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", branch = "ss/fee-upgrade" } hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" } -hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } -hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } -hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } +hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", branch = "ss/fee-upgrade" } +hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", branch = "ss/fee-upgrade" } +hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", branch = "ss/fee-upgrade" } # Push CDN imports cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", features = [ diff --git a/builder/src/lib.rs b/builder/src/lib.rs index 633be6fa5e..e98ec3974c 100644 --- a/builder/src/lib.rs +++ b/builder/src/lib.rs @@ -226,6 +226,10 @@ pub mod testing { known_nodes_with_stake.clone().len() as u64, known_nodes_with_stake.clone().len() as u64, ), + start_proposing_view: 0, + stop_proposing_view: 0, + start_voting_view: 0, + stop_voting_view: 0, }; Self { diff --git a/sequencer/src/api/data_source.rs b/sequencer/src/api/data_source.rs index f973be1c1c..76791bb2e0 100644 --- a/sequencer/src/api/data_source.rs +++ b/sequencer/src/api/data_source.rs @@ -216,6 +216,10 @@ pub struct PublicHotShotConfig { pub builder_timeout: Duration, pub data_request_delay: Duration, pub builder_url: Url, + pub start_proposing_view: u64, + pub stop_proposing_view: u64, + pub start_voting_view: u64, + pub stop_voting_view: u64, } impl From> for PublicHotShotConfig { @@ -244,6 +248,10 @@ impl From> for PublicHotShotConfig { builder_timeout, data_request_delay, builder_url, + start_proposing_view, + stop_proposing_view, + start_voting_view, + stop_voting_view, } = v; Self { @@ -267,6 +275,10 @@ impl From> for PublicHotShotConfig { builder_timeout, data_request_delay, builder_url, + start_proposing_view, + stop_proposing_view, + start_voting_view, + stop_voting_view, } } } diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 0c85ae3ab4..954709da99 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -599,6 +599,10 @@ pub mod testing { known_nodes_with_stake.clone().len() as u64, known_nodes_with_stake.clone().len() as u64, ), + start_proposing_view: 0, + stop_proposing_view: 0, + start_voting_view: 0, + stop_voting_view: 0, }; Self { From 8f5df887ef6a941b7ea8ee64abba9b7168b8e4c1 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 31 May 2024 01:38:20 +0500 Subject: [PATCH 09/48] make clippy happy --- builder/src/non_permissioned.rs | 1 - builder/src/permissioned.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/builder/src/non_permissioned.rs b/builder/src/non_permissioned.rs index 2cfabe6cb8..56f143d5e9 100644 --- a/builder/src/non_permissioned.rs +++ b/builder/src/non_permissioned.rs @@ -165,7 +165,6 @@ impl BuilderConfig { //???? Duration::from_secs(60), Arc::new(validated_state), - ); // spawn the builder event loop diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index ef2781e5da..9bbc448539 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -460,7 +460,6 @@ impl Date: Fri, 31 May 2024 02:01:40 +0500 Subject: [PATCH 10/48] update lockfile to point to specific hotshot commit --- Cargo.lock | 22 +++++++++++----------- sequencer/src/lib.rs | 3 +-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe67f522ae..dc452f6df1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4027,7 +4027,7 @@ dependencies = [ [[package]] name = "hotshot" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#74ee1daf26e5df8883a4dd6d77881886e087bf31" dependencies = [ "anyhow", "async-broadcast", @@ -4072,7 +4072,7 @@ dependencies = [ [[package]] name = "hotshot-builder-api" version = "0.1.7" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#74ee1daf26e5df8883a4dd6d77881886e087bf31" dependencies = [ "async-trait", "clap", @@ -4169,7 +4169,7 @@ dependencies = [ [[package]] name = "hotshot-example-types" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#74ee1daf26e5df8883a4dd6d77881886e087bf31" dependencies = [ "anyhow", "async-broadcast", @@ -4200,7 +4200,7 @@ dependencies = [ [[package]] name = "hotshot-macros" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#74ee1daf26e5df8883a4dd6d77881886e087bf31" dependencies = [ "derive_builder", "proc-macro2", @@ -4211,7 +4211,7 @@ dependencies = [ [[package]] name = "hotshot-orchestrator" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#74ee1daf26e5df8883a4dd6d77881886e087bf31" dependencies = [ "anyhow", "async-compatibility-layer", @@ -4294,7 +4294,7 @@ dependencies = [ [[package]] name = "hotshot-stake-table" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#74ee1daf26e5df8883a4dd6d77881886e087bf31" dependencies = [ "ark-bn254", "ark-ed-on-bn254", @@ -4365,7 +4365,7 @@ dependencies = [ [[package]] name = "hotshot-task" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#74ee1daf26e5df8883a4dd6d77881886e087bf31" dependencies = [ "anyhow", "async-broadcast", @@ -4380,7 +4380,7 @@ dependencies = [ [[package]] name = "hotshot-task-impls" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#74ee1daf26e5df8883a4dd6d77881886e087bf31" dependencies = [ "anyhow", "async-broadcast", @@ -4415,7 +4415,7 @@ dependencies = [ [[package]] name = "hotshot-testing" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#74ee1daf26e5df8883a4dd6d77881886e087bf31" dependencies = [ "anyhow", "async-broadcast", @@ -4456,7 +4456,7 @@ dependencies = [ [[package]] name = "hotshot-types" version = "0.1.11" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#74ee1daf26e5df8883a4dd6d77881886e087bf31" dependencies = [ "anyhow", "ark-bls12-381", @@ -5719,7 +5719,7 @@ dependencies = [ [[package]] name = "libp2p-networking" version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#95dc95f6ecd5bb2085c0b3588b70d08fe2397fae" +source = "git+https://github.com/EspressoSystems/hotshot?branch=main#74ee1daf26e5df8883a4dd6d77881886e087bf31" dependencies = [ "anyhow", "async-compatibility-layer", diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 0c85ae3ab4..087c24b023 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -514,7 +514,6 @@ pub mod testing { }; use hotshot::types::{EventType::Decide, Message}; use hotshot_stake_table::vec_based::StakeTable; - use hotshot_testing::block_builder::SimpleBuilderConfig; use hotshot_testing::block_builder::{ BuilderTask, SimpleBuilderImplementation, TestBuilderImplementation, }; @@ -534,7 +533,7 @@ pub mod testing { ) -> (Option>>, Url) { >::start( TestConfig::NUM_NODES, - SimpleBuilderConfig::default(), + (), ) .await } From 05629c10eacc920d2d28f3e2b2dbcd178a4f91f1 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 31 May 2024 02:12:59 +0500 Subject: [PATCH 11/48] update builder-core --- Cargo.lock | 2 +- builder/src/non_permissioned.rs | 3 +-- builder/src/permissioned.rs | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6461b92895..dd2af35398 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4091,7 +4091,7 @@ dependencies = [ [[package]] name = "hotshot-builder-core" version = "0.1.21" -source = "git+https://github.com/EspressoSystems/hotshot-builder-core?branch=abdul/update-hotshot#af1cca8d7ee1ea14cea36fd98b97c4ff6575df10" +source = "git+https://github.com/EspressoSystems/hotshot-builder-core?branch=abdul/update-hotshot#303348d22e59a805b254e59ebd87f01675b15a92" dependencies = [ "anyhow", "async-broadcast", diff --git a/builder/src/non_permissioned.rs b/builder/src/non_permissioned.rs index 56f143d5e9..6f4939d695 100644 --- a/builder/src/non_permissioned.rs +++ b/builder/src/non_permissioned.rs @@ -163,8 +163,7 @@ impl BuilderConfig { .context("the base fee exceeds the maximum amount that a builder can pay (defined by u64::MAX)")?, Arc::new(instance_state), //???? - Duration::from_secs(60), - Arc::new(validated_state), + Arc::new(validated_state), Duration::from_secs(60), ); // spawn the builder event loop diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index 9bbc448539..c21fc3f559 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -458,8 +458,8 @@ impl Date: Fri, 31 May 2024 03:10:02 +0500 Subject: [PATCH 12/48] arc SystemContextHandle --- Cargo.lock | 14 ++++++------- builder/src/lib.rs | 10 ++++----- builder/src/non_permissioned.rs | 1 - sequencer/src/api.rs | 15 +++++++++---- sequencer/src/genesis.rs | 37 ++++++++++++++++++++++++++++++++- sequencer/src/lib.rs | 8 +++---- sequencer/src/main.rs | 1 + 7 files changed, 64 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd2af35398..d22b7673e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4091,7 +4091,7 @@ dependencies = [ [[package]] name = "hotshot-builder-core" version = "0.1.21" -source = "git+https://github.com/EspressoSystems/hotshot-builder-core?branch=abdul/update-hotshot#303348d22e59a805b254e59ebd87f01675b15a92" +source = "git+https://github.com/EspressoSystems/hotshot-builder-core?branch=abdul/update-hotshot#d1c4b3c783d008cd8a46465aad91c9e772ebe059" dependencies = [ "anyhow", "async-broadcast", @@ -4240,7 +4240,7 @@ dependencies = [ [[package]] name = "hotshot-query-service" version = "0.1.27" -source = "git+https://github.com/EspressoSystems/hotshot-query-service?branch=abdul/update-hotshot#0fd5941387b2f293bc8effd4bea7739b50e16260" +source = "git+https://github.com/EspressoSystems/hotshot-query-service?branch=abdul/update-hotshot#e9f909f679a0613ce2a917d76ee7a6bf2853c402" dependencies = [ "anyhow", "ark-serialize", @@ -7465,7 +7465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.66", @@ -10012,9 +10012,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes 1.6.0", @@ -10042,9 +10042,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", diff --git a/builder/src/lib.rs b/builder/src/lib.rs index e98ec3974c..c05e797bd4 100644 --- a/builder/src/lib.rs +++ b/builder/src/lib.rs @@ -326,7 +326,7 @@ pub mod testing { bind_version: Ver, options: impl PersistenceOptions, ) -> Vec<( - SystemContextHandle>, + Arc>>, Option>, )> { let num_staked_nodes = self.num_staked_nodes(); @@ -352,7 +352,7 @@ pub mod testing { ) .await; // wrapped in some because need to take later - (hotshot_handle, Some(state_signer)) + (Arc::new(hotshot_handle), Some(state_signer)) } })) .await @@ -456,7 +456,7 @@ pub mod testing { hotshot_events_api_url: Url, known_nodes_with_stake: Vec>, num_non_staking_nodes: usize, - hotshot_context_handle: SystemContextHandle>, + hotshot_context_handle: Arc>>, ) { // create a event streamer let events_streamer = Arc::new(RwLock::new(EventsStreamer::new( @@ -589,7 +589,7 @@ pub mod testing { { pub async fn init_permissioned_builder( hotshot_test_config: HotShotTestConfig, - hotshot_handle: SystemContextHandle>, + hotshot_handle: Arc>>, node_id: u64, state_signer: StateSigner, hotshot_builder_api_url: Url, @@ -617,7 +617,7 @@ pub mod testing { let bootstrapped_view = ViewNumber::new(0); let builder_context = BuilderContext::init( - hotshot_handle, + hotshot_handle.into(), state_signer, node_id, key_pair, diff --git a/builder/src/non_permissioned.rs b/builder/src/non_permissioned.rs index 6f4939d695..81aacacbe1 100644 --- a/builder/src/non_permissioned.rs +++ b/builder/src/non_permissioned.rs @@ -283,7 +283,6 @@ mod test { [NonPermissionedBuilderTestConfig::SUBSCRIBED_DA_NODE_ID] .0 .clone(); - // hotshot event streaming api url let hotshot_events_streaming_api_url = HotShotTestConfig::hotshot_event_streaming_api_url(); diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index d9d2ed769e..cce823554a 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -744,7 +744,7 @@ mod api_tests { l1, ) .await; - let mut events = network.server.event_stream(); + let mut events = network.server.event_stream().await; // Connect client. let client: Client = @@ -1041,7 +1041,7 @@ mod test { .await; // Wait for replica 0 to reach a (non-genesis) decide, before disconnecting it. - let mut events = network.peers[0].event_stream(); + let mut events = network.peers[0].event_stream().await; loop { let event = events.next().await.unwrap(); let EventType::Decide { leaf_chain, .. } = event.event else { @@ -1063,6 +1063,7 @@ mod test { network .server .event_stream() + .await .filter(|event| future::ready(matches!(event.event, EventType::Decide { .. }))) .take(3) .collect::>() @@ -1083,7 +1084,7 @@ mod test { SEQUENCER_VERSION, ) .await; - let mut events = node.event_stream(); + let mut events = node.event_stream().await; // Wait for a (non-genesis) block proposed by each node, to prove that the lagging node has // caught up and all nodes are in sync. @@ -1181,7 +1182,13 @@ mod test { let decided_view = chain.last().unwrap().leaf().view_number(); // Get the most recent state, for catchup. - let state = network.server.consensus().decided_state().await; + let state = network + .server + .consensus() + .read() + .await + .decided_state() + .await; tracing::info!(?decided_view, ?state, "consensus state"); // Fully shut down the API servers. diff --git a/sequencer/src/genesis.rs b/sequencer/src/genesis.rs index 858e9e4810..dcddc4af2b 100644 --- a/sequencer/src/genesis.rs +++ b/sequencer/src/genesis.rs @@ -7,8 +7,12 @@ use anyhow::Context; use derive_more::{Display, From, Into}; use sequencer_utils::{impl_serde_from_string_or_integer, ser::FromStringOrInteger}; use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, path::Path}; +use std::{ + collections::{BTreeMap, HashMap}, + path::Path, +}; use time::{format_description::well_known::Rfc3339 as TimestampFormat, OffsetDateTime}; +use vbs::version::Version; /// Initial configuration of an Espresso stake table. #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -98,6 +102,12 @@ pub struct Genesis { pub accounts: HashMap, pub l1_finalized: Option, pub header: GenesisHeader, + pub upgrades: BTreeMap, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum Upgrade { + ChainConfig(ChainConfig), } impl Genesis { @@ -118,9 +128,34 @@ impl Genesis { #[cfg(test)] mod test { use super::*; + use es_version::SEQUENCER_VERSION; use ethers::prelude::{Address, H160, H256}; + use toml::toml; + #[test] + fn test_to_toml() { + let cf = ChainConfig { + chain_id: 777.into(), + max_block_size: 1054.into(), + ..Default::default() + }; + + let mut upgrades: BTreeMap = BTreeMap::new(); + upgrades.insert(SEQUENCER_VERSION.version(), Upgrade::ChainConfig(cf)); + + let genesis = Genesis { + upgrades, + chain_config: Default::default(), + stake_table: StakeTableConfig { capacity: 10 }, + accounts: Default::default(), + l1_finalized: Default::default(), + header: Default::default(), + }; + + genesis.to_file("../tmp/genesis"); + } + #[test] fn test_genesis_from_toml_with_optional_fields() { let toml = toml! { diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 954709da99..50d6f09e6c 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -823,10 +823,10 @@ mod test { // Hook the builder up to the event stream from the first node if let Some(builder_task) = builder_task { - builder_task.start(Box::new(handle_0.event_stream())); + builder_task.start(Box::new(handle_0.event_stream().await)); } - let mut events = handle_0.event_stream(); + let mut events = handle_0.event_stream().await; for handle in handles.iter() { handle.start_consensus().await; @@ -862,11 +862,11 @@ mod test { let handle_0 = &handles[0]; - let mut events = handle_0.event_stream(); + let mut events = handle_0.event_stream().await; // Hook the builder up to the event stream from the first node if let Some(builder_task) = builder_task { - builder_task.start(Box::new(handle_0.event_stream())); + builder_task.start(Box::new(handle_0.event_stream().await)); } for handle in handles.iter() { diff --git a/sequencer/src/main.rs b/sequencer/src/main.rs index dc5b422597..93ee9678f5 100644 --- a/sequencer/src/main.rs +++ b/sequencer/src/main.rs @@ -194,6 +194,7 @@ mod test { accounts: Default::default(), l1_finalized: Default::default(), header: Default::default(), + upgrades: Default::default(), }; genesis.to_file(&genesis_file).unwrap(); From 375e34a665bd3230311d2d2930b13bedc6c6d43a Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Mon, 3 Jun 2024 17:47:38 +0500 Subject: [PATCH 13/48] add upgrades to Genesis and Nodestate --- Cargo.lock | 1 + Cargo.toml | 1 + builder/src/lib.rs | 4 + builder/src/non_permissioned.rs | 1 + builder/src/permissioned.rs | 2 + sequencer/Cargo.toml | 1 + sequencer/src/api/options.rs | 2 +- sequencer/src/genesis.rs | 133 +++++++++++++++++++++++++------- sequencer/src/header.rs | 22 ++++-- sequencer/src/lib.rs | 28 ++++++- sequencer/src/state.rs | 110 ++++++++++++++++---------- 11 files changed, 228 insertions(+), 77 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d22b7673e1..dc1314a213 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8651,6 +8651,7 @@ dependencies = [ "time 0.3.36", "tokio-postgres", "toml", + "toml_edit 0.22.13", "tracing", "tracing-subscriber 0.3.18", "trait-set", diff --git a/Cargo.toml b/Cargo.toml index dfdcf0a5d6..17c8a57c29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,6 +111,7 @@ rand_distr = "0.4" reqwest = "0.12" serde = { version = "1.0.195", features = ["derive"] } toml = "0.8" +toml_edit = { version = "0.22", features = ["serde"] } url = "2.3" vbs = "0.1" vergen = { version = "8.3", features = ["git", "gitcl"] } diff --git a/builder/src/lib.rs b/builder/src/lib.rs index c05e797bd4..89fc422847 100644 --- a/builder/src/lib.rs +++ b/builder/src/lib.rs @@ -121,6 +121,7 @@ pub mod testing { ExecutionType, HotShotConfig, PeerConfig, ValidatorConfig, }; use portpicker::pick_unused_port; + use vbs::version::StaticVersion; //use sequencer::persistence::NoStorage; use async_broadcast::{ broadcast, Receiver as BroadcastReceiver, RecvError, Sender as BroadcastSender, @@ -397,6 +398,7 @@ pub mod testing { ChainConfig::default(), L1Client::new(self.anvil.endpoint().parse().unwrap(), 1), MockStateCatchup::default(), + Ver::version(), ) .with_genesis(ValidatedState::default()); @@ -538,6 +540,7 @@ pub mod testing { 1, ), MockStateCatchup::default(), + StaticVersion::<1, 0>::version(), ) .with_genesis(ValidatedState::default()); @@ -603,6 +606,7 @@ pub mod testing { 1, ), MockStateCatchup::default(), + StaticVersion::<1, 0>::version(), ) .with_genesis(ValidatedState::default()); diff --git a/builder/src/non_permissioned.rs b/builder/src/non_permissioned.rs index 81aacacbe1..c80b8f0632 100644 --- a/builder/src/non_permissioned.rs +++ b/builder/src/non_permissioned.rs @@ -70,6 +70,7 @@ pub fn build_instance_state( chain_config, l1_client, Arc::new(StatePeers::::from_urls(state_peers)), + Ver::version(), ); Ok(instance_state) } diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index c21fc3f559..687e0a923e 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -263,6 +263,8 @@ pub async fn init_node::from_urls(network_params.state_peers)), node_id: node_index, + upgrades: Default::default(), + sequencer_version: Ver::version(), }; let stake_table_commit = diff --git a/sequencer/Cargo.toml b/sequencer/Cargo.toml index 2236b92c32..7686392ae2 100644 --- a/sequencer/Cargo.toml +++ b/sequencer/Cargo.toml @@ -100,6 +100,7 @@ tokio-postgres = { version = "0.7", default-features = false, features = [ # dis "with-serde_json-1", ] } toml = { workspace = true } +toml_edit = { workspace = true } tracing = { workspace = true } tracing-subscriber = "0.3.18" trait-set = "0.3.0" diff --git a/sequencer/src/api/options.rs b/sequencer/src/api/options.rs index 3caeb66f40..08cc2e369c 100644 --- a/sequencer/src/api/options.rs +++ b/sequencer/src/api/options.rs @@ -371,7 +371,7 @@ impl Options { let get_node_state = async move { state.node_state().await.clone() }; tasks.spawn( "merklized state storage update loop", - update_state_storage_loop(ds, get_node_state), + update_state_storage_loop(ds, get_node_state, Ver::version()), ); } diff --git a/sequencer/src/genesis.rs b/sequencer/src/genesis.rs index dcddc4af2b..c0a8c2602c 100644 --- a/sequencer/src/genesis.rs +++ b/sequencer/src/genesis.rs @@ -102,12 +102,103 @@ pub struct Genesis { pub accounts: HashMap, pub l1_finalized: Option, pub header: GenesisHeader, + #[serde(rename = "upgrade", with = "upgrade_serialization")] + #[serde(default)] pub upgrades: BTreeMap, } #[derive(Clone, Debug, Deserialize, Serialize)] -pub enum Upgrade { - ChainConfig(ChainConfig), +#[serde(untagged)] +#[serde(rename_all = "snake_case")] +pub enum UpgradeType { + // Note: Wrapping this in a tuple variant causes deserialization to fail because + // the 'chain_config' name is also provided in the TOML input. + ChainConfig { chain_config: ChainConfig }, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Upgrade { + pub block: u64, + #[serde(flatten)] + pub upgrade_type: UpgradeType, +} + +mod upgrade_serialization { + use crate::genesis::{Upgrade, UpgradeType}; + use serde::ser::SerializeSeq; + use serde::{ + de::{SeqAccess, Visitor}, + Deserialize, Deserializer, Serializer, + }; + use std::{collections::BTreeMap, fmt}; + use vbs::version::Version; + + pub fn serialize(map: &BTreeMap, serializer: S) -> Result + where + S: Serializer, + { + let mut seq = serializer.serialize_seq(Some(map.len()))?; + for (version, upgrade) in map { + seq.serialize_element(&( + version.to_string(), + upgrade.block, + upgrade.upgrade_type.clone(), + ))?; + } + seq.end() + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + struct VecToHashMap; + + impl<'de> Visitor<'de> for VecToHashMap { + type Value = BTreeMap; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a vector of tuples (key-value pairs)") + } + + fn visit_seq(self, mut seq: A) -> Result, A::Error> + where + A: SeqAccess<'de>, + { + let mut map = BTreeMap::new(); + + #[derive(Deserialize)] + struct UpgradeFields { + version: String, + block: u64, + #[serde(flatten)] + upgrade_type: UpgradeType, + } + + while let Some(fields) = seq.next_element::()? { + // add try_from in Version + let version: Vec<_> = fields.version.split(".").collect(); + + let version = Version { + major: version[0].parse().expect("invalid version"), + minor: version[1].parse().expect("invalid version"), + }; + + map.insert( + version, + Upgrade { + block: fields.block, + upgrade_type: fields.upgrade_type, + }, + ); + } + + Ok(map) + } + } + + deserializer.deserialize_seq(VecToHashMap) + } } impl Genesis { @@ -121,6 +212,7 @@ impl Genesis { let path = path.as_ref(); let bytes = std::fs::read(path).context(format!("genesis file {}", path.display()))?; let text = std::str::from_utf8(&bytes).context("genesis file must be UTF-8")?; + toml::from_str(text).context("malformed genesis file") } } @@ -128,34 +220,10 @@ impl Genesis { #[cfg(test)] mod test { use super::*; - use es_version::SEQUENCER_VERSION; - use ethers::prelude::{Address, H160, H256}; + use ethers::prelude::{Address, H160, H256}; use toml::toml; - #[test] - fn test_to_toml() { - let cf = ChainConfig { - chain_id: 777.into(), - max_block_size: 1054.into(), - ..Default::default() - }; - - let mut upgrades: BTreeMap = BTreeMap::new(); - upgrades.insert(SEQUENCER_VERSION.version(), Upgrade::ChainConfig(cf)); - - let genesis = Genesis { - upgrades, - chain_config: Default::default(), - stake_table: StakeTableConfig { capacity: 10 }, - accounts: Default::default(), - l1_finalized: Default::default(), - header: Default::default(), - }; - - genesis.to_file("../tmp/genesis"); - } - #[test] fn test_genesis_from_toml_with_optional_fields() { let toml = toml! { @@ -180,6 +248,17 @@ mod test { number = 64 timestamp = "0x123def" hash = "0x80f5dd11f2bdda2814cb1ad94ef30a47de02cf28ad68c89e104c00c4e51bb7a5" + + [[upgrade]] + version = "1.0" + block = 1 + + [upgrade.chain_config] + chain_id = 12345 + max_block_size = 30000 + base_fee = 1 + fee_recipient = "0x0000000000000000000000000000000000000000" + fee_contract = "0x0000000000000000000000000000000000000000" } .to_string(); diff --git a/sequencer/src/header.rs b/sequencer/src/header.rs index 1b061a51b2..1b6029f2d6 100644 --- a/sequencer/src/header.rs +++ b/sequencer/src/header.rs @@ -288,12 +288,16 @@ impl BlockHeader for Header { _vid_common: VidCommon, _version: Version, ) -> Result { - let chain_config = instance_state.chain_config; let height = parent_leaf.height(); let view = parent_leaf.view_number(); let mut validated_state = parent_state.clone(); + let chain_config = validated_state + .chain_config + .resolve() + .unwrap_or_else(|| instance_state.chain_config); + // Fetch the latest L1 snapshot. let l1_snapshot = instance_state.l1_client.snapshot().await; // Fetch the new L1 deposits between parent and current finalized L1 block. @@ -496,6 +500,7 @@ mod test_headers { }; use hotshot_types::{traits::signature_key::BuilderSignatureKey, vid::vid_scheme}; use jf_vid::VidScheme; + use vbs::version::{StaticVersion, StaticVersionType}; #[derive(Debug, Default)] #[must_use] @@ -793,9 +798,11 @@ mod test_headers { parent_header.block_merkle_tree_root = block_merkle_tree_root; let mut proposal = parent_header.clone(); + let ver = StaticVersion::<1, 0>::version(); + // Pass a different chain config to trigger a chain config validation error. let state = validated_state - .apply_header(&genesis.instance_state, &parent_leaf, &proposal) + .apply_header(&genesis.instance_state, &parent_leaf, &proposal, ver) .await .unwrap() .0; @@ -817,7 +824,7 @@ mod test_headers { // Advance `proposal.height` to trigger validation error. let validated_state = validated_state - .apply_header(&genesis.instance_state, &parent_leaf, &proposal) + .apply_header(&genesis.instance_state, &parent_leaf, &proposal, ver) .await .unwrap() .0; @@ -838,7 +845,7 @@ mod test_headers { proposal.height += 1; let validated_state = validated_state - .apply_header(&genesis.instance_state, &parent_leaf, &proposal) + .apply_header(&genesis.instance_state, &parent_leaf, &proposal, ver) .await .unwrap() .0; @@ -935,7 +942,12 @@ mod test_headers { block_merkle_tree.push(proposal.commit()).unwrap(); let proposal_state = proposal_state - .apply_header(&genesis_state, &parent_leaf, &proposal) + .apply_header( + &genesis_state, + &parent_leaf, + &proposal, + StaticVersion::<1, 0>::version(), + ) .await .unwrap() .0; diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 50d6f09e6c..1827d19e8d 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -20,7 +20,7 @@ use block::entry::TxTableEntryWord; use catchup::{StateCatchup, StatePeers}; use context::SequencerContext; use ethers::types::U256; -use genesis::{GenesisHeader, L1Finalized}; +use genesis::{GenesisHeader, L1Finalized, Upgrade}; // Should move `STAKE_TABLE_CAPACITY` in the sequencer repo when we have variate stake table support @@ -73,7 +73,7 @@ use persistence::{PersistenceOptions, SequencerPersistence}; use serde::{Deserialize, Serialize}; use snafu::Snafu; use std::{collections::BTreeMap, fmt::Debug, marker::PhantomData, net::SocketAddr, sync::Arc}; -use vbs::version::StaticVersionType; +use vbs::version::{StaticVersion, StaticVersionType, Version}; #[cfg(feature = "libp2p")] use std::time::Duration; @@ -171,6 +171,8 @@ pub struct NodeState { pub genesis_header: GenesisHeader, pub genesis_state: ValidatedState, pub l1_genesis: Option, + pub upgrades: BTreeMap, + pub sequencer_version: Version, } impl NodeState { @@ -179,6 +181,7 @@ impl NodeState { chain_config: ChainConfig, l1_client: L1Client, catchup: impl StateCatchup + 'static, + sequencer_version: Version, ) -> Self { Self { node_id, @@ -188,16 +191,21 @@ impl NodeState { genesis_header: Default::default(), genesis_state: Default::default(), l1_genesis: None, + upgrades: Default::default(), + sequencer_version, } } #[cfg(any(test, feature = "testing"))] pub fn mock() -> Self { + use vbs::version::StaticVersion; + Self::new( 0, ChainConfig::default(), L1Client::new("http://localhost:3331".parse().unwrap(), 10000), catchup::mock::MockStateCatchup::default(), + StaticVersion::<1, 0>::version(), ) } @@ -227,6 +235,7 @@ impl Default for NodeState { ChainConfig::default(), L1Client::new("http://localhost:3331".parse().unwrap(), 10000), catchup::mock::MockStateCatchup::default(), + StaticVersion::<1, 0>::version(), ) } } @@ -341,7 +350,7 @@ pub async fn init_node( .with_context(|| "Failed to derive Libp2p peer ID")?; let mut persistence = persistence_opt.clone().create().await?; - let (config, wait_for_orchestrator) = match persistence.load_config().await? { + let (mut config, wait_for_orchestrator) = match persistence.load_config().await? { Some(config) => { tracing::info!("loaded network config from storage, rejoining existing network"); (config, false) @@ -372,6 +381,16 @@ pub async fn init_node( (config, true) } }; + + let version = Ver::version(); + if let Some(upgrade) = genesis.upgrades.get(&version) { + let block = upgrade.block; + // >>>> ????? + config.config.start_proposing_view = block; + config.config.stop_proposing_view = block + 20; + config.config.start_voting_view = block + 21; + config.config.stop_proposing_view = block + 40; + } let node_index = config.node_index; // If we are a DA node, we need to subscribe to the DA topic @@ -472,6 +491,8 @@ pub async fn init_node( ) .await, node_id: node_index, + upgrades: genesis.upgrades, + sequencer_version: Ver::version(), }; let mut ctx = SequencerContext::init( @@ -720,6 +741,7 @@ pub mod testing { ChainConfig::default(), L1Client::new(self.url.clone(), 1000), catchup::local_and_remote(persistence_opt.clone(), catchup).await, + Ver::version(), ) .with_genesis(state); diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 885d9562dc..ed00459ddd 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -1,7 +1,7 @@ use crate::{ api::data_source::CatchupDataSource, catchup::SqlStateCatchup, - chain_config::ResolvableChainConfig, eth_signature_key::EthKeyPair, ChainConfig, Header, Leaf, - NodeState, SeqTypes, + chain_config::ResolvableChainConfig, eth_signature_key::EthKeyPair, genesis::UpgradeType, + ChainConfig, Header, Leaf, NodeState, SeqTypes, }; use anyhow::{anyhow, bail, ensure, Context}; use ark_serialize::{ @@ -333,6 +333,7 @@ async fn compute_state_update( instance: &NodeState, parent_leaf: &LeafQueryData, proposed_leaf: &LeafQueryData, + version: Version, ) -> anyhow::Result<(ValidatedState, Delta)> { let proposed_leaf = proposed_leaf.leaf(); let parent_leaf = parent_leaf.leaf(); @@ -353,7 +354,9 @@ async fn compute_state_update( parent_header.fee_merkle_tree_root ); - state.apply_header(instance, parent_leaf, header).await + state + .apply_header(instance, parent_leaf, header, version) + .await } async fn store_state_update( @@ -435,10 +438,12 @@ async fn update_state_storage( instance: &NodeState, parent_leaf: &LeafQueryData, proposed_leaf: &LeafQueryData, + version: Version, ) -> anyhow::Result { - let (state, delta) = compute_state_update(parent_state, instance, parent_leaf, proposed_leaf) - .await - .context("computing state update")?; + let (state, delta) = + compute_state_update(parent_state, instance, parent_leaf, proposed_leaf, version) + .await + .context("computing state update")?; let mut storage = storage.write().await; if let Err(err) = store_state_update(&mut *storage, proposed_leaf.height(), &state, delta).await @@ -486,6 +491,7 @@ async fn store_genesis_state( pub(crate) async fn update_state_storage_loop( storage: Arc>, instance: impl Future, + version: Version, ) -> anyhow::Result<()> { let mut instance = instance.await; instance.peers = Arc::new(SqlStateCatchup::from(storage.clone())); @@ -520,8 +526,15 @@ pub(crate) async fn update_state_storage_loop( while let Some(leaf) = leaves.next().await { loop { - match update_state_storage(&parent_state, &storage, &instance, &parent_leaf, &leaf) - .await + match update_state_storage( + &parent_state, + &storage, + &instance, + &parent_leaf, + &leaf, + version, + ) + .await { Ok(state) => { parent_leaf = leaf; @@ -572,36 +585,55 @@ impl ValidatedState { /// If neither has the `ChainConfig`, it fetches the config from the peers. /// /// Returns an error if the proposed header's `ChainConfig` commitment does not match the `ValidatedState` `ChainConfig` commitment, - pub(crate) async fn get_chain_config( - &self, + pub(crate) async fn apply_chain_config( + &mut self, instance: &NodeState, header_cf: &ResolvableChainConfig, + version: Version, ) -> anyhow::Result { // TODO(abdul) : to be uncommented after we have upgrade mechanism - // if header_cf.commit() != self.chain_config.commit() { - // bail!( - // "Proposed header chain config commit={} expected={}", - // header_cf.commit(), - // self.chain_config.commit() - // ); - // } - - if self.chain_config.commit() == instance.chain_config.commit() { - return Ok(instance.chain_config); + if header_cf.commit() != self.chain_config.commit() { + bail!( + "Proposed header chain config commit={} expected={}", + header_cf.commit(), + self.chain_config.commit() + ); } - match (self.chain_config.resolve(), header_cf.resolve()) { - (Some(cf), _) => Ok(cf), - (_, Some(cf)) => Ok(cf), - (None, None) => { - instance - .peers - .as_ref() - .fetch_chain_config(self.chain_config.commit()) - .await + let current_version = instance.sequencer_version; + + if version > current_version { + let upgrades = &instance.upgrades; + if let Some(upgrade) = upgrades.get(&version) { + match upgrade.upgrade_type { + UpgradeType::ChainConfig { chain_config } => { + self.chain_config = chain_config.into(); + return Ok(chain_config); + } + } } } + + let cf = if self.chain_config.commit() == instance.chain_config.commit() { + instance.chain_config + } else { + match (self.chain_config.resolve(), header_cf.resolve()) { + (Some(cf), _) => cf, + (_, Some(cf)) => cf, + (None, None) => { + instance + .peers + .as_ref() + .fetch_chain_config(self.chain_config.commit()) + .await? + } + } + }; + + self.chain_config = cf.into(); + + Ok(cf) } pub(crate) async fn apply_header( @@ -609,6 +641,7 @@ impl ValidatedState { instance: &NodeState, parent_leaf: &Leaf, proposed_header: &Header, + version: Version, ) -> anyhow::Result<(Self, Delta)> { // Clone state to avoid mutation. Consumer can take update // through returned value. @@ -617,21 +650,16 @@ impl ValidatedState { let mut validated_state = self.clone(); let chain_config = validated_state - .get_chain_config(instance, &proposed_header.chain_config) + .apply_chain_config(instance, &proposed_header.chain_config, version) .await?; - validated_state.chain_config = chain_config.into(); - // Find missing fee state entries. We will need to use the builder account which is paying a // fee and the recipient account which is receiving it, plus any counts receiving deposits // in this block. let missing_accounts = self.forgotten_accounts( - [ - proposed_header.fee_info.account, - instance.chain_config.fee_recipient, - ] - .into_iter() - .chain(l1_deposits.iter().map(|fee_info| fee_info.account)), + [proposed_header.fee_info.account, chain_config.fee_recipient] + .into_iter() + .chain(l1_deposits.iter().map(|fee_info| fee_info.account)), ); let parent_height = parent_leaf.height(); @@ -693,7 +721,7 @@ impl ValidatedState { &mut validated_state, &mut delta, proposed_header.fee_info, - instance.chain_config.fee_recipient, + chain_config.fee_recipient, )?; Ok((validated_state, delta)) @@ -775,7 +803,7 @@ impl HotShotState for ValidatedState { parent_leaf: &Leaf, proposed_header: &Header, vid_common: VidCommon, - _version: Version, + version: Version, ) -> Result<(Self, Self::Delta), Self::Error> { //validate builder fee if let Err(err) = validate_builder_fee(proposed_header) { @@ -786,7 +814,7 @@ impl HotShotState for ValidatedState { // Unwrapping here is okay as we retry in a loop //so we should either get a validated state or until hotshot cancels the task let (validated_state, delta) = self - .apply_header(instance, parent_leaf, proposed_header) + .apply_header(instance, parent_leaf, proposed_header, version) .await .unwrap(); From 727a7cd35cd8c0a084a78a6e50d9e89469620bc0 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Mon, 3 Jun 2024 18:05:18 +0500 Subject: [PATCH 14/48] use fee_contract address from updated chain_config --- builder/src/lib.rs | 2 +- sequencer/src/genesis.rs | 2 +- sequencer/src/header.rs | 2 +- sequencer/src/state.rs | 14 +++++++++----- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/builder/src/lib.rs b/builder/src/lib.rs index 89fc422847..f31c1c13df 100644 --- a/builder/src/lib.rs +++ b/builder/src/lib.rs @@ -621,7 +621,7 @@ pub mod testing { let bootstrapped_view = ViewNumber::new(0); let builder_context = BuilderContext::init( - hotshot_handle.into(), + hotshot_handle, state_signer, node_id, key_pair, diff --git a/sequencer/src/genesis.rs b/sequencer/src/genesis.rs index c0a8c2602c..6c4ed5ddf5 100644 --- a/sequencer/src/genesis.rs +++ b/sequencer/src/genesis.rs @@ -177,7 +177,7 @@ mod upgrade_serialization { while let Some(fields) = seq.next_element::()? { // add try_from in Version - let version: Vec<_> = fields.version.split(".").collect(); + let version: Vec<_> = fields.version.split('.').collect(); let version = Version { major: version[0].parse().expect("invalid version"), diff --git a/sequencer/src/header.rs b/sequencer/src/header.rs index 1b6029f2d6..df20c20445 100644 --- a/sequencer/src/header.rs +++ b/sequencer/src/header.rs @@ -296,7 +296,7 @@ impl BlockHeader for Header { let chain_config = validated_state .chain_config .resolve() - .unwrap_or_else(|| instance_state.chain_config); + .unwrap_or(instance_state.chain_config); // Fetch the latest L1 snapshot. let l1_snapshot = instance_state.l1_client.snapshot().await; diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index ed00459ddd..1b3adaa5d4 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -646,12 +646,17 @@ impl ValidatedState { // Clone state to avoid mutation. Consumer can take update // through returned value. - let l1_deposits = get_l1_deposits(instance, proposed_header, parent_leaf).await; - let mut validated_state = self.clone(); let chain_config = validated_state .apply_chain_config(instance, &proposed_header.chain_config, version) .await?; + let l1_deposits = get_l1_deposits( + instance, + proposed_header, + parent_leaf, + chain_config.fee_contract, + ) + .await; // Find missing fee state entries. We will need to use the builder account which is paying a // fee and the recipient account which is receiving it, plus any counts receiving deposits @@ -732,10 +737,9 @@ pub async fn get_l1_deposits( instance: &NodeState, header: &Header, parent_leaf: &Leaf, + fee_contract_address: Option
, ) -> Vec { - if let (Some(addr), Some(block_info)) = - (instance.chain_config.fee_contract, header.l1_finalized) - { + if let (Some(addr), Some(block_info)) = (fee_contract_address, header.l1_finalized) { instance .l1_client .get_finalized_deposits( From cc80ce31365041b10d7b62cedb899dc6724a6f4f Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Mon, 3 Jun 2024 18:14:44 +0500 Subject: [PATCH 15/48] rename upgrade block field to view --- sequencer/src/genesis.rs | 10 +++++----- sequencer/src/lib.rs | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sequencer/src/genesis.rs b/sequencer/src/genesis.rs index 6c4ed5ddf5..11b08a346d 100644 --- a/sequencer/src/genesis.rs +++ b/sequencer/src/genesis.rs @@ -118,7 +118,7 @@ pub enum UpgradeType { #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Upgrade { - pub block: u64, + pub view: u64, #[serde(flatten)] pub upgrade_type: UpgradeType, } @@ -141,7 +141,7 @@ mod upgrade_serialization { for (version, upgrade) in map { seq.serialize_element(&( version.to_string(), - upgrade.block, + upgrade.view, upgrade.upgrade_type.clone(), ))?; } @@ -170,7 +170,7 @@ mod upgrade_serialization { #[derive(Deserialize)] struct UpgradeFields { version: String, - block: u64, + view: u64, #[serde(flatten)] upgrade_type: UpgradeType, } @@ -187,7 +187,7 @@ mod upgrade_serialization { map.insert( version, Upgrade { - block: fields.block, + view: fields.view, upgrade_type: fields.upgrade_type, }, ); @@ -251,7 +251,7 @@ mod test { [[upgrade]] version = "1.0" - block = 1 + view = 1 [upgrade.chain_config] chain_id = 12345 diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 1827d19e8d..74b0934042 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -384,12 +384,12 @@ pub async fn init_node( let version = Ver::version(); if let Some(upgrade) = genesis.upgrades.get(&version) { - let block = upgrade.block; + let view = upgrade.view; // >>>> ????? - config.config.start_proposing_view = block; - config.config.stop_proposing_view = block + 20; - config.config.start_voting_view = block + 21; - config.config.stop_proposing_view = block + 40; + config.config.start_proposing_view = view; + config.config.stop_proposing_view = view + 20; + config.config.start_voting_view = view + 21; + config.config.stop_proposing_view = view + 40; } let node_index = config.node_index; From c88dcead024fef61bf041c72d5490e7ca183b002 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Tue, 4 Jun 2024 07:32:54 +0500 Subject: [PATCH 16/48] Chain config persistence --- data/genesis/demo.toml | 11 +++ .../migrations/V17__chain_config_table.sql | 4 + sequencer/src/api.rs | 56 +++++++++++++- sequencer/src/api/sql.rs | 74 ++++++++++++++++++- sequencer/src/catchup.rs | 4 +- sequencer/src/header.rs | 44 +++++++++-- sequencer/src/lib.rs | 6 +- sequencer/src/persistence.rs | 15 +++- sequencer/src/persistence/sql.rs | 2 +- sequencer/src/state.rs | 30 ++++++-- 10 files changed, 220 insertions(+), 26 deletions(-) create mode 100644 sequencer/api/migrations/V17__chain_config_table.sql diff --git a/data/genesis/demo.toml b/data/genesis/demo.toml index 1f70a67860..9e9518f283 100644 --- a/data/genesis/demo.toml +++ b/data/genesis/demo.toml @@ -10,3 +10,14 @@ fee_contract = '0xa15bb66138824a1c7167f5e85b957d04dd34e468' [header] timestamp = "1970-01-01T00:00:00Z" + +[[upgrade]] +version = "1.0" +view = 1000 + +[upgrade.chain_config] +chain_id = 999999999 +base_fee = '1 wei' +max_block_size = '10mb' +fee_recipient = '0x0000000000000000000000000000000000000000' +fee_contract = '0xa15bb66138824a1c7167f5e85b957d04dd34e468' \ No newline at end of file diff --git a/sequencer/api/migrations/V17__chain_config_table.sql b/sequencer/api/migrations/V17__chain_config_table.sql new file mode 100644 index 0000000000..39cc44ac48 --- /dev/null +++ b/sequencer/api/migrations/V17__chain_config_table.sql @@ -0,0 +1,4 @@ +CREATE TABLE chain_config ( + commitment VARCHAR PRIMARY KEY, + data JSONB NOT NULL +); diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index cce823554a..524393fdeb 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -1,15 +1,16 @@ use self::data_source::{HotShotConfigDataSource, PublicHotShotConfig, StateSignatureDataSource}; use crate::{ network, - persistence::SequencerPersistence, + persistence::{ChainConfigPersistence, SequencerPersistence}, state::{BlockMerkleTree, FeeAccountProof}, state_signature::StateSigner, - Node, NodeState, PubKey, SeqTypes, SequencerContext, Transaction, + ChainConfig, Node, NodeState, PubKey, SeqTypes, SequencerContext, Transaction, }; -use anyhow::Context; +use anyhow::{bail, Context}; use async_once_cell::Lazy; use async_std::sync::{Arc, RwLock}; use async_trait::async_trait; +use committable::Commitment; use data_source::{CatchupDataSource, SubmitDataSource}; use derivative::Derivative; use ethers::prelude::{Address, U256}; @@ -217,6 +218,41 @@ impl< // Try storage. self.inner().get_frontier(height, view).await } + + async fn get_chain_config( + &self, + commitment: Commitment, + ) -> anyhow::Result { + // Check if we have the desired state in memory. + match self.as_ref().get_chain_config(commitment).await { + Ok(cf) => return Ok(cf), + Err(err) => { + tracing::info!("chain config is not in memory, trying storage: {err:#}"); + } + } + + // Try storage. + self.inner().get_chain_config(commitment).await + } +} + +#[async_trait] +impl< + N: network::Type, + Ver: StaticVersionType + 'static, + P: SequencerPersistence, + D: ChainConfigPersistence + Send + Sync, + > ChainConfigPersistence for StorageState +{ + async fn insert_chain_config(&mut self, chain_config: ChainConfig) -> anyhow::Result<()> { + self.inner_mut().insert_chain_config(chain_config).await + } + async fn load_chain_config( + &self, + commitment: Commitment, + ) -> anyhow::Result { + self.inner().load_chain_config(commitment).await + } } impl CatchupDataSource @@ -261,6 +297,20 @@ impl, + ) -> anyhow::Result { + let state = self.consensus().await.read().await.decided_state().await; + let chain_config = state.chain_config; + + if chain_config.commit() == commitment { + chain_config.resolve().context("chain config found") + } else { + bail!("chain config not found") + } + } } impl diff --git a/sequencer/src/api/sql.rs b/sequencer/src/api/sql.rs index 95d248fe32..34bbc614bc 100644 --- a/sequencer/src/api/sql.rs +++ b/sequencer/src/api/sql.rs @@ -3,19 +3,25 @@ use super::{ AccountQueryData, BlocksFrontier, }; use crate::{ - persistence::sql::Options, + persistence::{ + sql::{sql_param, Options}, + ChainConfigPersistence, + }, state::{BlockMerkleTree, FeeAccountProof, FeeMerkleTree}, - SeqTypes, + ChainConfig, SeqTypes, }; use anyhow::{bail, Context}; use async_trait::async_trait; +use committable::Commitment; use ethers::prelude::Address; use hotshot_query_service::{ data_source::{ - sql::{Config, SqlDataSource}, + sql::{Config, Query, SqlDataSource}, storage::SqlStorage, + VersionedDataSource, }, merklized_state::{MerklizedStateDataSource, Snapshot}, + Resolvable, }; use hotshot_types::data::ViewNumber; use jf_merkle_tree::{prelude::MerkleNode, MerkleTreeScheme}; @@ -85,6 +91,13 @@ impl CatchupDataSource for SqlStorage { .await .context(format!("fetching frontier at height {height}")) } + + async fn get_chain_config( + &self, + commitment: committable::Commitment, + ) -> anyhow::Result { + self.load_chain_config(commitment).await + } } impl CatchupDataSource for DataSource { @@ -104,6 +117,61 @@ impl CatchupDataSource for DataSource { } } +#[async_trait] +impl ChainConfigPersistence for SqlStorage { + async fn insert_chain_config(&mut self, chain_config: ChainConfig) -> anyhow::Result<()> { + let commitment = chain_config.commitment(); + let data = bincode::serialize(&chain_config)?; + + let mut transaction = self.transaction().await?; + + transaction + .upsert( + "chain_config", + ["commitment", "data"], + ["commitment"], + [[sql_param(&(commitment.to_string())), sql_param(&data)]], + ) + .await?; + + self.commit().await?; + + Ok(()) + } + + async fn load_chain_config( + &self, + commitment: committable::Commitment, + ) -> anyhow::Result { + let query = self + .query_one( + "SELECT * from chain_config where commitment = $1", + [&commitment.to_string()], + ) + .await?; + + let data: Vec = query.try_get("data")?; + + bincode::deserialize(&data[..]).context("failed to deserialize") + } +} + +#[async_trait] +impl ChainConfigPersistence for DataSource { + async fn insert_chain_config(&mut self, chain_config: ChainConfig) -> anyhow::Result<()> { + (*self.storage_mut().await) + .insert_chain_config(chain_config) + .await + } + + async fn load_chain_config( + &self, + commitment: Commitment, + ) -> anyhow::Result { + self.storage().await.load_chain_config(commitment).await + } +} + #[cfg(test)] mod impl_testable_data_source { use super::*; diff --git a/sequencer/src/catchup.rs b/sequencer/src/catchup.rs index 50f3ce1e2f..b9e1ade5e0 100644 --- a/sequencer/src/catchup.rs +++ b/sequencer/src/catchup.rs @@ -345,9 +345,9 @@ where async fn try_fetch_chain_config( &self, - _commitment: Commitment, + commitment: Commitment, ) -> anyhow::Result { - bail!("chain config catchup is not supported for SQL data source"); + self.db.read().await.get_chain_config(commitment).await } } diff --git a/sequencer/src/header.rs b/sequencer/src/header.rs index df20c20445..b05ece3e16 100644 --- a/sequencer/src/header.rs +++ b/sequencer/src/header.rs @@ -2,6 +2,7 @@ use crate::{ block::{entry::TxTableEntryWord, tables::NameSpaceTable, NsTable}, chain_config::ResolvableChainConfig, eth_signature_key::BuilderSignature, + genesis::UpgradeType, l1_client::L1Snapshot, state::{BlockMerkleCommitment, FeeAccount, FeeAmount, FeeInfo, FeeMerkleCommitment}, ChainConfig, L1BlockInfo, Leaf, NamespaceId, NodeState, SeqTypes, ValidatedState, @@ -9,7 +10,7 @@ use crate::{ use anyhow::{ensure, Context}; use ark_serialize::CanonicalSerialize; use committable::{Commitment, Committable, RawCommitmentBuilder}; -use hotshot_query_service::{availability::QueryableHeader, explorer::ExplorerHeader}; +use hotshot_query_service::{availability::QueryableHeader, explorer::ExplorerHeader, Resolvable}; use hotshot_types::{ traits::{ block_contents::{BlockHeader, BlockPayload, BuilderFee}, @@ -246,6 +247,26 @@ impl Header { builder_signature, }) } + + async fn get_chain_config( + validated_state: &ValidatedState, + instance_state: &NodeState, + ) -> anyhow::Result { + if validated_state.chain_config.commit() == instance_state.chain_config.commitment() { + Ok(instance_state.chain_config) + } else { + match validated_state.chain_config.resolve() { + Some(cf) => Ok(cf), + None => { + instance_state + .peers + .as_ref() + .fetch_chain_config(validated_state.chain_config.commit()) + .await + } + } + } + } } #[derive(Debug, Snafu)] @@ -286,17 +307,28 @@ impl BlockHeader for Header { metadata: <::BlockPayload as BlockPayload>::Metadata, builder_fee: BuilderFee, _vid_common: VidCommon, - _version: Version, + version: Version, ) -> Result { let height = parent_leaf.height(); let view = parent_leaf.view_number(); let mut validated_state = parent_state.clone(); - let chain_config = validated_state - .chain_config - .resolve() - .unwrap_or(instance_state.chain_config); + let chain_config = if version > instance_state.sequencer_version { + match instance_state + .upgrades + .get(&version) + .map(|upgrade| match upgrade.upgrade_type { + UpgradeType::ChainConfig { chain_config } => chain_config, + }) { + Some(cf) => cf, + None => Header::get_chain_config(&validated_state, instance_state).await?, + } + } else { + Header::get_chain_config(&validated_state, instance_state).await? + }; + + validated_state.chain_config = chain_config.into(); // Fetch the latest L1 snapshot. let l1_snapshot = instance_state.l1_client.snapshot().await; diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 74b0934042..a5c5ac8bfb 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -387,9 +387,9 @@ pub async fn init_node( let view = upgrade.view; // >>>> ????? config.config.start_proposing_view = view; - config.config.stop_proposing_view = view + 20; - config.config.start_voting_view = view + 21; - config.config.stop_proposing_view = view + 40; + config.config.stop_proposing_view = u64::MAX; + config.config.start_voting_view = 1; + config.config.stop_proposing_view = view + 1; } let node_index = config.node_index; diff --git a/sequencer/src/persistence.rs b/sequencer/src/persistence.rs index a116bf9777..3fe410635b 100644 --- a/sequencer/src/persistence.rs +++ b/sequencer/src/persistence.rs @@ -8,11 +8,13 @@ //! an extension that node operators can opt into. This module defines the minimum level of //! persistence which is _required_ to run a node. -use crate::{Leaf, NodeState, PubKey, SeqTypes, StateCatchup, ValidatedState, ViewNumber}; +use crate::{ + ChainConfig, Leaf, NodeState, PubKey, SeqTypes, StateCatchup, ValidatedState, ViewNumber, +}; use anyhow::{bail, ensure, Context}; use async_std::sync::Arc; use async_trait::async_trait; -use committable::Committable; +use committable::{Commitment, Committable}; use hotshot::{ traits::ValidatedState as _, types::{Event, EventType}, @@ -236,6 +238,15 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { ) -> anyhow::Result<()>; } +#[async_trait] +pub trait ChainConfigPersistence: Sized + Send + Sync + 'static { + async fn insert_chain_config(&mut self, chain_config: ChainConfig) -> anyhow::Result<()>; + async fn load_chain_config( + &self, + commitment: Commitment, + ) -> anyhow::Result; +} + #[cfg(test)] mod testing { use super::*; diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index 5eaf8e12a9..2f16d39ce9 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -542,7 +542,7 @@ impl SequencerPersistence for Persistence { } } -fn sql_param(param: &T) -> &(dyn ToSql + Sync) { +pub fn sql_param(param: &T) -> &(dyn ToSql + Sync) { param } diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 1b3adaa5d4..07eb9fd75c 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -1,7 +1,7 @@ use crate::{ api::data_source::CatchupDataSource, catchup::SqlStateCatchup, chain_config::ResolvableChainConfig, eth_signature_key::EthKeyPair, genesis::UpgradeType, - ChainConfig, Header, Leaf, NodeState, SeqTypes, + persistence::ChainConfigPersistence, ChainConfig, Header, Leaf, NodeState, SeqTypes, }; use anyhow::{anyhow, bail, ensure, Context}; use ark_serialize::{ @@ -26,6 +26,7 @@ use hotshot_query_service::{ explorer::MonetaryValue, merklized_state::{MerklizedState, MerklizedStateHeightPersistence, UpdateStateData}, types::HeightIndexed, + Resolvable, }; use hotshot_types::{ data::{BlockError, ViewNumber}, @@ -231,8 +232,8 @@ pub fn validate_proposal( proposal.chain_config.commit() == expected_chain_config.commit(), anyhow::anyhow!( "Invalid Chain Config: local={:?}, proposal={:?}", - expected_chain_config, - proposal.chain_config + expected_chain_config.commitment(), + proposal.chain_config.commit(), ) ); @@ -440,6 +441,8 @@ async fn update_state_storage( proposed_leaf: &LeafQueryData, version: Version, ) -> anyhow::Result { + let parent_chain_config = parent_state.chain_config; + let (state, delta) = compute_state_update(parent_state, instance, parent_leaf, proposed_leaf, version) .await @@ -452,6 +455,16 @@ async fn update_state_storage( return Err(err); } + if parent_chain_config != state.chain_config { + if let Err(err) = storage + .insert_chain_config(state.chain_config.resolve().unwrap()) + .await + { + storage.revert().await; + return Err(err); + } + } + Ok(state) } @@ -562,6 +575,7 @@ pub(crate) trait SequencerStateDataSource: + UpdateStateData + UpdateStateData + MerklizedStateHeightPersistence + + ChainConfigPersistence { } @@ -574,6 +588,7 @@ impl SequencerStateDataSource for T where + UpdateStateData + UpdateStateData + MerklizedStateHeightPersistence + + ChainConfigPersistence { } @@ -591,8 +606,6 @@ impl ValidatedState { header_cf: &ResolvableChainConfig, version: Version, ) -> anyhow::Result { - // TODO(abdul) : to be uncommented after we have upgrade mechanism - if header_cf.commit() != self.chain_config.commit() { bail!( "Proposed header chain config commit={} expected={}", @@ -822,10 +835,15 @@ impl HotShotState for ValidatedState { .await .unwrap(); + let chain_config = validated_state + .chain_config + .resolve() + .expect("Chain Config not found in validated state"); + // validate the proposal if let Err(err) = validate_proposal( &validated_state, - instance.chain_config, + chain_config, parent_leaf, proposed_header, &vid_common, From 510da55c556a72d5349da01decb504d5c672debc Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 5 Jun 2024 03:15:36 +0500 Subject: [PATCH 17/48] fix: use instance chain config for genesis validated_state --- sequencer/api/migrations/V17__chain_config_table.sql | 2 +- sequencer/src/api/sql.rs | 2 ++ sequencer/src/header.rs | 5 +++++ sequencer/src/lib.rs | 12 +++++++++--- sequencer/src/persistence.rs | 5 +++++ sequencer/src/state.rs | 11 ++++++++++- 6 files changed, 32 insertions(+), 5 deletions(-) diff --git a/sequencer/api/migrations/V17__chain_config_table.sql b/sequencer/api/migrations/V17__chain_config_table.sql index 39cc44ac48..f41825ac0a 100644 --- a/sequencer/api/migrations/V17__chain_config_table.sql +++ b/sequencer/api/migrations/V17__chain_config_table.sql @@ -1,4 +1,4 @@ CREATE TABLE chain_config ( commitment VARCHAR PRIMARY KEY, - data JSONB NOT NULL + data BYTEA NOT NULL ); diff --git a/sequencer/src/api/sql.rs b/sequencer/src/api/sql.rs index 34bbc614bc..8f4d27b670 100644 --- a/sequencer/src/api/sql.rs +++ b/sequencer/src/api/sql.rs @@ -174,8 +174,10 @@ impl ChainConfigPersistence for DataSource { #[cfg(test)] mod impl_testable_data_source { + use super::*; use crate::api::{self, data_source::testing::TestableSequencerDataSource}; + use hotshot_query_service::data_source::storage::sql::testing::TmpDb; fn tmp_options(db: &TmpDb) -> Options { diff --git a/sequencer/src/header.rs b/sequencer/src/header.rs index b05ece3e16..d2414ad59e 100644 --- a/sequencer/src/header.rs +++ b/sequencer/src/header.rs @@ -258,6 +258,11 @@ impl Header { match validated_state.chain_config.resolve() { Some(cf) => Ok(cf), None => { + tracing::info!( + "fetching chain config {} from peers", + validated_state.chain_config.commit() + ); + instance_state .peers .as_ref() diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index a5c5ac8bfb..ffe9091c9b 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -13,7 +13,7 @@ pub mod state_signature; mod message_compat_tests; mod reference_tests; -use anyhow::Context; +use anyhow::{Chain, Context}; use async_std::sync::RwLock; use async_trait::async_trait; use block::entry::TxTableEntryWord; @@ -189,7 +189,10 @@ impl NodeState { l1_client, peers: Arc::new(catchup), genesis_header: Default::default(), - genesis_state: Default::default(), + genesis_state: ValidatedState { + chain_config: chain_config.into(), + ..Default::default() + }, l1_genesis: None, upgrades: Default::default(), sequencer_version, @@ -465,7 +468,10 @@ pub async fn init_node( // crash horribly just because we're not using the P2P network yet. let _ = NetworkingMetricsValue::new(metrics); - let mut genesis_state = ValidatedState::default(); + let mut genesis_state = ValidatedState { + chain_config: genesis.chain_config.into(), + ..Default::default() + }; for (address, amount) in genesis.accounts { tracing::info!(%address, %amount, "Prefunding account for demo"); genesis_state.prefund_account(address, amount); diff --git a/sequencer/src/persistence.rs b/sequencer/src/persistence.rs index 3fe410635b..ee0e459316 100644 --- a/sequencer/src/persistence.rs +++ b/sequencer/src/persistence.rs @@ -249,6 +249,11 @@ pub trait ChainConfigPersistence: Sized + Send + Sync + 'static { #[cfg(test)] mod testing { + use hotshot_query_service::data_source::{ + sql::{testing::TmpDb, Config}, + storage::SqlStorage, + }; + use super::*; #[async_trait] diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 07eb9fd75c..da13d865d9 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -470,6 +470,7 @@ async fn update_state_storage( async fn store_genesis_state( storage: &mut impl SequencerStateDataSource, + chain_config: ChainConfig, state: &ValidatedState, ) -> anyhow::Result<()> { ensure!( @@ -498,6 +499,8 @@ async fn store_genesis_state( } storage.commit().await?; + + storage.insert_chain_config(chain_config).await?; Ok(()) } @@ -530,7 +533,13 @@ pub(crate) async fn update_state_storage_loop( // never the result of a state update and thus is not inserted in the loop below. tracing::info!("storing genesis merklized state"); let mut storage = storage.write().await; - if let Err(err) = store_genesis_state(&mut *storage, &instance.genesis_state).await { + if let Err(err) = store_genesis_state( + &mut *storage, + instance.chain_config, + &instance.genesis_state, + ) + .await + { tracing::error!("failed to store genesis state: {err:#}"); storage.revert().await; return Err(err); From 5cd07e95305f647129222ae8d23e2638d57ce9ad Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 5 Jun 2024 03:59:15 +0500 Subject: [PATCH 18/48] test for chain config catchup --- sequencer/src/api.rs | 126 ++++++++++++++++++++++++++++++++++- sequencer/src/lib.rs | 2 +- sequencer/src/persistence.rs | 4 -- 3 files changed, 126 insertions(+), 6 deletions(-) diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 524393fdeb..dc2f7b7b9f 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -935,7 +935,7 @@ mod test { }; use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use async_std::task::sleep; - use committable::Commitment; + use committable::{Commitment, Committable}; use es_version::{SequencerVersion, SEQUENCER_VERSION}; use ethers::utils::Anvil; use futures::future::{self, join_all}; @@ -1163,6 +1163,130 @@ mod test { } } + #[async_std::test] + async fn test_chain_config_from_instance() { + // This test uses a ValidatedState which only has the default chain config commitment. + // The NodeState has the full chain config. + // Both chain config commitments will match, so the ValidatedState should have the full chain config after a non-genesis block is decided. + setup_logging(); + setup_backtrace(); + + let port = pick_unused_port().expect("No ports free"); + let anvil = Anvil::new().spawn(); + let l1 = anvil.endpoint().parse().unwrap(); + + let chain_config: ChainConfig = ChainConfig::default(); + + let state = ValidatedState { + chain_config: chain_config.commit().into(), + ..Default::default() + }; + + let states = std::array::from_fn(|_| state.clone()); + + let mut network = TestNetwork::with_state( + Options::from(options::Http { port }).catchup(Default::default()), + states, + [no_storage::Options; TestConfig::NUM_NODES], + std::array::from_fn(|_| { + StatePeers::::from_urls(vec![format!("http://localhost:{port}") + .parse() + .unwrap()]) + }), + l1, + ) + .await; + + // Wait for few blocks to be decided. + network + .server + .event_stream() + .await + .filter(|event| future::ready(matches!(event.event, EventType::Decide { .. }))) + .take(3) + .collect::>() + .await; + + for peer in &network.peers { + let state = peer.consensus().read().await.decided_state().await; + + assert_eq!(state.chain_config.resolve().unwrap(), chain_config) + } + + network.server.shut_down().await; + drop(network); + } + + #[async_std::test] + async fn test_chain_config_catchup() { + // This test uses a ValidatedState with a non-default chain config + // so it will be different from the NodeState chain config used by the TestNetwork. + // However, for this test to work, at least one node should have a full chain config + // to allow other nodes to catch up. + + setup_logging(); + setup_backtrace(); + + let port = pick_unused_port().expect("No ports free"); + let anvil = Anvil::new().spawn(); + let l1 = anvil.endpoint().parse().unwrap(); + + let cf = ChainConfig { + max_block_size: 300.into(), + base_fee: 1.into(), + ..Default::default() + }; + + // State1 contains only the chain config commitment + let state1 = ValidatedState { + chain_config: cf.commit().into(), + ..Default::default() + }; + + //state 2 contains the full chain config + let state2 = ValidatedState { + chain_config: cf.into(), + ..Default::default() + }; + + let mut states = std::array::from_fn(|_| state1.clone()); + // only one node has the full chain config + // all the other nodes should do a catchup to get the full chain config from peer 0 + states[0] = state2; + + let mut network = TestNetwork::with_state( + Options::from(options::Http { port }).catchup(Default::default()), + states, + [no_storage::Options; TestConfig::NUM_NODES], + std::array::from_fn(|_| { + StatePeers::::from_urls(vec![format!("http://localhost:{port}") + .parse() + .unwrap()]) + }), + l1, + ) + .await; + + // Wait for a few blocks to be decided. + network + .server + .event_stream() + .await + .filter(|event| future::ready(matches!(event.event, EventType::Decide { .. }))) + .take(3) + .collect::>() + .await; + + for peer in &network.peers { + let state = peer.consensus().read().await.decided_state().await; + + assert_eq!(state.chain_config.resolve().unwrap(), cf) + } + + network.server.shut_down().await; + drop(network); + } + #[async_std::test] pub(crate) async fn test_restart() { setup_logging(); diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index ffe9091c9b..1ddbd49d5d 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -13,7 +13,7 @@ pub mod state_signature; mod message_compat_tests; mod reference_tests; -use anyhow::{Chain, Context}; +use anyhow::Context; use async_std::sync::RwLock; use async_trait::async_trait; use block::entry::TxTableEntryWord; diff --git a/sequencer/src/persistence.rs b/sequencer/src/persistence.rs index ee0e459316..56df56aa5a 100644 --- a/sequencer/src/persistence.rs +++ b/sequencer/src/persistence.rs @@ -249,10 +249,6 @@ pub trait ChainConfigPersistence: Sized + Send + Sync + 'static { #[cfg(test)] mod testing { - use hotshot_query_service::data_source::{ - sql::{testing::TmpDb, Config}, - storage::SqlStorage, - }; use super::*; From 3b15b051d6037a3e0cd939f13e31e30863ffbf9c Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 5 Jun 2024 04:54:27 +0500 Subject: [PATCH 19/48] remove version from NodeState --- builder/src/lib.rs | 3 -- builder/src/non_permissioned.rs | 1 - builder/src/permissioned.rs | 1 - sequencer/src/api.rs | 65 ++++++++++++++++++++++++++++++++ sequencer/src/header.rs | 20 ++++------ sequencer/src/lib.rs | 11 +----- sequencer/src/reference_tests.rs | 5 ++- sequencer/src/state.rs | 16 +++----- 8 files changed, 83 insertions(+), 39 deletions(-) diff --git a/builder/src/lib.rs b/builder/src/lib.rs index f31c1c13df..330c6b9a03 100644 --- a/builder/src/lib.rs +++ b/builder/src/lib.rs @@ -398,7 +398,6 @@ pub mod testing { ChainConfig::default(), L1Client::new(self.anvil.endpoint().parse().unwrap(), 1), MockStateCatchup::default(), - Ver::version(), ) .with_genesis(ValidatedState::default()); @@ -540,7 +539,6 @@ pub mod testing { 1, ), MockStateCatchup::default(), - StaticVersion::<1, 0>::version(), ) .with_genesis(ValidatedState::default()); @@ -606,7 +604,6 @@ pub mod testing { 1, ), MockStateCatchup::default(), - StaticVersion::<1, 0>::version(), ) .with_genesis(ValidatedState::default()); diff --git a/builder/src/non_permissioned.rs b/builder/src/non_permissioned.rs index c80b8f0632..81aacacbe1 100644 --- a/builder/src/non_permissioned.rs +++ b/builder/src/non_permissioned.rs @@ -70,7 +70,6 @@ pub fn build_instance_state( chain_config, l1_client, Arc::new(StatePeers::::from_urls(state_peers)), - Ver::version(), ); Ok(instance_state) } diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index 687e0a923e..a2c0e39adc 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -264,7 +264,6 @@ pub async fn init_node::from_urls(network_params.state_peers)), node_id: node_index, upgrades: Default::default(), - sequencer_version: Ver::version(), }; let stake_table_commit = diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index dc2f7b7b9f..119a7376e0 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -1287,6 +1287,71 @@ mod test { drop(network); } + #[async_std::test] + async fn test_chain_config_upgrade() { + setup_logging(); + setup_backtrace(); + + let port = pick_unused_port().expect("No ports free"); + let anvil = Anvil::new().spawn(); + let l1 = anvil.endpoint().parse().unwrap(); + + let cf = ChainConfig { + max_block_size: 300.into(), + base_fee: 1.into(), + ..Default::default() + }; + + // State1 contains only the chain config commitment + let state1 = ValidatedState { + chain_config: cf.commit().into(), + ..Default::default() + }; + + //state 2 contains the full chain config + let state2 = ValidatedState { + chain_config: cf.into(), + ..Default::default() + }; + + let mut states = std::array::from_fn(|_| state1.clone()); + // only one node has the full chain config + // all the other nodes should do a catchup to get the full chain config from peer 0 + states[0] = state2; + + let mut network = TestNetwork::with_state( + Options::from(options::Http { port }).catchup(Default::default()), + states, + [no_storage::Options; TestConfig::NUM_NODES], + std::array::from_fn(|_| { + StatePeers::::from_urls(vec![format!("http://localhost:{port}") + .parse() + .unwrap()]) + }), + l1, + ) + .await; + + // Wait for a few blocks to be decided. + network + .server + .event_stream() + .await + .filter(|event| future::ready(matches!(event.event, EventType::Decide { .. }))) + .take(3) + .collect::>() + .await; + + for peer in &network.peers { + let state = peer.consensus().read().await.decided_state().await; + + assert_eq!(state.chain_config.resolve().unwrap(), cf) + } + + network.server.shut_down().await; + drop(network); + } + #[async_std::test] pub(crate) async fn test_restart() { setup_logging(); diff --git a/sequencer/src/header.rs b/sequencer/src/header.rs index d2414ad59e..de925b1eaf 100644 --- a/sequencer/src/header.rs +++ b/sequencer/src/header.rs @@ -319,18 +319,14 @@ impl BlockHeader for Header { let mut validated_state = parent_state.clone(); - let chain_config = if version > instance_state.sequencer_version { - match instance_state - .upgrades - .get(&version) - .map(|upgrade| match upgrade.upgrade_type { - UpgradeType::ChainConfig { chain_config } => chain_config, - }) { - Some(cf) => cf, - None => Header::get_chain_config(&validated_state, instance_state).await?, - } - } else { - Header::get_chain_config(&validated_state, instance_state).await? + let chain_config = match instance_state + .upgrades + .get(&version) + .map(|upgrade| match upgrade.upgrade_type { + UpgradeType::ChainConfig { chain_config } => chain_config, + }) { + Some(cf) => cf, + None => Header::get_chain_config(&validated_state, instance_state).await?, }; validated_state.chain_config = chain_config.into(); diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 1ddbd49d5d..99073b6399 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -73,7 +73,7 @@ use persistence::{PersistenceOptions, SequencerPersistence}; use serde::{Deserialize, Serialize}; use snafu::Snafu; use std::{collections::BTreeMap, fmt::Debug, marker::PhantomData, net::SocketAddr, sync::Arc}; -use vbs::version::{StaticVersion, StaticVersionType, Version}; +use vbs::version::{StaticVersionType, Version}; #[cfg(feature = "libp2p")] use std::time::Duration; @@ -172,7 +172,6 @@ pub struct NodeState { pub genesis_state: ValidatedState, pub l1_genesis: Option, pub upgrades: BTreeMap, - pub sequencer_version: Version, } impl NodeState { @@ -181,7 +180,6 @@ impl NodeState { chain_config: ChainConfig, l1_client: L1Client, catchup: impl StateCatchup + 'static, - sequencer_version: Version, ) -> Self { Self { node_id, @@ -195,20 +193,16 @@ impl NodeState { }, l1_genesis: None, upgrades: Default::default(), - sequencer_version, } } #[cfg(any(test, feature = "testing"))] pub fn mock() -> Self { - use vbs::version::StaticVersion; - Self::new( 0, ChainConfig::default(), L1Client::new("http://localhost:3331".parse().unwrap(), 10000), catchup::mock::MockStateCatchup::default(), - StaticVersion::<1, 0>::version(), ) } @@ -238,7 +232,6 @@ impl Default for NodeState { ChainConfig::default(), L1Client::new("http://localhost:3331".parse().unwrap(), 10000), catchup::mock::MockStateCatchup::default(), - StaticVersion::<1, 0>::version(), ) } } @@ -498,7 +491,6 @@ pub async fn init_node( .await, node_id: node_index, upgrades: genesis.upgrades, - sequencer_version: Ver::version(), }; let mut ctx = SequencerContext::init( @@ -747,7 +739,6 @@ pub mod testing { ChainConfig::default(), L1Client::new(self.url.clone(), 1000), catchup::local_and_remote(persistence_opt.clone(), catchup).await, - Ver::version(), ) .with_genesis(state); diff --git a/sequencer/src/reference_tests.rs b/sequencer/src/reference_tests.rs index 67fc37b80a..714d46eaea 100644 --- a/sequencer/src/reference_tests.rs +++ b/sequencer/src/reference_tests.rs @@ -29,8 +29,9 @@ use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use committable::Committable; use es_version::SequencerVersion; use hotshot_types::traits::{ - block_contents::vid_commitment, block_contents::TestableBlock, - signature_key::BuilderSignatureKey, BlockPayload, EncodeBytes, + block_contents::{vid_commitment, TestableBlock}, + signature_key::BuilderSignatureKey, + BlockPayload, EncodeBytes, }; use jf_merkle_tree::MerkleTreeScheme; use pretty_assertions::assert_eq; diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index da13d865d9..49351bde5c 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -623,16 +623,12 @@ impl ValidatedState { ); } - let current_version = instance.sequencer_version; - - if version > current_version { - let upgrades = &instance.upgrades; - if let Some(upgrade) = upgrades.get(&version) { - match upgrade.upgrade_type { - UpgradeType::ChainConfig { chain_config } => { - self.chain_config = chain_config.into(); - return Ok(chain_config); - } + let upgrades = &instance.upgrades; + if let Some(upgrade) = upgrades.get(&version) { + match upgrade.upgrade_type { + UpgradeType::ChainConfig { chain_config } => { + self.chain_config = chain_config.into(); + return Ok(chain_config); } } } From 32eeb8e79e150ba0a7a8fa335ba3c467e6bebf85 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 5 Jun 2024 06:28:25 +0500 Subject: [PATCH 20/48] move proposal.chain_config and state chainconfig check below upgrade --- Cargo.lock | 17 ++----- data/genesis/demo.toml | 8 ++-- process-compose.yaml | 2 +- sequencer/src/api.rs | 104 ++++++++++++++++++++++++----------------- sequencer/src/lib.rs | 27 +++++++++-- sequencer/src/state.rs | 28 ++++++----- 6 files changed, 112 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc1314a213..c0efb1fcd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4733,7 +4733,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core", ] [[package]] @@ -10682,9 +10682,9 @@ dependencies = [ [[package]] name = "vbs" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1da07d9ca0c69892851ff70256392d1b0c6408a0007cc87409b1ca3064821e9b" +checksum = "6a9d83afb293d109ae418271619046cd9d87d63ba83e473e25a5e36101edc453" dependencies = [ "anyhow", "bincode", @@ -10969,7 +10969,7 @@ version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" dependencies = [ - "windows-core 0.51.1", + "windows-core", "windows-targets 0.48.5", ] @@ -10982,15 +10982,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ - "windows-targets 0.52.5", -] - [[package]] name = "windows-sys" version = "0.48.0" diff --git a/data/genesis/demo.toml b/data/genesis/demo.toml index 9e9518f283..c8bd51c906 100644 --- a/data/genesis/demo.toml +++ b/data/genesis/demo.toml @@ -12,12 +12,12 @@ fee_contract = '0xa15bb66138824a1c7167f5e85b957d04dd34e468' timestamp = "1970-01-01T00:00:00Z" [[upgrade]] -version = "1.0" -view = 1000 +version = "0.2" +view = 5 [upgrade.chain_config] chain_id = 999999999 -base_fee = '1 wei' -max_block_size = '10mb' +base_fee = '5 wei' +max_block_size = '1mb' fee_recipient = '0x0000000000000000000000000000000000000000' fee_contract = '0xa15bb66138824a1c7167f5e85b957d04dd34e468' \ No newline at end of file diff --git a/process-compose.yaml b/process-compose.yaml index 62e7ec8357..35dd1e6322 100644 --- a/process-compose.yaml +++ b/process-compose.yaml @@ -46,7 +46,7 @@ processes: condition: process_completed fund-builder: - command: bridge deposit + command: espresso-bridge deposit environment: - L1_PROVIDER=http://localhost:$ESPRESSO_SEQUENCER_L1_PORT - ESPRESSO_PROVIDER=http://localhost:$ESPRESSO_SEQUENCER1_API_PORT diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 119a7376e0..49108c2f2c 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -352,6 +352,7 @@ pub mod test_helpers { use super::*; use crate::{ catchup::{mock::MockStateCatchup, StateCatchup}, + genesis::Upgrade, persistence::{no_storage, PersistenceOptions, SequencerPersistence}, state::{BlockMerkleTree, ValidatedState}, testing::{run_test_builder, wait_for_decide_on_handle, TestConfig}, @@ -375,10 +376,11 @@ pub mod test_helpers { use itertools::izip; use jf_merkle_tree::{MerkleCommitment, MerkleTreeScheme}; use portpicker::pick_unused_port; - use std::time::Duration; + use std::{collections::BTreeMap, time::Duration}; use surf_disco::Client; use tide_disco::error::ServerError; use url::Url; + use vbs::version::Version; pub const STAKE_TABLE_CAPACITY_FOR_TEST: u64 = 10; @@ -388,6 +390,15 @@ pub mod test_helpers { pub cfg: TestConfig, } + #[derive(Clone, Debug)] + pub struct TestNetworkUpgrades { + pub upgrades: BTreeMap, + pub start_proposing_view: u64, + pub stop_proposing_view: u64, + pub start_voting_view: u64, + pub stop_voting_view: u64, + } + impl TestNetwork

{ pub async fn with_state( opt: Options, @@ -395,6 +406,7 @@ pub mod test_helpers { persistence: [impl PersistenceOptions; TestConfig::NUM_NODES], catchup: [impl StateCatchup + 'static; TestConfig::NUM_NODES], l1: Url, + upgrades: Option, ) -> Self { let mut cfg = TestConfig::default_with_l1(l1); @@ -402,9 +414,22 @@ pub mod test_helpers { cfg.set_builder_url(builder_url); + let mut upgrades_map = BTreeMap::default(); + if let Some(upgrades) = upgrades { + cfg.set_upgrade_parameters( + upgrades.start_proposing_view, + upgrades.stop_proposing_view, + upgrades.start_voting_view, + upgrades.stop_voting_view, + ); + + upgrades_map = upgrades.upgrades; + } + let mut nodes = join_all(izip!(state, persistence, catchup).enumerate().map( |(i, (state, persistence, catchup))| { let opt = opt.clone(); + let upgrades_map = upgrades_map.clone(); let cfg = &cfg; async move { if i == 0 { @@ -420,6 +445,7 @@ pub mod test_helpers { &*metrics, STAKE_TABLE_CAPACITY_FOR_TEST, SEQUENCER_VERSION, + upgrades_map, ) .await } @@ -438,6 +464,7 @@ pub mod test_helpers { &NoMetrics, STAKE_TABLE_CAPACITY_FOR_TEST, SEQUENCER_VERSION, + upgrades_map, ) .await } @@ -474,6 +501,7 @@ pub mod test_helpers { persistence, std::array::from_fn(|_| MockStateCatchup::default()), l1, + Default::default(), ) .await } @@ -928,6 +956,7 @@ mod test { use super::*; use crate::{ catchup::{mock::MockStateCatchup, StatePeers}, + genesis::{Upgrade, UpgradeType}, persistence::no_storage, state::{FeeAccount, FeeAmount, ValidatedState}, testing::TestConfig, @@ -946,16 +975,17 @@ mod test { types::HeightIndexed, }; use hotshot_types::{ + constants::{Version01, Version02}, event::LeafInfo, traits::{metrics::NoMetrics, node_implementation::ConsensusTime}, }; use jf_merkle_tree::prelude::{MerkleProof, Sha3Node}; use portpicker::pick_unused_port; - use std::time::Duration; + use std::{collections::BTreeMap, time::Duration}; use surf_disco::Client; use test_helpers::{ catchup_test_helper, state_signature_test_helper, status_test_helper, submit_test_helper, - TestNetwork, + TestNetwork, TestNetworkUpgrades, }; use tide_disco::{app::AppHealth, error::ServerError, healthcheck::HealthStatus}; @@ -1087,6 +1117,7 @@ mod test { .unwrap()]) }), l1, + Default::default(), ) .await; @@ -1132,6 +1163,7 @@ mod test { &NoMetrics, test_helpers::STAKE_TABLE_CAPACITY_FOR_TEST, SEQUENCER_VERSION, + Default::default(), ) .await; let mut events = node.event_stream().await; @@ -1194,6 +1226,7 @@ mod test { .unwrap()]) }), l1, + None, ) .await; @@ -1264,6 +1297,7 @@ mod test { .unwrap()]) }), l1, + None, ) .await; @@ -1296,32 +1330,32 @@ mod test { let anvil = Anvil::new().spawn(); let l1 = anvil.endpoint().parse().unwrap(); - let cf = ChainConfig { - max_block_size: 300.into(), - base_fee: 1.into(), - ..Default::default() - }; - - // State1 contains only the chain config commitment - let state1 = ValidatedState { - chain_config: cf.commit().into(), - ..Default::default() - }; + let mut map = BTreeMap::new(); + map.insert( + Version02::VERSION, + Upgrade { + view: 5, + upgrade_type: UpgradeType::ChainConfig { + chain_config: ChainConfig { + max_block_size: 300.into(), + base_fee: 1.into(), + ..Default::default() + }, + }, + }, + ); - //state 2 contains the full chain config - let state2 = ValidatedState { - chain_config: cf.into(), - ..Default::default() + let upgrades = TestNetworkUpgrades { + upgrades: map, + start_proposing_view: 5, + stop_proposing_view: u64::MAX, + start_voting_view: 1, + stop_voting_view: u64::MAX, }; - let mut states = std::array::from_fn(|_| state1.clone()); - // only one node has the full chain config - // all the other nodes should do a catchup to get the full chain config from peer 0 - states[0] = state2; - let mut network = TestNetwork::with_state( Options::from(options::Http { port }).catchup(Default::default()), - states, + Default::default(), [no_storage::Options; TestConfig::NUM_NODES], std::array::from_fn(|_| { StatePeers::::from_urls(vec![format!("http://localhost:{port}") @@ -1329,27 +1363,11 @@ mod test { .unwrap()]) }), l1, + Some(upgrades), ) .await; - // Wait for a few blocks to be decided. - network - .server - .event_stream() - .await - .filter(|event| future::ready(matches!(event.event, EventType::Decide { .. }))) - .take(3) - .collect::>() - .await; - - for peer in &network.peers { - let state = peer.consensus().read().await.decided_state().await; - - assert_eq!(state.chain_config.resolve().unwrap(), cf) - } - - network.server.shut_down().await; - drop(network); + sleep(Duration::from_secs(60 * 60)).await; } #[async_std::test] @@ -1377,6 +1395,7 @@ mod test { persistence, std::array::from_fn(|_| MockStateCatchup::default()), l1, + None, ) .await; @@ -1457,6 +1476,7 @@ mod test { .unwrap()]) }), l1, + None, ) .await; let client: Client = diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 99073b6399..b3d34210f4 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -220,6 +220,11 @@ impl NodeState { self.chain_config = cfg; self } + + pub fn with_upgrades(mut self, upgrades: BTreeMap) -> Self { + self.upgrades = upgrades; + self + } } // This allows us to turn on `Default` on InstanceState trait @@ -379,13 +384,13 @@ pub async fn init_node( }; let version = Ver::version(); - if let Some(upgrade) = genesis.upgrades.get(&version) { + if let Some(upgrade) = genesis.upgrades.get(&Version { major: 0, minor: 2 }) { let view = upgrade.view; // >>>> ????? config.config.start_proposing_view = view; config.config.stop_proposing_view = u64::MAX; config.config.start_voting_view = 1; - config.config.stop_proposing_view = view + 1; + config.config.stop_proposing_view = u64::MAX; } let node_index = config.node_index; @@ -656,6 +661,19 @@ pub mod testing { } } + pub fn set_upgrade_parameters( + &mut self, + start_proposing_view: u64, + stop_proposing_view: u64, + start_voting_view: u64, + stop_voting_view: u64, + ) { + self.config.start_proposing_view = start_proposing_view; + self.config.stop_proposing_view = stop_proposing_view; + self.config.start_voting_view = start_voting_view; + self.config.stop_voting_view = stop_voting_view; + } + pub async fn init_nodes( &self, bind_version: Ver, @@ -669,6 +687,7 @@ pub mod testing { &NoMetrics, STAKE_TABLE_CAPACITY_FOR_TEST, bind_version, + Default::default(), ) .await })) @@ -707,6 +726,7 @@ pub mod testing { metrics: &dyn Metrics, stake_table_capacity: u64, bind_version: Ver, + upgrades: BTreeMap, ) -> SequencerContext { let mut config = self.config.clone(); let my_peer_config = &config.known_nodes_with_stake[i]; @@ -740,7 +760,8 @@ pub mod testing { L1Client::new(self.url.clone(), 1000), catchup::local_and_remote(persistence_opt.clone(), catchup).await, ) - .with_genesis(state); + .with_genesis(state) + .with_upgrades(upgrades); tracing::info!( i, diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 49351bde5c..6cfbe62a2e 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -232,8 +232,8 @@ pub fn validate_proposal( proposal.chain_config.commit() == expected_chain_config.commit(), anyhow::anyhow!( "Invalid Chain Config: local={:?}, proposal={:?}", - expected_chain_config.commitment(), - proposal.chain_config.commit(), + expected_chain_config.commitment().to_string(), + proposal.chain_config.commit().to_string(), ) ); @@ -615,24 +615,28 @@ impl ValidatedState { header_cf: &ResolvableChainConfig, version: Version, ) -> anyhow::Result { - if header_cf.commit() != self.chain_config.commit() { - bail!( - "Proposed header chain config commit={} expected={}", - header_cf.commit(), - self.chain_config.commit() - ); - } + let header_cf_commit = header_cf.commit(); let upgrades = &instance.upgrades; if let Some(upgrade) = upgrades.get(&version) { match upgrade.upgrade_type { UpgradeType::ChainConfig { chain_config } => { - self.chain_config = chain_config.into(); - return Ok(chain_config); + if header_cf_commit == chain_config.commit() { + self.chain_config = chain_config.into(); + return Ok(chain_config); + } } } } + if header_cf_commit != self.chain_config.commit() { + bail!( + "Proposed header chain config commit={} expected={}", + header_cf.commit(), + self.chain_config.commit() + ); + } + let cf = if self.chain_config.commit() == instance.chain_config.commit() { instance.chain_config } else { @@ -827,6 +831,8 @@ impl HotShotState for ValidatedState { vid_common: VidCommon, version: Version, ) -> Result<(Self, Self::Delta), Self::Error> { + tracing::info!("{version:?}"); + //validate builder fee if let Err(err) = validate_builder_fee(proposed_header) { tracing::error!("invalid builder fee: {err:#}"); From 8079e64296aab4590d637f857c5665a9976b3eb7 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 5 Jun 2024 07:26:31 +0500 Subject: [PATCH 21/48] move current version back to NodeState --- builder/src/permissioned.rs | 1 + data/genesis/demo.toml | 4 +-- sequencer/src/api.rs | 55 ++----------------------------------- sequencer/src/header.rs | 20 ++++++++------ sequencer/src/lib.rs | 11 +++++--- sequencer/src/state.rs | 15 +++++----- 6 files changed, 32 insertions(+), 74 deletions(-) diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index a2c0e39adc..a1af93589e 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -264,6 +264,7 @@ pub async fn init_node::from_urls(network_params.state_peers)), node_id: node_index, upgrades: Default::default(), + current_version: Ver::VERSION, }; let stake_table_commit = diff --git a/data/genesis/demo.toml b/data/genesis/demo.toml index c8bd51c906..c9edf51a8e 100644 --- a/data/genesis/demo.toml +++ b/data/genesis/demo.toml @@ -12,12 +12,12 @@ fee_contract = '0xa15bb66138824a1c7167f5e85b957d04dd34e468' timestamp = "1970-01-01T00:00:00Z" [[upgrade]] -version = "0.2" +version = "0.1" view = 5 [upgrade.chain_config] chain_id = 999999999 -base_fee = '5 wei' +base_fee = '1 wei' max_block_size = '1mb' fee_recipient = '0x0000000000000000000000000000000000000000' fee_contract = '0xa15bb66138824a1c7167f5e85b957d04dd34e468' \ No newline at end of file diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 49108c2f2c..0e1265942a 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -956,7 +956,6 @@ mod test { use super::*; use crate::{ catchup::{mock::MockStateCatchup, StatePeers}, - genesis::{Upgrade, UpgradeType}, persistence::no_storage, state::{FeeAccount, FeeAmount, ValidatedState}, testing::TestConfig, @@ -975,17 +974,16 @@ mod test { types::HeightIndexed, }; use hotshot_types::{ - constants::{Version01, Version02}, event::LeafInfo, traits::{metrics::NoMetrics, node_implementation::ConsensusTime}, }; use jf_merkle_tree::prelude::{MerkleProof, Sha3Node}; use portpicker::pick_unused_port; - use std::{collections::BTreeMap, time::Duration}; + use std::time::Duration; use surf_disco::Client; use test_helpers::{ catchup_test_helper, state_signature_test_helper, status_test_helper, submit_test_helper, - TestNetwork, TestNetworkUpgrades, + TestNetwork, }; use tide_disco::{app::AppHealth, error::ServerError, healthcheck::HealthStatus}; @@ -1321,55 +1319,6 @@ mod test { drop(network); } - #[async_std::test] - async fn test_chain_config_upgrade() { - setup_logging(); - setup_backtrace(); - - let port = pick_unused_port().expect("No ports free"); - let anvil = Anvil::new().spawn(); - let l1 = anvil.endpoint().parse().unwrap(); - - let mut map = BTreeMap::new(); - map.insert( - Version02::VERSION, - Upgrade { - view: 5, - upgrade_type: UpgradeType::ChainConfig { - chain_config: ChainConfig { - max_block_size: 300.into(), - base_fee: 1.into(), - ..Default::default() - }, - }, - }, - ); - - let upgrades = TestNetworkUpgrades { - upgrades: map, - start_proposing_view: 5, - stop_proposing_view: u64::MAX, - start_voting_view: 1, - stop_voting_view: u64::MAX, - }; - - let mut network = TestNetwork::with_state( - Options::from(options::Http { port }).catchup(Default::default()), - Default::default(), - [no_storage::Options; TestConfig::NUM_NODES], - std::array::from_fn(|_| { - StatePeers::::from_urls(vec![format!("http://localhost:{port}") - .parse() - .unwrap()]) - }), - l1, - Some(upgrades), - ) - .await; - - sleep(Duration::from_secs(60 * 60)).await; - } - #[async_std::test] pub(crate) async fn test_restart() { setup_logging(); diff --git a/sequencer/src/header.rs b/sequencer/src/header.rs index de925b1eaf..7f834d823f 100644 --- a/sequencer/src/header.rs +++ b/sequencer/src/header.rs @@ -319,14 +319,18 @@ impl BlockHeader for Header { let mut validated_state = parent_state.clone(); - let chain_config = match instance_state - .upgrades - .get(&version) - .map(|upgrade| match upgrade.upgrade_type { - UpgradeType::ChainConfig { chain_config } => chain_config, - }) { - Some(cf) => cf, - None => Header::get_chain_config(&validated_state, instance_state).await?, + let chain_config = if version > instance_state.current_version { + match instance_state + .upgrades + .get(&version) + .map(|upgrade| match upgrade.upgrade_type { + UpgradeType::ChainConfig { chain_config } => chain_config, + }) { + Some(cf) => cf, + None => Header::get_chain_config(&validated_state, instance_state).await?, + } + } else { + Header::get_chain_config(&validated_state, instance_state).await? }; validated_state.chain_config = chain_config.into(); diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index b3d34210f4..6e8c9e4d95 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -52,6 +52,7 @@ use hotshot_orchestrator::{ }; use hotshot_types::{ consensus::CommitmentMap, + constants::Version01, data::{DaProposal, VidDisperseShare, ViewNumber}, event::HotShotAction, light_client::{StateKeyPair, StateSignKey}, @@ -172,6 +173,7 @@ pub struct NodeState { pub genesis_state: ValidatedState, pub l1_genesis: Option, pub upgrades: BTreeMap, + pub current_version: Version, } impl NodeState { @@ -193,6 +195,7 @@ impl NodeState { }, l1_genesis: None, upgrades: Default::default(), + current_version: Version01::VERSION, } } @@ -384,13 +387,12 @@ pub async fn init_node( }; let version = Ver::version(); - if let Some(upgrade) = genesis.upgrades.get(&Version { major: 0, minor: 2 }) { + if let Some(upgrade) = genesis.upgrades.get(&version) { let view = upgrade.view; - // >>>> ????? config.config.start_proposing_view = view; - config.config.stop_proposing_view = u64::MAX; + config.config.stop_proposing_view = view + 1; config.config.start_voting_view = 1; - config.config.stop_proposing_view = u64::MAX; + config.config.stop_voting_view = u64::MAX; } let node_index = config.node_index; @@ -496,6 +498,7 @@ pub async fn init_node( .await, node_id: node_index, upgrades: genesis.upgrades, + current_version: Ver::VERSION, }; let mut ctx = SequencerContext::init( diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 6cfbe62a2e..32845d5f03 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -617,13 +617,14 @@ impl ValidatedState { ) -> anyhow::Result { let header_cf_commit = header_cf.commit(); - let upgrades = &instance.upgrades; - if let Some(upgrade) = upgrades.get(&version) { - match upgrade.upgrade_type { - UpgradeType::ChainConfig { chain_config } => { - if header_cf_commit == chain_config.commit() { - self.chain_config = chain_config.into(); - return Ok(chain_config); + if version > instance.current_version { + if let Some(upgrade) = &instance.upgrades.get(&version) { + match upgrade.upgrade_type { + UpgradeType::ChainConfig { chain_config } => { + if header_cf_commit == chain_config.commit() { + self.chain_config = chain_config.into(); + return Ok(chain_config); + } } } } From f2d92d56b3169a21772d6a8111d1d92ffe291332 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 5 Jun 2024 07:32:41 +0500 Subject: [PATCH 22/48] get upgrade for current_version --- sequencer/src/header.rs | 2 +- sequencer/src/state.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sequencer/src/header.rs b/sequencer/src/header.rs index 7f834d823f..3ba1583ba2 100644 --- a/sequencer/src/header.rs +++ b/sequencer/src/header.rs @@ -322,7 +322,7 @@ impl BlockHeader for Header { let chain_config = if version > instance_state.current_version { match instance_state .upgrades - .get(&version) + .get(&instance_state.current_version) .map(|upgrade| match upgrade.upgrade_type { UpgradeType::ChainConfig { chain_config } => chain_config, }) { diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 32845d5f03..f2064c465b 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -618,7 +618,7 @@ impl ValidatedState { let header_cf_commit = header_cf.commit(); if version > instance.current_version { - if let Some(upgrade) = &instance.upgrades.get(&version) { + if let Some(upgrade) = &instance.upgrades.get(&instance.current_version) { match upgrade.upgrade_type { UpgradeType::ChainConfig { chain_config } => { if header_cf_commit == chain_config.commit() { From 81c82921bb7ab202dfb63badc03efda0dec23008 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 5 Jun 2024 09:02:53 +0500 Subject: [PATCH 23/48] update lockfile --- Cargo.lock | 13 ++++++------- Cargo.toml | 1 - builder/src/permissioned.rs | 5 ++--- process-compose.yaml | 2 +- scripts/demo-native | 2 +- sequencer/Cargo.toml | 1 - sequencer/src/lib.rs | 14 ++++++++++++-- 7 files changed, 22 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9396a0452c..1cd6087892 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4070,8 +4070,8 @@ dependencies = [ [[package]] name = "hotshot-builder-core" -version = "0.1.21" -source = "git+https://github.com/EspressoSystems/hotshot-builder-core?branch=abdul/update-hotshot#d1c4b3c783d008cd8a46465aad91c9e772ebe059" +version = "0.1.25" +source = "git+https://github.com/EspressoSystems/hotshot-builder-core?branch=abdul/update-hotshot#7c2c6e4eef52ee25ebd99e7d07d2ea2dc8e4cdbd" dependencies = [ "anyhow", "async-broadcast", @@ -4123,8 +4123,8 @@ dependencies = [ [[package]] name = "hotshot-events-service" -version = "0.1.22" -source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?branch=abdul/update-hotshot#b89e29d44881fec8f1fb21db853dfee26aeaa79d" +version = "0.1.25" +source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?branch=abdul/update-hotshot#241c8b3227bc6a12feb083a5509a08130a9cb22f" dependencies = [ "async-broadcast", "async-compatibility-layer", @@ -4219,8 +4219,8 @@ dependencies = [ [[package]] name = "hotshot-query-service" -version = "0.1.27" -source = "git+https://github.com/EspressoSystems/hotshot-query-service?branch=abdul/update-hotshot#e9f909f679a0613ce2a917d76ee7a6bf2853c402" +version = "0.1.30" +source = "git+https://github.com/EspressoSystems/hotshot-query-service?branch=abdul/update-hotshot#dc3401ebcf0404fc47f55e73f9f1ee88c1ae7bcd" dependencies = [ "anyhow", "ark-serialize", @@ -8633,7 +8633,6 @@ dependencies = [ "time 0.3.36", "tokio-postgres", "toml", - "toml_edit 0.22.14", "tracing", "tracing-subscriber 0.3.18", "trait-set", diff --git a/Cargo.toml b/Cargo.toml index 17c8a57c29..dfdcf0a5d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,7 +111,6 @@ rand_distr = "0.4" reqwest = "0.12" serde = { version = "1.0.195", features = ["derive"] } toml = "0.8" -toml_edit = { version = "0.22", features = ["serde"] } url = "2.3" vbs = "0.1" vergen = { version = "8.3", features = ["git", "gitcl"] } diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index f1ab860bdc..216807fbc3 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -466,8 +466,7 @@ impl Date: Wed, 5 Jun 2024 18:49:34 +0500 Subject: [PATCH 24/48] add test for chain config upgrade --- sequencer/src/api.rs | 95 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 982bbded34..7745213d19 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -932,6 +932,7 @@ mod test { use super::*; use crate::{ catchup::{mock::MockStateCatchup, StatePeers}, + genesis::{Upgrade, UpgradeType}, persistence::no_storage, state::{FeeAccount, FeeAmount, ValidatedState}, testing::TestConfig, @@ -959,9 +960,10 @@ mod test { use surf_disco::Client; use test_helpers::{ catchup_test_helper, state_signature_test_helper, status_test_helper, submit_test_helper, - TestNetwork, + TestNetwork, TestNetworkUpgrades, }; use tide_disco::{app::AppHealth, error::ServerError, healthcheck::HealthStatus}; + use vbs::version::Version; #[async_std::test] async fn test_healthcheck() { @@ -1295,6 +1297,97 @@ mod test { drop(network); } + #[async_std::test] + async fn test_chain_config_upgrade() { + setup_logging(); + setup_backtrace(); + + let port = pick_unused_port().expect("No ports free"); + let anvil = Anvil::new().spawn(); + let l1 = anvil.endpoint().parse().unwrap(); + + let chain_config_upgrade = ChainConfig { + max_block_size: 300.into(), + base_fee: 1.into(), + ..Default::default() + }; + let mut map = std::collections::BTreeMap::new(); + map.insert( + Version { major: 0, minor: 1 }, + Upgrade { + view: 5, + upgrade_type: UpgradeType::ChainConfig { + chain_config: chain_config_upgrade, + }, + }, + ); + + let upgrades = TestNetworkUpgrades { + upgrades: map, + start_proposing_view: 5, + stop_proposing_view: 6, + start_voting_view: 1, + stop_voting_view: 100, + }; + + let mut network = TestNetwork::with_state( + Options::from(options::Http { port }).catchup(Default::default()), + Default::default(), + [no_storage::Options; TestConfig::NUM_NODES], + std::array::from_fn(|_| { + StatePeers::::from_urls(vec![format!("http://localhost:{port}") + .parse() + .unwrap()]) + }), + l1, + Some(upgrades), + ) + .await; + + let mut events = network.server.event_stream().await; + loop { + let event = events.next().await.unwrap(); + + match event.event { + EventType::UpgradeProposal { proposal, .. } => { + let upgrade = proposal.data.upgrade_proposal; + let new_version = upgrade.new_version; + assert_eq!(new_version, Version { major: 0, minor: 2 }); + break; + } + _ => continue, + } + } + + let mut upgrades = [false; TestConfig::NUM_NODES]; + loop { + let event = events.next().await.unwrap(); + let EventType::Decide { leaf_chain, .. } = event.event else { + continue; + }; + for LeafInfo { leaf, state, .. } in leaf_chain.iter().rev() { + let height = leaf.height(); + let leaf_builder = (leaf.view_number().u64() as usize) % TestConfig::NUM_NODES; + if height == 0 { + continue; + } + + if let Some(cf) = state.chain_config.resolve() { + if cf == chain_config_upgrade { + upgrades[leaf_builder] = true; + } + } + } + + if upgrades.iter().all(|upgrade_completed| *upgrade_completed) { + break; + } + } + + network.server.shut_down().await; + drop(network); + } + #[async_std::test] pub(crate) async fn test_restart() { setup_logging(); From 4ab7e3c73fa10a84b50d470b36f99cd8a9ca608c Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 5 Jun 2024 18:57:24 +0500 Subject: [PATCH 25/48] pass None for builder port --- sequencer/src/api.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index b682d182f4..3f9ab991b3 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -1132,7 +1132,7 @@ mod test { .unwrap()]) }), l1, - Default::default(), + None, None, ) .await; @@ -1243,6 +1243,7 @@ mod test { }), l1, None, + None, ) .await; @@ -1314,6 +1315,7 @@ mod test { }), l1, None, + None, ) .await; @@ -1381,6 +1383,7 @@ mod test { }), l1, Some(upgrades), + None, ) .await; @@ -1454,6 +1457,7 @@ mod test { std::array::from_fn(|_| MockStateCatchup::default()), l1, None, + None, ) .await; @@ -1530,6 +1534,7 @@ mod test { }), l1, None, + None, ) .await; let client: Client = From f186c45c4199581118e3962bd1594ee49ee3d2b0 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Wed, 5 Jun 2024 19:12:31 +0500 Subject: [PATCH 26/48] merge main --- data/genesis/demo.toml | 10 ++++++++++ sequencer/src/header.rs | 5 +---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/data/genesis/demo.toml b/data/genesis/demo.toml index da3ef9aad9..6f20dab136 100644 --- a/data/genesis/demo.toml +++ b/data/genesis/demo.toml @@ -11,10 +11,20 @@ fee_contract = '0xa15bb66138824a1c7167f5e85b957d04dd34e468' [header] timestamp = "1970-01-01T00:00:00Z" +[network] +# bootstrap_nodes = [ +# "/ip4/127.0.0.1/udp/7003/quic-v1/p2p/12D3KooWFLKhKr61FhtdcMNR2wV4XHnceJQRFo6KKwQvwvRCRvy6", +# "/ip4/127.0.0.1/udp/7001/quic-v1/p2p/12D3KooWR9caJf3eqwyB3rZQX7DFUYTjM1jvXZJJxYbwhkAnPf4w", +# "/ip4/127.0.0.1/udp/7002/quic-v1/p2p/12D3KooWGbEJ2VsfX8KdUZgVaLYFndAxmtdScC91EritK57FbE2N", +# "/ip4/127.0.0.1/udp/7000/quic-v1/p2p/12D3KooWDtGECieXrqKoVxfDhU7afYnS6toj1GqWXuEDfcaGPDxa", +# "/ip4/127.0.0.1/udp/1769/quic-v1/p2p/12D3KooWCzRqAGWkUib7EqHgaWsJsGvKnujXSLsm4xf8u36nJKhH" +# ] + [[upgrade]] version = "0.1" view = 5 + [upgrade.chain_config] chain_id = 999999999 base_fee = '1 wei' diff --git a/sequencer/src/header.rs b/sequencer/src/header.rs index db17979b75..0e4cbee38c 100644 --- a/sequencer/src/header.rs +++ b/sequencer/src/header.rs @@ -810,10 +810,7 @@ mod test_headers { async fn default() -> Self { let instance_state = NodeState::mock(); let validated_state = ValidatedState::genesis(&instance_state).0; - - //TODO(abdul): Change this - let leaf = async_std::task::block_on(Leaf::genesis(&validated_state, &instance_state)); - + let leaf = Leaf::genesis(&validated_state, &instance_state).await; let header = leaf.block_header().clone(); let ns_table = leaf.block_payload().unwrap().get_ns_table().clone(); Self { From 353a80daa89a565652a22fbab8de32d370140d08 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Thu, 6 Jun 2024 17:27:39 +0500 Subject: [PATCH 27/48] fix test --- sequencer/src/api.rs | 46 +++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 3f9ab991b3..e7d6461a95 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -1364,16 +1364,19 @@ mod test { }, ); + let stop_voting_view = 100; let upgrades = TestNetworkUpgrades { upgrades: map, start_proposing_view: 5, stop_proposing_view: 6, start_voting_view: 1, - stop_voting_view: 100, + stop_voting_view, }; let mut network = TestNetwork::with_state( - Options::from(options::Http { port }).catchup(Default::default()), + Options::from(options::Http { port }) + .catchup(Default::default()) + .status(Default::default()), Default::default(), [no_storage::Options; TestConfig::NUM_NODES], std::array::from_fn(|_| { @@ -1402,29 +1405,32 @@ mod test { } } - let mut upgrades = [false; TestConfig::NUM_NODES]; - loop { - let event = events.next().await.unwrap(); - let EventType::Decide { leaf_chain, .. } = event.event else { - continue; - }; - for LeafInfo { leaf, state, .. } in leaf_chain.iter().rev() { - let height = leaf.height(); - let leaf_builder = (leaf.view_number().u64() as usize) % TestConfig::NUM_NODES; - if height == 0 { - continue; - } + let client: Client = + Client::new(format!("http://localhost:{port}").parse().unwrap()); + client.connect(None).await; + tracing::info!(port, "server running"); + + 'outer: loop { + let height = client + .get::("status/block-height") + .send() + .await + .unwrap(); + + for peer in &network.peers { + let state = peer.consensus().read().await.decided_state().await; - if let Some(cf) = state.chain_config.resolve() { - if cf == chain_config_upgrade { - upgrades[leaf_builder] = true; + match state.chain_config.resolve() { + Some(cf) => { + if cf != chain_config_upgrade && height as u64 > stop_voting_view { + panic!("failed to upgrade chain config"); + } } + None => continue 'outer, } } - if upgrades.iter().all(|upgrade_completed| *upgrade_completed) { - break; - } + break; } network.server.shut_down().await; From af27263fa84fa94ac60038d207db4c58029db8ce Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 7 Jun 2024 03:06:35 +0500 Subject: [PATCH 28/48] update migration version to V31 and cleanup --- builder/src/bin/permissionless-builder.rs | 3 ++- builder/src/permissioned.rs | 5 +++- data/genesis/demo.toml | 4 ++-- ..._table.sql => V31__chain_config_table.sql} | 0 sequencer/src/api.rs | 2 +- sequencer/src/api/sql.rs | 3 --- sequencer/src/block.rs | 13 +++++----- sequencer/src/catchup.rs | 17 ++++--------- sequencer/src/header.rs | 12 +++++----- sequencer/src/persistence/sql.rs | 2 +- sequencer/src/state.rs | 24 ++++++++++--------- 11 files changed, 40 insertions(+), 45 deletions(-) rename sequencer/api/migrations/{V17__chain_config_table.sql => V31__chain_config_table.sql} (100%) diff --git a/builder/src/bin/permissionless-builder.rs b/builder/src/bin/permissionless-builder.rs index 44587f485a..886a2108b1 100644 --- a/builder/src/bin/permissionless-builder.rs +++ b/builder/src/bin/permissionless-builder.rs @@ -3,6 +3,7 @@ use builder::non_permissioned::{build_instance_state, BuilderConfig}; use clap::Parser; use cld::ClDuration; use es_version::SEQUENCER_VERSION; +use hotshot::traits::ValidatedState; use hotshot_types::data::ViewNumber; use hotshot_types::traits::node_implementation::ConsensusTime; use sequencer::{eth_signature_key::EthKeyPair, Genesis, L1Params}; @@ -121,7 +122,7 @@ async fn main() -> anyhow::Result<()> { ) .unwrap(); - let validated_state = Default::default(); + let validated_state = ValidatedState::genesis(&instance_state).0; let api_response_timeout_duration = opt.max_api_timeout_duration; diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index 216807fbc3..41882d60a5 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -247,7 +247,10 @@ pub async fn init_node for Payload { } else { match validated_state_cf.resolve() { Some(cf) => cf, - None => instance_state - .peers - .as_ref() - .fetch_chain_config(validated_state_cf.commit()) - .await - .unwrap(), // TODO: + None => { + instance_state + .peers + .as_ref() + .fetch_chain_config(validated_state_cf.commit()) + .await + } } }; diff --git a/sequencer/src/catchup.rs b/sequencer/src/catchup.rs index b9e1ade5e0..f5b4ba1e6f 100644 --- a/sequencer/src/catchup.rs +++ b/sequencer/src/catchup.rs @@ -149,16 +149,13 @@ pub trait StateCatchup: Send + Sync + std::fmt::Debug { commitment: Commitment, ) -> anyhow::Result; - async fn fetch_chain_config( - &self, - commitment: Commitment, - ) -> anyhow::Result { + async fn fetch_chain_config(&self, commitment: Commitment) -> ChainConfig { // Retry until we succeed. let mut delay = MIN_RETRY_DELAY; loop { match self.try_fetch_chain_config(commitment).await { - Ok(cf) => return Ok(cf), + Ok(cf) => return cf, Err(err) => { tracing::warn!( ?delay, @@ -404,10 +401,7 @@ impl StateCatchup for Box { (**self).try_fetch_chain_config(commitment).await } - async fn fetch_chain_config( - &self, - commitment: Commitment, - ) -> anyhow::Result { + async fn fetch_chain_config(&self, commitment: Commitment) -> ChainConfig { (**self).fetch_chain_config(commitment).await } } @@ -465,10 +459,7 @@ impl StateCatchup for Arc { (**self).try_fetch_chain_config(commitment).await } - async fn fetch_chain_config( - &self, - commitment: Commitment, - ) -> anyhow::Result { + async fn fetch_chain_config(&self, commitment: Commitment) -> ChainConfig { (**self).fetch_chain_config(commitment).await } } diff --git a/sequencer/src/header.rs b/sequencer/src/header.rs index 0e4cbee38c..c1b4d90f28 100644 --- a/sequencer/src/header.rs +++ b/sequencer/src/header.rs @@ -251,12 +251,12 @@ impl Header { async fn get_chain_config( validated_state: &ValidatedState, instance_state: &NodeState, - ) -> anyhow::Result { + ) -> ChainConfig { if validated_state.chain_config.commit() == instance_state.chain_config.commitment() { - Ok(instance_state.chain_config) + instance_state.chain_config } else { match validated_state.chain_config.resolve() { - Some(cf) => Ok(cf), + Some(cf) => cf, None => { tracing::info!( "fetching chain config {} from peers", @@ -322,15 +322,15 @@ impl BlockHeader for Header { let chain_config = if version > instance_state.current_version { match instance_state .upgrades - .get(&instance_state.current_version) + .get(&version) .map(|upgrade| match upgrade.upgrade_type { UpgradeType::ChainConfig { chain_config } => chain_config, }) { Some(cf) => cf, - None => Header::get_chain_config(&validated_state, instance_state).await?, + None => Header::get_chain_config(&validated_state, instance_state).await, } } else { - Header::get_chain_config(&validated_state, instance_state).await? + Header::get_chain_config(&validated_state, instance_state).await }; validated_state.chain_config = chain_config.into(); diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index bab4449a7c..2db0dfe852 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -550,7 +550,7 @@ impl SequencerPersistence for Persistence { } } -pub fn sql_param(param: &T) -> &(dyn ToSql + Sync) { +pub(crate) fn sql_param(param: &T) -> &(dyn ToSql + Sync) { param } diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 5d9909dfad..4e114d6efa 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -231,9 +231,9 @@ pub fn validate_proposal( anyhow::ensure!( proposal.chain_config.commit() == expected_chain_config.commit(), anyhow::anyhow!( - "Invalid Chain Config: local={:?}, proposal={:?}", - expected_chain_config.commitment().to_string(), - proposal.chain_config.commit().to_string(), + "Invalid Chain Config: local={}, proposal={}", + expected_chain_config.commitment(), + proposal.chain_config.commit(), ) ); @@ -456,10 +456,12 @@ async fn update_state_storage( } if parent_chain_config != state.chain_config { - if let Err(err) = storage - .insert_chain_config(state.chain_config.resolve().unwrap()) - .await - { + let cf = state + .chain_config + .resolve() + .context("failed to resolve to chain config")?; + + if let Err(err) = storage.insert_chain_config(cf).await { storage.revert().await; return Err(err); } @@ -498,9 +500,9 @@ async fn store_genesis_state( .context("failed to store fee merkle nodes")?; } - storage.commit().await?; - storage.insert_chain_config(chain_config).await?; + + storage.commit().await?; Ok(()) } @@ -618,7 +620,7 @@ impl ValidatedState { let header_cf_commit = header_cf.commit(); if version > instance.current_version { - if let Some(upgrade) = &instance.upgrades.get(&instance.current_version) { + if let Some(upgrade) = &instance.upgrades.get(&version) { match upgrade.upgrade_type { UpgradeType::ChainConfig { chain_config } => { if header_cf_commit == chain_config.commit() { @@ -649,7 +651,7 @@ impl ValidatedState { .peers .as_ref() .fetch_chain_config(self.chain_config.commit()) - .await? + .await } } }; From d9366a0574c3f4996cc9b402c980196ee04a4244 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 7 Jun 2024 05:34:22 +0500 Subject: [PATCH 29/48] apply_upgrade() and get_chain_config() --- sequencer/src/api/sql.rs | 3 + sequencer/src/state.rs | 124 +++++++++++++++++++++------------------ 2 files changed, 69 insertions(+), 58 deletions(-) diff --git a/sequencer/src/api/sql.rs b/sequencer/src/api/sql.rs index 532fcfe6c4..67a6bf81a9 100644 --- a/sequencer/src/api/sql.rs +++ b/sequencer/src/api/sql.rs @@ -18,6 +18,7 @@ use hotshot_query_service::{ data_source::{ sql::{Config, Query, SqlDataSource}, storage::SqlStorage, + VersionedDataSource, }, merklized_state::{MerklizedStateDataSource, Snapshot}, Resolvable, @@ -141,6 +142,8 @@ impl ChainConfigPersistence for SqlStorage { ) .await?; + self.commit().await?; + Ok(()) } diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 4e114d6efa..6eda1c15f9 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -604,63 +604,6 @@ impl SequencerStateDataSource for T where } impl ValidatedState { - /// Retrieves the `ChainConfig`. - /// - /// Returns the `NodeState` `ChainConfig` if the `ValidatedState` `ChainConfig` commitment matches the `NodeState` `ChainConfig`` commitment. - /// If the commitments do not match, it returns the `ChainConfig` available in either `ValidatedState` or proposed header. - /// If neither has the `ChainConfig`, it fetches the config from the peers. - /// - /// Returns an error if the proposed header's `ChainConfig` commitment does not match the `ValidatedState` `ChainConfig` commitment, - pub(crate) async fn apply_chain_config( - &mut self, - instance: &NodeState, - header_cf: &ResolvableChainConfig, - version: Version, - ) -> anyhow::Result { - let header_cf_commit = header_cf.commit(); - - if version > instance.current_version { - if let Some(upgrade) = &instance.upgrades.get(&version) { - match upgrade.upgrade_type { - UpgradeType::ChainConfig { chain_config } => { - if header_cf_commit == chain_config.commit() { - self.chain_config = chain_config.into(); - return Ok(chain_config); - } - } - } - } - } - - if header_cf_commit != self.chain_config.commit() { - bail!( - "Proposed header chain config commit={} expected={}", - header_cf.commit(), - self.chain_config.commit() - ); - } - - let cf = if self.chain_config.commit() == instance.chain_config.commit() { - instance.chain_config - } else { - match (self.chain_config.resolve(), header_cf.resolve()) { - (Some(cf), _) => cf, - (_, Some(cf)) => cf, - (None, None) => { - instance - .peers - .as_ref() - .fetch_chain_config(self.chain_config.commit()) - .await - } - } - }; - - self.chain_config = cf.into(); - - Ok(cf) - } - pub(crate) async fn apply_header( &self, instance: &NodeState, @@ -672,9 +615,13 @@ impl ValidatedState { // through returned value. let mut validated_state = self.clone(); + validated_state.apply_upgrade(instance, proposed_header, version); + let chain_config = validated_state - .apply_chain_config(instance, &proposed_header.chain_config, version) + .get_chain_config(instance, &proposed_header.chain_config) .await?; + validated_state.chain_config = chain_config.into(); + let l1_deposits = get_l1_deposits( instance, proposed_header, @@ -756,6 +703,67 @@ impl ValidatedState { Ok((validated_state, delta)) } + + /// Updates the `ValidatedState` if a protocol upgrade has occurred. + pub(crate) fn apply_upgrade( + &mut self, + instance: &NodeState, + header: &Header, + version: Version, + ) { + // NOTE: This function currently only supports ChainConfig upgrade. + // You might need to add additional logic in the future to handle other upgrade types. + + // Check for protocol upgrade based on sequencer version + if version <= instance.current_version { + return; + } + + let Some(upgrade) = instance.upgrades.get(&version) else { + return; + }; + + match upgrade.upgrade_type { + UpgradeType::ChainConfig { chain_config } => { + if header.chain_config.commit() == chain_config.commit() { + self.chain_config = chain_config.into(); + } + } + } + } + + /// Retrieves the `ChainConfig`. + /// + /// Returns the `NodeState` `ChainConfig` if the `ValidatedState` `ChainConfig` commitment matches the `NodeState` `ChainConfig`` commitment. + /// If the commitments do not match, it returns the `ChainConfig` available in either `ValidatedState` or proposed header. + /// If neither has the `ChainConfig`, it fetches the config from the peers. + /// + /// Returns an error if it fails to fetch the `ChainConfig` from the peers. + pub(crate) async fn get_chain_config( + &self, + instance: &NodeState, + header_cf: &ResolvableChainConfig, + ) -> anyhow::Result { + let state_cf = self.chain_config; + + if state_cf.commit() == instance.chain_config.commit() { + return Ok(instance.chain_config); + } + + let cf = match (state_cf.resolve(), header_cf.resolve()) { + (Some(cf), _) => cf, + (_, Some(cf)) if cf.commit() == state_cf.commit() => cf, + (_, Some(_)) | (None, None) => { + instance + .peers + .as_ref() + .fetch_chain_config(state_cf.commit()) + .await + } + }; + + Ok(cf) + } } pub async fn get_l1_deposits( From b585bac2b15bf4647f0a9913164fd26e986efb6d Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 7 Jun 2024 05:50:21 +0500 Subject: [PATCH 30/48] update only if chain config is different --- sequencer/src/state.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 6eda1c15f9..01a10f5fd6 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -620,7 +620,10 @@ impl ValidatedState { let chain_config = validated_state .get_chain_config(instance, &proposed_header.chain_config) .await?; - validated_state.chain_config = chain_config.into(); + + if Some(chain_config) != validated_state.chain_config.resolve() { + validated_state.chain_config = chain_config.into(); + } let l1_deposits = get_l1_deposits( instance, From c54d1666bfd39afb09549c9a02ccf998417bb677 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 7 Jun 2024 06:13:27 +0500 Subject: [PATCH 31/48] return early --- builder/src/permissioned.rs | 3 ++- sequencer/src/header.rs | 34 +++++++++++++++++----------------- sequencer/src/state.rs | 3 --- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index 41882d60a5..76b0e4ef80 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -470,7 +470,8 @@ impl ChainConfig { - if validated_state.chain_config.commit() == instance_state.chain_config.commitment() { - instance_state.chain_config - } else { - match validated_state.chain_config.resolve() { - Some(cf) => cf, - None => { - tracing::info!( - "fetching chain config {} from peers", - validated_state.chain_config.commit() - ); - - instance_state - .peers - .as_ref() - .fetch_chain_config(validated_state.chain_config.commit()) - .await - } + let validated_cf = validated_state.chain_config; + let instance_cf = instance_state.chain_config; + + if validated_cf.commit() == instance_cf.commitment() { + return instance_cf; + } + + match validated_cf.resolve() { + Some(cf) => cf, + None => { + tracing::info!("fetching chain config {} from peers", validated_cf.commit()); + + instance_state + .peers + .as_ref() + .fetch_chain_config(validated_cf.commit()) + .await } } } diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 01a10f5fd6..bb31c1b867 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -714,9 +714,6 @@ impl ValidatedState { header: &Header, version: Version, ) { - // NOTE: This function currently only supports ChainConfig upgrade. - // You might need to add additional logic in the future to handle other upgrade types. - // Check for protocol upgrade based on sequencer version if version <= instance.current_version { return; From 9b3952a8a3d4805de22ff43719dcdfb9c340a52b Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 14 Jun 2024 03:49:29 +0500 Subject: [PATCH 32/48] use Base::instance() --- builder/src/lib.rs | 4 ++-- sequencer/src/state.rs | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/builder/src/lib.rs b/builder/src/lib.rs index ee7de9f526..b2f62ce92e 100644 --- a/builder/src/lib.rs +++ b/builder/src/lib.rs @@ -88,7 +88,7 @@ pub fn run_builder_api_service(url: Url, source: ProxyGlobalState) { app.register_module("txn_submit", private_mempool_api) .expect("Failed to register the private mempool API"); - async_spawn(app.serve(url, vbs::version::StaticVersion::<0, 1> {})); + async_spawn(app.serve(url, Base::instance())); } #[cfg(test)] @@ -446,7 +446,7 @@ pub mod testing { app.register_module("hotshot-events", hotshot_events_api) .expect("Failed to register hotshot events API"); - async_spawn(app.serve(url, vbs::version::StaticVersion::<0, 1> {})); + async_spawn(app.serve(url, Base::instance())); } // enable hotshot event streaming pub fn enable_hotshot_node_event_streaming( diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 199990e593..0afc58b24d 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -905,8 +905,6 @@ impl HotShotState for ValidatedState { vid_common: VidCommon, version: Version, ) -> Result<(Self, Self::Delta), Self::Error> { - tracing::info!("{version:?}"); - //validate builder fee if let Err(err) = validate_builder_fee(proposed_header) { tracing::error!("invalid builder fee: {err:#}"); From ef2e4913018cf3d088423e8b364d0e4f3ff87485 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 14 Jun 2024 04:08:05 +0500 Subject: [PATCH 33/48] add propose_window, commit and revert in caller, remove header cf commit check --- data/genesis/demo.toml | 1 + sequencer/src/api.rs | 9 ++++++--- sequencer/src/api/sql.rs | 3 --- sequencer/src/genesis.rs | 4 ++++ sequencer/src/state.rs | 17 ++++++++++++----- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/data/genesis/demo.toml b/data/genesis/demo.toml index a43823633a..a9971b0258 100644 --- a/data/genesis/demo.toml +++ b/data/genesis/demo.toml @@ -14,6 +14,7 @@ timestamp = "1970-01-01T00:00:00Z" [[upgrade]] version = "0.2" view = 5 +propose_window = 100 [upgrade.chain_config] diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index becb484446..4207a4c0e5 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -1374,10 +1374,13 @@ mod test { ..Default::default() }; let mut map = std::collections::BTreeMap::new(); + let view = 5; + let propose_window = 100; map.insert( Version { major: 0, minor: 2 }, Upgrade { - view: 5, + view, + propose_window, upgrade_type: UpgradeType::ChainConfig { chain_config: chain_config_upgrade, }, @@ -1387,8 +1390,8 @@ mod test { let stop_voting_view = 100; let upgrades = TestNetworkUpgrades { upgrades: map, - start_proposing_view: 5, - stop_proposing_view: 6, + start_proposing_view: view, + stop_proposing_view: view + propose_window, start_voting_view: 1, stop_voting_view, }; diff --git a/sequencer/src/api/sql.rs b/sequencer/src/api/sql.rs index 67a6bf81a9..532fcfe6c4 100644 --- a/sequencer/src/api/sql.rs +++ b/sequencer/src/api/sql.rs @@ -18,7 +18,6 @@ use hotshot_query_service::{ data_source::{ sql::{Config, Query, SqlDataSource}, storage::SqlStorage, - VersionedDataSource, }, merklized_state::{MerklizedStateDataSource, Snapshot}, Resolvable, @@ -142,8 +141,6 @@ impl ChainConfigPersistence for SqlStorage { ) .await?; - self.commit().await?; - Ok(()) } diff --git a/sequencer/src/genesis.rs b/sequencer/src/genesis.rs index 11b08a346d..52f65088da 100644 --- a/sequencer/src/genesis.rs +++ b/sequencer/src/genesis.rs @@ -119,6 +119,7 @@ pub enum UpgradeType { #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Upgrade { pub view: u64, + pub propose_window: u64, #[serde(flatten)] pub upgrade_type: UpgradeType, } @@ -171,6 +172,7 @@ mod upgrade_serialization { struct UpgradeFields { version: String, view: u64, + propose_window: u64, #[serde(flatten)] upgrade_type: UpgradeType, } @@ -188,6 +190,7 @@ mod upgrade_serialization { version, Upgrade { view: fields.view, + propose_window: fields.propose_window, upgrade_type: fields.upgrade_type, }, ); @@ -252,6 +255,7 @@ mod test { [[upgrade]] version = "1.0" view = 1 + propose_window = 100 [upgrade.chain_config] chain_id = 12345 diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 0afc58b24d..fe867bf95c 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -524,9 +524,16 @@ async fn update_state_storage( .resolve() .context("failed to resolve to chain config")?; - if let Err(err) = storage.insert_chain_config(cf).await { - storage.revert().await; - return Err(err); + match storage.insert_chain_config(cf).await { + Err(err) => { + return Err(err); + } + Ok(()) => { + if let Err(err) = storage.commit().await { + storage.revert().await; + return Err(err.into()); + } + } } } @@ -815,8 +822,8 @@ impl ValidatedState { let cf = match (state_cf.resolve(), header_cf.resolve()) { (Some(cf), _) => cf, - (_, Some(cf)) if cf.commit() == state_cf.commit() => cf, - (_, Some(_)) | (None, None) => { + (_, Some(cf)) => cf, + (None, None) => { instance .peers .as_ref() From c3625aa46ecb430ba0d7845e3bed029bef0164ef Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 14 Jun 2024 04:15:12 +0500 Subject: [PATCH 34/48] lint --- hotshot-state-prover/src/mock_ledger.rs | 30 +++++++++++-------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/hotshot-state-prover/src/mock_ledger.rs b/hotshot-state-prover/src/mock_ledger.rs index 0e9ec11904..0b7b061f76 100644 --- a/hotshot-state-prover/src/mock_ledger.rs +++ b/hotshot-state-prover/src/mock_ledger.rs @@ -477,23 +477,19 @@ pub fn gen_plonk_proof_for_test( let mut proofs = vec![]; let mut extra_msgs = vec![]; - circuits - .iter() - .zip(prove_keys.iter()) - .enumerate() - .for_each(|(_, (cs, pk))| { - let extra_msg = Some(vec![]); // We set extra_msg="" for the contract tests to pass - proofs.push( - PlonkKzgSnark::::prove::<_, _, SolidityTranscript>( - rng, - cs, - pk, - extra_msg.clone(), - ) - .unwrap(), - ); - extra_msgs.push(extra_msg); - }); + circuits.iter().zip(prove_keys.iter()).for_each(|(cs, pk)| { + let extra_msg = Some(vec![]); // We set extra_msg="" for the contract tests to pass + proofs.push( + PlonkKzgSnark::::prove::<_, _, SolidityTranscript>( + rng, + cs, + pk, + extra_msg.clone(), + ) + .unwrap(), + ); + extra_msgs.push(extra_msg); + }); let public_inputs: Vec> = circuits .iter() From 17d927702b5cfaa10f5b37392b69164288344455 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 14 Jun 2024 04:27:54 +0500 Subject: [PATCH 35/48] lint --- sequencer/src/transaction.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sequencer/src/transaction.rs b/sequencer/src/transaction.rs index e4ac5f4ae1..7bea5b643f 100644 --- a/sequencer/src/transaction.rs +++ b/sequencer/src/transaction.rs @@ -84,11 +84,11 @@ impl NamespaceId { impl Namespace for NamespaceId { fn max() -> Self { - Self(u32::max_value() as u64) + Self(u32::MAX as u64) } fn min() -> Self { - Self(u32::min_value() as u64) + Self(u32::MIN as u64) } } From 56bbcfcaf8c717d88e477571269aaf7334d7d6df Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 14 Jun 2024 05:04:59 +0500 Subject: [PATCH 36/48] fix migrations version and use propose_window --- ...{V31__chain_config_table.sql => V33__chain_config_table.sql} | 0 sequencer/src/lib.rs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename sequencer/api/migrations/{V31__chain_config_table.sql => V33__chain_config_table.sql} (100%) diff --git a/sequencer/api/migrations/V31__chain_config_table.sql b/sequencer/api/migrations/V33__chain_config_table.sql similarity index 100% rename from sequencer/api/migrations/V31__chain_config_table.sql rename to sequencer/api/migrations/V33__chain_config_table.sql diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 643843121b..fa2ccfa490 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -402,7 +402,7 @@ pub async fn init_node( if let Some(upgrade) = genesis.upgrades.get(&version) { let view = upgrade.view; config.config.start_proposing_view = view; - config.config.stop_proposing_view = view + 100; + config.config.stop_proposing_view = view + upgrade.propose_window; config.config.start_voting_view = 1; config.config.stop_voting_view = u64::MAX; } From 1f49651519f712355dae26d7efe3d91c0b2ea121 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 14 Jun 2024 17:53:17 +0500 Subject: [PATCH 37/48] fix: enforce_max_block_size test --- docker-compose.yaml | 2 +- process-compose.yaml | 2 +- sequencer/api/public-env-vars.toml | 2 +- sequencer/src/bin/orchestrator.rs | 2 +- sequencer/src/block/test.rs | 30 ++++++++++++++++++++++-------- sequencer/src/genesis.rs | 1 + 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index a0abc0bc93..7d1c4fcdc8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -71,7 +71,7 @@ services: ports: - "$ESPRESSO_ORCHESTRATOR_PORT:$ESPRESSO_ORCHESTRATOR_PORT" environment: - - ESPRESSO_ORCHESTRATOR_BUILDER_URL=http://permissionless-builder:$ESPRESSO_BUILDER_SERVER_PORT + - ESPRESSO_ORCHESTRATOR_BUILDER_URLS=http://permissionless-builder:$ESPRESSO_BUILDER_SERVER_PORT - ESPRESSO_ORCHESTRATOR_PORT - ESPRESSO_ORCHESTRATOR_NUM_NODES - ESPRESSO_ORCHESTRATOR_START_DELAY diff --git a/process-compose.yaml b/process-compose.yaml index cdb0e9a80f..67796668b2 100644 --- a/process-compose.yaml +++ b/process-compose.yaml @@ -64,7 +64,7 @@ processes: orchestrator: command: orchestrator environment: - - ESPRESSO_ORCHESTRATOR_BUILDER_URL=http://localhost:$ESPRESSO_BUILDER_SERVER_PORT + - ESPRESSO_ORCHESTRATOR_BUILDER_URLS=http://localhost:$ESPRESSO_BUILDER_SERVER_PORT readiness_probe: http_get: scheme: http diff --git a/sequencer/api/public-env-vars.toml b/sequencer/api/public-env-vars.toml index 34be7819b4..88f37c8b06 100644 --- a/sequencer/api/public-env-vars.toml +++ b/sequencer/api/public-env-vars.toml @@ -42,7 +42,7 @@ variables = [ "ESPRESSO_NASTY_CLIENT_WEIGHT_QUERY_NAMESPACE", "ESPRESSO_NASTY_CLIENT_WEIGHT_QUERY_WINDOW", "ESPRESSO_ORCHESTRATOR_BUILDER_TIMEOUT", - "ESPRESSO_ORCHESTRATOR_BUILDER_URL", + "ESPRESSO_ORCHESTRATOR_BUILDER_URLS", "ESPRESSO_ORCHESTRATOR_LIBP2P_MESH_N", "ESPRESSO_ORCHESTRATOR_NEXT_VIEW_TIMEOUT", "ESPRESSO_ORCHESTRATOR_NUM_NODES", diff --git a/sequencer/src/bin/orchestrator.rs b/sequencer/src/bin/orchestrator.rs index f420579a69..f51b79c263 100644 --- a/sequencer/src/bin/orchestrator.rs +++ b/sequencer/src/bin/orchestrator.rs @@ -77,7 +77,7 @@ struct Args { keygen_seed: [u8; 32], /// HotShot builder URL - #[arg(long, env = "ESPRESSO_ORCHESTRATOR_BUILDER_URL", value_delimiter = ',')] + #[arg(long, env = "ESPRESSO_ORCHESTRATOR_BUILDER_URLS", num_args = 1.., value_delimiter = ',')] builder_urls: Vec, /// The maximum amount of time a leader can wait to get a block from a builder. diff --git a/sequencer/src/block/test.rs b/sequencer/src/block/test.rs index 02adf867d8..993d016847 100644 --- a/sequencer/src/block/test.rs +++ b/sequencer/src/block/test.rs @@ -4,7 +4,7 @@ use crate::{ namespace_payload::TxProof, }, chain_config::BlockSize, - ChainConfig, NamespaceId, NodeState, Transaction, + ChainConfig, NamespaceId, NodeState, Transaction, ValidatedState, }; use async_compatibility_layer::logging::{setup_backtrace, setup_logging}; use hotshot::traits::BlockPayload; @@ -117,15 +117,21 @@ async fn enforce_max_block_size() { let test = ValidTest::from_tx_lengths(test_case, &mut rng); let tx_count_expected = test.all_txs().len(); - // test: actual block size equals max block size - let instance_state = NodeState::default().with_chain_config(ChainConfig { + let chain_config = ChainConfig { max_block_size: BlockSize::from( (payload_byte_len_expected + ns_table_byte_len_expected) as u64, ), ..Default::default() - }); + }; + + // test: actual block size equals max block size + let instance_state = NodeState::default().with_chain_config(chain_config); - let block = Payload::from_transactions(test.all_txs(), &Default::default(), &instance_state) + let validated_state = ValidatedState { + chain_config: chain_config.into(), + ..Default::default() + }; + let block = Payload::from_transactions(test.all_txs(), &validated_state, &instance_state) .await .unwrap() .0; @@ -135,13 +141,21 @@ async fn enforce_max_block_size() { // test: actual block size exceeds max block size, so 1 tx is dropped // WARN log should be emitted - let instance_state = NodeState::default().with_chain_config(ChainConfig { + + let chain_config = ChainConfig { max_block_size: BlockSize::from( (payload_byte_len_expected + ns_table_byte_len_expected - 1) as u64, ), ..Default::default() - }); - let block = Payload::from_transactions(test.all_txs(), &Default::default(), &instance_state) + }; + let instance_state = NodeState::default().with_chain_config(chain_config); + + let validated_state = ValidatedState { + chain_config: chain_config.into(), + ..Default::default() + }; + + let block = Payload::from_transactions(test.all_txs(), &validated_state, &instance_state) .await .unwrap() .0; diff --git a/sequencer/src/genesis.rs b/sequencer/src/genesis.rs index 52f65088da..59d4c67359 100644 --- a/sequencer/src/genesis.rs +++ b/sequencer/src/genesis.rs @@ -143,6 +143,7 @@ mod upgrade_serialization { seq.serialize_element(&( version.to_string(), upgrade.view, + upgrade.propose_window, upgrade.upgrade_type.clone(), ))?; } From 7f976ff75d93307e037c9162d173b6885b00c5c9 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 14 Jun 2024 20:14:11 +0500 Subject: [PATCH 38/48] change TestConfig nodes to 5 --- sequencer/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index fa2ccfa490..6192f8d286 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -689,7 +689,7 @@ pub mod testing { } impl TestConfig { - pub const NUM_NODES: usize = 4; + pub const NUM_NODES: usize = 5; pub fn num_nodes(&self) -> usize { self.priv_keys.len() From 03b4e3c6e6808b9302b7fae212f5314fe43a3094 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 14 Jun 2024 20:34:25 +0500 Subject: [PATCH 39/48] set propose window to 10 --- data/genesis/demo.toml | 2 +- sequencer/src/api.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/genesis/demo.toml b/data/genesis/demo.toml index a9971b0258..5c52dc3861 100644 --- a/data/genesis/demo.toml +++ b/data/genesis/demo.toml @@ -14,7 +14,7 @@ timestamp = "1970-01-01T00:00:00Z" [[upgrade]] version = "0.2" view = 5 -propose_window = 100 +propose_window = 10 [upgrade.chain_config] diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 4207a4c0e5..25aa49f9c4 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -1375,7 +1375,7 @@ mod test { }; let mut map = std::collections::BTreeMap::new(); let view = 5; - let propose_window = 100; + let propose_window = 10; map.insert( Version { major: 0, minor: 2 }, Upgrade { From 119f60a5a4e8526903ba8fff11f2b9a72aa86e4d Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 14 Jun 2024 21:07:55 +0500 Subject: [PATCH 40/48] use transaction() --- sequencer/src/api.rs | 6 +----- sequencer/src/api/sql.rs | 29 ++++++++++++++++------------- sequencer/src/genesis.rs | 2 +- sequencer/src/persistence/sql.rs | 32 +++++++++++++++++++------------- sequencer/src/state.rs | 12 +----------- 5 files changed, 38 insertions(+), 43 deletions(-) diff --git a/sequencer/src/api.rs b/sequencer/src/api.rs index 25aa49f9c4..abba9131fb 100644 --- a/sequencer/src/api.rs +++ b/sequencer/src/api.rs @@ -1245,11 +1245,7 @@ mod test { let states = std::array::from_fn(|_| state.clone()); let mut network = TestNetwork::with_state( - Options::from(options::Http { - port, - max_connections: None, - }) - .catchup(Default::default()), + Options::with_port(port).catchup(Default::default()), states, [no_storage::Options; TestConfig::NUM_NODES], std::array::from_fn(|_| { diff --git a/sequencer/src/api/sql.rs b/sequencer/src/api/sql.rs index 532fcfe6c4..716190d5f0 100644 --- a/sequencer/src/api/sql.rs +++ b/sequencer/src/api/sql.rs @@ -4,7 +4,7 @@ use super::{ }; use crate::{ persistence::{ - sql::{sql_param, Options}, + sql::{sql_param, transaction, Options}, ChainConfigPersistence, }, state::{BlockMerkleTree, FeeAccountProof, FeeMerkleTree}, @@ -14,6 +14,7 @@ use anyhow::{bail, Context}; use async_trait::async_trait; use committable::Commitment; use ethers::prelude::Address; +use futures::FutureExt; use hotshot_query_service::{ data_source::{ sql::{Config, Query, SqlDataSource}, @@ -130,18 +131,20 @@ impl ChainConfigPersistence for SqlStorage { let commitment = chain_config.commitment(); let data = bincode::serialize(&chain_config)?; - let mut transaction = self.transaction().await?; - - transaction - .upsert( - "chain_config", - ["commitment", "data"], - ["commitment"], - [[sql_param(&(commitment.to_string())), sql_param(&data)]], - ) - .await?; - - Ok(()) + transaction(self, |mut tx| { + async move { + tx.upsert( + "chain_config", + ["commitment", "data"], + ["commitment"], + [[sql_param(&(commitment.to_string())), sql_param(&data)]], + ) + .await?; + Ok(()) + } + .boxed() + }) + .await } async fn load_chain_config( diff --git a/sequencer/src/genesis.rs b/sequencer/src/genesis.rs index 59d4c67359..9c013090d3 100644 --- a/sequencer/src/genesis.rs +++ b/sequencer/src/genesis.rs @@ -256,7 +256,7 @@ mod test { [[upgrade]] version = "1.0" view = 1 - propose_window = 100 + propose_window = 10 [upgrade.chain_config] chain_id = 12345 diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index 8c48a5595b..209d1af99c 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -253,19 +253,25 @@ pub struct Persistence { store_undecided_state: bool, } -async fn transaction( - persistence: &mut Persistence, +pub(crate) async fn transaction( + sql: &mut SqlStorage, f: impl FnOnce(Transaction) -> BoxFuture>, ) -> anyhow::Result<()> { - let tx = persistence.db.transaction().await?; + let tx = sql.transaction().await?; match f(tx).await { Ok(_) => { - persistence.db.commit().await?; + if let Err(err) = sql.commit().await { + tracing::warn!("transaction failed, reverting: {err:#}"); + sql.revert().await; + + return Err(err.into()); + } + Ok(()) } Err(err) => { tracing::warn!("transaction failed, reverting: {err:#}"); - persistence.db.revert().await; + sql.revert().await; Err(err) } } @@ -299,7 +305,7 @@ impl SequencerPersistence for Persistence { tracing::info!("saving config to Postgres"); let json = serde_json::to_value(cfg)?; - transaction(self, |mut tx| { + transaction(&mut self.db, |mut tx| { async move { tx.execute_one_with_retries( "INSERT INTO network_config (config) VALUES ($1)", @@ -314,7 +320,7 @@ impl SequencerPersistence for Persistence { } async fn collect_garbage(&mut self, view: ViewNumber) -> anyhow::Result<()> { - transaction(self, |mut tx| { + transaction(&mut self.db, |mut tx| { async move { let stmt1 = "DELETE FROM vid_share where view <= $1"; tx.execute(stmt1, [&(view.u64() as i64)]).await?; @@ -360,7 +366,7 @@ impl SequencerPersistence for Persistence { let leaf_bytes = bincode::serialize(leaf)?; let qc_bytes = bincode::serialize(qc)?; - transaction(self, |mut tx| { + transaction(&mut self.db, |mut tx| { async move { tx.execute_one_with_retries( stmt, @@ -502,7 +508,7 @@ impl SequencerPersistence for Persistence { let view = data.view_number().u64(); let data_bytes = bincode::serialize(proposal).unwrap(); - transaction(self, |mut tx| { + transaction(&mut self.db, |mut tx| { async move { tx.upsert( "vid_share", @@ -525,7 +531,7 @@ impl SequencerPersistence for Persistence { let view = data.view_number().u64(); let data_bytes = bincode::serialize(proposal).unwrap(); - transaction(self, |mut tx| { + transaction(&mut self.db, |mut tx| { async move { tx.upsert( "da_proposal", @@ -549,7 +555,7 @@ impl SequencerPersistence for Persistence { INSERT INTO highest_voted_view (id, view) VALUES (0, $1) ON CONFLICT (id) DO UPDATE SET view = GREATEST(highest_voted_view.view, excluded.view)"; - transaction(self, |mut tx| { + transaction(&mut self.db, |mut tx| { async move { tx.execute_one_with_retries(stmt, [view.u64() as i64]) .await?; @@ -571,7 +577,7 @@ impl SequencerPersistence for Persistence { let leaves_bytes = bincode::serialize(&leaves).context("serializing leaves")?; let state_bytes = bincode::serialize(&state).context("serializing state")?; - transaction(self, |mut tx| { + transaction(&mut self.db, |mut tx| { async move { tx.upsert( "undecided_state", @@ -596,7 +602,7 @@ impl SequencerPersistence for Persistence { ) -> anyhow::Result<()> { let view_number = proposal.data.view_number().u64(); let proposal_bytes = bincode::serialize(&proposal).context("serializing proposal")?; - transaction(self, |mut tx| { + transaction(&mut self.db, |mut tx| { async move { tx.upsert( "quorum_proposals", diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index fe867bf95c..861316271a 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -524,17 +524,7 @@ async fn update_state_storage( .resolve() .context("failed to resolve to chain config")?; - match storage.insert_chain_config(cf).await { - Err(err) => { - return Err(err); - } - Ok(()) => { - if let Err(err) = storage.commit().await { - storage.revert().await; - return Err(err.into()); - } - } - } + storage.insert_chain_config(cf).await? } Ok(state) From 4d1698898976a32b73a56d3908c40e13b042bd90 Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 14 Jun 2024 21:17:04 +0500 Subject: [PATCH 41/48] remove header cf commit check --- sequencer/src/api/sql.rs | 4 ++-- sequencer/src/state.rs | 13 +++---------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/sequencer/src/api/sql.rs b/sequencer/src/api/sql.rs index 716190d5f0..6b9afac5f7 100644 --- a/sequencer/src/api/sql.rs +++ b/sequencer/src/api/sql.rs @@ -139,8 +139,8 @@ impl ChainConfigPersistence for SqlStorage { ["commitment"], [[sql_param(&(commitment.to_string())), sql_param(&data)]], ) - .await?; - Ok(()) + .await + .map_err(Into::into) } .boxed() }) diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 861316271a..5577fde194 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -675,7 +675,7 @@ impl ValidatedState { // through returned value. let mut validated_state = self.clone(); - validated_state.apply_upgrade(instance, proposed_header, version); + validated_state.apply_upgrade(instance, version); let chain_config = validated_state .get_chain_config(instance, &proposed_header.chain_config) @@ -768,12 +768,7 @@ impl ValidatedState { } /// Updates the `ValidatedState` if a protocol upgrade has occurred. - pub(crate) fn apply_upgrade( - &mut self, - instance: &NodeState, - header: &Header, - version: Version, - ) { + pub(crate) fn apply_upgrade(&mut self, instance: &NodeState, version: Version) { // Check for protocol upgrade based on sequencer version if version <= instance.current_version { return; @@ -785,9 +780,7 @@ impl ValidatedState { match upgrade.upgrade_type { UpgradeType::ChainConfig { chain_config } => { - if header.chain_config.commit() == chain_config.commit() { - self.chain_config = chain_config.into(); - } + self.chain_config = chain_config.into(); } } } From e9fbe2da24754ee8cf3180cfb09cb5a2cc81276a Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 14 Jun 2024 21:30:35 +0500 Subject: [PATCH 42/48] update hotshot and cdn --- Cargo.lock | 309 +++++++++++++++++++++++------------------------------ Cargo.toml | 26 ++--- 2 files changed, 144 insertions(+), 191 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9209a53061..143fff0878 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -535,30 +535,14 @@ dependencies = [ "term", ] -[[package]] -name = "asn1-rs" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" -dependencies = [ - "asn1-rs-derive 0.4.0", - "asn1-rs-impl 0.1.0", - "displaydoc", - "nom", - "num-traits", - "rusticata-macros", - "thiserror", - "time 0.3.36", -] - [[package]] name = "asn1-rs" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ad1373757efa0f70ec53939aabc7152e1591cb485208052993070ac8d2429d" dependencies = [ - "asn1-rs-derive 0.5.0", - "asn1-rs-impl 0.2.0", + "asn1-rs-derive", + "asn1-rs-impl", "displaydoc", "nom", "num-traits", @@ -567,18 +551,6 @@ dependencies = [ "time 0.3.36", ] -[[package]] -name = "asn1-rs-derive" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", - "synstructure 0.12.6", -] - [[package]] name = "asn1-rs-derive" version = "0.5.0" @@ -591,17 +563,6 @@ dependencies = [ "synstructure 0.13.1", ] -[[package]] -name = "asn1-rs-impl" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "asn1-rs-impl" version = "0.2.0" @@ -674,7 +635,7 @@ dependencies = [ "async-std", "async-trait", "color-eyre", - "console-subscriber", + "console-subscriber 0.2.0", "flume 0.11.0", "futures", "tokio", @@ -1550,12 +1511,12 @@ dependencies = [ [[package]] name = "cdn-broker" version = "0.1.0" -source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.3.6#569d6a6ec3ccbbd41c041a4c80bd260bc5c928ff" +source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.3.12#7bf490b34fb5b996d7e98d3d4be6d88028a076ac" dependencies = [ "async-std", "cdn-proto", "clap", - "console-subscriber", + "console-subscriber 0.3.0", "dashmap", "derivative", "jf-signature", @@ -1574,7 +1535,7 @@ dependencies = [ [[package]] name = "cdn-client" version = "0.1.0" -source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.3.6#569d6a6ec3ccbbd41c041a4c80bd260bc5c928ff" +source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.3.12#7bf490b34fb5b996d7e98d3d4be6d88028a076ac" dependencies = [ "async-std", "cdn-proto", @@ -1589,7 +1550,7 @@ dependencies = [ [[package]] name = "cdn-marshal" version = "0.1.0" -source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.3.6#569d6a6ec3ccbbd41c041a4c80bd260bc5c928ff" +source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.3.12#7bf490b34fb5b996d7e98d3d4be6d88028a076ac" dependencies = [ "async-std", "cdn-proto", @@ -1603,7 +1564,7 @@ dependencies = [ [[package]] name = "cdn-proto" version = "0.1.0" -source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.3.6#569d6a6ec3ccbbd41c041a4c80bd260bc5c928ff" +source = "git+https://github.com/EspressoSystems/Push-CDN?tag=0.3.12#7bf490b34fb5b996d7e98d3d4be6d88028a076ac" dependencies = [ "anyhow", "ark-serialize", @@ -1618,15 +1579,17 @@ dependencies = [ "num_enum", "pem 3.0.4", "prometheus", - "quinn 0.10.2", + "quinn", "rand 0.8.5", - "rcgen 0.12.1", + "rcgen 0.13.1", "redis", "rkyv", - "rustls 0.21.12", + "rustls 0.23.10", + "rustls-pki-types", "sqlx", "thiserror", "tokio", + "tokio-rustls 0.26.0", "tracing", "url", "warp", @@ -1898,7 +1861,20 @@ dependencies = [ "futures-core", "prost", "prost-types", - "tonic", + "tonic 0.10.2", + "tracing-core", +] + +[[package]] +name = "console-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a257c22cd7e487dd4a13d413beabc512c5052f0bc048db0da6a84c3d8a6142fd" +dependencies = [ + "futures-core", + "prost", + "prost-types", + "tonic 0.11.0", "tracing-core", ] @@ -1908,7 +1884,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7481d4c57092cd1c19dd541b92bdce883de840df30aa5d03fd48a3935c01842e" dependencies = [ - "console-api", + "console-api 0.6.0", "crossbeam-channel", "crossbeam-utils", "futures-task", @@ -1920,7 +1896,32 @@ dependencies = [ "thread_local", "tokio", "tokio-stream", - "tonic", + "tonic 0.10.2", + "tracing", + "tracing-core", + "tracing-subscriber 0.3.18", +] + +[[package]] +name = "console-subscriber" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c4cc54bae66f7d9188996404abdf7fdfa23034ef8e43478c8810828abad758" +dependencies = [ + "console-api 0.7.0", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime", + "prost", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic 0.11.0", "tracing", "tracing-core", "tracing-subscriber 0.3.18", @@ -2422,27 +2423,13 @@ dependencies = [ "zeroize", ] -[[package]] -name = "der-parser" -version = "8.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" -dependencies = [ - "asn1-rs 0.5.2", - "displaydoc", - "nom", - "num-bigint", - "num-traits", - "rusticata-macros", -] - [[package]] name = "der-parser" version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" dependencies = [ - "asn1-rs 0.6.1", + "asn1-rs", "displaydoc", "nom", "num-bigint", @@ -4018,7 +4005,7 @@ dependencies = [ [[package]] name = "hotshot" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#1f28a2e9a54ee11f5559bfd226f90411b3510fc2" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" dependencies = [ "anyhow", "async-broadcast", @@ -4064,7 +4051,7 @@ dependencies = [ [[package]] name = "hotshot-builder-api" version = "0.1.7" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#1f28a2e9a54ee11f5559bfd226f90411b3510fc2" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" dependencies = [ "async-trait", "clap", @@ -4083,7 +4070,7 @@ dependencies = [ [[package]] name = "hotshot-builder-core" version = "0.1.26" -source = "git+https://github.com/EspressoSystems/hotshot-builder-core?branch=daily-build#d52c87f098e1ee0d9114316a742ce61f26f6b01b" +source = "git+https://github.com/EspressoSystems/hotshot-builder-core?tag=rc-0.1.28#0bebe9a3490e55c0563cf96e839dde20205e144e" dependencies = [ "anyhow", "async-broadcast", @@ -4136,7 +4123,7 @@ dependencies = [ [[package]] name = "hotshot-events-service" version = "0.1.27" -source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?branch=daily-build#f54e094d42316ca510275a4b07675516fe4984d6" +source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?tag=rc-0.1.28#3e14bbe20e1a227272e3d79bb52a8ba1ff27a15e" dependencies = [ "async-broadcast", "async-compatibility-layer", @@ -4161,7 +4148,7 @@ dependencies = [ [[package]] name = "hotshot-example-types" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#1f28a2e9a54ee11f5559bfd226f90411b3510fc2" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" dependencies = [ "anyhow", "async-broadcast", @@ -4192,7 +4179,7 @@ dependencies = [ [[package]] name = "hotshot-macros" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#1f28a2e9a54ee11f5559bfd226f90411b3510fc2" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" dependencies = [ "derive_builder", "proc-macro2", @@ -4203,7 +4190,7 @@ dependencies = [ [[package]] name = "hotshot-orchestrator" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#1f28a2e9a54ee11f5559bfd226f90411b3510fc2" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" dependencies = [ "anyhow", "async-compatibility-layer", @@ -4233,7 +4220,7 @@ dependencies = [ [[package]] name = "hotshot-query-service" version = "0.1.39" -source = "git+https://github.com/EspressoSystems/hotshot-query-service?branch=daily-build#ba2a6e1f8917e96f21fdcfddd2eeca45451c1f8c" +source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=rc-0.1.40#0a2481db04df0a2076613546413f592e160ec1be" dependencies = [ "anyhow", "ark-serialize", @@ -4288,7 +4275,7 @@ dependencies = [ [[package]] name = "hotshot-stake-table" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#1f28a2e9a54ee11f5559bfd226f90411b3510fc2" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" dependencies = [ "ark-bn254", "ark-ed-on-bn254", @@ -4360,7 +4347,7 @@ dependencies = [ [[package]] name = "hotshot-task" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#1f28a2e9a54ee11f5559bfd226f90411b3510fc2" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" dependencies = [ "anyhow", "async-broadcast", @@ -4375,7 +4362,7 @@ dependencies = [ [[package]] name = "hotshot-task-impls" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#1f28a2e9a54ee11f5559bfd226f90411b3510fc2" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" dependencies = [ "anyhow", "async-broadcast", @@ -4410,7 +4397,7 @@ dependencies = [ [[package]] name = "hotshot-testing" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#1f28a2e9a54ee11f5559bfd226f90411b3510fc2" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" dependencies = [ "anyhow", "async-broadcast", @@ -4452,7 +4439,7 @@ dependencies = [ [[package]] name = "hotshot-types" version = "0.1.11" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#1f28a2e9a54ee11f5559bfd226f90411b3510fc2" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" dependencies = [ "anyhow", "ark-bls12-381", @@ -4668,7 +4655,7 @@ dependencies = [ "hyper 0.14.29", "rustls 0.21.12", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", ] [[package]] @@ -5836,7 +5823,7 @@ dependencies = [ [[package]] name = "libp2p-networking" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?branch=main#1f28a2e9a54ee11f5559bfd226f90411b3510fc2" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" dependencies = [ "anyhow", "async-compatibility-layer", @@ -5954,7 +5941,7 @@ dependencies = [ "libp2p-identity", "libp2p-tls", "parking_lot", - "quinn 0.11.2", + "quinn", "rand 0.8.5", "ring 0.17.8", "rustls 0.23.10", @@ -6105,7 +6092,7 @@ dependencies = [ "rustls 0.23.10", "rustls-webpki 0.101.7", "thiserror", - "x509-parser 0.16.0", + "x509-parser", "yasna", ] @@ -6401,9 +6388,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.3" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0d8b92cd8358e8d229c11df9358decae64d137c5be540952c5ca7b25aea768" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memoize" @@ -6850,22 +6837,13 @@ dependencies = [ "memchr", ] -[[package]] -name = "oid-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" -dependencies = [ - "asn1-rs 0.5.2", -] - [[package]] name = "oid-registry" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c958dd45046245b9c3c2547369bb634eb461670b2e7e0de552905801a648d1d" dependencies = [ - "asn1-rs 0.6.1", + "asn1-rs", ] [[package]] @@ -7027,7 +7005,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.1", + "redox_syscall 0.5.2", "smallvec", "windows-targets 0.52.5", ] @@ -7611,7 +7589,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.66", @@ -7693,23 +7671,6 @@ dependencies = [ "unsigned-varint 0.8.0", ] -[[package]] -name = "quinn" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cc2c5017e4b43d5995dcea317bc46c1e09404c0a9664d2908f7f02dfe943d75" -dependencies = [ - "bytes 1.6.0", - "pin-project-lite 0.2.14", - "quinn-proto 0.10.6", - "quinn-udp 0.4.1", - "rustc-hash", - "rustls 0.21.12", - "thiserror", - "tokio", - "tracing", -] - [[package]] name = "quinn" version = "0.11.2" @@ -7721,8 +7682,8 @@ dependencies = [ "bytes 1.6.0", "futures-io", "pin-project-lite 0.2.14", - "quinn-proto 0.11.3", - "quinn-udp 0.5.2", + "quinn-proto", + "quinn-udp", "rustc-hash", "rustls 0.23.10", "thiserror", @@ -7730,23 +7691,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "quinn-proto" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "141bf7dfde2fbc246bfd3fe12f2455aa24b0fbd9af535d8c86c7bd1381ff2b1a" -dependencies = [ - "bytes 1.6.0", - "rand 0.8.5", - "ring 0.16.20", - "rustc-hash", - "rustls 0.21.12", - "slab", - "thiserror", - "tinyvec", - "tracing", -] - [[package]] name = "quinn-proto" version = "0.11.3" @@ -7764,19 +7708,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "quinn-udp" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "055b4e778e8feb9f93c4e439f71dc2156ef13360b432b799e179a8c4cdf0b1d7" -dependencies = [ - "bytes 1.6.0", - "libc", - "socket2 0.5.7", - "tracing", - "windows-sys 0.48.0", -] - [[package]] name = "quinn-udp" version = "0.5.2" @@ -7929,22 +7860,23 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.12.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48406db8ac1f3cbc7dcdb56ec355343817958a356ff430259bb07baf7607e1e1" +checksum = "54077e1872c46788540de1ea3d7f4ccb1983d12f9aa909b234468676c1a36779" dependencies = [ "pem 3.0.4", "ring 0.17.8", + "rustls-pki-types", "time 0.3.36", - "x509-parser 0.15.1", + "x509-parser", "yasna", ] [[package]] name = "redis" -version = "0.24.0" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c580d9cbbe1d1b479e8d67cf9daf6a62c957e6846048408b80b43ac3f6af84cd" +checksum = "e0d7a6955c7511f60f3ba9e86c6d02b3c3f144f8c24b288d1f4e18074ab8bbec" dependencies = [ "arc-swap", "async-trait", @@ -7973,9 +7905,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ "bitflags 2.5.0", ] @@ -8120,7 +8052,7 @@ dependencies = [ "sync_wrapper", "system-configuration", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", @@ -10272,6 +10204,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls 0.23.10", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.15" @@ -10293,7 +10236,7 @@ dependencies = [ "log", "rustls 0.21.12", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", "tungstenite 0.20.1", "webpki-roots 0.25.4", ] @@ -10383,6 +10326,33 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.7", + "bytes 1.6.0", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.29", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -11404,36 +11374,19 @@ dependencies = [ "zeroize", ] -[[package]] -name = "x509-parser" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" -dependencies = [ - "asn1-rs 0.5.2", - "data-encoding", - "der-parser 8.2.0", - "lazy_static", - "nom", - "oid-registry 0.6.1", - "ring 0.16.20", - "rusticata-macros", - "thiserror", - "time 0.3.36", -] - [[package]] name = "x509-parser" version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" dependencies = [ - "asn1-rs 0.6.1", + "asn1-rs", "data-encoding", - "der-parser 9.0.0", + "der-parser", "lazy_static", "nom", - "oid-registry 0.7.0", + "oid-registry", + "ring 0.17.8", "rusticata-macros", "thiserror", "time 0.3.36", diff --git a/Cargo.toml b/Cargo.toml index a3f37283c6..13c6449769 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,29 +47,29 @@ dotenvy = "0.15" ethers = { version = "2.0", features = ["solc"] } futures = "0.3" -hotshot = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } +hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.58" } # Hotshot imports -hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", branch = "main" } -hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", branch = "daily-build" } -hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } -hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } +hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "rc-0.5.58" } +hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "rc-0.1.28" } +hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "rc-0.1.28" } +hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.58" } +hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "rc-0.1.40" } +hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.58" } hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" } -hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } -hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } -hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", branch = "main" } +hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.58" } +hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.58" } +hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.58" } hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" } -hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", branch = "daily-build" } -hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", branch = "daily-build" } # Push CDN imports cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", features = [ "runtime-async-std", "global-permits", -], tag = "0.3.6", package = "cdn-broker" } +], tag = "0.3.12", package = "cdn-broker" } cdn-marshal = { git = "https://github.com/EspressoSystems/Push-CDN", features = [ "runtime-async-std", "global-permits", -], tag = "0.3.6", package = "cdn-marshal" } +], tag = "0.3.12", package = "cdn-marshal" } jf-plonk = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.5", features = [ "test-apis", @@ -120,4 +120,4 @@ vergen = { version = "8.3", features = ["git", "gitcl"] } zeroize = "1.7" committable = "0.2" portpicker = "0.1.1" -pretty_assertions = "1.4" +pretty_assertions = "1.4" \ No newline at end of file From 1ac5cf261e9178bc93d2ab37b0eb090c2ab9d788 Mon Sep 17 00:00:00 2001 From: rob-maron <132852777+rob-maron@users.noreply.github.com> Date: Fri, 14 Jun 2024 12:51:57 -0400 Subject: [PATCH 43/48] Update CDN in `ab/fee-upgrade` (#1606) * update the CDN * update comment * fix latency calculation * query service update --- Cargo.lock | 6 ++++-- builder/src/lib.rs | 7 ++----- builder/src/permissioned.rs | 12 ++++-------- sequencer/Cargo.toml | 2 ++ sequencer/src/bin/cdn-broker.rs | 13 +++++++++++++ sequencer/src/bin/cdn-marshal.rs | 14 +++++++++++++- sequencer/src/bin/dev-cdn.rs | 2 ++ sequencer/src/lib.rs | 15 +++++---------- sequencer/src/network/cdn.rs | 32 +++++++++++++++++++++++--------- 9 files changed, 68 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 143fff0878..d5b6fba0d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4220,7 +4220,7 @@ dependencies = [ [[package]] name = "hotshot-query-service" version = "0.1.39" -source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=rc-0.1.40#0a2481db04df0a2076613546413f592e160ec1be" +source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=rc-0.1.40#a41c0a50aef228b1a0ea017bdb4341d06c902725" dependencies = [ "anyhow", "ark-serialize", @@ -7589,7 +7589,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.66", @@ -8712,6 +8712,7 @@ dependencies = [ "jf-vid", "libp2p", "num-traits", + "num_enum", "paste", "portpicker", "pretty_assertions", @@ -8724,6 +8725,7 @@ dependencies = [ "serde_json", "sha2 0.10.8", "snafu 0.8.3", + "static_assertions", "strum", "surf-disco", "tagged-base64", diff --git a/builder/src/lib.rs b/builder/src/lib.rs index b2f62ce92e..883a66e74b 100644 --- a/builder/src/lib.rs +++ b/builder/src/lib.rs @@ -9,9 +9,7 @@ use futures::{ stream::{Stream, StreamExt}, }; use hotshot::{ - traits::{ - election::static_committee::GeneralStaticCommittee, implementations::NetworkingMetricsValue, - }, + traits::election::static_committee::GeneralStaticCommittee, types::{SignatureKey, SystemContextHandle}, HotShotInitializer, Memberships, Networks, SystemContext, }; @@ -379,8 +377,7 @@ pub mod testing { let network = Arc::new(MemoryNetwork::new( config.my_own_validator_config.public_key, - NetworkingMetricsValue::new(metrics), - self.master_map.clone(), + &self.master_map, None, )); let networks = Networks { diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index 30dd571f15..93a5d60731 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -12,8 +12,8 @@ use hotshot::{ traits::{ election::static_committee::GeneralStaticCommittee, implementations::{ - derive_libp2p_peer_id, CombinedNetworks, KeyPair, Libp2pNetwork, - NetworkingMetricsValue, PushCdnNetwork, Topic, WrappedSignatureKey, + derive_libp2p_peer_id, CdnMetricsValue, CombinedNetworks, KeyPair, Libp2pNetwork, + PushCdnNetwork, Topic, WrappedSignatureKey, }, }, types::{SignatureKey, SystemContextHandle}, @@ -209,6 +209,7 @@ pub async fn init_node Result<()> { @@ -111,6 +123,7 @@ async fn main() -> Result<()> { public_advertise_endpoint: args.public_advertise_endpoint, private_bind_endpoint: args.private_bind_endpoint, private_advertise_endpoint: args.private_advertise_endpoint, + global_memory_pool_size: Some(args.global_memory_pool_size), }; // Create new `Broker` diff --git a/sequencer/src/bin/cdn-marshal.rs b/sequencer/src/bin/cdn-marshal.rs index a9646d8b7d..4a5030deb6 100644 --- a/sequencer/src/bin/cdn-marshal.rs +++ b/sequencer/src/bin/cdn-marshal.rs @@ -4,7 +4,7 @@ use anyhow::Result; use cdn_marshal::{Config, Marshal}; use clap::Parser; -use sequencer::{network::cdn::ProductionDef, SeqTypes}; +use sequencer::{network::cdn::ProductionDef, options::parse_size, SeqTypes}; use tracing_subscriber::EnvFilter; #[derive(Parser, Debug)] @@ -38,6 +38,17 @@ struct Args { /// If not provided, a local, pinned CA is used #[arg(long, env = "ESPRESSO_CDN_MARSHAL_CA_KEY_PATH")] ca_key_path: Option, + + /// The size of the global memory pool. This is the maximum number of bytes that + /// can be allocated at once for all connections. A connection will block if it + /// tries to allocate more than this amount until some memory is freed. + #[arg( + long, + default_value = "1GB", + value_parser = parse_size, + env = "ESPRESSO_CDN_MARSHAL_GLOBAL_MEMORY_POOL_SIZE" + )] + global_memory_pool_size: usize, } #[async_std::main] @@ -64,6 +75,7 @@ async fn main() -> Result<()> { metrics_bind_endpoint: args.metrics_bind_endpoint, ca_cert_path: args.ca_cert_path, ca_key_path: args.ca_key_path, + global_memory_pool_size: Some(args.global_memory_pool_size), }; // Create new `Marshal` from the config diff --git a/sequencer/src/bin/dev-cdn.rs b/sequencer/src/bin/dev-cdn.rs index 73def9e41c..064142d713 100644 --- a/sequencer/src/bin/dev-cdn.rs +++ b/sequencer/src/bin/dev-cdn.rs @@ -75,6 +75,7 @@ async fn main() -> Result<()> { ca_cert_path: None, ca_key_path: None, + global_memory_pool_size: Some(1024 * 1024 * 1024), }; // Configure the marshal @@ -84,6 +85,7 @@ async fn main() -> Result<()> { discovery_endpoint: discovery_endpoint.clone(), ca_cert_path: None, ca_key_path: None, + global_memory_pool_size: Some(1024 * 1024 * 1024), }; // Create a new `Broker` diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 6192f8d286..5f6fbb9f64 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -42,8 +42,8 @@ use hotshot::{ traits::{ election::static_committee::GeneralStaticCommittee, implementations::{ - derive_libp2p_peer_id, KeyPair, MemoryNetwork, NetworkingMetricsValue, PushCdnNetwork, - Topic, WrappedSignatureKey, + derive_libp2p_peer_id, CdnMetricsValue, KeyPair, MemoryNetwork, PushCdnNetwork, Topic, + WrappedSignatureKey, }, }, types::SignatureKey, @@ -445,6 +445,7 @@ pub async fn init_node( public_key: WrappedSignatureKey(my_config.public_key), private_key: my_config.private_key.clone(), }, + CdnMetricsValue::new(metrics), ) .with_context(|| "Failed to create CDN network")?; @@ -458,6 +459,7 @@ pub async fn init_node( // We need the private key so we can derive our Libp2p keypair // (using https://docs.rs/blake3/latest/blake3/fn.derive_key.html) &my_config.private_key, + hotshot::traits::implementations::Libp2pMetricsValue::new(metrics), ) .await .with_context(|| "Failed to create libp2p network")?; @@ -503,12 +505,6 @@ pub async fn init_node( _pd: Default::default(), }; - // The web server network doesn't have any metrics. By creating and dropping a - // `NetworkingMetricsValue`, we ensure the networking metrics are created, but just not - // populated, so that monitoring software built to work with network-related metrics doesn't - // crash horribly just because we're not using the P2P network yet. - let _ = NetworkingMetricsValue::new(metrics); - let mut genesis_state = ValidatedState { chain_config: genesis.chain_config.into(), ..Default::default() @@ -789,8 +785,7 @@ pub mod testing { let network = Arc::new(MemoryNetwork::new( config.my_own_validator_config.public_key, - NetworkingMetricsValue::new(metrics), - self.master_map.clone(), + &self.master_map, None, )); let networks = Networks { diff --git a/sequencer/src/network/cdn.rs b/sequencer/src/network/cdn.rs index e0b998e41c..a3e6b7074b 100644 --- a/sequencer/src/network/cdn.rs +++ b/sequencer/src/network/cdn.rs @@ -3,16 +3,33 @@ use std::marker::PhantomData; use bincode::Options; use cdn_broker::reexports::{ - connection::{ - protocols::{Quic, Tcp}, - NoMiddleware, TrustedMiddleware, UntrustedMiddleware, - }, + connection::protocols::{Quic, Tcp}, crypto::signature::{Serializable, SignatureScheme}, - def::{ConnectionDef, RunDef}, + def::{ConnectionDef, RunDef, Topic as TopicTrait}, discovery::{Embedded, Redis}, }; -use hotshot::{traits::implementations::Topic, types::SignatureKey}; +use hotshot::{traits::implementations::Topic as HotShotTopic, types::SignatureKey}; use hotshot_types::{traits::node_implementation::NodeType, utils::bincode_opts}; +use num_enum::{IntoPrimitive, TryFromPrimitive}; +use static_assertions::const_assert_eq; + +/// The enum for the topics we can subscribe to in the Push CDN +#[repr(u8)] +#[derive(IntoPrimitive, TryFromPrimitive, Clone, PartialEq, Eq)] +pub enum Topic { + /// The global topic + Global = 0, + /// The DA topic + Da = 1, +} + +// Make sure the topics are the same as defined in `HotShot`. +const_assert_eq!(Topic::Global as u8, HotShotTopic::Global as u8); +const_assert_eq!(Topic::Da as u8, HotShotTopic::Da as u8); + +/// Implement the `TopicTrait` for our `Topic` enum. This lets us define +/// compatible topics at the broker-level. Others will be rejected. +impl TopicTrait for Topic {} /// A wrapped `SignatureKey`. We need to implement the Push CDN's `SignatureScheme` /// trait in order to sign and verify messages to/from the CDN. @@ -70,7 +87,6 @@ pub struct UserDef(PhantomData); impl ConnectionDef for UserDef { type Scheme = WrappedSignatureKey; type Protocol = Quic; - type Middleware = UntrustedMiddleware; } /// The broker definition for the Push CDN. @@ -79,7 +95,6 @@ pub struct BrokerDef(PhantomData); impl ConnectionDef for BrokerDef { type Scheme = WrappedSignatureKey; type Protocol = Tcp; - type Middleware = TrustedMiddleware; } /// The client definition for the Push CDN. Uses the Quic @@ -90,7 +105,6 @@ pub struct ClientDef(PhantomData); impl ConnectionDef for ClientDef { type Scheme = WrappedSignatureKey; type Protocol = Quic; - type Middleware = NoMiddleware; } /// The testing run definition for the Push CDN. From f6b1d6cff38f86eb7dfebaed43f7b35392c7215e Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 14 Jun 2024 21:56:21 +0500 Subject: [PATCH 44/48] cargo sort --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 13c6449769..ff58fdbc5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,4 +120,4 @@ vergen = { version = "8.3", features = ["git", "gitcl"] } zeroize = "1.7" committable = "0.2" portpicker = "0.1.1" -pretty_assertions = "1.4" \ No newline at end of file +pretty_assertions = "1.4" From 98ad78b470cd38a59bcd5157b1ab5b40049b730c Mon Sep 17 00:00:00 2001 From: Abdul Basit Date: Fri, 14 Jun 2024 22:23:15 +0500 Subject: [PATCH 45/48] add commit check back --- sequencer/src/state.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sequencer/src/state.rs b/sequencer/src/state.rs index 5577fde194..4916461cfb 100644 --- a/sequencer/src/state.rs +++ b/sequencer/src/state.rs @@ -805,8 +805,8 @@ impl ValidatedState { let cf = match (state_cf.resolve(), header_cf.resolve()) { (Some(cf), _) => cf, - (_, Some(cf)) => cf, - (None, None) => { + (_, Some(cf)) if cf.commit() == state_cf.commit() => cf, + (_, Some(_)) | (None, None) => { instance .peers .as_ref() From 257727b7489af2f92a53c2d84188433af175a68e Mon Sep 17 00:00:00 2001 From: Rob Date: Fri, 14 Jun 2024 14:13:30 -0400 Subject: [PATCH 46/48] `try_cast` --- sequencer/src/bin/cdn-broker.rs | 15 ++++++++++++--- sequencer/src/bin/cdn-marshal.rs | 13 +++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/sequencer/src/bin/cdn-broker.rs b/sequencer/src/bin/cdn-broker.rs index 9c6e5e93dd..0e6266261d 100644 --- a/sequencer/src/bin/cdn-broker.rs +++ b/sequencer/src/bin/cdn-broker.rs @@ -1,6 +1,6 @@ //! The following is the main `Broker` binary, which just instantiates and runs //! a `Broker` object. -use anyhow::Result; +use anyhow::{Context, Result}; use cdn_broker::reexports::crypto::signature::KeyPair; use cdn_broker::{Broker, Config}; use clap::Parser; @@ -83,7 +83,7 @@ struct Args { value_parser = parse_size, env = "ESPRESSO_CDN_BROKER_GLOBAL_MEMORY_POOL_SIZE" )] - global_memory_pool_size: usize, + global_memory_pool_size: u64, } #[async_std::main] async fn main() -> Result<()> { @@ -107,6 +107,15 @@ async fn main() -> Result<()> { let (public_key, private_key) = ::SignatureKey::generated_from_seed_indexed(key_hash.into(), 1337); + // Cast the memory pool size to a `usize` + let global_memory_pool_size = + usize::try_from(args.global_memory_pool_size).with_context(|| { + format!( + "Failed to convert global memory pool size to usize: {}", + args.global_memory_pool_size + ) + })?; + // Create config let broker_config: Config> = Config { ca_cert_path: args.ca_cert_path, @@ -123,7 +132,7 @@ async fn main() -> Result<()> { public_advertise_endpoint: args.public_advertise_endpoint, private_bind_endpoint: args.private_bind_endpoint, private_advertise_endpoint: args.private_advertise_endpoint, - global_memory_pool_size: Some(args.global_memory_pool_size), + global_memory_pool_size: Some(global_memory_pool_size), }; // Create new `Broker` diff --git a/sequencer/src/bin/cdn-marshal.rs b/sequencer/src/bin/cdn-marshal.rs index 4a5030deb6..549faa1b8b 100644 --- a/sequencer/src/bin/cdn-marshal.rs +++ b/sequencer/src/bin/cdn-marshal.rs @@ -1,7 +1,7 @@ //! The following is the main `Marshal` binary, which just instantiates and runs //! a `Marshal` object. -use anyhow::Result; +use anyhow::{Context, Result}; use cdn_marshal::{Config, Marshal}; use clap::Parser; use sequencer::{network::cdn::ProductionDef, options::parse_size, SeqTypes}; @@ -68,6 +68,15 @@ async fn main() -> Result<()> { .init(); } + // Cast the memory pool size to a `usize` + let global_memory_pool_size = + usize::try_from(args.global_memory_pool_size).with_context(|| { + format!( + "Failed to convert global memory pool size to usize: {}", + args.global_memory_pool_size + ) + })?; + // Create a new `Config` let config = Config { discovery_endpoint: args.discovery_endpoint, @@ -75,7 +84,7 @@ async fn main() -> Result<()> { metrics_bind_endpoint: args.metrics_bind_endpoint, ca_cert_path: args.ca_cert_path, ca_key_path: args.ca_key_path, - global_memory_pool_size: Some(args.global_memory_pool_size), + global_memory_pool_size: Some(global_memory_pool_size), }; // Create new `Marshal` from the config From 8d9d7a7d90a8565264ecb88e633422055a5447fc Mon Sep 17 00:00:00 2001 From: Rob Date: Fri, 14 Jun 2024 14:16:26 -0400 Subject: [PATCH 47/48] clippy --- sequencer/src/bin/cdn-marshal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequencer/src/bin/cdn-marshal.rs b/sequencer/src/bin/cdn-marshal.rs index 549faa1b8b..7b15f23789 100644 --- a/sequencer/src/bin/cdn-marshal.rs +++ b/sequencer/src/bin/cdn-marshal.rs @@ -48,7 +48,7 @@ struct Args { value_parser = parse_size, env = "ESPRESSO_CDN_MARSHAL_GLOBAL_MEMORY_POOL_SIZE" )] - global_memory_pool_size: usize, + global_memory_pool_size: u64, } #[async_std::main] From 57e9cfbff2a9d7007c7c005a40ce6197835eb441 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Fri, 14 Jun 2024 12:50:50 -0600 Subject: [PATCH 48/48] bump --- Cargo.lock | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5b6fba0d6..3819b7e396 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4005,7 +4005,7 @@ dependencies = [ [[package]] name = "hotshot" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#9b642d35e88db3083dbbe1ab6e2aa8bd32ac68ea" dependencies = [ "anyhow", "async-broadcast", @@ -4051,7 +4051,7 @@ dependencies = [ [[package]] name = "hotshot-builder-api" version = "0.1.7" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#9b642d35e88db3083dbbe1ab6e2aa8bd32ac68ea" dependencies = [ "async-trait", "clap", @@ -4070,7 +4070,7 @@ dependencies = [ [[package]] name = "hotshot-builder-core" version = "0.1.26" -source = "git+https://github.com/EspressoSystems/hotshot-builder-core?tag=rc-0.1.28#0bebe9a3490e55c0563cf96e839dde20205e144e" +source = "git+https://github.com/EspressoSystems/hotshot-builder-core?tag=rc-0.1.28#ba3b264d1cfe534f11742038578520dac44d0f6d" dependencies = [ "anyhow", "async-broadcast", @@ -4123,7 +4123,7 @@ dependencies = [ [[package]] name = "hotshot-events-service" version = "0.1.27" -source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?tag=rc-0.1.28#3e14bbe20e1a227272e3d79bb52a8ba1ff27a15e" +source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?tag=rc-0.1.28#7bec1bd3740418914f6c07f237b4b22adab3fe0e" dependencies = [ "async-broadcast", "async-compatibility-layer", @@ -4148,7 +4148,7 @@ dependencies = [ [[package]] name = "hotshot-example-types" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#9b642d35e88db3083dbbe1ab6e2aa8bd32ac68ea" dependencies = [ "anyhow", "async-broadcast", @@ -4179,7 +4179,7 @@ dependencies = [ [[package]] name = "hotshot-macros" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#9b642d35e88db3083dbbe1ab6e2aa8bd32ac68ea" dependencies = [ "derive_builder", "proc-macro2", @@ -4190,7 +4190,7 @@ dependencies = [ [[package]] name = "hotshot-orchestrator" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#9b642d35e88db3083dbbe1ab6e2aa8bd32ac68ea" dependencies = [ "anyhow", "async-compatibility-layer", @@ -4220,7 +4220,7 @@ dependencies = [ [[package]] name = "hotshot-query-service" version = "0.1.39" -source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=rc-0.1.40#a41c0a50aef228b1a0ea017bdb4341d06c902725" +source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=rc-0.1.40#38b7c5f8e27d76768024990ca9109988468699e2" dependencies = [ "anyhow", "ark-serialize", @@ -4275,7 +4275,7 @@ dependencies = [ [[package]] name = "hotshot-stake-table" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#9b642d35e88db3083dbbe1ab6e2aa8bd32ac68ea" dependencies = [ "ark-bn254", "ark-ed-on-bn254", @@ -4347,7 +4347,7 @@ dependencies = [ [[package]] name = "hotshot-task" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#9b642d35e88db3083dbbe1ab6e2aa8bd32ac68ea" dependencies = [ "anyhow", "async-broadcast", @@ -4362,7 +4362,7 @@ dependencies = [ [[package]] name = "hotshot-task-impls" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#9b642d35e88db3083dbbe1ab6e2aa8bd32ac68ea" dependencies = [ "anyhow", "async-broadcast", @@ -4397,7 +4397,7 @@ dependencies = [ [[package]] name = "hotshot-testing" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#9b642d35e88db3083dbbe1ab6e2aa8bd32ac68ea" dependencies = [ "anyhow", "async-broadcast", @@ -4439,7 +4439,7 @@ dependencies = [ [[package]] name = "hotshot-types" version = "0.1.11" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#9b642d35e88db3083dbbe1ab6e2aa8bd32ac68ea" dependencies = [ "anyhow", "ark-bls12-381", @@ -5823,7 +5823,7 @@ dependencies = [ [[package]] name = "libp2p-networking" version = "0.5.57" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#855ab7dfb60ae0de8e7c5174e11b2d589f16833a" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.58#9b642d35e88db3083dbbe1ab6e2aa8bd32ac68ea" dependencies = [ "anyhow", "async-compatibility-layer", @@ -7589,7 +7589,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.66",