From 3bcc5cf971c7c79418b0ed48f2dd9ab72c9eaece Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Tue, 4 Jun 2024 08:32:05 -0600 Subject: [PATCH 01/14] merge main, rebuild, update --- Cargo.lock | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a22c24c944..2259c75e08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1117,7 +1117,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.29", "itoa", "matchit", "memchr", @@ -3433,9 +3433,9 @@ dependencies = [ [[package]] name = "futures-bounded" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e2774cc104e198ef3d3e1ff4ab40f86fa3245d6cb6a3a46174f21463cee173" +checksum = "91f328e7fb845fc832912fb6a34f40cf6d1888c92f974d1893a54e97b5ff542e" dependencies = [ "futures-timer", "futures-util", @@ -4598,9 +4598,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" dependencies = [ "bytes 1.6.0", "futures-channel", @@ -4648,7 +4648,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.28", + "hyper 0.14.29", "rustls 0.21.12", "tokio", "tokio-rustls", @@ -4660,7 +4660,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.28", + "hyper 0.14.29", "pin-project-lite 0.2.14", "tokio", "tokio-io-timeout", @@ -4792,7 +4792,7 @@ dependencies = [ "bytes 1.6.0", "futures", "http 0.2.12", - "hyper 0.14.28", + "hyper 0.14.29", "log", "rand 0.8.5", "tokio", @@ -7937,7 +7937,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.29", "hyper-rustls", "ipnet", "js-sys", @@ -10129,14 +10129,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.13" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e43f8cc456c9704c851ae29c67e17ef65d2c30017c17a9765b89c382dc8bba" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.13", + "toml_edit 0.22.14", ] [[package]] @@ -10161,15 +10161,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.13" +version = "0.22.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c127785850e8c20836d49732ae6abfa47616e60bf9d9f57c43c250361a9db96c" +checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" dependencies = [ "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.9", + "winnow 0.6.11", ] [[package]] @@ -10186,7 +10186,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.29", "hyper-timeout", "percent-encoding", "pin-project", @@ -10661,9 +10661,9 @@ dependencies = [ [[package]] name = "vbs" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1da07d9ca0c69892851ff70256392d1b0c6408a0007cc87409b1ca3064821e9b" +checksum = "6a9d83afb293d109ae418271619046cd9d87d63ba83e473e25a5e36101edc453" dependencies = [ "anyhow", "bincode", @@ -10738,7 +10738,7 @@ dependencies = [ "futures-util", "headers", "http 0.2.12", - "hyper 0.14.28", + "hyper 0.14.29", "log", "mime", "mime_guess", @@ -11120,9 +11120,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.9" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86c949fede1d13936a99f14fafd3e76fd642b556dd2ce96287fbe2e0151bfac6" +checksum = "56c52728401e1dc672a56e81e593e912aa54c78f40246869f78359a2bf24d29d" dependencies = [ "memchr", ] From 2779c6e0b995c424d9b6213cba419b214eb4d136 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Thu, 6 Jun 2024 13:50:31 -0600 Subject: [PATCH 02/14] update --- Cargo.lock | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2259c75e08..9cf34f39c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -202,9 +202,9 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" dependencies = [ "windows-sys 0.52.0", ] @@ -2874,9 +2874,9 @@ dependencies = [ [[package]] name = "escargot" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f474c6844cbd04e783d0f25757583db4f491770ca618bedf2fb01815fc79939" +checksum = "650eb5f6eeda986377996e9ed570cbc20cc16d30440696f82f129c863e4e3e83" dependencies = [ "log", "once_cell", @@ -5878,9 +5878,9 @@ dependencies = [ [[package]] name = "libp2p-request-response" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6946e5456240b3173187cc37a17cb40c3cd1f7138c76e2c773e0d792a42a8de1" +checksum = "c314fe28368da5e3a262553fb0ad575c1c8934c461e10de10265551478163836" dependencies = [ "async-trait", "cbor4ii", @@ -9478,9 +9478,9 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.26.3" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7993a8e3a9e88a00351486baae9522c91b123a088f76469e5bd5cc17198ea87" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -10169,7 +10169,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.11", + "winnow 0.6.12", ] [[package]] @@ -10568,7 +10568,7 @@ dependencies = [ "rustls-pki-types", "rustls-webpki 0.102.4", "url", - "webpki-roots 0.26.1", + "webpki-roots 0.26.2", ] [[package]] @@ -10887,9 +10887,9 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "webpki-roots" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +checksum = "3c452ad30530b54a4d8e71952716a212b08efd0f3562baa66c29a618b07da7c3" dependencies = [ "rustls-pki-types", ] @@ -11120,9 +11120,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c52728401e1dc672a56e81e593e912aa54c78f40246869f78359a2bf24d29d" +checksum = "41ff33f391015ecab21cd092389215eb265ef9496a9a07b6bee7d3529831deda" dependencies = [ "memchr", ] From d1d9c668ac1c668b3257f5da596118ae3e0b13e7 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Fri, 7 Jun 2024 09:26:46 -0600 Subject: [PATCH 03/14] wip implementing storage --- Cargo.lock | 158 +++++++++++++++++++++++--------- Cargo.toml | 20 ++-- sequencer/src/lib.rs | 9 +- sequencer/src/persistence.rs | 20 +++- sequencer/src/persistence/fs.rs | 72 ++++++++++++++- 5 files changed, 223 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed3dc0e799..805244d733 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1385,9 +1385,9 @@ dependencies = [ "serde", "snafu 0.8.3", "surf", - "surf-disco", + "surf-disco 0.7.0", "tagged-base64", - "tide-disco", + "tide-disco 0.7.0", "tracing", "url", "vbs", @@ -4005,8 +4005,8 @@ dependencies = [ [[package]] name = "hotshot" -version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.56#1138cfeafc061839494391eb52a46d0259008cf0" +version = "0.5.56" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.57#828f71864bec2600a2740e24c86468c5a485585b" dependencies = [ "anyhow", "async-broadcast", @@ -4042,7 +4042,7 @@ dependencies = [ "serde", "sha2 0.10.8", "snafu 0.8.3", - "surf-disco", + "surf-disco 0.8.0", "time 0.3.36", "tokio", "tracing", @@ -4052,7 +4052,7 @@ dependencies = [ [[package]] name = "hotshot-builder-api" version = "0.1.7" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.56#1138cfeafc061839494391eb52a46d0259008cf0" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.57#828f71864bec2600a2740e24c86468c5a485585b" dependencies = [ "async-trait", "clap", @@ -4063,15 +4063,15 @@ dependencies = [ "serde", "snafu 0.8.3", "tagged-base64", - "tide-disco", + "tide-disco 0.8.0", "toml", "vbs", ] [[package]] name = "hotshot-builder-core" -version = "0.1.25" -source = "git+https://github.com/EspressoSystems/hotshot-builder-core?tag=0.1.25#0862a0a1335ca07e1b5bb3d619c1cd996a886382" +version = "0.1.26" +source = "git+https://github.com/EspressoSystems/hotshot-builder-core?tag=0.1.26#aa38257cc17f90745fd5e5caa899b76f42968c26" dependencies = [ "anyhow", "async-broadcast", @@ -4090,9 +4090,9 @@ dependencies = [ "serde", "sha2 0.10.8", "snafu 0.8.3", - "surf-disco", + "surf-disco 0.7.0", "tagged-base64", - "tide-disco", + "tide-disco 0.7.0", "tokio", "tracing", "vbs", @@ -4123,8 +4123,8 @@ dependencies = [ [[package]] name = "hotshot-events-service" -version = "0.1.25" -source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?tag=0.1.25#964c175d9145dcde354cbaf407f26f05976c5348" +version = "0.1.26" +source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?tag=0.1.26#cb0b05ae81c6c03a059d06a19aaa208b630fb76a" dependencies = [ "async-broadcast", "async-compatibility-layer", @@ -4140,7 +4140,7 @@ dependencies = [ "serde", "snafu 0.8.3", "tagged-base64", - "tide-disco", + "tide-disco 0.7.0", "toml", "tracing", "vbs", @@ -4148,8 +4148,8 @@ dependencies = [ [[package]] name = "hotshot-example-types" -version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.56#1138cfeafc061839494391eb52a46d0259008cf0" +version = "0.5.56" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.57#828f71864bec2600a2740e24c86468c5a485585b" dependencies = [ "anyhow", "async-broadcast", @@ -4179,8 +4179,8 @@ dependencies = [ [[package]] name = "hotshot-macros" -version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.56#1138cfeafc061839494391eb52a46d0259008cf0" +version = "0.5.56" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.57#828f71864bec2600a2740e24c86468c5a485585b" dependencies = [ "derive_builder", "proc-macro2", @@ -4190,8 +4190,8 @@ dependencies = [ [[package]] name = "hotshot-orchestrator" -version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.56#1138cfeafc061839494391eb52a46d0259008cf0" +version = "0.5.56" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.57#828f71864bec2600a2740e24c86468c5a485585b" dependencies = [ "anyhow", "async-compatibility-layer", @@ -4208,9 +4208,9 @@ dependencies = [ "serde", "serde-inline-default", "serde_json", - "surf-disco", + "surf-disco 0.8.0", "thiserror", - "tide-disco", + "tide-disco 0.8.0", "tokio", "toml", "tracing", @@ -4220,7 +4220,7 @@ dependencies = [ [[package]] name = "hotshot-query-service" version = "0.1.34" -source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=0.1.34#7b363e18965303bbe719a53aec757dbadeee23b7" +source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=0.1.35#8c6ccb20cbd7afe817292371414a07695e973bf4" dependencies = [ "anyhow", "ark-serialize", @@ -4257,10 +4257,10 @@ dependencies = [ "serde_json", "snafu 0.8.3", "spin_sleep", - "surf-disco", + "surf-disco 0.7.0", "tagged-base64", "tempfile", - "tide-disco", + "tide-disco 0.7.0", "time 0.3.36", "tokio", "tokio-postgres", @@ -4273,8 +4273,8 @@ dependencies = [ [[package]] name = "hotshot-stake-table" -version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.56#1138cfeafc061839494391eb52a46d0259008cf0" +version = "0.5.56" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.57#828f71864bec2600a2740e24c86468c5a485585b" dependencies = [ "ark-bn254", "ark-ed-on-bn254", @@ -4332,9 +4332,9 @@ dependencies = [ "sequencer-utils", "serde", "snafu 0.8.3", - "surf-disco", + "surf-disco 0.7.0", "tagged-base64", - "tide-disco", + "tide-disco 0.7.0", "time 0.3.36", "toml", "tracing", @@ -4344,8 +4344,8 @@ dependencies = [ [[package]] name = "hotshot-task" -version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.56#1138cfeafc061839494391eb52a46d0259008cf0" +version = "0.5.56" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.57#828f71864bec2600a2740e24c86468c5a485585b" dependencies = [ "anyhow", "async-broadcast", @@ -4359,8 +4359,8 @@ dependencies = [ [[package]] name = "hotshot-task-impls" -version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.56#1138cfeafc061839494391eb52a46d0259008cf0" +version = "0.5.56" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.57#828f71864bec2600a2740e24c86468c5a485585b" dependencies = [ "anyhow", "async-broadcast", @@ -4384,7 +4384,7 @@ dependencies = [ "serde", "sha2 0.10.8", "snafu 0.8.3", - "surf-disco", + "surf-disco 0.8.0", "tagged-base64", "time 0.3.36", "tokio", @@ -4394,8 +4394,8 @@ dependencies = [ [[package]] name = "hotshot-testing" -version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.56#1138cfeafc061839494391eb52a46d0259008cf0" +version = "0.5.56" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.57#828f71864bec2600a2740e24c86468c5a485585b" dependencies = [ "anyhow", "async-broadcast", @@ -4427,7 +4427,7 @@ dependencies = [ "sha3", "snafu 0.8.3", "tagged-base64", - "tide-disco", + "tide-disco 0.8.0", "tokio", "tracing", "vbs", @@ -4436,7 +4436,7 @@ dependencies = [ [[package]] name = "hotshot-types" version = "0.1.11" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.56#1138cfeafc061839494391eb52a46d0259008cf0" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.57#828f71864bec2600a2740e24c86468c5a485585b" dependencies = [ "anyhow", "ark-bls12-381", @@ -5698,8 +5698,8 @@ dependencies = [ [[package]] name = "libp2p-networking" -version = "0.5.43" -source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.56#1138cfeafc061839494391eb52a46d0259008cf0" +version = "0.5.56" +source = "git+https://github.com/EspressoSystems/hotshot?tag=rc-0.5.57#828f71864bec2600a2740e24c86468c5a485585b" dependencies = [ "anyhow", "async-compatibility-layer", @@ -8625,10 +8625,10 @@ dependencies = [ "sha2 0.10.8", "snafu 0.8.3", "strum", - "surf-disco", + "surf-disco 0.7.0", "tagged-base64", "tempfile", - "tide-disco", + "tide-disco 0.7.0", "time 0.3.36", "tokio-postgres", "toml", @@ -9533,7 +9533,26 @@ dependencies = [ "reqwest 0.12.4", "serde", "serde_json", - "tide-disco", + "tide-disco 0.7.0", + "tracing", + "vbs", +] + +[[package]] +name = "surf-disco" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55c871338479a1c27b5e2ae15e5ae0fe70769306d2ef5a0c2d30896a09ac8a3c" +dependencies = [ + "async-std", + "async-tungstenite", + "derivative", + "futures", + "hex", + "reqwest 0.12.4", + "serde", + "serde_json", + "tide-disco 0.8.0", "tracing", "vbs", ] @@ -9878,6 +9897,59 @@ dependencies = [ "vbs", ] +[[package]] +name = "tide-disco" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715558234f09576333f92ee769bd160e6c4bb1d156d25613cf62be9a592f291a" +dependencies = [ + "anyhow", + "async-h1", + "async-lock 3.4.0", + "async-std", + "async-trait", + "clap", + "config", + "derivative", + "derive_more", + "dirs", + "edit-distance", + "futures", + "futures-util", + "http 1.1.0", + "include_dir", + "itertools 0.12.1", + "lazy_static", + "libc", + "markdown", + "maud", + "parking_lot", + "prometheus", + "reqwest 0.12.4", + "routefinder", + "semver 1.0.23", + "serde", + "serde_json", + "serde_with", + "shellexpand", + "signal-hook", + "signal-hook-async-std", + "snafu 0.8.3", + "strum", + "strum_macros", + "tagged-base64", + "tide", + "tide-websockets", + "toml", + "tracing", + "tracing-distributed", + "tracing-futures", + "tracing-log", + "tracing-subscriber 0.3.18", + "url", + "vbs", +] + [[package]] name = "tide-websockets" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index ff09ad58c0..d7d2721d32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,18 +47,18 @@ dotenvy = "0.15" ethers = { version = "2.0", features = ["solc"] } futures = "0.3" -hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" } +hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" } # Hotshot imports -hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "rc-0.5.56" } -hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.25" } -hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.25" } -hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" } -hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.34" } -hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" } +hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "rc-0.5.57" } +hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.26" } +hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.26" } +hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" } +hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.35" } +hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" } hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" } -hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" } -hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" } -hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.56" } +hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" } +hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" } +hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" } hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" } # Push CDN imports diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 2346945ba7..750bf85155 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -54,7 +54,7 @@ use hotshot_orchestrator::{ }; use hotshot_types::{ consensus::CommitmentMap, - data::{DaProposal, VidDisperseShare, ViewNumber}, + data::{DaProposal, QuorumProposal, VidDisperseShare, ViewNumber}, event::HotShotAction, light_client::{StateKeyPair, StateSignKey}, message::Proposal, @@ -162,6 +162,13 @@ impl Storage for Arc> { .update_undecided_state(leaves, state) .await } + + async fn append_proposal( + &self, + proposal: &Proposal>, + ) -> anyhow::Result<()> { + self.write().await.append_quorum_proposal(proposal).await + } } #[derive(Debug, Clone)] diff --git a/sequencer/src/persistence.rs b/sequencer/src/persistence.rs index c8a9f50766..10aa1d3ff7 100644 --- a/sequencer/src/persistence.rs +++ b/sequencer/src/persistence.rs @@ -20,7 +20,7 @@ use hotshot::{ }; use hotshot_types::{ consensus::CommitmentMap, - data::{DaProposal, VidDisperseShare}, + data::{DaProposal, QuorumProposal, VidDisperseShare}, event::{HotShotAction, LeafInfo}, message::Proposal, simple_certificate::QuorumCertificate, @@ -87,6 +87,11 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { &self, ) -> anyhow::Result, BTreeMap>)>>; + /// Load the proposals saved by consensus + async fn load_quorum_proposals( + &self, + ) -> anyhow::Result>>>>; + async fn load_vid_share( &self, view: ViewNumber, @@ -168,6 +173,13 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { .context("loading undecided state")? .unwrap_or_default(); + let saved_proposals = self + .load_quorum_proposals() + .await + .context("loading saved proposals") + .unwrap_or_default() + .unwrap_or_default(); + tracing::info!( ?leaf, ?view, @@ -175,6 +187,7 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { ?validated_state, ?undecided_leaves, ?undecided_state, + ?saved_proposals, "loaded consensus state" ); Ok(HotShotInitializer::from_reload( @@ -182,6 +195,7 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { state, validated_state, view, + saved_proposals, high_qc, undecided_leaves.into_values().collect(), undecided_state, @@ -233,6 +247,10 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { leaves: CommitmentMap, state: BTreeMap>, ) -> anyhow::Result<()>; + async fn append_quorum_proposal( + &mut self, + proposal: &Proposal>, + ) -> anyhow::Result<()>; } #[cfg(test)] diff --git a/sequencer/src/persistence/fs.rs b/sequencer/src/persistence/fs.rs index 03aa995da9..24420787d3 100644 --- a/sequencer/src/persistence/fs.rs +++ b/sequencer/src/persistence/fs.rs @@ -6,7 +6,7 @@ use clap::Parser; use hotshot_types::{ consensus::CommitmentMap, - data::{DaProposal, VidDisperseShare}, + data::{DaProposal, QuorumProposal, VidDisperseShare}, event::HotShotAction, message::Proposal, simple_certificate::QuorumCertificate, @@ -99,6 +99,10 @@ impl Persistence { self.path.join("undecided_state") } + fn proposal_dir_path(&self) -> PathBuf { + self.path.join("proposal") + } + /// Overwrite a file if a condition is met. /// /// The file at `path`, if it exists, is opened in read mode and passed to `pred`. If `pred` @@ -408,6 +412,72 @@ impl SequencerPersistence for Persistence { }, ) } + async fn append_quorum_proposal( + &mut self, + proposal: &Proposal>, + ) -> anyhow::Result<()> { + let view_number = proposal.data.view_number().u64(); + let dir_path = self.proposal_dir_path(); + + fs::create_dir_all(dir_path.clone()).context("failed to create proposals dir")?; + + let file_path = dir_path.join(view_number.to_string()).with_extension("txt"); + self.replace( + &file_path, + |_| { + // Don't overwrite the file, we want to append since we can get multiple + // saved quorum proposals. + Ok(false) + }, + |mut file| { + let proposal_bytes = bincode::serialize(&proposal).context("serialize proposal")?; + + file.write_all(&proposal_bytes)?; + Ok(()) + }, + ) + } + async fn load_quorum_proposals( + &self, + ) -> anyhow::Result>>>> + { + // First, get the proposal directory. + let dir_path = self.proposal_dir_path(); + + // Then, we want to get the entries in this directory since they'll be the + // key/value pairs for our map. + let files: Vec = fs::read_dir(dir_path)? + .filter_map(|entry| { + entry + .ok() + .filter(|e| e.file_type().map(|ft| ft.is_file()).unwrap_or(false)) + }) + .collect(); + + // Do we have any entries? + if files.is_empty() { + // Don't both continuing if we don't have any data. + return Ok(None); + } + + // Read all of the files + let proposal_files = files + .into_iter() + .map(|entry| dir_path.join(entry.file_name()).with_extension("txt")); + + for file in proposal_files.into_iter() { + if let Some(file_name) = file.file_name() { + let view_number = ViewNumber::new( + file_name + .to_string_lossy() + .parse::() + .context("convert file name to u64")?, + ); + } + } + + Ok(None) + } } #[cfg(test)] From a39381f54b73189cbb43334b3e5e51725dd4068b Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Fri, 7 Jun 2024 10:03:51 -0600 Subject: [PATCH 04/14] almost done --- sequencer/src/persistence/fs.rs | 28 ++++++++++++---- sequencer/src/persistence/no_storage.rs | 15 ++++++++- sequencer/src/persistence/sql.rs | 44 ++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 9 deletions(-) diff --git a/sequencer/src/persistence/fs.rs b/sequencer/src/persistence/fs.rs index 24420787d3..a3cae2093f 100644 --- a/sequencer/src/persistence/fs.rs +++ b/sequencer/src/persistence/fs.rs @@ -99,8 +99,8 @@ impl Persistence { self.path.join("undecided_state") } - fn proposal_dir_path(&self) -> PathBuf { - self.path.join("proposal") + fn proposals_dir_path(&self) -> PathBuf { + self.path.join("proposals") } /// Overwrite a file if a condition is met. @@ -417,7 +417,7 @@ impl SequencerPersistence for Persistence { proposal: &Proposal>, ) -> anyhow::Result<()> { let view_number = proposal.data.view_number().u64(); - let dir_path = self.proposal_dir_path(); + let dir_path = self.proposals_dir_path(); fs::create_dir_all(dir_path.clone()).context("failed to create proposals dir")?; @@ -442,11 +442,11 @@ impl SequencerPersistence for Persistence { ) -> anyhow::Result>>>> { // First, get the proposal directory. - let dir_path = self.proposal_dir_path(); + let dir_path = self.proposals_dir_path(); // Then, we want to get the entries in this directory since they'll be the // key/value pairs for our map. - let files: Vec = fs::read_dir(dir_path)? + let files: Vec = fs::read_dir(dir_path.clone())? .filter_map(|entry| { entry .ok() @@ -465,18 +465,32 @@ impl SequencerPersistence for Persistence { .into_iter() .map(|entry| dir_path.join(entry.file_name()).with_extension("txt")); + let mut map = BTreeMap::new(); for file in proposal_files.into_iter() { - if let Some(file_name) = file.file_name() { + // This operation shouldn't fail, but we don't want to panic here if the filesystem + // somehow gets corrupted. We get the stem to remove the ".txt" from the end. + if let Some(file_name) = file.file_stem() { + // We need to convert the filename (which corresponds to the view) let view_number = ViewNumber::new( file_name .to_string_lossy() .parse::() .context("convert file name to u64")?, ); + + // Now, we'll try and load the proposal associated with this function. + let proposal_bytes = fs::read(file)?; + + // Then, deserialize. + let proposal: Proposal> = + bincode::deserialize(&proposal_bytes)?; + + // Push to the map and we're done. + map.insert(view_number, proposal); } } - Ok(None) + Ok(Some(map)) } } diff --git a/sequencer/src/persistence/no_storage.rs b/sequencer/src/persistence/no_storage.rs index 315e82d1a5..9a39296ebd 100644 --- a/sequencer/src/persistence/no_storage.rs +++ b/sequencer/src/persistence/no_storage.rs @@ -6,7 +6,7 @@ use crate::{Leaf, SeqTypes, ViewNumber}; use async_trait::async_trait; use hotshot_types::{ consensus::CommitmentMap, - data::{DaProposal, VidDisperseShare}, + data::{DaProposal, QuorumProposal, VidDisperseShare}, event::HotShotAction, message::Proposal, simple_certificate::QuorumCertificate, @@ -85,6 +85,13 @@ impl SequencerPersistence for NoStorage { Ok(None) } + async fn load_quorum_proposals( + &self, + ) -> anyhow::Result>>>> + { + Ok(None) + } + async fn append_vid( &mut self, _proposal: &Proposal>, @@ -111,4 +118,10 @@ impl SequencerPersistence for NoStorage { ) -> anyhow::Result<()> { Ok(()) } + async fn append_quorum_proposal( + &mut self, + _proposal: &Proposal>, + ) -> anyhow::Result<()> { + Ok(()) + } } diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index 818258292a..c3dbe66a59 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -19,7 +19,7 @@ use hotshot_query_service::data_source::{ }; use hotshot_types::{ consensus::CommitmentMap, - data::{DaProposal, VidDisperseShare}, + data::{DaProposal, QuorumProposal, VidDisperseShare}, event::HotShotAction, message::Proposal, simple_certificate::QuorumCertificate, @@ -452,6 +452,27 @@ impl SequencerPersistence for Persistence { .transpose() } + async fn load_quorum_proposals( + &self, + ) -> anyhow::Result>>>> + { + let result = self + .db + .query_static("SELECT * FROM quorum_proposals") + .await?; + + let entry_pairs = result + .map(|row| { + let view_number = row.get("view") as u64; + let bytes: Vec = row.get("proposal"); + let proposal: Proposal> = + bincode::deserialize(&bytes)?; + Ok((ViewNumber::new(view_number), proposal)) + }) + .transpose() + .collect::>)>>(); + } + async fn append_vid( &mut self, proposal: &Proposal>, @@ -548,6 +569,27 @@ impl SequencerPersistence for Persistence { }) .await } + async fn append_quorum_proposal( + &mut self, + proposal: &Proposal>, + ) -> anyhow::Result<()> { + let view_number = proposal.data.view_number().u64(); + let proposal_bytes = bincode::serialize(&proposal).context("serializing proposal")?; + transaction(self, |mut tx| { + async move { + tx.upsert( + "quorum_proposals", + ["view", "proposal"], + ["view"], + [[sql_param(&(view_number as i64)), sql_param(&proposal_bytes)]], + ) + .await?; + Ok(()) + } + .boxed() + }) + .await + } } fn sql_param(param: &T) -> &(dyn ToSql + Sync) { From 79b6c24309ca195546a3f9240cdb4017803f0439 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Fri, 7 Jun 2024 10:34:17 -0600 Subject: [PATCH 05/14] broken query --- sequencer/src/persistence/sql.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index c3dbe66a59..16f06c4fac 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -9,7 +9,10 @@ use async_std::sync::{Arc, RwLock}; use async_trait::async_trait; use clap::Parser; use derivative::Derivative; -use futures::future::{BoxFuture, FutureExt}; +use futures::{ + future::{BoxFuture, FutureExt}, + StreamExt, +}; use hotshot_query_service::data_source::{ storage::{ pruning::PrunerCfg, @@ -456,21 +459,23 @@ impl SequencerPersistence for Persistence { &self, ) -> anyhow::Result>>>> { - let result = self + let rows = self .db .query_static("SELECT * FROM quorum_proposals") .await?; - let entry_pairs = result + let entry_pairs = rows .map(|row| { - let view_number = row.get("view") as u64; + let row = row?; + let view: i64 = row.get("view"); + let view_number: ViewNumber = ViewNumber::new(view.try_into()?); let bytes: Vec = row.get("proposal"); let proposal: Proposal> = bincode::deserialize(&bytes)?; - Ok((ViewNumber::new(view_number), proposal)) + Ok((view_number, proposal)) }) - .transpose() - .collect::>)>>(); + .collect::>>(); + Ok(None) } async fn append_vid( From 2567226cb46950aa4ee676e706367f7d94628331 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Fri, 7 Jun 2024 11:57:35 -0600 Subject: [PATCH 06/14] integrate new versions --- Cargo.lock | 129 +++--------------- Cargo.toml | 10 +- sequencer/src/api/endpoints.rs | 14 +- sequencer/src/hotshot_commitment.rs | 2 +- sequencer/src/persistence/sql.rs | 19 +-- sequencer/src/state_signature/relay_server.rs | 6 +- 6 files changed, 48 insertions(+), 132 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 805244d733..e9767d9cfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1385,9 +1385,9 @@ dependencies = [ "serde", "snafu 0.8.3", "surf", - "surf-disco 0.7.0", + "surf-disco", "tagged-base64", - "tide-disco 0.7.0", + "tide-disco", "tracing", "url", "vbs", @@ -4042,7 +4042,7 @@ dependencies = [ "serde", "sha2 0.10.8", "snafu 0.8.3", - "surf-disco 0.8.0", + "surf-disco", "time 0.3.36", "tokio", "tracing", @@ -4063,7 +4063,7 @@ dependencies = [ "serde", "snafu 0.8.3", "tagged-base64", - "tide-disco 0.8.0", + "tide-disco", "toml", "vbs", ] @@ -4071,7 +4071,7 @@ dependencies = [ [[package]] name = "hotshot-builder-core" version = "0.1.26" -source = "git+https://github.com/EspressoSystems/hotshot-builder-core?tag=0.1.26#aa38257cc17f90745fd5e5caa899b76f42968c26" +source = "git+https://github.com/EspressoSystems/hotshot-builder-core?tag=0.1.27#592ddf5f807a01835d40b3f1ea1f8484906ba7d1" dependencies = [ "anyhow", "async-broadcast", @@ -4090,9 +4090,9 @@ dependencies = [ "serde", "sha2 0.10.8", "snafu 0.8.3", - "surf-disco 0.7.0", + "surf-disco", "tagged-base64", - "tide-disco 0.7.0", + "tide-disco", "tokio", "tracing", "vbs", @@ -4124,7 +4124,7 @@ dependencies = [ [[package]] name = "hotshot-events-service" version = "0.1.26" -source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?tag=0.1.26#cb0b05ae81c6c03a059d06a19aaa208b630fb76a" +source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?tag=0.1.27#e4703477c687be50dea4faa4235aafa67a068cd4" dependencies = [ "async-broadcast", "async-compatibility-layer", @@ -4140,7 +4140,7 @@ dependencies = [ "serde", "snafu 0.8.3", "tagged-base64", - "tide-disco 0.7.0", + "tide-disco", "toml", "tracing", "vbs", @@ -4208,9 +4208,9 @@ dependencies = [ "serde", "serde-inline-default", "serde_json", - "surf-disco 0.8.0", + "surf-disco", "thiserror", - "tide-disco 0.8.0", + "tide-disco", "tokio", "toml", "tracing", @@ -4219,8 +4219,8 @@ dependencies = [ [[package]] name = "hotshot-query-service" -version = "0.1.34" -source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=0.1.35#8c6ccb20cbd7afe817292371414a07695e973bf4" +version = "0.1.36" +source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=0.1.36#225db2352dba0c38fa07f7d2788a5b15b1a0bca4" dependencies = [ "anyhow", "ark-serialize", @@ -4257,10 +4257,10 @@ dependencies = [ "serde_json", "snafu 0.8.3", "spin_sleep", - "surf-disco 0.7.0", + "surf-disco", "tagged-base64", "tempfile", - "tide-disco 0.7.0", + "tide-disco", "time 0.3.36", "tokio", "tokio-postgres", @@ -4332,9 +4332,9 @@ dependencies = [ "sequencer-utils", "serde", "snafu 0.8.3", - "surf-disco 0.7.0", + "surf-disco", "tagged-base64", - "tide-disco 0.7.0", + "tide-disco", "time 0.3.36", "toml", "tracing", @@ -4384,7 +4384,7 @@ dependencies = [ "serde", "sha2 0.10.8", "snafu 0.8.3", - "surf-disco 0.8.0", + "surf-disco", "tagged-base64", "time 0.3.36", "tokio", @@ -4427,7 +4427,7 @@ dependencies = [ "sha3", "snafu 0.8.3", "tagged-base64", - "tide-disco 0.8.0", + "tide-disco", "tokio", "tracing", "vbs", @@ -6594,17 +6594,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" -[[package]] -name = "num-derive" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "num-integer" version = "0.1.46" @@ -8625,10 +8614,10 @@ dependencies = [ "sha2 0.10.8", "snafu 0.8.3", "strum", - "surf-disco 0.7.0", + "surf-disco", "tagged-base64", "tempfile", - "tide-disco 0.7.0", + "tide-disco", "time 0.3.36", "tokio-postgres", "toml", @@ -9519,25 +9508,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "surf-disco" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e4318abbbbc4d640c6cb45d29cd3c70cf517c751fae9b68f2f76a6ff1950da7" -dependencies = [ - "async-std", - "async-tungstenite", - "derivative", - "futures", - "hex", - "reqwest 0.12.4", - "serde", - "serde_json", - "tide-disco 0.7.0", - "tracing", - "vbs", -] - [[package]] name = "surf-disco" version = "0.8.0" @@ -9552,7 +9522,7 @@ dependencies = [ "reqwest 0.12.4", "serde", "serde_json", - "tide-disco 0.8.0", + "tide-disco", "tracing", "vbs", ] @@ -9842,61 +9812,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "tide-disco" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c9d803bc734a5cced767b36bb1caab87e9eb7eb144cffcab38160b084a8cef" -dependencies = [ - "anyhow", - "async-h1", - "async-lock 3.4.0", - "async-std", - "async-trait", - "clap", - "config", - "derivative", - "derive_more", - "dirs", - "edit-distance", - "futures", - "futures-util", - "http 1.1.0", - "include_dir", - "itertools 0.12.1", - "lazy_static", - "libc", - "markdown", - "maud", - "num-derive", - "num-traits", - "parking_lot", - "prometheus", - "reqwest 0.12.4", - "routefinder", - "semver 1.0.23", - "serde", - "serde_json", - "serde_with", - "shellexpand", - "signal-hook", - "signal-hook-async-std", - "snafu 0.8.3", - "strum", - "strum_macros", - "tagged-base64", - "tide", - "tide-websockets", - "toml", - "tracing", - "tracing-distributed", - "tracing-futures", - "tracing-log", - "tracing-subscriber 0.3.18", - "url", - "vbs", -] - [[package]] name = "tide-disco" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index d7d2721d32..95beef04c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,10 +50,10 @@ futures = "0.3" hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" } # Hotshot imports hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "rc-0.5.57" } -hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.26" } -hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.26" } +hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.27" } +hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.27" } hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" } -hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.35" } +hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.36" } hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" } hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" } hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" } @@ -99,9 +99,9 @@ jf-relation = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4 jf-utils = { git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.5" } snafu = "0.8" strum = { version = "0.26", features = ["derive"] } -surf-disco = "0.7" +surf-disco = "0.8" tagged-base64 = "0.4" -tide-disco = "0.7" +tide-disco = "0.8" time = "0.3" tracing = "0.1" bytesize = "1.3" diff --git a/sequencer/src/api/endpoints.rs b/sequencer/src/api/endpoints.rs index 9980933939..ccb073c3e7 100644 --- a/sequencer/src/api/endpoints.rs +++ b/sequencer/src/api/endpoints.rs @@ -108,7 +108,7 @@ where ) .context(CustomSnafu { message: format!("failed to make proof for namespace {ns_id}"), - status: StatusCode::NotFound, + status: StatusCode::NOT_FOUND, })?; let transactions = if let NamespaceProof::Existence { @@ -184,7 +184,7 @@ where if tx.namespace() > NamespaceId::from(u32::MAX as u64) { return Err(Error::Custom { message: "Transaction namespace > u32::MAX".to_string(), - status: StatusCode::BadRequest, + status: StatusCode::BAD_REQUEST, }); } @@ -221,7 +221,7 @@ where .get_state_signature(height) .await .ok_or(tide_disco::Error::catch_all( - StatusCode::NotFound, + StatusCode::NOT_FOUND, "Signature not found.".to_owned(), )) } @@ -252,7 +252,7 @@ where .map_err(Error::from_request_error)?; let account = account.parse().map_err(|err| { Error::catch_all( - StatusCode::BadRequest, + StatusCode::BAD_REQUEST, format!("malformed account {account}: {err}"), ) })?; @@ -260,7 +260,7 @@ where state .get_account(height, ViewNumber::new(view), account) .await - .map_err(|err| Error::catch_all(StatusCode::NotFound, format!("{err:#}"))) + .map_err(|err| Error::catch_all(StatusCode::NOT_FOUND, format!("{err:#}"))) } .boxed() })? @@ -276,7 +276,7 @@ where state .get_frontier(height, ViewNumber::new(view)) .await - .map_err(|err| Error::catch_all(StatusCode::NotFound, format!("{err:#}"))) + .map_err(|err| Error::catch_all(StatusCode::NOT_FOUND, format!("{err:#}"))) } .boxed() })?; @@ -314,7 +314,7 @@ where let mut api = Api::::new(toml)?; let env_variables = get_public_env_vars() - .map_err(|err| Error::catch_all(StatusCode::InternalServerError, format!("{err:#}")))?; + .map_err(|err| Error::catch_all(StatusCode::INTERNAL_SERVER_ERROR, format!("{err:#}")))?; api.get("hotshot", |_, state| { async move { Ok(state.get_config().await) }.boxed() diff --git a/sequencer/src/hotshot_commitment.rs b/sequencer/src/hotshot_commitment.rs index 798ac3f456..cf4759d252 100644 --- a/sequencer/src/hotshot_commitment.rs +++ b/sequencer/src/hotshot_commitment.rs @@ -320,7 +320,7 @@ mod test { .flatten() .ok_or_else(|| { Self::Error::catch_all( - StatusCode::NotFound, + StatusCode::NOT_FOUND, format!("no leaf for height {height}"), ) }) diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index 16f06c4fac..2e047cf4a5 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -5,14 +5,14 @@ use crate::{ Leaf, SeqTypes, ViewNumber, }; use anyhow::Context; -use async_std::sync::{Arc, RwLock}; +use async_std::{ + stream::StreamExt, + sync::{Arc, RwLock}, +}; use async_trait::async_trait; use clap::Parser; use derivative::Derivative; -use futures::{ - future::{BoxFuture, FutureExt}, - StreamExt, -}; +use futures::future::{BoxFuture, FutureExt}; use hotshot_query_service::data_source::{ storage::{ pruning::PrunerCfg, @@ -464,8 +464,8 @@ impl SequencerPersistence for Persistence { .query_static("SELECT * FROM quorum_proposals") .await?; - let entry_pairs = rows - .map(|row| { + Ok(Some(BTreeMap::from_iter( + rows.map(|row| { let row = row?; let view: i64 = row.get("view"); let view_number: ViewNumber = ViewNumber::new(view.try_into()?); @@ -474,8 +474,9 @@ impl SequencerPersistence for Persistence { bincode::deserialize(&bytes)?; Ok((view_number, proposal)) }) - .collect::>>(); - Ok(None) + .collect::>>() + .await?, + ))) } async fn append_vid( diff --git a/sequencer/src/state_signature/relay_server.rs b/sequencer/src/state_signature/relay_server.rs index 4dd0263f89..9c78c7c568 100644 --- a/sequencer/src/state_signature/relay_server.rs +++ b/sequencer/src/state_signature/relay_server.rs @@ -88,7 +88,7 @@ impl StateRelayServerDataSource for StateRelayServerState { match &self.latest_available_bundle { Some(bundle) => Ok(bundle.clone()), None => Err(tide_disco::error::ServerError::catch_all( - StatusCode::NotFound, + StatusCode::NOT_FOUND, "The light client state signatures are not ready.".to_owned(), )), } @@ -115,7 +115,7 @@ impl StateRelayServerDataSource for StateRelayServerState { let state_msg: [FieldType; 7] = (&state).into(); if StateSignatureScheme::verify(&(), &key, state_msg, &signature).is_err() { return Err(tide_disco::error::ServerError::catch_all( - StatusCode::BadRequest, + StatusCode::BAD_REQUEST, "The posted signature is not valid.".to_owned(), )); } @@ -141,7 +141,7 @@ impl StateRelayServerDataSource for StateRelayServerState { std::collections::hash_map::Entry::Occupied(_) => { // A signature is already posted for this key with this state return Err(tide_disco::error::ServerError::catch_all( - StatusCode::BadRequest, + StatusCode::BAD_REQUEST, "A signature of this light client state is already posted at this block height for this key.".to_owned(), )); } From d0499f385da48185a5b0ec4e4985165b9e140e2f Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Fri, 7 Jun 2024 12:06:05 -0600 Subject: [PATCH 07/14] bump query service --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5055f5c92e..7a25780e03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4219,8 +4219,8 @@ dependencies = [ [[package]] name = "hotshot-query-service" -version = "0.1.36" -source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=0.1.36#225db2352dba0c38fa07f7d2788a5b15b1a0bca4" +version = "0.1.37" +source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=0.1.37#1907a0dda66ed02a28097c07ea7445c1b7eca88a" dependencies = [ "anyhow", "ark-serialize", diff --git a/Cargo.toml b/Cargo.toml index 7d1bca7a25..4d174d024c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.27" } hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.27" } hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" } -hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.36" } +hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.37" } hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" } hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" } hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.57" } From 4f7f831fd758fe2e02a282b24c68efe690cdfe21 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Fri, 7 Jun 2024 14:01:07 -0600 Subject: [PATCH 08/14] add migration, change name --- sequencer/api/migrations/V32__saved_proposals.sql | 4 ++++ sequencer/src/persistence/sql.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 sequencer/api/migrations/V32__saved_proposals.sql diff --git a/sequencer/api/migrations/V32__saved_proposals.sql b/sequencer/api/migrations/V32__saved_proposals.sql new file mode 100644 index 0000000000..7c782f99d8 --- /dev/null +++ b/sequencer/api/migrations/V32__saved_proposals.sql @@ -0,0 +1,4 @@ +CREATE TABLE quorum_proposals ( + view BIGINT PRIMARY KEY, + data BYTEA +); \ No newline at end of file diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index 2e047cf4a5..697f850e5b 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -585,7 +585,7 @@ impl SequencerPersistence for Persistence { async move { tx.upsert( "quorum_proposals", - ["view", "proposal"], + ["view", "data"], ["view"], [[sql_param(&(view_number as i64)), sql_param(&proposal_bytes)]], ) From 3f3c54c00221cc864508319eab41fd91f3d5e6f3 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Fri, 7 Jun 2024 14:05:14 -0600 Subject: [PATCH 09/14] improve fs interface --- sequencer/src/persistence/fs.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sequencer/src/persistence/fs.rs b/sequencer/src/persistence/fs.rs index a3cae2093f..140ea118b3 100644 --- a/sequencer/src/persistence/fs.rs +++ b/sequencer/src/persistence/fs.rs @@ -99,8 +99,8 @@ impl Persistence { self.path.join("undecided_state") } - fn proposals_dir_path(&self) -> PathBuf { - self.path.join("proposals") + fn quorum_proposals_dir_path(&self) -> PathBuf { + self.path.join("quorum_proposals") } /// Overwrite a file if a condition is met. @@ -191,7 +191,8 @@ impl SequencerPersistence for Persistence { }; delete_files(self.da_dir_path())?; - delete_files(self.vid_dir_path()) + delete_files(self.vid_dir_path())?; + delete_files(self.quorum_proposals_dir_path()) } async fn load_latest_acted_view(&self) -> anyhow::Result> { @@ -417,7 +418,7 @@ impl SequencerPersistence for Persistence { proposal: &Proposal>, ) -> anyhow::Result<()> { let view_number = proposal.data.view_number().u64(); - let dir_path = self.proposals_dir_path(); + let dir_path = self.quorum_proposals_dir_path(); fs::create_dir_all(dir_path.clone()).context("failed to create proposals dir")?; @@ -425,9 +426,8 @@ impl SequencerPersistence for Persistence { self.replace( &file_path, |_| { - // Don't overwrite the file, we want to append since we can get multiple - // saved quorum proposals. - Ok(false) + // Always overwrite the previous file + Ok(true) }, |mut file| { let proposal_bytes = bincode::serialize(&proposal).context("serialize proposal")?; @@ -442,7 +442,7 @@ impl SequencerPersistence for Persistence { ) -> anyhow::Result>>>> { // First, get the proposal directory. - let dir_path = self.proposals_dir_path(); + let dir_path = self.quorum_proposals_dir_path(); // Then, we want to get the entries in this directory since they'll be the // key/value pairs for our map. From f9e517755000c5b453f2a9094b6332b873e67c6f Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Fri, 7 Jun 2024 14:26:37 -0600 Subject: [PATCH 10/14] add sql gc --- sequencer/src/persistence/sql.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index 697f850e5b..3e183b3402 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -309,6 +309,9 @@ impl SequencerPersistence for Persistence { let stmt2 = "DELETE FROM da_proposal where view <= $1"; tx.execute(stmt2, [&(view.u64() as i64)]).await?; + + let stmt3 = "DELETE FROM quorum_proposals where view <= $1"; + tx.execute(stmt3, [&(view.u64() as i64)]).await?; Ok(()) } .boxed() From 6b2baaf82895f7d49d57e3ee301afc28bb90cdc5 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Fri, 7 Jun 2024 14:41:02 -0600 Subject: [PATCH 11/14] update tests to include quorum proposal saving and loadin --- sequencer/src/persistence.rs | 89 ++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/sequencer/src/persistence.rs b/sequencer/src/persistence.rs index 10aa1d3ff7..8c083d53f4 100644 --- a/sequencer/src/persistence.rs +++ b/sequencer/src/persistence.rs @@ -386,6 +386,7 @@ mod persistence_tests { let bytes = payload.encode().to_vec(); let disperse = vid_scheme(2).disperse(bytes).unwrap(); let (pubkey, privkey) = BLSPubKey::generated_from_seed_indexed([0; 32], 1); + let signature = PubKey::sign(&privkey, &[]).unwrap(); let mut vid = VidDisperseShare:: { view_number: ViewNumber::new(1), payload_commitment: Default::default(), @@ -393,6 +394,21 @@ mod persistence_tests { common: disperse.common, recipient_key: pubkey, }; + let mut quorum_proposal = Proposal { + data: QuorumProposal:: { + block_header: leaf.block_header().clone(), + view_number: ViewNumber::genesis(), + justify_qc: QuorumCertificate::genesis( + &ValidatedState::default(), + &NodeState::mock(), + ) + .await, + upgrade_certificate: None, + proposal_certificate: None, + }, + signature, + _pd: Default::default(), + }; let vid_share1 = vid.clone().to_proposal(&privkey).unwrap().clone(); @@ -475,6 +491,70 @@ mod persistence_tests { Some(da_proposal3.clone()) ); + let quorum_proposal1 = quorum_proposal.clone(); + storage + .append_quorum_proposal(&quorum_proposal1) + .await + .unwrap(); + + assert_eq!( + storage.load_quorum_proposals().await.unwrap(), + Some(BTreeMap::from_iter([( + ViewNumber::genesis(), + quorum_proposal1.clone() + )])) + ); + + quorum_proposal.data.view_number = ViewNumber::new(1); + let quorum_proposal2 = quorum_proposal.clone(); + storage + .append_quorum_proposal(&quorum_proposal2) + .await + .unwrap(); + + assert_eq!( + storage.load_quorum_proposals().await.unwrap(), + Some(BTreeMap::from_iter([ + (ViewNumber::genesis(), quorum_proposal1.clone()), + (ViewNumber::new(1), quorum_proposal2.clone()) + ])) + ); + + quorum_proposal.data.view_number = ViewNumber::new(2); + let quorum_proposal3 = quorum_proposal.clone(); + storage + .append_quorum_proposal(&quorum_proposal3) + .await + .unwrap(); + + assert_eq!( + storage.load_quorum_proposals().await.unwrap(), + Some(BTreeMap::from_iter([ + (ViewNumber::genesis(), quorum_proposal1.clone()), + (ViewNumber::new(1), quorum_proposal2.clone()), + (ViewNumber::new(2), quorum_proposal3.clone()) + ])) + ); + + quorum_proposal.data.view_number = ViewNumber::new(10); + + // This one should stick around after GC runs. + let quorum_proposal4 = quorum_proposal.clone(); + storage + .append_quorum_proposal(&quorum_proposal3) + .await + .unwrap(); + + assert_eq!( + storage.load_quorum_proposals().await.unwrap(), + Some(BTreeMap::from_iter([ + (ViewNumber::genesis(), quorum_proposal1), + (ViewNumber::new(1), quorum_proposal2), + (ViewNumber::new(2), quorum_proposal3), + (ViewNumber::new(10), quorum_proposal4.clone()) + ])) + ); + // Test garbage collection // Deleting da proposals and vid shares with view number <=2 storage.collect_garbage(ViewNumber::new(2)).await.unwrap(); @@ -500,5 +580,14 @@ mod persistence_tests { storage.load_vid_share(ViewNumber::new(3)).await.unwrap(), Some(vid_share3) ); + + let proposals = storage.load_quorum_proposals().await.unwrap(); + assert_eq!( + proposals, + Some(BTreeMap::from_iter([( + ViewNumber::new(10), + quorum_proposal4 + )])) + ) } } From 4de78d4d41ddc3d32fae93e9c0781e9b2545bfa3 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Fri, 7 Jun 2024 14:41:25 -0600 Subject: [PATCH 12/14] typo --- sequencer/src/persistence/sql.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index 3e183b3402..39319e64ca 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -472,7 +472,7 @@ impl SequencerPersistence for Persistence { let row = row?; let view: i64 = row.get("view"); let view_number: ViewNumber = ViewNumber::new(view.try_into()?); - let bytes: Vec = row.get("proposal"); + let bytes: Vec = row.get("data"); let proposal: Proposal> = bincode::deserialize(&bytes)?; Ok((view_number, proposal)) From fd4208d4b0e441bfecb460771429158524dddbf4 Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Fri, 7 Jun 2024 14:43:14 -0600 Subject: [PATCH 13/14] typo --- sequencer/src/persistence.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequencer/src/persistence.rs b/sequencer/src/persistence.rs index 8c083d53f4..b78106396b 100644 --- a/sequencer/src/persistence.rs +++ b/sequencer/src/persistence.rs @@ -541,7 +541,7 @@ mod persistence_tests { // This one should stick around after GC runs. let quorum_proposal4 = quorum_proposal.clone(); storage - .append_quorum_proposal(&quorum_proposal3) + .append_quorum_proposal(&quorum_proposal4) .await .unwrap(); From dbf62ace374efc3a3755516be1968c132ba4eebd Mon Sep 17 00:00:00 2001 From: Jarred Parr Date: Fri, 7 Jun 2024 14:45:31 -0600 Subject: [PATCH 14/14] typo --- sequencer/src/persistence/fs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequencer/src/persistence/fs.rs b/sequencer/src/persistence/fs.rs index 140ea118b3..a238a3da1d 100644 --- a/sequencer/src/persistence/fs.rs +++ b/sequencer/src/persistence/fs.rs @@ -456,7 +456,7 @@ impl SequencerPersistence for Persistence { // Do we have any entries? if files.is_empty() { - // Don't both continuing if we don't have any data. + // Don't bother continuing if we don't have any data. return Ok(None); }