Merge pull request #2130 from subspace/auditing-refactoring

Auditing refactoring

crates/pallet-subspace/src/mock.rs

@@ -24,6 +24,7 @@ use crate::{
 use frame_support::parameter_types;
 use frame_support::traits::{ConstU128, ConstU16, ConstU32, ConstU64, OnInitialize};
 use futures::executor::block_on;
+use futures::{FutureExt, StreamExt};
 use rand::Rng;
 use schnorrkel::Keypair;
 use sp_consensus_slots::Slot;
@@ -54,7 +55,7 @@ use subspace_core_primitives::{
     Solution, SolutionRange, REWARD_SIGNING_CONTEXT,
 };
 use subspace_erasure_coding::ErasureCoding;
-use subspace_farmer_components::auditing::audit_sector;
+use subspace_farmer_components::auditing::audit_sector_sync;
 use subspace_farmer_components::plotting::{
     plot_sector, PieceGetterRetryPolicy, PlotSectorOptions,
 };
@@ -478,9 +479,8 @@ pub fn create_signed_vote(
             .derive_global_randomness()
             .derive_global_challenge(slot.into());
 
-        let maybe_audit_result = audit_sector(
+        let maybe_audit_result = audit_sector_sync(
             &public_key,
-            sector_index,
             &global_challenge,
             vote_solution_range,
             &plotted_sector_bytes,
@@ -497,8 +497,12 @@ pub fn create_signed_vote(
             .into_solutions(&reward_address, kzg, erasure_coding, |seed: &PosSeed| {
                 table_generator.generate_parallel(seed)
             })
+            .now_or_never()
+            .unwrap()
             .unwrap()
             .next()
+            .now_or_never()
+            .unwrap()
             .unwrap()
             .unwrap();
 

crates/sp-lightclient/src/tests.rs

@@ -197,7 +197,6 @@ fn valid_header(
 
         let maybe_solution_candidates = audit_sector(
             &public_key,
-            sector_index,
             &global_challenge,
             SolutionRange::MAX,
             &plotted_sector_bytes,

crates/subspace-farmer-components/Cargo.toml

@@ -25,6 +25,7 @@ futures = "0.3.28"
 hex = "0.4.3"
 libc = "0.2.146"
 parity-scale-codec = "3.6.5"
+pin-project = "1.1.3"
 rand = "0.8.5"
 rayon = "1.8.0"
 schnorrkel = "0.9.1"
@@ -45,6 +46,7 @@ winapi = "0.3.9"
 [dev-dependencies]
 criterion = "0.5.1"
 futures = "0.3.28"
+parking_lot = "0.12.1"
 subspace-archiving = { version = "0.1.0", path = "../subspace-archiving" }
 subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space" }
 

crates/subspace-farmer-components/benches/auditing.rs

@@ -4,25 +4,23 @@ use rand::prelude::*;
 use std::fs::OpenOptions;
 use std::io::Write;
 use std::num::{NonZeroU64, NonZeroUsize};
-use std::time::Instant;
-use std::{env, fs};
+use std::{env, fs, slice};
 use subspace_archiving::archiver::Archiver;
 use subspace_core_primitives::crypto::kzg;
 use subspace_core_primitives::crypto::kzg::Kzg;
 use subspace_core_primitives::{
-    Blake3Hash, HistorySize, PublicKey, Record, RecordedHistorySegment, SectorId, SectorIndex,
-    SolutionRange,
+    Blake3Hash, HistorySize, PublicKey, Record, RecordedHistorySegment, SectorId, SolutionRange,
 };
 use subspace_erasure_coding::ErasureCoding;
-use subspace_farmer_components::auditing::audit_sector;
+use subspace_farmer_components::auditing::audit_plot_sync;
 use subspace_farmer_components::file_ext::{FileExt, OpenOptionsExt};
 use subspace_farmer_components::plotting::{
     plot_sector, PieceGetterRetryPolicy, PlotSectorOptions, PlottedSector,
 };
 use subspace_farmer_components::sector::{
     sector_size, SectorContentsMap, SectorMetadata, SectorMetadataChecksummed,
 };
-use subspace_farmer_components::{FarmerProtocolInfo, ReadAt};
+use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_proof_of_space::chia::ChiaTable;
 use subspace_proof_of_space::Table;
 
@@ -45,7 +43,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         .map(|sectors_count| sectors_count.parse().unwrap())
         .unwrap_or(10);
 
-    let public_key = PublicKey::default();
+    let public_key = &PublicKey::default();
     let sector_index = 0;
     let mut input = RecordedHistorySegment::new_boxed();
     StdRng::seed_from_u64(42).fill(AsMut::<[u8]>::as_mut(input.as_mut()));
@@ -78,7 +76,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         ),
         min_sector_lifetime: HistorySize::from(NonZeroU64::new(4).unwrap()),
     };
-    let global_challenge = Blake3Hash::default();
+    let global_challenge = &Blake3Hash::default();
     let solution_range = SolutionRange::MAX;
 
     let sector_size = sector_size(pieces_in_sector);
@@ -120,7 +118,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         let mut plotted_sector_metadata_bytes = Vec::new();
 
         let plotted_sector = block_on(plot_sector::<PosTable, _>(PlotSectorOptions {
-            public_key: &public_key,
+            public_key,
             sector_index,
             piece_getter: &archived_history_segment,
             piece_getter_retry_policy: PieceGetterRetryPolicy::default(),
@@ -151,16 +149,16 @@ pub fn criterion_benchmark(c: &mut Criterion) {
 
     let mut group = c.benchmark_group("auditing");
     group.throughput(Throughput::Elements(1));
-    group.bench_function("memory", |b| {
-        b.iter(|| {
-            audit_sector(
-                black_box(&public_key),
-                black_box(sector_index),
-                black_box(&global_challenge),
+    group.bench_function("memory/sync", |b| {
+        b.iter(|| async {
+            black_box(audit_plot_sync(
+                black_box(public_key),
+                black_box(global_challenge),
                 black_box(solution_range),
                 black_box(&plotted_sector_bytes),
-                black_box(&plotted_sector.sector_metadata),
-            );
+                black_box(slice::from_ref(&plotted_sector.sector_metadata)),
+                black_box(None),
+            ));
         })
     });
 
@@ -188,24 +186,21 @@ pub fn criterion_benchmark(c: &mut Criterion) {
                 .unwrap();
         }
 
+        let sectors_metadata = (0..sectors_count)
+            .map(|_| plotted_sector.sector_metadata.clone())
+            .collect::<Vec<_>>();
+
         group.throughput(Throughput::Elements(sectors_count));
-        group.bench_function("disk", |b| {
-            b.iter_custom(|iters| {
-                let start = Instant::now();
-                for _i in 0..iters {
-                    for sector_index in 0..sectors_count as usize {
-                        let sector = plot_file.offset(sector_index * sector_size);
-                        audit_sector(
-                            black_box(&public_key),
-                            black_box(sector_index as SectorIndex),
-                            black_box(&global_challenge),
-                            black_box(solution_range),
-                            black_box(&sector),
-                            black_box(&plotted_sector.sector_metadata),
-                        );
-                    }
-                }
-                start.elapsed()
+        group.bench_function("disk/sync", |b| {
+            b.iter(|| {
+                black_box(audit_plot_sync(
+                    black_box(public_key),
+                    black_box(global_challenge),
+                    black_box(solution_range),
+                    black_box(&plot_file),
+                    black_box(&sectors_metadata),
+                    black_box(None),
+                ));
             });
         });