diff --git a/.github/workflows/iroha2-profiling-image.yml b/.github/workflows/iroha2-profiling-image.yml index 57b47eca93f..2f14c841cca 100644 --- a/.github/workflows/iroha2-profiling-image.yml +++ b/.github/workflows/iroha2-profiling-image.yml @@ -16,7 +16,13 @@ on: default: profiling IROHA2_RUSTFLAGS: required: false - default: -C force-frame-pointers=on --cfg wasm_profiling + default: -C force-frame-pointers=on + IROHA2_FEATURES: + required: false + default: profiling + IROHA2_CARGOFLAGS: + required: false + default: -Z build-std jobs: registry: @@ -51,6 +57,8 @@ jobs: build-args: | "PROFILE=${{ github.event.inputs.IROHA2_PROFILE }}" "RUSTFLAGS=${{ github.event.inputs.IROHA2_RUSTFLAGS }}" + "FEATURES=${{ github.event.inputs.IROHA2_FEATURES }}" + "CARGOFLAGS=${{ github.event.inputs.IROHA2_CARGOFLAGS }}" file: ${{ github.event.inputs.IROHA2_DOCKERFILE }} # This context specification is required context: . diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a8ec2028ffd..5ef9c16a5ec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -272,10 +272,10 @@ tokio-console http://127.0.0.1:5555 To optimize performance it's useful to profile iroha. -To do that you should compile iroha with `profiling` profile and with `wasm_profiling` compilation flag: +To do that you should compile iroha with `profiling` profile and with `profiling` feature: ```bash -RUSTFLAGS="-C force-frame-pointers=on --cfg wasm_profiling" cargo build --profile profiling +RUSTFLAGS="-C force-frame-pointers=on" cargo +nightly -Z build-std build --target your-desired-target --profile profiling --features profiling ``` Then start iroha and attach profiler of your choice to the iroha pid. @@ -283,7 +283,7 @@ Then start iroha and attach profiler of your choice to the iroha pid. Alternatively it's possible to build iroha inside docker with profiler support and profile iroha this way. ```bash -docker build -f Dockerfile.glibc --build-arg="PROFILE=profiling" --build-arg='RUSTFLAGS=-C force-frame-pointers=on --cfg wasm_profiling' -t iroha2:profiling . +docker build -f Dockerfile.glibc --build-arg="PROFILE=profiling" --build-arg='RUSTFLAGS=-C force-frame-pointers=on' --build-arg='FEATURES=profiling' --build-arg='CARGOFLAGS=-Z build-std' -t iroha2:profiling . ``` E.g. using perf (available only on linux): @@ -303,6 +303,15 @@ It can be done by running: cargo run --bin iroha_wasm_builder_cli -- build ./path/to/executor --outfile executor.wasm ``` +With profiling feature enabled iroha exposes endpoint to scrap pprof profiles: + +```bash +# profile iroha for 30 seconds and get protobuf profile +curl host:port/debug/pprof/profile?seconds=30 -o profile.pb +# analyze profile in browser (required installed go) +go tool pprof -web profile.pb +``` + ## Style Guides diff --git a/Cargo.lock b/Cargo.lock index fa430e31109..4b76a0de150 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -859,6 +859,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "cpp_demangle" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpufeatures" version = "0.2.11" @@ -1569,6 +1578,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -3276,6 +3297,7 @@ dependencies = [ "iroha_torii_derive", "iroha_version", "parity-scale-codec", + "pprof", "serde", "serde_json", "thiserror", @@ -3819,6 +3841,17 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -4275,6 +4308,27 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "pprof" +version = "0.13.0" +source = "git+https://github.com/Erigara/pprof-rs?branch=fix_pointer_align#55fa41916b9bb7f2029643e26168556efda19333" +dependencies = [ + "backtrace", + "cfg-if", + "findshlibs", + "libc", + "log", + "nix", + "once_cell", + "parking_lot", + "protobuf", + "protobuf-codegen-pure", + "smallvec", + "symbolic-demangle", + "tempfile", + "thiserror", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -4407,6 +4461,31 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + +[[package]] +name = "protobuf-codegen" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "033460afb75cf755fcfc16dfaed20b86468082a2ea24e05ac35ab4a099a017d6" +dependencies = [ + "protobuf", +] + +[[package]] +name = "protobuf-codegen-pure" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a29399fc94bcd3eeaa951c715f7bea69409b2445356b00519740bcd6ddd865" +dependencies = [ + "protobuf", + "protobuf-codegen", +] + [[package]] name = "psm" version = "0.1.21" @@ -5245,6 +5324,29 @@ dependencies = [ "is_ci", ] +[[package]] +name = "symbolic-common" +version = "12.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cccfffbc6bb3bb2d3a26cd2077f4d055f6808d266f9d4d158797a4c60510dfe" +dependencies = [ + "debugid", + "memmap2 0.9.0", + "stable_deref_trait", + "uuid", +] + +[[package]] +name = "symbolic-demangle" +version = "12.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a99812da4020a67e76c4eb41f08c87364c14170495ff780f30dd519c221a68" +dependencies = [ + "cpp_demangle 0.4.3", + "rustc-demangle", + "symbolic-common", +] + [[package]] name = "syn" version = "1.0.109" @@ -6341,7 +6443,7 @@ dependencies = [ "anyhow", "bincode", "cfg-if", - "cpp_demangle", + "cpp_demangle 0.3.5", "gimli", "ittapi", "log", diff --git a/Cargo.toml b/Cargo.toml index 3eb3627a5ca..b961c3d7966 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -256,4 +256,4 @@ lto = true [profile.profiling] inherits = "release" -debug = "line-tables-only" +debug = "limited" diff --git a/Dockerfile.glibc b/Dockerfile.glibc index 014cfca20b2..fd4668a6e7e 100644 --- a/Dockerfile.glibc +++ b/Dockerfile.glibc @@ -18,7 +18,9 @@ WORKDIR /iroha COPY . . ARG PROFILE="deploy" ARG RUSTFLAGS="" -RUN RUSTFLAGS="${RUSTFLAGS}" mold --run cargo build --target x86_64-unknown-linux-gnu --profile "${PROFILE}" +ARG FEATURES="" +ARG CARGOFLAGS="" +RUN RUSTFLAGS="${RUSTFLAGS}" mold --run cargo ${CARGOFLAGS} build --target x86_64-unknown-linux-gnu --profile "${PROFILE}" --features "${FEATURES}" # final image FROM debian:bookworm-slim diff --git a/config/src/torii.rs b/config/src/torii.rs index 231d5045484..d77457f0ddb 100644 --- a/config/src/torii.rs +++ b/config/src/torii.rs @@ -74,6 +74,8 @@ pub mod uri { pub const SCHEMA: &str = "schema"; /// URI for getting the API version currently used pub const API_VERSION: &str = "api_version"; + /// URI for getting cpu profile + pub const PROFILE: &str = "debug/pprof/profile"; } #[cfg(test)] diff --git a/core/Cargo.toml b/core/Cargo.toml index dc040225591..e2f86e46f47 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -29,6 +29,8 @@ cli = [] dev-telemetry = ["telemetry", "iroha_telemetry/dev-telemetry"] # Support Prometheus metrics. See https://prometheus.io/. expensive-telemetry = ["iroha_telemetry/metric-instrumentation"] +# Profiler integration for wasmtime +profiling = [] [badges] is-it-maintained-issue-resolution = { repository = "https://github.com/hyperledger/iroha" } diff --git a/core/src/smartcontracts/wasm.rs b/core/src/smartcontracts/wasm.rs index 6a06dc3261a..cdc3ebd1d60 100644 --- a/core/src/smartcontracts/wasm.rs +++ b/core/src/smartcontracts/wasm.rs @@ -274,7 +274,7 @@ fn create_config() -> Result { .consume_fuel(true) .cache_config_load_default() .map_err(Error::Initialization)?; - #[cfg(wasm_profiling)] + #[cfg(feature = "profiling")] { config.profiler(wasmtime::ProfilingStrategy::PerfMap); } diff --git a/torii/Cargo.toml b/torii/Cargo.toml index e363f9f5e4a..aa9359f97f1 100644 --- a/torii/Cargo.toml +++ b/torii/Cargo.toml @@ -20,6 +20,8 @@ workspace = true [features] # Enables Telemetry (i.e. Status, Metrics, and API Version) endpoints telemetry = ["iroha_telemetry", "iroha_core/telemetry", "serde_json"] +# Enables profiling endpoint +profiling = ["pprof"] # Enables Data Model Schema endpoint schema = ["iroha_schema_gen"] @@ -46,3 +48,5 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, optional = true } async-trait = { workspace = true } parity-scale-codec = { workspace = true, features = ["derive"] } +# TODO: switch to original crate once fix is merged (https://github.com/tikv/pprof-rs/pull/241) +pprof = { git = " https://github.com/Erigara/pprof-rs", branch = "fix_pointer_align", optional = true, default-features = false, features = ["protobuf-codec", "frame-pointer", "cpp"] } diff --git a/torii/src/lib.rs b/torii/src/lib.rs index ef0168f6b07..68507798239 100644 --- a/torii/src/lib.rs +++ b/torii/src/lib.rs @@ -127,6 +127,30 @@ impl Torii { let get_router = get_router.or(warp::path(uri::SCHEMA) .and_then(|| async { Ok::<_, Infallible>(routing::handle_schema().await) })); + #[cfg(feature = "profiling")] + let get_router = { + // `warp` panics if there is `/` in the string given to the `warp::path` filter + // Path filter has to be boxed to have a single uniform type during iteration + let profile_router_path = uri::PROFILE + .split('/') + .skip_while(|p| p.is_empty()) + .fold(warp::any().boxed(), |path_filter, path| { + path_filter.and(warp::path(path)).boxed() + }); + + let profiling_lock = std::sync::Arc::new(tokio::sync::Mutex::new(())); + get_router.or(profile_router_path + .and(warp::query::()) + .and_then(move |params| { + let profiling_lock = Arc::clone(&profiling_lock); + async move { + Ok::<_, Infallible>( + routing::profiling::handle_profile(params, profiling_lock).await, + ) + } + })) + }; + let post_router = warp::post() .and( endpoint4( @@ -279,6 +303,9 @@ pub enum Error { #[cfg(feature = "telemetry")] /// Failed to get Prometheus metrics Prometheus(#[source] eyre::Report), + #[cfg(feature = "profiling")] + /// Failed to get pprof profile + Pprof(#[source] eyre::Report), #[cfg(feature = "telemetry")] /// Failed to get status StatusFailure(#[source] eyre::Report), @@ -315,6 +342,8 @@ impl Error { }, #[cfg(feature = "telemetry")] Prometheus(_) | StatusFailure(_) => StatusCode::INTERNAL_SERVER_ERROR, + #[cfg(feature = "profiling")] + Pprof(_) => StatusCode::INTERNAL_SERVER_ERROR, ConfigurationFailure(_) => StatusCode::INTERNAL_SERVER_ERROR, } } diff --git a/torii/src/routing.rs b/torii/src/routing.rs index 522f6aae0b9..cc5a7ab590e 100644 --- a/torii/src/routing.rs +++ b/torii/src/routing.rs @@ -386,3 +386,81 @@ pub fn handle_status( Ok(reply) } } + +#[cfg(feature = "profiling")] +pub mod profiling { + use std::num::{NonZeroU16, NonZeroU64}; + + use pprof::protos::Message; + use serde::{Deserialize, Serialize}; + + use super::*; + + /// Query params used to configure profile gathering + #[derive(Serialize, Deserialize, Clone, Copy)] + pub struct ProfileParams { + /// How often to sample iroha + #[serde(default = "ProfileParams::default_frequency")] + frequency: NonZeroU16, + /// How long to sample iroha + #[serde(default = "ProfileParams::default_seconds")] + seconds: NonZeroU64, + } + + impl ProfileParams { + fn default_frequency() -> NonZeroU16 { + NonZeroU16::new(99).unwrap() + } + + fn default_seconds() -> NonZeroU64 { + NonZeroU64::new(10).unwrap() + } + } + + /// Serve pprof protobuf profiles + pub async fn handle_profile( + ProfileParams { frequency, seconds }: ProfileParams, + profiling_lock: std::sync::Arc>, + ) -> Result> { + match profiling_lock.try_lock() { + Ok(_guard) => { + let mut body = Vec::new(); + { + // Create profiler guard + let guard = pprof::ProfilerGuardBuilder::default() + .frequency(frequency.get() as i32) + .blocklist(&["libc", "libgcc", "pthread", "vdso"]) + .build() + .map_err(|e| { + Error::Pprof(eyre::eyre!( + "pprof::ProfilerGuardBuilder::build fail: {}", + e + )) + })?; + + // Collect profiles for seconds + tokio::time::sleep(tokio::time::Duration::from_secs(seconds.get())).await; + + let report = guard + .report() + .build() + .map_err(|e| Error::Pprof(eyre::eyre!("generate report fail: {}", e)))?; + + let profile = report.pprof().map_err(|e| { + Error::Pprof(eyre::eyre!("generate pprof from report fail: {}", e)) + })?; + + profile.write_to_vec(&mut body).map_err(|e| { + Error::Pprof(eyre::eyre!("encode pprof into bytes fail: {}", e)) + })?; + } + + Ok(body) + } + Err(_) => { + // profile already running return error + Err(Error::Pprof(eyre::eyre!("profiling already running"))) + } + } + } +}