diff --git a/Cargo.lock b/Cargo.lock index d44ea8fd..5fac5e9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -227,21 +227,6 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" -[[package]] -name = "bit-set" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - [[package]] name = "bitflags" version = "1.3.2" @@ -943,12 +928,6 @@ version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" -[[package]] -name = "libm" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" - [[package]] name = "linux-raw-sys" version = "0.4.7" @@ -1049,7 +1028,7 @@ dependencies = [ [[package]] name = "ndc-client" version = "0.1.0" -source = "git+http://github.com/hasura/ndc-spec.git?tag=v0.1.0-rc.18#608ecc4b6719753c186a37494bbe95ae26e64f45" +source = "git+http://github.com/hasura/ndc-spec.git?tag=v0.1.0#8892f0524affd37e94097c2ce43da8740fc57aca" dependencies = [ "async-trait", "indexmap 2.1.0", @@ -1102,14 +1081,14 @@ dependencies = [ [[package]] name = "ndc-test" version = "0.1.0" -source = "git+http://github.com/hasura/ndc-spec.git?tag=v0.1.0-rc.18#608ecc4b6719753c186a37494bbe95ae26e64f45" +source = "git+http://github.com/hasura/ndc-spec.git?tag=v0.1.0#8892f0524affd37e94097c2ce43da8740fc57aca" dependencies = [ "async-trait", "clap", "colored", "indexmap 2.1.0", "ndc-client", - "proptest", + "rand", "reqwest", "semver", "serde", @@ -1135,7 +1114,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -1434,26 +1412,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "proptest" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c003ac8c77cb07bb74f5f198bce836a689bcd5a42574612bf14d17bfd08c20e" -dependencies = [ - "bit-set", - "bit-vec", - "bitflags 2.4.1", - "lazy_static", - "num-traits", - "rand", - "rand_chacha", - "rand_xorshift", - "regex-syntax 0.7.5", - "rusty-fork", - "tempfile", - "unarray", -] - [[package]] name = "prost" version = "0.11.9" @@ -1483,12 +1441,6 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - [[package]] name = "quote" version = "1.0.33" @@ -1528,15 +1480,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rand_xorshift" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" -dependencies = [ - "rand_core", -] - [[package]] name = "redox_syscall" version = "0.3.5" @@ -1656,18 +1599,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" -[[package]] -name = "rusty-fork" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" -dependencies = [ - "fnv", - "quick-error", - "tempfile", - "wait-timeout", -] - [[package]] name = "ryu" version = "1.0.15" @@ -2368,12 +2299,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" -[[package]] -name = "unarray" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" - [[package]] name = "unicase" version = "2.7.0" @@ -2451,15 +2376,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "wait-timeout" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" -dependencies = [ - "libc", -] - [[package]] name = "want" version = "0.3.1" diff --git a/rust-connector-sdk/Cargo.toml b/rust-connector-sdk/Cargo.toml index 9d74a70c..7c65b4e5 100644 --- a/rust-connector-sdk/Cargo.toml +++ b/rust-connector-sdk/Cargo.toml @@ -11,10 +11,14 @@ path = "src/lib.rs" name = "ndc_hub_example" path = "bin/main.rs" +[features] +ndc-test = ["dep:ndc-test"] +default = ["ndc-test"] + [dependencies] gdc_rust_types = { git = "https://github.com/hasura/gdc_rust_types.git", rev = "3273434" } -ndc-client = { git = "http://github.com/hasura/ndc-spec.git", tag = "v0.1.0-rc.18" } -ndc-test = { git = "http://github.com/hasura/ndc-spec.git", tag = "v0.1.0-rc.18" } +ndc-client = { git = "http://github.com/hasura/ndc-spec.git", tag = "v0.1.0" } +ndc-test = { git = "http://github.com/hasura/ndc-spec.git", tag = "v0.1.0", optional = true } async-trait = "^0.1.74" axum = "^0.6.20" diff --git a/rust-connector-sdk/src/default_main.rs b/rust-connector-sdk/src/default_main.rs index 25cd644d..a226c492 100644 --- a/rust-connector-sdk/src/default_main.rs +++ b/rust-connector-sdk/src/default_main.rs @@ -3,9 +3,7 @@ mod v2_compat; use std::error::Error; use std::net; use std::path::{Path, PathBuf}; -use std::process::exit; -use async_trait::async_trait; use axum::{ body::Body, extract::State, @@ -23,7 +21,6 @@ use ndc_client::models::{ CapabilitiesResponse, ErrorResponse, ExplainResponse, MutationRequest, MutationResponse, QueryRequest, QueryResponse, SchemaResponse, }; -use ndc_test::report; use crate::check_health; use crate::connector::{Connector, ConnectorSetup}; @@ -43,10 +40,15 @@ enum Command { #[command()] Serve(ServeCommand), #[command()] + #[cfg(feature = "ndc-test")] Test(TestCommand), #[command()] + #[cfg(feature = "ndc-test")] Replay(ReplayCommand), #[command()] + #[cfg(feature = "ndc-test")] + Bench(BenchCommand), + #[command()] CheckHealth(CheckHealthCommand), } @@ -89,6 +91,27 @@ struct ReplayCommand { snapshots_dir: PathBuf, } +#[derive(Clone, Parser)] +struct BenchCommand { + #[arg(long, value_name = "DIRECTORY", env = "HASURA_CONFIGURATION_DIRECTORY")] + configuration: PathBuf, + #[arg( + long, + value_name = "COUNT", + help = "the number of samples to collect per test", + default_value = "100" + )] + samples: u32, + #[arg( + long, + value_name = "TOLERANCE", + help = "tolerable deviation from previous report, in standard deviations from the mean" + )] + tolerance: Option, + #[arg(long, value_name = "DIRECTORY", env = "HASURA_SNAPSHOTS_DIR")] + snapshots_dir: PathBuf, +} + #[derive(Clone, Parser)] struct CheckHealthCommand { #[arg(long, value_name = "HOST")] @@ -182,9 +205,13 @@ where match command { Command::Serve(serve_command) => serve(setup, serve_command).await, - Command::Test(test_command) => test(setup, test_command).await, - Command::Replay(replay_command) => replay(setup, replay_command).await, Command::CheckHealth(check_health_command) => check_health(check_health_command).await, + #[cfg(feature = "ndc-test")] + Command::Test(test_command) => ndc_test_commands::test(setup, test_command).await, + #[cfg(feature = "ndc-test")] + Command::Bench(bench_command) => ndc_test_commands::bench(setup, bench_command).await, + #[cfg(feature = "ndc-test")] + Command::Replay(replay_command) => ndc_test_commands::replay(setup, replay_command).await, } } @@ -475,108 +502,159 @@ async fn post_query( routes::post_query::(&state.configuration, &state.state, request).await } -struct ConnectorAdapter { - configuration: C::Configuration, - state: C::State, -} - -#[async_trait] -impl ndc_test::Connector for ConnectorAdapter { - async fn get_capabilities( - &self, - ) -> Result { - C::get_capabilities() - .await - .into_value::>() - .map_err(|err| ndc_test::Error::OtherError(err)) +#[cfg(feature = "ndc-test")] +mod ndc_test_commands { + use super::{BenchCommand, Connector, ConnectorSetup}; + use crate::json_response::JsonResponse; + use async_trait::async_trait; + use ndc_test::reporter::{ConsoleReporter, TestResults}; + use prometheus::Registry; + use std::error::Error; + use std::path::PathBuf; + use std::process::exit; + + struct ConnectorAdapter { + configuration: C::Configuration, + state: C::State, } - async fn get_schema(&self) -> Result { - match C::get_schema(&self.configuration).await { - Ok(response) => response + #[async_trait(?Send)] + impl ndc_test::connector::Connector for ConnectorAdapter { + async fn get_capabilities( + &self, + ) -> Result { + C::get_capabilities() + .await .into_value::>() - .map_err(|err| ndc_test::Error::OtherError(err)), - Err(err) => Err(ndc_test::Error::OtherError(err.into())), + .map_err(|err| ndc_test::error::Error::OtherError(err)) } - } - async fn query( - &self, - request: ndc_client::models::QueryRequest, - ) -> Result { - match C::query(&self.configuration, &self.state, request) - .await - .and_then(JsonResponse::into_value) - { - Ok(response) => Ok(response), - Err(err) => Err(ndc_test::Error::OtherError(err.into())), + async fn get_schema( + &self, + ) -> Result { + match C::get_schema(&self.configuration).await { + Ok(response) => response + .into_value::>() + .map_err(|err| ndc_test::error::Error::OtherError(err)), + Err(err) => Err(ndc_test::error::Error::OtherError(err.into())), + } } - } - async fn mutation( - &self, - request: ndc_client::models::MutationRequest, - ) -> Result { - match C::mutation(&self.configuration, &self.state, request) - .await - .and_then(JsonResponse::into_value) - { - Ok(response) => Ok(response), - Err(err) => Err(ndc_test::Error::OtherError(err.into())), + async fn query( + &self, + request: ndc_client::models::QueryRequest, + ) -> Result { + match C::query(&self.configuration, &self.state, request) + .await + .and_then(JsonResponse::into_value) + { + Ok(response) => Ok(response), + Err(err) => Err(ndc_test::error::Error::OtherError(err.into())), + } + } + + async fn mutation( + &self, + request: ndc_client::models::MutationRequest, + ) -> Result { + match C::mutation(&self.configuration, &self.state, request) + .await + .and_then(JsonResponse::into_value) + { + Ok(response) => Ok(response), + Err(err) => Err(ndc_test::error::Error::OtherError(err.into())), + } } } -} -async fn test( - setup: Setup, - command: TestCommand, -) -> Result<(), Box> { - let test_configuration = ndc_test::TestConfiguration { - seed: command.seed, - snapshots_dir: command.snapshots_dir, - }; + pub(super) async fn test( + setup: Setup, + command: super::TestCommand, + ) -> Result<(), Box> { + let test_configuration = ndc_test::configuration::TestConfiguration { + seed: command.seed.map(|s| s.as_bytes().try_into()).transpose()?, + snapshots_dir: command.snapshots_dir, + gen_config: ndc_test::configuration::TestGenerationConfiguration::default(), + }; - let connector = make_connector_adapter(setup, command.configuration).await?; - let results = ndc_test::test_connector(&test_configuration, &connector).await; + let connector = make_connector_adapter(setup, command.configuration).await?; + let mut reporter = (ConsoleReporter::new(), TestResults::default()); - if !results.failures.is_empty() { - println!(); - println!("{}", report(results)); + ndc_test::test_connector(&test_configuration, &connector, &mut reporter).await; - exit(1) + if !reporter.1.failures.is_empty() { + println!(); + println!("{}", reporter.1.report()); + + exit(1) + } + + Ok(()) } - Ok(()) -} + pub(super) async fn replay( + setup: Setup, + command: super::ReplayCommand, + ) -> Result<(), Box> { + let connector = make_connector_adapter(setup, command.configuration).await?; + let mut reporter = (ConsoleReporter::new(), TestResults::default()); -async fn replay( - setup: Setup, - command: ReplayCommand, -) -> Result<(), Box> { - let connector = make_connector_adapter(setup, command.configuration).await?; - let results = ndc_test::test_snapshots_in_directory(&connector, command.snapshots_dir).await; + ndc_test::test_snapshots_in_directory(&connector, &mut reporter, command.snapshots_dir) + .await; - if !results.failures.is_empty() { - println!(); - println!("{}", report(results)); + if !reporter.1.failures.is_empty() { + println!(); + println!("{}", reporter.1.report()); - exit(1) + exit(1) + } + + Ok(()) } - Ok(()) -} + pub(super) async fn bench( + setup: Setup, + command: BenchCommand, + ) -> Result<(), Box> { + let configuration = ndc_test::ReportConfiguration { + samples: command.samples, + tolerance: command.tolerance, + }; + + let connector = make_connector_adapter(setup, command.configuration).await?; + let mut reporter = (ConsoleReporter::new(), TestResults::default()); + + let reports = ndc_test::bench_snapshots_in_directory( + &configuration, + &connector, + &mut reporter, + command.snapshots_dir, + ) + .await + .map_err(|e| e.to_string())?; -async fn make_connector_adapter( - setup: Setup, - configuration_path: PathBuf, -) -> Result, Box> { - let mut metrics = Registry::new(); - let configuration = setup.parse_configuration(configuration_path).await?; - let state = setup.try_init_state(&configuration, &mut metrics).await?; - Ok(ConnectorAdapter { - configuration, - state, - }) + println!(); + println!("{}", ndc_test::benchmark_report(&configuration, reports)); + + if !reporter.1.failures.is_empty() { + exit(1); + } + + Ok(()) + } + + async fn make_connector_adapter( + setup: Setup, + configuration_path: PathBuf, + ) -> Result, Box> { + let mut metrics = Registry::new(); + let configuration = setup.parse_configuration(configuration_path).await?; + let state = setup.try_init_state(&configuration, &mut metrics).await?; + Ok(ConnectorAdapter { + configuration, + state, + }) + } } async fn check_health(