From ba74bdf88d8128a0a9d37555fb5af1a31d0b6e7e Mon Sep 17 00:00:00 2001 From: Alberto Schiabel Date: Tue, 21 Nov 2023 12:12:06 +0100 Subject: [PATCH 1/3] feat(driver-adapters): enable Wasm on `request-handlers` (#4455) * feat(quaint): allow wasm32-unknown-unknown compilation; currently fails on native * feat(quaint): split postgres connector into native and wasm submodules * feat(quaint): split mysql connector into native and wasm submodules * feat(quaint): recover wasm error for mysql * feat(quaint): split mssql connector into native and wasm submodules * feat(quaint): split sqlite connector into native and wasm submodules * chore(quaint): fix clippy when compiling natively * chore(quaint): fix clippy when compiling to wasm32-unknown-unknown * chore(quaint): update README * chore(quaint): rename "*-connector" feature flag to "*-native" * feat(quaint): enable pure Wasm SqliteError * feat(query-connect): allow wasm32-unknown-unknown compilation * feat(sql-query-connector): allow wasm32-unknown-unknown compilation * chore(query-engine-wasm): add currently unused local crates to test wasm32-unknown-unknown compilation * chore: update Cargo.lock * chore: remove leftover comments * feat(query-core): allow wasm32-unknown-unknown compilation * chore(sql-query-connector): fix clipppy on wasm32 * chore: remove leftover comment * feat(driver-adapters): enable Wasm on request-handlers * WIP: refactor mysql module to flatten its structure * feat(quaint): flatten mssql connector module * feat(quaint): flatten postgres connector module * feat(quaint): flatten sqlite connector module * chore(quaint): export all public definitions in connector "url" modules * chore(quaint): refactor tests for connectors, addressing feedback * chore: add comment on MysqlAsyncError * chore: add comment on ffi.rs for sqlite * chore: replace awkward "super::super::" with "crate::..." * chore: add comments around "query_core::executor::task" * chore: add "request-handlers" to "query-engine-wasm" * chore: move "task" module into its own file * fix(driver-adapters): ci for "request-handlers" * fix(driver-adapters): ci for "request-handlers" --------- Co-authored-by: Miguel Fernandez --- Cargo.lock | 1 + query-engine/query-engine-wasm/Cargo.toml | 1 + query-engine/request-handlers/Cargo.toml | 7 +- .../request-handlers/src/connector_mode.rs | 1 + .../request-handlers/src/load_executor.rs | 162 +++++++++--------- 5 files changed, 91 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d788c5f344c6..cd2fe6c6b5c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3827,6 +3827,7 @@ dependencies = [ "quaint", "query-connector", "query-core", + "request-handlers", "serde", "serde-wasm-bindgen", "serde_json", diff --git a/query-engine/query-engine-wasm/Cargo.toml b/query-engine/query-engine-wasm/Cargo.toml index b8c12dc91f5c..dff7985af27c 100644 --- a/query-engine/query-engine-wasm/Cargo.toml +++ b/query-engine/query-engine-wasm/Cargo.toml @@ -15,6 +15,7 @@ user-facing-errors = { path = "../../libs/user-facing-errors" } psl.workspace = true prisma-models = { path = "../prisma-models" } quaint = { path = "../../quaint" } +request-handlers = { path = "../request-handlers", default-features = false, features = ["sql", "driver-adapters"] } connector = { path = "../connectors/query-connector", package = "query-connector" } sql-query-connector = { path = "../connectors/sql-query-connector" } query-core = { path = "../core" } diff --git a/query-engine/request-handlers/Cargo.toml b/query-engine/request-handlers/Cargo.toml index e6545eda2234..3eb6954e03c1 100644 --- a/query-engine/request-handlers/Cargo.toml +++ b/query-engine/request-handlers/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" prisma-models = { path = "../prisma-models" } query-core = { path = "../core", features = ["metrics"] } user-facing-errors = { path = "../../libs/user-facing-errors" } +quaint = { path = "../../quaint" } psl.workspace = true dmmf_crate = { path = "../dmmf", package = "dmmf" } itertools = "0.10" @@ -20,7 +21,6 @@ thiserror = "1" tracing = "0.1" url = "2" connection-string.workspace = true -quaint.workspace = true once_cell = "1.15" mongodb-query-connector = { path = "../connectors/mongodb-query-connector", optional = true } @@ -32,10 +32,11 @@ schema = { path = "../schema" } codspeed-criterion-compat = "1.1.0" [features] -default = ["mongodb", "sql"] +default = ["sql", "mongodb", "native"] mongodb = ["mongodb-query-connector"] sql = ["sql-query-connector"] -driver-adapters = ["sql-query-connector"] +driver-adapters = ["sql-query-connector/driver-adapters"] +native = ["mongodb", "sql-query-connector", "quaint/native", "query-core/metrics"] [[bench]] name = "query_planning_bench" diff --git a/query-engine/request-handlers/src/connector_mode.rs b/query-engine/request-handlers/src/connector_mode.rs index 00e0515a596e..be03fbab5820 100644 --- a/query-engine/request-handlers/src/connector_mode.rs +++ b/query-engine/request-handlers/src/connector_mode.rs @@ -1,6 +1,7 @@ #[derive(Copy, Clone, PartialEq, Eq)] pub enum ConnectorMode { /// Indicates that Rust drivers are used in Query Engine. + #[cfg(feature = "native")] Rust, /// Indicates that JS drivers are used in Query Engine. diff --git a/query-engine/request-handlers/src/load_executor.rs b/query-engine/request-handlers/src/load_executor.rs index 652ad3108f0d..26728605f92a 100644 --- a/query-engine/request-handlers/src/load_executor.rs +++ b/query-engine/request-handlers/src/load_executor.rs @@ -1,14 +1,12 @@ +#![allow(unused_imports)] + use psl::{builtin_connectors::*, Datasource, PreviewFeatures}; use query_core::{executor::InterpretingExecutor, Connector, QueryExecutor}; use sql_query_connector::*; use std::collections::HashMap; use std::env; -use tracing::trace; use url::Url; -#[cfg(feature = "mongodb")] -use mongodb_query_connector::MongoDb; - use super::ConnectorMode; /// Loads a query executor based on the parsed Prisma schema (datasource). @@ -27,6 +25,7 @@ pub async fn load( driver_adapter(source, url, features).await } + #[cfg(feature = "native")] ConnectorMode::Rust => { if let Ok(value) = env::var("PRISMA_DISABLE_QUAINT_EXECUTORS") { let disable = value.to_uppercase(); @@ -36,14 +35,14 @@ pub async fn load( } match source.active_provider { - p if SQLITE.is_provider(p) => sqlite(source, url, features).await, - p if MYSQL.is_provider(p) => mysql(source, url, features).await, - p if POSTGRES.is_provider(p) => postgres(source, url, features).await, - p if MSSQL.is_provider(p) => mssql(source, url, features).await, - p if COCKROACH.is_provider(p) => postgres(source, url, features).await, + p if SQLITE.is_provider(p) => native::sqlite(source, url, features).await, + p if MYSQL.is_provider(p) => native::mysql(source, url, features).await, + p if POSTGRES.is_provider(p) => native::postgres(source, url, features).await, + p if MSSQL.is_provider(p) => native::mssql(source, url, features).await, + p if COCKROACH.is_provider(p) => native::postgres(source, url, features).await, #[cfg(feature = "mongodb")] - p if MONGODB.is_provider(p) => mongodb(source, url, features).await, + p if MONGODB.is_provider(p) => native::mongodb(source, url, features).await, x => Err(query_core::CoreError::ConfigurationError(format!( "Unsupported connector type: {x}" @@ -53,57 +52,88 @@ pub async fn load( } } -async fn sqlite( +#[cfg(feature = "driver-adapters")] +async fn driver_adapter( source: &Datasource, url: &str, features: PreviewFeatures, -) -> query_core::Result> { - trace!("Loading SQLite query connector..."); - let sqlite = Sqlite::from_source(source, url, features).await?; - trace!("Loaded SQLite query connector."); - Ok(executor_for(sqlite, false)) +) -> Result, query_core::CoreError> { + let js = Js::from_source(source, url, features).await?; + Ok(executor_for(js, false)) } -async fn postgres( - source: &Datasource, - url: &str, - features: PreviewFeatures, -) -> query_core::Result> { - trace!("Loading Postgres query connector..."); - let database_str = url; - let psql = PostgreSql::from_source(source, url, features).await?; - - let url = Url::parse(database_str) - .map_err(|err| query_core::CoreError::ConfigurationError(format!("Error parsing connection string: {err}")))?; - let params: HashMap = url.query_pairs().into_owned().collect(); - - let force_transactions = params - .get("pgbouncer") - .and_then(|flag| flag.parse().ok()) - .unwrap_or(false); - trace!("Loaded Postgres query connector."); - Ok(executor_for(psql, force_transactions)) -} +#[cfg(feature = "native")] +mod native { + use super::*; + use tracing::trace; + + pub(crate) async fn sqlite( + source: &Datasource, + url: &str, + features: PreviewFeatures, + ) -> query_core::Result> { + trace!("Loading SQLite query connector..."); + let sqlite = Sqlite::from_source(source, url, features).await?; + trace!("Loaded SQLite query connector."); + Ok(executor_for(sqlite, false)) + } -async fn mysql( - source: &Datasource, - url: &str, - features: PreviewFeatures, -) -> query_core::Result> { - let mysql = Mysql::from_source(source, url, features).await?; - trace!("Loaded MySQL query connector."); - Ok(executor_for(mysql, false)) -} + pub(crate) async fn postgres( + source: &Datasource, + url: &str, + features: PreviewFeatures, + ) -> query_core::Result> { + trace!("Loading Postgres query connector..."); + let database_str = url; + let psql = PostgreSql::from_source(source, url, features).await?; + + let url = Url::parse(database_str).map_err(|err| { + query_core::CoreError::ConfigurationError(format!("Error parsing connection string: {err}")) + })?; + let params: HashMap = url.query_pairs().into_owned().collect(); + + let force_transactions = params + .get("pgbouncer") + .and_then(|flag| flag.parse().ok()) + .unwrap_or(false); + trace!("Loaded Postgres query connector."); + Ok(executor_for(psql, force_transactions)) + } -async fn mssql( - source: &Datasource, - url: &str, - features: PreviewFeatures, -) -> query_core::Result> { - trace!("Loading SQL Server query connector..."); - let mssql = Mssql::from_source(source, url, features).await?; - trace!("Loaded SQL Server query connector."); - Ok(executor_for(mssql, false)) + pub(crate) async fn mysql( + source: &Datasource, + url: &str, + features: PreviewFeatures, + ) -> query_core::Result> { + let mysql = Mysql::from_source(source, url, features).await?; + trace!("Loaded MySQL query connector."); + Ok(executor_for(mysql, false)) + } + + pub(crate) async fn mssql( + source: &Datasource, + url: &str, + features: PreviewFeatures, + ) -> query_core::Result> { + trace!("Loading SQL Server query connector..."); + let mssql = Mssql::from_source(source, url, features).await?; + trace!("Loaded SQL Server query connector."); + Ok(executor_for(mssql, false)) + } + + #[cfg(feature = "mongodb")] + pub(crate) async fn mongodb( + source: &Datasource, + url: &str, + _features: PreviewFeatures, + ) -> query_core::Result> { + use mongodb_query_connector::MongoDb; + + trace!("Loading MongoDB query connector..."); + let mongo = MongoDb::new(source, url).await?; + trace!("Loaded MongoDB query connector."); + Ok(executor_for(mongo, false)) + } } fn executor_for(connector: T, force_transactions: bool) -> Box @@ -112,27 +142,3 @@ where { Box::new(InterpretingExecutor::new(connector, force_transactions)) } - -#[cfg(feature = "mongodb")] -async fn mongodb( - source: &Datasource, - url: &str, - _features: PreviewFeatures, -) -> query_core::Result> { - trace!("Loading MongoDB query connector..."); - let mongo = MongoDb::new(source, url).await?; - trace!("Loaded MongoDB query connector."); - Ok(executor_for(mongo, false)) -} - -#[cfg(feature = "driver-adapters")] -async fn driver_adapter( - source: &Datasource, - url: &str, - features: PreviewFeatures, -) -> Result, query_core::CoreError> { - trace!("Loading driver adapter..."); - let js = Js::from_source(source, url, features).await?; - trace!("Loaded driver adapter..."); - Ok(executor_for(js, false)) -} From 63a4fd930b2737b5802f17da8282a5bbbbdeeaba Mon Sep 17 00:00:00 2001 From: Serhii Tatarintsev Date: Tue, 21 Nov 2023 18:20:20 +0100 Subject: [PATCH 2/3] query-engine-wasm: Fix build & ensure it stays fixed (#4488) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * query-engine-wasm: Fix build & ensure it stays fixed Current version in main does not compile to wasm correctly. This PR fixes the error and adds CI check to ensure it compiles before it is merged. * Add name to the job * Update .github/workflows/qe-wasm-check.yml Co-authored-by: Joël Galeran --------- Co-authored-by: Joël Galeran --- .github/workflows/qe-wasm-check.yml | 27 ++++++++++++++++++++++++ query-engine/request-handlers/Cargo.toml | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/qe-wasm-check.yml diff --git a/.github/workflows/qe-wasm-check.yml b/.github/workflows/qe-wasm-check.yml new file mode 100644 index 000000000000..f67d2d247b27 --- /dev/null +++ b/.github/workflows/qe-wasm-check.yml @@ -0,0 +1,27 @@ +name: WASM engine compile check +on: + push: + branches: + - main + pull_request: + paths-ignore: + - '.github/**' + - '!.github/workflows/qe-wasm-check.yml' + - '.buildkite/**' + - '*.md' + - 'LICENSE' + - 'CODEOWNERS' + - 'renovate.json' + +jobs: + build: + name: 'Compilation check for query-engine-wasm' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: Install wasm-pack + run: cargo install wasm-pack + - name: Build wasm query engine + run: ./build.sh + working-directory: ./query-engine/query-engine-wasm diff --git a/query-engine/request-handlers/Cargo.toml b/query-engine/request-handlers/Cargo.toml index 3eb6954e03c1..f04d742c448e 100644 --- a/query-engine/request-handlers/Cargo.toml +++ b/query-engine/request-handlers/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] prisma-models = { path = "../prisma-models" } -query-core = { path = "../core", features = ["metrics"] } +query-core = { path = "../core" } user-facing-errors = { path = "../../libs/user-facing-errors" } quaint = { path = "../../quaint" } psl.workspace = true From 2c867f452d73681500eb8426a83b37ad1bc1eda7 Mon Sep 17 00:00:00 2001 From: Serhii Tatarintsev Date: Wed, 22 Nov 2023 11:07:03 +0100 Subject: [PATCH 3/3] qe: Allow to run query engine test suite against wasm engines (#4467) * qe: Allow to run query engine test suite against wasm engines Intoroduces new environment variable/test config option, `EXTERNAL_TEST_EXECUTOR_ENGINE` that can be either `NAPI` (default) or `WASM` that affects which engine external test executor will start with. Adds a bunch of additional test configs for driver adapters that duplicate existing ones, but run against WASM engine. Close prisma/team-orm#551 * Fixes & clippy * Add GH Actions jobs * Install wasm-pack * Comment out WASM tests * Remove `external_test_executor_engine` option Specify engine option in `external_test_executor` * fix native tests * Adjust makefile and fix wasm build * Docs * Uncomment dependency --- .envrc | 2 +- .../query-engine-driver-adapters.yml | 24 ++++-- Makefile | 38 +++++++++- query-engine/connector-test-kit-rs/README.md | 5 +- .../query-tests-setup/src/config.rs | 74 +++++++++++-------- .../src/connector_tag/js/external_process.rs | 18 +++-- .../test-configs/libsql-sqlite | 2 +- .../test-configs/libsql-sqlite-wasm | 5 ++ .../test-configs/neon-ws-postgres13 | 2 +- .../test-configs/neon-ws-postgres13-wasm | 7 ++ .../test-configs/pg-postgres13 | 2 +- .../test-configs/pg-postgres13-wasm | 6 ++ .../test-configs/planetscale-vitess8 | 2 +- .../test-configs/planetscale-vitess8-wasm | 7 ++ .../connector-test-kit-executor/package.json | 5 ++ .../connector-test-kit-executor/src/index.ts | 7 +- .../connector-test-kit-executor/src/qe.ts | 50 ++++++++++--- .../connector-test-kit-executor/src/wasm.ts | 14 ++++ .../connector-test-kit-executor/tsconfig.json | 4 +- query-engine/query-engine-wasm/Cargo.toml | 7 +- 20 files changed, 209 insertions(+), 72 deletions(-) create mode 100644 query-engine/connector-test-kit-rs/test-configs/libsql-sqlite-wasm create mode 100644 query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13-wasm create mode 100644 query-engine/connector-test-kit-rs/test-configs/pg-postgres13-wasm create mode 100644 query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8-wasm create mode 100644 query-engine/driver-adapters/connector-test-kit-executor/src/wasm.ts diff --git a/.envrc b/.envrc index 48b1254c1700..5488da9e10e7 100644 --- a/.envrc +++ b/.envrc @@ -23,7 +23,7 @@ export QE_LOG_LEVEL=debug # Set it to "trace" to enable query-graph debugging lo # export FMT_SQL=1 # Uncomment it to enable logging formatted SQL queries ### Uncomment to run driver adapters tests. See query-engine-driver-adapters.yml workflow for how tests run in CI. -# export EXTERNAL_TEST_EXECUTOR="$(pwd)/query-engine/driver-adapters/js/connector-test-kit-executor/script/start_node.sh" +# export EXTERNAL_TEST_EXECUTOR="napi" # export DRIVER_ADAPTER=pg # Set to pg, neon or planetscale # export PRISMA_DISABLE_QUAINT_EXECUTORS=1 # Disable quaint executors for driver adapters # export DRIVER_ADAPTER_URL_OVERRIDE ="postgres://USER:PASSWORD@DATABASExxxx" # Override the database url for the driver adapter tests diff --git a/.github/workflows/query-engine-driver-adapters.yml b/.github/workflows/query-engine-driver-adapters.yml index 3de0238aa0e7..f6eec5ffc102 100644 --- a/.github/workflows/query-engine-driver-adapters.yml +++ b/.github/workflows/query-engine-driver-adapters.yml @@ -25,12 +25,22 @@ jobs: fail-fast: false matrix: adapter: - - name: 'pg' + - name: 'pg (napi)' setup_task: 'dev-pg-postgres13' - - name: 'neon:ws' + - name: 'neon:ws (napi)' setup_task: 'dev-neon-ws-postgres13' - - name: 'libsql' + - name: 'libsql (napi)' setup_task: 'dev-libsql-sqlite' + # TODO: uncomment when WASM engine is functional + # - name: 'pg (wasm)' + # setup_task: 'dev-pg-postgres13-wasm' + # needs_wasm_pack: true + # - name: 'neon:ws (wasm)' + # setup_task: 'dev-neon-ws-postgres13-wasm' + # needs_wasm_pack: true + # - name: 'libsql (wasm)' + # setup_task: 'dev-libsql-sqlite-wasm' + # needs_wasm_pack: true node_version: ['18'] env: LOG_LEVEL: 'info' # Set to "debug" to trace the query engine and node process running the driver adapter @@ -85,9 +95,13 @@ jobs: echo "DRIVER_ADAPTERS_BRANCH=$branch" >> "$GITHUB_ENV" fi - - run: make ${{ matrix.adapter.setup_task }} - - uses: dtolnay/rust-toolchain@stable + - name: 'Install wasm-pack' + if: ${{ matrix.adapter.needs_wasm_pack }} + run: cargo install wasm-pack + + - run: make ${{ matrix.adapter.setup_task }} + - name: 'Run tests' run: cargo test --package query-engine-tests -- --test-threads=1 diff --git a/Makefile b/Makefile index b4516e645817..94b3d539f971 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,11 @@ ifndef DRIVER_ADAPTER cargo test --package query-engine-tests else @echo "Executing query engine tests with $(DRIVER_ADAPTER) driver adapter"; \ - # Add your actual command for the "test-driver-adapter" task here - $(MAKE) test-driver-adapter-$(DRIVER_ADAPTER); + if [ "$(ENGINE)" = "wasm" ]; then \ + $(MAKE) test-driver-adapter-$(DRIVER_ADAPTER)-wasm; \ + else \ + $(MAKE) test-driver-adapter-$(DRIVER_ADAPTER); \ + fi endif test-qe-verbose: @@ -91,6 +94,12 @@ test-libsql-sqlite: dev-libsql-sqlite test-qe-st test-driver-adapter-libsql: test-libsql-sqlite +dev-libsql-sqlite-wasm: build-qe-wasm build-connector-kit-js + cp $(CONFIG_PATH)/libsql-sqlite-wasm $(CONFIG_FILE) + +test-libsql-sqlite-wasm: dev-libsql-sqlite-wasm test-qe-st +test-driver-adapter-libsql-sqlite-wasm: test-libsql-sqlite-wasm + start-postgres9: docker compose -f docker-compose.yml up --wait -d --remove-orphans postgres9 @@ -121,14 +130,20 @@ start-postgres13: dev-postgres13: start-postgres13 cp $(CONFIG_PATH)/postgres13 $(CONFIG_FILE) -start-pg-postgres13: build-qe-napi build-connector-kit-js start-postgres13 +start-pg-postgres13: start-postgres13 -dev-pg-postgres13: start-pg-postgres13 +dev-pg-postgres13: start-pg-postgres13 build-qe-napi build-connector-kit-js cp $(CONFIG_PATH)/pg-postgres13 $(CONFIG_FILE) test-pg-postgres13: dev-pg-postgres13 test-qe-st +dev-pg-postgres13-wasm: start-pg-postgres13 build-qe-wasm build-connector-kit-js + cp $(CONFIG_PATH)/pg-postgres13-wasm $(CONFIG_FILE) + +test-pg-postgres13-wasm: dev-pg-postgres13-wasm test-qe-st + test-driver-adapter-pg: test-pg-postgres13 +test-driver-adapter-pg-wasm: test-pg-postgres13-wasm start-neon-postgres13: docker compose -f docker-compose.yml up --wait -d --remove-orphans neon-postgres13 @@ -138,7 +153,13 @@ dev-neon-ws-postgres13: start-neon-postgres13 build-qe-napi build-connector-kit- test-neon-ws-postgres13: dev-neon-ws-postgres13 test-qe-st +dev-neon-ws-postgres13-wasm: start-neon-postgres13 build-qe-wasm build-connector-kit-js + cp $(CONFIG_PATH)/neon-ws-postgres13-wasm $(CONFIG_FILE) + +test-neon-ws-postgres13-wasm: dev-neon-ws-postgres13-wasm test-qe-st + test-driver-adapter-neon: test-neon-ws-postgres13 +test-driver-adapter-neon-wasm: test-neon-ws-postgres13-wasm start-postgres14: docker compose -f docker-compose.yml up --wait -d --remove-orphans postgres14 @@ -270,7 +291,13 @@ dev-planetscale-vitess8: start-planetscale-vitess8 build-qe-napi build-connector test-planetscale-vitess8: dev-planetscale-vitess8 test-qe-st +dev-planetscale-vitess8-wasm: start-planetscale-vitess8 build-qe-wasm build-connector-kit-js + cp $(CONFIG_PATH)/planetscale-vitess8-wasm $(CONFIG_FILE) + +test-planetscale-vitess8-wasm: dev-planetscale-vitess8-wasm test-qe-st + test-driver-adapter-planetscale: test-planetscale-vitess8 +test-driver-adapter-planetscale-wasm: test-planetscale-vitess8-wasm ###################### # Local dev commands # @@ -279,6 +306,9 @@ test-driver-adapter-planetscale: test-planetscale-vitess8 build-qe-napi: cargo build --package query-engine-node-api +build-qe-wasm: + cd query-engine/query-engine-wasm && ./build.sh + build-connector-kit-js: build-driver-adapters cd query-engine/driver-adapters && pnpm i && pnpm build diff --git a/query-engine/connector-test-kit-rs/README.md b/query-engine/connector-test-kit-rs/README.md index 97d19467879a..993f636e0d28 100644 --- a/query-engine/connector-test-kit-rs/README.md +++ b/query-engine/connector-test-kit-rs/README.md @@ -82,15 +82,16 @@ drivers the code that actually communicates with the databases. See [`adapter-*` To run tests through a driver adapters, you should also configure the following environment variables: -* `EXTERNAL_TEST_EXECUTOR`: tells the query engine test kit to use an external process to run the queries, this is a node process running a program that will read the queries to run from STDIN, and return responses to STDOUT. The connector kit follows a protocol over JSON RPC for this communication. * `DRIVER_ADAPTER`: tells the test executor to use a particular driver adapter. Set to `neon`, `planetscale` or any other supported adapter. * `DRIVER_ADAPTER_CONFIG`: a json string with the configuration for the driver adapter. This is adapter specific. See the [github workflow for driver adapter tests](.github/workflows/query-engine-driver-adapters.yml) for examples on how to configure the driver adapters. +* `ENGINE`: can be used to run either `wasm` or `napi` version of the engine. Example: ```shell export EXTERNAL_TEST_EXECUTOR="$WORKSPACE_ROOT/query-engine/driver-adapters/connector-test-kit-executor/script/start_node.sh" export DRIVER_ADAPTER=neon +export ENGINE=wasm export DRIVER_ADAPTER_CONFIG ='{ "proxyUrl": "127.0.0.1:5488/v1" }' ```` @@ -98,7 +99,7 @@ We have provided helpers to run the query-engine tests with driver adapters, the variables for you: ```shell -DRIVER_ADAPTER=$adapter make test-qe +DRIVER_ADAPTER=$adapter ENGINE=$engine make test-qe ``` Where `$adapter` is one of the supported adapters: `neon`, `planetscale`, `libsql`. diff --git a/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs b/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs index 4af4e763298a..07ceca784ff9 100644 --- a/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs +++ b/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs @@ -3,10 +3,25 @@ use crate::{ PostgresConnectorTag, SqlServerConnectorTag, SqliteConnectorTag, TestResult, VitessConnectorTag, }; use serde::Deserialize; -use std::{convert::TryFrom, env, fs::File, io::Read, path::PathBuf}; +use std::{convert::TryFrom, env, fmt::Display, fs::File, io::Read, path::PathBuf}; static TEST_CONFIG_FILE_NAME: &str = ".test_config"; +#[derive(Debug, Deserialize, Clone)] +pub enum TestExecutor { + Napi, + Wasm, +} + +impl Display for TestExecutor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TestExecutor::Napi => f.write_str("Napi"), + TestExecutor::Wasm => f.write_str("Wasm"), + } + } +} + /// The central test configuration. #[derive(Debug, Default, Deserialize)] pub struct TestConfig { @@ -24,8 +39,9 @@ pub struct TestConfig { /// Used when testing driver adapters, this process is expected to be a javascript process /// loading the library engine (as a library, or WASM modules) and providing it with a /// driver adapter. + /// Possible values: Napi, Wasm /// Env key: `EXTERNAL_TEST_EXECUTOR` - external_test_executor: Option, + external_test_executor: Option, /// The driver adapter to use when running tests, will be forwarded to the external test /// executor by setting the `DRIVER_ADAPTER` env var when spawning the executor process @@ -85,12 +101,11 @@ fn exit_with_message(msg: &str) -> ! { impl TestConfig { /// Loads a configuration. File-based config has precedence over env config. pub(crate) fn load() -> Self { - let mut config = match Self::from_file().or_else(Self::from_env) { + let config = match Self::from_file().or_else(Self::from_env) { Some(config) => config, None => exit_with_message(CONFIG_LOAD_FAILED), }; - config.fill_defaults(); config.validate(); config.log_info(); @@ -107,8 +122,8 @@ impl TestConfig { self.connector_version().unwrap_or_default() ); println!("* CI? {}", self.is_ci); - if self.external_test_executor.as_ref().is_some() { - println!("* External test executor: {}", self.external_test_executor().unwrap_or_default()); + if let Some(external_test_executor) = self.external_test_executor.as_ref() { + println!("* External test executor: {}", external_test_executor); println!("* Driver adapter: {}", self.driver_adapter().unwrap_or_default()); println!("* Driver adapter url override: {}", self.json_stringify_driver_adapter_config()); } @@ -118,7 +133,10 @@ impl TestConfig { fn from_env() -> Option { let connector = std::env::var("TEST_CONNECTOR").ok(); let connector_version = std::env::var("TEST_CONNECTOR_VERSION").ok(); - let external_test_executor = std::env::var("EXTERNAL_TEST_EXECUTOR").ok(); + let external_test_executor = std::env::var("EXTERNAL_TEST_EXECUTOR") + .map(|value| serde_json::from_str::(&value).ok()) + .unwrap_or_default(); + let driver_adapter = std::env::var("DRIVER_ADAPTER").ok(); let driver_adapter_config = std::env::var("DRIVER_ADAPTER_CONFIG") .map(|config| serde_json::from_str::(config.as_str()).ok()) @@ -155,31 +173,24 @@ impl TestConfig { }) } - /// if the loaded value for external_test_executor is "default" (case insensitive), - /// and the workspace_root is set, then use the default external test executor. - fn fill_defaults(&mut self) { + fn workspace_root() -> Option { + env::var("WORKSPACE_ROOT").ok().map(PathBuf::from) + } + + pub fn external_test_executor_path(&self) -> Option { const DEFAULT_TEST_EXECUTOR: &str = "query-engine/driver-adapters/connector-test-kit-executor/script/start_node.sh"; - - if self - .external_test_executor + self.external_test_executor .as_ref() - .filter(|s| s.eq_ignore_ascii_case("default")) - .is_some() - { - self.external_test_executor = Self::workspace_root() - .map(|path| path.join(DEFAULT_TEST_EXECUTOR)) - .or_else(|| { + .and_then(|_| { + Self::workspace_root().or_else(|| { exit_with_message( "WORKSPACE_ROOT needs to be correctly set to the root of the prisma-engines repository", ) }) - .and_then(|path| path.to_str().map(|s| s.to_owned())); - } - } - - fn workspace_root() -> Option { - env::var("WORKSPACE_ROOT").ok().map(PathBuf::from) + }) + .map(|path| path.join(DEFAULT_TEST_EXECUTOR)) + .and_then(|path| path.to_str().map(|s| s.to_owned())) } fn validate(&self) { @@ -206,7 +217,7 @@ impl TestConfig { Err(err) => exit_with_message(&err.to_string()), } - if let Some(file) = self.external_test_executor.as_ref() { + if let Some(file) = self.external_test_executor_path().as_ref() { let path = PathBuf::from(file); let md = path.metadata(); if !path.exists() || md.is_err() || !md.unwrap().is_file() { @@ -259,8 +270,8 @@ impl TestConfig { self.is_ci } - pub fn external_test_executor(&self) -> Option<&str> { - self.external_test_executor.as_deref() + pub fn external_test_executor(&self) -> Option { + self.external_test_executor.clone() } pub fn driver_adapter(&self) -> Option<&str> { @@ -294,11 +305,16 @@ impl TestConfig { vec!( ( "DRIVER_ADAPTER".to_string(), - self.driver_adapter.clone().unwrap_or_default()), + self.driver_adapter.clone().unwrap_or_default() + ), ( "DRIVER_ADAPTER_CONFIG".to_string(), self.json_stringify_driver_adapter_config() ), + ( + "EXTERNAL_TEST_EXECUTOR".to_string(), + self.external_test_executor.clone().unwrap_or(TestExecutor::Napi).to_string(), + ), ( "PRISMA_DISABLE_QUAINT_EXECUTORS".to_string(), "1".to_string(), diff --git a/query-engine/connector-test-kit-rs/query-tests-setup/src/connector_tag/js/external_process.rs b/query-engine/connector-test-kit-rs/query-tests-setup/src/connector_tag/js/external_process.rs index 583d5058c62e..1abfedbaf8ee 100644 --- a/query-engine/connector-test-kit-rs/query-tests-setup/src/connector_tag/js/external_process.rs +++ b/query-engine/connector-test-kit-rs/query-tests-setup/src/connector_tag/js/external_process.rs @@ -74,7 +74,7 @@ impl ExecutorProcess { }; self.task_handle.send((method_call, sender)).await?; - let raw_response = receiver.await?; + let raw_response = receiver.await??; tracing::debug!(%raw_response); let response = serde_json::from_value(raw_response)?; Ok(response) @@ -91,14 +91,17 @@ pub(super) static EXTERNAL_PROCESS: Lazy = } }); -type ReqImpl = (jsonrpc_core::MethodCall, oneshot::Sender); +type ReqImpl = ( + jsonrpc_core::MethodCall, + oneshot::Sender>, +); fn start_rpc_thread(mut receiver: mpsc::Receiver) -> Result<()> { use std::process::Stdio; use tokio::process::Command; let path = crate::CONFIG - .external_test_executor() + .external_test_executor_path() .unwrap_or_else(|| exit_with_message(1, "start_rpc_thread() error: external test executor is not set")); tokio::runtime::Builder::new_current_thread() @@ -106,7 +109,7 @@ fn start_rpc_thread(mut receiver: mpsc::Receiver) -> Result<()> { .build() .unwrap() .block_on(async move { - let process = match Command::new(path) + let process = match Command::new(&path) .envs(CONFIG.for_external_executor()) .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -119,7 +122,7 @@ fn start_rpc_thread(mut receiver: mpsc::Receiver) -> Result<()> { let mut stdout = BufReader::new(process.stdout.unwrap()).lines(); let mut stdin = process.stdin.unwrap(); - let mut pending_requests: HashMap> = + let mut pending_requests: HashMap>> = HashMap::new(); loop { @@ -140,10 +143,11 @@ fn start_rpc_thread(mut receiver: mpsc::Receiver) -> Result<()> { // The other end may be dropped if the whole // request future was dropped and not polled to // completion, so we ignore send errors here. - _ = sender.send(success.result); + _ = sender.send(Ok(success.result)); } jsonrpc_core::Output::Failure(err) => { - panic!("error response from jsonrpc: {err:?}") + tracing::error!("error response from jsonrpc: {err:?}"); + _ = sender.send(Err(Box::new(err.error))); } } } diff --git a/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite b/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite index 9638e3a22840..d1532fc12584 100644 --- a/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite +++ b/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite @@ -1,5 +1,5 @@ { "connector": "sqlite", "driver_adapter": "libsql", - "external_test_executor": "default" + "external_test_executor": "Napi" } \ No newline at end of file diff --git a/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite-wasm b/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite-wasm new file mode 100644 index 000000000000..b93966875dea --- /dev/null +++ b/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite-wasm @@ -0,0 +1,5 @@ +{ + "connector": "sqlite", + "driver_adapter": "libsql", + "external_test_executor": "Wasm" +} \ No newline at end of file diff --git a/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13 b/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13 index 0097d8c91f57..bb2034d0e460 100644 --- a/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13 +++ b/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13 @@ -3,5 +3,5 @@ "version": "13", "driver_adapter": "neon:ws", "driver_adapter_config": { "proxyUrl": "127.0.0.1:5488/v1" }, - "external_test_executor": "default" + "external_test_executor": "Napi" } \ No newline at end of file diff --git a/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13-wasm b/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13-wasm new file mode 100644 index 000000000000..6b1e9c0d1286 --- /dev/null +++ b/query-engine/connector-test-kit-rs/test-configs/neon-ws-postgres13-wasm @@ -0,0 +1,7 @@ +{ + "connector": "postgres", + "version": "13", + "driver_adapter": "neon:ws", + "driver_adapter_config": { "proxyUrl": "127.0.0.1:5488/v1" }, + "external_test_executor": "Wasm" +} \ No newline at end of file diff --git a/query-engine/connector-test-kit-rs/test-configs/pg-postgres13 b/query-engine/connector-test-kit-rs/test-configs/pg-postgres13 index 00f0c75ed736..4a2653dd3d2e 100644 --- a/query-engine/connector-test-kit-rs/test-configs/pg-postgres13 +++ b/query-engine/connector-test-kit-rs/test-configs/pg-postgres13 @@ -2,5 +2,5 @@ "connector": "postgres", "version": "13", "driver_adapter": "pg", - "external_test_executor": "default" + "external_test_executor": "Napi" } \ No newline at end of file diff --git a/query-engine/connector-test-kit-rs/test-configs/pg-postgres13-wasm b/query-engine/connector-test-kit-rs/test-configs/pg-postgres13-wasm new file mode 100644 index 000000000000..b5d8ac3c7b15 --- /dev/null +++ b/query-engine/connector-test-kit-rs/test-configs/pg-postgres13-wasm @@ -0,0 +1,6 @@ +{ + "connector": "postgres", + "version": "13", + "driver_adapter": "pg", + "external_test_executor": "Wasm" +} \ No newline at end of file diff --git a/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8 b/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8 index 48c89c79427c..b823cc106997 100644 --- a/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8 +++ b/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8 @@ -3,5 +3,5 @@ "version": "8.0", "driver_adapter": "planetscale", "driver_adapter_config": { "proxyUrl": "http://root:root@127.0.0.1:8085" }, - "external_test_executor": "default" + "external_test_executor": "Napi" } diff --git a/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8-wasm b/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8-wasm new file mode 100644 index 000000000000..d4ee0627759a --- /dev/null +++ b/query-engine/connector-test-kit-rs/test-configs/planetscale-vitess8-wasm @@ -0,0 +1,7 @@ +{ + "connector": "vitess", + "version": "8.0", + "driver_adapter": "planetscale", + "driver_adapter_config": { "proxyUrl": "http://root:root@127.0.0.1:8085" }, + "external_test_executor": "Wasm" +} diff --git a/query-engine/driver-adapters/connector-test-kit-executor/package.json b/query-engine/driver-adapters/connector-test-kit-executor/package.json index 4648887f5063..e872d5684450 100644 --- a/query-engine/driver-adapters/connector-test-kit-executor/package.json +++ b/query-engine/driver-adapters/connector-test-kit-executor/package.json @@ -12,6 +12,11 @@ "scripts": { "build": "tsup ./src/index.ts --format esm --dts" }, + "tsup": { + "external": [ + "../../../query-engine-wasm/pkg/query_engine_bg.js" + ] + }, "keywords": [], "author": "", "sideEffects": false, diff --git a/query-engine/driver-adapters/connector-test-kit-executor/src/index.ts b/query-engine/driver-adapters/connector-test-kit-executor/src/index.ts index 2318c0525760..3ea8aaf147b9 100644 --- a/query-engine/driver-adapters/connector-test-kit-executor/src/index.ts +++ b/query-engine/driver-adapters/connector-test-kit-executor/src/index.ts @@ -1,5 +1,4 @@ import * as qe from './qe' -import * as engines from './engines/Library' import * as readline from 'node:readline' import * as jsonRpc from './jsonRpc' @@ -76,7 +75,7 @@ async function main(): Promise { } const state: Record = {} @@ -215,10 +214,10 @@ function respondOk(requestId: number, payload: unknown) { console.log(JSON.stringify(msg)) } -async function initQe(url: string, prismaSchema: string, logCallback: qe.QueryLogCallback): Promise<[engines.QueryEngineInstance, ErrorCapturingDriverAdapter]> { +async function initQe(url: string, prismaSchema: string, logCallback: qe.QueryLogCallback): Promise<[qe.QueryEngine, ErrorCapturingDriverAdapter]> { const adapter = await adapterFromEnv(url) as DriverAdapter const errorCapturingAdapter = bindAdapter(adapter) - const engineInstance = qe.initQueryEngine(errorCapturingAdapter, prismaSchema, logCallback, debug) + const engineInstance = await qe.initQueryEngine(errorCapturingAdapter, prismaSchema, logCallback, debug) return [engineInstance, errorCapturingAdapter]; } diff --git a/query-engine/driver-adapters/connector-test-kit-executor/src/qe.ts b/query-engine/driver-adapters/connector-test-kit-executor/src/qe.ts index 186d7a9e80d2..20e9a4917fb5 100644 --- a/query-engine/driver-adapters/connector-test-kit-executor/src/qe.ts +++ b/query-engine/driver-adapters/connector-test-kit-executor/src/qe.ts @@ -1,22 +1,24 @@ import type { ErrorCapturingDriverAdapter } from '@prisma/driver-adapter-utils' -import * as lib from './engines/Library' +import * as napi from './engines/Library' import * as os from 'node:os' import * as path from 'node:path' +import { fileURLToPath } from 'node:url' -export type QueryLogCallback = (log: string) => void +const dirname = path.dirname(fileURLToPath(import.meta.url)) -export function initQueryEngine(adapter: ErrorCapturingDriverAdapter, datamodel: string, queryLogCallback: QueryLogCallback, debug: (...args: any[]) => void): lib.QueryEngineInstance { - // I assume nobody will run this on Windows ¯\_(ツ)_/¯ - const libExt = os.platform() === 'darwin' ? 'dylib' : 'so' - const dirname = path.dirname(new URL(import.meta.url).pathname) +export interface QueryEngine { + connect(trace: string): Promise + disconnect(trace: string): Promise; + query(body: string, trace: string, tx_id?: string): Promise; + startTransaction(input: string, trace: string): Promise; + commitTransaction(tx_id: string, trace: string): Promise; + rollbackTransaction(tx_id: string, trace: string): Promise; +} - const libQueryEnginePath = path.join(dirname, `../../../../target/debug/libquery_engine.${libExt}`) +export type QueryLogCallback = (log: string) => void - const libqueryEngine = { exports: {} as unknown as lib.Library } - // @ts-ignore - process.dlopen(libqueryEngine, libQueryEnginePath) - const QueryEngine = libqueryEngine.exports.QueryEngine +export async function initQueryEngine(adapter: ErrorCapturingDriverAdapter, datamodel: string, queryLogCallback: QueryLogCallback, debug: (...args: any[]) => void): QueryEngine { const queryEngineOptions = { datamodel, @@ -37,5 +39,29 @@ export function initQueryEngine(adapter: ErrorCapturingDriverAdapter, datamodel: debug(parsed) } - return new QueryEngine(queryEngineOptions, logCallback, adapter) + const engineFromEnv = process.env.EXTERNAL_TEST_EXECUTOR ?? 'Napi' + if (engineFromEnv === 'Wasm') { + const { WasmQueryEngine } = await import('./wasm') + return new WasmQueryEngine(queryEngineOptions, logCallback, adapter) + } else if (engineFromEnv === 'Napi') { + const { QueryEngine } = loadNapiEngine() + return new QueryEngine(queryEngineOptions, logCallback, adapter) + } else { + throw new TypeError(`Invalid EXTERNAL_TEST_EXECUTOR value: ${engineFromEnv}. Expected Napi or Wasm`) + } + + } + +function loadNapiEngine(): napi.Library { + // I assume nobody will run this on Windows ¯\_(ツ)_/¯ + const libExt = os.platform() === 'darwin' ? 'dylib' : 'so' + + const libQueryEnginePath = path.join(dirname, `../../../../target/debug/libquery_engine.${libExt}`) + + const libqueryEngine = { exports: {} as unknown as napi.Library } + // @ts-ignore + process.dlopen(libqueryEngine, libQueryEnginePath) + + return libqueryEngine.exports +} \ No newline at end of file diff --git a/query-engine/driver-adapters/connector-test-kit-executor/src/wasm.ts b/query-engine/driver-adapters/connector-test-kit-executor/src/wasm.ts new file mode 100644 index 000000000000..439fd0c3f94f --- /dev/null +++ b/query-engine/driver-adapters/connector-test-kit-executor/src/wasm.ts @@ -0,0 +1,14 @@ +import * as wasm from '../../../query-engine-wasm/pkg/query_engine_bg.js' +import fs from 'node:fs/promises' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const dirname = path.dirname(fileURLToPath(import.meta.url)) + +const bytes = await fs.readFile(path.resolve(dirname, '..', '..', '..', 'query-engine-wasm', 'pkg', 'query_engine_bg.wasm')) +const module = new WebAssembly.Module(bytes) +const instance = new WebAssembly.Instance(module, { './query_engine_bg.js': wasm }) +wasm.__wbg_set_wasm(instance.exports); +wasm.init() + +export const WasmQueryEngine = wasm.QueryEngine \ No newline at end of file diff --git a/query-engine/driver-adapters/connector-test-kit-executor/tsconfig.json b/query-engine/driver-adapters/connector-test-kit-executor/tsconfig.json index 516c114b3e15..20fc4bd62ff7 100644 --- a/query-engine/driver-adapters/connector-test-kit-executor/tsconfig.json +++ b/query-engine/driver-adapters/connector-test-kit-executor/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "ES2022", "module": "ESNext", - "lib": ["ES2022"], + "lib": ["ES2022", "DOM"], "moduleResolution": "Bundler", "esModuleInterop": false, "isolatedModules": true, @@ -17,7 +17,7 @@ "skipDefaultLibCheck": true, "skipLibCheck": true, "emitDeclarationOnly": true, - "resolveJsonModule": true + "resolveJsonModule": true, }, "exclude": ["**/dist", "**/declaration", "**/node_modules", "**/src/__tests__"] } \ No newline at end of file diff --git a/query-engine/query-engine-wasm/Cargo.toml b/query-engine/query-engine-wasm/Cargo.toml index dff7985af27c..fdccc773eaf3 100644 --- a/query-engine/query-engine-wasm/Cargo.toml +++ b/query-engine/query-engine-wasm/Cargo.toml @@ -15,13 +15,16 @@ user-facing-errors = { path = "../../libs/user-facing-errors" } psl.workspace = true prisma-models = { path = "../prisma-models" } quaint = { path = "../../quaint" } -request-handlers = { path = "../request-handlers", default-features = false, features = ["sql", "driver-adapters"] } +request-handlers = { path = "../request-handlers", default-features = false, features = [ + "sql", + "driver-adapters", +] } connector = { path = "../connectors/query-connector", package = "query-connector" } sql-query-connector = { path = "../connectors/sql-query-connector" } query-core = { path = "../core" } thiserror = "1" -connection-string.workspace = true +connection-string.workspace = true url = "2" serde_json.workspace = true serde.workspace = true