From d771e84df0f15ee0705c48e12048bd743e02fca1 Mon Sep 17 00:00:00 2001 From: Serhii Tatarintsev Date: Wed, 7 Feb 2024 14:49:07 +0100 Subject: [PATCH] qe-wasm: split by connector (#4681) * qe-wasm: Split query engine by connector Intoroduces separate features for mysql, postgresql and sqlite connectors and builds 3 individual engines for each. All engines are put and published as a single package (with connector-specific) engine in subdirectory. It is up to a client to pick the correct one. Contributes to prisma/team-orm#891 * feat: some of the review comments * Fix size job * Fix size again --- .github/workflows/test-unit-tests.yml | 2 +- .github/workflows/wasm-size.yml | 19 +- libs/user-facing-errors/Cargo.toml | 2 +- nix/all-engines.nix | 15 +- nix/publish-engine-size.nix | 8 +- quaint/src/ast/function.rs | 2 +- quaint/src/error/mod.rs | 3 + .../connectors/sql-query-connector/Cargo.toml | 8 +- query-engine/driver-adapters/Cargo.toml | 7 +- .../driver-adapters/executor/package.json | 4 +- .../driver-adapters/executor/src/qe.ts | 3 +- .../driver-adapters/executor/src/wasm.ts | 34 ++- .../driver-adapters/src/conversion/mod.rs | 3 + query-engine/driver-adapters/src/error.rs | 19 +- query-engine/driver-adapters/src/lib.rs | 3 + query-engine/driver-adapters/src/queryable.rs | 13 +- query-engine/driver-adapters/src/types.rs | 12 ++ query-engine/query-engine-node-api/Cargo.toml | 17 +- query-engine/query-engine-wasm/Cargo.toml | 15 +- query-engine/query-engine-wasm/build.sh | 193 ++++++++---------- query-engine/query-engine-wasm/package.json | 5 + .../query-engine-wasm/src/wasm/engine.rs | 13 +- query-engine/request-handlers/Cargo.toml | 2 +- 23 files changed, 249 insertions(+), 153 deletions(-) create mode 100644 query-engine/query-engine-wasm/package.json diff --git a/.github/workflows/test-unit-tests.yml b/.github/workflows/test-unit-tests.yml index 39f1dcb7e32d..d4fea49b9c0a 100644 --- a/.github/workflows/test-unit-tests.yml +++ b/.github/workflows/test-unit-tests.yml @@ -28,7 +28,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - run: | - cargo test --workspace \ + cargo test --workspace --all-features \ --exclude=quaint \ --exclude=query-engine \ --exclude=query-engine-node-api \ diff --git a/.github/workflows/wasm-size.yml b/.github/workflows/wasm-size.yml index 20513356061f..7688ae9d36d4 100644 --- a/.github/workflows/wasm-size.yml +++ b/.github/workflows/wasm-size.yml @@ -1,14 +1,15 @@ name: "QE: WASM size" on: - pull_request: - paths-ignore: - - ".github/**" - - "!.github/workflows/wasm-size.yml" - - ".buildkite/**" - - "*.md" - - "LICENSE" - - "CODEOWNERS" - - "renovate.json" + workflow_dispatch: + # pull_request: + # paths-ignore: + # - ".github/**" + # - "!.github/workflows/wasm-size.yml" + # - ".buildkite/**" + # - "*.md" + # - "LICENSE" + # - "CODEOWNERS" + # - "renovate.json" concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/libs/user-facing-errors/Cargo.toml b/libs/user-facing-errors/Cargo.toml index 8e715e7d66c9..cbfdde2ead48 100644 --- a/libs/user-facing-errors/Cargo.toml +++ b/libs/user-facing-errors/Cargo.toml @@ -11,7 +11,7 @@ backtrace = "0.3.40" tracing = "0.1" indoc.workspace = true itertools.workspace = true -quaint = { path = "../../quaint", optional = true } +quaint = { path = "../../quaint", default-features = false, optional = true } [features] default = [] diff --git a/nix/all-engines.nix b/nix/all-engines.nix index 0d7e08bb9836..b509d67229ad 100644 --- a/nix/all-engines.nix +++ b/nix/all-engines.nix @@ -117,7 +117,7 @@ in packages.build-engine-wasm = pkgs.writeShellApplication { name = "build-engine-wasm"; - runtimeInputs = with pkgs; [ git rustup wasm-pack wasm-bindgen-cli binaryen jq iconv]; + runtimeInputs = with pkgs; [ git rustup wasm-bindgen-cli binaryen jq iconv ]; text = '' cd query-engine/query-engine-wasm WASM_BUILD_PROFILE=release ./build.sh "$1" "$2" @@ -128,19 +128,26 @@ in ({ profile }: stdenv.mkDerivation { name = "query-engine-wasm-gz"; inherit src; + buildInputs = with pkgs; [ iconv ]; buildPhase = '' export HOME=$(mktemp -dt wasm-engine-home-XXXX) OUT_FOLDER=$(mktemp -dt wasm-engine-out-XXXX) ${self'.packages.build-engine-wasm}/bin/build-engine-wasm "0.0.0" "$OUT_FOLDER" - gzip -ckn "$OUT_FOLDER/query_engine_bg.wasm" > query_engine_bg.wasm.gz + + for provider in "postgresql" "mysql" "sqlite"; do + gzip -ckn "$OUT_FOLDER/$provider/query_engine_bg.wasm" > "query-engine-$provider.wasm.gz" + done ''; installPhase = '' + set +x mkdir -p $out - cp "$OUT_FOLDER/query_engine_bg.wasm" $out/ - cp query_engine_bg.wasm.gz $out/ + for provider in "postgresql" "mysql" "sqlite"; do + cp "$OUT_FOLDER/$provider/query_engine_bg.wasm" "$out/query-engine-$provider.wasm" + cp "query-engine-$provider.wasm.gz" "$out/" + done ''; }) { profile = "release"; }; diff --git a/nix/publish-engine-size.nix b/nix/publish-engine-size.nix index 6d2f7c4a1eef..b49c093d9a31 100644 --- a/nix/publish-engine-size.nix +++ b/nix/publish-engine-size.nix @@ -46,8 +46,12 @@ ${self'.packages.update-engine-size}/bin/update-engine-size \ ${self'.packages.query-engine-bin-and-lib}/bin/query-engine \ ${self'.packages.query-engine-bin-and-lib}/lib/libquery_engine.node \ - ${self'.packages.query-engine-wasm-gz}/query_engine_bg.wasm.gz \ - ${self'.packages.query-engine-wasm-gz}/query_engine_bg.wasm + ${self'.packages.query-engine-wasm-gz}/query-engine-postgresql.wasm.gz \ + ${self'.packages.query-engine-wasm-gz}/query-engine-postgresql.wasm \ + ${self'.packages.query-engine-wasm-gz}/query-engine-mysql.wasm.gz \ + ${self'.packages.query-engine-wasm-gz}/query-engine-mysql.wasm \ + ${self'.packages.query-engine-wasm-gz}/query-engine-sqlite.wasm.gz \ + ${self'.packages.query-engine-wasm-gz}/query-engine-sqlite.wasm git add "$CSV_PATH" git commit --quiet -m "update engines size for $CURRENT_COMMIT_SHORT" diff --git a/quaint/src/ast/function.rs b/quaint/src/ast/function.rs index 659cf03bfac3..246ea762b34e 100644 --- a/quaint/src/ast/function.rs +++ b/quaint/src/ast/function.rs @@ -44,7 +44,7 @@ pub use minimum::*; pub use row_number::*; #[cfg(feature = "postgresql")] pub use row_to_json::*; -#[cfg(feature = "mysql")] +#[cfg(any(feature = "mysql", feature = "postgresql"))] pub use search::*; pub use sum::*; pub use upper::*; diff --git a/quaint/src/error/mod.rs b/quaint/src/error/mod.rs index aec1adc1648a..c28e97970ebc 100644 --- a/quaint/src/error/mod.rs +++ b/quaint/src/error/mod.rs @@ -15,8 +15,11 @@ use std::time::Duration; #[cfg(not(target_arch = "wasm32"))] pub use native::NativeErrorKind; +#[cfg(feature = "mysql")] pub use crate::connector::mysql::MysqlError; +#[cfg(feature = "postgresql")] pub use crate::connector::postgres::PostgresError; +#[cfg(feature = "sqlite")] pub use crate::connector::sqlite::SqliteError; pub(crate) use name::Name; diff --git a/query-engine/connectors/sql-query-connector/Cargo.toml b/query-engine/connectors/sql-query-connector/Cargo.toml index 354ec5bc0887..aae27b671c44 100644 --- a/query-engine/connectors/sql-query-connector/Cargo.toml +++ b/query-engine/connectors/sql-query-connector/Cargo.toml @@ -4,7 +4,11 @@ name = "sql-query-connector" version = "0.1.0" [features] +default = ["postgresql", "sqlite", "mysql"] vendored-openssl = ["quaint/vendored-openssl"] +postgresql = ["quaint/postgresql"] +sqlite = ["quaint/sqlite"] +mysql = ["quaint/mysql"] # Enable Driver Adapters driver-adapters = [] @@ -18,7 +22,7 @@ futures = "0.3" itertools.workspace = true once_cell = "1.3" rand = "0.7" -serde_json = {version = "1.0", features = ["float_roundtrip"]} +serde_json = { version = "1.0", features = ["float_roundtrip"] } thiserror = "1.0" tokio = { version = "1.0", features = ["macros", "time"] } tracing = "0.1" @@ -32,7 +36,7 @@ cuid = { git = "https://github.com/prisma/cuid-rust", branch = "wasm32-support" quaint.workspace = true [target.'cfg(target_arch = "wasm32")'.dependencies] -quaint = { path = "../../../quaint" } +quaint = { path = "../../../quaint", default-features = false } [dependencies.connector-interface] package = "query-connector" diff --git a/query-engine/driver-adapters/Cargo.toml b/query-engine/driver-adapters/Cargo.toml index 9f9db91287f0..e2c051204cf0 100644 --- a/query-engine/driver-adapters/Cargo.toml +++ b/query-engine/driver-adapters/Cargo.toml @@ -3,6 +3,11 @@ name = "driver-adapters" version = "0.1.0" edition = "2021" +[features] +mysql = ["quaint/mysql"] +sqlite = ["quaint/sqlite"] +postgresql = ["quaint/postgresql"] + [dependencies] async-trait = "0.1" once_cell = "1.15" @@ -28,7 +33,7 @@ napi-derive.workspace = true quaint.workspace = true [target.'cfg(target_arch = "wasm32")'.dependencies] -quaint = { path = "../../quaint" } +quaint = { path = "../../quaint", default-features = false } js-sys.workspace = true serde-wasm-bindgen.workspace = true wasm-bindgen.workspace = true diff --git a/query-engine/driver-adapters/executor/package.json b/query-engine/driver-adapters/executor/package.json index cf35f2997452..cac623ac0930 100644 --- a/query-engine/driver-adapters/executor/package.json +++ b/query-engine/driver-adapters/executor/package.json @@ -12,7 +12,9 @@ }, "tsup": { "external": [ - "../../../query-engine-wasm/pkg/query_engine_bg.js" + "../../../query-engine-wasm/pkg/postgresql/query_engine_bg.js", + "../../../query-engine-wasm/pkg/mysql/query_engine_bg.js", + "../../../query-engine-wasm/pkg/sqlite/query_engine_bg.js" ] }, "keywords": [], diff --git a/query-engine/driver-adapters/executor/src/qe.ts b/query-engine/driver-adapters/executor/src/qe.ts index 6937c02cd4d5..e95f76ff05c5 100644 --- a/query-engine/driver-adapters/executor/src/qe.ts +++ b/query-engine/driver-adapters/executor/src/qe.ts @@ -35,7 +35,8 @@ export async function initQueryEngine( const options = queryEngineOptions(datamodel); if (engineType === "Wasm") { - const { WasmQueryEngine } = await import("./wasm"); + const { getEngineForProvider } = await import("./wasm"); + const WasmQueryEngine = await getEngineForProvider(adapter.provider) return new WasmQueryEngine(options, logCallback, adapter); } else { const { QueryEngine } = loadNapiEngine(); diff --git a/query-engine/driver-adapters/executor/src/wasm.ts b/query-engine/driver-adapters/executor/src/wasm.ts index 6eea2ee36cef..c9040b398395 100644 --- a/query-engine/driver-adapters/executor/src/wasm.ts +++ b/query-engine/driver-adapters/executor/src/wasm.ts @@ -1,13 +1,35 @@ -import * as wasm from '../../../query-engine-wasm/pkg/query_engine_bg.js' +import * as wasmPostgres from '../../../query-engine-wasm/pkg/postgresql/query_engine_bg.js' +import * as wasmMysql from '../../../query-engine-wasm/pkg/mysql/query_engine_bg.js' +import * as wasmSqlite from '../../../query-engine-wasm/pkg/sqlite/query_engine_bg.js' import fs from 'node:fs/promises' import path from 'node:path' import { fileURLToPath } from 'node:url' const dirname = path.dirname(fileURLToPath(import.meta.url)) -const bytes = await fs.readFile(path.resolve(dirname, '..', '..', '..', 'query-engine-wasm', 'pkg', 'query_engine_bg.wasm')) -const module = new WebAssembly.Module(bytes) -const instance = new WebAssembly.Instance(module, { './query_engine_bg.js': wasm }) -wasm.__wbg_set_wasm(instance.exports); +const wasm = { + postgres: wasmPostgres, + mysql: wasmMysql, + sqlite: wasmSqlite +} -export const WasmQueryEngine = wasm.QueryEngine \ No newline at end of file +type EngineName = keyof typeof wasm; + +const initializedModules = new Set() + + + +export async function getEngineForProvider(provider: EngineName) { + const engine = wasm[provider] + if (!initializedModules.has(provider)) { + const subDir = provider === 'postgres' ? 'postgresql' : provider + const bytes = await fs.readFile(path.resolve(dirname, '..', '..', '..', 'query-engine-wasm', 'pkg', subDir, 'query_engine_bg.wasm')) + console.error(bytes) + const module = new WebAssembly.Module(bytes) + const instance = new WebAssembly.Instance(module, { './query_engine_bg.js': engine }) + engine.__wbg_set_wasm(instance.exports); + initializedModules.add(provider) + } + + return engine.QueryEngine +} diff --git a/query-engine/driver-adapters/src/conversion/mod.rs b/query-engine/driver-adapters/src/conversion/mod.rs index 34ad8d3ae66c..8b7808517ba4 100644 --- a/query-engine/driver-adapters/src/conversion/mod.rs +++ b/query-engine/driver-adapters/src/conversion/mod.rs @@ -1,8 +1,11 @@ pub(crate) mod js_arg; pub(crate) mod js_to_quaint; +#[cfg(feature = "mysql")] pub(crate) mod mysql; +#[cfg(feature = "postgresql")] pub(crate) mod postgres; +#[cfg(feature = "sqlite")] pub(crate) mod sqlite; pub use js_arg::JSArg; diff --git a/query-engine/driver-adapters/src/error.rs b/query-engine/driver-adapters/src/error.rs index fa01759d9213..d60285259135 100644 --- a/query-engine/driver-adapters/src/error.rs +++ b/query-engine/driver-adapters/src/error.rs @@ -1,6 +1,14 @@ -use quaint::error::{MysqlError, PostgresError, SqliteError}; +#[cfg(feature = "mysql")] +use quaint::error::MysqlError; + +#[cfg(feature = "postgresql")] +use quaint::error::PostgresError; + +#[cfg(feature = "sqlite")] +use quaint::error::SqliteError; use serde::Deserialize; +#[cfg(feature = "postgresql")] #[derive(Deserialize)] #[serde(remote = "PostgresError")] pub struct PostgresErrorDef { @@ -12,6 +20,7 @@ pub struct PostgresErrorDef { hint: Option, } +#[cfg(feature = "mysql")] #[derive(Deserialize)] #[serde(remote = "MysqlError")] pub struct MysqlErrorDef { @@ -20,6 +29,7 @@ pub struct MysqlErrorDef { pub state: String, } +#[cfg(feature = "sqlite")] #[derive(Deserialize)] #[serde(remote = "SqliteError", rename_all = "camelCase")] pub struct SqliteErrorDef { @@ -32,14 +42,15 @@ pub struct SqliteErrorDef { /// Wrapper for JS-side errors pub(crate) enum DriverAdapterError { /// Unexpected JS exception - GenericJs { - id: i32, - }, + GenericJs { id: i32 }, UnsupportedNativeDataType { #[serde(rename = "type")] native_type: String, }, + #[cfg(feature = "postgresql")] Postgres(#[serde(with = "PostgresErrorDef")] PostgresError), + #[cfg(feature = "mysql")] Mysql(#[serde(with = "MysqlErrorDef")] MysqlError), + #[cfg(feature = "sqlite")] Sqlite(#[serde(with = "SqliteErrorDef")] SqliteError), } diff --git a/query-engine/driver-adapters/src/lib.rs b/query-engine/driver-adapters/src/lib.rs index b5b1cf666e06..55c7de41eb8d 100644 --- a/query-engine/driver-adapters/src/lib.rs +++ b/query-engine/driver-adapters/src/lib.rs @@ -34,8 +34,11 @@ impl From for QuaintError { .build() } DriverAdapterError::GenericJs { id } => QuaintError::external_error(id), + #[cfg(feature = "postgresql")] DriverAdapterError::Postgres(e) => e.into(), + #[cfg(feature = "mysql")] DriverAdapterError::Mysql(e) => e.into(), + #[cfg(feature = "sqlite")] DriverAdapterError::Sqlite(e) => e.into(), // in future, more error types would be added and we'll need to convert them to proper QuaintErrors here } diff --git a/query-engine/driver-adapters/src/queryable.rs b/query-engine/driver-adapters/src/queryable.rs index a4599019003e..6250d137fbb6 100644 --- a/query-engine/driver-adapters/src/queryable.rs +++ b/query-engine/driver-adapters/src/queryable.rs @@ -41,8 +41,11 @@ impl JsBaseQueryable { /// visit a quaint query AST according to the provider of the JS connector fn visit_quaint_query<'a>(&self, q: QuaintQuery<'a>) -> quaint::Result<(String, Vec>)> { match self.provider { + #[cfg(feature = "mysql")] AdapterFlavour::Mysql => visitor::Mysql::build(q), + #[cfg(feature = "postgresql")] AdapterFlavour::Postgres => visitor::Postgres::build(q), + #[cfg(feature = "sqlite")] AdapterFlavour::Sqlite => visitor::Sqlite::build(q), } } @@ -51,8 +54,11 @@ impl JsBaseQueryable { let sql: String = sql.to_string(); let converter = match self.provider { + #[cfg(feature = "postgresql")] AdapterFlavour::Postgres => conversion::postgres::value_to_js_arg, + #[cfg(feature = "sqlite")] AdapterFlavour::Sqlite => conversion::sqlite::value_to_js_arg, + #[cfg(feature = "mysql")] AdapterFlavour::Mysql => conversion::mysql::value_to_js_arg, }; @@ -125,6 +131,7 @@ impl QuaintQueryable for JsBaseQueryable { return Err(Error::builder(ErrorKind::invalid_isolation_level(&isolation_level)).build()); } + #[cfg(feature = "sqlite")] if self.provider == AdapterFlavour::Sqlite { return match isolation_level { IsolationLevel::Serializable => Ok(()), @@ -138,8 +145,12 @@ impl QuaintQueryable for JsBaseQueryable { fn requires_isolation_first(&self) -> bool { match self.provider { + #[cfg(feature = "mysql")] AdapterFlavour::Mysql => true, - AdapterFlavour::Postgres | AdapterFlavour::Sqlite => false, + #[cfg(feature = "postgresql")] + AdapterFlavour::Postgres => false, + #[cfg(feature = "sqlite")] + AdapterFlavour::Sqlite => false, } } } diff --git a/query-engine/driver-adapters/src/types.rs b/query-engine/driver-adapters/src/types.rs index f79742bfe0c7..6a63edd5aebe 100644 --- a/query-engine/driver-adapters/src/types.rs +++ b/query-engine/driver-adapters/src/types.rs @@ -16,8 +16,11 @@ use serde_repr::{Deserialize_repr, Serialize_repr}; #[cfg_attr(target_arch = "wasm32", derive(Deserialize))] #[derive(Debug, Eq, PartialEq, Clone)] pub enum AdapterFlavour { + #[cfg(feature = "mysql")] Mysql, + #[cfg(feature = "postgresql")] Postgres, + #[cfg(feature = "sqlite")] Sqlite, } @@ -26,8 +29,11 @@ impl FromStr for AdapterFlavour { fn from_str(s: &str) -> Result { match s { + #[cfg(feature = "postgresql")] "postgres" => Ok(Self::Postgres), + #[cfg(feature = "mysql")] "mysql" => Ok(Self::Mysql), + #[cfg(feature = "sqlite")] "sqlite" => Ok(Self::Sqlite), _ => Err(format!("Unsupported adapter flavour: {:?}", s)), } @@ -37,8 +43,11 @@ impl FromStr for AdapterFlavour { impl From for SqlFamily { fn from(value: AdapterFlavour) -> Self { match value { + #[cfg(feature = "mysql")] AdapterFlavour::Mysql => SqlFamily::Mysql, + #[cfg(feature = "postgresql")] AdapterFlavour::Postgres => SqlFamily::Postgres, + #[cfg(feature = "sqlite")] AdapterFlavour::Sqlite => SqlFamily::Sqlite, } } @@ -67,8 +76,11 @@ impl JsConnectionInfo { fn default_schema_name(&self, provider: &AdapterFlavour) -> &str { match provider { + #[cfg(feature = "mysql")] AdapterFlavour::Mysql => quaint::connector::DEFAULT_MYSQL_DB, + #[cfg(feature = "postgresql")] AdapterFlavour::Postgres => quaint::connector::DEFAULT_POSTGRES_SCHEMA, + #[cfg(feature = "sqlite")] AdapterFlavour::Sqlite => quaint::connector::DEFAULT_SQLITE_DATABASE, } } diff --git a/query-engine/query-engine-node-api/Cargo.toml b/query-engine/query-engine-node-api/Cargo.toml index 10168eafa25d..e477626702fe 100644 --- a/query-engine/query-engine-node-api/Cargo.toml +++ b/query-engine/query-engine-node-api/Cargo.toml @@ -11,7 +11,10 @@ name = "query_engine" [features] default = ["driver-adapters"] vendored-openssl = ["sql-connector/vendored-openssl"] -driver-adapters = ["request-handlers/driver-adapters", "sql-connector/driver-adapters"] +driver-adapters = [ + "request-handlers/driver-adapters", + "sql-connector/driver-adapters", +] [dependencies] anyhow = "1" @@ -24,12 +27,16 @@ user-facing-errors = { path = "../../libs/user-facing-errors" } psl.workspace = true sql-connector = { path = "../connectors/sql-query-connector", package = "sql-query-connector" } query-structure = { path = "../query-structure" } -driver-adapters = { path = "../driver-adapters" } +driver-adapters = { path = "../driver-adapters", features = [ + "postgresql", + "sqlite", + "mysql", +] } napi.workspace = true napi-derive.workspace = true thiserror = "1" -connection-string.workspace = true +connection-string.workspace = true url = "2" serde_json.workspace = true serde.workspace = true @@ -38,12 +45,12 @@ tracing = "0.1" tracing-subscriber = { version = "0.3" } tracing-futures = "0.2" tracing-opentelemetry = "0.17.3" -opentelemetry = { version = "0.17"} +opentelemetry = { version = "0.17" } quaint.workspace = true tokio.workspace = true futures = "0.3" -query-engine-metrics = {path = "../metrics"} +query-engine-metrics = { path = "../metrics" } [build-dependencies] napi-build = "1" diff --git a/query-engine/query-engine-wasm/Cargo.toml b/query-engine/query-engine-wasm/Cargo.toml index 9dc877893669..931f04c399f0 100644 --- a/query-engine/query-engine-wasm/Cargo.toml +++ b/query-engine/query-engine-wasm/Cargo.toml @@ -8,6 +8,11 @@ doc = false crate-type = ["cdylib"] name = "query_engine_wasm" +[features] +sqlite = ["driver-adapters/sqlite", "sql-connector/sqlite"] +postgresql = ["driver-adapters/postgresql", "sql-connector/postgresql"] +mysql = ["driver-adapters/mysql", "sql-connector/mysql"] + [dependencies] query-connector = { path = "../connectors/query-connector" } @@ -17,14 +22,14 @@ async-trait = "0.1" user-facing-errors = { path = "../../libs/user-facing-errors" } psl.workspace = true query-structure = { path = "../query-structure" } -quaint = { path = "../../quaint" } -sql-connector = { path = "../connectors/sql-query-connector", package = "sql-query-connector" } +sql-connector = { path = "../connectors/sql-query-connector", package = "sql-query-connector", default-features = false } request-handlers = { path = "../request-handlers", default-features = false, features = [ "sql", "driver-adapters", ] } query-core = { path = "../core" } driver-adapters = { path = "../driver-adapters" } +quaint = { path = "../../quaint", default-features = false } connection-string.workspace = true js-sys.workspace = true @@ -45,10 +50,10 @@ tracing = "0.1" tracing-subscriber = { version = "0.3" } tracing-futures = "0.2" tracing-opentelemetry = "0.17.3" -opentelemetry = { version = "0.17"} +opentelemetry = { version = "0.17" } [package.metadata.wasm-pack.profile.release] -wasm-opt = false # use wasm-opt explicitly in `./build.sh` +wasm-opt = false # use wasm-opt explicitly in `./build.sh` [package.metadata.wasm-pack.profile.profiling] -wasm-opt = false # use wasm-opt explicitly in `./build.sh` +wasm-opt = false # use wasm-opt explicitly in `./build.sh` diff --git a/query-engine/query-engine-wasm/build.sh b/query-engine/query-engine-wasm/build.sh index 22ab20d844fe..c3f129bb276b 100755 --- a/query-engine/query-engine-wasm/build.sh +++ b/query-engine/query-engine-wasm/build.sh @@ -2,25 +2,37 @@ # Call this script as `./build.sh ` set -euo pipefail -OUT_VERSION="${1:-}" -OUT_NPM_NAME="@prisma/query-engine-wasm" CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" REPO_ROOT="$( cd "$( dirname "$CURRENT_DIR/../../../" )" >/dev/null 2>&1 && pwd )" - OUT_VERSION="${1:-"0.0.0"}" OUT_FOLDER="${2:-"query-engine/query-engine-wasm/pkg"}" +OUT_TARGET="bundler" +# wasm-opt pass +WASM_OPT_ARGS=( + "-Os" # execute size-focused optimization passes (-Oz actually increases size by 1KB) + "--vacuum" # removes obviously unneeded code + "--duplicate-function-elimination" # removes duplicate functions + "--duplicate-import-elimination" # removes duplicate imports + "--remove-unused-module-elements" # removes unused module elements + "--dae-optimizing" # removes arguments to calls in an lto-like manner + "--remove-unused-names" # removes names from location that are never branched to + "--rse" # removes redundant local.sets + "--gsi" # global struct inference, to optimize constant values + "--gufa-optimizing" # optimize the entire program using type monomorphization + "--strip-dwarf" # removes DWARF debug information + "--strip-producers" # removes the "producers" section + "--strip-target-features" # removes the "target_features" section +) + # if it's a relative path, let it be relative to the repo root if [[ "$OUT_FOLDER" != /* ]]; then OUT_FOLDER="$REPO_ROOT/$OUT_FOLDER" fi +OUT_JSON="${OUT_FOLDER}/package.json" echo "ℹ️ target version: $OUT_VERSION" echo "ℹ️ out folder: $OUT_FOLDER" -OUT_NPM_NAME="@prisma/query-engine-wasm" -OUT_TARGET="bundler" -OUT_JSON="${OUT_FOLDER}/package.json" - if [[ -z "${WASM_BUILD_PROFILE:-}" ]]; then # use `wasm-pack build --release` by default on CI only if [[ -z "${BUILDKITE:-}" ]] && [[ -z "${GITHUB_ACTIONS:-}" ]]; then @@ -30,114 +42,85 @@ if [[ -z "${WASM_BUILD_PROFILE:-}" ]]; then fi fi -echo "Using build profile: \"${WASM_BUILD_PROFILE}\"" - -if ! command -v wasm-pack &> /dev/null -then - echo "wasm-pack could not be found, installing now..." - curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh +if [ "$WASM_BUILD_PROFILE" = "dev" ]; then + WASM_TARGET_SUBDIR="debug" +else + WASM_TARGET_SUBDIR="release" fi +echo "Using build profile: \"${WASM_BUILD_PROFILE}\"" + echo "ℹ️ Configuring rust toolchain to use nightly and rust-src component" rustup default nightly-2024-01-25 rustup target add wasm32-unknown-unknown rustup component add rust-src --target wasm32-unknown-unknown - - -echo "Building query-engine-wasm using $WASM_BUILD_PROFILE profile" export RUSTFLAGS="-Zlocation-detail=none" -CARGO_PROFILE_RELEASE_OPT_LEVEL="z" wasm-pack build "--$WASM_BUILD_PROFILE" --target "$OUT_TARGET" --out-dir "$OUT_FOLDER" --out-name query_engine . \ - -Zbuild-std=std,panic_abort -Zbuild-std-features=panic_immediate_abort +CARGO_TARGET_DIR=$(cargo metadata --format-version 1 | jq -r .target_directory) + +build() { + local CONNECTOR="$1" + echo "🔨 Building $CONNECTOR" + CARGO_PROFILE_RELEASE_OPT_LEVEL="z" cargo build \ + -p query-engine-wasm \ + --profile "$WASM_BUILD_PROFILE" \ + --features "$CONNECTOR" \ + --target wasm32-unknown-unknown \ + -Zbuild-std=std,panic_abort -Zbuild-std-features=panic_immediate_abort + + local IN_FILE="$CARGO_TARGET_DIR/wasm32-unknown-unknown/$WASM_TARGET_SUBDIR/query_engine_wasm.wasm" + local OUT_FILE="$OUT_FOLDER/$CONNECTOR/query_engine_bg.wasm" + + wasm-bindgen --target "$OUT_TARGET" --out-name query_engine --out-dir "$OUT_FOLDER/$CONNECTOR" "$IN_FILE" + optimize "$OUT_FILE" + + if ! command -v wasm2wat &> /dev/null; then + echo "Skipping wasm2wat, as it is not installed." + else + wasm2wat "$OUT_FILE" -o "./query_engine.$CONNECTOR.wat" + fi +} -# wasm-opt pass -WASM_OPT_ARGS=( - "-Os" # execute size-focused optimization passes (-Oz actually increases size by 1KB) - "--vacuum" # removes obviously unneeded code - "--duplicate-function-elimination" # removes duplicate functions - "--duplicate-import-elimination" # removes duplicate imports - "--remove-unused-module-elements" # removes unused module elements - "--dae-optimizing" # removes arguments to calls in an lto-like manner - "--remove-unused-names" # removes names from location that are never branched to - "--rse" # removes redundant local.sets - "--gsi" # global struct inference, to optimize constant values - "--gufa-optimizing" # optimize the entire program using type monomorphization - "--strip-dwarf" # removes DWARF debug information - "--strip-producers" # removes the "producers" section - "--strip-target-features" # removes the "target_features" section -) +optimize() { + local OUT_FILE="$1" + case "$WASM_BUILD_PROFILE" in + release) + # In release mode, we want to strip the debug symbols. + wasm-opt "${WASM_OPT_ARGS[@]}" \ + "--strip-debug" \ + "$OUT_FILE" \ + -o "$OUT_FILE" + ;; + profiling) + # In profiling mode, we want to keep the debug symbols. + wasm-opt "${WASM_OPT_ARGS[@]}" \ + "--debuginfo" \ + "${OUT_FILE}" \ + -o "${OUT_FILE}" + ;; + *) + # In other modes (e.g., "dev"), skip wasm-opt. + echo "Skipping wasm-opt." + ;; + esac +} -echo "🗜️ Optimizing with wasm-opt with $WASM_BUILD_PROFILE profile..." -echo "ℹ️ before raw: $(du -h "${OUT_FOLDER}/query_engine_bg.wasm")" -echo "ℹ️ before zip: $(gzip -c "${OUT_FOLDER}/query_engine_bg.wasm" | wc -c) bytes" -case "$WASM_BUILD_PROFILE" in - release) - # In release mode, we want to strip the debug symbols. - wasm-opt "${WASM_OPT_ARGS[@]}" \ - "--strip-debug" \ - "${OUT_FOLDER}/query_engine_bg.wasm" \ - -o "${OUT_FOLDER}/query_engine_bg.wasm" - ;; - profiling) - # In profiling mode, we want to keep the debug symbols. - wasm-opt "${WASM_OPT_ARGS[@]}" \ - "--debuginfo" \ - "${OUT_FOLDER}/query_engine_bg.wasm" \ - -o "${OUT_FOLDER}/query_engine_bg.wasm" - ;; - *) - # In other modes (e.g., "dev"), skip wasm-opt. - echo "Skipping wasm-opt." - ;; -esac -echo "ℹ️ after raw: $(du -h "${OUT_FOLDER}/query_engine_bg.wasm")" -echo "ℹ️ after zip: $(gzip -c "${OUT_FOLDER}/query_engine_bg.wasm" | wc -c) bytes" - -# Convert the `.wasm` file to its human-friendly `.wat` representation for debugging purposes, if `wasm2wat` is installed -if ! command -v wasm2wat &> /dev/null; then - echo "Skipping wasm2wat, as it is not installed." -else - wasm2wat "${OUT_FOLDER}/query_engine_bg.wasm" -o "./query_engine.wat" -fi +report_size() { + local CONNECTOR="$1" -sleep 1 -# Mark the package as a ES module, set the entry point to the query_engine.js file, mark the package as public -printf '%s\n' "$(jq '. + {"type": "module"} + {"main": "./query_engine.js"} + {"private": false}' "$OUT_JSON")" > "$OUT_JSON" - -# Add the version -printf '%s\n' "$(jq --arg version "$OUT_VERSION" '. + {"version": $version}' "$OUT_JSON")" > "$OUT_JSON" - -# Add the package name -printf '%s\n' "$(jq --arg name "$OUT_NPM_NAME" '. + {"name": $name}' "$OUT_JSON")" > "$OUT_JSON" - -# Some info: enabling Cloudflare Workers in the bindings generated by wasm-package -# is useful for local experiments, but it's not needed here. -# `@prisma/client` has its own `esbuild` plugin for CF-compatible bindings -# and import of `.wasm` files. -enable_cf_in_bindings() { - # Enable Cloudflare Workers in the generated JS bindings. - # The generated bindings are compatible with: - # - Node.js - # - Cloudflare Workers / Miniflare - - local FILE="$1" # e.g., `query_engine.js` - local BG_FILE="${FILE%.js}_bg.js" - local OUTPUT_FILE="${OUT_FOLDER}/${FILE}" - - cat < "$OUTPUT_FILE" -import * as imports from "./${BG_FILE}"; - -// switch between both syntax for Node.js and for workers (Cloudflare Workers) -import * as wkmod from "./${BG_FILE%.js}.wasm"; -import * as nodemod from "./${BG_FILE%.js}.wasm"; -if ((typeof process !== 'undefined') && (process.release.name === 'node')) { - imports.__wbg_set_wasm(nodemod); -} else { - const instance = new WebAssembly.Instance(wkmod.default, { "./${BG_FILE}": imports }); - imports.__wbg_set_wasm(instance.exports); + echo "$CONNECTOR:" + echo "ℹ️ raw: $(du -h "${OUT_FOLDER}/$CONNECTOR/query_engine_bg.wasm")" + echo "ℹ️ zip: $(gzip -c "${OUT_FOLDER}/$CONNECTOR/query_engine_bg.wasm" | wc -c) bytes" + echo "" } -export * from "./${BG_FILE}"; -EOF -} +echo "Building query-engine-wasm using $WASM_BUILD_PROFILE profile" + +build "postgresql" +build "sqlite" +build "mysql" + +jq '.version=$version' --arg version "$OUT_VERSION" package.json > "$OUT_JSON" -enable_cf_in_bindings "query_engine.js" +report_size "postgresql" +report_size "sqlite" +report_size "mysql" diff --git a/query-engine/query-engine-wasm/package.json b/query-engine/query-engine-wasm/package.json new file mode 100644 index 000000000000..79dbeb9014f2 --- /dev/null +++ b/query-engine/query-engine-wasm/package.json @@ -0,0 +1,5 @@ +{ + "name": "@prisma/query-engine-wasm", + "version": "0.0.0", + "type": "module" +} \ No newline at end of file diff --git a/query-engine/query-engine-wasm/src/wasm/engine.rs b/query-engine/query-engine-wasm/src/wasm/engine.rs index 49652823a15b..57a1d469d5a6 100644 --- a/query-engine/query-engine-wasm/src/wasm/engine.rs +++ b/query-engine/query-engine-wasm/src/wasm/engine.rs @@ -7,7 +7,6 @@ use crate::{ }; use driver_adapters::JsObject; use js_sys::Function as JsFunction; -use psl::builtin_connectors::{MYSQL, POSTGRES, SQLITE}; use psl::ConnectorRegistry; use quaint::connector::ExternalConnector; use query_core::{ @@ -25,6 +24,15 @@ use tracing::{field, instrument::WithSubscriber, Instrument, Level, Span}; use tracing_subscriber::filter::LevelFilter; use wasm_bindgen::prelude::wasm_bindgen; +const CONNECTOR_REGISTRY: ConnectorRegistry<'_> = &[ + #[cfg(feature = "postgresql")] + psl::builtin_connectors::POSTGRES, + #[cfg(feature = "mysql")] + psl::builtin_connectors::MYSQL, + #[cfg(feature = "sqlite")] + psl::builtin_connectors::SQLITE, +]; + /// The main query engine used by JS #[wasm_bindgen] pub struct QueryEngine { @@ -51,8 +59,7 @@ impl QueryEngine { } = options; // Note: if we used `psl::validate`, we'd add ~1MB to the Wasm artifact (before gzip). - let connector_registry: ConnectorRegistry<'_> = &[POSTGRES, MYSQL, SQLITE]; - let schema = psl::parse_without_validation(datamodel.into(), connector_registry); + let schema = psl::parse_without_validation(datamodel.into(), CONNECTOR_REGISTRY); let js_queryable = Arc::new(driver_adapters::from_js(adapter)); diff --git a/query-engine/request-handlers/Cargo.toml b/query-engine/request-handlers/Cargo.toml index 4a550f325039..4200980202c5 100644 --- a/query-engine/request-handlers/Cargo.toml +++ b/query-engine/request-handlers/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" query-structure = { path = "../query-structure" } query-core = { path = "../core" } user-facing-errors = { path = "../../libs/user-facing-errors" } -quaint = { path = "../../quaint" } +quaint = { path = "../../quaint", default-features = false } psl.workspace = true dmmf_crate = { path = "../dmmf", package = "dmmf" } itertools.workspace = true