diff --git a/Cargo.lock b/Cargo.lock index cb9d754e2ad..c3cd6e22f2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4671,6 +4671,23 @@ dependencies = [ "user-facing-errors", ] +[[package]] +name = "schema-engine-wasm" +version = "0.1.0" +dependencies = [ + "build-utils", + "driver-adapters", + "js-sys", + "psl", + "quaint", + "serde", + "tracing", + "tsify", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-rs-dbg", +] + [[package]] name = "scopeguard" version = "1.2.0" diff --git a/Cargo.toml b/Cargo.toml index 53c478bb120..b78f4801fd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,9 +6,10 @@ members = [ "schema-engine/connectors/*", "schema-engine/datamodel-renderer", "schema-engine/json-rpc-api-build", + "schema-engine/mongodb-schema-describer", "schema-engine/sql-migration-tests", "schema-engine/sql-introspection-tests", - "schema-engine/mongodb-schema-describer", + "schema-engine/schema-engine-wasm", "schema-engine/sql-schema-describer", "query-engine/connectors/*", "query-engine/connector-test-kit-rs/qe-setup", diff --git a/Makefile b/Makefile index 94066b92d8d..7358317bb00 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,10 @@ clean-qe-wasm: @echo "Cleaning query-engine/query-engine-wasm/pkg" && \ cd query-engine/query-engine-wasm/pkg && find . ! -name '.' ! -name '..' ! -name 'README.md' -exec rm -rf {} + +clean-se-wasm: + @echo "Cleaning schema-engine/schema-engine-wasm/pkg" && \ + cd schema-engine/schema-engine-wasm/pkg && find . ! -name '.' ! -name '..' ! -name 'README.md' -exec rm -rf {} + + clean-cargo: @echo "Cleaning cargo" && \ cargo clean @@ -71,6 +75,10 @@ integrate-qe-wasm: cd query-engine/query-engine-wasm && \ ./build.sh $(QE_WASM_VERSION) ../prisma/packages/client/node_modules/@prisma/query-engine-wasm +build-se-wasm: + cd schema-engine/schema-engine-wasm && \ + ./build.sh $(QE_WASM_VERSION) schema-engine/schema-engine-wasm/pkg + build-schema-wasm: @printf '%s\n' "🛠️ Building the Rust crate" cargo build --profile $(PROFILE) --target=wasm32-unknown-unknown -p prisma-schema-build @@ -85,7 +93,7 @@ build-schema-wasm: pedantic: RUSTFLAGS="-D warnings" cargo fmt -- --check RUSTFLAGS="-D warnings" cargo clippy --all-features --all-targets - RUSTFLAGS="-D warnings" cargo clippy --all-features --all-targets -p query-engine-wasm -p prisma-schema-build --target wasm32-unknown-unknown + RUSTFLAGS="-D warnings" cargo clippy --all-features --all-targets -p query-engine-wasm -p schema-engine-wasm -p prisma-schema-build --target wasm32-unknown-unknown release: cargo build --release diff --git a/query-engine/driver-adapters/executor/.gitignore b/query-engine/driver-adapters/executor/.gitignore index 37b61ff565c..ea7cd1b095b 100644 --- a/query-engine/driver-adapters/executor/.gitignore +++ b/query-engine/driver-adapters/executor/.gitignore @@ -1,3 +1,4 @@ node_modules pnpm-debug.log dist/ +./db.sqlite diff --git a/query-engine/driver-adapters/executor/package.json b/query-engine/driver-adapters/executor/package.json index 016a5f0a6a5..185797bfea2 100644 --- a/query-engine/driver-adapters/executor/package.json +++ b/query-engine/driver-adapters/executor/package.json @@ -10,13 +10,17 @@ "scripts": { "build": "tsup ./src/testd-qe.ts ./src/bench.ts --format esm --dts", "test:qe": "node --import tsx ./src/testd-qe.ts", + "demo:se": "node --import tsx ./src/demo-se.ts", "clean:d1": "rm -rf ../../connector-test-kit-rs/query-engine-tests/.wrangler" }, "tsup": { "external": [ "../../../query-engine-wasm/pkg/postgresql/query_engine_bg.js", "../../../query-engine-wasm/pkg/mysql/query_engine_bg.js", - "../../../query-engine-wasm/pkg/sqlite/query_engine_bg.js" + "../../../query-engine-wasm/pkg/sqlite/query_engine_bg.js", + "../../../schema-engine-wasm/pkg/postgresql/schema_engine_bg.js", + "../../../schema-engine-wasm/pkg/mysql/schema_engine_bg.js", + "../../../schema-engine-wasm/pkg/sqlite/schema_engine_bg.js" ] }, "keywords": [], diff --git a/query-engine/driver-adapters/executor/src/demo-se.ts b/query-engine/driver-adapters/executor/src/demo-se.ts new file mode 100644 index 00000000000..35868031b4e --- /dev/null +++ b/query-engine/driver-adapters/executor/src/demo-se.ts @@ -0,0 +1,93 @@ +import * as S from '@effect/schema/Schema' +import { bindAdapter } from '@prisma/driver-adapter-utils' + +import type { DriverAdaptersManager } from './driver-adapters-manager' +import { Env } from './types' +import * as se from './schema-engine' +import { err } from './utils' +import { setupDriverAdaptersManager } from './setup' + +/** + * Example run: `DRIVER_ADAPTER="libsql" pnpm demo:se` + */ +async function main(): Promise { + const env = S.decodeUnknownSync(Env)(process.env) + console.log('[env]', env) + + /** + * Static input for demo purposes. + */ + + const url = 'file:./db.sqlite' + + const schema = /* prisma */ ` + generator client { + provider = "prisma-client-js" + } + + datasource db { + provider = "sqlite" + url = "file:./db.sqlite" + } + + model User { + id Int @id @default(autoincrement()) + email String @unique + name String? + posts Post[] + } + + model Post { + id Int @id @default(autoincrement()) + title String + content String + author User @relation(fields: [authorId], references: [id]) + authorId Int + } + ` + + const driverAdapterManager = await setupDriverAdaptersManager( + env, + ) + + const { engine, adapter } = await initSE({ + env, + driverAdapterManager, + url, + schema, + }) + + console.log('[adapter]', adapter) + + // TODO: use `engine`. +} + +type InitQueryEngineParams = { + env: Env + driverAdapterManager: DriverAdaptersManager + url: string + schema: string +} + +async function initSE({ + env, + driverAdapterManager, + url, + schema, +}: InitQueryEngineParams) { + const adapter = await driverAdapterManager.connect({ url }) + const errorCapturingAdapter = bindAdapter(adapter) + const engineInstance = await se.initSchemaEngine( + { + datamodel: schema, + }, + adapter, + ) + + return { + engine: engineInstance, + adapter: errorCapturingAdapter, + } +} + +main().catch(err) diff --git a/query-engine/driver-adapters/executor/src/schema-engine-wasm.ts b/query-engine/driver-adapters/executor/src/schema-engine-wasm.ts new file mode 100644 index 00000000000..d589ce26bd6 --- /dev/null +++ b/query-engine/driver-adapters/executor/src/schema-engine-wasm.ts @@ -0,0 +1,30 @@ +import * as wasmPostgres from '../../../../schema-engine/schema-engine-wasm/pkg/postgresql/schema_engine_bg.js' +import * as wasmMysql from '../../../../schema-engine/schema-engine-wasm/pkg/mysql/schema_engine_bg.js' +import * as wasmSqlite from '../../../../schema-engine/schema-engine-wasm/pkg/sqlite/schema_engine_bg.js' +import fs from 'node:fs/promises' +import path from 'node:path' +import { __dirname } from './utils.js' + +const wasm = { + postgres: wasmPostgres, + mysql: wasmMysql, + sqlite: wasmSqlite +} + +type EngineName = keyof typeof wasm + +const initializedModules = new Set() + +export async function getSchemaEngineForProvider(provider: EngineName) { + const engine = wasm[provider] + if (!initializedModules.has(provider)) { + const subDir = provider === 'postgres' ? 'postgresql' : provider + const bytes = await fs.readFile(path.resolve(__dirname, '..', '..', '..', '..', 'schema-engine', 'schema-engine-wasm', 'pkg', subDir, 'schema_engine_bg.wasm')) + const module = new WebAssembly.Module(bytes) + const instance = new WebAssembly.Instance(module, { './schema_engine_bg.js': engine }) + engine.__wbg_set_wasm(instance.exports) + initializedModules.add(provider) + } + + return engine.SchemaEngine +} diff --git a/query-engine/driver-adapters/executor/src/schema-engine.ts b/query-engine/driver-adapters/executor/src/schema-engine.ts new file mode 100644 index 00000000000..f3881647b65 --- /dev/null +++ b/query-engine/driver-adapters/executor/src/schema-engine.ts @@ -0,0 +1,25 @@ +import type { DriverAdapter } from '@prisma/driver-adapter-utils' +import { __dirname } from './utils' + +export type SchemaEngineParams = { + // TODO: support multiple datamodels + datamodel: string +} + +export interface SchemaEngine { + new(params: SchemaEngineParams, adapter: DriverAdapter): SchemaEngine + debugPanic(): Promise + version(): Promise + reset(): Promise +} + +export type QueryLogCallback = (log: string) => void + +export async function initSchemaEngine( + params: SchemaEngineParams, + adapter: DriverAdapter, +): Promise { + const { getSchemaEngineForProvider: getEngineForProvider } = await import('./schema-engine-wasm') + const WasmSchemaEngine = (await getEngineForProvider(adapter.provider)) as SchemaEngine + return new WasmSchemaEngine(params, adapter) +} diff --git a/schema-engine/schema-engine-wasm/.gitignore b/schema-engine/schema-engine-wasm/.gitignore new file mode 100644 index 00000000000..a6f0e4dca12 --- /dev/null +++ b/schema-engine/schema-engine-wasm/.gitignore @@ -0,0 +1,7 @@ +/target +**/*.rs.bk +Cargo.lock +bin/ +pkg/ +wasm-pack.log +node_modules/ \ No newline at end of file diff --git a/schema-engine/schema-engine-wasm/Cargo.toml b/schema-engine/schema-engine-wasm/Cargo.toml new file mode 100644 index 00000000000..70444bb52e3 --- /dev/null +++ b/schema-engine/schema-engine-wasm/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "schema-engine-wasm" +version = "0.1.0" +edition = "2021" + +[lib] +doc = false +crate-type = ["cdylib"] +name = "schema_engine_wasm" + +[features] +sqlite = ["driver-adapters/sqlite", "psl/sqlite"] +postgresql = ["driver-adapters/postgresql", "psl/postgresql"] +mysql = ["driver-adapters/mysql", "psl/mysql"] + +[dependencies] +psl.workspace = true +quaint.workspace = true +tracing.workspace = true + +js-sys.workspace = true +serde.workspace = true +tsify.workspace = true +wasm-bindgen.workspace = true +wasm-bindgen-futures.workspace = true +wasm-rs-dbg.workspace = true +driver-adapters = { path = "../../query-engine/driver-adapters" } + +[build-dependencies] +build-utils.path = "../../libs/build-utils" + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false # use wasm-opt explicitly in `./build.sh` + +[package.metadata.wasm-pack.profile.profiling] +wasm-opt = false # use wasm-opt explicitly in `./build.sh` diff --git a/schema-engine/schema-engine-wasm/build.rs b/schema-engine/schema-engine-wasm/build.rs new file mode 100644 index 00000000000..33aded23a4a --- /dev/null +++ b/schema-engine/schema-engine-wasm/build.rs @@ -0,0 +1,3 @@ +fn main() { + build_utils::store_git_commit_hash_in_env(); +} diff --git a/schema-engine/schema-engine-wasm/build.sh b/schema-engine/schema-engine-wasm/build.sh new file mode 100755 index 00000000000..b5151f4915a --- /dev/null +++ b/schema-engine/schema-engine-wasm/build.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Call this script as `./build.sh ` +# +# Note: this script started as a copy of the `query-engine-wasm`'s `build.sh` script, but will likely diverge over time. +# For this reason, we're avoiding premature refactoring and keeping the two scripts separate. + +set -euo pipefail + +CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +REPO_ROOT="$( cd "$( dirname "$CURRENT_DIR/../../../" )" >/dev/null 2>&1 && pwd )" +OUT_VERSION="${1:-"0.0.0"}" +OUT_FOLDER="${2:-"schema-engine/schema-engine-wasm/pkg"}" +OUT_TARGET="bundler" +# wasm-opt pass +WASM_OPT_ARGS=( + "-Os" # execute size-focused optimization passes (-Oz actually increases size by 1KB) + "--vacuum" # removes obviously unneeded code + "--duplicate-function-elimination" # removes duplicate functions + "--duplicate-import-elimination" # removes duplicate imports + "--remove-unused-module-elements" # removes unused module elements + "--dae-optimizing" # removes arguments to calls in an lto-like manner + "--remove-unused-names" # removes names from location that are never branched to + "--rse" # removes redundant local.sets + "--gsi" # global struct inference, to optimize constant values + "--gufa-optimizing" # optimize the entire program using type monomorphization + "--strip-dwarf" # removes DWARF debug information + "--strip-producers" # removes the "producers" section + "--strip-target-features" # removes the "target_features" section +) + +# if it's a relative path, let it be relative to the repo root +if [[ "$OUT_FOLDER" != /* ]]; then + OUT_FOLDER="$REPO_ROOT/$OUT_FOLDER" +fi +OUT_JSON="${OUT_FOLDER}/package.json" + +echo "ℹ️ target version: $OUT_VERSION" +echo "ℹ️ out folder: $OUT_FOLDER" + +if [[ -z "${WASM_BUILD_PROFILE:-}" ]]; then + if [[ -z "${BUILDKITE:-}" ]] && [[ -z "${GITHUB_ACTIONS:-}" ]]; then + WASM_BUILD_PROFILE="dev" + else + WASM_BUILD_PROFILE="release" + fi +fi + +if [ "$WASM_BUILD_PROFILE" = "dev" ]; then + WASM_TARGET_SUBDIR="debug" +else + WASM_TARGET_SUBDIR="$WASM_BUILD_PROFILE" +fi + + + +build() { + echo "ℹ️ Note that schema-engine compiled to WASM uses a different Rust toolchain" + cargo --version + + local CONNECTOR="$1" + local CARGO_TARGET_DIR + CARGO_TARGET_DIR=$(cargo metadata --format-version 1 | jq -r .target_directory) + echo "🔨 Building $CONNECTOR" + CARGO_PROFILE_RELEASE_OPT_LEVEL="z" cargo build \ + -p schema-engine-wasm \ + --profile "$WASM_BUILD_PROFILE" \ + --features "$CONNECTOR" \ + --target wasm32-unknown-unknown + + local IN_FILE="$CARGO_TARGET_DIR/wasm32-unknown-unknown/$WASM_TARGET_SUBDIR/schema_engine_wasm.wasm" + local OUT_FILE="$OUT_FOLDER/$CONNECTOR/schema_engine_bg.wasm" + + wasm-bindgen --target "$OUT_TARGET" --out-name schema_engine --out-dir "$OUT_FOLDER/$CONNECTOR" "$IN_FILE" + optimize "$OUT_FILE" + + if ! command -v wasm2wat &> /dev/null; then + echo "Skipping wasm2wat, as it is not installed." + else + wasm2wat "$OUT_FILE" -o "./schema_engine.$CONNECTOR.wat" + fi +} + +optimize() { + local OUT_FILE="$1" + case "$WASM_BUILD_PROFILE" in + release) + # In release mode, we want to strip the debug symbols. + wasm-opt "${WASM_OPT_ARGS[@]}" \ + "--strip-debug" \ + "$OUT_FILE" \ + -o "$OUT_FILE" + ;; + profiling) + # In profiling mode, we want to keep the debug symbols. + wasm-opt "${WASM_OPT_ARGS[@]}" \ + "--debuginfo" \ + "${OUT_FILE}" \ + -o "${OUT_FILE}" + ;; + *) + # In other modes (e.g., "dev"), skip wasm-opt. + echo "Skipping wasm-opt." + ;; + esac +} + +report_size() { + local CONNECTOR + local GZ_SIZE + local FORMATTED_GZ_SIZE + + CONNECTOR="$1" + GZ_SIZE=$(gzip -c "${OUT_FOLDER}/$CONNECTOR/schema_engine_bg.wasm" | wc -c) + FORMATTED_GZ_SIZE=$(echo "$GZ_SIZE"|numfmt --format '%.3f' --to=iec-i --suffix=B) + + echo "$CONNECTOR:" + echo "ℹ️ raw: $(du -h "${OUT_FOLDER}/$CONNECTOR/schema_engine_bg.wasm")" + echo "ℹ️ zip: $GZ_SIZE bytes ($FORMATTED_GZ_SIZE)" + echo "" +} + +echo "Building schema-engine-wasm using $WASM_BUILD_PROFILE profile" + +build "postgresql" +build "sqlite" +build "mysql" + +jq '.version=$version' --arg version "$OUT_VERSION" package.json > "$OUT_JSON" + +report_size "postgresql" +report_size "sqlite" +report_size "mysql" diff --git a/schema-engine/schema-engine-wasm/package.json b/schema-engine/schema-engine-wasm/package.json new file mode 100644 index 00000000000..a116fefc814 --- /dev/null +++ b/schema-engine/schema-engine-wasm/package.json @@ -0,0 +1,5 @@ +{ + "name": "@prisma/schema-engine-wasm", + "version": "0.0.0", + "type": "module" +} diff --git a/schema-engine/schema-engine-wasm/src/lib.rs b/schema-engine/schema-engine-wasm/src/lib.rs new file mode 100644 index 00000000000..6364592af8b --- /dev/null +++ b/schema-engine/schema-engine-wasm/src/lib.rs @@ -0,0 +1,21 @@ +#[cfg(not(target_arch = "wasm32"))] +mod arch { + // This crate only works in a Wasm environment. + // This conditional compilation block is here to make commands like + // `cargo clippy --all-features` happy, as `clippy` doesn't support the + // `--exclude` option (see: https://github.com/rust-lang/rust-clippy/issues/9555). + // + // This crate can still be inspected by `clippy` via: + // `cargo clippy --all-features -p schema-engine-wasm --target wasm32-unknown-unknown` +} + +#[cfg(target_arch = "wasm32")] +mod wasm; + +#[cfg(target_arch = "wasm32")] +mod arch { + pub use super::wasm::*; +} + +#[cfg_attr(not(target_arch = "wasm32"), allow(unused_imports))] +pub use arch::*; diff --git a/schema-engine/schema-engine-wasm/src/wasm.rs b/schema-engine/schema-engine-wasm/src/wasm.rs new file mode 100644 index 00000000000..702e611f14e --- /dev/null +++ b/schema-engine/schema-engine-wasm/src/wasm.rs @@ -0,0 +1 @@ +pub mod engine; diff --git a/schema-engine/schema-engine-wasm/src/wasm/engine.rs b/schema-engine/schema-engine-wasm/src/wasm/engine.rs new file mode 100644 index 00000000000..04fc4f363a7 --- /dev/null +++ b/schema-engine/schema-engine-wasm/src/wasm/engine.rs @@ -0,0 +1,95 @@ +#![allow(dead_code)] +#![allow(unused_variables)] + +use driver_adapters::JsObject; +use psl::{ConnectorRegistry, ValidatedSchema}; +use quaint::connector::ExternalConnector; +use serde::Deserialize; +use std::sync::Arc; +use tsify::Tsify; +use wasm_bindgen::prelude::wasm_bindgen; + +const CONNECTOR_REGISTRY: ConnectorRegistry<'_> = &[ + #[cfg(feature = "postgresql")] + psl::builtin_connectors::POSTGRES, + #[cfg(feature = "mysql")] + psl::builtin_connectors::MYSQL, + #[cfg(feature = "sqlite")] + psl::builtin_connectors::SQLITE, +]; + +#[wasm_bindgen] +extern "C" { + /// This function registers the reason for a Wasm panic via the + /// JS function `globalThis.PRISMA_WASM_PANIC_REGISTRY.set_message()` + #[wasm_bindgen(js_namespace = ["global", "PRISMA_WASM_PANIC_REGISTRY"], js_name = "set_message")] + fn prisma_set_wasm_panic_message(s: &str); +} + +/// Registers a singleton panic hook that will register the reason for the Wasm panic in JS. +/// Without this, the panic message would be lost: you'd see `RuntimeError: unreachable` message in JS, +/// with no reference to the Rust function and line that panicked. +/// This function should be manually called before any other public function in this module. +/// Note: no method is safe to call after a panic has occurred. +fn register_panic_hook() { + use std::sync::Once; + static SET_HOOK: Once = Once::new(); + + SET_HOOK.call_once(|| { + std::panic::set_hook(Box::new(|info| { + let message = &info.to_string(); + prisma_set_wasm_panic_message(message); + })); + }); +} + +/// The main query engine used by JS +#[wasm_bindgen] +pub struct SchemaEngine { + schema: ValidatedSchema, + adapter: Arc, +} + +#[derive(Deserialize, Tsify)] +#[tsify(from_wasm_abi)] +pub struct SchemaEngineParams { + // TODO: support multiple datamodels + datamodel: String, +} + +#[wasm_bindgen] +impl SchemaEngine { + #[wasm_bindgen(constructor)] + pub fn new(params: SchemaEngineParams, adapter: JsObject) -> Result { + let SchemaEngineParams { datamodel, .. } = params; + + // Note: if we used `psl::validate`, we'd add ~1MB to the Wasm artifact (before gzip). + let schema = psl::parse_without_validation(datamodel.into(), CONNECTOR_REGISTRY); + let js_queryable = Arc::new(driver_adapters::from_js(adapter)); + + tracing::info!(git_hash = env!("GIT_HASH"), "Starting schema-engine-wasm"); + register_panic_hook(); + + Ok(Self { + schema, + adapter: js_queryable, + }) + } + + /// Debugging method that only panics, for tests. + #[wasm_bindgen(js_name = "debugPanic")] + pub fn debug_panic(&self) { + panic!("This is the debugPanic artificial panic") + } + + /// Return the database version as a string. + #[wasm_bindgen] + pub async fn version(&self) -> Result, wasm_bindgen::JsError> { + Err(wasm_bindgen::JsError::new("Not yet available.")) + } + + /// Reset a database to an empty state (no data, no schema). + pub async fn reset(&self) -> Result<(), wasm_bindgen::JsError> { + Err(wasm_bindgen::JsError::new("Not yet available.")) + } +}