Insertion ORM w/ prepared statements works (#966)

* fix diesel function registration

* query building works

* try insert with orml

* compile with sqlite default fix

* compile but still ahve static lifetimes

* fix unimplemented traits

* ORM compiles!

* some cleanup, setup tracing in wasm for test

* readme

* upgrade wa-sqlite version

* temp DebugQuery to help with debugging

* make debug query unsafe

* progress on statement issue

* progress for destructor order

* logs/fix async drop for BoundStatement

* remove task spawn for statementfactory

diesel-wasm-sqlite/Cargo.toml

@@ -2,39 +2,50 @@
 name = "diesel-wasm-sqlite"
 version = "0.1.1"
 edition = "2021"
+resolver = "2"
 
 [dependencies]
 # diesel = { git = "https://github.com/xmtp/diesel", branch = "insipx/wasm-backend", default-features = false, features = ["i-implement-a-third-party-backend-and-opt-into-breaking-changes"] }
-diesel = "2.2"
+diesel = { version = "2.2", features = ["i-implement-a-third-party-backend-and-opt-into-breaking-changes"] }
 diesel-async = { git = "https://github.com/insipx/diesel_async", branch = "insipx/make-stmt-cache-public", features = ["stmt-cache"] }
+diesel_derives = "2.2"
 wasm-bindgen = "0.2.92"
 wasm-bindgen-futures = "0.4.42"
-log = "0.4"
+tracing = "0.1"
 rand = "0.8"
 getrandom = { version = "0.2", features = ["js"] }
 web-sys = { version = "0.3", features = ["console"] }
+js-sys = { version = "0.3" }
 # wee_alloc = { version = "0.4.2", optional = true }
 console_error_panic_hook = { version = "0.1", optional = true }
-tokio = { version = "1.38", default-features = false, features = ["rt", "macros", "sync", "io-util", "time"] }
+tokio = { version = "1.38", default-features = false, features = ["rt", "macros", "sync"] }
 futures = "0.3"
 futures-util = "0.3"
 async-trait = "0.1"
 bitflags = "2.6"
 serde = { version = "1.0", features = ["derive"] }
 serde-wasm-bindgen = "0.6"
+thiserror = "1"
 
 [dev-dependencies]
 rand = "0.8"
 getrandom = { version = "0.2", features = ["js"] }
 wasm-bindgen-test = "0.3.42"
 web-sys = { version = "0.3", features = ["console"] }
+chrono = { version = "0.4", features = ["wasmbind", "serde"] }
+diesel_migrations = "2.2"
+diesel = { version = "2.2", features = ["chrono"]}
+tracing-wasm = { version = "0.2" }
 
 
 [lib]
 crate-type = ["cdylib", "rlib"]
 
 [features]
 default = ["console_error_panic_hook"]
+# enables a `DebugQueryWrapper` for diesel
+# but is unsafe because of mem::transmute
+unsafe-debug-query = []
 
 [build]
 target = "wasm32-unknown-unknown"

diesel-wasm-sqlite/README.md

@@ -1,25 +1,29 @@
 # Custom Diesel Backend for Wasm wa-sqlite
 
-#### Bundle the javascript in `package.js` to rust
+### Compile rust code without creating a npm package
 
-`yarn run build`
+`cargo build --target wasm32-unknown-unknown`
 
-#### Build the JS WASM interface
+#### Build the JS WASM interfaces
 
 `wasm-pack build`
 
 #### Run the Wasm Tests
 
-wasm-pack test --chrome --headless
+wasm-pack test --chrome
+
+navigate to `http://localhost:8000` to observe test output
+
+(headless tests don't work yet)
 
 # TODO
 
 - [ ] wa-sqlite should be included in `pkg` build w/o manual copy (wasm-pack
       issue?)
 - [ ] OPFS
 
-# Notes
+# Setting up the project in VSCode
 
-- rust-analyzer doesn't like crates with different targets in the same
-  workspace. If you want this to work well with your LSP, open
-  `diesel-wasm-sqlite` as it's own project.
+rust-analyzer does not like crates with different targets in the same workspace.
+If you want this to work well with your LSP, open `diesel-wasm-sqlite` as it's
+own project in VSCode.

diesel-wasm-sqlite/build.rs

@@ -3,6 +3,7 @@ use std::process::Command;
 
 fn main() {
     println!("cargo::rerun-if-changed=package.js");
+    println!("cargo::rerun-if-changed=package.json");
 
     Command::new("yarn")
         .args(["run", "build"])

diesel-wasm-sqlite/package.js

@@ -67,7 +67,7 @@ export class SQLite {
     try {
       return this.sqlite3.bind(stmt, i, value);
     } catch (error) {
-      console.log("bind err");
+      console.log(`bind err ${error}`);
       throw error;
     }
   }
@@ -185,7 +185,7 @@ export class SQLite {
   }
 
   value_type(pValue) {
-    this.sqlite3.value_type(pValue);
+    return this.sqlite3.value_type(pValue);
   }
 
   async open_v2(database_url, iflags) {
@@ -226,7 +226,12 @@ export class SQLite {
   }
 
   clear_bindings(stmt) {
-    return this.sqlite3.clear_bindings(stmt);
+    try {
+      return this.sqlite3.clear_bindings(stmt);
+    } catch (error) {
+      console.log("sqlite3.clear_bindings error");
+      throw error;
+    }
   }
 
   async close(db) {
@@ -251,6 +256,19 @@ export class SQLite {
     }
   }
 
+  // there should be a way to do this from Rust
+  // If we pass the statement we get from 'next'
+  // it does not work.
+  async get_stmt_from_iterator(iterator) {
+    try {
+      const stmt = await iterator.next();
+      return stmt;
+    } catch (error) {
+      console.log("sqlite prepare error");
+      throw error;
+    }
+  }
+
   async step(stmt) {
     try {
       return await this.sqlite3.step(stmt);
@@ -268,12 +286,11 @@ export class SQLite {
     return this.sqlite3.column_count(stmt);
   }
 
-  batch_execute(database, query) {
+  async batch_execute(database, query) {
     try {
-      return this.sqlite3.exec(database, query);
-      console.log("Batch exec'ed");
+      return await this.sqlite3.exec(database, query);
     } catch (error) {
-      console.log("exec err");
+      console.log("batch exec err");
       throw error;
     }
   }
@@ -315,7 +332,7 @@ export class SQLite {
         1,
         WasmSQLiteLibrary.SQLITE_UTF8,
         0,
-        (context, values) => {
+        async (context, values) => {
           const table_name = this.sqlite3.value_text(values[0]);
 
           await this.sqlite3.exec(

diesel-wasm-sqlite/package.json

@@ -8,7 +8,7 @@
     "build": "rollup -c"
   },
   "dependencies": {
-    "@xmtp/wa-sqlite": "^1.0.1"
+    "@xmtp/wa-sqlite": ">=1.0.3"
   },
   "packageManager": "[email protected]",
   "engines": {

diesel-wasm-sqlite/rollup.config.js

@@ -1,5 +1,5 @@
 import { defineConfig } from "rollup";
-import resolve from "@rollup/plugin-node-resolve";
+import { nodeResolve } from "@rollup/plugin-node-resolve";
 import { base64 } from "rollup-plugin-base64";
 
 export default defineConfig([
@@ -10,7 +10,7 @@ export default defineConfig([
       format: "es",
     },
     plugins: [
-      resolve(),
+      nodeResolve(),
       base64({ include: "**/*.wasm" }),
     ],
     // external: ["@xmtp/wa-sqlite", "@xmtp/wa-sqlite/build"],

diesel-wasm-sqlite/src/backend.rs

@@ -1,4 +1,4 @@
-//! The SQLite backend
+//! The WasmSQLite backend
 
 use super::connection::SqliteBindCollector;
 use super::connection::SqliteValue;
@@ -49,9 +49,6 @@ impl TypeMetadata for WasmSqlite {
 }
 
 impl SqlDialect for WasmSqlite {
-    #[cfg(not(feature = "returning_clauses_for_sqlite_3_35"))]
-    type ReturningClause = sql_dialect::returning_clause::DoesNotSupportReturningClause;
-    #[cfg(feature = "returning_clauses_for_sqlite_3_35")]
     type ReturningClause = SqliteReturningClause;
 
     type OnConflictClause = SqliteOnConflictClause;

diesel-wasm-sqlite/src/connection/bind_collector.rs

@@ -6,11 +6,6 @@ use diesel::{
     sql_types::HasSqlType,
 };
 use serde::{Deserialize, Serialize};
-use wasm_bindgen::JsValue;
-
-// TODO:insipx this file remains largely unchanged other than the ffi::. Candidate for shared code?
-
-pub type BindValue = JsValue;
 
 #[derive(Debug, Default)]
 pub struct SqliteBindCollector<'a> {
@@ -131,7 +126,7 @@ impl std::fmt::Display for InternalSqliteBindValue<'_> {
         f.write_str(n)
     }
 }
-
+/*
 impl InternalSqliteBindValue<'_> {
     #[allow(unsafe_code)] // ffi function calls
     pub(crate) fn result_of(self, ctx: &mut i32) {
@@ -148,6 +143,7 @@ impl InternalSqliteBindValue<'_> {
         }
     }
 }
+*/
 
 impl<'a> BindCollector<'a, WasmSqlite> for SqliteBindCollector<'a> {
     type Buffer = SqliteBindValue<'a>;

diesel-wasm-sqlite/src/connection/mod.rs

@@ -33,36 +33,34 @@ pub use diesel_async::{AnsiTransactionManager, AsyncConnection, SimpleAsyncConne
 
 use crate::{get_sqlite_unchecked, WasmSqlite, WasmSqliteError};
 
+// This relies on the invariant that RawConnection or Statement are never
+// leaked. If a reference to one of those was held on a different thread, this
+// would not be thread safe.
+// Web is in one thread. Web workers can establish & hold a WasmSqliteConnection
+// separately.
+#[allow(unsafe_code)]
+unsafe impl Send for WasmSqliteConnection {}
+
 pub struct WasmSqliteConnection {
     // statement_cache needs to be before raw_connection
     // otherwise we will get errors about open statements before closing the
     // connection itself
     statement_cache: StmtCache<WasmSqlite, Statement>,
-    pub raw_connection: RawConnection,
+    raw_connection: RawConnection,
     transaction_manager: AnsiTransactionManager,
     // this exists for the sole purpose of implementing `WithMetadataLookup` trait
     // and avoiding static mut which will be deprecated in 2024 edition
     instrumentation: Arc<Mutex<Option<Box<dyn Instrumentation>>>>,
 }
 
-
-// This relies on the invariant that RawConnection or Statement are never
-// leaked. If a reference to one of those was held on a different thread, this
-// would not be thread safe.
-// Web is in one thread. Web workers can be used to hold a WasmSqliteConnection
-// separately.
-
-#[allow(unsafe_code)]
-unsafe impl Send for WasmSqliteConnection {}
-
-
 impl ConnectionSealed for WasmSqliteConnection {}
 
  #[async_trait::async_trait(?Send)]
 impl SimpleAsyncConnection for WasmSqliteConnection {
     async fn batch_execute(&mut self, query: &str) -> diesel::prelude::QueryResult<()> {
         get_sqlite_unchecked()
             .batch_execute(&self.raw_connection.internal_connection, query)
+            .await
             .map_err(WasmSqliteError::from)
             .map_err(Into::into)
     }
@@ -88,6 +86,7 @@ impl AsyncConnection for WasmSqliteConnection {
     {
         let query = source.as_query();
         self.with_prepared_statement(query, |_, statement| async move {
+            tracing::debug!("Loading statement!");
             Ok(StatementStream::new(statement).stream())
         })
     }
@@ -100,6 +99,7 @@ impl AsyncConnection for WasmSqliteConnection {
         T: QueryFragment<Self::Backend> + QueryId + 'query,
     {
         self.with_prepared_statement(query, |conn, statement| async move {
+            tracing::debug!("Running statement!");
             statement.run().await.map(|_| {
                 conn.rows_affected_by_last_query()
             })
@@ -113,11 +113,11 @@ impl AsyncConnection for WasmSqliteConnection {
     }
 
     fn instrumentation(&mut self) -> &mut dyn Instrumentation {
-        todo!()
+        Arc::get_mut(&mut self.instrumentation).expect("arc ref").get_mut().expect("Mutex poison")
     }
 
     fn set_instrumentation(&mut self, _instrumentation: impl Instrumentation) {
-        todo!()
+        tracing::debug!("Set instrumentation");
     }
 }
 
@@ -237,6 +237,7 @@ impl WasmSqliteConnection {
         Q: QueryFragment<WasmSqlite> + QueryId,
         F: Future<Output = QueryResult<R>>,
     {
+        tracing::info!("WITH PREPARED STATEMENT");
         let WasmSqliteConnection {
             ref mut raw_connection,
             ref mut statement_cache,
@@ -260,16 +261,22 @@ impl WasmSqliteConnection {
         let sql = query.to_sql(&mut qb, &WasmSqlite).map(|()| qb.finish()); 
 
         async move {
+            tracing::info!("IN STATEMENT PREPARE FUTURE");
             let (statement, conn) = statement_cache.cached_prepared_statement(
                 cache_key?,
                 sql?,
                 is_safe_to_cache_prepared?,
                 &[],
                 raw_connection,
                 &instrumentation,
-            ).await?; // Cloned RawConnection is dropped here
+            ).await?;
             let statement = StatementUse::bind(statement, bind_collector?, instrumentation)?;
-            callback(conn, statement).await
+            let result = callback(conn, statement).await;
+            // statement is dropped here
+            // we need to yield back to the executor and allow
+            // the destructor for BoundStatement to run
+            tokio::task::yield_now().await;
+            result
         }.boxed_local()
     }