Migrate driver adapters to prisma/prisma (#4380)

* Promote connector-test-kit to the driver-adapters directory and remove js

* Remove node_modules from connector-test-kit-executor

* Remove dist from connector-test-kit-executor

* Ignore non-relevant files

* Sort out dependencies

* Makefile to setup driver adapters from a checkout of prisma

* Only clone prisma/prisma shallowly

* Delete driver-adapter-smoke-tests.yml

* DRIVER_ADAPTERS_BRANCH=driver-adapters-migration see if this works

* DRIVER_ADAPTERS_BRANCH=driver-adapters-migration change deprecated set-output command

* DRIVER_ADAPTERS_BRANCH=driver-adapters-migration tmp remove

* DRIVER_ADAPTERS_BRANCH=driver-adapters-migration clearer detection of branch

* DRIVER_ADAPTERS_BRANCH=driver-adapters-migration Build executor separately

* DRIVER_ADAPTERS_BRANCH=driver-adapters-migration Add make tasks to test driver adapters

* Document and ease running driver adapter tests

* Revert "DRIVER_ADAPTERS_BRANCH=driver-adapters-migration tmp remove"

This reverts commit 463775a.

* Move documentation to where it belongs

* Document how to do integration testing in shorter loops in CI.

* chore(driver-adapters): remove outdated symlink to tsconfig file

* fix(driver-adapters): use ws, making connector-test-kit-executor compatible with Node.js 16+

* fix(driver-adapters): remove warning "import.meta" is not available with the "cjs" output format

* chore(driver-adapters): remove references to query-engine-driver-adapters.yml

* Revert "chore(driver-adapters): remove references to query-engine-driver-adapters.yml"

This reverts commit eeaaa8f.

* Remove publish-driver-adapters workflow

* Fix using main branch

* Take back conditional on docker login after bad main merge

---------

Co-authored-by: jkomyno <[email protected]>

.github/workflows/query-engine-driver-adapters.yml

@@ -49,6 +49,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: 'Setup Node.js'
         uses: actions/setup-node@v3
@@ -76,6 +78,15 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Extract Branch Name
+        id: extract-branch
+        run: |
+          branch="$(git show -s --format=%s | grep -o "DRIVER_ADAPTERS_BRANCH=[^ ]*" | cut -f2 -d=)"
+          if [ -n "$branch" ]; then
+            echo "Using $branch branch of driver adapters"
+            echo "DRIVER_ADAPTERS_BRANCH=$branch" >> "$GITHUB_ENV"
+          fi
+
       - run: make ${{ matrix.adapter.setup_task }}
 
       - uses: dtolnay/rust-toolchain@stable

.gitignore

@@ -46,3 +46,6 @@ dmmf.json
 graph.dot
 
 prisma-schema-wasm/nodejs
+
+# This symlink looks orphan here, but it comes from prisma/prisma where driver adapters reference a file in their parent directory
+tsconfig.build.adapter.json

Makefile

@@ -2,6 +2,7 @@ CONFIG_PATH = ./query-engine/connector-test-kit-rs/test-configs
 CONFIG_FILE = .test_config
 SCHEMA_EXAMPLES_PATH = ./query-engine/example_schemas
 DEV_SCHEMA_FILE = dev_datamodel.prisma
+DRIVER_ADAPTERS_BRANCH ?= main
 
 LIBRARY_EXT := $(shell                            \
     case "$$(uname -s)" in                        \
@@ -44,7 +45,13 @@ release:
 #################
 
 test-qe:
+ifndef DRIVER_ADAPTER
 	cargo test --package query-engine-tests
+else
+	@echo "Executing query engine tests with $(DRIVER_ADAPTER) driver adapter"; \
+	# Add your actual command for the "test-driver-adapter" task here
+	$(MAKE) test-driver-adapter-$(DRIVER_ADAPTER);
+endif
 
 test-qe-verbose:
 	cargo test --package query-engine-tests -- --nocapture
@@ -80,6 +87,10 @@ dev-sqlite:
 dev-libsql-sqlite: build-qe-napi build-connector-kit-js
 	cp $(CONFIG_PATH)/libsql-sqlite $(CONFIG_FILE)
 
+test-libsql-sqlite: dev-libsql-sqlite test-qe-st
+
+test-driver-adapter-libsql: test-libsql-sqlite
+
 start-postgres9:
 	docker compose -f docker-compose.yml up --wait -d --remove-orphans postgres9
 
@@ -115,12 +126,20 @@ start-pg-postgres13: build-qe-napi build-connector-kit-js start-postgres13
 dev-pg-postgres13: start-pg-postgres13
 	cp $(CONFIG_PATH)/pg-postgres13 $(CONFIG_FILE)
 
+test-pg-postgres13: dev-pg-postgres13 test-qe-st
+
+test-driver-adapter-pg: test-pg-postgres13
+
 start-neon-postgres13: build-qe-napi build-connector-kit-js
 	docker compose -f docker-compose.yml up --wait -d --remove-orphans neon-postgres13
 
 dev-neon-ws-postgres13: start-neon-postgres13
 	cp $(CONFIG_PATH)/neon-ws-postgres13 $(CONFIG_FILE)
 
+test-neon-ws-postgres13: dev-neon-ws-postgres13 test-qe-st
+
+test-driver-adapter-neon: test-neon-ws-postgres13
+
 start-postgres14:
 	docker compose -f docker-compose.yml up --wait -d --remove-orphans postgres14
 
@@ -255,15 +274,47 @@ start-planetscale-vitess8: build-qe-napi build-connector-kit-js
 dev-planetscale-vitess8: start-planetscale-vitess8
 	cp $(CONFIG_PATH)/planetscale-vitess8 $(CONFIG_FILE)
 
+test-planetscale-vitess8: dev-planetscale-vitess8 test-qe-st
+
+test-driver-adapter-planetscale: test-planetscale-vitess8
+
 ######################
 # Local dev commands #
 ######################
 
 build-qe-napi:
 	cargo build --package query-engine-node-api
 
-build-connector-kit-js:
-	cd query-engine/driver-adapters/js && pnpm i && pnpm build
+build-connector-kit-js: build-driver-adapters symlink-driver-adapters
+	cd query-engine/driver-adapters/connector-test-kit-executor && pnpm i && pnpm build
+
+build-driver-adapters: ensure-prisma-present
+	@echo "Building driver adapters..."
+	@cd ../prisma && pnpm --filter "*adapter*" i && pnpm --filter "*adapter*" build
+	@echo "Driver adapters build completed.";
+
+symlink-driver-adapters: ensure-prisma-present
+	@echo "Creating symbolic links for driver adapters..."
+	@for dir in $(wildcard $(realpath ../prisma)/packages/*adapter*); do \
+        if [ -d "$$dir" ]; then \
+            dir_name=$$(basename "$$dir"); \
+            ln -sfn "$$dir" "$(realpath .)/query-engine/driver-adapters/$$dir_name"; \
+            echo "Created symbolic link for $$dir_name"; \
+        fi; \
+	done;
+	echo "Symbolic links creation completed.";
+
+ensure-prisma-present:
+	@if [ -d ../prisma ]; then \
+		cd "$(realpath ../prisma)" && git fetch origin main; \
+		LOCAL_CHANGES=$$(git diff --name-only HEAD origin/main -- 'packages/*adapter*'); \
+		if [ -n "$$LOCAL_CHANGES" ]; then \
+		  echo "⚠️ ../prisma diverges from prisma/prisma main branch. Test results might diverge from those in CI ⚠️ "; \
+		fi \
+	else \
+		echo "git clone --depth=1 https://github.com/prisma/prisma.git --branch=$(DRIVER_ADAPTERS_BRANCH) ../prisma"; \
+		git clone --depth=1 https://github.com/prisma/prisma.git --branch=$(DRIVER_ADAPTERS_BRANCH) "../prisma" && echo "Prisma repository has been cloned to ../prisma"; \
+	fi;
 
 # Quick schema validation of whatever you have in the dev_datamodel.prisma file.
 validate:

README.md

@@ -203,6 +203,7 @@ integration tests.
   - Alternatively: Load the defined environment in `./.envrc` manually in your shell.
 
 **Setup:**
+
 There are helper `make` commands to set up a test environment for a specific
 database connector you want to test. The commands set up a container (if needed)
 and write the `.test_config` file, which is picked up by the integration
@@ -234,6 +235,31 @@ Other variables may or may not be useful.
 
 Run `cargo test` in the repository root.
 
+### Testing driver adapters
+
+Please refer to the [Testing driver adapters](./query-engine/connector-test-kit-rs/README.md#testing-driver-adapters) section in the connector-test-kit-rs README.
+
+**ℹ️ Important note on developing features that require changes to the both the query engine, and driver adapters code**
+
+As explained in [Testing driver adapters](./query-engine/connector-test-kit-rs/README.md#testing-driver-adapters), running `DRIVER_ADAPTER=$adapter make qe-test`
+will ensure you have prisma checked out in your filesystem in the same directory as prisma-engines. This is needed because the driver adapters code is symlinked in prisma-engines.
+
+When working on a feature or bugfix spanning adapters code and query-engine code, you will need to open sibling PRs in `prisma/prisma` and `prisma/prisma-engines` respectively.
+Locally, each time you run `DRIVER_ADAPTER=$adapter make qe-test` tests will run using the driver adapters built from the source code in the working copy of prisma/prisma. All good.
+
+In CI, tho', we need to denote which branch of prisma/prisma we want to use for tests. In CI, there's no working copy of prisma/prisma before tests run.
+The CI jobs clones prisma/prisma `main` branch by default, which doesn't include your local changes. To test in integration, we can tell CI to use the branch of prisma/prisma containing
+the changes in adapters. To do it, you can use a simple convention in commit messages. Like this:
+
+```
+git commit -m "DRIVER_ADAPTERS_BRANCH=prisma-branch-with-changes-in-adapters [...]"
+```
+
+GitHub actions will then pick up the branch name and use it to clone that branch's code of prisma/prisma, and build the driver adapters code from there.
+
+When it's time to merge the sibling PRs, you'll need to merge the prisma/prisma PR first, so when merging the engines PR you have the code of the adapters ready in prisma/prisma `main` branch.
+
+
 ## Parallel rust-analyzer builds
 
 When rust-analzyer runs `cargo check` it will lock the build directory and stop any cargo commands from running until it has completed. This makes the build process feel a lot longer. It is possible to avoid this by setting a different build path for