From e895a789d81ee3ee9f583df54fdf5acb31cad28a Mon Sep 17 00:00:00 2001 From: Marcin Antas Date: Fri, 22 Mar 2024 17:11:26 +0100 Subject: [PATCH] Add Weaviate module --- docs/modules/weaviate.md | 17 +++ mkdocs.yml | 1 + package-lock.json | 116 +++++++++++++++++- packages/modules/weaviate/jest.config.ts | 11 ++ packages/modules/weaviate/package.json | 38 ++++++ packages/modules/weaviate/src/index.ts | 1 + .../weaviate/src/weaviate-container.test.ts | 40 ++++++ .../weaviate/src/weaviate-container.ts | 40 ++++++ packages/modules/weaviate/tsconfig.build.json | 14 +++ packages/modules/weaviate/tsconfig.json | 21 ++++ 10 files changed, 296 insertions(+), 3 deletions(-) create mode 100644 docs/modules/weaviate.md create mode 100644 packages/modules/weaviate/jest.config.ts create mode 100644 packages/modules/weaviate/package.json create mode 100644 packages/modules/weaviate/src/index.ts create mode 100644 packages/modules/weaviate/src/weaviate-container.test.ts create mode 100644 packages/modules/weaviate/src/weaviate-container.ts create mode 100644 packages/modules/weaviate/tsconfig.build.json create mode 100644 packages/modules/weaviate/tsconfig.json diff --git a/docs/modules/weaviate.md b/docs/modules/weaviate.md new file mode 100644 index 000000000..771da1f7f --- /dev/null +++ b/docs/modules/weaviate.md @@ -0,0 +1,17 @@ +# Weaviate Module + +[Weaviate](https://weaviate.io) is an open source, AI-native vector database that helps +developers create intuitive and reliable AI-powered applications. + +## Install + +```bash +npm install @testcontainers/weaviate --save-dev +``` + +## Examples + + +[Connect to Weaviate:](../../packages/modules/weaviate/src/weaviate-container.test.ts) +inside_block:connect + diff --git a/mkdocs.yml b/mkdocs.yml index 369978232..0d1fee84b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -58,4 +58,5 @@ nav: - Qdrant: modules/qdrant.md - Redis: modules/redis.md - Selenium: modules/selenium.md + - Weaviate: modules/weaviate.md - Configuration: configuration.md diff --git a/package-lock.json b/package-lock.json index 65a11bfe8..93500f248 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2344,6 +2344,15 @@ "fxparser": "src/cli/cli.js" } }, + "node_modules/@graphql-typed-document-node/core": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@graphql-typed-document-node/core/-/core-3.2.0.tgz", + "integrity": "sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==", + "dev": true, + "peerDependencies": { + "graphql": "^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" + } + }, "node_modules/@grpc/grpc-js": { "version": "1.10.3", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.10.3.tgz", @@ -4768,6 +4777,10 @@ "resolved": "packages/modules/selenium", "link": true }, + "node_modules/@testcontainers/weaviate": { + "resolved": "packages/modules/weaviate", + "link": true + }, "node_modules/@tokenizer/token": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", @@ -5147,9 +5160,9 @@ "dev": true }, "node_modules/@types/qs": { - "version": "6.9.13", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.13.tgz", - "integrity": "sha512-iLR+1vTTJ3p0QaOUq6ACbY1mzKTODFDT/XedZI8BksOotFmL4ForwDfRQ/DZeuTHR7/2i4lI1D203gdfxuqTlA==", + "version": "6.9.14", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.14.tgz", + "integrity": "sha512-5khscbd3SwWMhFqylJBLQ0zIu7c1K6Vz0uBIt915BI3zV0q1nfjRQD3RqSBcPaO6PHEF4ov/t9y89fSiyThlPA==", "dev": true }, "node_modules/@types/range-parser": { @@ -7257,6 +7270,15 @@ "yarn": ">=1" } }, + "node_modules/cross-fetch": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", + "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==", + "dev": true, + "dependencies": { + "node-fetch": "^2.6.12" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -8213,6 +8235,18 @@ "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", "dev": true }, + "node_modules/extract-files": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/extract-files/-/extract-files-9.0.0.tgz", + "integrity": "sha512-CvdFfHkC95B4bBBk36hcEmvdR2awOdhhVUYH6S/zrVj3477zven/fJMYg7121h4T1xHZC+tetUpubpAhxwI7hQ==", + "dev": true, + "engines": { + "node": "^10.17.0 || ^12.0.0 || >= 13.7.0" + }, + "funding": { + "url": "https://github.com/sponsors/jaydenseric" + } + }, "node_modules/fast_array_intersect": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fast_array_intersect/-/fast_array_intersect-1.1.0.tgz", @@ -9871,6 +9905,45 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, + "node_modules/graphql": { + "version": "16.8.1", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz", + "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==", + "dev": true, + "peer": true, + "engines": { + "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" + } + }, + "node_modules/graphql-request": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/graphql-request/-/graphql-request-5.2.0.tgz", + "integrity": "sha512-pLhKIvnMyBERL0dtFI3medKqWOz/RhHdcgbZ+hMMIb32mEPa5MJSzS4AuXxfI4sRAu6JVVk5tvXuGfCWl9JYWQ==", + "dev": true, + "dependencies": { + "@graphql-typed-document-node/core": "^3.1.1", + "cross-fetch": "^3.1.5", + "extract-files": "^9.0.0", + "form-data": "^3.0.0" + }, + "peerDependencies": { + "graphql": "14 - 16" + } + }, + "node_modules/graphql-request/node_modules/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/gtoken": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", @@ -17646,6 +17719,32 @@ "makeerror": "1.0.12" } }, + "node_modules/weaviate-ts-client": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/weaviate-ts-client/-/weaviate-ts-client-2.1.1.tgz", + "integrity": "sha512-d8yc2KnIEIV1beHAU8mhrElT3BoROoXGDsLlqFX8QGx3G+gOiPTRMc7SLy4F17+LvaUaTD0XkHvWX++4iehnsg==", + "dev": true, + "dependencies": { + "graphql-request": "^5.2.0", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/weaviate-ts-client/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", @@ -18293,6 +18392,17 @@ "selenium-webdriver": "^4.18.1" } }, + "packages/modules/weaviate": { + "name": "@testcontainers/weaviate", + "version": "10.7.2", + "license": "MIT", + "dependencies": { + "testcontainers": "^10.7.2" + }, + "devDependencies": { + "weaviate-ts-client": "^2.1.0" + } + }, "packages/testcontainers": { "version": "10.7.2", "license": "MIT", diff --git a/packages/modules/weaviate/jest.config.ts b/packages/modules/weaviate/jest.config.ts new file mode 100644 index 000000000..1f677baaf --- /dev/null +++ b/packages/modules/weaviate/jest.config.ts @@ -0,0 +1,11 @@ +import type { Config } from "jest"; +import * as path from "path"; + +const config: Config = { + preset: "ts-jest", + moduleNameMapper: { + "^testcontainers$": path.resolve(__dirname, "../../testcontainers/src"), + }, +}; + +export default config; diff --git a/packages/modules/weaviate/package.json b/packages/modules/weaviate/package.json new file mode 100644 index 000000000..a24dffe5a --- /dev/null +++ b/packages/modules/weaviate/package.json @@ -0,0 +1,38 @@ +{ + "name": "@testcontainers/weaviate", + "version": "10.7.2", + "license": "MIT", + "keywords": [ + "weaviate", + "testing", + "docker", + "testcontainers" + ], + "description": "Weaviate module for Testcontainers", + "homepage": "https://github.com/testcontainers/testcontainers-node#readme", + "repository": { + "type": "git", + "url": "https://github.com/testcontainers/testcontainers-node" + }, + "bugs": { + "url": "https://github.com/testcontainers/testcontainers-node/issues" + }, + "main": "build/index.js", + "files": [ + "build" + ], + "publishConfig": { + "access": "public" + }, + "scripts": { + "prepack": "shx cp ../../../README.md . && shx cp ../../../LICENSE .", + "build": "tsc --project tsconfig.build.json" + }, + + "devDependencies": { + "weaviate-ts-client": "^2.1.0" + }, + "dependencies": { + "testcontainers": "^10.7.2" + } +} diff --git a/packages/modules/weaviate/src/index.ts b/packages/modules/weaviate/src/index.ts new file mode 100644 index 000000000..cb9945e80 --- /dev/null +++ b/packages/modules/weaviate/src/index.ts @@ -0,0 +1 @@ +export { WeaviateContainer, StartedWeaviateContainer } from "./weaviate-container"; diff --git a/packages/modules/weaviate/src/weaviate-container.test.ts b/packages/modules/weaviate/src/weaviate-container.test.ts new file mode 100644 index 000000000..dc8d4f845 --- /dev/null +++ b/packages/modules/weaviate/src/weaviate-container.test.ts @@ -0,0 +1,40 @@ +import { WeaviateContainer } from "./weaviate-container"; +import weaviate from "weaviate-ts-client"; + +describe("WeaviateContainer", () => { + jest.setTimeout(100_000); + + // expose { + it("should expose ports", async () => { + const container = await new WeaviateContainer().start(); + + expect(container.getRestHostAddress()).toBeDefined() + expect(container.getGrpcHostAddress()).toBeDefined() + + await container.stop(); + }); + // } + + // connect { + it("should connect to Weaviate", async () => { + const container = await new WeaviateContainer().start(); + + const client = weaviate.client({ + scheme: 'http', + host: container.getRestHostAddress(), + }); + + client.misc + .metaGetter() + .do() + .then((res: any) => { + expect(res.version).toBeDefined(); + }) + .catch((e: any) => { + throw new Error('it should not have errord: ' + e); + }); + + await container.stop(); + }); + // } +}); diff --git a/packages/modules/weaviate/src/weaviate-container.ts b/packages/modules/weaviate/src/weaviate-container.ts new file mode 100644 index 000000000..b8c08030a --- /dev/null +++ b/packages/modules/weaviate/src/weaviate-container.ts @@ -0,0 +1,40 @@ +import { AbstractStartedContainer, GenericContainer, StartedTestContainer, Wait } from "testcontainers"; + +const WEAVIATE_REST_PORT = 8080; +const WEAVIATE_GRPC_PORT = 50051; + +export class WeaviateContainer extends GenericContainer { + constructor(image = "semitechnologies/weaviate:1.24.5") { + super(image); + this.withCommand(["--host", "0.0.0.0", "--scheme", "http", "--port", "8080"]) + this.withExposedPorts(WEAVIATE_REST_PORT, WEAVIATE_GRPC_PORT); + this.withEnvironment({ + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: "true", + PERSISTENCE_DATA_PATH: "/var/lib/weaviate", + }) + this.withWaitStrategy( + Wait.forAll([ + Wait.forListeningPorts(), + Wait.forHttp("/v1/.well-known/ready", WEAVIATE_REST_PORT), + ]).withStartupTimeout(5_000) + ); + } + + public override async start(): Promise { + return new StartedWeaviateContainer(await super.start()); + } +} + +export class StartedWeaviateContainer extends AbstractStartedContainer { + constructor(startedTestContainer: StartedTestContainer) { + super(startedTestContainer); + } + + public getRestHostAddress(): string { + return `${this.getHost()}:${this.getMappedPort(WEAVIATE_REST_PORT)}` + } + + public getGrpcHostAddress(): string { + return `${this.getHost()}:${this.getMappedPort(WEAVIATE_GRPC_PORT)}` + } +} diff --git a/packages/modules/weaviate/tsconfig.build.json b/packages/modules/weaviate/tsconfig.build.json new file mode 100644 index 000000000..8aa91624b --- /dev/null +++ b/packages/modules/weaviate/tsconfig.build.json @@ -0,0 +1,14 @@ +{ + "extends": "./tsconfig.json", + "exclude": [ + "build", + "jest.config.ts", + "src/**/*.test.ts", + "src/test_config.yaml" + ], + "references": [ + { + "path": "../../testcontainers" + } + ] +} diff --git a/packages/modules/weaviate/tsconfig.json b/packages/modules/weaviate/tsconfig.json new file mode 100644 index 000000000..5b04a1a44 --- /dev/null +++ b/packages/modules/weaviate/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "build", + "paths": { + "testcontainers": [ + "../../testcontainers/src" + ] + } + }, + "exclude": [ + "build", + "jest.config.ts" + ], + "references": [ + { + "path": "../../testcontainers" + } + ] +}