From 3321ac9cf7f63a041b2fa9263f1ee44fc3e37c2a Mon Sep 17 00:00:00 2001 From: Dave New Date: Mon, 24 Jun 2024 20:15:05 +0200 Subject: [PATCH] chore: neon dialect (#1502) --- packages/functions-runtime/package.json | 5 +- packages/functions-runtime/pnpm-lock.yaml | 99 ++++++++++++++++++++++ packages/functions-runtime/src/database.js | 47 +++++----- 3 files changed, 131 insertions(+), 20 deletions(-) diff --git a/packages/functions-runtime/package.json b/packages/functions-runtime/package.json index 2d708f35b..bb8f6e6d9 100644 --- a/packages/functions-runtime/package.json +++ b/packages/functions-runtime/package.json @@ -19,6 +19,7 @@ "vitest": "^0.34.6" }, "dependencies": { + "@neondatabase/serverless": "^0.9.3", "@opentelemetry/api": "^1.7.0", "@opentelemetry/exporter-trace-otlp-proto": "^0.46.0", "@opentelemetry/resources": "^1.19.0", @@ -28,7 +29,9 @@ "json-rpc-2.0": "^1.7.0", "ksuid": "^3.0.0", "kysely": "^0.25.0", + "kysely-neon": "^1.3.0", "pg": "^8.11.3", - "traceparent": "^1.0.0" + "traceparent": "^1.0.0", + "ws": "^8.17.1" } } diff --git a/packages/functions-runtime/pnpm-lock.yaml b/packages/functions-runtime/pnpm-lock.yaml index 69e050706..ff9326cab 100644 --- a/packages/functions-runtime/pnpm-lock.yaml +++ b/packages/functions-runtime/pnpm-lock.yaml @@ -5,6 +5,9 @@ settings: excludeLinksFromLockfile: false dependencies: + '@neondatabase/serverless': + specifier: ^0.9.3 + version: 0.9.3 '@opentelemetry/api': specifier: ^1.7.0 version: 1.7.0 @@ -32,12 +35,18 @@ dependencies: kysely: specifier: ^0.25.0 version: 0.25.0 + kysely-neon: + specifier: ^1.3.0 + version: 1.3.0(@neondatabase/serverless@0.9.3)(kysely@0.25.0)(ws@8.17.1) pg: specifier: ^8.11.3 version: 8.11.3 traceparent: specifier: ^1.0.0 version: 1.0.0 + ws: + specifier: ^8.17.1 + version: 8.17.1 devDependencies: prettier: @@ -267,6 +276,12 @@ packages: resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} dev: true + /@neondatabase/serverless@0.9.3: + resolution: {integrity: sha512-6ZBK8asl2Z3+ADEaELvbaVVGVlmY1oAzkxxZfpmXPKFuJhbDN+5fU3zYBamsahS/Ch1zE+CVWB3R+8QEI2LMSw==} + dependencies: + '@types/pg': 8.11.6 + dev: false + /@opentelemetry/api-logs@0.46.0: resolution: {integrity: sha512-+9BcqfiEDGPXEIo+o3tso/aqGM5dGbGwAkGVp3FPpZ8GlkK1YlaKRd9gMVyPaeRATwvO5wYGGnCsAc/sMMM9Qw==} engines: {node: '>=14'} @@ -604,6 +619,14 @@ packages: dependencies: undici-types: 5.26.5 + /@types/pg@8.11.6: + resolution: {integrity: sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==} + dependencies: + '@types/node': 20.10.5 + pg-protocol: 1.6.0 + pg-types: 4.0.2 + dev: false + /@vitest/expect@0.34.6: resolution: {integrity: sha512-QUzKpUQRc1qC7qdGo7rMK3AkETI7w18gTCUrsNnyjjJKYiuUB9+TQK3QnR1unhCnWRC0AbKv2omLGQDF/mIjOw==} dependencies: @@ -830,6 +853,21 @@ packages: base-convert-int-array: 1.0.1 dev: false + /kysely-neon@1.3.0(@neondatabase/serverless@0.9.3)(kysely@0.25.0)(ws@8.17.1): + resolution: {integrity: sha512-CIIlbmqpIXVJDdBEYtEOwbmALag0jmqYrGfBeM4cHKb9AgBGs+X1SvXUZ8TqkDacQEqEZN2XtsDoUkcMIISjHw==} + peerDependencies: + '@neondatabase/serverless': ^0.4.3 + kysely: 0.x.x + ws: ^8.13.0 + peerDependenciesMeta: + ws: + optional: true + dependencies: + '@neondatabase/serverless': 0.9.3 + kysely: 0.25.0 + ws: 8.17.1 + dev: false + /kysely@0.25.0: resolution: {integrity: sha512-srn0efIMu5IoEBk0tBmtGnoUss4uwvxtbFQWG/U2MosfqIace1l43IFP1PmEpHRDp+Z79xIcKEqmHH3dAvQdQA==} engines: {node: '>=14.0.0'} @@ -900,6 +938,10 @@ packages: tslib: 2.6.2 dev: false + /obuf@1.1.2: + resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} + dev: false + /p-limit@4.0.0: resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -955,6 +997,11 @@ packages: engines: {node: '>=4.0.0'} dev: false + /pg-numeric@1.0.2: + resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} + engines: {node: '>=4'} + dev: false + /pg-pool@3.6.1(pg@8.11.3): resolution: {integrity: sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==} peerDependencies: @@ -978,6 +1025,19 @@ packages: postgres-interval: 1.2.0 dev: false + /pg-types@4.0.2: + resolution: {integrity: sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng==} + engines: {node: '>=10'} + dependencies: + pg-int8: 1.0.1 + pg-numeric: 1.0.2 + postgres-array: 3.0.2 + postgres-bytea: 3.0.0 + postgres-date: 2.1.0 + postgres-interval: 3.0.0 + postgres-range: 1.1.4 + dev: false + /pg@8.11.3: resolution: {integrity: sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==} engines: {node: '>= 8.0.0'} @@ -1030,16 +1090,33 @@ packages: engines: {node: '>=4'} dev: false + /postgres-array@3.0.2: + resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==} + engines: {node: '>=12'} + dev: false + /postgres-bytea@1.0.0: resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==} engines: {node: '>=0.10.0'} dev: false + /postgres-bytea@3.0.0: + resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} + engines: {node: '>= 6'} + dependencies: + obuf: 1.1.2 + dev: false + /postgres-date@1.0.7: resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} engines: {node: '>=0.10.0'} dev: false + /postgres-date@2.1.0: + resolution: {integrity: sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==} + engines: {node: '>=12'} + dev: false + /postgres-interval@1.2.0: resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} engines: {node: '>=0.10.0'} @@ -1047,6 +1124,15 @@ packages: xtend: 4.0.2 dev: false + /postgres-interval@3.0.0: + resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} + engines: {node: '>=12'} + dev: false + + /postgres-range@1.1.4: + resolution: {integrity: sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==} + dev: false + /prettier@3.1.1: resolution: {integrity: sha512-22UbSzg8luF4UuZtzgiUOfcGM8s4tjBv6dJRT7j275NXsy2jb4aJa4NNveul5x4eqlF1wuhuR2RElK71RvmVaw==} engines: {node: '>=14'} @@ -1341,6 +1427,19 @@ packages: stackback: 0.0.2 dev: true + /ws@8.17.1: + resolution: {integrity: sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + dev: false + /xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} diff --git a/packages/functions-runtime/src/database.js b/packages/functions-runtime/src/database.js index 3f34c68f2..c156d5fad 100644 --- a/packages/functions-runtime/src/database.js +++ b/packages/functions-runtime/src/database.js @@ -4,6 +4,8 @@ const { AuditContextPlugin } = require("./auditing"); const pg = require("pg"); const { PROTO_ACTION_TYPES } = require("./consts"); const { withSpan } = require("./tracing"); +import { NeonDialect } from "kysely-neon"; +import ws from 'ws'; // withDatabase is responsible for setting the correct database client in our AsyncLocalStorage // so that the the code in a custom function uses the correct client. @@ -148,27 +150,34 @@ function getDialect() { return parseFloat(val); }); - return new PostgresDialect({ - pool: new InstrumentedPool({ - Client: InstrumentedClient, - // Increased idle time before closing a connection in the local pool (from 10s default). - // Establising a new connection on (almost) every functions query can be expensive, so this - // will reduce having to open connections as regularly. https://node-postgres.com/apis/pool - // - // NOTE: We should consider setting this to 0 (i.e. never pool locally) and open and close - // connections with each invocation. This is because the freeze/thaw nature of lambdas can cause problems - // with long-lived connections - see https://github.com/brianc/node-postgres/issues/2718 - // Once we're "fully regional" this should not be a performance problem anymore. - // - // Although I doubt we will run into these freeze/thaw issues if idleTimeoutMillis is always shorter than the - // time is takes for a lambda to freeze (which is not a constant, but could be as short as several minutes, - // https://www.pluralsight.com/resources/blog/cloud/how-long-does-aws-lambda-keep-your-idle-functions-around-before-a-cold-start) - idleTimeoutMillis: 0, - connectionTimeoutMillis: 0, - connectionString: mustEnv("KEEL_DB_CONN"), - }), + return new NeonDialect({ + connectionString: "postgresql://neondb_owner:MKdyUwxQIj48@ep-aged-silence-a2c04fyv.eu-central-1.aws.neon.tech/neondb?sslmode=require",//mustEnv("KEEL_DB_CONN"), + webSocketConstructor: ws, }); + // return new PostgresDialect({ + // connectisonString: mustEnv("KEEL_DB_CONN"), + // client: new InstrumentedClient, + // // pool: new InstrumentedPool({ + // // Client: InstrumentedClient, + // // // Increased idle time before closing a connection in the local pool (from 10s default). + // // // Establising a new connection on (almost) every functions query can be expensive, so this + // // // will reduce having to open connections as regularly. https://node-postgres.com/apis/pool + // // // + // // // NOTE: We should consider setting this to 0 (i.e. never pool locally) and open and close + // // // connections with each invocation. This is because the freeze/thaw nature of lambdas can cause problems + // // // with long-lived connections - see https://github.com/brianc/node-postgres/issues/2718 + // // // Once we're "fully regional" this should not be a performance problem anymore. + // // // + // // // Although I doubt we will run into these freeze/thaw issues if idleTimeoutMillis is always shorter than the + // // // time is takes for a lambda to freeze (which is not a constant, but could be as short as several minutes, + // // // https://www.pluralsight.com/resources/blog/cloud/how-long-does-aws-lambda-keep-your-idle-functions-around-before-a-cold-start) + // // idleTimeoutMillis: 0, + // // connectionTimeoutMillis: 0, + // // connectionString: mustEnv("KEEL_DB_CONN"), + // // }), + // }); + default: throw Error("unexpected KEEL_DB_CONN_TYPE: " + dbConnType); }