From a23e4efd8cf2c5e9d4034a0cab9dcd2a14178997 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 20 May 2022 17:41:27 +0100 Subject: [PATCH 1/5] Fix for Postgres connections, seemed to be some issues with out of order, connections not opening in time. --- packages/server/src/integrations/postgres.ts | 48 ++++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 430a3f3461..a3e49687b6 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -136,7 +136,7 @@ module PostgresModule { : undefined, } this.client = new Client(newConfig) - this.setSchema() + this.open = false } getBindingIdentifier(): string { @@ -147,7 +147,34 @@ module PostgresModule { return parts.join(" || ") } + async openConnection() { + await this.client.connect() + if (!this.config.schema) { + this.config.schema = "public" + } + this.client.query(`SET search_path TO ${this.config.schema}`) + this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'` + this.open = true + } + + closeConnection() { + const pg = this + return new Promise((resolve, reject) => { + this.client.end((err: any) => { + pg.open = false + if (err) { + reject(err) + } else { + resolve() + } + }) + }) + } + async internalQuery(query: SqlQuery, close: boolean = true) { + if (!this.open) { + await this.openConnection() + } const client = this.client this.index = 1 // need to handle a specific issue with json data types in postgres, @@ -164,23 +191,16 @@ module PostgresModule { try { return await client.query(query.sql, query.bindings || []) } catch (err) { - await this.client.end() + await this.closeConnection() // @ts-ignore throw new Error(err) } finally { - if (close) await this.client.end() + if (close) { + await this.closeConnection() + } } } - async setSchema() { - await this.client.connect() - if (!this.config.schema) { - this.config.schema = "public" - } - this.client.query(`SET search_path TO ${this.config.schema}`) - this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'` - } - /** * Fetches the tables from the postgres table and assigns them to the datasource. * @param {*} datasourceId - datasourceId to fetch @@ -251,7 +271,7 @@ module PostgresModule { // @ts-ignore throw new Error(err) } finally { - await this.client.end() + await this.closeConnection() } } @@ -283,7 +303,7 @@ module PostgresModule { for (let query of input) { responses.push(await this.internalQuery(query, false)) } - await this.client.end() + await this.closeConnection() return responses } else { const response = await this.internalQuery(input) From 5f8cfcdb5c609c514550dc60877414232b5a4786 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 20 May 2022 17:43:47 +0100 Subject: [PATCH 2/5] Opening connection for building schema. --- packages/server/src/integrations/postgres.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index a3e49687b6..220f35dae5 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -208,6 +208,7 @@ module PostgresModule { */ async buildSchema(datasourceId: string, entities: Record) { let tableKeys: { [key: string]: string[] } = {} + await this.openConnection() try { const primaryKeysResponse = await this.client.query( this.PRIMARY_KEYS_SQL From 5b1ae1f6fc02506d91276230621bf2e4b2e320ab Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 20 May 2022 18:08:48 +0100 Subject: [PATCH 3/5] Fixing mock. --- packages/server/__mocks__/pg.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/__mocks__/pg.ts b/packages/server/__mocks__/pg.ts index 44aeabcb38..110933ad52 100644 --- a/packages/server/__mocks__/pg.ts +++ b/packages/server/__mocks__/pg.ts @@ -14,7 +14,9 @@ module PgMock { function Client() {} Client.prototype.query = query - Client.prototype.end = jest.fn() + Client.prototype.end = jest.fn(cb => { + if (cb) cb() + }) Client.prototype.connect = jest.fn() Client.prototype.release = jest.fn() From eae5d4dc4dc5d019065b0cacb0136bc21d1ed88f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 20 May 2022 18:29:37 +0100 Subject: [PATCH 4/5] Removing the use of pouchdb-all-dbs from testing environment, it was causing more harm than good and was entirely avoidable. --- packages/backend-core/package.json | 3 +-- packages/backend-core/src/db/index.js | 12 +++++++++--- packages/backend-core/src/db/pouch.js | 8 +------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index c0265f110a..bbd7a1af6b 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -41,8 +41,7 @@ "devDependencies": { "ioredis-mock": "^5.5.5", "jest": "^26.6.3", - "pouchdb-adapter-memory": "^7.2.2", - "pouchdb-all-dbs": "^1.0.2" + "pouchdb-adapter-memory": "^7.2.2" }, "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" } diff --git a/packages/backend-core/src/db/index.js b/packages/backend-core/src/db/index.js index 7d54b881b1..d179186988 100644 --- a/packages/backend-core/src/db/index.js +++ b/packages/backend-core/src/db/index.js @@ -3,13 +3,13 @@ const env = require("../environment") let PouchDB let initialised = false +const dbList = new Set() const put = dbPut => async (doc, options = {}) => { - const response = await dbPut(doc, options) // TODO: add created / updated - return response + return await dbPut(doc, options) } const checkInitialised = () => { @@ -28,6 +28,9 @@ exports.init = opts => { // in situations that using the function doWithDB does not work exports.dangerousGetDB = (dbName, opts) => { checkInitialised() + if (env.isTest()) { + dbList.add(dbName) + } const db = new PouchDB(dbName, opts) const dbPut = db.put db.put = put(dbPut) @@ -63,6 +66,9 @@ exports.doWithDB = async (dbName, cb, opts) => { } exports.allDbs = () => { + if (!env.isTest()) { + throw new Error("Cannot be used outside test environment.") + } checkInitialised() - return PouchDB.allDbs() + return [...dbList] } diff --git a/packages/backend-core/src/db/pouch.js b/packages/backend-core/src/db/pouch.js index 9c1ada8d76..76390ac644 100644 --- a/packages/backend-core/src/db/pouch.js +++ b/packages/backend-core/src/db/pouch.js @@ -92,11 +92,5 @@ exports.getPouch = (opts = {}) => { PouchDB.plugin(find) } - const Pouch = PouchDB.defaults(POUCH_DB_DEFAULTS) - if (opts.allDbs) { - const allDbs = require("pouchdb-all-dbs") - allDbs(Pouch) - } - - return Pouch + return PouchDB.defaults(POUCH_DB_DEFAULTS) } From be4535f5b7cefdec1bd21a01dccda7ecf9397d09 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 20 May 2022 18:38:11 +0100 Subject: [PATCH 5/5] Updating yarn locks. --- packages/server/yarn.lock | 18 +++++++++--------- packages/worker/yarn.lock | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 690f4ba0ef..e8a1186adf 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1014,10 +1014,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.164": - version "1.0.164" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164.tgz#08c111dcebf5c74159a3c18218c7b3a0716de4f6" - integrity sha512-lpMudezndUD1hHBLfT9LDNKCunj8rQNlaJb30/xggdIUvp718u/jVP54hXF26NYxXOTMZ0EvMwCsIS4AucJ1Mg== +"@budibase/backend-core@1.0.167": + version "1.0.167" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.167.tgz#6ea4b90f8b8b8ec3cbbe05e39853d44d40938879" + integrity sha512-IG9GZUdjFiqOKbgpZiwGotyT3BttFlChXs7mT8GaOkX7XvlyxxrG/nSI1duglBd6X2iafGESKQU8e6tKKQsxuw== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -1091,12 +1091,12 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@1.0.164": - version "1.0.164" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164.tgz#780ae38893d0609c87bf51fe96cc2c35bbdb431a" - integrity sha512-PgF7q2vADPPYzet4Wdma+THWuQPrEnN1+TfRly4l0oS9SUxutog3hYn0TlPmPS0AHgrqG/1v65TcEdC4ucX8TA== +"@budibase/pro@1.0.167": + version "1.0.167" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.167.tgz#62fe54b58206eb04151a786a5516351137bd2f84" + integrity sha512-AdWWOub58LMxZoZzXm3jy1ZSVOR1teSH+lwLisdGWLnoVAUV8e46pD7iOyJDM1SKuSyNWeQ1lXj8tcLHVK1+OA== dependencies: - "@budibase/backend-core" "1.0.164" + "@budibase/backend-core" "1.0.167" node-fetch "^2.6.1" "@budibase/standard-components@^0.9.139": diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 6ec47f0e09..e52f6e6d9d 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -293,10 +293,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.164": - version "1.0.164" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.164.tgz#08c111dcebf5c74159a3c18218c7b3a0716de4f6" - integrity sha512-lpMudezndUD1hHBLfT9LDNKCunj8rQNlaJb30/xggdIUvp718u/jVP54hXF26NYxXOTMZ0EvMwCsIS4AucJ1Mg== +"@budibase/backend-core@1.0.167": + version "1.0.167" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.167.tgz#6ea4b90f8b8b8ec3cbbe05e39853d44d40938879" + integrity sha512-IG9GZUdjFiqOKbgpZiwGotyT3BttFlChXs7mT8GaOkX7XvlyxxrG/nSI1duglBd6X2iafGESKQU8e6tKKQsxuw== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -321,12 +321,12 @@ uuid "^8.3.2" zlib "^1.0.5" -"@budibase/pro@1.0.164": - version "1.0.164" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.164.tgz#780ae38893d0609c87bf51fe96cc2c35bbdb431a" - integrity sha512-PgF7q2vADPPYzet4Wdma+THWuQPrEnN1+TfRly4l0oS9SUxutog3hYn0TlPmPS0AHgrqG/1v65TcEdC4ucX8TA== +"@budibase/pro@1.0.167": + version "1.0.167" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.167.tgz#62fe54b58206eb04151a786a5516351137bd2f84" + integrity sha512-AdWWOub58LMxZoZzXm3jy1ZSVOR1teSH+lwLisdGWLnoVAUV8e46pD7iOyJDM1SKuSyNWeQ1lXj8tcLHVK1+OA== dependencies: - "@budibase/backend-core" "1.0.164" + "@budibase/backend-core" "1.0.167" node-fetch "^2.6.1" "@cspotcode/source-map-consumer@0.8.0":