From 79dfe56d1164fece17c182d05ddbe32307d556d4 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 2 Mar 2022 22:43:41 +0100 Subject: [PATCH 1/4] allowing iframes from HTTPS URLs --- hosting/nginx.prod.conf.hbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs index f446c928fb..8d8f8c2d0c 100644 --- a/hosting/nginx.prod.conf.hbs +++ b/hosting/nginx.prod.conf.hbs @@ -55,7 +55,7 @@ http { add_header X-Frame-Options SAMEORIGIN always; add_header X-Content-Type-Options nosniff always; add_header X-XSS-Protection "1; mode=block" always; - add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' data https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com; frame-src 'self'; img-src http: https: data; manifest-src 'self'; media-src 'self'; worker-src 'none';" always; + add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' data https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com; frame-src 'self' https:; img-src http: https: data; manifest-src 'self'; media-src 'self'; worker-src 'none';" always; # upstreams set $apps {{ apps }}; From 07551c54c170885f5de2b383335828a68fe2542a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 3 Mar 2022 19:20:26 +0000 Subject: [PATCH 2/4] Attempting to fix mysql issue by changing our usage of mysql2 to use the promise version, making sure disconnection always occurs correctly and using a slightly different syntax/approach. --- packages/server/src/integrations/mysql.ts | 175 +++++++++++----------- packages/server/src/integrations/utils.ts | 2 +- 2 files changed, 92 insertions(+), 85 deletions(-) diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 24a55a273d..82377aa21a 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -16,7 +16,7 @@ import { import { DatasourcePlus } from "./base/datasourcePlus" module MySQLModule { - const mysql = require("mysql2") + const mysql = require("mysql2/promise") const Sql = require("./base/sql") interface MySQLConfig { @@ -29,7 +29,7 @@ module MySQLModule { } const SCHEMA: Integration = { - docs: "https://github.com/mysqljs/mysql", + docs: "https://github.com/sidorares/node-mysql2", plus: true, friendlyName: "MySQL", description: @@ -80,36 +80,9 @@ module MySQLModule { }, } - function internalQuery( - client: any, - query: SqlQuery, - connect: boolean = true - ): Promise { - // Node MySQL is callback based, so we must wrap our call in a promise - return new Promise((resolve, reject) => { - if (connect) { - client.connect() - } - return client.query( - query.sql, - query.bindings || {}, - (error: any, results: object[]) => { - if (error) { - reject(error) - } else { - resolve(results) - } - if (connect) { - client.end() - } - } - ) - }) - } - class MySQLIntegration extends Sql implements DatasourcePlus { private config: MySQLConfig - private readonly client: any + private client: any public tables: Record = {} public schemaErrors: Record = {} @@ -119,93 +92,127 @@ module MySQLModule { if (config.ssl && Object.keys(config.ssl).length === 0) { delete config.ssl } - this.client = mysql.createConnection(config) + this.config = config + } + + async connect() { + this.client = await mysql.createConnection(this.config) + } + + async disconnect() { + await this.client.end() + } + + async internalQuery( + query: SqlQuery, + connect: boolean = true + ): Promise { + try { + if (connect) { + await this.connect() + } + // Node MySQL is callback based, so we must wrap our call in a promise + const response = await this.client.query( + query.sql, + query.bindings || [] + ) + return response[0] + } finally { + if (connect) { + await this.disconnect() + } + } } async buildSchema(datasourceId: string, entities: Record) { const tables: { [key: string]: Table } = {} const database = this.config.database - this.client.connect() + await this.connect() - // get the tables first - const tablesResp = await internalQuery( - this.client, - { sql: "SHOW TABLES;" }, - false - ) - const tableNames = tablesResp.map( - (obj: any) => - obj[`Tables_in_${database}`] || - obj[`Tables_in_${database.toLowerCase()}`] - ) - for (let tableName of tableNames) { - const primaryKeys = [] - const schema: TableSchema = {} - const descResp = await internalQuery( - this.client, - { sql: `DESCRIBE \`${tableName}\`;` }, + try { + // get the tables first + const tablesResp = await this.internalQuery( + { sql: "SHOW TABLES;" }, false ) - for (let column of descResp) { - const columnName = column.Field - if (column.Key === "PRI" && primaryKeys.indexOf(column.Key) === -1) { - primaryKeys.push(columnName) + const tableNames = tablesResp.map( + (obj: any) => + obj[`Tables_in_${database}`] || + obj[`Tables_in_${database.toLowerCase()}`] + ) + for (let tableName of tableNames) { + const primaryKeys = [] + const schema: TableSchema = {} + const descResp = await this.internalQuery( + { sql: `DESCRIBE \`${tableName}\`;` }, + false + ) + for (let column of descResp) { + const columnName = column.Field + if ( + column.Key === "PRI" && + primaryKeys.indexOf(column.Key) === -1 + ) { + primaryKeys.push(columnName) + } + const constraints = { + presence: column.Null !== "YES", + } + const isAuto: boolean = + typeof column.Extra === "string" && + (column.Extra === "auto_increment" || + column.Extra.toLowerCase().includes("generated")) + schema[columnName] = { + name: columnName, + autocolumn: isAuto, + type: convertSqlType(column.Type), + constraints, + } } - const constraints = { - presence: column.Null !== "YES", - } - const isAuto: boolean = - typeof column.Extra === "string" && - (column.Extra === "auto_increment" || - column.Extra.toLowerCase().includes("generated")) - schema[columnName] = { - name: columnName, - autocolumn: isAuto, - type: convertSqlType(column.Type), - constraints, - } - } - if (!tables[tableName]) { - tables[tableName] = { - _id: buildExternalTableId(datasourceId, tableName), - primary: primaryKeys, - name: tableName, - schema, + if (!tables[tableName]) { + tables[tableName] = { + _id: buildExternalTableId(datasourceId, tableName), + primary: primaryKeys, + name: tableName, + schema, + } } } + } finally { + await this.disconnect() } - - this.client.end() const final = finaliseExternalTables(tables, entities) this.tables = final.tables this.schemaErrors = final.errors } async create(query: SqlQuery | string) { - const results = await internalQuery(this.client, getSqlQuery(query)) + const results = await this.internalQuery(getSqlQuery(query)) return results.length ? results : [{ created: true }] } async read(query: SqlQuery | string) { - return internalQuery(this.client, getSqlQuery(query)) + return this.internalQuery(getSqlQuery(query)) } async update(query: SqlQuery | string) { - const results = await internalQuery(this.client, getSqlQuery(query)) + const results = await this.internalQuery(getSqlQuery(query)) return results.length ? results : [{ updated: true }] } async delete(query: SqlQuery | string) { - const results = await internalQuery(this.client, getSqlQuery(query)) + const results = await this.internalQuery(getSqlQuery(query)) return results.length ? results : [{ deleted: true }] } async query(json: QueryJson) { - this.client.connect() - const queryFn = (query: any) => internalQuery(this.client, query, false) - const output = await this.queryWithReturning(json, queryFn) - this.client.end() - return output + await this.connect() + try { + const queryFn = (query: any) => this.internalQuery(query, false) + return await this.queryWithReturning(json, queryFn) + } finally { + await this.disconnect() + } } } diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 1341f5abca..26e35f300f 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -40,7 +40,7 @@ const SQL_TYPE_MAP = { export enum SqlClients { MS_SQL = "mssql", POSTGRES = "pg", - MY_SQL = "mysql", + MY_SQL = "mysql2", ORACLE = "oracledb", } From 7179cf978abc2e9d608a9b13ac7d06c895f205db Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Thu, 3 Mar 2022 23:50:46 +0000 Subject: [PATCH 3/4] Updating test case to handle new promise library. --- packages/server/__mocks__/mysql2/promise.ts | 17 +++++++++++++++++ .../server/src/integrations/tests/mysql.spec.js | 8 ++++---- 2 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 packages/server/__mocks__/mysql2/promise.ts diff --git a/packages/server/__mocks__/mysql2/promise.ts b/packages/server/__mocks__/mysql2/promise.ts new file mode 100644 index 0000000000..8a8fb7fcf0 --- /dev/null +++ b/packages/server/__mocks__/mysql2/promise.ts @@ -0,0 +1,17 @@ +module MySQLMock { + const mysql: any = {} + + const client = { + connect: jest.fn(), + end: jest.fn(), + query: jest.fn(async () => { + return [[]] + }), + } + + mysql.createConnection = jest.fn(async () => { + return client + }) + + module.exports = mysql +} diff --git a/packages/server/src/integrations/tests/mysql.spec.js b/packages/server/src/integrations/tests/mysql.spec.js index 47ca3688f0..8304771d5d 100644 --- a/packages/server/src/integrations/tests/mysql.spec.js +++ b/packages/server/src/integrations/tests/mysql.spec.js @@ -19,7 +19,7 @@ describe("MySQL Integration", () => { await config.integration.create({ sql }) - expect(config.integration.client.query).toHaveBeenCalledWith(sql, {}, expect.any(Function)) + expect(config.integration.client.query).toHaveBeenCalledWith(sql, []) }) it("calls the read method with the correct params", async () => { @@ -27,7 +27,7 @@ describe("MySQL Integration", () => { await config.integration.read({ sql }) - expect(config.integration.client.query).toHaveBeenCalledWith(sql, {}, expect.any(Function)) + expect(config.integration.client.query).toHaveBeenCalledWith(sql, []) }) it("calls the update method with the correct params", async () => { @@ -35,7 +35,7 @@ describe("MySQL Integration", () => { await config.integration.update({ sql }) - expect(config.integration.client.query).toHaveBeenCalledWith(sql, {}, expect.any(Function)) + expect(config.integration.client.query).toHaveBeenCalledWith(sql, []) }) it("calls the delete method with the correct params", async () => { @@ -43,7 +43,7 @@ describe("MySQL Integration", () => { await config.integration.delete({ sql }) - expect(config.integration.client.query).toHaveBeenCalledWith(sql, {}, expect.any(Function)) + expect(config.integration.client.query).toHaveBeenCalledWith(sql, []) }) describe("no rows returned", () => { From 89e4c59e3cb151e0acb46d5abdb9fd00be0b5809 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Fri, 4 Mar 2022 10:13:53 +0000 Subject: [PATCH 4/4] v1.0.79 --- lerna.json | 2 +- packages/backend-core/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 2 +- packages/client/package.json | 8 ++++---- packages/frontend-core/package.json | 4 ++-- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 10 files changed, 23 insertions(+), 23 deletions(-) diff --git a/lerna.json b/lerna.json index d4147c70f0..af7c5930bd 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.78", + "version": "1.0.79", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 851773f355..9e19ea3517 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.78", + "version": "1.0.79", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 7d2bef9ab9..6dcb6a2b76 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.78", + "version": "1.0.79", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index ab387222bd..ce6fe862ad 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.78", + "version": "1.0.79", "license": "GPL-3.0", "private": true, "scripts": { @@ -64,10 +64,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.78", - "@budibase/client": "^1.0.78", - "@budibase/frontend-core": "^1.0.78", - "@budibase/string-templates": "^1.0.78", + "@budibase/bbui": "^1.0.79", + "@budibase/client": "^1.0.79", + "@budibase/frontend-core": "^1.0.79", + "@budibase/string-templates": "^1.0.79", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 767bc291b8..e0f0013909 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.78", + "version": "1.0.79", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 065b43b4b5..08ed7e8224 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.78", + "version": "1.0.79", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.78", - "@budibase/frontend-core": "^1.0.78", - "@budibase/string-templates": "^1.0.78", + "@budibase/bbui": "^1.0.79", + "@budibase/frontend-core": "^1.0.79", + "@budibase/string-templates": "^1.0.79", "regexparam": "^1.3.0", "rollup-plugin-polyfill-node": "^0.8.0", "shortid": "^2.2.15", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index ca47be4eac..ba86320542 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "1.0.78", + "version": "1.0.79", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^1.0.78", + "@budibase/bbui": "^1.0.79", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/server/package.json b/packages/server/package.json index 9e79e24b0b..66d6e9bd6e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.78", + "version": "1.0.79", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -73,9 +73,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.78", - "@budibase/client": "^1.0.78", - "@budibase/string-templates": "^1.0.78", + "@budibase/backend-core": "^1.0.79", + "@budibase/client": "^1.0.79", + "@budibase/string-templates": "^1.0.79", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index f9c7560de4..5134f7fb89 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.78", + "version": "1.0.79", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 1aa5978c76..76bfe29ccd 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.78", + "version": "1.0.79", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -34,8 +34,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.78", - "@budibase/string-templates": "^1.0.78", + "@budibase/backend-core": "^1.0.79", + "@budibase/string-templates": "^1.0.79", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0",