Merge branch 'develop' of github.com:Budibase/budibase into fix/sql-issues

hosting/nginx.prod.conf.hbs

@@ -49,7 +49,7 @@ http {
     add_header X-Frame-Options SAMEORIGIN always;
     add_header X-Content-Type-Options nosniff always;
     add_header X-XSS-Protection "1; mode=block" always;
-    add_header Content-Security-Policy  "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' data https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com; frame-src 'self'; img-src http: https: data; manifest-src 'self'; media-src 'self'; worker-src 'none';" always;
+    add_header Content-Security-Policy  "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io; font-src 'self' data https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com; frame-src 'self' https:; img-src http: https: data; manifest-src 'self'; media-src 'self'; worker-src 'none';" always;
 
     # upstreams
     set $apps {{ apps }};

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.79-alpha.11",
+  "version": "1.0.80-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.79-alpha.11",
+  "version": "1.0.80-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.79-alpha.11",
+  "version": "1.0.80-alpha.0",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "^1.0.79-alpha.11",
+    "@budibase/string-templates": "^1.0.80-alpha.0",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.79-alpha.11",
+  "version": "1.0.80-alpha.0",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.79-alpha.11",
+    "@budibase/bbui": "^1.0.80-alpha.0",
-    "@budibase/client": "^1.0.79-alpha.11",
+    "@budibase/client": "^1.0.80-alpha.0",
-    "@budibase/frontend-core": "^1.0.79-alpha.11",
+    "@budibase/frontend-core": "^1.0.80-alpha.0",
-    "@budibase/string-templates": "^1.0.79-alpha.11",
+    "@budibase/string-templates": "^1.0.80-alpha.0",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.79-alpha.11",
+  "version": "1.0.80-alpha.0",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.79-alpha.11",
+  "version": "1.0.80-alpha.0",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.79-alpha.11",
+    "@budibase/bbui": "^1.0.80-alpha.0",
-    "@budibase/frontend-core": "^1.0.79-alpha.11",
+    "@budibase/frontend-core": "^1.0.80-alpha.0",
-    "@budibase/string-templates": "^1.0.79-alpha.11",
+    "@budibase/string-templates": "^1.0.80-alpha.0",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "1.0.79-alpha.11",
+  "version": "1.0.80-alpha.0",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "^1.0.79-alpha.11",
+    "@budibase/bbui": "^1.0.80-alpha.0",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/server/__mocks__/mysql2/promise.ts

@@ -0,0 +1,17 @@
+module MySQLMock {
+  const mysql: any = {}
+
+  const client = {
+    connect: jest.fn(),
+    end: jest.fn(),
+    query: jest.fn(async () => {
+      return [[]]
+    }),
+  }
+
+  mysql.createConnection = jest.fn(async () => {
+    return client
+  })
+
+  module.exports = mysql
+}

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.79-alpha.11",
+  "version": "1.0.80-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -71,9 +71,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.79-alpha.11",
+    "@budibase/backend-core": "^1.0.80-alpha.0",
-    "@budibase/client": "^1.0.79-alpha.11",
+    "@budibase/client": "^1.0.80-alpha.0",
-    "@budibase/string-templates": "^1.0.79-alpha.11",
+    "@budibase/string-templates": "^1.0.80-alpha.0",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/integrations/mysql.ts

@@ -16,7 +16,7 @@ import {
 import { DatasourcePlus } from "./base/datasourcePlus"
 
 module MySQLModule {
-  const mysql = require("mysql2")
+  const mysql = require("mysql2/promise")
   const Sql = require("./base/sql")
 
   interface MySQLConfig {
@@ -29,7 +29,7 @@ module MySQLModule {
   }
 
   const SCHEMA: Integration = {
-    docs: "https://github.com/mysqljs/mysql",
+    docs: "https://github.com/sidorares/node-mysql2",
     plus: true,
     friendlyName: "MySQL",
     description:
@@ -82,7 +82,7 @@ module MySQLModule {
 
   class MySQLIntegration extends Sql implements DatasourcePlus {
     private config: MySQLConfig
-    private readonly client: any
+    private client: any
     public tables: Record<string, Table> = {}
     public schemaErrors: Record<string, string> = {}
 
@@ -92,92 +92,99 @@ module MySQLModule {
       if (config.ssl && Object.keys(config.ssl).length === 0) {
         delete config.ssl
       }
-      this.client = mysql.createConnection(config)
+      this.config = config
     }
 
     getBindingIdentifier(): string {
       return "?"
     }
 
-    internalQuery(
+    async connect() {
+      this.client = await mysql.createConnection(this.config)
+    }
+
+    async disconnect() {
+      await this.client.end()
+    }
+
+    async internalQuery(
       query: SqlQuery,
       connect: boolean = true
     ): Promise<any[] | any> {
-      const client = this.client
+      try {
-      // Node MySQL is callback based, so we must wrap our call in a promise
-      return new Promise((resolve, reject) => {
         if (connect) {
-          client.connect()
+          await this.connect()
         }
-        return client.query(
+        // Node MySQL is callback based, so we must wrap our call in a promise
+        const response = await this.client.query(
           query.sql,
-          query.bindings || {},
+          query.bindings || []
-          (error: any, results: object[]) => {
-            if (error) {
-              reject(error)
-            } else {
-              resolve(results)
-            }
-            if (connect) {
-              client.end()
-            }
-          }
         )
-      })
+        return response[0]
+      } finally {
+        if (connect) {
+          await this.disconnect()
+        }
+      }
     }
 
     async buildSchema(datasourceId: string, entities: Record<string, Table>) {
       const tables: { [key: string]: Table } = {}
       const database = this.config.database
-      this.client.connect()
+      await this.connect()
 
-      // get the tables first
+      try {
-      const tablesResp = await this.internalQuery(
+        // get the tables first
-        { sql: "SHOW TABLES;" },
+        const tablesResp = await this.internalQuery(
-        false
+          { sql: "SHOW TABLES;" },
-      )
-      const tableNames = tablesResp.map(
-        (obj: any) =>
-          obj[`Tables_in_${database}`] ||
-          obj[`Tables_in_${database.toLowerCase()}`]
-      )
-      for (let tableName of tableNames) {
-        const primaryKeys = []
-        const schema: TableSchema = {}
-        const descResp = await this.internalQuery(
-          { sql: `DESCRIBE \`${tableName}\`;` },
           false
         )
-        for (let column of descResp) {
+        const tableNames = tablesResp.map(
-          const columnName = column.Field
+          (obj: any) =>
-          if (column.Key === "PRI" && primaryKeys.indexOf(column.Key) === -1) {
+            obj[`Tables_in_${database}`] ||
-            primaryKeys.push(columnName)
+            obj[`Tables_in_${database.toLowerCase()}`]
+        )
+        for (let tableName of tableNames) {
+          const primaryKeys = []
+          const schema: TableSchema = {}
+          const descResp = await this.internalQuery(
+            { sql: `DESCRIBE \`${tableName}\`;` },
+            false
+          )
+          for (let column of descResp) {
+            const columnName = column.Field
+            if (
+              column.Key === "PRI" &&
+              primaryKeys.indexOf(column.Key) === -1
+            ) {
+              primaryKeys.push(columnName)
+            }
+            const constraints = {
+              presence: column.Null !== "YES",
+            }
+            const isAuto: boolean =
+              typeof column.Extra === "string" &&
+              (column.Extra === "auto_increment" ||
+                column.Extra.toLowerCase().includes("generated"))
+            schema[columnName] = {
+              name: columnName,
+              autocolumn: isAuto,
+              type: convertSqlType(column.Type),
+              constraints,
+            }
           }
-          const constraints = {
+          if (!tables[tableName]) {
-            presence: column.Null !== "YES",
+            tables[tableName] = {
-          }
+              _id: buildExternalTableId(datasourceId, tableName),
-          const isAuto: boolean =
+              primary: primaryKeys,
-            typeof column.Extra === "string" &&
+              name: tableName,
-            (column.Extra === "auto_increment" ||
+              schema,
-              column.Extra.toLowerCase().includes("generated"))
+            }
-          schema[columnName] = {
-            name: columnName,
-            autocolumn: isAuto,
-            type: convertSqlType(column.Type),
-            constraints,
-          }
-        }
-        if (!tables[tableName]) {
-          tables[tableName] = {
-            _id: buildExternalTableId(datasourceId, tableName),
-            primary: primaryKeys,
-            name: tableName,
-            schema,
           }
         }
+      } finally {
+        await this.disconnect()
       }
-
-      this.client.end()
       const final = finaliseExternalTables(tables, entities)
       this.tables = final.tables
       this.schemaErrors = final.errors
@@ -203,11 +210,13 @@ module MySQLModule {
     }
 
     async query(json: QueryJson) {
-      this.client.connect()
+      await this.connect()
-      const queryFn = (query: any) => this.internalQuery(query, false)
+      try {
-      const output = await this.queryWithReturning(json, queryFn)
+        const queryFn = (query: any) => this.internalQuery(query, false)
-      this.client.end()
+        return await this.queryWithReturning(json, queryFn)
-      return output
+      } finally {
+        await this.disconnect()
+      }
     }
   }
 

packages/server/src/integrations/tests/mysql.spec.js

@@ -19,7 +19,7 @@ describe("MySQL Integration", () => {
     await config.integration.create({
       sql
     })
-    expect(config.integration.client.query).toHaveBeenCalledWith(sql, {}, expect.any(Function))
+    expect(config.integration.client.query).toHaveBeenCalledWith(sql, [])
   })
 
   it("calls the read method with the correct params", async () => {
@@ -27,7 +27,7 @@ describe("MySQL Integration", () => {
     await config.integration.read({
       sql
     })
-    expect(config.integration.client.query).toHaveBeenCalledWith(sql, {}, expect.any(Function))
+    expect(config.integration.client.query).toHaveBeenCalledWith(sql, [])
   })
 
   it("calls the update method with the correct params", async () => {
@@ -35,7 +35,7 @@ describe("MySQL Integration", () => {
     await config.integration.update({
       sql
     })
-    expect(config.integration.client.query).toHaveBeenCalledWith(sql, {}, expect.any(Function))
+    expect(config.integration.client.query).toHaveBeenCalledWith(sql, [])
   })
 
   it("calls the delete method with the correct params", async () => {
@@ -43,7 +43,7 @@ describe("MySQL Integration", () => {
     await config.integration.delete({
       sql
     })
-    expect(config.integration.client.query).toHaveBeenCalledWith(sql, {}, expect.any(Function))
+    expect(config.integration.client.query).toHaveBeenCalledWith(sql, [])
   })
 
   describe("no rows returned", () => {

packages/server/src/integrations/utils.ts

@@ -41,7 +41,7 @@ const SQL_TYPE_MAP = {
 export enum SqlClients {
   MS_SQL = "mssql",
   POSTGRES = "pg",
-  MY_SQL = "mysql",
+  MY_SQL = "mysql2",
   ORACLE = "oracledb",
 }
 

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.0.79-alpha.11",
+  "version": "1.0.80-alpha.0",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.79-alpha.11",
+  "version": "1.0.80-alpha.0",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -34,8 +34,8 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.79-alpha.11",
+    "@budibase/backend-core": "^1.0.80-alpha.0",
-    "@budibase/string-templates": "^1.0.79-alpha.11",
+    "@budibase/string-templates": "^1.0.80-alpha.0",
     "@koa/router": "^8.0.0",
     "@sentry/node": "^6.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",