Merge branch 'master' into budi-7710-user-groups-do-not-fully-support-custom-roles-3

packages/account-portal

@@ -1 +1 @@
-Subproject commit 19f7a5829f4d23cbc694136e45d94482a59a475a
+Subproject commit 0c050591c21d3b67dc0c9225d60cc9e2324c8dac

packages/builder/src/stores/builder/tables.js

@@ -147,6 +147,12 @@ export function createTablesStore() {
     if (indexes) {
       draft.indexes = indexes
     }
+    // Add object to indicate if column is being added
+    if (draft.schema[field.name] === undefined) {
+      draft._add = {
+        name: field.name,
+      }
+    }
     draft.schema = {
       ...draft.schema,
       [field.name]: cloneDeep(field),

packages/pro

@@ -1 +1 @@
-Subproject commit 183b35d3acd42433dcb2d32bcd89a36abe13afec
+Subproject commit 22a278da720d92991dabdcd4cb6c96e7abe29781

packages/server/src/api/controllers/table/external.ts

@@ -6,6 +6,7 @@ import {
   BulkImportRequest,
   BulkImportResponse,
   Operation,
+  RenameColumn,
   SaveTableRequest,
   SaveTableResponse,
   Table,
@@ -25,9 +26,12 @@ function getDatasourceId(table: Table) {
   return breakExternalTableId(table._id).datasourceId
 }
 
-export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
+export async function save(
+  ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
+  renaming?: RenameColumn
+) {
   const inputs = ctx.request.body
-  const renaming = inputs?._rename
+  const adding = inputs?._add
   // can't do this right now
   delete inputs.rows
   const tableId = ctx.request.body._id
@@ -40,7 +44,7 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
     const { datasource, table } = await sdk.tables.external.save(
       datasourceId!,
       inputs,
-      { tableId, renaming }
+      { tableId, renaming, adding }
     )
     builderSocket?.emitDatasourceUpdate(ctx, datasource)
     return table

packages/server/src/api/controllers/table/index.ts

@@ -74,8 +74,15 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   const appId = ctx.appId
   const table = ctx.request.body
   const isImport = table.rows
+  const renaming = ctx.request.body._rename
 
-  let savedTable = await pickApi({ table }).save(ctx)
+  const api = pickApi({ table })
+  // do not pass _rename or _add if saving to CouchDB
+  if (api === internal) {
+    delete ctx.request.body._add
+    delete ctx.request.body._rename
+  }
+  let savedTable = await api.save(ctx, renaming)
   if (!table._id) {
     await events.table.created(savedTable)
     savedTable = sdk.tables.enrichViewSchemas(savedTable)

packages/server/src/api/controllers/table/internal.ts

@@ -12,11 +12,12 @@ import {
 } from "@budibase/types"
 import sdk from "../../../sdk"
 
-export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
+export async function save(
+  ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
+  renaming?: RenameColumn
+) {
   const { rows, ...rest } = ctx.request.body
-  let tableToSave: Table & {
-    _rename?: RenameColumn
-  } = {
+  let tableToSave: Table = {
     _id: generateTableID(),
     ...rest,
     // Ensure these fields are populated, even if not sent in the request
@@ -28,15 +29,12 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
     tableToSave.views = {}
   }
 
-  const renaming = tableToSave._rename
-  delete tableToSave._rename
-
   try {
     const { table } = await sdk.tables.internal.save(tableToSave, {
       user: ctx.user,
       rowsToImport: rows,
       tableId: ctx.request.body._id,
-      renaming: renaming,
+      renaming,
     })
 
     return table

packages/server/src/api/routes/tests/table.spec.ts

@@ -26,6 +26,7 @@ import { TableToBuild } from "../../../tests/utilities/TestConfiguration"
 tk.freeze(mocks.date.MOCK_DATE)
 
 const { basicTable } = setup.structures
+const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
 
 describe("/tables", () => {
   let request = setup.getRequest()
@@ -285,6 +286,35 @@ describe("/tables", () => {
         expect(res.body.schema.roleId).toBeDefined()
       })
     })
+
+    it("should add a new column for an internal DB table", async () => {
+      const saveTableRequest: SaveTableRequest = {
+        _add: {
+          name: "NEW_COLUMN",
+        },
+        ...basicTable(),
+      }
+
+      const response = await request
+        .post(`/api/tables`)
+        .send(saveTableRequest)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+
+      const expectedResponse = {
+        ...saveTableRequest,
+        _rev: expect.stringMatching(/^\d-.+/),
+        _id: expect.stringMatching(/^ta_.+/),
+        createdAt: expect.stringMatching(ISO_REGEX_PATTERN),
+        updatedAt: expect.stringMatching(ISO_REGEX_PATTERN),
+        views: {},
+      }
+      delete expectedResponse._add
+
+      expect(response.status).toBe(200)
+      expect(response.body).toEqual(expectedResponse)
+    })
   })
 
   describe("import", () => {

packages/server/src/integration-test/mysql.spec.ts

@@ -0,0 +1,363 @@
+import fetch from "node-fetch"
+import {
+  generateMakeRequest,
+  MakeRequestResponse,
+} from "../api/routes/public/tests/utils"
+import { v4 as uuidv4 } from "uuid"
+import * as setup from "../api/routes/tests/utilities"
+import {
+  Datasource,
+  FieldType,
+  Table,
+  TableRequest,
+  TableSourceType,
+} from "@budibase/types"
+import _ from "lodash"
+import { databaseTestProviders } from "../integrations/tests/utils"
+import mysql from "mysql2/promise"
+import { builderSocket } from "../websockets"
+// @ts-ignore
+fetch.mockSearch()
+
+const config = setup.getConfig()!
+
+jest.unmock("mysql2/promise")
+jest.mock("../websockets", () => ({
+  clientAppSocket: jest.fn(),
+  gridAppSocket: jest.fn(),
+  initialise: jest.fn(),
+  builderSocket: {
+    emitTableUpdate: jest.fn(),
+    emitTableDeletion: jest.fn(),
+    emitDatasourceUpdate: jest.fn(),
+    emitDatasourceDeletion: jest.fn(),
+    emitScreenUpdate: jest.fn(),
+    emitAppMetadataUpdate: jest.fn(),
+    emitAppPublish: jest.fn(),
+  },
+}))
+
+describe("mysql integrations", () => {
+  let makeRequest: MakeRequestResponse,
+    mysqlDatasource: Datasource,
+    primaryMySqlTable: Table
+
+  beforeAll(async () => {
+    await config.init()
+    const apiKey = await config.generateApiKey()
+
+    makeRequest = generateMakeRequest(apiKey, true)
+
+    mysqlDatasource = await config.api.datasource.create(
+      await databaseTestProviders.mysql.datasource()
+    )
+  })
+
+  afterAll(async () => {
+    await databaseTestProviders.mysql.stop()
+  })
+
+  beforeEach(async () => {
+    primaryMySqlTable = await config.createTable({
+      name: uuidv4(),
+      type: "table",
+      primary: ["id"],
+      schema: {
+        id: {
+          name: "id",
+          type: FieldType.AUTO,
+          autocolumn: true,
+        },
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+        },
+        description: {
+          name: "description",
+          type: FieldType.STRING,
+        },
+        value: {
+          name: "value",
+          type: FieldType.NUMBER,
+        },
+      },
+      sourceId: mysqlDatasource._id,
+      sourceType: TableSourceType.EXTERNAL,
+    })
+  })
+
+  afterAll(config.end)
+
+  it("validate table schema", async () => {
+    const res = await makeRequest(
+      "get",
+      `/api/datasources/${mysqlDatasource._id}`
+    )
+
+    expect(res.status).toBe(200)
+    expect(res.body).toEqual({
+      config: {
+        database: "mysql",
+        host: mysqlDatasource.config!.host,
+        password: "--secret-value--",
+        port: mysqlDatasource.config!.port,
+        user: "root",
+      },
+      plus: true,
+      source: "MYSQL",
+      type: "datasource_plus",
+      _id: expect.any(String),
+      _rev: expect.any(String),
+      createdAt: expect.any(String),
+      updatedAt: expect.any(String),
+      entities: expect.any(Object),
+    })
+  })
+
+  describe("POST /api/datasources/verify", () => {
+    it("should be able to verify the connection", async () => {
+      await config.api.datasource.verify(
+        {
+          datasource: await databaseTestProviders.mysql.datasource(),
+        },
+        {
+          body: {
+            connected: true,
+          },
+        }
+      )
+    })
+
+    it("should state an invalid datasource cannot connect", async () => {
+      const dbConfig = await databaseTestProviders.mysql.datasource()
+      await config.api.datasource.verify(
+        {
+          datasource: {
+            ...dbConfig,
+            config: {
+              ...dbConfig.config,
+              password: "wrongpassword",
+            },
+          },
+        },
+        {
+          body: {
+            connected: false,
+            error:
+              "Access denied for the specified user. User does not have the necessary privileges or the provided credentials are incorrect. Please verify the credentials, and ensure that the user has appropriate permissions.",
+          },
+        }
+      )
+    })
+  })
+
+  describe("POST /api/datasources/info", () => {
+    it("should fetch information about mysql datasource", async () => {
+      const primaryName = primaryMySqlTable.name
+      const response = await makeRequest("post", "/api/datasources/info", {
+        datasource: mysqlDatasource,
+      })
+      expect(response.status).toBe(200)
+      expect(response.body.tableNames).toBeDefined()
+      expect(response.body.tableNames.indexOf(primaryName)).not.toBe(-1)
+    })
+  })
+
+  describe("Integration compatibility with mysql search_path", () => {
+    let client: mysql.Connection, pathDatasource: Datasource
+    const database = "test1"
+    const database2 = "test-2"
+
+    beforeAll(async () => {
+      const dsConfig = await databaseTestProviders.mysql.datasource()
+      const dbConfig = dsConfig.config!
+
+      client = await mysql.createConnection(dbConfig)
+      await client.query(`CREATE DATABASE \`${database}\`;`)
+      await client.query(`CREATE DATABASE \`${database2}\`;`)
+
+      const pathConfig: any = {
+        ...dsConfig,
+        config: {
+          ...dbConfig,
+          database,
+        },
+      }
+      pathDatasource = await config.api.datasource.create(pathConfig)
+    })
+
+    afterAll(async () => {
+      await client.query(`DROP DATABASE \`${database}\`;`)
+      await client.query(`DROP DATABASE \`${database2}\`;`)
+      await client.end()
+    })
+
+    it("discovers tables from any schema in search path", async () => {
+      await client.query(
+        `CREATE TABLE \`${database}\`.table1 (id1 SERIAL PRIMARY KEY);`
+      )
+      const response = await makeRequest("post", "/api/datasources/info", {
+        datasource: pathDatasource,
+      })
+      expect(response.status).toBe(200)
+      expect(response.body.tableNames).toBeDefined()
+      expect(response.body.tableNames).toEqual(
+        expect.arrayContaining(["table1"])
+      )
+    })
+
+    it("does not mix columns from different tables", async () => {
+      const repeated_table_name = "table_same_name"
+      await client.query(
+        `CREATE TABLE \`${database}\`.${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);`
+      )
+      await client.query(
+        `CREATE TABLE \`${database2}\`.${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);`
+      )
+      const response = await makeRequest(
+        "post",
+        `/api/datasources/${pathDatasource._id}/schema`,
+        {
+          tablesFilter: [repeated_table_name],
+        }
+      )
+      expect(response.status).toBe(200)
+      expect(
+        response.body.datasource.entities[repeated_table_name].schema
+      ).toBeDefined()
+      const schema =
+        response.body.datasource.entities[repeated_table_name].schema
+      expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
+    })
+  })
+
+  describe("POST /api/tables/", () => {
+    let client: mysql.Connection
+    const emitDatasourceUpdateMock = jest.fn()
+
+    beforeEach(async () => {
+      client = await mysql.createConnection(
+        (
+          await databaseTestProviders.mysql.datasource()
+        ).config!
+      )
+      mysqlDatasource = await config.api.datasource.create(
+        await databaseTestProviders.mysql.datasource()
+      )
+    })
+
+    afterEach(async () => {
+      await client.end()
+    })
+
+    it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => {
+      const addColumnToTable: TableRequest = {
+        type: "table",
+        sourceType: TableSourceType.EXTERNAL,
+        name: "table",
+        sourceId: mysqlDatasource._id!,
+        primary: ["id"],
+        schema: {
+          id: {
+            type: FieldType.AUTO,
+            name: "id",
+            autocolumn: true,
+          },
+          new_column: {
+            type: FieldType.NUMBER,
+            name: "new_column",
+          },
+        },
+        _add: {
+          name: "new_column",
+        },
+      }
+
+      jest
+        .spyOn(builderSocket!, "emitDatasourceUpdate")
+        .mockImplementation(emitDatasourceUpdateMock)
+
+      await makeRequest("post", "/api/tables/", addColumnToTable)
+
+      const expectedTable: TableRequest = {
+        ...addColumnToTable,
+        schema: {
+          id: {
+            type: FieldType.NUMBER,
+            name: "id",
+            autocolumn: true,
+            constraints: {
+              presence: false,
+            },
+            externalType: "int unsigned",
+          },
+          new_column: {
+            type: FieldType.NUMBER,
+            name: "new_column",
+            autocolumn: false,
+            constraints: {
+              presence: false,
+            },
+            externalType: "float(8,2)",
+          },
+        },
+        created: true,
+        _id: `${mysqlDatasource._id}__table`,
+      }
+      delete expectedTable._add
+
+      expect(emitDatasourceUpdateMock).toBeCalledTimes(1)
+      const emittedDatasource: Datasource =
+        emitDatasourceUpdateMock.mock.calls[0][1]
+      expect(emittedDatasource.entities!["table"]).toEqual(expectedTable)
+    })
+
+    it("will rename a column", async () => {
+      await makeRequest("post", "/api/tables/", primaryMySqlTable)
+
+      let renameColumnOnTable: TableRequest = {
+        ...primaryMySqlTable,
+        schema: {
+          id: {
+            name: "id",
+            type: FieldType.AUTO,
+            autocolumn: true,
+            externalType: "unsigned integer",
+          },
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+            externalType: "text",
+          },
+          description: {
+            name: "description",
+            type: FieldType.STRING,
+            externalType: "text",
+          },
+          age: {
+            name: "age",
+            type: FieldType.NUMBER,
+            externalType: "float(8,2)",
+          },
+        },
+      }
+
+      const response = await makeRequest(
+        "post",
+        "/api/tables/",
+        renameColumnOnTable
+      )
+      mysqlDatasource = (
+        await makeRequest(
+          "post",
+          `/api/datasources/${mysqlDatasource._id}/schema`
+        )
+      ).body.datasource
+
+      expect(response.status).toEqual(200)
+      expect(
+        Object.keys(mysqlDatasource.entities![primaryMySqlTable.name].schema)
+      ).toEqual(["id", "name", "description", "age"])
+    })
+  })
+})

packages/server/src/sdk/app/tables/external/index.ts

@@ -3,6 +3,7 @@ import {
   Operation,
   RelationshipType,
   RenameColumn,
+  AddColumn,
   Table,
   TableRequest,
   ViewV2,
@@ -32,7 +33,7 @@ import * as viewSdk from "../../views"
 export async function save(
   datasourceId: string,
   update: Table,
-  opts?: { tableId?: string; renaming?: RenameColumn }
+  opts?: { tableId?: string; renaming?: RenameColumn; adding?: AddColumn }
 ) {
   let tableToSave: TableRequest = {
     ...update,
@@ -165,8 +166,17 @@ export async function save(
 
   // remove the rename prop
   delete tableToSave._rename
+
+  // if adding a new column, we need to rebuild the schema for that table to get the 'externalType' of the column
+  if (opts?.adding) {
+    datasource.entities[tableToSave.name] = (
+      await datasourceSdk.buildFilteredSchema(datasource, [tableToSave.name])
+    ).tables[tableToSave.name]
+  } else {
+    datasource.entities[tableToSave.name] = tableToSave
+  }
+
   // store it into couch now for budibase reference
-  datasource.entities[tableToSave.name] = tableToSave
   await db.put(populateExternalTableSchemas(datasource))
 
   // Since tables are stored inside datasources, we need to notify clients

packages/types/src/documents/app/table/table.ts

@@ -1,6 +1,6 @@
 import { Document } from "../../document"
 import { View, ViewV2 } from "../view"
-import { RenameColumn } from "../../../sdk"
+import { AddColumn, RenameColumn } from "../../../sdk"
 import { TableSchema } from "./schema"
 
 export const INTERNAL_TABLE_SOURCE_ID = "bb_internal"
@@ -29,5 +29,6 @@ export interface Table extends Document {
 
 export interface TableRequest extends Table {
   _rename?: RenameColumn
+  _add?: AddColumn
   created?: boolean
 }

packages/types/src/sdk/search.ts

@@ -60,6 +60,10 @@ export interface RenameColumn {
   updated: string
 }
 
+export interface AddColumn {
+  name: string
+}
+
 export interface RelationshipsJson {
   through?: string
   from?: string