Merge pull request #13238 from Budibase/budi-8067-sql-testing-more-datasource-types

Update row.spec.ts to test against more real databases.

packages/server/src/api/routes/tests/row.spec.ts

@@ -38,11 +38,18 @@ import * as uuid from "uuid"
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
 
+jest.unmock("mysql2")
+jest.unmock("mysql2/promise")
+jest.unmock("mssql")
+
 const { basicRow } = setup.structures
 
 describe.each([
   ["internal", undefined],
   ["postgres", databaseTestProviders.postgres],
+  ["mysql", databaseTestProviders.mysql],
+  ["mssql", databaseTestProviders.mssql],
+  ["mariadb", databaseTestProviders.mariadb],
 ])("/rows (%s)", (__, dsProvider) => {
   const isInternal = !dsProvider
 
@@ -70,7 +77,7 @@ describe.each([
 
   const generateTableConfig: () => SaveTableRequest = () => {
     return {
-      name: uuid.v4(),
+      name: uuid.v4().substring(0, 16),
       type: "table",
       primary: ["id"],
       primaryDisplay: "name",
@@ -467,7 +474,6 @@ describe.each([
       const createRowResponse = await config.api.row.save(
         createViewResponse.id,
         {
-          OrderID: "1111",
           Country: "Aussy",
           Story: "aaaaa",
         }
@@ -477,7 +483,7 @@ describe.each([
       expect(row.Story).toBeUndefined()
       expect(row).toEqual({
         ...defaultRowFields,
-        OrderID: 1111,
+        OrderID: createRowResponse.OrderID,
         Country: "Aussy",
         _id: createRowResponse._id,
         _rev: createRowResponse._rev,
@@ -641,7 +647,7 @@ describe.each([
       const createdRow = await config.createRow()
 
       const res = await config.api.row.bulkDelete(table._id!, {
-        rows: [createdRow, { _id: "2" }],
+        rows: [createdRow, { _id: "9999999" }],
       })
 
       expect(res[0]._id).toEqual(createdRow._id)

packages/server/src/integrations/base/sql.ts

@@ -4,11 +4,17 @@ import { QueryOptions } from "../../definitions/datasource"
 import { isIsoDateString, SqlClient, isValidFilter } from "../utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import {
+  BBReferenceFieldMetadata,
+  FieldSchema,
+  FieldSubtype,
+  FieldType,
+  JsonFieldMetadata,
   Operation,
   QueryJson,
   RelationshipsJson,
   SearchFilters,
   SortDirection,
+  Table,
 } from "@budibase/types"
 import environment from "../../environment"
 
@@ -691,6 +697,37 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     return results.length ? results : [{ [operation.toLowerCase()]: true }]
   }
 
+  convertJsonStringColumns(
+    table: Table,
+    results: Record<string, any>[]
+  ): Record<string, any>[] {
+    for (const [name, field] of Object.entries(table.schema)) {
+      if (!this._isJsonColumn(field)) {
+        continue
+      }
+      const fullName = `${table.name}.${name}`
+      for (let row of results) {
+        if (typeof row[fullName] === "string") {
+          row[fullName] = JSON.parse(row[fullName])
+        }
+        if (typeof row[name] === "string") {
+          row[name] = JSON.parse(row[name])
+        }
+      }
+    }
+    return results
+  }
+
+  _isJsonColumn(
+    field: FieldSchema
+  ): field is JsonFieldMetadata | BBReferenceFieldMetadata {
+    return (
+      field.type === FieldType.JSON ||
+      (field.type === FieldType.BB_REFERENCE &&
+        field.subtype === FieldSubtype.USERS)
+    )
+  }
+
   log(query: string, values?: any[]) {
     if (!environment.SQL_LOGGING_ENABLE) {
       return

packages/server/src/integrations/microsoftSqlServer.ts

@@ -14,6 +14,8 @@ import {
   Schema,
   TableSourceType,
   DatasourcePlusQueryResponse,
+  FieldType,
+  FieldSubtype,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -502,8 +504,14 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
     }
     const operation = this._operation(json)
     const queryFn = (query: any, op: string) => this.internalQuery(query, op)
-    const processFn = (result: any) =>
-      result.recordset ? result.recordset : [{ [operation]: true }]
+    const processFn = (result: any) => {
+      if (json?.meta?.table && result.recordset) {
+        return this.convertJsonStringColumns(json.meta.table, result.recordset)
+      } else if (result.recordset) {
+        return result.recordset
+      }
+      return [{ [operation]: true }]
+    }
     return this.queryWithReturning(json, queryFn, processFn)
   }
 

packages/server/src/integrations/mysql.ts

@@ -13,6 +13,8 @@ import {
   Schema,
   TableSourceType,
   DatasourcePlusQueryResponse,
+  FieldType,
+  FieldSubtype,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -386,7 +388,13 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     try {
       const queryFn = (query: any) =>
         this.internalQuery(query, { connect: false, disableCoercion: true })
-      return await this.queryWithReturning(json, queryFn)
+      const processFn = (result: any) => {
+        if (json?.meta?.table && Array.isArray(result)) {
+          return this.convertJsonStringColumns(json.meta.table, result)
+        }
+        return result
+      }
+      return await this.queryWithReturning(json, queryFn, processFn)
     } finally {
       await this.disconnect()
     }

packages/server/src/integrations/tests/utils/index.ts

@@ -4,6 +4,8 @@ import { Datasource } from "@budibase/types"
 import * as postgres from "./postgres"
 import * as mongodb from "./mongodb"
 import * as mysql from "./mysql"
+import * as mssql from "./mssql"
+import * as mariadb from "./mariadb"
 import { StartedTestContainer } from "testcontainers"
 
 jest.setTimeout(30000)
@@ -14,4 +16,10 @@ export interface DatabaseProvider {
   datasource(): Promise<Datasource>
 }
 
-export const databaseTestProviders = { postgres, mongodb, mysql }
+export const databaseTestProviders = {
+  postgres,
+  mongodb,
+  mysql,
+  mssql,
+  mariadb,
+}

packages/server/src/integrations/tests/utils/mariadb.ts

@@ -0,0 +1,58 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
+
+let container: StartedTestContainer | undefined
+
+class MariaDBWaitStrategy extends AbstractWaitStrategy {
+  async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
+    // Because MariaDB first starts itself up, runs an init script, then restarts,
+    // it's possible for the mysqladmin ping to succeed early and then tests to
+    // run against a MariaDB that's mid-restart and fail. To get around this, we
+    // wait for logs and then do a ping check.
+
+    const logs = Wait.forLogMessage("mariadbd: ready for connections", 2)
+    await logs.waitUntilReady(container, boundPorts, startTime)
+
+    const command = Wait.forSuccessfulCommand(
+      `mysqladmin ping -h localhost -P 3306 -u root -ppassword`
+    )
+    await command.waitUntilReady(container)
+  }
+}
+
+export async function start(): Promise<StartedTestContainer> {
+  return await new GenericContainer("mariadb:lts")
+    .withExposedPorts(3306)
+    .withEnvironment({ MARIADB_ROOT_PASSWORD: "password" })
+    .withWaitStrategy(new MariaDBWaitStrategy())
+    .start()
+}
+
+export async function datasource(): Promise<Datasource> {
+  if (!container) {
+    container = await start()
+  }
+  const host = container.getHost()
+  const port = container.getMappedPort(3306)
+
+  return {
+    type: "datasource_plus",
+    source: SourceName.MYSQL,
+    plus: true,
+    config: {
+      host,
+      port,
+      user: "root",
+      password: "password",
+      database: "mysql",
+    },
+  }
+}
+
+export async function stop() {
+  if (container) {
+    await container.stop()
+    container = undefined
+  }
+}

packages/server/src/integrations/tests/utils/mssql.ts

@@ -0,0 +1,53 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+
+let container: StartedTestContainer | undefined
+
+export async function start(): Promise<StartedTestContainer> {
+  return await new GenericContainer(
+    "mcr.microsoft.com/mssql/server:2022-latest"
+  )
+    .withExposedPorts(1433)
+    .withEnvironment({
+      ACCEPT_EULA: "Y",
+      MSSQL_SA_PASSWORD: "Password_123",
+      // This is important, as Microsoft allow us to use the "Developer" edition
+      // of SQL Server for development and testing purposes. We can't use other
+      // versions without a valid license, and we cannot use the Developer
+      // version in production.
+      MSSQL_PID: "Developer",
+    })
+    .withWaitStrategy(
+      Wait.forSuccessfulCommand(
+        "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'"
+      )
+    )
+    .start()
+}
+
+export async function datasource(): Promise<Datasource> {
+  if (!container) {
+    container = await start()
+  }
+  const host = container.getHost()
+  const port = container.getMappedPort(1433)
+
+  return {
+    type: "datasource_plus",
+    source: SourceName.SQL_SERVER,
+    plus: true,
+    config: {
+      server: host,
+      port,
+      user: "sa",
+      password: "Password_123",
+    },
+  }
+}
+
+export async function stop() {
+  if (container) {
+    await container.stop()
+    container = undefined
+  }
+}

packages/server/src/sdk/app/rows/search.ts

@@ -1,10 +1,4 @@
-import {
-  Row,
-  SearchFilters,
-  SearchParams,
-  SortOrder,
-  SortType,
-} from "@budibase/types"
+import { Row, SearchFilters, SearchParams, SortOrder } from "@budibase/types"
 import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"