Merge branch 'master' into BUDI-8508/sql-support-for-logical-operators

.github/workflows/budibase_ci.yml

@@ -175,6 +175,7 @@ jobs:
           docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }} &
           docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }} &
           docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }} &
+          docker pull budibase/oracle-database:23.2-slim-faststart &
           docker pull minio/minio &
           docker pull redis &
           docker pull testcontainers/ryuk:0.5.1 &

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.29.27",
+  "version": "2.29.29",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/sql/sqlStatements.ts

@@ -1,87 +0,0 @@
-import { FieldType, Table, FieldSchema, SqlClient } from "@budibase/types"
-import { Knex } from "knex"
-
-export class SqlStatements {
-  client: string
-  table: Table
-  allOr: boolean | undefined
-  columnPrefix: string | undefined
-
-  constructor(
-    client: string,
-    table: Table,
-    { allOr, columnPrefix }: { allOr?: boolean; columnPrefix?: string } = {}
-  ) {
-    this.client = client
-    this.table = table
-    this.allOr = allOr
-    this.columnPrefix = columnPrefix
-  }
-
-  getField(key: string): FieldSchema | undefined {
-    const fieldName = key.split(".")[1]
-    let found = this.table.schema[fieldName]
-    if (!found && this.columnPrefix) {
-      const prefixRemovedFieldName = fieldName.replace(this.columnPrefix, "")
-      found = this.table.schema[prefixRemovedFieldName]
-    }
-    return found
-  }
-
-  between(
-    query: Knex.QueryBuilder,
-    key: string,
-    low: number | string,
-    high: number | string
-  ) {
-    // Use a between operator if we have 2 valid range values
-    const field = this.getField(key)
-    if (
-      field?.type === FieldType.BIGINT &&
-      this.client === SqlClient.SQL_LITE
-    ) {
-      query = query.whereRaw(
-        `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
-        [low, high]
-      )
-    } else {
-      const fnc = this.allOr ? "orWhereBetween" : "whereBetween"
-      query = query[fnc](key, [low, high])
-    }
-    return query
-  }
-
-  lte(query: Knex.QueryBuilder, key: string, low: number | string) {
-    // Use just a single greater than operator if we only have a low
-    const field = this.getField(key)
-    if (
-      field?.type === FieldType.BIGINT &&
-      this.client === SqlClient.SQL_LITE
-    ) {
-      query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
-        low,
-      ])
-    } else {
-      const fnc = this.allOr ? "orWhere" : "where"
-      query = query[fnc](key, ">=", low)
-    }
-    return query
-  }
-
-  gte(query: Knex.QueryBuilder, key: string, high: number | string) {
-    const field = this.getField(key)
-    // Use just a single less than operator if we only have a high
-    if (
-      field?.type === FieldType.BIGINT &&
-      this.client === SqlClient.SQL_LITE
-    ) {
-      query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
-        high,
-      ])
-    } else {
-      const fnc = this.allOr ? "orWhere" : "where"
-      query = query[fnc](key, "<=", high)
-    }
-    return query
-  }
-}

packages/backend-core/src/sql/utils.ts

@@ -22,6 +22,7 @@ export function getNativeSql(
   query: Knex.SchemaBuilder | Knex.QueryBuilder
 ): SqlQuery | SqlQuery[] {
   let sql = query.toSQL()
+
   if (Array.isArray(sql)) {
     return sql as SqlQuery[]
   }

packages/server/__mocks__/oracledb.ts

@@ -1,21 +0,0 @@
-const executeMock = jest.fn(() => ({
-  rows: [
-    {
-      a: "string",
-      b: 1,
-    },
-  ],
-}))
-
-const closeMock = jest.fn()
-
-class Connection {
-  execute = executeMock
-  close = closeMock
-}
-
-module.exports = {
-  getConnection: jest.fn(() => new Connection()),
-  executeMock,
-  closeMock,
-}

packages/server/scripts/integrations/oracle/docker-compose.yml

@@ -6,9 +6,9 @@ services:
   db:
     restart: unless-stopped
     platform: linux/x86_64
-    image: container-registry.oracle.com/database/express:18.4.0-xe
+    image: gvenzl/oracle-free:23.2-slim-faststart
     environment:
-      ORACLE_PWD: oracle
+      ORACLE_PWD: Password1
     ports:
       - 1521:1521
       - 5500:5500

packages/server/src/api/routes/tests/queries/generic-sql.spec.ts

@@ -22,9 +22,13 @@ describe.each(
     DatabaseName.MYSQL,
     DatabaseName.SQL_SERVER,
     DatabaseName.MARIADB,
+    DatabaseName.ORACLE,
   ].map(name => [name, getDatasource(name)])
 )("queries (%s)", (dbName, dsProvider) => {
   const config = setup.getConfig()
+  const isOracle = dbName === DatabaseName.ORACLE
+  const isMsSQL = dbName === DatabaseName.SQL_SERVER
+
   let rawDatasource: Datasource
   let datasource: Datasource
   let client: Knex
@@ -97,7 +101,7 @@ describe.each(
         const query = await createQuery({
           name: "New Query",
           fields: {
-            sql: "SELECT * FROM test_table",
+            sql: client("test_table").select("*").toString(),
           },
         })
 
@@ -106,7 +110,7 @@ describe.each(
           name: "New Query",
           parameters: [],
           fields: {
-            sql: "SELECT * FROM test_table",
+            sql: client("test_table").select("*").toString(),
           },
           schema: {},
           queryVerb: "read",
@@ -125,7 +129,7 @@ describe.each(
       it("should be able to update a query", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table",
+            sql: client("test_table").select("*").toString(),
           },
         })
 
@@ -135,7 +139,7 @@ describe.each(
           ...query,
           name: "Updated Query",
           fields: {
-            sql: "SELECT * FROM test_table WHERE id = 1",
+            sql: client("test_table").where({ id: 1 }).toString(),
           },
         })
 
@@ -144,7 +148,7 @@ describe.each(
           name: "Updated Query",
           parameters: [],
           fields: {
-            sql: "SELECT * FROM test_table WHERE id = 1",
+            sql: client("test_table").where({ id: 1 }).toString(),
           },
           schema: {},
           queryVerb: "read",
@@ -161,7 +165,7 @@ describe.each(
       it("should be able to delete a query", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table",
+            sql: client("test_table").select("*").toString(),
           },
         })
 
@@ -180,7 +184,7 @@ describe.each(
       it("should be able to list queries", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table",
+            sql: client("test_table").select("*").toString(),
           },
         })
 
@@ -191,7 +195,7 @@ describe.each(
       it("should strip sensitive fields for prod apps", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table",
+            sql: client("test_table").select("*").toString(),
           },
         })
 
@@ -212,7 +216,7 @@ describe.each(
         datasourceId: datasource._id!,
         queryVerb: "read",
         fields: {
-          sql: `SELECT * FROM test_table WHERE id = 1`,
+          sql: client("test_table").where({ id: 1 }).toString(),
         },
         parameters: [],
         transformer: "return data",
@@ -270,7 +274,7 @@ describe.each(
         name: "Test Query",
         queryVerb: "read",
         fields: {
-          sql: `SELECT * FROM ${tableName}`,
+          sql: client(tableName).select("*").toString(),
         },
         parameters: [],
         transformer: "return data",
@@ -284,11 +288,13 @@ describe.each(
         })
       )
 
+      await client(tableName).delete()
       await client.schema.alterTable(tableName, table => {
         table.string("data").alter()
       })
 
-      await client(tableName).update({
+      await client(tableName).insert({
+        name: "test",
         data: "string value",
       })
 
@@ -297,7 +303,7 @@ describe.each(
         name: "Test Query",
         queryVerb: "read",
         fields: {
-          sql: `SELECT * FROM ${tableName}`,
+          sql: client(tableName).select("*").toString(),
         },
         parameters: [],
         transformer: "return data",
@@ -311,6 +317,7 @@ describe.each(
         })
       )
     })
+
     it("should work with static variables", async () => {
       await config.api.datasource.update({
         ...datasource,
@@ -326,7 +333,7 @@ describe.each(
         datasourceId: datasource._id!,
         queryVerb: "read",
         fields: {
-          sql: `SELECT '{{ foo }}' as foo`,
+          sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
         },
         parameters: [],
         transformer: "return data",
@@ -337,16 +344,17 @@ describe.each(
 
       const response = await config.api.query.preview(request)
 
+      let key = isOracle ? "FOO" : "foo"
       expect(response.schema).toEqual({
-        foo: {
+        [key]: {
-          name: "foo",
+          name: key,
           type: "string",
         },
       })
 
       expect(response.rows).toEqual([
         {
-          foo: "bar",
+          [key]: "bar",
         },
       ])
     })
@@ -354,7 +362,7 @@ describe.each(
     it("should work with dynamic variables", async () => {
       const basedOnQuery = await createQuery({
         fields: {
-          sql: "SELECT name FROM test_table WHERE id = 1",
+          sql: client("test_table").select("name").where({ id: 1 }).toString(),
         },
       })
 
@@ -376,7 +384,7 @@ describe.each(
         datasourceId: datasource._id!,
         queryVerb: "read",
         fields: {
-          sql: `SELECT '{{ foo }}' as foo`,
+          sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
         },
         parameters: [],
         transformer: "return data",
@@ -385,16 +393,17 @@ describe.each(
         readable: true,
       })
 
+      let key = isOracle ? "FOO" : "foo"
       expect(preview.schema).toEqual({
-        foo: {
+        [key]: {
-          name: "foo",
+          name: key,
           type: "string",
         },
       })
 
       expect(preview.rows).toEqual([
         {
-          foo: "one",
+          [key]: "one",
         },
       ])
     })
@@ -402,7 +411,7 @@ describe.each(
     it("should handle the dynamic base query being deleted", async () => {
       const basedOnQuery = await createQuery({
         fields: {
-          sql: "SELECT name FROM test_table WHERE id = 1",
+          sql: client("test_table").select("name").where({ id: 1 }).toString(),
         },
       })
 
@@ -426,7 +435,7 @@ describe.each(
         datasourceId: datasource._id!,
         queryVerb: "read",
         fields: {
-          sql: `SELECT '{{ foo }}' as foo`,
+          sql: `SELECT '{{ foo }}' AS foo ${isOracle ? "FROM dual" : ""}`,
         },
         parameters: [],
         transformer: "return data",
@@ -435,16 +444,17 @@ describe.each(
         readable: true,
       })
 
+      let key = isOracle ? "FOO" : "foo"
       expect(preview.schema).toEqual({
-        foo: {
+        [key]: {
-          name: "foo",
+          name: key,
           type: "string",
         },
       })
 
       expect(preview.rows).toEqual([
         {
-          foo: datasource.source === SourceName.SQL_SERVER ? "" : null,
+          [key]: datasource.source === SourceName.SQL_SERVER ? "" : null,
         },
       ])
     })
@@ -455,7 +465,7 @@ describe.each(
       it("should be able to insert with bindings", async () => {
         const query = await createQuery({
           fields: {
-            sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
+            sql: client("test_table").insert({ name: "{{ foo }}" }).toString(),
           },
           parameters: [
             {
@@ -488,7 +498,7 @@ describe.each(
       it("should not allow handlebars as parameters", async () => {
         const query = await createQuery({
           fields: {
-            sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
+            sql: client("test_table").insert({ name: "{{ foo }}" }).toString(),
           },
           parameters: [
             {
@@ -516,46 +526,55 @@ describe.each(
         )
       })
 
-      it.each(["2021-02-05T12:01:00.000Z", "2021-02-05"])(
+      // Oracle doesn't automatically coerce strings into dates.
-        "should coerce %s into a date",
+      !isOracle &&
-        async datetimeStr => {
+        it.each(["2021-02-05T12:01:00.000Z", "2021-02-05"])(
-          const date = new Date(datetimeStr)
+          "should coerce %s into a date",
-          const query = await createQuery({
+          async datetimeStr => {
-            fields: {
+            const date = new Date(datetimeStr)
-              sql: `INSERT INTO test_table (name, birthday) VALUES ('foo', {{ birthday }})`,
+            const query = await createQuery({
-            },
+              fields: {
-            parameters: [
+                sql: client("test_table")
-              {
+                  .insert({
-                name: "birthday",
+                    name: "foo",
-                default: "",
+                    birthday: client.raw("{{ birthday }}"),
+                  })
+                  .toString(),
               },
-            ],
+              parameters: [
-            queryVerb: "create",
+                {
-          })
+                  name: "birthday",
+                  default: "",
+                },
+              ],
+              queryVerb: "create",
+            })
 
-          const result = await config.api.query.execute(query._id!, {
+            const result = await config.api.query.execute(query._id!, {
-            parameters: { birthday: datetimeStr },
+              parameters: { birthday: datetimeStr },
-          })
+            })
 
-          expect(result.data).toEqual([{ created: true }])
+            expect(result.data).toEqual([{ created: true }])
 
-          const rows = await client("test_table")
+            const rows = await client("test_table")
-            .where({ birthday: datetimeStr })
+              .where({ birthday: datetimeStr })
-            .select()
+              .select()
-          expect(rows).toHaveLength(1)
+            expect(rows).toHaveLength(1)
 
-          for (const row of rows) {
+            for (const row of rows) {
-            expect(new Date(row.birthday)).toEqual(date)
+              expect(new Date(row.birthday)).toEqual(date)
+            }
           }
-        }
+        )
-      )
 
       it.each(["2021,02,05", "202205-1500"])(
         "should not coerce %s as a date",
         async notDateStr => {
           const query = await createQuery({
             fields: {
-              sql: "INSERT INTO test_table (name) VALUES ({{ name }})",
+              sql: client("test_table")
+                .insert({ name: client.raw("{{ name }}") })
+                .toString(),
             },
             parameters: [
               {
@@ -586,7 +605,7 @@ describe.each(
       it("should execute a query", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table ORDER BY id",
+            sql: client("test_table").select("*").orderBy("id").toString(),
           },
         })
 
@@ -629,7 +648,7 @@ describe.each(
       it("should be able to transform a query", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table WHERE id = 1",
+            sql: client("test_table").where({ id: 1 }).select("*").toString(),
           },
           transformer: `
             data[0].id = data[0].id + 1;
@@ -652,7 +671,10 @@ describe.each(
       it("should coerce numeric bindings", async () => {
         const query = await createQuery({
           fields: {
-            sql: "SELECT * FROM test_table WHERE id = {{ id }}",
+            sql: client("test_table")
+              .where({ id: client.raw("{{ id }}") })
+              .select("*")
+              .toString(),
           },
           parameters: [
             {
@@ -683,7 +705,10 @@ describe.each(
       it("should be able to update rows", async () => {
         const query = await createQuery({
           fields: {
-            sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}",
+            sql: client("test_table")
+              .update({ name: client.raw("{{ name }}") })
+              .where({ id: client.raw("{{ id }}") })
+              .toString(),
           },
           parameters: [
             {
@@ -698,19 +723,13 @@ describe.each(
           queryVerb: "update",
         })
 
-        const result = await config.api.query.execute(query._id!, {
+        await config.api.query.execute(query._id!, {
           parameters: {
             id: "1",
             name: "foo",
           },
         })
 
-        expect(result.data).toEqual([
-          {
-            updated: true,
-          },
-        ])
-
         const rows = await client("test_table").where({ id: 1 }).select()
         expect(rows).toEqual([
           { id: 1, name: "foo", birthday: null, number: null },
@@ -720,35 +739,34 @@ describe.each(
       it("should be able to execute an update that updates no rows", async () => {
         const query = await createQuery({
           fields: {
-            sql: "UPDATE test_table SET name = 'updated' WHERE id = 100",
+            sql: client("test_table")
+              .update({ name: "updated" })
+              .where({ id: 100 })
+              .toString(),
           },
           queryVerb: "update",
         })
 
-        const result = await config.api.query.execute(query._id!)
+        await config.api.query.execute(query._id!)
 
-        expect(result.data).toEqual([
+        const rows = await client("test_table").select()
-          {
+        for (const row of rows) {
-            updated: true,
+          expect(row.name).not.toEqual("updated")
-          },
+        }
-        ])
       })
 
       it("should be able to execute a delete that deletes no rows", async () => {
         const query = await createQuery({
           fields: {
-            sql: "DELETE FROM test_table WHERE id = 100",
+            sql: client("test_table").where({ id: 100 }).delete().toString(),
           },
           queryVerb: "delete",
         })
 
-        const result = await config.api.query.execute(query._id!)
+        await config.api.query.execute(query._id!)
 
-        expect(result.data).toEqual([
+        const rows = await client("test_table").select()
-          {
+        expect(rows).toHaveLength(5)
-            deleted: true,
-          },
-        ])
       })
     })
 
@@ -756,7 +774,10 @@ describe.each(
       it("should be able to delete rows", async () => {
         const query = await createQuery({
           fields: {
-            sql: "DELETE FROM test_table WHERE id = {{ id }}",
+            sql: client("test_table")
+              .where({ id: client.raw("{{ id }}") })
+              .delete()
+              .toString(),
           },
           parameters: [
             {
@@ -767,18 +788,12 @@ describe.each(
           queryVerb: "delete",
         })
 
-        const result = await config.api.query.execute(query._id!, {
+        await config.api.query.execute(query._id!, {
           parameters: {
             id: "1",
           },
         })
 
-        expect(result.data).toEqual([
-          {
-            deleted: true,
-          },
-        ])
-
         const rows = await client("test_table").where({ id: 1 }).select()
         expect(rows).toHaveLength(0)
       })
@@ -823,72 +838,63 @@ describe.each(
       })
     })
 
-    it("should be able to execute an update that updates no rows", async () => {
+    // this parameter really only impacts SQL queries
-      const query = await createQuery({
+    describe("confirm nullDefaultSupport", () => {
-        fields: {
+      let queryParams: Partial<Query>
-          sql: "UPDATE test_table SET name = 'updated' WHERE id = 100",
+      beforeAll(async () => {
-        },
+        queryParams = {
-        queryVerb: "update",
+          fields: {
+            sql: client("test_table")
+              .insert({
+                name: client.raw("{{ bindingName }}"),
+                number: client.raw("{{ bindingNumber }}"),
+              })
+              .toString(),
+          },
+          parameters: [
+            {
+              name: "bindingName",
+              default: "",
+            },
+            {
+              name: "bindingNumber",
+              default: "",
+            },
+          ],
+          queryVerb: "create",
+        }
       })
 
-      const result = await config.api.query.execute(query._id!, {})
+      it("should error for old queries", async () => {
+        const query = await createQuery(queryParams)
+        await config.api.query.save({ ...query, nullDefaultSupport: false })
+        let error: string | undefined
+        try {
+          await config.api.query.execute(query._id!, {
+            parameters: {
+              bindingName: "testing",
+            },
+          })
+        } catch (err: any) {
+          error = err.message
+        }
+        if (isMsSQL || isOracle) {
+          expect(error).toBeUndefined()
+        } else {
+          expect(error).toBeDefined()
+          expect(error).toContain("integer")
+        }
+      })
 
-      expect(result.data).toEqual([
+      it("should not error for new queries", async () => {
-        {
+        const query = await createQuery(queryParams)
-          updated: true,
+        const results = await config.api.query.execute(query._id!, {
-        },
-      ])
-    })
-  })
-
-  // this parameter really only impacts SQL queries
-  describe("confirm nullDefaultSupport", () => {
-    const queryParams = {
-      fields: {
-        sql: "INSERT INTO test_table (name, number) VALUES ({{ bindingName }}, {{ bindingNumber }})",
-      },
-      parameters: [
-        {
-          name: "bindingName",
-          default: "",
-        },
-        {
-          name: "bindingNumber",
-          default: "",
-        },
-      ],
-      queryVerb: "create",
-    }
-
-    it("should error for old queries", async () => {
-      const query = await createQuery(queryParams)
-      await config.api.query.save({ ...query, nullDefaultSupport: false })
-      let error: string | undefined
-      try {
-        await config.api.query.execute(query._id!, {
           parameters: {
             bindingName: "testing",
           },
         })
-      } catch (err: any) {
+        expect(results).toEqual({ data: [{ created: true }] })
-        error = err.message
-      }
-      if (dbName === "mssql") {
-        expect(error).toBeUndefined()
-      } else {
-        expect(error).toBeDefined()
-        expect(error).toContain("integer")
-      }
-    })
-
-    it("should not error for new queries", async () => {
-      const query = await createQuery(queryParams)
-      const results = await config.api.query.execute(query._id!, {
-        parameters: {
-          bindingName: "testing",
-        },
       })
-      expect(results).toEqual({ data: [{ created: true }] })
     })
   })
 })

packages/server/src/api/routes/tests/row.spec.ts

@@ -72,9 +72,11 @@ describe.each([
   [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
   [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
   [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("/rows (%s)", (providerType, dsProvider) => {
   const isInternal = dsProvider === undefined
   const isMSSQL = providerType === DatabaseName.SQL_SERVER
+  const isOracle = providerType === DatabaseName.ORACLE
   const config = setup.getConfig()
 
   let table: Table
@@ -129,7 +131,8 @@ describe.each([
       primary: ["id"],
       schema: defaultSchema,
     }
-    return merge(req, ...overrides)
+    const merged = merge(req, ...overrides)
+    return merged
   }
 
   function defaultTable(
@@ -1406,9 +1409,10 @@ describe.each([
         await assertRowUsage(rowUsage + 3)
       })
 
-    // Upserting isn't yet supported in MSSQL, see:
+    // Upserting isn't yet supported in MSSQL / Oracle, see:
     //   https://github.com/knex/knex/pull/6050
     !isMSSQL &&
+      !isOracle &&
       it("should be able to update existing rows with bulkImport", async () => {
         const table = await config.api.table.save(
           saveTableRequest({
@@ -1478,9 +1482,10 @@ describe.each([
         expect(rows[2].description).toEqual("Row 3 description")
       })
 
-    // Upserting isn't yet supported in MSSQL, see:
+    // Upserting isn't yet supported in MSSQL or Oracle, see:
     //   https://github.com/knex/knex/pull/6050
     !isMSSQL &&
+      !isOracle &&
       !isInternal &&
       it("should be able to update existing rows with composite primary keys with bulkImport", async () => {
         const tableName = uuid.v4()
@@ -1547,9 +1552,10 @@ describe.each([
         expect(rows[2].description).toEqual("Row 3 description")
       })
 
-    // Upserting isn't yet supported in MSSQL, see:
+    // Upserting isn't yet supported in MSSQL/Oracle, see:
     //   https://github.com/knex/knex/pull/6050
     !isMSSQL &&
+      !isOracle &&
       !isInternal &&
       it("should be able to update existing rows an autoID primary key", async () => {
         const tableName = uuid.v4()

packages/server/src/api/routes/tests/search.spec.ts

@@ -48,6 +48,7 @@ describe.each([
   [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
   [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
   [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("search (%s)", (name, dsProvider) => {
   const isSqs = name === "sqs"
   const isLucene = name === "lucene"
@@ -1594,7 +1595,10 @@ describe.each([
     const MEDIUM = "10000000"
 
     // Our bigints are int64s in most datasources.
-    const BIG = "9223372036854775807"
+    let BIG = "9223372036854775807"
+    if (name === DatabaseName.ORACLE) {
+      // BIG = "9223372036854775808"
+    }
 
     beforeAll(async () => {
       table = await createTable({

packages/server/src/api/routes/tests/table.spec.ts

@@ -40,7 +40,8 @@ describe.each([
   [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
   [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
   [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-])("/tables (%s)", (_, dsProvider) => {
+  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
+])("/tables (%s)", (name, dsProvider) => {
   const isInternal: boolean = !dsProvider
   let datasource: Datasource | undefined
   let config = setup.getConfig()
@@ -59,15 +60,20 @@ describe.each([
       jest.clearAllMocks()
     })
 
-    it.each([
+    let names = [
       "alphanum",
       "with spaces",
       "with-dashes",
       "with_underscores",
-      'with "double quotes"',
-      "with 'single quotes'",
       "with `backticks`",
-    ])("creates a table with name: %s", async name => {
+    ]
+
+    if (name !== DatabaseName.ORACLE) {
+      names.push(`with "double quotes"`)
+      names.push(`with 'single quotes'`)
+    }
+
+    it.each(names)("creates a table with name: %s", async name => {
       const table = await config.api.table.save(
         tableForDatasource(datasource, { name })
       )

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -33,6 +33,7 @@ describe.each([
   [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
   [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
   [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("/v2/views (%s)", (name, dsProvider) => {
   const config = setup.getConfig()
   const isSqs = name === "sqs"

packages/server/src/automations/tests/executeQuery.spec.ts

@@ -1,27 +1,20 @@
-import { Datasource, Query, SourceName } from "@budibase/types"
+import { Datasource, Query } from "@budibase/types"
 import * as setup from "./utilities"
-import { DatabaseName, getDatasource } from "../../integrations/tests/utils"
+import {
-import knex, { Knex } from "knex"
+  DatabaseName,
+  getDatasource,
+  knexClient,
+} from "../../integrations/tests/utils"
+import { Knex } from "knex"
 import { generator } from "@budibase/backend-core/tests"
 
-function getKnexClientName(source: SourceName) {
-  switch (source) {
-    case SourceName.MYSQL:
-      return "mysql2"
-    case SourceName.SQL_SERVER:
-      return "mssql"
-    case SourceName.POSTGRES:
-      return "pg"
-  }
-  throw new Error(`Unsupported source: ${source}`)
-}
-
 describe.each(
   [
     DatabaseName.POSTGRES,
     DatabaseName.MYSQL,
     DatabaseName.SQL_SERVER,
     DatabaseName.MARIADB,
+    DatabaseName.ORACLE,
   ].map(name => [name, getDatasource(name)])
 )("execute query action (%s)", (_, dsProvider) => {
   let tableName: string
@@ -35,10 +28,7 @@ describe.each(
 
     const ds = await dsProvider
     datasource = await config.api.datasource.create(ds)
-    client = knex({
+    client = await knexClient(ds)
-      client: getKnexClientName(ds.source),
-      connection: ds.config,
-    })
   })
 
   beforeEach(async () => {

packages/server/src/integrations/base/types.ts

@@ -104,6 +104,37 @@ export interface OracleColumnsResponse {
   SEARCH_CONDITION: null | string
 }
 
+export enum TriggeringEvent {
+  INSERT = "INSERT",
+  DELETE = "DELETE",
+  UPDATE = "UPDATE",
+  LOGON = "LOGON",
+  LOGOFF = "LOGOFF",
+  STARTUP = "STARTUP",
+  SHUTDOWN = "SHUTDOWN",
+  SERVERERROR = "SERVERERROR",
+  SCHEMA = "SCHEMA",
+  ALTER = "ALTER",
+  DROP = "DROP",
+}
+
+export enum TriggerType {
+  BEFORE_EACH_ROW = "BEFORE EACH ROW",
+  AFTER_EACH_ROW = "AFTER EACH ROW",
+  BEFORE_STATEMENT = "BEFORE STATEMENT",
+  AFTER_STATEMENT = "AFTER STATEMENT",
+  INSTEAD_OF = "INSTEAD OF",
+  COMPOUND = "COMPOUND",
+}
+
+export interface OracleTriggersResponse {
+  TABLE_NAME: string
+  TRIGGER_NAME: string
+  TRIGGER_TYPE: TriggerType
+  TRIGGERING_EVENT: TriggeringEvent
+  TRIGGER_BODY: string
+}
+
 /**
  * An oracle constraint
  */

packages/server/src/integrations/oracle.ts

@@ -31,7 +31,14 @@ import oracledb, {
   ExecuteOptions,
   Result,
 } from "oracledb"
-import { OracleTable, OracleColumn, OracleColumnsResponse } from "./base/types"
+import {
+  OracleTable,
+  OracleColumn,
+  OracleColumnsResponse,
+  OracleTriggersResponse,
+  TriggeringEvent,
+  TriggerType,
+} from "./base/types"
 import { sql } from "@budibase/backend-core"
 
 const Sql = sql.Sql
@@ -98,7 +105,7 @@ const SCHEMA: Integration = {
   },
 }
 
-const UNSUPPORTED_TYPES = ["BLOB", "CLOB", "NCLOB"]
+const UNSUPPORTED_TYPES = ["BLOB", "NCLOB"]
 
 const OracleContraintTypes = {
   PRIMARY: "P",
@@ -111,7 +118,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
   private readonly config: OracleConfig
   private index: number = 1
 
-  private readonly COLUMNS_SQL = `
+  private static readonly COLUMNS_SQL = `
     SELECT
       tabs.table_name,
       cols.column_name,
@@ -139,6 +146,19 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       (cons.status = 'ENABLED'
         OR cons.status IS NULL)
   `
+
+  private static readonly TRIGGERS_SQL = `
+    SELECT 
+      table_name, 
+      trigger_name, 
+      trigger_type, 
+      triggering_event, 
+      trigger_body 
+    FROM 
+      all_triggers 
+    WHERE status = 'ENABLED'
+  `
+
   constructor(config: OracleConfig) {
     super(SqlClient.ORACLE)
     this.config = config
@@ -211,6 +231,75 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     return oracleTables
   }
 
+  private getTriggersFor(
+    tableName: string,
+    triggersResponse: Result<OracleTriggersResponse>,
+    opts?: { event?: TriggeringEvent; type?: TriggerType }
+  ): OracleTriggersResponse[] {
+    const triggers: OracleTriggersResponse[] = []
+    for (const trigger of triggersResponse.rows || []) {
+      if (trigger.TABLE_NAME !== tableName) {
+        continue
+      }
+      if (opts?.event && opts.event !== trigger.TRIGGERING_EVENT) {
+        continue
+      }
+      if (opts?.type && opts.type !== trigger.TRIGGER_TYPE) {
+        continue
+      }
+      triggers.push(trigger)
+    }
+    return triggers
+  }
+
+  private markAutoIncrementColumns(
+    triggersResponse: Result<OracleTriggersResponse>,
+    tables: Record<string, Table>
+  ) {
+    for (const table of Object.values(tables)) {
+      const triggers = this.getTriggersFor(table.name, triggersResponse, {
+        type: TriggerType.BEFORE_EACH_ROW,
+        event: TriggeringEvent.INSERT,
+      })
+
+      // This is the trigger body Knex generates for an auto increment column
+      // called "id" on a table called "foo":
+      //
+      //   declare checking number := 1;
+      //   begin if (:new. "id" is null) then while checking >= 1 loop
+      //   select
+      //       "foo_seq".nextval into :new. "id"
+      //   from
+      //       dual;
+      //   select
+      //       count("id") into checking
+      //   from
+      //       "foo"
+      //   where
+      //       "id" = :new. "id";
+      //   end loop;
+      //   end if;
+      //   end;
+      for (const [columnName, schema] of Object.entries(table.schema)) {
+        const autoIncrementTriggers = triggers.filter(
+          trigger =>
+            // This is a bit heuristic, but I think it's the best we can do with
+            // the information we have. We're looking for triggers that run
+            // before each row is inserted, and that have a body that contains a
+            // call to a function that generates a new value for the column.  We
+            // also check that the column name is in the trigger body, to make
+            // sure we're not picking up triggers that don't affect the column.
+            trigger.TRIGGER_BODY.includes(`"${columnName}"`) &&
+            trigger.TRIGGER_BODY.includes(`.nextval`)
+        )
+
+        if (autoIncrementTriggers.length > 0) {
+          schema.autocolumn = true
+        }
+      }
+    }
+  }
+
   private static isSupportedColumn(column: OracleColumn) {
     return !UNSUPPORTED_TYPES.includes(column.type)
   }
@@ -255,7 +344,10 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     entities: Record<string, Table>
   ): Promise<Schema> {
     const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
-      sql: this.COLUMNS_SQL,
+      sql: OracleIntegration.COLUMNS_SQL,
+    })
+    const triggersResponse = await this.internalQuery<OracleTriggersResponse>({
+      sql: OracleIntegration.TRIGGERS_SQL,
     })
     const oracleTables = this.mapColumns(columnsResponse)
 
@@ -318,6 +410,8 @@ class OracleIntegration extends Sql implements DatasourcePlus {
         })
     })
 
+    this.markAutoIncrementColumns(triggersResponse, tables)
+
     let externalTables = finaliseExternalTables(tables, entities)
     let errors = checkExternalTables(externalTables)
     return { tables: externalTables, errors }
@@ -325,7 +419,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
 
   async getTableNames() {
     const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
-      sql: this.COLUMNS_SQL,
+      sql: OracleIntegration.COLUMNS_SQL,
     })
     return (columnsResponse.rows || []).map(row => row.TABLE_NAME)
   }
@@ -360,11 +454,32 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       this.index = 1
       connection = await this.getConnection()
 
-      const options: ExecuteOptions = { autoCommit: true }
+      const options: ExecuteOptions = {
+        autoCommit: true,
+        fetchTypeHandler: function (metaData) {
+          if (metaData.dbType === oracledb.CLOB) {
+            return { type: oracledb.STRING }
+          } else if (
+            // When we create a new table in OracleDB from Budibase, bigints get
+            // created as NUMBER(20,0). Budibase expects bigints to be returned
+            // as strings, which is what we're doing here. However, this is
+            // likely to be brittle if we connect to externally created
+            // databases that have used different precisions and scales.
+            // We shold find a way to do better.
+            metaData.dbType === oracledb.NUMBER &&
+            metaData.precision === 20 &&
+            metaData.scale === 0
+          ) {
+            return { type: oracledb.STRING }
+          }
+          return undefined
+        },
+      }
       const bindings: BindParameters = query.bindings || []
 
       this.log(query.sql, bindings)
-      return await connection.execute<T>(query.sql, bindings, options)
+      const result = await connection.execute(query.sql, bindings, options)
+      return result as Result<T>
     } finally {
       if (connection) {
         try {
@@ -377,7 +492,6 @@ class OracleIntegration extends Sql implements DatasourcePlus {
   }
 
   private getConnection = async (): Promise<Connection> => {
-    //connectString : "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = localhost)(PORT = 1521))(CONNECT_DATA =(SID= ORCL)))"
     const connectString = `${this.config.host}:${this.config.port || 1521}/${
       this.config.database
     }`
@@ -386,7 +500,10 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       password: this.config.password,
       connectString,
     }
-    return oracledb.getConnection(attributes)
+    const tz = Intl.DateTimeFormat().resolvedOptions().timeZone
+    const connection = await oracledb.getConnection(attributes)
+    await connection.execute(`ALTER SESSION SET TIME_ZONE = '${tz}'`)
+    return connection
   }
 
   async create(query: SqlQuery | string): Promise<any[]> {

packages/server/src/integrations/tests/oracle.spec.ts

@@ -1,100 +0,0 @@
-const oracledb = require("oracledb")
-
-import { default as OracleIntegration } from "../oracle"
-
-jest.mock("oracledb")
-
-class TestConfiguration {
-  integration: any
-
-  constructor(config: any = {}) {
-    this.integration = new OracleIntegration.integration(config)
-  }
-}
-
-const options = { autoCommit: true }
-
-describe("Oracle Integration", () => {
-  let config: any
-
-  beforeEach(() => {
-    jest.clearAllMocks()
-    config = new TestConfiguration()
-  })
-
-  it("calls the create method with the correct params", async () => {
-    const sql = "insert into users (name, age) values ('Joe', 123);"
-    await config.integration.create({
-      sql,
-    })
-    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
-    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-    expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-  })
-
-  it("calls the read method with the correct params", async () => {
-    const sql = "select * from users;"
-    await config.integration.read({
-      sql,
-    })
-    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
-    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-    expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-  })
-
-  it("calls the update method with the correct params", async () => {
-    const sql = "update table users set name = 'test';"
-    await config.integration.update({
-      sql,
-    })
-    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
-    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-    expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-  })
-
-  it("calls the delete method with the correct params", async () => {
-    const sql = "delete from users where name = 'todelete';"
-    await config.integration.delete({
-      sql,
-    })
-    expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options)
-    expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-    expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-  })
-
-  describe("no rows returned", () => {
-    beforeEach(() => {
-      oracledb.executeMock.mockImplementation(() => ({ rows: [] }))
-    })
-
-    it("returns the correct response when the create response has no rows", async () => {
-      const sql = "insert into users (name, age) values ('Joe', 123);"
-      const response = await config.integration.create({
-        sql,
-      })
-      expect(response).toEqual([{ created: true }])
-      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-      expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-    })
-
-    it("returns the correct response when the update response has no rows", async () => {
-      const sql = "update table users set name = 'test';"
-      const response = await config.integration.update({
-        sql,
-      })
-      expect(response).toEqual([{ updated: true }])
-      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-      expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-    })
-
-    it("returns the correct response when the delete response has no rows", async () => {
-      const sql = "delete from users where name = 'todelete';"
-      const response = await config.integration.delete({
-        sql,
-      })
-      expect(response).toEqual([{ deleted: true }])
-      expect(oracledb.executeMock).toHaveBeenCalledTimes(1)
-      expect(oracledb.closeMock).toHaveBeenCalledTimes(1)
-    })
-  })
-})

packages/server/src/integrations/tests/sql.spec.ts

@@ -1,12 +1,16 @@
 import {
   FieldType,
   Operation,
+  PaginationJson,
   QueryJson,
+  SearchFilters,
+  SortJson,
+  SqlClient,
   Table,
   TableSourceType,
-  SqlClient,
 } from "@budibase/types"
 import { sql } from "@budibase/backend-core"
+import { merge } from "lodash"
 
 const Sql = sql.Sql
 
@@ -25,7 +29,16 @@ const TABLE: Table = {
   primary: ["id"],
 }
 
-function endpoint(table: any, operation: any) {
+const ORACLE_TABLE: Partial<Table> = {
+  schema: {
+    name: {
+      name: "name",
+      type: FieldType.STRING,
+    },
+  },
+}
+
+function endpoint(table: string, operation: Operation) {
   return {
     datasourceId: "Postgres",
     operation: operation,
@@ -39,19 +52,25 @@ function generateReadJson({
   filters,
   sort,
   paginate,
-}: any = {}): QueryJson {
+}: {
-  const tableObj = { ...TABLE }
+  table?: Partial<Table>
+  fields?: string[]
+  filters?: SearchFilters
+  sort?: SortJson
+  paginate?: PaginationJson
+} = {}): QueryJson {
+  let tableObj: Table = { ...TABLE }
   if (table) {
-    tableObj.name = table
+    tableObj = merge(TABLE, table)
   }
   return {
-    endpoint: endpoint(table || TABLE_NAME, "READ"),
+    endpoint: endpoint(tableObj.name || TABLE_NAME, Operation.READ),
     resource: {
       fields: fields || [],
     },
     filters: filters || {},
     sort: sort || {},
-    paginate: paginate || {},
+    paginate: paginate || undefined,
     meta: {
       table: tableObj,
     },
@@ -191,7 +210,7 @@ describe("SQL query builder", () => {
     )
     expect(query).toEqual({
       bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit, 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where (COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2) and (COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4) order by "test"."id" asc) where rownum <= :5) "test" order by "test"."id" asc) where rownum <= :6`,
+      sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" order by "test"."id" asc) where rownum <= :6`,
     })
 
     query = new Sql(SqlClient.ORACLE, limit)._query(
@@ -212,6 +231,7 @@ describe("SQL query builder", () => {
   it("should use an oracle compatible coalesce query for oracle when using the equals filter", () => {
     let query = new Sql(SqlClient.ORACLE, limit)._query(
       generateReadJson({
+        table: ORACLE_TABLE,
         filters: {
           equal: {
             name: "John",
@@ -222,13 +242,14 @@ describe("SQL query builder", () => {
 
     expect(query).toEqual({
       bindings: ["John", limit, 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) = :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
+      sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
     })
   })
 
   it("should use an oracle compatible coalesce query for oracle when using the not equals filter", () => {
     let query = new Sql(SqlClient.ORACLE, limit)._query(
       generateReadJson({
+        table: ORACLE_TABLE,
         filters: {
           notEqual: {
             name: "John",
@@ -239,7 +260,7 @@ describe("SQL query builder", () => {
 
     expect(query).toEqual({
       bindings: ["John", limit, 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where COALESCE("test"."name", -1) != :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
+      sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
     })
   })
 })

packages/server/src/integrations/tests/utils/index.ts

@@ -5,6 +5,7 @@ import * as mongodb from "./mongodb"
 import * as mysql from "./mysql"
 import * as mssql from "./mssql"
 import * as mariadb from "./mariadb"
+import * as oracle from "./oracle"
 import { GenericContainer, StartedTestContainer } from "testcontainers"
 import { testContainerUtils } from "@budibase/backend-core/tests"
 import cloneDeep from "lodash/cloneDeep"
@@ -17,6 +18,7 @@ export enum DatabaseName {
   MYSQL = "mysql",
   SQL_SERVER = "mssql",
   MARIADB = "mariadb",
+  ORACLE = "oracle",
 }
 
 const providers: Record<DatabaseName, DatasourceProvider> = {
@@ -25,6 +27,7 @@ const providers: Record<DatabaseName, DatasourceProvider> = {
   [DatabaseName.MYSQL]: mysql.getDatasource,
   [DatabaseName.SQL_SERVER]: mssql.getDatasource,
   [DatabaseName.MARIADB]: mariadb.getDatasource,
+  [DatabaseName.ORACLE]: oracle.getDatasource,
 }
 
 export function getDatasourceProviders(
@@ -60,6 +63,9 @@ export async function knexClient(ds: Datasource) {
     case SourceName.SQL_SERVER: {
       return mssql.knexClient(ds)
     }
+    case SourceName.ORACLE: {
+      return oracle.knexClient(ds)
+    }
     default: {
       throw new Error(`Unsupported source: ${ds.source}`)
     }

packages/server/src/integrations/tests/utils/oracle.ts

@@ -0,0 +1,78 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait } from "testcontainers"
+import { generator, testContainerUtils } from "@budibase/backend-core/tests"
+import { startContainer } from "."
+import knex from "knex"
+
+let ports: Promise<testContainerUtils.Port[]>
+
+export async function getDatasource(): Promise<Datasource> {
+  // password needs to conform to Oracle standards
+  const password = "password"
+  if (!ports) {
+    // couldn't build 19.3.0 for X64
+    let image = "budibase/oracle-database:23.2-slim-faststart"
+    if (process.arch.startsWith("arm")) {
+      // there isn't an ARM compatible 23.2 build
+      image = "budibase/oracle-database:19.3.0-ee-slim-faststart"
+    }
+
+    ports = startContainer(
+      new GenericContainer(image)
+        .withExposedPorts(1521)
+        .withEnvironment({
+          ORACLE_PASSWORD: password,
+        })
+        .withWaitStrategy(Wait.forLogMessage("DATABASE IS READY TO USE!"))
+    )
+  }
+
+  const port = (await ports).find(x => x.container === 1521)?.host
+  if (!port) {
+    throw new Error("Oracle port not found")
+  }
+
+  const host = "127.0.0.1"
+  const user = "SYSTEM"
+
+  const datasource: Datasource = {
+    type: "datasource_plus",
+    source: SourceName.ORACLE,
+    plus: true,
+    config: { host, port, user, password, database: "FREEPDB1" },
+  }
+
+  const newUser = "a" + generator.guid().replaceAll("-", "")
+  const client = await knexClient(datasource)
+  await client.raw(`CREATE USER ${newUser} IDENTIFIED BY password`)
+  await client.raw(
+    `GRANT CONNECT, RESOURCE, CREATE VIEW, CREATE SESSION TO ${newUser}`
+  )
+  await client.raw(`GRANT UNLIMITED TABLESPACE TO ${newUser}`)
+  datasource.config!.user = newUser
+
+  return datasource
+}
+
+export async function knexClient(ds: Datasource) {
+  if (!ds.config) {
+    throw new Error("Datasource config is missing")
+  }
+  if (ds.source !== SourceName.ORACLE) {
+    throw new Error("Datasource source is not Oracle")
+  }
+
+  const db = ds.config.database || "FREEPDB1"
+  const connectString = `${ds.config.host}:${ds.config.port}/${db}`
+
+  const c = knex({
+    client: "oracledb",
+    connection: {
+      connectString,
+      user: ds.config.user,
+      password: ds.config.password,
+    },
+  })
+
+  return c
+}

packages/server/src/sdk/app/rows/search/external.ts

@@ -147,7 +147,8 @@ export async function search(
   } catch (err: any) {
     if (err.message && err.message.includes("does not exist")) {
       throw new Error(
-        `Table updated externally, please re-fetch - ${err.message}`
+        `Table updated externally, please re-fetch - ${err.message}`,
+        { cause: err }
       )
     } else {
       throw err

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -20,12 +20,11 @@ import {
   buildInternalRelationships,
   sqlOutputProcessing,
 } from "../../../../../api/controllers/row/utils"
+import sdk from "../../../../index"
 import {
-  decodeNonAscii,
   mapToUserColumn,
   USER_COLUMN_PREFIX,
 } from "../../../tables/internal/sqs"
-import sdk from "../../../../index"
 import {
   context,
   sql,
@@ -42,7 +41,11 @@ import {
   getRelationshipColumns,
   getTableIDList,
 } from "../filters"
-import { dataFilters, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
+import {
+  dataFilters,
+  helpers,
+  PROTECTED_INTERNAL_COLUMNS,
+} from "@budibase/shared-core"
 import { isSearchingByRowID } from "../utils"
 import tracer from "dd-trace"
 
@@ -191,7 +194,7 @@ function reverseUserColumnMapping(rows: Row[]) {
       if (index !== -1) {
         // cut out the prefix
         const newKey = key.slice(0, index) + key.slice(index + prefixLength)
-        const decoded = decodeNonAscii(newKey)
+        const decoded = helpers.schema.decodeNonAscii(newKey)
         finalRow[decoded] = row[key]
       } else {
         finalRow[key] = row[key]

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -13,7 +13,7 @@ import tablesSdk from "../"
 import { generateJunctionTableID } from "../../../../db/utils"
 import { isEqual } from "lodash"
 import { DEFAULT_TABLES } from "../../../../db/defaultData/datasource_bb_default"
-import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
+import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
 
 const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.BOOLEAN]: SQLiteType.NUMERIC,
@@ -63,29 +63,10 @@ function buildRelationshipDefinitions(
 
 export const USER_COLUMN_PREFIX = "data_"
 
-// SQS does not support non-ASCII characters in column names, so we need to
-// replace them with unicode escape sequences.
-function encodeNonAscii(str: string): string {
-  return str
-    .split("")
-    .map(char => {
-      return char.charCodeAt(0) > 127
-        ? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")
-        : char
-    })
-    .join("")
-}
-
-export function decodeNonAscii(str: string): string {
-  return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) =>
-    String.fromCharCode(parseInt(p1, 16))
-  )
-}
-
 // utility function to denote that columns in SQLite are mapped to avoid overlap issues
 // the overlaps can occur due to case insensitivity and some of the columns which Budibase requires
 export function mapToUserColumn(key: string) {
-  return `${USER_COLUMN_PREFIX}${encodeNonAscii(key)}`
+  return `${USER_COLUMN_PREFIX}${helpers.schema.encodeNonAscii(key)}`
 }
 
 // this can generate relationship tables as part of the mapping

packages/server/src/utilities/rowProcessor/index.ts

@@ -315,6 +315,21 @@ export async function outputProcessing<T extends Row[] | Row>(
           column.subtype
         )
       }
+    } else if (column.type === FieldType.DATETIME && column.timeOnly) {
+      for (let row of enriched) {
+        if (row[property] instanceof Date) {
+          const hours = row[property].getUTCHours().toString().padStart(2, "0")
+          const minutes = row[property]
+            .getUTCMinutes()
+            .toString()
+            .padStart(2, "0")
+          const seconds = row[property]
+            .getUTCSeconds()
+            .toString()
+            .padStart(2, "0")
+          row[property] = `${hours}:${minutes}:${seconds}`
+        }
+      }
     }
   }
 

packages/shared-core/src/filters.ts

@@ -22,6 +22,7 @@ import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
 import { deepGet, schema } from "./helpers"
 import { isPlainObject, isEmpty } from "lodash"
+import { decodeNonAscii } from "./helpers/schema"
 
 const HBS_REGEX = /{{([^{].*?)}}/g
 
@@ -181,8 +182,16 @@ export class ColumnSplitter {
   tableIds: string[]
   relationshipColumnNames: string[]
   relationships: string[]
+  aliases?: Record<string, string>
+  columnPrefix?: string
 
-  constructor(tables: Table[]) {
+  constructor(
+    tables: Table[],
+    opts?: {
+      aliases?: Record<string, string>
+      columnPrefix?: string
+    }
+  ) {
     this.tableNames = tables.map(table => table.name)
     this.tableIds = tables.map(table => table._id!)
     this.relationshipColumnNames = tables.flatMap(table =>
@@ -195,16 +204,38 @@ export class ColumnSplitter {
       .concat(this.relationshipColumnNames)
       // sort by length - makes sure there's no mis-matches due to similarities (sub column names)
       .sort((a, b) => b.length - a.length)
+
+    if (opts?.aliases) {
+      this.aliases = {}
+      for (const [key, value] of Object.entries(opts.aliases)) {
+        this.aliases[value] = key
+      }
+    }
+
+    this.columnPrefix = opts?.columnPrefix
   }
 
   run(key: string): {
     numberPrefix?: string
     relationshipPrefix?: string
+    tableName?: string
     column: string
   } {
     let { prefix, key: splitKey } = getKeyNumbering(key)
+
+    let tableName: string | undefined = undefined
+    if (this.aliases) {
+      for (const possibleAlias of Object.keys(this.aliases || {})) {
+        const withDot = `${possibleAlias}.`
+        if (splitKey.startsWith(withDot)) {
+          tableName = this.aliases[possibleAlias]!
+          splitKey = splitKey.slice(withDot.length)
+        }
+      }
+    }
+
     let relationship: string | undefined
-    for (let possibleRelationship of this.relationships) {
+    for (const possibleRelationship of this.relationships) {
       const withDot = `${possibleRelationship}.`
       if (splitKey.startsWith(withDot)) {
         const finalKeyParts = splitKey.split(withDot)
@@ -214,7 +245,15 @@ export class ColumnSplitter {
         break
       }
     }
+
+    if (this.columnPrefix) {
+      if (splitKey.startsWith(this.columnPrefix)) {
+        splitKey = decodeNonAscii(splitKey.slice(this.columnPrefix.length))
+      }
+    }
+
     return {
+      tableName,
       numberPrefix: prefix,
       relationshipPrefix: relationship,
       column: splitKey,

packages/shared-core/src/helpers/schema.ts

@@ -26,3 +26,22 @@ export function isRequired(constraints: FieldConstraints | undefined) {
       constraints.presence === true)
   return isRequired
 }
+
+// SQS does not support non-ASCII characters in column names, so we need to
+// replace them with unicode escape sequences.
+export function encodeNonAscii(str: string): string {
+  return str
+    .split("")
+    .map(char => {
+      return char.charCodeAt(0) > 127
+        ? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")
+        : char
+    })
+    .join("")
+}
+
+export function decodeNonAscii(str: string): string {
+  return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) =>
+    String.fromCharCode(parseInt(p1, 16))
+  )
+}

packages/worker/src/sdk/tenants/tenants.ts

@@ -48,10 +48,13 @@ async function removeTenantUsers(tenantId: string) {
   try {
     const allUsers = await getTenantUsers(tenantId)
     const allEmails = allUsers.rows.map((row: any) => row.doc.email)
+    const allSsoIds = allUsers.rows
+      .map((row: any) => row.doc.ssoId)
+      .filter(id => !!id)
 
     // get the id and email doc ids
     let keys = allUsers.rows.map((row: any) => row.id)
-    keys = keys.concat(allEmails)
+    keys = keys.concat(allEmails).concat(allSsoIds)
 
     const platformDb = platform.getPlatformDB()