Merge pull request #13714 from Budibase/budi-8195/unable-to-load-grid-when-using-external-postgres

Unable to load grid when using external postgres

packages/bbui/src/Form/Core/DatePicker/NumberInput.svelte

@@ -4,13 +4,14 @@
   export let max
   export let hideArrows = false
   export let width
+  export let type = "number"
 
   $: style = width ? `width:${width}px;` : ""
 </script>
 
 <input
   class:hide-arrows={hideArrows}
-  type="number"
+  {type}
   {style}
   {value}
   {min}
@@ -51,4 +52,7 @@
   input.hide-arrows {
     -moz-appearance: textfield;
   }
+  input[type="time"]::-webkit-calendar-picker-indicator {
+    display: none;
+  }
 </style>

packages/bbui/src/Form/Core/DatePicker/TimePicker.svelte

@@ -1,5 +1,4 @@
 <script>
-  import { cleanInput } from "./utils"
   import dayjs from "dayjs"
   import NumberInput from "./NumberInput.svelte"
   import { createEventDispatcher } from "svelte"
@@ -8,39 +7,26 @@
 
   const dispatch = createEventDispatcher()
 
-  $: displayValue = value || dayjs()
+  $: displayValue = value?.format("HH:mm")
 
-  const handleHourChange = e => {
+  const handleChange = e => {
-    dispatch("change", displayValue.hour(parseInt(e.target.value)))
+    if (!e.target.value) {
+      dispatch("change", undefined)
+      return
     }
 
-  const handleMinuteChange = e => {
+    const [hour, minute] = e.target.value.split(":").map(x => parseInt(x))
-    dispatch("change", displayValue.minute(parseInt(e.target.value)))
+    dispatch("change", (value || dayjs()).hour(hour).minute(minute))
   }
-
-  const cleanHour = cleanInput({ max: 23, pad: 2, fallback: "00" })
-  const cleanMinute = cleanInput({ max: 59, pad: 2, fallback: "00" })
 </script>
 
 <div class="time-picker">
   <NumberInput
     hideArrows
-    value={displayValue.hour().toString().padStart(2, "0")}
+    type={"time"}
-    min={0}
+    value={displayValue}
-    max={23}
+    on:input={handleChange}
-    width={20}
+    on:change={handleChange}
-    on:input={cleanHour}
-    on:change={handleHourChange}
-  />
-  <span>:</span>
-  <NumberInput
-    hideArrows
-    value={displayValue.minute().toString().padStart(2, "0")}
-    min={0}
-    max={59}
-    width={20}
-    on:input={cleanMinute}
-    on:change={handleMinuteChange}
   />
 </div>
 
@@ -50,10 +36,4 @@
     flex-direction: row;
     align-items: center;
   }
-  .time-picker span {
-    font-weight: bold;
-    font-size: 18px;
-    z-index: 0;
-    margin-bottom: 1px;
-  }
 </style>

packages/bbui/src/helpers.js

@@ -166,7 +166,7 @@ export const stringifyDate = (
   const offsetForTimezone = (enableTime && ignoreTimezones) || timeOnly
   if (offsetForTimezone) {
     // Ensure we use the correct offset for the date
-    const referenceDate = timeOnly ? new Date() : value.toDate()
+    const referenceDate = value.toDate()
     const offset = referenceDate.getTimezoneOffset() * 60000
     return new Date(value.valueOf() - offset).toISOString().slice(0, -1)
   }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -1,3 +1,4 @@
+import dayjs from "dayjs"
 import {
   AutoFieldSubType,
   AutoReason,
@@ -285,13 +286,10 @@ export class ExternalRequest<T extends Operation> {
       // parse floats/numbers
       if (field.type === FieldType.NUMBER && !isNaN(parseFloat(row[key]))) {
         newRow[key] = parseFloat(row[key])
-      }
+      } else if (field.type === FieldType.LINK) {
-      // if its not a link then just copy it over
+        const { tableName: linkTableName } = breakExternalTableId(
-      if (field.type !== FieldType.LINK) {
+          field?.tableId
-        newRow[key] = row[key]
+        )
-        continue
-      }
-      const { tableName: linkTableName } = breakExternalTableId(field?.tableId)
         // table has to exist for many to many
         if (!linkTableName || !this.tables[linkTableName]) {
           continue
@@ -306,7 +304,8 @@ export class ExternalRequest<T extends Operation> {
             if (typeof row[key] === "string") {
               id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1]
             }
-          newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0]
+            newRow[field.foreignKey || linkTablePrimary] =
+              breakRowIdField(id)[0]
           } else {
             // Removing from both new and row, as we don't know if it has already been processed
             row[field.foreignKey || linkTablePrimary] = null
@@ -345,6 +344,16 @@ export class ExternalRequest<T extends Operation> {
             })
           }
         }
+      } else if (
+        field.type === FieldType.DATETIME &&
+        field.timeOnly &&
+        row[key] &&
+        dayjs(row[key]).isValid()
+      ) {
+        newRow[key] = dayjs(row[key]).format("HH:mm")
+      } else {
+        newRow[key] = row[key]
+      }
     }
     // we return the relationships that may need to be created in the through table
     // we do this so that if the ID is generated by the DB it can be inserted

packages/server/src/api/routes/tests/search.spec.ts

@@ -79,13 +79,14 @@ describe.each([
   }
 
   async function createRows(rows: Record<string, any>[]) {
-    await config.api.row.bulkImport(table._id!, { rows })
+    // Shuffling to avoid false positives given a fixed order
+    await config.api.row.bulkImport(table._id!, { rows: _.shuffle(rows) })
   }
 
   class SearchAssertion {
     constructor(private readonly query: RowSearchParams) {}
 
-    private findRow(expectedRow: any, foundRows: any[]) {
+    private popRow(expectedRow: any, foundRows: any[]) {
       const row = foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
       if (!row) {
         const fields = Object.keys(expectedRow)
@@ -98,6 +99,9 @@ describe.each([
           )} in ${JSON.stringify(searchedObjects)}`
         )
       }
+
+      // Ensuring the same row is not matched twice
+      foundRows.splice(foundRows.indexOf(row), 1)
       return row
     }
 
@@ -114,9 +118,9 @@ describe.each([
       // eslint-disable-next-line jest/no-standalone-expect
       expect(foundRows).toHaveLength(expectedRows.length)
       // eslint-disable-next-line jest/no-standalone-expect
-      expect(foundRows).toEqual(
+      expect([...foundRows]).toEqual(
         expectedRows.map((expectedRow: any) =>
-          expect.objectContaining(this.findRow(expectedRow, foundRows))
+          expect.objectContaining(this.popRow(expectedRow, foundRows))
         )
       )
     }
@@ -133,10 +137,10 @@ describe.each([
       // eslint-disable-next-line jest/no-standalone-expect
       expect(foundRows).toHaveLength(expectedRows.length)
       // eslint-disable-next-line jest/no-standalone-expect
-      expect(foundRows).toEqual(
+      expect([...foundRows]).toEqual(
         expect.arrayContaining(
           expectedRows.map((expectedRow: any) =>
-            expect.objectContaining(this.findRow(expectedRow, foundRows))
+            expect.objectContaining(this.popRow(expectedRow, foundRows))
           )
         )
       )
@@ -152,10 +156,10 @@ describe.each([
       })
 
       // eslint-disable-next-line jest/no-standalone-expect
-      expect(foundRows).toEqual(
+      expect([...foundRows]).toEqual(
         expect.arrayContaining(
           expectedRows.map((expectedRow: any) =>
-            expect.objectContaining(this.findRow(expectedRow, foundRows))
+            expect.objectContaining(this.popRow(expectedRow, foundRows))
           )
         )
       )
@@ -995,6 +999,159 @@ describe.each([
     })
   })
 
+  !isInternal &&
+    describe("datetime - time only", () => {
+      const T_1000 = "10:00"
+      const T_1045 = "10:45"
+      const T_1200 = "12:00"
+      const T_1530 = "15:30"
+      const T_0000 = "00:00"
+
+      const UNEXISTING_TIME = "10:01"
+
+      const NULL_TIME__ID = `null_time__id`
+
+      beforeAll(async () => {
+        await createTable({
+          timeid: { name: "timeid", type: FieldType.STRING },
+          time: { name: "time", type: FieldType.DATETIME, timeOnly: true },
+        })
+
+        await createRows([
+          { timeid: NULL_TIME__ID, time: null },
+          { time: T_1000 },
+          { time: T_1045 },
+          { time: T_1200 },
+          { time: T_1530 },
+          { time: T_0000 },
+        ])
+      })
+
+      describe("equal", () => {
+        it("successfully finds a row", () =>
+          expectQuery({ equal: { time: T_1000 } }).toContainExactly([
+            { time: "10:00:00" },
+          ]))
+
+        it("fails to find nonexistent row", () =>
+          expectQuery({ equal: { time: UNEXISTING_TIME } }).toFindNothing())
+      })
+
+      describe("notEqual", () => {
+        it("successfully finds a row", () =>
+          expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([
+            { time: "10:45:00" },
+            { time: "12:00:00" },
+            { time: "15:30:00" },
+            { time: "00:00:00" },
+          ]))
+
+        it("return all when requesting non-existing", () =>
+          expectQuery({ notEqual: { time: UNEXISTING_TIME } }).toContainExactly(
+            [
+              { time: "10:00:00" },
+              { time: "10:45:00" },
+              { time: "12:00:00" },
+              { time: "15:30:00" },
+              { time: "00:00:00" },
+            ]
+          ))
+      })
+
+      describe("oneOf", () => {
+        it("successfully finds a row", () =>
+          expectQuery({ oneOf: { time: [T_1000] } }).toContainExactly([
+            { time: "10:00:00" },
+          ]))
+
+        it("fails to find nonexistent row", () =>
+          expectQuery({ oneOf: { time: [UNEXISTING_TIME] } }).toFindNothing())
+      })
+
+      describe("range", () => {
+        it("successfully finds a row", () =>
+          expectQuery({
+            range: { time: { low: T_1045, high: T_1045 } },
+          }).toContainExactly([{ time: "10:45:00" }]))
+
+        it("successfully finds multiple rows", () =>
+          expectQuery({
+            range: { time: { low: T_1045, high: T_1530 } },
+          }).toContainExactly([
+            { time: "10:45:00" },
+            { time: "12:00:00" },
+            { time: "15:30:00" },
+          ]))
+
+        it("successfully finds no rows", () =>
+          expectQuery({
+            range: { time: { low: UNEXISTING_TIME, high: UNEXISTING_TIME } },
+          }).toFindNothing())
+      })
+
+      describe("sort", () => {
+        it("sorts ascending", () =>
+          expectSearch({
+            query: {},
+            sort: "time",
+            sortOrder: SortOrder.ASCENDING,
+          }).toMatchExactly([
+            { timeid: NULL_TIME__ID },
+            { time: "00:00:00" },
+            { time: "10:00:00" },
+            { time: "10:45:00" },
+            { time: "12:00:00" },
+            { time: "15:30:00" },
+          ]))
+
+        it("sorts descending", () =>
+          expectSearch({
+            query: {},
+            sort: "time",
+            sortOrder: SortOrder.DESCENDING,
+          }).toMatchExactly([
+            { time: "15:30:00" },
+            { time: "12:00:00" },
+            { time: "10:45:00" },
+            { time: "10:00:00" },
+            { time: "00:00:00" },
+            { timeid: NULL_TIME__ID },
+          ]))
+
+        describe("sortType STRING", () => {
+          it("sorts ascending", () =>
+            expectSearch({
+              query: {},
+              sort: "time",
+              sortType: SortType.STRING,
+              sortOrder: SortOrder.ASCENDING,
+            }).toMatchExactly([
+              { timeid: NULL_TIME__ID },
+              { time: "00:00:00" },
+              { time: "10:00:00" },
+              { time: "10:45:00" },
+              { time: "12:00:00" },
+              { time: "15:30:00" },
+            ]))
+
+          it("sorts descending", () =>
+            expectSearch({
+              query: {},
+              sort: "time",
+              sortType: SortType.STRING,
+              sortOrder: SortOrder.DESCENDING,
+            }).toMatchExactly([
+              { time: "15:30:00" },
+              { time: "12:00:00" },
+              { time: "10:45:00" },
+              { time: "10:00:00" },
+              { time: "00:00:00" },
+              { timeid: NULL_TIME__ID },
+            ]))
+        })
+      })
+    })
+
   describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
     beforeAll(async () => {
       await createTable({

packages/server/src/integrations/base/sql.ts

@@ -122,11 +122,8 @@ function generateSelectStatement(
     const fieldNames = field.split(/\./g)
     const tableName = fieldNames[0]
     const columnName = fieldNames[1]
-    if (
+    const columnSchema = schema?.[columnName]
-      columnName &&
+    if (columnSchema && knex.client.config.client === SqlClient.POSTGRES) {
-      schema?.[columnName] &&
-      knex.client.config.client === SqlClient.POSTGRES
-    ) {
       const externalType = schema[columnName].externalType
       if (externalType?.includes("money")) {
         return knex.raw(
@@ -134,6 +131,14 @@ function generateSelectStatement(
         )
       }
     }
+    if (
+      knex.client.config.client === SqlClient.MS_SQL &&
+      columnSchema?.type === FieldType.DATETIME &&
+      columnSchema.timeOnly
+    ) {
+      // Time gets returned as timestamp from mssql, not matching the expected HH:mm format
+      return knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
+    }
     return `${field} as ${field}`
   })
 }
@@ -383,7 +388,13 @@ class InternalBuilder {
       for (let [key, value] of Object.entries(sort)) {
         const direction =
           value.direction === SortDirection.ASCENDING ? "asc" : "desc"
-        query = query.orderBy(`${aliased}.${key}`, direction)
+        let nulls
+        if (this.client === SqlClient.POSTGRES) {
+          // All other clients already sort this as expected by default, and adding this to the rest of the clients is causing issues
+          nulls = value.direction === SortDirection.ASCENDING ? "first" : "last"
+        }
+
+        query = query.orderBy(`${aliased}.${key}`, direction, nulls)
       }
     } else if (this.client === SqlClient.MS_SQL && paginate?.limit) {
       // @ts-ignore
@@ -634,12 +645,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
    */
   _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] {
     const sqlClient = this.getSqlClient()
-    const config: { client: string; useNullAsDefault?: boolean } = {
+    const config: Knex.Config = {
       client: sqlClient,
     }
     if (sqlClient === SqlClient.SQL_LITE) {
       config.useNullAsDefault = true
     }
+
     const client = knex(config)
     let query: Knex.QueryBuilder
     const builder = new InternalBuilder(sqlClient)

packages/server/src/integrations/base/sqlTable.ts

@@ -79,9 +79,13 @@ function generateSchema(
         schema.boolean(key)
         break
       case FieldType.DATETIME:
+        if (!column.timeOnly) {
           schema.datetime(key, {
             useTz: !column.ignoreTimezones,
           })
+        } else {
+          schema.time(key)
+        }
         break
       case FieldType.ARRAY:
       case FieldType.BB_REFERENCE:

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -61,9 +61,9 @@ describe("Captures of real examples", () => {
               "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid", 
               "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", 
               "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
-              from (select * from "persons" as "a" order by "a"."firstname" asc limit $1) as "a" 
+              from (select * from "persons" as "a" order by "a"."firstname" asc nulls first limit $1) as "a" 
               left join "tasks" as "b" on "a"."personid" = "b"."qaid" or "a"."personid" = "b"."executorid" 
-              order by "a"."firstname" asc limit $2`),
+              order by "a"."firstname" asc nulls first limit $2`),
       })
     })
 
@@ -75,10 +75,10 @@ describe("Captures of real examples", () => {
         sql: multiline(`select "a"."productname" as "a.productname", "a"."productid" as "a.productid", 
               "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", 
               "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
-              from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" 
+              from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" 
               left join "products_tasks" as "c" on "a"."productid" = "c"."productid" 
               left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 
-              order by "a"."productname" asc limit $3`),
+              order by "a"."productname" asc nulls first limit $3`),
       })
     })
 
@@ -90,10 +90,10 @@ describe("Captures of real examples", () => {
         sql: multiline(`select "a"."productname" as "a.productname", "a"."productid" as "a.productid", 
               "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", 
               "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
-              from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" 
+              from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" 
               left join "products_tasks" as "c" on "a"."productid" = "c"."productid" 
               left join "tasks" as "b" on "b"."taskid" = "c"."taskid" 
-              order by "a"."productname" asc limit $2`),
+              order by "a"."productname" asc nulls first limit $2`),
       })
     })
 
@@ -138,11 +138,11 @@ describe("Captures of real examples", () => {
              "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", 
              "c"."city" as "c.city", "c"."lastname" as "c.lastname" 
              from (select * from "tasks" as "a" where not "a"."completed" = $1 
-             order by "a"."taskname" asc limit $2) as "a" 
+             order by "a"."taskname" asc nulls first limit $2) as "a" 
              left join "products_tasks" as "d" on "a"."taskid" = "d"."taskid" 
              left join "products" as "b" on "b"."productid" = "d"."productid" 
              left join "persons" as "c" on "a"."executorid" = "c"."personid" or "a"."qaid" = "c"."personid" 
-             where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc limit $6`),
+             where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc nulls first limit $6`),
       })
     })
   })

packages/server/src/integrations/utils/utils.ts

@@ -71,7 +71,11 @@ const SQL_DATE_TYPE_MAP: Record<string, PrimitiveTypes> = {
 }
 
 const SQL_DATE_ONLY_TYPES = ["date"]
-const SQL_TIME_ONLY_TYPES = ["time"]
+const SQL_TIME_ONLY_TYPES = [
+  "time",
+  "time without time zone",
+  "time with time zone",
+]
 
 const SQL_STRING_TYPE_MAP: Record<string, PrimitiveTypes> = {
   varchar: FieldType.STRING,

packages/server/src/utilities/schema.ts

@@ -129,11 +129,12 @@ export function parse(rows: Rows, schema: TableSchema): Rows {
         return
       }
 
-      const { type: columnType } = schema[columnName]
+      const columnSchema = schema[columnName]
+      const { type: columnType } = columnSchema
       if (columnType === FieldType.NUMBER) {
         // If provided must be a valid number
         parsedRow[columnName] = columnData ? Number(columnData) : columnData
-      } else if (columnType === FieldType.DATETIME) {
+      } else if (columnType === FieldType.DATETIME && !columnSchema.timeOnly) {
         // If provided must be a valid date
         parsedRow[columnName] = columnData
           ? new Date(columnData).toISOString()