Merge pull request #14104 from Budibase/fix/invalid-time-value

Solving some invalid time filters

packages/backend-core/src/db/constants.ts

@@ -1,5 +0,0 @@
-export {
-  CONSTANT_INTERNAL_ROW_COLS,
-  CONSTANT_EXTERNAL_ROW_COLS,
-  isInternalColumnName,
-} from "@budibase/shared-core"

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -13,6 +13,7 @@ import {
   isDocument,
   RowResponse,
   RowValue,
+  SqlClient,
   SQLiteDefinition,
   SqlQueryBinding,
 } from "@budibase/types"
@@ -25,6 +26,7 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 import { DDInstrumentedDatabase } from "../instrumentation"
 import { checkSlashesInUrl } from "../../helpers"
 import env from "../../environment"
+import { sqlLog } from "../../sql/utils"
 
 const DATABASE_NOT_FOUND = "Database does not exist."
 
@@ -322,6 +324,7 @@ export class DatabaseImpl implements Database {
   ): Promise<T[]> {
     const dbName = this.name
     const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}`
+    sqlLog(SqlClient.SQL_LITE, sql, parameters)
     return await this._sqlQuery<T[]>(url, "POST", {
       query: sql,
       args: parameters,

packages/backend-core/src/db/couch/index.ts

@@ -2,4 +2,3 @@ export * from "./connections"
 export * from "./DatabaseImpl"
 export * from "./utils"
 export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB"
-export * from "../constants"

packages/backend-core/src/sql/sql.ts

@@ -3,8 +3,10 @@ import * as dbCore from "../db"
 import {
   getNativeSql,
   isExternalTable,
-  isIsoDateString,
+  isValidISODateString,
   isValidFilter,
+  sqlLog,
+  isInvalidISODateString,
 } from "./utils"
 import { SqlStatements } from "./sqlStatements"
 import SqlTableQueryBuilder from "./sqlTable"
@@ -38,10 +40,6 @@ const envLimit = environment.SQL_MAX_ROWS
   : null
 const BASE_LIMIT = envLimit || 5000
 
-// these are invalid dates sent by the client, need to convert them to a real max date
-const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z"
-const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z"
-
 function likeKey(client: string, key: string): string {
   let start: string, end: string
   switch (client) {
@@ -75,10 +73,10 @@ function parse(input: any) {
   if (typeof input !== "string") {
     return input
   }
-  if (input === MAX_ISO_DATE || input === MIN_ISO_DATE) {
+  if (isInvalidISODateString(input)) {
     return null
   }
-  if (isIsoDateString(input)) {
+  if (isValidISODateString(input)) {
     return new Date(input.trim())
   }
   return input
@@ -938,15 +936,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
   }
 
   log(query: string, values?: SqlQueryBinding) {
-    if (!environment.SQL_LOGGING_ENABLE) {
+    sqlLog(this.getSqlClient(), query, values)
-      return
-    }
-    const sqlClient = this.getSqlClient()
-    let string = `[SQL] [${sqlClient.toUpperCase()}] query="${query}"`
-    if (values) {
-      string += ` values="${values.join(", ")}"`
-    }
-    console.log(string)
   }
 }
 

packages/backend-core/src/sql/utils.ts

@@ -2,10 +2,12 @@ import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types"
 import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
 import { Knex } from "knex"
 import { SEPARATOR } from "../db"
+import environment from "../environment"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
 const ENCODED_SPACE = encodeURIComponent(" ")
+const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/
 
 export function isExternalTableID(tableId: string) {
   return tableId.startsWith(DocumentType.DATASOURCE + SEPARATOR)
@@ -120,15 +122,38 @@ export function breakRowIdField(_id: string | { _id: string }): any[] {
   }
 }
 
-export function isIsoDateString(str: string) {
+export function isInvalidISODateString(str: string) {
   const trimmedValue = str.trim()
-  if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) {
+  if (!ISO_DATE_REGEX.test(trimmedValue)) {
     return false
   }
   let d = new Date(trimmedValue)
+  return isNaN(d.getTime())
+}
+
+export function isValidISODateString(str: string) {
+  const trimmedValue = str.trim()
+  if (!ISO_DATE_REGEX.test(trimmedValue)) {
+    return false
+  }
+  let d = new Date(trimmedValue)
+  if (isNaN(d.getTime())) {
+    return false
+  }
   return d.toISOString() === trimmedValue
 }
 
 export function isValidFilter(value: any) {
   return value != null && value !== ""
 }
+
+export function sqlLog(client: string, query: string, values?: any[]) {
+  if (!environment.SQL_LOGGING_ENABLE) {
+    return
+  }
+  let string = `[SQL] [${client.toUpperCase()}] query="${query}"`
+  if (values) {
+    string += ` values="${values.join(", ")}"`
+  }
+  console.log(string)
+}

packages/backend-core/tests/core/utilities/jestUtils.ts

@@ -1,4 +1,7 @@
-import { db } from "../../../src"
+import {
+  CONSTANT_EXTERNAL_ROW_COLS,
+  CONSTANT_INTERNAL_ROW_COLS,
+} from "@budibase/shared-core"
 
 export function expectFunctionWasCalledTimesWith(
   jestFunction: any,
@@ -11,7 +14,7 @@ export function expectFunctionWasCalledTimesWith(
 }
 
 export const expectAnyInternalColsAttributes: {
-  [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any
+  [K in (typeof CONSTANT_INTERNAL_ROW_COLS)[number]]: any
 } = {
   tableId: expect.anything(),
   type: expect.anything(),
@@ -22,7 +25,7 @@ export const expectAnyInternalColsAttributes: {
 }
 
 export const expectAnyExternalColsAttributes: {
-  [K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any
+  [K in (typeof CONSTANT_EXTERNAL_ROW_COLS)[number]]: any
 } = {
   tableId: expect.anything(),
   _id: expect.anything(),

packages/server/src/api/routes/tests/search.spec.ts

@@ -2167,6 +2167,35 @@ describe.each([
     }
   )
 
+  describe.each([
+    { low: "2024-07-03T00:00:00.000Z", high: "9999-00-00T00:00:00.000Z" },
+    { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" },
+    { low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
+    { low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
+  ])("date special cases", ({ low, high }) => {
+    const earlyDate = "2024-07-03T10:00:00.000Z",
+      laterDate = "2024-07-03T11:00:00.000Z"
+    beforeAll(async () => {
+      table = await createTable({
+        date: {
+          name: "date",
+          type: FieldType.DATETIME,
+        },
+      })
+      await createRows([{ date: earlyDate }, { date: laterDate }])
+    })
+
+    it("should be able to handle a date search", async () => {
+      await expectSearch({
+        query: {
+          range: {
+            "1:date": { low, high },
+          },
+        },
+      }).toContainExactly([{ date: earlyDate }, { date: laterDate }])
+    })
+  })
+
   describe.each([
     "名前", // Japanese for "name"
     "Benutzer-ID", // German for "user ID", includes a hyphen

packages/server/src/sdk/app/rows/search/external.ts

@@ -16,9 +16,9 @@ import {
   breakExternalTableId,
   breakRowIdField,
 } from "../../../../integrations/utils"
-import { utils } from "@budibase/shared-core"
+import { utils, CONSTANT_EXTERNAL_ROW_COLS } from "@budibase/shared-core"
 import { ExportRowsParams, ExportRowsResult } from "./types"
-import { db, HTTPError } from "@budibase/backend-core"
+import { HTTPError } from "@budibase/backend-core"
 import pick from "lodash/pick"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
 import sdk from "../../../"
@@ -99,7 +99,7 @@ export async function search(
     }
 
     if (options.fields) {
-      const fields = [...options.fields, ...db.CONSTANT_EXTERNAL_ROW_COLS]
+      const fields = [...options.fields, ...CONSTANT_EXTERNAL_ROW_COLS]
       rows = rows.map((r: any) => pick(r, fields))
     }
 

packages/server/src/sdk/app/rows/search/internal.ts

@@ -1,4 +1,5 @@
-import { context, db, HTTPError } from "@budibase/backend-core"
+import { context, HTTPError } from "@budibase/backend-core"
+import { CONSTANT_INTERNAL_ROW_COLS } from "@budibase/shared-core"
 import env from "../../../../environment"
 import { fullSearch, paginatedSearch } from "./utils"
 import { getRowParams, InternalTables } from "../../../../db/utils"
@@ -74,7 +75,7 @@ export async function search(
     }
 
     if (options.fields) {
-      const fields = [...options.fields, ...db.CONSTANT_INTERNAL_ROW_COLS]
+      const fields = [...options.fields, ...CONSTANT_INTERNAL_ROW_COLS]
       response.rows = response.rows.map((r: any) => pick(r, fields))
     }
 

packages/server/src/sdk/app/tables/migration.ts

@@ -18,6 +18,7 @@ import sdk from "../../../sdk"
 import { isExternalTableID } from "../../../integrations/utils"
 import { EventType, updateLinks } from "../../../db/linkedRows"
 import { cloneDeep } from "lodash"
+import { isInternalColumnName } from "@budibase/shared-core"
 
 export interface MigrationResult {
   tablesUpdated: Table[]
@@ -36,7 +37,7 @@ export async function migrate(
     throw new BadRequestError(`Column name cannot be empty`)
   }
 
-  if (dbCore.isInternalColumnName(newColumnName)) {
+  if (isInternalColumnName(newColumnName)) {
     throw new BadRequestError(`Column name cannot be a reserved column name`)
   }
 

packages/server/src/sdk/app/views/index.ts

@@ -6,9 +6,13 @@ import {
   ViewV2,
   ViewV2Enriched,
 } from "@budibase/types"
-import { HTTPError, db as dbCore } from "@budibase/backend-core"
+import { HTTPError } from "@budibase/backend-core"
 import { features } from "@budibase/pro"
-import { helpers } from "@budibase/shared-core"
+import {
+  helpers,
+  CONSTANT_EXTERNAL_ROW_COLS,
+  CONSTANT_INTERNAL_ROW_COLS,
+} from "@budibase/shared-core"
 import { cloneDeep } from "lodash/fp"
 
 import * as utils from "../../../db/utils"
@@ -144,8 +148,8 @@ export function allowedFields(view: View | ViewV2) {
       const fieldSchema = view.schema![key]
       return fieldSchema.visible && !fieldSchema.readonly
     }),
-    ...dbCore.CONSTANT_EXTERNAL_ROW_COLS,
+    ...CONSTANT_EXTERNAL_ROW_COLS,
-    ...dbCore.CONSTANT_INTERNAL_ROW_COLS,
+    ...CONSTANT_INTERNAL_ROW_COLS,
   ]
 }