Merge branch 'master' into fix/default-table-sqs-indexing

packages/backend-core/src/db/constants.ts

@@ -1,5 +0,0 @@
-export {
-  CONSTANT_INTERNAL_ROW_COLS,
-  CONSTANT_EXTERNAL_ROW_COLS,
-  isInternalColumnName,
-} from "@budibase/shared-core"

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -13,6 +13,7 @@ import {
   isDocument,
   RowResponse,
   RowValue,
+  SqlClient,
   SQLiteDefinition,
   SqlQueryBinding,
 } from "@budibase/types"
@@ -25,6 +26,7 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 import { DDInstrumentedDatabase } from "../instrumentation"
 import { checkSlashesInUrl } from "../../helpers"
 import env from "../../environment"
+import { sqlLog } from "../../sql/utils"
 
 const DATABASE_NOT_FOUND = "Database does not exist."
 
@@ -322,6 +324,7 @@ export class DatabaseImpl implements Database {
   ): Promise<T[]> {
     const dbName = this.name
     const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}`
+    sqlLog(SqlClient.SQL_LITE, sql, parameters)
     return await this._sqlQuery<T[]>(url, "POST", {
       query: sql,
       args: parameters,

packages/backend-core/src/db/couch/index.ts

@@ -2,4 +2,3 @@ export * from "./connections"
 export * from "./DatabaseImpl"
 export * from "./utils"
 export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB"
-export * from "../constants"

packages/backend-core/src/sql/sql.ts

@@ -3,8 +3,10 @@ import * as dbCore from "../db"
 import {
   getNativeSql,
   isExternalTable,
-  isIsoDateString,
+  isValidISODateString,
   isValidFilter,
+  sqlLog,
+  isInvalidISODateString,
 } from "./utils"
 import { SqlStatements } from "./sqlStatements"
 import SqlTableQueryBuilder from "./sqlTable"
@@ -38,10 +40,6 @@ const envLimit = environment.SQL_MAX_ROWS
   : null
 const BASE_LIMIT = envLimit || 5000
 
-// these are invalid dates sent by the client, need to convert them to a real max date
-const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z"
-const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z"
-
 function likeKey(client: string, key: string): string {
   let start: string, end: string
   switch (client) {
@@ -75,10 +73,10 @@ function parse(input: any) {
   if (typeof input !== "string") {
     return input
   }
-  if (input === MAX_ISO_DATE || input === MIN_ISO_DATE) {
+  if (isInvalidISODateString(input)) {
     return null
   }
-  if (isIsoDateString(input)) {
+  if (isValidISODateString(input)) {
     return new Date(input.trim())
   }
   return input
@@ -938,15 +936,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
   }
 
   log(query: string, values?: SqlQueryBinding) {
-    if (!environment.SQL_LOGGING_ENABLE) {
+    sqlLog(this.getSqlClient(), query, values)
-      return
-    }
-    const sqlClient = this.getSqlClient()
-    let string = `[SQL] [${sqlClient.toUpperCase()}] query="${query}"`
-    if (values) {
-      string += ` values="${values.join(", ")}"`
-    }
-    console.log(string)
   }
 }
 

packages/backend-core/src/sql/utils.ts

@@ -2,10 +2,12 @@ import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types"
 import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
 import { Knex } from "knex"
 import { SEPARATOR } from "../db"
+import environment from "../environment"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
 const ENCODED_SPACE = encodeURIComponent(" ")
+const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/
 
 export function isExternalTableID(tableId: string) {
   return tableId.startsWith(DocumentType.DATASOURCE + SEPARATOR)
@@ -120,15 +122,38 @@ export function breakRowIdField(_id: string | { _id: string }): any[] {
   }
 }
 
-export function isIsoDateString(str: string) {
+export function isInvalidISODateString(str: string) {
   const trimmedValue = str.trim()
-  if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) {
+  if (!ISO_DATE_REGEX.test(trimmedValue)) {
     return false
   }
   let d = new Date(trimmedValue)
+  return isNaN(d.getTime())
+}
+
+export function isValidISODateString(str: string) {
+  const trimmedValue = str.trim()
+  if (!ISO_DATE_REGEX.test(trimmedValue)) {
+    return false
+  }
+  let d = new Date(trimmedValue)
+  if (isNaN(d.getTime())) {
+    return false
+  }
   return d.toISOString() === trimmedValue
 }
 
 export function isValidFilter(value: any) {
   return value != null && value !== ""
 }
+
+export function sqlLog(client: string, query: string, values?: any[]) {
+  if (!environment.SQL_LOGGING_ENABLE) {
+    return
+  }
+  let string = `[SQL] [${client.toUpperCase()}] query="${query}"`
+  if (values) {
+    string += ` values="${values.join(", ")}"`
+  }
+  console.log(string)
+}

packages/backend-core/tests/core/utilities/jestUtils.ts

@@ -1,4 +1,7 @@
-import { db } from "../../../src"
+import {
+  CONSTANT_EXTERNAL_ROW_COLS,
+  CONSTANT_INTERNAL_ROW_COLS,
+} from "@budibase/shared-core"
 
 export function expectFunctionWasCalledTimesWith(
   jestFunction: any,
@@ -11,7 +14,7 @@ export function expectFunctionWasCalledTimesWith(
 }
 
 export const expectAnyInternalColsAttributes: {
-  [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any
+  [K in (typeof CONSTANT_INTERNAL_ROW_COLS)[number]]: any
 } = {
   tableId: expect.anything(),
   type: expect.anything(),
@@ -22,7 +25,7 @@ export const expectAnyInternalColsAttributes: {
 }
 
 export const expectAnyExternalColsAttributes: {
-  [K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any
+  [K in (typeof CONSTANT_EXTERNAL_ROW_COLS)[number]]: any
 } = {
   tableId: expect.anything(),
   _id: expect.anything(),

packages/builder/src/components/backend/DataTable/TableDataTable.svelte

@@ -5,6 +5,7 @@
   import { TableNames } from "constants"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
+  import GridCreateAutomationButton from "./buttons/grid/GridCreateAutomationButton.svelte"
   import GridAddColumnModal from "components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
   import GridEditUserModal from "components/backend/DataTable/modals/grid/GridEditUserModal.svelte"
@@ -81,6 +82,9 @@
         <GridCreateViewButton />
       {/if}
       <GridManageAccessButton />
+      {#if !isUsersTable}
+        <GridCreateAutomationButton />
+      {/if}
       {#if relationshipsEnabled}
         <GridRelationshipButton />
       {/if}

packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte

@@ -0,0 +1,101 @@
+<script>
+  import {
+    ActionButton,
+    Popover,
+    Menu,
+    MenuItem,
+    notifications,
+  } from "@budibase/bbui"
+  import { getContext } from "svelte"
+  import { automationStore, tables, builderStore } from "stores/builder"
+  import { TriggerStepID } from "constants/backend/automations"
+  import { goto } from "@roxi/routify"
+
+  const { datasource } = getContext("grid")
+
+  $: triggers = $automationStore.blockDefinitions.TRIGGER
+
+  $: table = $tables.list.find(table => table._id === $datasource.tableId)
+
+  async function createAutomation(type) {
+    const triggerType = triggers[type]
+    if (!triggerType) {
+      console.error("Invalid trigger type", type)
+      notifications.error("Invalid automation trigger type")
+      return
+    }
+
+    if (!table) {
+      notifications.error("Invalid table, cannot create automation")
+      return
+    }
+
+    const automationName = `${table.name} : Row ${
+      type === TriggerStepID.ROW_SAVED ? "created" : "updated"
+    }`
+    const triggerBlock = automationStore.actions.constructBlock(
+      "TRIGGER",
+      triggerType.stepId,
+      triggerType
+    )
+
+    triggerBlock.inputs = { tableId: $datasource.tableId }
+
+    try {
+      const response = await automationStore.actions.create(
+        automationName,
+        triggerBlock
+      )
+      builderStore.setPreviousTopNavPath(
+        "/builder/app/:application/data",
+        window.location.pathname
+      )
+      $goto(`/builder/app/${response.appId}/automation/${response.id}`)
+      notifications.success(`Automation created`)
+    } catch (e) {
+      console.error("Error creating automation", e)
+      notifications.error("Error creating automation")
+    }
+  }
+
+  let anchor
+  let open
+</script>
+
+<div bind:this={anchor}>
+  <ActionButton
+    icon="MagicWand"
+    quiet
+    size="M"
+    on:click={() => (open = !open)}
+    selected={open}
+  >
+    Generate
+  </ActionButton>
+</div>
+
+<Popover bind:open {anchor} align="left">
+  <Menu>
+    <MenuItem
+      icon="ShareAndroid"
+      on:click={() => {
+        open = false
+        createAutomation(TriggerStepID.ROW_SAVED)
+      }}
+    >
+      Automation: when row is created
+    </MenuItem>
+    <MenuItem
+      icon="ShareAndroid"
+      on:click={() => {
+        open = false
+        createAutomation(TriggerStepID.ROW_UPDATED)
+      }}
+    >
+      Automation: when row is updated
+    </MenuItem>
+  </Menu>
+</Popover>
+
+<style>
+</style>

packages/server/src/api/routes/tests/search.spec.ts

@@ -2186,6 +2186,35 @@ describe.each([
       })
     })
 
+  describe.each([
+    { low: "2024-07-03T00:00:00.000Z", high: "9999-00-00T00:00:00.000Z" },
+    { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" },
+    { low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
+    { low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
+  ])("date special cases", ({ low, high }) => {
+    const earlyDate = "2024-07-03T10:00:00.000Z",
+      laterDate = "2024-07-03T11:00:00.000Z"
+    beforeAll(async () => {
+      table = await createTable({
+        date: {
+          name: "date",
+          type: FieldType.DATETIME,
+        },
+      })
+      await createRows([{ date: earlyDate }, { date: laterDate }])
+    })
+
+    it("should be able to handle a date search", async () => {
+      await expectSearch({
+        query: {
+          range: {
+            "1:date": { low, high },
+          },
+        },
+      }).toContainExactly([{ date: earlyDate }, { date: laterDate }])
+    })
+  })
+
   describe.each([
     "名前", // Japanese for "name"
     "Benutzer-ID", // German for "user ID", includes a hyphen

packages/server/src/sdk/app/rows/search/external.ts

@@ -16,9 +16,9 @@ import {
   breakExternalTableId,
   breakRowIdField,
 } from "../../../../integrations/utils"
-import { utils } from "@budibase/shared-core"
+import { utils, CONSTANT_EXTERNAL_ROW_COLS } from "@budibase/shared-core"
 import { ExportRowsParams, ExportRowsResult } from "./types"
-import { db, HTTPError } from "@budibase/backend-core"
+import { HTTPError } from "@budibase/backend-core"
 import pick from "lodash/pick"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
 import sdk from "../../../"
@@ -99,7 +99,7 @@ export async function search(
     }
 
     if (options.fields) {
-      const fields = [...options.fields, ...db.CONSTANT_EXTERNAL_ROW_COLS]
+      const fields = [...options.fields, ...CONSTANT_EXTERNAL_ROW_COLS]
       rows = rows.map((r: any) => pick(r, fields))
     }
 

packages/server/src/sdk/app/rows/search/internal.ts

@@ -1,4 +1,5 @@
-import { context, db, HTTPError } from "@budibase/backend-core"
+import { context, HTTPError } from "@budibase/backend-core"
+import { CONSTANT_INTERNAL_ROW_COLS } from "@budibase/shared-core"
 import env from "../../../../environment"
 import { fullSearch, paginatedSearch } from "./utils"
 import { getRowParams, InternalTables } from "../../../../db/utils"
@@ -74,7 +75,7 @@ export async function search(
     }
 
     if (options.fields) {
-      const fields = [...options.fields, ...db.CONSTANT_INTERNAL_ROW_COLS]
+      const fields = [...options.fields, ...CONSTANT_INTERNAL_ROW_COLS]
       response.rows = response.rows.map((r: any) => pick(r, fields))
     }
 

packages/server/src/sdk/app/tables/migration.ts

@@ -18,6 +18,7 @@ import sdk from "../../../sdk"
 import { isExternalTableID } from "../../../integrations/utils"
 import { EventType, updateLinks } from "../../../db/linkedRows"
 import { cloneDeep } from "lodash"
+import { isInternalColumnName } from "@budibase/shared-core"
 
 export interface MigrationResult {
   tablesUpdated: Table[]
@@ -36,7 +37,7 @@ export async function migrate(
     throw new BadRequestError(`Column name cannot be empty`)
   }
 
-  if (dbCore.isInternalColumnName(newColumnName)) {
+  if (isInternalColumnName(newColumnName)) {
     throw new BadRequestError(`Column name cannot be a reserved column name`)
   }
 

packages/server/src/sdk/app/views/index.ts

@@ -6,9 +6,13 @@ import {
   ViewV2,
   ViewV2Enriched,
 } from "@budibase/types"
-import { HTTPError, db as dbCore } from "@budibase/backend-core"
+import { HTTPError } from "@budibase/backend-core"
 import { features } from "@budibase/pro"
-import { helpers } from "@budibase/shared-core"
+import {
+  helpers,
+  CONSTANT_EXTERNAL_ROW_COLS,
+  CONSTANT_INTERNAL_ROW_COLS,
+} from "@budibase/shared-core"
 import { cloneDeep } from "lodash/fp"
 
 import * as utils from "../../../db/utils"
@@ -144,8 +148,8 @@ export function allowedFields(view: View | ViewV2) {
       const fieldSchema = view.schema![key]
       return fieldSchema.visible && !fieldSchema.readonly
     }),
-    ...dbCore.CONSTANT_EXTERNAL_ROW_COLS,
+    ...CONSTANT_EXTERNAL_ROW_COLS,
-    ...dbCore.CONSTANT_INTERNAL_ROW_COLS,
+    ...CONSTANT_INTERNAL_ROW_COLS,
   ]
 }