From 21b28a3de85c8fc4aac0c9f3e042d0b4d979b0de Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 4 Jul 2024 12:05:56 +0100 Subject: [PATCH 1/7] Added new Generate automation button and added it to the table --- .../backend/DataTable/TableDataTable.svelte | 4 + .../grid/GridCreateAutomationButton.svelte | 112 ++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte diff --git a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte index e8e1008e3c..525421f996 100644 --- a/packages/builder/src/components/backend/DataTable/TableDataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/TableDataTable.svelte @@ -5,6 +5,7 @@ import { TableNames } from "constants" import { Grid } from "@budibase/frontend-core" import { API } from "api" + import GridCreateAutomationButton from "./buttons/grid/GridCreateAutomationButton.svelte" import GridAddColumnModal from "components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte" import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte" import GridEditUserModal from "components/backend/DataTable/modals/grid/GridEditUserModal.svelte" @@ -81,6 +82,9 @@ {/if} + {#if !isUsersTable} + + {/if} {#if relationshipsEnabled} {/if} diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte new file mode 100644 index 0000000000..60aae9e195 --- /dev/null +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte @@ -0,0 +1,112 @@ + + +
+ (open = !open)} + selected={open} + > + Generate + +
+ + + + { + open = false + createAutomation(TriggerStepID.ROW_SAVED) + console.log("create") + }} + > + Automation: when row is created + + { + open = false + createAutomation(TriggerStepID.ROW_UPDATED) + console.log("update") + }} + > + Automation: when row is updated + + + + + From a764bfb6a54ac1fda4be86ff295ce8136ffce0e5 Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 4 Jul 2024 14:30:51 +0100 Subject: [PATCH 2/7] Tidying up debugging console statements and comments --- .../grid/GridCreateAutomationButton.svelte | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte index 60aae9e195..3ee1edc4b6 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte @@ -7,22 +7,15 @@ notifications, } from "@budibase/bbui" import { getContext } from "svelte" - import { automationStore } from "stores/builder" + import { automationStore, tables } from "stores/builder" import { TriggerStepID } from "constants/backend/automations" - import { tables } from "stores/builder" import { goto } from "@roxi/routify" const { datasource } = getContext("grid") - // ROW_SAVED - // ROW_UPDATED - $: console.log($datasource) $: triggers = $automationStore.blockDefinitions.TRIGGER $: table = $tables.list.find(table => table._id === $datasource.tableId) - $: console.log("table", table) - // $: rowCreateTrigger = triggers[TriggerStepID.ROW_SAVED] - // $: rowUpdateTrigger = triggers[TriggerStepID.ROW_UPDATED] async function createAutomation(type) { const triggerType = triggers[type] @@ -48,13 +41,7 @@ triggerBlock.inputs = { tableId: $datasource.tableId } - // need to set inputs to { "tableId": "ta_bb_employee" }, - try { - console.log("REQ", { - automationName, - triggerBlock, - }) const response = await automationStore.actions.create( automationName, triggerBlock @@ -90,7 +77,6 @@ on:click={() => { open = false createAutomation(TriggerStepID.ROW_SAVED) - console.log("create") }} > Automation: when row is created @@ -100,7 +86,6 @@ on:click={() => { open = false createAutomation(TriggerStepID.ROW_UPDATED) - console.log("update") }} > Automation: when row is updated From 77abe6da839657709566745a41f232177292c763 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 4 Jul 2024 18:29:08 +0100 Subject: [PATCH 3/7] Handling invalid time values when ISO strings are input as filter options. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 3 ++ packages/backend-core/src/sql/sql.ts | 22 ++++---------- packages/backend-core/src/sql/utils.ts | 29 +++++++++++++++++-- .../src/api/routes/tests/search.spec.ts | 29 +++++++++++++++++++ 4 files changed, 65 insertions(+), 18 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 274c09438d..f3c3beeaab 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -13,6 +13,7 @@ import { isDocument, RowResponse, RowValue, + SqlClient, SQLiteDefinition, SqlQueryBinding, } from "@budibase/types" @@ -25,6 +26,7 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants" import { DDInstrumentedDatabase } from "../instrumentation" import { checkSlashesInUrl } from "../../helpers" import env from "../../environment" +import { sqlLog } from "../../sql/utils" const DATABASE_NOT_FOUND = "Database does not exist." @@ -322,6 +324,7 @@ export class DatabaseImpl implements Database { ): Promise { const dbName = this.name const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}` + sqlLog(SqlClient.SQL_LITE, sql, parameters) return await this._sqlQuery(url, "POST", { query: sql, args: parameters, diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 615753efc3..e8881c22d4 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -3,8 +3,10 @@ import * as dbCore from "../db" import { getNativeSql, isExternalTable, - isIsoDateString, + isValidISODateString, isValidFilter, + sqlLog, + isInvalidISODateString, } from "./utils" import { SqlStatements } from "./sqlStatements" import SqlTableQueryBuilder from "./sqlTable" @@ -38,10 +40,6 @@ const envLimit = environment.SQL_MAX_ROWS : null const BASE_LIMIT = envLimit || 5000 -// these are invalid dates sent by the client, need to convert them to a real max date -const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z" -const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z" - function likeKey(client: string, key: string): string { let start: string, end: string switch (client) { @@ -75,10 +73,10 @@ function parse(input: any) { if (typeof input !== "string") { return input } - if (input === MAX_ISO_DATE || input === MIN_ISO_DATE) { + if (isInvalidISODateString(input)) { return null } - if (isIsoDateString(input)) { + if (isValidISODateString(input)) { return new Date(input.trim()) } return input @@ -938,15 +936,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { } log(query: string, values?: SqlQueryBinding) { - if (!environment.SQL_LOGGING_ENABLE) { - return - } - const sqlClient = this.getSqlClient() - let string = `[SQL] [${sqlClient.toUpperCase()}] query="${query}"` - if (values) { - string += ` values="${values.join(", ")}"` - } - console.log(string) + sqlLog(this.getSqlClient(), query, values) } } diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts index 45ab510948..e73c6ac445 100644 --- a/packages/backend-core/src/sql/utils.ts +++ b/packages/backend-core/src/sql/utils.ts @@ -2,10 +2,12 @@ import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types" import { DEFAULT_BB_DATASOURCE_ID } from "../constants" import { Knex } from "knex" import { SEPARATOR } from "../db" +import environment from "../environment" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const ROW_ID_REGEX = /^\[.*]$/g const ENCODED_SPACE = encodeURIComponent(" ") +const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/ export function isExternalTableID(tableId: string) { return tableId.startsWith(DocumentType.DATASOURCE + SEPARATOR) @@ -120,15 +122,38 @@ export function breakRowIdField(_id: string | { _id: string }): any[] { } } -export function isIsoDateString(str: string) { +export function isInvalidISODateString(str: string) { const trimmedValue = str.trim() - if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) { + if (!ISO_DATE_REGEX.test(trimmedValue)) { return false } let d = new Date(trimmedValue) + return isNaN(d.getTime()) +} + +export function isValidISODateString(str: string) { + const trimmedValue = str.trim() + if (!ISO_DATE_REGEX.test(trimmedValue)) { + return false + } + let d = new Date(trimmedValue) + if (isNaN(d.getTime())) { + return false + } return d.toISOString() === trimmedValue } export function isValidFilter(value: any) { return value != null && value !== "" } + +export function sqlLog(client: string, query: string, values?: any[]) { + if (!environment.SQL_LOGGING_ENABLE) { + return + } + let string = `[SQL] [${client.toUpperCase()}] query="${query}"` + if (values) { + string += ` values="${values.join(", ")}"` + } + console.log(string) +} diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 145db9b4a3..bdc8fc1af6 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2167,6 +2167,35 @@ describe.each([ } ) + describe.each([ + { low: "2024-07-03T00:00:00.000Z", high: "9999-00-00T00:00:00.000Z" }, + { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" }, + { low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, + { low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, + ])("date special cases", ({ low, high }) => { + const earlyDate = "2024-07-03T10:00:00.000Z", + laterDate = "2024-07-03T11:00:00.000Z" + beforeAll(async () => { + table = await createTable({ + date: { + name: "date", + type: FieldType.DATETIME, + }, + }) + await createRows([{ date: earlyDate }, { date: laterDate }]) + }) + + it("should be able to handle a date search", async () => { + await expectSearch({ + query: { + range: { + "1:date": { low, high }, + }, + }, + }).toContainExactly([{ date: earlyDate }, { date: laterDate }]) + }) + }) + describe.each([ "名前", // Japanese for "name" "Benutzer-ID", // German for "user ID", includes a hyphen From 9518680d1277f1b5cbf9733dd63a5c6102bd9311 Mon Sep 17 00:00:00 2001 From: Dean Date: Fri, 5 Jul 2024 14:33:09 +0100 Subject: [PATCH 4/7] Update the topnav history when navigating to the automation sections on create --- .../buttons/grid/GridCreateAutomationButton.svelte | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte index 3ee1edc4b6..94ac41fd24 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte @@ -7,9 +7,9 @@ notifications, } from "@budibase/bbui" import { getContext } from "svelte" - import { automationStore, tables } from "stores/builder" + import { automationStore, tables, builderStore } from "stores/builder" import { TriggerStepID } from "constants/backend/automations" - import { goto } from "@roxi/routify" + import { goto, layout, isActive } from "@roxi/routify" const { datasource } = getContext("grid") @@ -46,10 +46,14 @@ automationName, triggerBlock ) - + builderStore.setPreviousTopNavPath( + "/builder/app/:application/data", + window.location.pathname + ) $goto(`/builder/app/${response.appId}/automation/${response.id}`) notifications.success(`Automation created`) - } catch { + } catch (e) { + console.error("Error creating automation", e) notifications.error("Error creating automation") } } From 4d11f62e007899abb8317e305ea057f27160aa08 Mon Sep 17 00:00:00 2001 From: Dean Date: Fri, 5 Jul 2024 14:36:20 +0100 Subject: [PATCH 5/7] Lint --- .../DataTable/buttons/grid/GridCreateAutomationButton.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte index 94ac41fd24..8e3d90be41 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte @@ -9,7 +9,7 @@ import { getContext } from "svelte" import { automationStore, tables, builderStore } from "stores/builder" import { TriggerStepID } from "constants/backend/automations" - import { goto, layout, isActive } from "@roxi/routify" + import { goto } from "@roxi/routify" const { datasource } = getContext("grid") From e4375c21966aee6c99018889d54862db27399ee2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Jul 2024 15:27:54 +0100 Subject: [PATCH 6/7] Fixing a build issue uncovered by tests. --- packages/backend-core/src/db/constants.ts | 5 ----- packages/backend-core/src/db/couch/index.ts | 1 - .../backend-core/tests/core/utilities/jestUtils.ts | 9 ++++++--- packages/server/src/sdk/app/rows/search/internal.ts | 5 +++-- packages/server/src/sdk/app/tables/migration.ts | 3 ++- packages/server/src/sdk/app/views/index.ts | 12 ++++++++---- 6 files changed, 19 insertions(+), 16 deletions(-) delete mode 100644 packages/backend-core/src/db/constants.ts diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts deleted file mode 100644 index 69c98fe569..0000000000 --- a/packages/backend-core/src/db/constants.ts +++ /dev/null @@ -1,5 +0,0 @@ -export { - CONSTANT_INTERNAL_ROW_COLS, - CONSTANT_EXTERNAL_ROW_COLS, - isInternalColumnName, -} from "@budibase/shared-core" diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index 932efed3f7..c731d20d6c 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -2,4 +2,3 @@ export * from "./connections" export * from "./DatabaseImpl" export * from "./utils" export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB" -export * from "../constants" diff --git a/packages/backend-core/tests/core/utilities/jestUtils.ts b/packages/backend-core/tests/core/utilities/jestUtils.ts index 4a3da8db8c..a49c2a795e 100644 --- a/packages/backend-core/tests/core/utilities/jestUtils.ts +++ b/packages/backend-core/tests/core/utilities/jestUtils.ts @@ -1,4 +1,7 @@ -import { db } from "../../../src" +import { + CONSTANT_EXTERNAL_ROW_COLS, + CONSTANT_INTERNAL_ROW_COLS, +} from "@budibase/shared-core" export function expectFunctionWasCalledTimesWith( jestFunction: any, @@ -11,7 +14,7 @@ export function expectFunctionWasCalledTimesWith( } export const expectAnyInternalColsAttributes: { - [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any + [K in (typeof CONSTANT_INTERNAL_ROW_COLS)[number]]: any } = { tableId: expect.anything(), type: expect.anything(), @@ -22,7 +25,7 @@ export const expectAnyInternalColsAttributes: { } export const expectAnyExternalColsAttributes: { - [K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any + [K in (typeof CONSTANT_EXTERNAL_ROW_COLS)[number]]: any } = { tableId: expect.anything(), _id: expect.anything(), diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 906ca016d1..097b16b104 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -1,4 +1,5 @@ -import { context, db, HTTPError } from "@budibase/backend-core" +import { context, HTTPError } from "@budibase/backend-core" +import { CONSTANT_INTERNAL_ROW_COLS } from "@budibase/shared-core" import env from "../../../../environment" import { fullSearch, paginatedSearch } from "./utils" import { getRowParams, InternalTables } from "../../../../db/utils" @@ -74,7 +75,7 @@ export async function search( } if (options.fields) { - const fields = [...options.fields, ...db.CONSTANT_INTERNAL_ROW_COLS] + const fields = [...options.fields, ...CONSTANT_INTERNAL_ROW_COLS] response.rows = response.rows.map((r: any) => pick(r, fields)) } diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts index d0decf01f6..8e084381aa 100644 --- a/packages/server/src/sdk/app/tables/migration.ts +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -18,6 +18,7 @@ import sdk from "../../../sdk" import { isExternalTableID } from "../../../integrations/utils" import { EventType, updateLinks } from "../../../db/linkedRows" import { cloneDeep } from "lodash" +import { isInternalColumnName } from "@budibase/shared-core" export interface MigrationResult { tablesUpdated: Table[] @@ -36,7 +37,7 @@ export async function migrate( throw new BadRequestError(`Column name cannot be empty`) } - if (dbCore.isInternalColumnName(newColumnName)) { + if (isInternalColumnName(newColumnName)) { throw new BadRequestError(`Column name cannot be a reserved column name`) } diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index fce57a390d..3bdfec7448 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -6,9 +6,13 @@ import { ViewV2, ViewV2Enriched, } from "@budibase/types" -import { HTTPError, db as dbCore } from "@budibase/backend-core" +import { HTTPError } from "@budibase/backend-core" import { features } from "@budibase/pro" -import { helpers } from "@budibase/shared-core" +import { + helpers, + CONSTANT_EXTERNAL_ROW_COLS, + CONSTANT_INTERNAL_ROW_COLS, +} from "@budibase/shared-core" import { cloneDeep } from "lodash/fp" import * as utils from "../../../db/utils" @@ -144,8 +148,8 @@ export function allowedFields(view: View | ViewV2) { const fieldSchema = view.schema![key] return fieldSchema.visible && !fieldSchema.readonly }), - ...dbCore.CONSTANT_EXTERNAL_ROW_COLS, - ...dbCore.CONSTANT_INTERNAL_ROW_COLS, + ...CONSTANT_EXTERNAL_ROW_COLS, + ...CONSTANT_INTERNAL_ROW_COLS, ] } From c48f5c6d8016f082324bc49b4447557393220430 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Jul 2024 15:45:00 +0100 Subject: [PATCH 7/7] Fixing build issue. --- packages/server/src/sdk/app/rows/search/external.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 93c46d8cc3..1b6638f671 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -16,9 +16,9 @@ import { breakExternalTableId, breakRowIdField, } from "../../../../integrations/utils" -import { utils } from "@budibase/shared-core" +import { utils, CONSTANT_EXTERNAL_ROW_COLS } from "@budibase/shared-core" import { ExportRowsParams, ExportRowsResult } from "./types" -import { db, HTTPError } from "@budibase/backend-core" +import { HTTPError } from "@budibase/backend-core" import pick from "lodash/pick" import { outputProcessing } from "../../../../utilities/rowProcessor" import sdk from "../../../" @@ -99,7 +99,7 @@ export async function search( } if (options.fields) { - const fields = [...options.fields, ...db.CONSTANT_EXTERNAL_ROW_COLS] + const fields = [...options.fields, ...CONSTANT_EXTERNAL_ROW_COLS] rows = rows.map((r: any) => pick(r, fields)) }