diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index d86d301507..463074e836 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -23,6 +23,7 @@ jobs: PAYLOAD_BRANCH: ${{ github.head_ref }} PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} PAYLOAD_LICENSE_TYPE: "free" + PAYLOAD_DEPLOY: true with: repository: budibase/budibase-deploys event: featurebranch-qa-deploy diff --git a/.tool-versions b/.tool-versions index 946d5198ce..cf78481d93 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ nodejs 20.10.0 python 3.10.0 -yarn 1.22.19 +yarn 1.22.22 diff --git a/lerna.json b/lerna.json index 10d36c9eaf..a4bcb56d38 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.8", + "version": "2.32.11", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/account-portal b/packages/account-portal index 558a32dfd1..3e24f6293f 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 558a32dfd1f55bd894804a503e7e1090937df88c +Subproject commit 3e24f6293ff5ee5f9b42822e001504e3bbf19cc0 diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index a52a17dd53..25b273e51c 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -10,7 +10,7 @@ import { StaticDatabases, DEFAULT_TENANT_ID, } from "../constants" -import { Database, IdentityContext, Snippet, App } from "@budibase/types" +import { Database, IdentityContext, Snippet, App, Table } from "@budibase/types" import { ContextMap } from "./types" let TEST_APP_ID: string | null = null @@ -394,3 +394,20 @@ export function setFeatureFlags(key: string, value: Record) { context.featureFlagCache ??= {} context.featureFlagCache[key] = value } + +export function getTableForView(viewId: string): Table | undefined { + const context = getCurrentContext() + if (!context) { + return + } + return context.viewToTableCache?.[viewId] +} + +export function setTableForView(viewId: string, table: Table) { + const context = getCurrentContext() + if (!context) { + return + } + context.viewToTableCache ??= {} + context.viewToTableCache[viewId] = table +} diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index fe6072e85c..ee84b49459 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -1,4 +1,4 @@ -import { IdentityContext, Snippet, VM } from "@budibase/types" +import { IdentityContext, Snippet, Table, VM } from "@budibase/types" import { OAuth2Client } from "google-auth-library" import { GoogleSpreadsheet } from "google-spreadsheet" @@ -21,4 +21,5 @@ export type ContextMap = { featureFlagCache?: { [key: string]: Record } + viewToTableCache?: Record } diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index f5ad7e6433..0206bb2140 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -612,7 +612,6 @@ async function runQuery( * limit {number} The number of results to fetch * bookmark {string|null} Current bookmark in the recursive search * rows {array|null} Current results in the recursive search - * @returns {Promise<*[]|*>} */ async function recursiveSearch( dbName: string, diff --git a/packages/backend-core/src/docIds/params.ts b/packages/backend-core/src/docIds/params.ts index 093724b55e..016604b69b 100644 --- a/packages/backend-core/src/docIds/params.ts +++ b/packages/backend-core/src/docIds/params.ts @@ -6,7 +6,7 @@ import { ViewName, } from "../constants" import { getProdAppID } from "./conversions" -import { DatabaseQueryOpts } from "@budibase/types" +import { DatabaseQueryOpts, VirtualDocumentType } from "@budibase/types" /** * If creating DB allDocs/query params with only a single top level ID this can be used, this @@ -66,9 +66,8 @@ export function getQueryIndex(viewName: ViewName) { /** * Check if a given ID is that of a table. - * @returns {boolean} */ -export const isTableId = (id: string) => { +export const isTableId = (id: string): boolean => { // this includes datasource plus tables return ( !!id && @@ -77,13 +76,16 @@ export const isTableId = (id: string) => { ) } +export function isViewId(id: string): boolean { + return !!id && id.startsWith(`${VirtualDocumentType.VIEW}${SEPARATOR}`) +} + /** * Check if a given ID is that of a datasource or datasource plus. - * @returns {boolean} */ -export const isDatasourceId = (id: string) => { +export const isDatasourceId = (id: string): boolean => { // this covers both datasources and datasource plus - return id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`) + return !!id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`) } /** diff --git a/packages/backend-core/src/sql/index.ts b/packages/backend-core/src/sql/index.ts index 16b718d2e6..816b3d60a5 100644 --- a/packages/backend-core/src/sql/index.ts +++ b/packages/backend-core/src/sql/index.ts @@ -1,5 +1,5 @@ export * as utils from "./utils" -export { default as Sql } from "./sql" +export { default as Sql, COUNT_FIELD_NAME } from "./sql" export { default as SqlTable } from "./sqlTable" export * as designDoc from "./designDoc" diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 2b20938981..3585dacbed 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -11,10 +11,12 @@ import { } from "./utils" import SqlTableQueryBuilder from "./sqlTable" import { + Aggregation, AnySearchFilter, ArrayOperator, BasicOperator, BBReferenceFieldMetadata, + CalculationType, FieldSchema, FieldType, INTERNAL_TABLE_SOURCE_ID, @@ -41,6 +43,8 @@ import { cloneDeep } from "lodash" type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any +export const COUNT_FIELD_NAME = "__bb_total" + function getBaseLimit() { const envLimit = environment.SQL_MAX_ROWS ? parseInt(environment.SQL_MAX_ROWS) @@ -69,18 +73,6 @@ function prioritisedArraySort(toSort: string[], priorities: string[]) { }) } -function getTableName(table?: Table): string | undefined { - // SQS uses the table ID rather than the table name - if ( - table?.sourceType === TableSourceType.INTERNAL || - table?.sourceId === INTERNAL_TABLE_SOURCE_ID - ) { - return table?._id - } else { - return table?.name - } -} - function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { if (Array.isArray(query)) { return query.map((q: SqlQuery) => convertBooleans(q) as SqlQuery) @@ -97,6 +89,13 @@ function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { return query } +function isSqs(table: Table): boolean { + return ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) +} + class InternalBuilder { private readonly client: SqlClient private readonly query: QueryJson @@ -178,15 +177,13 @@ class InternalBuilder { } private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { meta, endpoint, resource, tableAliases } = this.query + const { meta, endpoint, resource } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" } - const alias = tableAliases?.[endpoint.entityId] - ? tableAliases?.[endpoint.entityId] - : endpoint.entityId + const alias = this.getTableName(endpoint.entityId) const schema = meta.table.schema if (!this.isFullSelectStatementRequired()) { return [this.knex.raw(`${this.quote(alias)}.*`)] @@ -811,26 +808,88 @@ class InternalBuilder { return query } + isSqs(): boolean { + return isSqs(this.table) + } + + getTableName(tableOrName?: Table | string): string { + let table: Table + if (typeof tableOrName === "string") { + const name = tableOrName + if (this.query.table?.name === name) { + table = this.query.table + } else if (this.query.meta.table?.name === name) { + table = this.query.meta.table + } else if (!this.query.meta.tables?.[name]) { + // This can legitimately happen in custom queries, where the user is + // querying against a table that may not have been imported into + // Budibase. + return name + } else { + table = this.query.meta.tables[name] + } + } else if (tableOrName) { + table = tableOrName + } else { + table = this.table + } + + let name = table.name + if (isSqs(table) && table._id) { + // SQS uses the table ID rather than the table name + name = table._id + } + const aliases = this.query.tableAliases || {} + return aliases[name] ? aliases[name] : name + } + addDistinctCount(query: Knex.QueryBuilder): Knex.QueryBuilder { - const primary = this.table.primary - const aliases = this.query.tableAliases - const aliased = - this.table.name && aliases?.[this.table.name] - ? aliases[this.table.name] - : this.table.name - if (!primary) { + if (!this.table.primary) { throw new Error("SQL counting requires primary key to be supplied") } - return query.countDistinct(`${aliased}.${primary[0]} as total`) + return query.countDistinct( + `${this.getTableName()}.${this.table.primary[0]} as ${COUNT_FIELD_NAME}` + ) + } + + addAggregations( + query: Knex.QueryBuilder, + aggregations: Aggregation[] + ): Knex.QueryBuilder { + const fields = this.query.resource?.fields || [] + const tableName = this.getTableName() + if (fields.length > 0) { + query = query.groupBy(fields.map(field => `${tableName}.${field}`)) + query = query.select(fields.map(field => `${tableName}.${field}`)) + } + for (const aggregation of aggregations) { + const op = aggregation.calculationType + const field = `${tableName}.${aggregation.field} as ${aggregation.name}` + switch (op) { + case CalculationType.COUNT: + query = query.count(field) + break + case CalculationType.SUM: + query = query.sum(field) + break + case CalculationType.AVG: + query = query.avg(field) + break + case CalculationType.MIN: + query = query.min(field) + break + case CalculationType.MAX: + query = query.max(field) + break + } + } + return query } addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder { - let { sort } = this.query + let { sort, resource } = this.query const primaryKey = this.table.primary - const tableName = getTableName(this.table) - const aliases = this.query.tableAliases - const aliased = - tableName && aliases?.[tableName] ? aliases[tableName] : this.table?.name + const aliased = this.getTableName() if (!Array.isArray(primaryKey)) { throw new Error("Sorting requires primary key to be specified for table") } @@ -862,7 +921,8 @@ class InternalBuilder { // add sorting by the primary key if the result isn't already sorted by it, // to make sure result is deterministic - if (!sort || sort[primaryKey[0]] === undefined) { + const hasAggregations = (resource?.aggregations?.length ?? 0) > 0 + if (!hasAggregations && (!sort || sort[primaryKey[0]] === undefined)) { query = query.orderBy(`${aliased}.${primaryKey[0]}`) } return query @@ -1246,10 +1306,15 @@ class InternalBuilder { } } - // if counting, use distinct count, else select - query = !counting - ? query.select(this.generateSelectStatement()) - : this.addDistinctCount(query) + const aggregations = this.query.resource?.aggregations || [] + if (counting) { + query = this.addDistinctCount(query) + } else if (aggregations.length > 0) { + query = this.addAggregations(query, aggregations) + } else { + query = query.select(this.generateSelectStatement()) + } + // have to add after as well (this breaks MS-SQL) if (!counting) { query = this.addSorting(query) @@ -1468,23 +1533,40 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return results.length ? results : [{ [operation.toLowerCase()]: true }] } + private getTableName( + table: Table, + aliases?: Record + ): string | undefined { + let name = table.name + if ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) { + if (!table._id) { + return + } + // SQS uses the table ID rather than the table name + name = table._id + } + return aliases?.[name] || name + } + convertJsonStringColumns>( table: Table, results: T[], aliases?: Record ): T[] { - const tableName = getTableName(table) + const tableName = this.getTableName(table, aliases) for (const [name, field] of Object.entries(table.schema)) { if (!this._isJsonColumn(field)) { continue } - const aliasedTableName = (tableName && aliases?.[tableName]) || tableName - const fullName = `${aliasedTableName}.${name}` + const fullName = `${tableName}.${name}` as keyof T for (let row of results) { - if (typeof row[fullName as keyof T] === "string") { - row[fullName as keyof T] = JSON.parse(row[fullName]) + if (typeof row[fullName] === "string") { + row[fullName] = JSON.parse(row[fullName]) } - if (typeof row[name as keyof T] === "string") { + if (typeof row[name] === "string") { row[name as keyof T] = JSON.parse(row[name]) } } diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 0c994d8287..d8546afa8b 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -17,11 +17,8 @@ import { ContextUser, CouchFindOptions, DatabaseQueryOpts, - SearchFilters, SearchUsersRequest, User, - BasicOperator, - ArrayOperator, } from "@budibase/types" import * as context from "../context" import { getGlobalDB } from "../context" @@ -45,32 +42,6 @@ function removeUserPassword(users: User | User[]) { return users } -export function isSupportedUserSearch(query: SearchFilters) { - const allowed = [ - { op: BasicOperator.STRING, key: "email" }, - { op: BasicOperator.EQUAL, key: "_id" }, - { op: ArrayOperator.ONE_OF, key: "_id" }, - ] - for (let [key, operation] of Object.entries(query)) { - if (typeof operation !== "object") { - return false - } - const fields = Object.keys(operation || {}) - // this filter doesn't contain options - ignore - if (fields.length === 0) { - continue - } - const allowedOperation = allowed.find( - allow => - allow.op === key && fields.length === 1 && fields[0] === allow.key - ) - if (!allowedOperation) { - return false - } - } - return true -} - export async function bulkGetGlobalUsersById( userIds: string[], opts?: GetOpts diff --git a/packages/builder/assets/MagicWand.svelte b/packages/builder/assets/MagicWand.svelte new file mode 100644 index 0000000000..14a06ebaec --- /dev/null +++ b/packages/builder/assets/MagicWand.svelte @@ -0,0 +1,29 @@ + + + + + + + + diff --git a/packages/builder/package.json b/packages/builder/package.json index f9e6becbab..aec0b509f0 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -67,6 +67,7 @@ "@spectrum-css/vars": "^3.0.1", "@zerodevx/svelte-json-view": "^1.0.7", "codemirror": "^5.65.16", + "cron-parser": "^4.9.0", "dayjs": "^1.10.8", "downloadjs": "1.4.7", "fast-json-patch": "^3.1.1", diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index aceb980786..bc65c234e9 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -641,6 +641,8 @@ let hasUserDefinedName = automation.stepNames?.[allSteps[idx]?.id] if (isLoopBlock) { runtimeName = `loop.${name}` + } else if (idx === 0) { + runtimeName = `trigger.${name}` } else if (block.name.startsWith("JS")) { runtimeName = hasUserDefinedName ? `stepsByName["${bindingName}"].${name}` @@ -650,7 +652,7 @@ ? `stepsByName.${bindingName}.${name}` : `steps.${idx - loopBlockCount}.${name}` } - return idx === 0 ? `trigger.${name}` : runtimeName + return runtimeName } const determineCategoryName = (idx, isLoopBlock, bindingName) => { @@ -677,7 +679,7 @@ ) return { readableBinding: - bindingName && !isLoopBlock + bindingName && !isLoopBlock && idx !== 0 ? `steps.${bindingName}.${name}` : runtimeBinding, runtimeBinding, @@ -1048,7 +1050,7 @@ {:else if value.customType === "cron"} onChange({ [key]: e.detail })} - value={inputData[key]} + cronExpression={inputData[key]} /> {:else if value.customType === "automationFields"} - import { Button, Select, Input, Label } from "@budibase/bbui" + import { + Select, + InlineAlert, + Input, + Label, + Layout, + notifications, + } from "@budibase/bbui" import { onMount, createEventDispatcher } from "svelte" import { flags } from "stores/builder" + import { licensing } from "stores/portal" + import { API } from "api" + import MagicWand from "../../../../assets/MagicWand.svelte" + import { helpers, REBOOT_CRON } from "@budibase/shared-core" const dispatch = createEventDispatcher() - export let value + export let cronExpression + let error + let nextExecutions + // AI prompt + let aiCronPrompt = "" + let loadingAICronExpression = false + + $: aiEnabled = + $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled $: { - const exists = CRON_EXPRESSIONS.some(cron => cron.value === value) - const customIndex = CRON_EXPRESSIONS.findIndex( - cron => cron.label === "Custom" - ) - - if (!exists && customIndex === -1) { - CRON_EXPRESSIONS[0] = { label: "Custom", value: value } - } else if (exists && customIndex !== -1) { - CRON_EXPRESSIONS.splice(customIndex, 1) + if (cronExpression) { + try { + nextExecutions = helpers.cron + .getNextExecutionDates(cronExpression) + .join("\n") + } catch (err) { + nextExecutions = null + } } } const onChange = e => { - if (value !== REBOOT_CRON) { + if (e.detail !== REBOOT_CRON) { error = helpers.cron.validate(e.detail).err } - if (e.detail === value || error) { + if (e.detail === cronExpression || error) { return } - value = e.detail + cronExpression = e.detail dispatch("change", e.detail) } + const updatePreset = e => { + aiCronPrompt = "" + onChange(e) + } + + const updateCronExpression = e => { + aiCronPrompt = "" + cronExpression = null + nextExecutions = null + onChange(e) + } + let touched = false - let presets = false const CRON_EXPRESSIONS = [ { @@ -64,45 +93,130 @@ }) } }) + + async function generateAICronExpression() { + loadingAICronExpression = true + try { + const response = await API.generateCronExpression({ + prompt: aiCronPrompt, + }) + cronExpression = response.message + dispatch("change", response.message) + } catch (err) { + notifications.error(err.message) + } finally { + loadingAICronExpression = false + } + } -
+ + + + + {#if aiCronPrompt} +
+ +
+ {/if} +
+ {/if} (touched = true)} updateOnChange={false} /> - {#if touched && !value} + {#if touched && !cronExpression} {/if} -
- - {#if presets} -