diff --git a/.tool-versions b/.tool-versions index 946d5198ce..cf78481d93 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ nodejs 20.10.0 python 3.10.0 -yarn 1.22.19 +yarn 1.22.22 diff --git a/lerna.json b/lerna.json index 5a279b3e44..092e9a133e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.9", + "version": "2.32.10", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/builder/assets/MagicWand.svelte b/packages/builder/assets/MagicWand.svelte new file mode 100644 index 0000000000..14a06ebaec --- /dev/null +++ b/packages/builder/assets/MagicWand.svelte @@ -0,0 +1,29 @@ + + + + + + + + diff --git a/packages/builder/package.json b/packages/builder/package.json index f9e6becbab..aec0b509f0 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -67,6 +67,7 @@ "@spectrum-css/vars": "^3.0.1", "@zerodevx/svelte-json-view": "^1.0.7", "codemirror": "^5.65.16", + "cron-parser": "^4.9.0", "dayjs": "^1.10.8", "downloadjs": "1.4.7", "fast-json-patch": "^3.1.1", diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index f6f5308f27..9dddaad578 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -871,7 +871,7 @@ {:else if value.customType === "cron"} onChange({ [key]: e.detail })} - value={inputData[key]} + cronExpression={inputData[key]} /> {:else if value.customType === "automationFields"} - import { Button, Select, Input, Label } from "@budibase/bbui" + import { + Select, + InlineAlert, + Input, + Label, + Layout, + notifications, + } from "@budibase/bbui" import { onMount, createEventDispatcher } from "svelte" import { flags } from "stores/builder" + import { licensing } from "stores/portal" + import { API } from "api" + import MagicWand from "../../../../assets/MagicWand.svelte" + import { helpers, REBOOT_CRON } from "@budibase/shared-core" const dispatch = createEventDispatcher() - export let value + export let cronExpression + let error + let nextExecutions + // AI prompt + let aiCronPrompt = "" + let loadingAICronExpression = false + + $: aiEnabled = + $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled $: { - const exists = CRON_EXPRESSIONS.some(cron => cron.value === value) - const customIndex = CRON_EXPRESSIONS.findIndex( - cron => cron.label === "Custom" - ) - - if (!exists && customIndex === -1) { - CRON_EXPRESSIONS[0] = { label: "Custom", value: value } - } else if (exists && customIndex !== -1) { - CRON_EXPRESSIONS.splice(customIndex, 1) + if (cronExpression) { + try { + nextExecutions = helpers.cron + .getNextExecutionDates(cronExpression) + .join("\n") + } catch (err) { + nextExecutions = null + } } } const onChange = e => { - if (value !== REBOOT_CRON) { + if (e.detail !== REBOOT_CRON) { error = helpers.cron.validate(e.detail).err } - if (e.detail === value || error) { + if (e.detail === cronExpression || error) { return } - value = e.detail + cronExpression = e.detail dispatch("change", e.detail) } + const updatePreset = e => { + aiCronPrompt = "" + onChange(e) + } + + const updateCronExpression = e => { + aiCronPrompt = "" + cronExpression = null + nextExecutions = null + onChange(e) + } + let touched = false - let presets = false const CRON_EXPRESSIONS = [ { @@ -64,45 +93,130 @@ }) } }) + + async function generateAICronExpression() { + loadingAICronExpression = true + try { + const response = await API.generateCronExpression({ + prompt: aiCronPrompt, + }) + cronExpression = response.message + dispatch("change", response.message) + } catch (err) { + notifications.error(err.message) + } finally { + loadingAICronExpression = false + } + } - + + + + + {#if aiEnabled} + + + {#if aiCronPrompt} + + + + {/if} + + {/if} (touched = true)} updateOnChange={false} /> - {#if touched && !value} + {#if touched && !cronExpression} Please specify a CRON expression {/if} - - (presets = !presets)} - >{presets ? "Hide" : "Show"} Presets - {#if presets} - - {/if} - - + {#if nextExecutions} + + {/if} + diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 0130c39715..a1bd54715b 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -21,6 +21,7 @@ PROTECTED_EXTERNAL_COLUMNS, canHaveDefaultColumn, } from "@budibase/shared-core" + import { makePropSafe } from "@budibase/string-templates" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/builder" @@ -46,6 +47,7 @@ import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" import OptionsEditor from "./OptionsEditor.svelte" import { isEnabled } from "helpers/featureFlags" + import { getUserBindings } from "dataBinding" const AUTO_TYPE = FieldType.AUTO const FORMULA_TYPE = FieldType.FORMULA @@ -191,6 +193,19 @@ fieldId: makeFieldId(t.type, t.subtype), ...t, })) + $: defaultValueBindings = [ + { + type: "context", + runtimeBinding: `${makePropSafe("now")}`, + readableBinding: `Date`, + category: "Date", + icon: "Date", + display: { + name: "Server date", + }, + }, + ...getUserBindings(), + ] const fieldDefinitions = Object.values(FIELDS).reduce( // Storing the fields by complex field id @@ -781,9 +796,8 @@ setRequired(false) } }} - bindings={getBindings({ table })} + bindings={defaultValueBindings} allowJS - context={rowGoldenSample} /> {/if} diff --git a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte index 2040f66706..d031e752cd 100644 --- a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte @@ -2,7 +2,12 @@ import { getContext } from "svelte" import CreateEditColumn from "components/backend/DataTable/modals/CreateEditColumn.svelte" - const { datasource } = getContext("grid") + const { datasource, rows } = getContext("grid") + + const onUpdate = async () => { + await datasource.actions.refreshDefinition() + await rows.actions.refreshData() + } - + diff --git a/packages/frontend-core/src/api/ai.js b/packages/frontend-core/src/api/ai.js new file mode 100644 index 0000000000..7fa756a19e --- /dev/null +++ b/packages/frontend-core/src/api/ai.js @@ -0,0 +1,11 @@ +export const buildAIEndpoints = API => ({ + /** + * Generates a cron expression from a prompt + */ + generateCronExpression: async ({ prompt }) => { + return await API.post({ + url: "/api/ai/cron", + body: { prompt }, + }) + }, +}) diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js index 9558ca36cb..1ca8b7c3f4 100644 --- a/packages/frontend-core/src/api/index.js +++ b/packages/frontend-core/src/api/index.js @@ -2,6 +2,7 @@ import { Helpers } from "@budibase/bbui" import { Header } from "@budibase/shared-core" import { ApiVersion } from "../constants" import { buildAnalyticsEndpoints } from "./analytics" +import { buildAIEndpoints } from "./ai" import { buildAppEndpoints } from "./app" import { buildAttachmentEndpoints } from "./attachments" import { buildAuthEndpoints } from "./auth" @@ -269,6 +270,7 @@ export const createAPIClient = config => { // Attach all endpoints return { ...API, + ...buildAIEndpoints(API), ...buildAnalyticsEndpoints(API), ...buildAppEndpoints(API), ...buildAttachmentEndpoints(API), diff --git a/packages/pro b/packages/pro index e2fe0f9cc8..aca9828117 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit e2fe0f9cc856b4ee1a97df96d623b2d87d4e8733 +Subproject commit aca9828117bb97f54f40ee359f1a3f6e259174e7 diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 334e897a0e..f804c650f9 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -3,15 +3,9 @@ import { ViewV2, SearchRowResponse, SearchViewRowRequest, - SearchFilterKey, - LogicalOperator, - SearchFilter, } from "@budibase/types" -import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" -import { db, context, features } from "@budibase/backend-core" -import { enrichSearchContext } from "./utils" -import { isExternalTableID } from "../../../integrations/utils" +import { context } from "@budibase/backend-core" export async function searchView( ctx: UserCtx @@ -28,70 +22,23 @@ export async function searchView( const { body } = ctx.request - const sqsEnabled = await features.flags.isEnabled("SQS") - const supportsLogicalOperators = isExternalTableID(view.tableId) || sqsEnabled - - // Enrich saved query with ephemeral query params. - // We prevent searching on any fields that are saved as part of the query, as - // that could let users find rows they should not be allowed to access. - let query = supportsLogicalOperators - ? dataFilters.buildQuery(view.query) - : dataFilters.buildQueryLegacy(view.query) - - delete query?.onEmptyFilter - - if (body.query) { - // Delete extraneous search params that cannot be overridden - delete body.query.onEmptyFilter - - if (!supportsLogicalOperators) { - // In the unlikely event that a Grouped Filter is in a non-SQS environment - // It needs to be ignored entirely - let queryFilters: SearchFilter[] = Array.isArray(view.query) - ? view.query - : [] - - // Extract existing fields - const existingFields = - queryFilters - ?.filter(filter => filter.field) - .map(filter => db.removeKeyNumbering(filter.field)) || [] - - // Carry over filters for unused fields - Object.keys(body.query).forEach(key => { - const operator = key as Exclude - Object.keys(body.query[operator] || {}).forEach(field => { - if (query && !existingFields.includes(db.removeKeyNumbering(field))) { - query[operator]![field] = body.query[operator]![field] - } - }) - }) - } else { - const conditions = query ? [query] : [] - query = { - $and: { - conditions: [...conditions, body.query], - }, - } - } - } - await context.ensureSnippetContext(true) - const enrichedQuery = await enrichSearchContext(query || {}, { - user: sdk.users.getUserContextBindings(ctx.user), - }) - - const result = await sdk.rows.search({ - viewId: view.id, - tableId: view.tableId, - query: enrichedQuery, - ...getSortOptions(body, view), - limit: body.limit, - bookmark: body.bookmark, - paginate: body.paginate, - countRows: body.countRows, - }) + const result = await sdk.rows.search( + { + viewId: view.id, + tableId: view.tableId, + query: body.query, + ...getSortOptions(body, view), + limit: body.limit, + bookmark: body.bookmark, + paginate: body.paginate, + countRows: body.countRows, + }, + { + user: sdk.users.getUserContextBindings(ctx.user), + } + ) result.rows.forEach(r => (r._viewId = view.id)) ctx.body = result diff --git a/packages/server/src/api/routes/index.ts b/packages/server/src/api/routes/index.ts index 2079eb01fd..e05bc44893 100644 --- a/packages/server/src/api/routes/index.ts +++ b/packages/server/src/api/routes/index.ts @@ -33,6 +33,7 @@ import rowActionRoutes from "./rowAction" export { default as staticRoutes } from "./static" export { default as publicRoutes } from "./public" +const aiRoutes = pro.ai const appBackupRoutes = pro.appBackups const environmentVariableRoutes = pro.environmentVariables @@ -67,6 +68,7 @@ export const mainRoutes: Router[] = [ debugRoutes, environmentVariableRoutes, rowActionRoutes, + aiRoutes, // these need to be handled last as they still use /api/:tableId // this could be breaking as koa may recognise other routes as this tableRoutes, diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index aab846e704..09273abdce 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -1738,6 +1738,40 @@ describe.each([ }) }) + it("views filters are respected even if the column is hidden", async () => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + const two = await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + query: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", + }, + ], + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: false }, + }, + }) + + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual([ + expect.objectContaining({ _id: two._id }), + ]) + }) + it("views without data can be returned", async () => { const response = await config.api.viewV2.search(view.id) expect(response.rows).toHaveLength(0) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 809bd73d1f..8de5818805 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,7 +1,10 @@ import { EmptyFilterOption, + LogicalOperator, Row, RowSearchParams, + SearchFilterKey, + SearchFilters, SearchResponse, SortOrder, Table, @@ -14,9 +17,10 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" import { searchInputMapping } from "./search/utils" -import { features } from "@budibase/backend-core" +import { db, features } from "@budibase/backend-core" import tracer from "dd-trace" import { getQueryableFields, removeInvalidFilters } from "./queryUtils" +import { enrichSearchContext } from "../../../api/controllers/row/utils" export { isValidFilter } from "../../../integrations/utils" @@ -34,7 +38,8 @@ function pickApi(tableId: any) { } export async function search( - options: RowSearchParams + options: RowSearchParams, + context?: Record ): Promise> { return await tracer.trace("search", async span => { span?.addTags({ @@ -51,7 +56,73 @@ export async function search( countRows: options.countRows, }) - options.query = dataFilters.cleanupQuery(options.query || {}) + let source: Table | ViewV2 + let table: Table + if (options.viewId) { + source = await sdk.views.get(options.viewId) + table = await sdk.views.getTable(source) + options = searchInputMapping(table, options) + } else if (options.tableId) { + source = await sdk.tables.getTable(options.tableId) + table = source + } else { + throw new Error(`Must supply either a view ID or a table ID`) + } + + const isExternalTable = isExternalTableID(table._id!) + + if (options.query) { + const visibleFields = ( + options.fields || Object.keys(table.schema) + ).filter(field => table.schema[field].visible !== false) + + const queryableFields = await getQueryableFields(table, visibleFields) + options.query = removeInvalidFilters(options.query, queryableFields) + } else { + options.query = {} + } + + if (options.viewId) { + const view = await sdk.views.get(options.viewId) + // Enrich saved query with ephemeral query params. + // We prevent searching on any fields that are saved as part of the query, as + // that could let users find rows they should not be allowed to access. + let viewQuery = dataFilters.buildQuery(view.query || []) + + if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) { + // Lucene does not accept conditional filters, so we need to keep the old logic + const query: SearchFilters = viewQuery + + // Extract existing fields + const existingFields = + view.query + ?.filter(filter => filter.field) + .map(filter => db.removeKeyNumbering(filter.field)) || [] + + // Carry over filters for unused fields + Object.keys(options.query || {}).forEach(key => { + const operator = key as Exclude + Object.keys(options.query[operator] || {}).forEach(field => { + if (!existingFields.includes(db.removeKeyNumbering(field))) { + query[operator]![field] = options.query[operator]![field] + } + }) + }) + options.query = query + } else { + options.query = { + $and: { + conditions: [viewQuery, options.query], + }, + } + } + } + + if (context) { + options.query = await enrichSearchContext(options.query, context) + } + + options.query = dataFilters.cleanupQuery(options.query) options.query = dataFilters.fixupFilterArrays(options.query) span.addTags({ @@ -72,30 +143,8 @@ export async function search( options.sortOrder = options.sortOrder.toLowerCase() as SortOrder } - let source: Table | ViewV2 - let table: Table - if (options.viewId) { - source = await sdk.views.get(options.viewId) - table = await sdk.views.getTable(source) - options = searchInputMapping(table, options) - } else if (options.tableId) { - source = await sdk.tables.getTable(options.tableId) - table = source - options = searchInputMapping(table, options) - } else { - throw new Error(`Must supply either a view ID or a table ID`) - } + options = searchInputMapping(table, options) - if (options.query) { - const visibleFields = ( - options.fields || Object.keys(table.schema) - ).filter(field => table.schema[field].visible !== false) - - const queryableFields = await getQueryableFields(table, visibleFields) - options.query = removeInvalidFilters(options.query, queryableFields) - } - - const isExternalTable = isExternalTableID(table._id!) let result: SearchResponse if (isExternalTable) { span?.addTags({ searchType: "external" }) diff --git a/packages/server/src/sdk/users/utils.ts b/packages/server/src/sdk/users/utils.ts index 0194e900fe..74389a1444 100644 --- a/packages/server/src/sdk/users/utils.ts +++ b/packages/server/src/sdk/users/utils.ts @@ -130,6 +130,26 @@ export function getUserContextBindings(user: ContextUser) { return {} } // Current user context for bindable search - const { _id, _rev, firstName, lastName, email, status, roleId } = user - return { _id, _rev, firstName, lastName, email, status, roleId } + const { + _id, + _rev, + firstName, + lastName, + email, + status, + roleId, + globalId, + userId, + } = user + return { + _id, + _rev, + firstName, + lastName, + email, + status, + roleId, + globalId, + userId, + } } diff --git a/packages/shared-core/src/helpers/cron.ts b/packages/shared-core/src/helpers/cron.ts index ca1f1badb7..15abda3483 100644 --- a/packages/shared-core/src/helpers/cron.ts +++ b/packages/shared-core/src/helpers/cron.ts @@ -1,4 +1,5 @@ import cronValidate from "cron-validate" +import cronParser from "cron-parser" const INPUT_CRON_START = "(Input cron: " const ERROR_SWAPS = { @@ -30,6 +31,19 @@ function improveErrors(errors: string[]): string[] { return finalErrors } +export function getNextExecutionDates( + cronExpression: string, + limit: number = 4 +): string[] { + const parsed = cronParser.parseExpression(cronExpression) + const nextRuns = [] + for (let i = 0; i < limit; i++) { + nextRuns.push(parsed.next().toString()) + } + + return nextRuns +} + export function validate( cronExpression: string ): { valid: false; err: string[] } | { valid: true } { diff --git a/yarn.lock b/yarn.lock index 46ac61d119..0cddbf981f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -817,23 +817,23 @@ tslib "^2.2.0" "@azure/msal-browser@^3.11.1": - version "3.23.0" - resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.23.0.tgz#446aaf268247e5943f464f007d3aa3a04abfe95b" - integrity sha512-+QgdMvaeEpdtgRTD7AHHq9aw8uga7mXVHV1KshO1RQ2uI5B55xJ4aEpGlg/ga3H+0arEVcRfT4ZVmX7QLXiCVw== + version "3.24.0" + resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.24.0.tgz#3208047672d0b0c943b0bef5f995d510d6582ae4" + integrity sha512-JGNV9hTYAa7lsum9IMIibn2kKczAojNihGo1hi7pG0kNrcKej530Fl6jxwM05A44/6I079CSn6WxYxbVhKUmWg== dependencies: - "@azure/msal-common" "14.14.2" + "@azure/msal-common" "14.15.0" -"@azure/msal-common@14.14.2": - version "14.14.2" - resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.14.2.tgz#583b4ac9c089953718d7a5e2f3b8df2d4dbb17f4" - integrity sha512-XV0P5kSNwDwCA/SjIxTe9mEAsKB0NqGNSuaVrkCCE2lAyBr/D6YtD80Vkdp4tjWnPFwjzkwldjr1xU/facOJog== +"@azure/msal-common@14.15.0": + version "14.15.0" + resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.15.0.tgz#0e27ac0bb88fe100f4f8d1605b64d5c268636a55" + integrity sha512-ImAQHxmpMneJ/4S8BRFhjt1MZ3bppmpRPYYNyzeQPeFN288YKbb8TmmISQEbtfkQ1BPASvYZU5doIZOPBAqENQ== "@azure/msal-node@^2.9.2": - version "2.13.1" - resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.13.1.tgz#f144371275b7c3cbe564762b84772a9732457a47" - integrity sha512-sijfzPNorKt6+9g1/miHwhj6Iapff4mPQx1azmmZExgzUROqWTM1o3ACyxDja0g47VpowFy/sxTM/WsuCyXTiw== + version "2.14.0" + resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.14.0.tgz#7881895d41b03d8b9b38a29550ba3bbb15f73b3c" + integrity sha512-rrfzIpG3Q1rHjVYZmHAEDidWAZZ2cgkxlIcMQ8dHebRISaZ2KCV33Q8Vs+uaV6lxweROabNxKFlR2lIKagZqYg== dependencies: - "@azure/msal-common" "14.14.2" + "@azure/msal-common" "14.15.0" jsonwebtoken "^9.0.0" uuid "^8.3.0" @@ -9167,6 +9167,13 @@ cron-parser@^4.2.1: dependencies: luxon "^3.2.1" +cron-parser@^4.9.0: + version "4.9.0" + resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.9.0.tgz#0340694af3e46a0894978c6f52a6dbb5c0f11ad5" + integrity sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q== + dependencies: + luxon "^3.2.1" + cron-validate@1.4.5: version "1.4.5" resolved "https://registry.yarnpkg.com/cron-validate/-/cron-validate-1.4.5.tgz#eceb221f7558e6302e5f84c7b3a454fdf4d064c3"