diff --git a/packages/backend-core/src/security/permissions.ts b/packages/backend-core/src/security/permissions.ts index 4ed2cd3954..929ae92909 100644 --- a/packages/backend-core/src/security/permissions.ts +++ b/packages/backend-core/src/security/permissions.ts @@ -1,4 +1,8 @@ -import { PermissionLevel, PermissionType } from "@budibase/types" +import { + PermissionLevel, + PermissionType, + BuiltinPermissionID, +} from "@budibase/types" import flatten from "lodash/flatten" import cloneDeep from "lodash/fp/cloneDeep" @@ -57,14 +61,6 @@ export function getAllowedLevels(userPermLevel: PermissionLevel): string[] { } } -export enum BuiltinPermissionID { - PUBLIC = "public", - READ_ONLY = "read_only", - WRITE = "write", - ADMIN = "admin", - POWER = "power", -} - export const BUILTIN_PERMISSIONS: { [key in keyof typeof BuiltinPermissionID]: { _id: (typeof BuiltinPermissionID)[key] diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 7d9a11db48..df7f41e6ce 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -1,5 +1,4 @@ import semver from "semver" -import { BuiltinPermissionID, PermissionLevel } from "./permissions" import { prefixRoleID, getRoleParams, @@ -14,6 +13,8 @@ import { RoleUIMetadata, Database, App, + BuiltinPermissionID, + PermissionLevel, } from "@budibase/types" import cloneDeep from "lodash/fp/cloneDeep" import { RoleColor, helpers } from "@budibase/shared-core" @@ -50,7 +51,7 @@ export class Role implements RoleDoc { _id: string _rev?: string name: string - permissionId: string + permissionId: BuiltinPermissionID inherits?: string | string[] version?: string permissions: Record = {} @@ -59,7 +60,7 @@ export class Role implements RoleDoc { constructor( id: string, name: string, - permissionId: string, + permissionId: BuiltinPermissionID, uiMetadata?: RoleUIMetadata ) { this._id = id diff --git a/packages/backend-core/src/security/tests/permissions.spec.ts b/packages/backend-core/src/security/tests/permissions.spec.ts index 39348646fb..f98833c7cd 100644 --- a/packages/backend-core/src/security/tests/permissions.spec.ts +++ b/packages/backend-core/src/security/tests/permissions.spec.ts @@ -1,6 +1,7 @@ import cloneDeep from "lodash/cloneDeep" import * as permissions from "../permissions" import { BUILTIN_ROLE_IDS } from "../roles" +import { BuiltinPermissionID } from "@budibase/types" describe("levelToNumber", () => { it("should return 0 for EXECUTE", () => { @@ -77,7 +78,7 @@ describe("doesHaveBasePermission", () => { const rolesHierarchy = [ { roleId: BUILTIN_ROLE_IDS.ADMIN, - permissionId: permissions.BuiltinPermissionID.ADMIN, + permissionId: BuiltinPermissionID.ADMIN, }, ] expect( @@ -91,7 +92,7 @@ describe("doesHaveBasePermission", () => { const rolesHierarchy = [ { roleId: BUILTIN_ROLE_IDS.PUBLIC, - permissionId: permissions.BuiltinPermissionID.PUBLIC, + permissionId: BuiltinPermissionID.PUBLIC, }, ] expect( @@ -129,7 +130,7 @@ describe("getBuiltinPermissions", () => { describe("getBuiltinPermissionByID", () => { it("returns correct permission object for valid ID", () => { const expectedPermission = { - _id: permissions.BuiltinPermissionID.PUBLIC, + _id: BuiltinPermissionID.PUBLIC, name: "Public", permissions: [ new permissions.Permission( diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte index aca3e950c3..873e769e21 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte @@ -3,7 +3,7 @@ automationStore, selectedAutomation, permissions, - selectedAutomationDisplayData, + tables, } from "stores/builder" import { Icon, @@ -17,6 +17,7 @@ AbsTooltip, InlineAlert, } from "@budibase/bbui" + import { sdk } from "@budibase/shared-core" import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte" import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte" import ActionModal from "./ActionModal.svelte" @@ -51,7 +52,12 @@ $: isAppAction && setPermissions(role) $: isAppAction && getPermissions(automationId) - $: triggerInfo = $selectedAutomationDisplayData?.triggerInfo + $: triggerInfo = sdk.automations.isRowAction($selectedAutomation) && { + title: "Automation trigger", + tableName: $tables.list.find( + x => x._id === $selectedAutomation.definition.trigger.inputs?.tableId + )?.name, + } async function setPermissions(role) { if (!role || !automationId) { @@ -187,10 +193,10 @@ {block} {webhookModal} /> - {#if isTrigger && triggerInfo} + {#if triggerInfo} {/if} {#if lastStep} diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte index 50814897b4..4e9ca5fd53 100644 --- a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte +++ b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte @@ -112,7 +112,7 @@ iconColor={automation.disabled ? "var(--spectrum-global-color-gray-600)" : "var(--spectrum-global-color-gray-900)"} - text={automation.displayName} + text={automation.name} selected={automation._id === $selectedAutomation?._id} hovering={automation._id === $contextMenuStore.id} on:click={() => automationStore.actions.select(automation._id)} diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte index 6b96c4ebf5..a26efdf243 100644 --- a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte +++ b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte @@ -25,15 +25,9 @@ automation.name.toLowerCase().includes(searchString.toLowerCase()) ) }) - .map(automation => ({ - ...automation, - displayName: - $automationStore.automationDisplayData[automation._id]?.displayName || - automation.name, - })) .sort((a, b) => { - const lowerA = a.displayName.toLowerCase() - const lowerB = b.displayName.toLowerCase() + const lowerA = a.name.toLowerCase() + const lowerB = b.name.toLowerCase() return lowerA > lowerB ? 1 : -1 }) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index cbac1bf2ff..b422154290 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -798,7 +798,9 @@ break } } - return utils.processSearchFilters(filters) + return Array.isArray(filters) + ? utils.processSearchFilters(filters) + : filters } function saveFilters(key) { diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte index e616e27467..2071ab8b86 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte @@ -5,7 +5,6 @@ import { getUserBindings } from "dataBinding" import { makePropSafe } from "@budibase/string-templates" import { search } from "@budibase/frontend-core" - import { utils } from "@budibase/shared-core" import { tables } from "stores/builder" export let schema @@ -17,19 +16,16 @@ let drawer - $: localFilters = utils.processSearchFilters(filters) - + $: localFilters = filters $: schemaFields = search.getFields( $tables.list, Object.values(schema || {}), { allowLinks: true } ) - $: filterCount = localFilters?.groups?.reduce((acc, group) => { return (acc += group.filters.filter(filter => filter.field).length) }, 0) || 0 - $: bindings = [ { type: "context", @@ -61,7 +57,7 @@ title="Filtering" on:drawerHide on:drawerShow={() => { - localFilters = utils.processSearchFilters(filters) + localFilters = filters }} forceModal > diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js b/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js index a5d24d7435..4927f51a8d 100644 --- a/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js +++ b/packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js @@ -79,6 +79,8 @@ describe("Export Modal", () => { props: propsCfg, }) + expect(propsCfg.filters[0].field).toBe("1:Cost") + expect(screen.getByTestId("filters-applied")).toBeVisible() expect(screen.getByTestId("filters-applied").textContent).toBe( "Filters applied" diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte index b62c8af03d..8a5484e2b2 100644 --- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte +++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte @@ -92,6 +92,7 @@ disabled={error || !name || (rows.length && (!allValid || displayColumn == null))} + size="M" > ({ }, fetch: async () => { const [automationResponse, definitions] = await Promise.all([ - API.getAutomations({ enrich: true }), + API.getAutomations(), API.getAutomationDefinitions(), ]) store.update(state => { @@ -91,7 +90,6 @@ const automationActions = store => ({ state.automations.sort((a, b) => { return a.name < b.name ? -1 : 1 }) - state.automationDisplayData = automationResponse.builderData state.blockDefinitions = getFinalDefinitions( definitions.trigger, definitions.action @@ -153,8 +151,6 @@ const automationActions = store => ({ state.selectedAutomationId = state.automations[0]?._id || null } - // Clear out automationDisplayData for the automation - delete state.automationDisplayData[automation._id] return state }) }, @@ -432,13 +428,3 @@ export const selectedAutomation = derived(automationStore, $automationStore => { x => x._id === $automationStore.selectedAutomationId ) }) - -export const selectedAutomationDisplayData = derived( - [automationStore, selectedAutomation], - ([$automationStore, $selectedAutomation]) => { - if (!$selectedAutomation?._id) { - return null - } - return $automationStore.automationDisplayData[$selectedAutomation._id] - } -) diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js index 6c2c438f0c..faa6a086ca 100644 --- a/packages/builder/src/stores/builder/components.js +++ b/packages/builder/src/stores/builder/components.js @@ -207,7 +207,6 @@ export class ComponentStore extends BudiStore { ) for (let setting of filterableTypes || []) { const isLegacy = Array.isArray(enrichedComponent[setting.key]) - if (isLegacy) { const processedSetting = utils.processSearchFilters( enrichedComponent[setting.key] diff --git a/packages/builder/src/stores/builder/index.js b/packages/builder/src/stores/builder/index.js index dbde739951..158cd29973 100644 --- a/packages/builder/src/stores/builder/index.js +++ b/packages/builder/src/stores/builder/index.js @@ -11,7 +11,6 @@ import { automationStore, selectedAutomation, automationHistoryStore, - selectedAutomationDisplayData, } from "./automations.js" import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js" import { deploymentStore } from "./deployments.js" @@ -46,7 +45,6 @@ export { previewStore, automationStore, selectedAutomation, - selectedAutomationDisplayData, automationHistoryStore, sortedScreens, userStore, diff --git a/packages/builder/src/stores/builder/viewsV2.js b/packages/builder/src/stores/builder/viewsV2.js index 7c0a940c1b..9bd32f4a24 100644 --- a/packages/builder/src/stores/builder/viewsV2.js +++ b/packages/builder/src/stores/builder/viewsV2.js @@ -1,30 +1,6 @@ import { writable, derived, get } from "svelte/store" import { tables } from "./tables" import { API } from "api" -import { dataFilters } from "@budibase/shared-core" - -function convertToSearchFilters(view) { - // convert from SearchFilterGroup type - if (view?.query) { - return { - ...view, - queryUI: view.query, - query: dataFilters.buildQuery(view.query), - } - } - return view -} - -function convertToSearchFilterGroup(view) { - if (view?.queryUI) { - return { - ...view, - query: view.queryUI, - queryUI: undefined, - } - } - return view -} export function createViewsV2Store() { const store = writable({ @@ -36,7 +12,7 @@ export function createViewsV2Store() { const views = Object.values(table?.views || {}).filter(view => { return view.version === 2 }) - list = list.concat(views.map(view => convertToSearchFilterGroup(view))) + list = list.concat(views) }) return { ...$store, @@ -58,7 +34,6 @@ export function createViewsV2Store() { } const create = async view => { - view = convertToSearchFilters(view) const savedViewResponse = await API.viewV2.create(view) const savedView = savedViewResponse.data replaceView(savedView.id, savedView) @@ -66,7 +41,6 @@ export function createViewsV2Store() { } const save = async view => { - view = convertToSearchFilters(view) const res = await API.viewV2.update(view) const savedView = res?.data replaceView(view.id, savedView) @@ -77,7 +51,6 @@ export function createViewsV2Store() { if (!viewId) { return } - view = convertToSearchFilterGroup(view) const existingView = get(derivedStore).list.find(view => view.id === viewId) const tableIndex = get(tables).list.findIndex(table => { return table._id === view?.tableId || table._id === existingView?.tableId diff --git a/packages/frontend-core/src/api/automations.js b/packages/frontend-core/src/api/automations.js index ce5f7d3aba..37a834cf04 100644 --- a/packages/frontend-core/src/api/automations.js +++ b/packages/frontend-core/src/api/automations.js @@ -26,14 +26,9 @@ export const buildAutomationEndpoints = API => ({ /** * Gets a list of all automations. */ - getAutomations: async ({ enrich }) => { - const params = new URLSearchParams() - if (enrich) { - params.set("enrich", true) - } - + getAutomations: async () => { return await API.get({ - url: `/api/automations?${params.toString()}`, + url: "/api/automations", }) }, diff --git a/packages/frontend-core/src/components/CoreFilterBuilder.svelte b/packages/frontend-core/src/components/CoreFilterBuilder.svelte index 8a2a19b0dc..a28cffd85d 100644 --- a/packages/frontend-core/src/components/CoreFilterBuilder.svelte +++ b/packages/frontend-core/src/components/CoreFilterBuilder.svelte @@ -10,12 +10,13 @@ } from "@budibase/bbui" import { FieldType, - FilterGroupLogicalOperator, + UILogicalOperator, EmptyFilterOption, } from "@budibase/types" import { QueryUtils, Constants } from "@budibase/frontend-core" import { getContext, createEventDispatcher } from "svelte" import FilterField from "./FilterField.svelte" + import { utils } from "@budibase/shared-core" const dispatch = createEventDispatcher() const { @@ -39,8 +40,7 @@ export let toReadable export let toRuntime - $: editableFilters = filters ? Helpers.cloneDeep(filters) : null - + $: editableFilters = migrateFilters(filters) $: { if ( tables.find( @@ -54,6 +54,16 @@ } } + // We still may need to migrate this even though the backend does it automatically now + // for query definitions. This is because we might be editing saved filter definitions + // from old screens, which will still be of type LegacyFilter[]. + const migrateFilters = filters => { + if (Array.isArray(filters)) { + return utils.processSearchFilters(filters) + } + return Helpers.cloneDeep(filters) + } + const filterOperatorOptions = Object.values(FilterOperator).map(entry => { return { value: entry, label: Helpers.capitalise(entry) } }) @@ -222,7 +232,7 @@ } else if (addGroup) { if (!editable?.groups?.length) { editable = { - logicalOperator: FilterGroupLogicalOperator.ALL, + logicalOperator: UILogicalOperator.ALL, onEmptyFilter: EmptyFilterOption.RETURN_NONE, groups: [], } diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js index 4645603479..4a4a91658c 100644 --- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js +++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js @@ -1,26 +1,13 @@ import { get } from "svelte/store" -import { dataFilters } from "@budibase/shared-core" import { SortOrder } from "@budibase/types" -function convertToSearchFilters(view) { - // convert from SearchFilterGroup type - if (view?.query) { - return { - ...view, - queryUI: view.query, - query: dataFilters.buildQuery(view.query), - } - } - return view -} - const SuppressErrors = true export const createActions = context => { const { API, datasource, columns } = context const saveDefinition = async newDefinition => { - await API.viewV2.update(convertToSearchFilters(newDefinition)) + await API.viewV2.update(newDefinition) } const saveRow = async row => { @@ -139,7 +126,7 @@ export const initialise = context => { } // Only override filter state if we don't have an initial filter if (!get(initialFilter)) { - filter.set($definition.queryUI || $definition.query) + filter.set($definition.queryUI) } }) ) @@ -198,7 +185,7 @@ export const initialise = context => { if (JSON.stringify($filter) !== JSON.stringify($view.queryUI)) { await datasource.actions.saveDefinition({ ...$view, - query: $filter, + queryUI: $filter, }) // Refresh data since view definition changed diff --git a/packages/frontend-core/src/components/grid/stores/filter.js b/packages/frontend-core/src/components/grid/stores/filter.js index 6e6c37da87..e7adc356ae 100644 --- a/packages/frontend-core/src/components/grid/stores/filter.js +++ b/packages/frontend-core/src/components/grid/stores/filter.js @@ -1,5 +1,5 @@ import { get, derived } from "svelte/store" -import { FieldType, FilterGroupLogicalOperator } from "@budibase/types" +import { FieldType, UILogicalOperator } from "@budibase/types" import { memo } from "../../../utils/memo" export const createStores = context => { @@ -25,10 +25,10 @@ export const deriveStores = context => { return $filter } let allFilters = { - logicalOperator: FilterGroupLogicalOperator.ALL, + logicalOperator: UILogicalOperator.ALL, groups: [ { - logicalOperator: FilterGroupLogicalOperator.ALL, + logicalOperator: UILogicalOperator.ALL, filters: $inlineFilters, }, ], diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index b19218647b..d8bc9d6b21 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -76,16 +76,8 @@ export async function update(ctx: UserCtx) { } export async function fetch(ctx: UserCtx) { - const query: { enrich?: string } = ctx.request.query || {} - const enrich = query.enrich === "true" - const automations = await sdk.automations.fetch() ctx.body = { automations } - if (enrich) { - ctx.body.builderData = await sdk.automations.utils.getBuilderData( - automations - ) - } } export async function find(ctx: UserCtx) { diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index 1047711983..7f5ac6b92f 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -18,7 +18,7 @@ import { UserCtx, UserMetadata, DocumentType, - PermissionLevel, + BuiltinPermissionID, } from "@budibase/types" import { RoleColor, sdk as sharedSdk, helpers } from "@budibase/shared-core" import sdk from "../../sdk" @@ -134,7 +134,13 @@ export async function save(ctx: UserCtx) { } // assume write permission level for newly created roles if (isCreate && !permissionId) { - permissionId = PermissionLevel.WRITE + permissionId = BuiltinPermissionID.WRITE + } else if (!permissionId && dbRole?.permissionId) { + permissionId = dbRole.permissionId + } + + if (!permissionId) { + ctx.throw(400, "Role requires permissionId to be specified.") } const role = new roles.Role(_id, name, permissionId, { diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts index 47b6776610..729f899379 100644 --- a/packages/server/src/api/routes/tests/application.spec.ts +++ b/packages/server/src/api/routes/tests/application.spec.ts @@ -16,7 +16,7 @@ import * as setup from "./utilities" import { AppStatus } from "../../../db/utils" import { events, utils, context, features } from "@budibase/backend-core" import env from "../../../environment" -import { type App } from "@budibase/types" +import { type App, BuiltinPermissionID } from "@budibase/types" import tk from "timekeeper" import * as uuid from "uuid" import { structures } from "@budibase/backend-core/tests" @@ -80,7 +80,7 @@ describe("/applications", () => { const role = await config.api.roles.save({ name: "Test", inherits: "PUBLIC", - permissionId: "read_only", + permissionId: BuiltinPermissionID.READ_ONLY, version: "name", }) @@ -112,7 +112,7 @@ describe("/applications", () => { const role = await config.api.roles.save({ name: roleName, inherits: "PUBLIC", - permissionId: "read_only", + permissionId: BuiltinPermissionID.READ_ONLY, version: "name", }) diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts index 7a9bb2f373..193d018c24 100644 --- a/packages/server/src/api/routes/tests/permissions.spec.ts +++ b/packages/server/src/api/routes/tests/permissions.spec.ts @@ -1,5 +1,12 @@ import { roles } from "@budibase/backend-core" -import { Document, PermissionLevel, Role, Row, Table } from "@budibase/types" +import { + BuiltinPermissionID, + Document, + PermissionLevel, + Role, + Row, + Table, +} from "@budibase/types" import * as setup from "./utilities" import { generator, mocks } from "@budibase/backend-core/tests" @@ -304,7 +311,7 @@ describe("/permission", () => { role1 = await config.api.roles.save( { name: "test_1", - permissionId: PermissionLevel.WRITE, + permissionId: BuiltinPermissionID.WRITE, inherits: BUILTIN_ROLE_IDS.BASIC, }, { status: 200 } @@ -312,7 +319,7 @@ describe("/permission", () => { role2 = await config.api.roles.save( { name: "test_2", - permissionId: PermissionLevel.WRITE, + permissionId: BuiltinPermissionID.WRITE, inherits: BUILTIN_ROLE_IDS.BASIC, }, { status: 200 } @@ -345,7 +352,7 @@ describe("/permission", () => { it("should be able to fetch two tables, with different roles, using multi-inheritance", async () => { const role3 = await config.api.roles.save({ name: "role3", - permissionId: PermissionLevel.WRITE, + permissionId: BuiltinPermissionID.WRITE, inherits: [role1._id!, role2._id!], }) diff --git a/packages/server/src/api/routes/tests/role.spec.ts b/packages/server/src/api/routes/tests/role.spec.ts index adb83ca793..ee5ba7840d 100644 --- a/packages/server/src/api/routes/tests/role.spec.ts +++ b/packages/server/src/api/routes/tests/role.spec.ts @@ -1,15 +1,9 @@ -import { - roles, - events, - permissions, - db as dbCore, -} from "@budibase/backend-core" +import { roles, events, db as dbCore } from "@budibase/backend-core" import * as setup from "./utilities" -import { PermissionLevel } from "@budibase/types" +import { PermissionLevel, BuiltinPermissionID } from "@budibase/types" const { basicRole } = setup.structures const { BUILTIN_ROLE_IDS } = roles -const { BuiltinPermissionID } = permissions const LOOP_ERROR = "Role inheritance contains a loop, this is not supported" @@ -58,6 +52,19 @@ describe("/roles", () => { }) expect(res.inherits).toEqual([BUILTIN_ROLE_IDS.BASIC]) }) + + it("save role without permissionId", async () => { + const res = await config.api.roles.save( + { + ...basicRole(), + permissionId: undefined, + }, + { + status: 200, + } + ) + expect(res.permissionId).toEqual(PermissionLevel.WRITE) + }) }) describe("update", () => { @@ -149,7 +156,7 @@ describe("/roles", () => { _id: id1, name: id1, permissions: {}, - permissionId: "write", + permissionId: BuiltinPermissionID.WRITE, version: "name", inherits: ["POWER"], }) @@ -157,7 +164,7 @@ describe("/roles", () => { _id: id2, permissions: {}, name: id2, - permissionId: "write", + permissionId: BuiltinPermissionID.WRITE, version: "name", inherits: [id1], }) @@ -176,10 +183,25 @@ describe("/roles", () => { inherits: [BUILTIN_ROLE_IDS.ADMIN], }) // remove the roles so that it will default back to DB roles, then save again - delete res.inherits - const updatedRes = await config.api.roles.save(res) + const updatedRes = await config.api.roles.save({ + ...res, + inherits: undefined, + }) expect(updatedRes.inherits).toEqual([BUILTIN_ROLE_IDS.ADMIN]) }) + + it("handle updating a role, without its permissionId", async () => { + const res = await config.api.roles.save({ + ...basicRole(), + permissionId: BuiltinPermissionID.READ_ONLY, + }) + // permission ID can be removed during update + const updatedRes = await config.api.roles.save({ + ...res, + permissionId: undefined, + }) + expect(updatedRes.permissionId).toEqual(BuiltinPermissionID.READ_ONLY) + }) }) describe("fetch", () => { @@ -210,9 +232,7 @@ describe("/roles", () => { const customRoleFetched = res.find(r => r._id === customRole.name) expect(customRoleFetched).toBeDefined() expect(customRoleFetched!.inherits).toEqual(BUILTIN_ROLE_IDS.BASIC) - expect(customRoleFetched!.permissionId).toEqual( - BuiltinPermissionID.READ_ONLY - ) + expect(customRoleFetched!.permissionId).toEqual(BuiltinPermissionID.WRITE) }) it("should be able to get the role with a permission added", async () => { @@ -316,7 +336,7 @@ describe("/roles", () => { await config.api.roles.save({ name: customRoleName, inherits: roles.BUILTIN_ROLE_IDS.BASIC, - permissionId: permissions.BuiltinPermissionID.READ_ONLY, + permissionId: BuiltinPermissionID.READ_ONLY, version: "name", }) await config.withHeaders( @@ -356,19 +376,19 @@ describe("/roles", () => { const { _id: roleId1 } = await config.api.roles.save({ name: role1, inherits: roles.BUILTIN_ROLE_IDS.BASIC, - permissionId: permissions.BuiltinPermissionID.WRITE, + permissionId: BuiltinPermissionID.WRITE, version: "name", }) const { _id: roleId2 } = await config.api.roles.save({ name: role2, inherits: roles.BUILTIN_ROLE_IDS.POWER, - permissionId: permissions.BuiltinPermissionID.POWER, + permissionId: BuiltinPermissionID.POWER, version: "name", }) await config.api.roles.save({ name: role3, inherits: [roleId1!, roleId2!], - permissionId: permissions.BuiltinPermissionID.READ_ONLY, + permissionId: BuiltinPermissionID.READ_ONLY, version: "name", }) const headers = await config.roleHeaders({ diff --git a/packages/server/src/api/routes/tests/screen.spec.ts b/packages/server/src/api/routes/tests/screen.spec.ts index 894710ca27..82a550f2fd 100644 --- a/packages/server/src/api/routes/tests/screen.spec.ts +++ b/packages/server/src/api/routes/tests/screen.spec.ts @@ -1,7 +1,7 @@ import { checkBuilderEndpoint } from "./utilities/TestFunctions" import * as setup from "./utilities" import { events, roles } from "@budibase/backend-core" -import { Screen, PermissionLevel, Role } from "@budibase/types" +import { Screen, Role, BuiltinPermissionID } from "@budibase/types" const { basicScreen } = setup.structures @@ -40,17 +40,17 @@ describe("/screens", () => { role1 = await config.api.roles.save({ name: "role1", inherits: roles.BUILTIN_ROLE_IDS.BASIC, - permissionId: PermissionLevel.WRITE, + permissionId: BuiltinPermissionID.WRITE, }) role2 = await config.api.roles.save({ name: "role2", inherits: roles.BUILTIN_ROLE_IDS.BASIC, - permissionId: PermissionLevel.WRITE, + permissionId: BuiltinPermissionID.WRITE, }) multiRole = await config.api.roles.save({ name: "multiRole", inherits: [role1._id!, role2._id!], - permissionId: PermissionLevel.WRITE, + permissionId: BuiltinPermissionID.WRITE, }) screen1 = await config.api.screen.save( { diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index f0b27e232f..d915776ad1 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1688,7 +1688,7 @@ describe.each([ }) }) - describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => { + describe("arrays", () => { beforeAll(async () => { tableOrViewId = await createTableOrView({ numbers: { diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index dfd4f50bd1..2af11b513b 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -27,12 +27,20 @@ import { ViewV2Schema, ViewV2Type, JsonTypes, + EmptyFilterOption, + JsonFieldSubType, + UISearchFilter, + LegacyFilter, + SearchViewRowRequest, + ArrayOperator, + UILogicalOperator, + SearchFilters, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" import merge from "lodash/merge" import { quotas } from "@budibase/pro" -import { db, roles, features } from "@budibase/backend-core" +import { db, roles, features, context } from "@budibase/backend-core" describe.each([ ["lucene", undefined], @@ -49,7 +57,8 @@ describe.each([ const isInternal = isSqs || isLucene let table: Table - let datasource: Datasource + let rawDatasource: Datasource | undefined + let datasource: Datasource | undefined let envCleanup: (() => void) | undefined function saveTableRequest( @@ -106,8 +115,9 @@ describe.each([ }) if (dsProvider) { + rawDatasource = await dsProvider datasource = await config.createDatasource({ - datasource: await dsProvider, + datasource: rawDatasource, }) } table = await config.api.table.save(priceTable()) @@ -145,6 +155,71 @@ describe.each([ }) it("can persist views with all fields", async () => { + const newView: Required> = { + name: generator.name(), + tableId: table._id!, + primaryDisplay: "id", + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.EQUAL, + field: "field", + value: "value", + }, + ], + }, + ], + }, + sort: { + field: "fieldToSort", + order: SortOrder.DESCENDING, + type: SortType.STRING, + }, + schema: { + id: { visible: true }, + Price: { + visible: true, + }, + }, + } + const res = await config.api.viewV2.create(newView) + + const expected: ViewV2 = { + ...newView, + schema: { + id: { visible: true }, + Price: { + visible: true, + }, + }, + query: { + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { + field: "value", + }, + }, + ], + }, + }, + ], + }, + }, + id: expect.any(String), + version: 2, + } + + expect(res).toEqual(expected) + }) + + it("can create a view with just a query field, no queryUI, for backwards compatibility", async () => { const newView: Required> = { name: generator.name(), tableId: table._id!, @@ -170,7 +245,7 @@ describe.each([ } const res = await config.api.viewV2.create(newView) - expect(res).toEqual({ + const expected: ViewV2 = { ...newView, schema: { id: { visible: true }, @@ -178,9 +253,27 @@ describe.each([ visible: true, }, }, + queryUI: { + logicalOperator: UILogicalOperator.ALL, + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + groups: [ + { + logicalOperator: UILogicalOperator.ALL, + filters: [ + { + operator: BasicOperator.EQUAL, + field: "field", + value: "value", + }, + ], + }, + ], + }, id: expect.any(String), version: 2, - }) + } + + expect(res).toEqual(expected) }) it("persist only UI schema overrides", async () => { @@ -832,6 +925,7 @@ describe.each([ describe("update", () => { let view: ViewV2 + let table: Table beforeEach(async () => { table = await config.api.table.save(priceTable()) @@ -858,14 +952,37 @@ describe.each([ ], }) - expect((await config.api.table.get(tableId)).views).toEqual({ - [view.name]: { - ...view, - query: [ - { operator: "equal", field: "newField", value: "thatValue" }, + const expected: ViewV2 = { + ...view, + query: [ + { + operator: BasicOperator.EQUAL, + field: "newField", + value: "thatValue", + }, + ], + // Should also update queryUI because query was not previously set. + queryUI: { + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + logicalOperator: UILogicalOperator.ALL, + groups: [ + { + logicalOperator: UILogicalOperator.ALL, + filters: [ + { + operator: BasicOperator.EQUAL, + field: "newField", + value: "thatValue", + }, + ], + }, ], - schema: expect.anything(), }, + schema: expect.anything(), + } + + expect((await config.api.table.get(tableId)).views).toEqual({ + [view.name]: expected, }) }) @@ -883,8 +1000,8 @@ describe.each([ query: [ { operator: BasicOperator.EQUAL, - field: generator.word(), - value: generator.word(), + field: "newField", + value: "newValue", }, ], sort: { @@ -905,23 +1022,42 @@ describe.each([ } await config.api.viewV2.update(updatedData) - expect((await config.api.table.get(tableId)).views).toEqual({ - [view.name]: { - ...updatedData, - schema: { - ...table.schema, - id: expect.objectContaining({ - visible: true, - }), - Category: expect.objectContaining({ - visible: false, - }), - Price: expect.objectContaining({ - visible: true, - readonly: true, - }), - }, + const expected: ViewV2 = { + ...updatedData, + // queryUI gets generated from query + queryUI: { + logicalOperator: UILogicalOperator.ALL, + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + groups: [ + { + logicalOperator: UILogicalOperator.ALL, + filters: [ + { + operator: BasicOperator.EQUAL, + field: "newField", + value: "newValue", + }, + ], + }, + ], }, + schema: { + ...table.schema, + id: expect.objectContaining({ + visible: true, + }), + Category: expect.objectContaining({ + visible: false, + }), + Price: expect.objectContaining({ + visible: true, + readonly: true, + }), + }, + } + + expect((await config.api.table.get(tableId)).views).toEqual({ + [view.name]: expected, }) }) @@ -1153,6 +1289,146 @@ describe.each([ ) }) + it("can update queryUI field and query gets regenerated", async () => { + await config.api.viewV2.update({ + ...view, + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.EQUAL, + field: "field", + value: "value", + }, + ], + }, + ], + }, + }) + + let updatedView = await config.api.viewV2.get(view.id) + let expected: SearchFilters = { + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { field: "value" }, + }, + ], + }, + }, + ], + }, + } + expect(updatedView.query).toEqual(expected) + + await config.api.viewV2.update({ + ...updatedView, + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.EQUAL, + field: "newField", + value: "newValue", + }, + ], + }, + ], + }, + }) + + updatedView = await config.api.viewV2.get(view.id) + expected = { + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + $and: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { newField: "newValue" }, + }, + ], + }, + }, + ], + }, + } + expect(updatedView.query).toEqual(expected) + }) + + it("can delete either query and it will get regenerated from queryUI", async () => { + await config.api.viewV2.update({ + ...view, + query: [ + { + operator: BasicOperator.EQUAL, + field: "field", + value: "value", + }, + ], + }) + + let updatedView = await config.api.viewV2.get(view.id) + expect(updatedView.queryUI).toBeDefined() + + await config.api.viewV2.update({ + ...updatedView, + query: undefined, + }) + + updatedView = await config.api.viewV2.get(view.id) + expect(updatedView.query).toBeDefined() + }) + + // This is because the conversion from queryUI -> query loses data, so you + // can't accurately reproduce the original queryUI from the query. If + // query is a LegacyFilter[] we allow it, because for Budibase v3 + // everything in the db had query set to a LegacyFilter[], and there's no + // loss of information converting from a LegacyFilter[] to a + // UISearchFilter. But we convert to a SearchFilters and that can't be + // accurately converted to a UISearchFilter. + it("can't regenerate queryUI from a query once it has been generated from a queryUI", async () => { + await config.api.viewV2.update({ + ...view, + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.EQUAL, + field: "field", + value: "value", + }, + ], + }, + ], + }, + }) + + let updatedView = await config.api.viewV2.get(view.id) + expect(updatedView.query).toBeDefined() + + await config.api.viewV2.update( + { + ...updatedView, + queryUI: undefined, + }, + { + status: 400, + body: { + message: "view is missing queryUI field", + }, + } + ) + }) + !isLucene && describe("calculation views", () => { let table: Table @@ -1506,6 +1782,65 @@ describe.each([ expect.objectContaining({ visible: true, readonly: true }) ) }) + + it("should fill in the queryUI field if it's missing", async () => { + const res = await config.api.viewV2.create({ + name: generator.name(), + tableId: tableId, + query: [ + { + operator: BasicOperator.EQUAL, + field: "one", + value: "1", + }, + ], + schema: { + id: { visible: true }, + one: { visible: true }, + }, + }) + + const table = await config.api.table.get(tableId) + const rawView = table.views![res.name] as ViewV2 + delete rawView.queryUI + + await context.doInAppContext(config.getAppId(), async () => { + const db = context.getAppDB() + + if (!rawDatasource) { + await db.put(table) + } else { + const ds = await config.api.datasource.get(datasource!._id!) + ds.entities![table.name] = table + const updatedDs = { + ...rawDatasource, + _id: ds._id, + _rev: ds._rev, + entities: ds.entities, + } + await db.put(updatedDs) + } + }) + + const view = await getDelegate(res) + const expected: UISearchFilter = { + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + logicalOperator: UILogicalOperator.ALL, + groups: [ + { + logicalOperator: UILogicalOperator.ALL, + filters: [ + { + operator: BasicOperator.EQUAL, + field: "one", + value: "1", + }, + ], + }, + ], + } + expect(view.queryUI).toEqual(expected) + }) }) describe("updating table schema", () => { @@ -2301,548 +2636,385 @@ describe.each([ }) }) - describe("search", () => { - it("returns empty rows from view when no schema is passed", async () => { - const rows = await Promise.all( - Array.from({ length: 10 }, () => config.api.row.save(table._id!, {})) - ) - const response = await config.api.viewV2.search(view.id) - expect(response.rows).toHaveLength(10) - expect(response).toEqual({ - rows: expect.arrayContaining( - rows.map(r => ({ - _viewId: view.id, - tableId: table._id, - id: r.id, - _id: r._id, - _rev: r._rev, - ...(isInternal - ? { - type: "row", - updatedAt: expect.any(String), - createdAt: expect.any(String), - } - : {}), - })) - ), - ...(isInternal - ? {} - : { - hasNextPage: false, - }), - }) - }) - - it("searching respects the view filters", async () => { - await config.api.row.save(table._id!, { - one: "foo", - two: "bar", - }) - const two = await config.api.row.save(table._id!, { - one: "foo2", - two: "bar2", - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - query: [ - { - operator: BasicOperator.EQUAL, - field: "two", - value: "bar2", - }, - ], - schema: { - id: { visible: true }, - one: { visible: false }, - two: { visible: true }, - }, - }) - - const response = await config.api.viewV2.search(view.id) - expect(response.rows).toHaveLength(1) - expect(response).toEqual({ - rows: expect.arrayContaining([ - { - _viewId: view.id, - tableId: table._id, - id: two.id, - two: two.two, - _id: two._id, - _rev: two._rev, - ...(isInternal - ? { - type: "row", - createdAt: expect.any(String), - updatedAt: expect.any(String), - } - : {}), - }, - ]), - ...(isInternal - ? {} - : { - hasNextPage: false, - }), - }) - }) - - it("views filters are respected even if the column is hidden", async () => { - await config.api.row.save(table._id!, { - one: "foo", - two: "bar", - }) - const two = await config.api.row.save(table._id!, { - one: "foo2", - two: "bar2", - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - query: [ - { - operator: BasicOperator.EQUAL, - field: "two", - value: "bar2", - }, - ], - schema: { - id: { visible: true }, - one: { visible: false }, - two: { visible: false }, - }, - }) - - const response = await config.api.viewV2.search(view.id) - expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual([ - expect.objectContaining({ _id: two._id }), - ]) - }) - - it("views without data can be returned", async () => { - const response = await config.api.viewV2.search(view.id) - expect(response.rows).toHaveLength(0) - }) - - it("respects the limit parameter", async () => { - await Promise.all( - Array.from({ length: 10 }, () => config.api.row.save(table._id!, {})) - ) - const limit = generator.integer({ min: 1, max: 8 }) - const response = await config.api.viewV2.search(view.id, { - limit, - query: {}, - }) - expect(response.rows).toHaveLength(limit) - }) - - it("can handle pagination", async () => { - await Promise.all( - Array.from({ length: 10 }, () => config.api.row.save(table._id!, {})) - ) - const rows = (await config.api.viewV2.search(view.id)).rows - - const page1 = await config.api.viewV2.search(view.id, { - paginate: true, - limit: 4, - query: {}, - countRows: true, - }) - expect(page1).toEqual({ - rows: expect.arrayContaining(rows.slice(0, 4)), - hasNextPage: true, - bookmark: expect.anything(), - totalRows: 10, - }) - - const page2 = await config.api.viewV2.search(view.id, { - paginate: true, - limit: 4, - bookmark: page1.bookmark, - query: {}, - countRows: true, - }) - expect(page2).toEqual({ - rows: expect.arrayContaining(rows.slice(4, 8)), - hasNextPage: true, - bookmark: expect.anything(), - totalRows: 10, - }) - - const page3 = await config.api.viewV2.search(view.id, { - paginate: true, - limit: 4, - bookmark: page2.bookmark, - query: {}, - countRows: true, - }) - const expectation: SearchResponse = { - rows: expect.arrayContaining(rows.slice(8)), - hasNextPage: false, - totalRows: 10, - } - if (isLucene) { - expectation.bookmark = expect.anything() - } - expect(page3).toEqual(expectation) - }) - - const sortTestOptions: [ - { - field: string - order?: SortOrder - type?: SortType - }, - string[] - ][] = [ - [ - { - field: "name", - order: SortOrder.ASCENDING, - type: SortType.STRING, - }, - ["Alice", "Bob", "Charly", "Danny"], - ], - [ - { - field: "name", - }, - ["Alice", "Bob", "Charly", "Danny"], - ], - [ - { - field: "name", - order: SortOrder.DESCENDING, - }, - ["Danny", "Charly", "Bob", "Alice"], - ], - [ - { - field: "name", - order: SortOrder.DESCENDING, - type: SortType.STRING, - }, - ["Danny", "Charly", "Bob", "Alice"], - ], - [ - { - field: "age", - order: SortOrder.ASCENDING, - type: SortType.NUMBER, - }, - ["Danny", "Alice", "Charly", "Bob"], - ], - [ - { - field: "age", - order: SortOrder.ASCENDING, - }, - ["Danny", "Alice", "Charly", "Bob"], - ], - [ - { - field: "age", - order: SortOrder.DESCENDING, - }, - ["Bob", "Charly", "Alice", "Danny"], - ], - [ - { - field: "age", - order: SortOrder.DESCENDING, - type: SortType.NUMBER, - }, - ["Bob", "Charly", "Alice", "Danny"], - ], - ] - - describe("sorting", () => { - let table: Table - const viewSchema = { - id: { visible: true }, - age: { visible: true }, - name: { visible: true }, - } - - beforeAll(async () => { - table = await config.api.table.save( - saveTableRequest({ - type: "table", - schema: { - name: { - type: FieldType.STRING, - name: "name", - }, - surname: { - type: FieldType.STRING, - name: "surname", - }, - age: { - type: FieldType.NUMBER, - name: "age", - }, - address: { - type: FieldType.STRING, - name: "address", - }, - jobTitle: { - type: FieldType.STRING, - name: "jobTitle", - }, - }, - }) + !isLucene && + describe("search", () => { + it("returns empty rows from view when no schema is passed", async () => { + const rows = await Promise.all( + Array.from({ length: 10 }, () => + config.api.row.save(table._id!, {}) + ) ) - - const users = [ - { name: "Alice", age: 25 }, - { name: "Bob", age: 30 }, - { name: "Charly", age: 27 }, - { name: "Danny", age: 15 }, - ] - await Promise.all( - users.map(u => - config.api.row.save(table._id!, { + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toHaveLength(10) + expect(response).toEqual({ + rows: expect.arrayContaining( + rows.map(r => ({ + _viewId: view.id, tableId: table._id, - ...u, - }) - ) - ) + id: r.id, + _id: r._id, + _rev: r._rev, + ...(isInternal + ? { + type: "row", + updatedAt: expect.any(String), + createdAt: expect.any(String), + } + : {}), + })) + ), + ...(isInternal + ? {} + : { + hasNextPage: false, + }), + }) }) - it.each(sortTestOptions)( - "allow sorting (%s)", - async (sortParams, expected) => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - sort: sortParams, - schema: viewSchema, - }) + it("searching respects the view filters", async () => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + const two = await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) - const response = await config.api.viewV2.search(view.id) - - expect(response.rows).toHaveLength(4) - expect(response.rows).toEqual( - expected.map(name => expect.objectContaining({ name })) - ) - } - ) - - it.each(sortTestOptions)( - "allow override the default view sorting (%s)", - async (sortParams, expected) => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - sort: { - field: "name", - order: SortOrder.ASCENDING, - type: SortType.STRING, - }, - schema: viewSchema, - }) - - const response = await config.api.viewV2.search(view.id, { - sort: sortParams.field, - sortOrder: sortParams.order, - sortType: sortParams.type, - query: {}, - }) - - expect(response.rows).toHaveLength(4) - expect(response.rows).toEqual( - expected.map(name => expect.objectContaining({ name })) - ) - } - ) - }) - - it("can query on top of the view filters", async () => { - await config.api.row.save(table._id!, { - one: "foo", - two: "bar", - }) - await config.api.row.save(table._id!, { - one: "foo2", - two: "bar2", - }) - const three = await config.api.row.save(table._id!, { - one: "foo3", - two: "bar3", - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - query: [ - { - operator: BasicOperator.NOT_EQUAL, - field: "one", - value: "foo2", - }, - ], - schema: { - id: { visible: true }, - one: { visible: true }, - two: { visible: true }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: { - [BasicOperator.EQUAL]: { - two: "bar3", - }, - [BasicOperator.NOT_EMPTY]: { - two: null, - }, - }, - }) - expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual( - expect.arrayContaining([expect.objectContaining({ _id: three._id })]) - ) - }) - - it("can query on top of the view filters (using or filters)", async () => { - const one = await config.api.row.save(table._id!, { - one: "foo", - two: "bar", - }) - await config.api.row.save(table._id!, { - one: "foo2", - two: "bar2", - }) - const three = await config.api.row.save(table._id!, { - one: "foo3", - two: "bar3", - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - query: [ - { - operator: BasicOperator.NOT_EQUAL, - field: "two", - value: "bar2", - }, - ], - schema: { - id: { visible: true }, - one: { visible: false }, - two: { visible: true }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: { - allOr: true, - [BasicOperator.NOT_EQUAL]: { - two: "bar", - }, - [BasicOperator.NOT_EMPTY]: { - two: null, - }, - }, - }) - expect(response.rows).toHaveLength(2) - expect(response.rows).toEqual( - expect.arrayContaining([ - expect.objectContaining({ _id: one._id }), - expect.objectContaining({ _id: three._id }), - ]) - ) - }) - - isLucene && - it.each([true, false])( - "in lucene, cannot override a view filter", - async allOr => { - await config.api.row.save(table._id!, { - one: "foo", - two: "bar", - }) - const two = await config.api.row.save(table._id!, { - one: "foo2", - two: "bar2", - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - query: [ + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + queryUI: { + groups: [ { - operator: BasicOperator.EQUAL, - field: "two", - value: "bar2", + filters: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", + }, + ], }, ], - schema: { - id: { visible: true }, - one: { visible: false }, - two: { visible: true }, - }, - }) + }, + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: true }, + }, + }) - const response = await config.api.viewV2.search(view.id, { - query: { - allOr, - equal: { - two: "bar", + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toHaveLength(1) + expect(response).toEqual({ + rows: expect.arrayContaining([ + { + _viewId: view.id, + tableId: table._id, + id: two.id, + two: two.two, + _id: two._id, + _rev: two._rev, + ...(isInternal + ? { + type: "row", + createdAt: expect.any(String), + updatedAt: expect.any(String), + } + : {}), + }, + ]), + ...(isInternal + ? {} + : { + hasNextPage: false, + }), + }) + }) + + it("views filters are respected even if the column is hidden", async () => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + const two = await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", + }, + ], }, - }, - }) - expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual([ - expect.objectContaining({ _id: two._id }), - ]) + ], + }, + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: false }, + }, + }) + + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual([ + expect.objectContaining({ _id: two._id }), + ]) + }) + + it("views without data can be returned", async () => { + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toHaveLength(0) + }) + + it("respects the limit parameter", async () => { + await Promise.all( + Array.from({ length: 10 }, () => + config.api.row.save(table._id!, {}) + ) + ) + const limit = generator.integer({ min: 1, max: 8 }) + const response = await config.api.viewV2.search(view.id, { + limit, + query: {}, + }) + expect(response.rows).toHaveLength(limit) + }) + + it("can handle pagination", async () => { + await Promise.all( + Array.from({ length: 10 }, () => + config.api.row.save(table._id!, {}) + ) + ) + const rows = (await config.api.viewV2.search(view.id)).rows + + const page1 = await config.api.viewV2.search(view.id, { + paginate: true, + limit: 4, + query: {}, + countRows: true, + }) + expect(page1).toEqual({ + rows: expect.arrayContaining(rows.slice(0, 4)), + hasNextPage: true, + bookmark: expect.anything(), + totalRows: 10, + }) + + const page2 = await config.api.viewV2.search(view.id, { + paginate: true, + limit: 4, + bookmark: page1.bookmark, + query: {}, + countRows: true, + }) + expect(page2).toEqual({ + rows: expect.arrayContaining(rows.slice(4, 8)), + hasNextPage: true, + bookmark: expect.anything(), + totalRows: 10, + }) + + const page3 = await config.api.viewV2.search(view.id, { + paginate: true, + limit: 4, + bookmark: page2.bookmark, + query: {}, + countRows: true, + }) + const expectation: SearchResponse = { + rows: expect.arrayContaining(rows.slice(8)), + hasNextPage: false, + totalRows: 10, } - ) + if (isLucene) { + expectation.bookmark = expect.anything() + } + expect(page3).toEqual(expectation) + }) - !isLucene && - it.each([true, false])( - "can filter a view without a view filter", - async allOr => { - const one = await config.api.row.save(table._id!, { - one: "foo", - two: "bar", - }) - await config.api.row.save(table._id!, { - one: "foo2", - two: "bar2", - }) + const sortTestOptions: [ + { + field: string + order?: SortOrder + type?: SortType + }, + string[] + ][] = [ + [ + { + field: "name", + order: SortOrder.ASCENDING, + type: SortType.STRING, + }, + ["Alice", "Bob", "Charly", "Danny"], + ], + [ + { + field: "name", + }, + ["Alice", "Bob", "Charly", "Danny"], + ], + [ + { + field: "name", + order: SortOrder.DESCENDING, + }, + ["Danny", "Charly", "Bob", "Alice"], + ], + [ + { + field: "name", + order: SortOrder.DESCENDING, + type: SortType.STRING, + }, + ["Danny", "Charly", "Bob", "Alice"], + ], + [ + { + field: "age", + order: SortOrder.ASCENDING, + type: SortType.NUMBER, + }, + ["Danny", "Alice", "Charly", "Bob"], + ], + [ + { + field: "age", + order: SortOrder.ASCENDING, + }, + ["Danny", "Alice", "Charly", "Bob"], + ], + [ + { + field: "age", + order: SortOrder.DESCENDING, + }, + ["Bob", "Charly", "Alice", "Danny"], + ], + [ + { + field: "age", + order: SortOrder.DESCENDING, + type: SortType.NUMBER, + }, + ["Bob", "Charly", "Alice", "Danny"], + ], + ] - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - schema: { - id: { visible: true }, - one: { visible: false }, - two: { visible: true }, - }, - }) + describe("sorting", () => { + let table: Table + const viewSchema = { + id: { visible: true }, + age: { visible: true }, + name: { visible: true }, + } - const response = await config.api.viewV2.search(view.id, { - query: { - allOr, - equal: { - two: "bar", + beforeAll(async () => { + table = await config.api.table.save( + saveTableRequest({ + type: "table", + schema: { + name: { + type: FieldType.STRING, + name: "name", + }, + surname: { + type: FieldType.STRING, + name: "surname", + }, + age: { + type: FieldType.NUMBER, + name: "age", + }, + address: { + type: FieldType.STRING, + name: "address", + }, + jobTitle: { + type: FieldType.STRING, + name: "jobTitle", + }, }, - }, - }) - expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual([ - expect.objectContaining({ _id: one._id }), - ]) - } - ) + }) + ) - !isLucene && - it.each([true, false])("cannot bypass a view filter", async allOr => { + const users = [ + { name: "Alice", age: 25 }, + { name: "Bob", age: 30 }, + { name: "Charly", age: 27 }, + { name: "Danny", age: 15 }, + ] + await Promise.all( + users.map(u => + config.api.row.save(table._id!, { + tableId: table._id, + ...u, + }) + ) + ) + }) + + it.each(sortTestOptions)( + "allow sorting (%s)", + async (sortParams, expected) => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + sort: sortParams, + schema: viewSchema, + }) + + const response = await config.api.viewV2.search(view.id) + + expect(response.rows).toHaveLength(4) + expect(response.rows).toEqual( + expected.map(name => expect.objectContaining({ name })) + ) + } + ) + + it.each(sortTestOptions)( + "allow override the default view sorting (%s)", + async (sortParams, expected) => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + sort: { + field: "name", + order: SortOrder.ASCENDING, + type: SortType.STRING, + }, + schema: viewSchema, + }) + + const response = await config.api.viewV2.search(view.id, { + sort: sortParams.field, + sortOrder: sortParams.order, + sortType: sortParams.type, + query: {}, + }) + + expect(response.rows).toHaveLength(4) + expect(response.rows).toEqual( + expected.map(name => expect.objectContaining({ name })) + ) + } + ) + }) + + it("can query on top of the view filters", async () => { await config.api.row.save(table._id!, { one: "foo", two: "bar", @@ -2851,17 +3023,82 @@ describe.each([ one: "foo2", two: "bar2", }) + const three = await config.api.row.save(table._id!, { + one: "foo3", + two: "bar3", + }) const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), - query: [ - { - operator: BasicOperator.EQUAL, - field: "two", - value: "bar2", + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.NOT_EQUAL, + field: "one", + value: "foo2", + }, + ], + }, + ], + }, + schema: { + id: { visible: true }, + one: { visible: true }, + two: { visible: true }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: { + [BasicOperator.EQUAL]: { + two: "bar3", }, - ], + [BasicOperator.NOT_EMPTY]: { + two: null, + }, + }, + }) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ _id: three._id }), + ]) + ) + }) + + it("can query on top of the view filters (using or filters)", async () => { + const one = await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + const three = await config.api.row.save(table._id!, { + one: "foo3", + two: "bar3", + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.NOT_EQUAL, + field: "one", + value: "foo2", + }, + ], + }, + ], + }, schema: { id: { visible: true }, one: { visible: false }, @@ -2871,217 +3108,775 @@ describe.each([ const response = await config.api.viewV2.search(view.id, { query: { - allOr, - equal: { + allOr: true, + [BasicOperator.NOT_EQUAL]: { two: "bar", }, + [BasicOperator.NOT_EMPTY]: { + two: null, + }, }, }) - expect(response.rows).toHaveLength(0) - }) - - describe("foreign relationship columns", () => { - let envCleanup: () => void - beforeAll(() => { - envCleanup = features.testutils.setFeatureFlags("*", { - ENRICHED_RELATIONSHIPS: true, - }) - }) - - afterAll(() => { - envCleanup?.() - }) - - const createMainTable = async ( - links: { - name: string - tableId: string - fk: string - }[] - ) => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { title: { name: "title", type: FieldType.STRING } }, - }) + expect(response.rows).toHaveLength(2) + expect(response.rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ _id: one._id }), + expect.objectContaining({ _id: three._id }), + ]) ) - await config.api.table.save({ - ...table, - schema: { - ...table.schema, - ...links.reduce((acc, c) => { - acc[c.name] = { - name: c.name, - relationshipType: RelationshipType.ONE_TO_MANY, - type: FieldType.LINK, - tableId: c.tableId, - fieldName: c.fk, - constraints: { type: "array" }, - } - return acc - }, {}), - }, - }) - return table - } - const createAuxTable = (schema: TableSchema) => - config.api.table.save( - saveTableRequest({ - primaryDisplay: "name", - schema: { - ...schema, - name: { name: "name", type: FieldType.STRING }, - }, - }) - ) - - it("returns squashed fields respecting the view config", async () => { - const auxTable = await createAuxTable({ - age: { name: "age", type: FieldType.NUMBER }, - }) - const auxRow = await config.api.row.save(auxTable._id!, { - name: generator.name(), - age: generator.age(), - }) - - const table = await createMainTable([ - { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, - ]) - await config.api.row.save(table._id!, { - title: generator.word(), - aux: [auxRow], - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - schema: { - title: { visible: true }, - aux: { - visible: true, - columns: { - name: { visible: false, readonly: false }, - age: { visible: true, readonly: true }, - }, - }, - }, - }) - - const response = await config.api.viewV2.search(view.id) - expect(response.rows).toEqual([ - expect.objectContaining({ - aux: [ - { - _id: auxRow._id, - primaryDisplay: auxRow.name, - age: auxRow.age, - }, - ], - }), - ]) }) - it("enriches squashed fields", async () => { - const auxTable = await createAuxTable({ - user: { - name: "user", - type: FieldType.BB_REFERENCE_SINGLE, - subtype: BBReferenceFieldSubType.USER, - constraints: { presence: true }, - }, - }) - const table = await createMainTable([ - { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, - ]) + !isLucene && + it.each([true, false])( + "can filter a view without a view filter", + async allOr => { + const one = await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) - const user = config.getUser() - const auxRow = await config.api.row.save(auxTable._id!, { - name: generator.name(), - user: user._id, - }) - await config.api.row.save(table._id!, { - title: generator.word(), - aux: [auxRow], - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - schema: { - title: { visible: true }, - aux: { - visible: true, - columns: { - name: { visible: true, readonly: true }, - user: { visible: true, readonly: true }, + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: true }, }, - }, - }, - }) + }) - const response = await config.api.viewV2.search(view.id) - - expect(response.rows).toEqual([ - expect.objectContaining({ - aux: [ - { - _id: auxRow._id, - primaryDisplay: auxRow.name, - name: auxRow.name, - user: { - _id: user._id, - email: user.email, - firstName: user.firstName, - lastName: user.lastName, - primaryDisplay: user.email, + const response = await config.api.viewV2.search(view.id, { + query: { + allOr, + equal: { + two: "bar", }, }, - ], - }), - ]) - }) - }) + }) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual([ + expect.objectContaining({ _id: one._id }), + ]) + } + ) - !isLucene && - describe("calculations", () => { - let table: Table - let rows: Row[] + !isLucene && + it.each([true, false])("cannot bypass a view filter", async allOr => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) - beforeAll(async () => { - table = await config.api.table.save( + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", + }, + ], + }, + ], + }, + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: true }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: { + allOr, + equal: { + two: "bar", + }, + }, + }) + expect(response.rows).toHaveLength(0) + }) + + describe("foreign relationship columns", () => { + let envCleanup: () => void + beforeAll(() => { + envCleanup = features.testutils.setFeatureFlags("*", { + ENRICHED_RELATIONSHIPS: true, + }) + }) + + afterAll(() => { + envCleanup?.() + }) + + const createMainTable = async ( + links: { + name: string + tableId: string + fk: string + }[] + ) => { + const table = await config.api.table.save( saveTableRequest({ + schema: { title: { name: "title", type: FieldType.STRING } }, + }) + ) + await config.api.table.save({ + ...table, + schema: { + ...table.schema, + ...links.reduce((acc, c) => { + acc[c.name] = { + name: c.name, + relationshipType: RelationshipType.ONE_TO_MANY, + type: FieldType.LINK, + tableId: c.tableId, + fieldName: c.fk, + constraints: { type: "array" }, + } + return acc + }, {}), + }, + }) + return table + } + const createAuxTable = (schema: TableSchema) => + config.api.table.save( + saveTableRequest({ + primaryDisplay: "name", + schema: { + ...schema, + name: { name: "name", type: FieldType.STRING }, + }, + }) + ) + + it("returns squashed fields respecting the view config", async () => { + const auxTable = await createAuxTable({ + age: { name: "age", type: FieldType.NUMBER }, + }) + const auxRow = await config.api.row.save(auxTable._id!, { + name: generator.name(), + age: generator.age(), + }) + + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + ]) + await config.api.row.save(table._id!, { + title: generator.word(), + aux: [auxRow], + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + title: { visible: true }, + aux: { + visible: true, + columns: { + name: { visible: false, readonly: false }, + age: { visible: true, readonly: true }, + }, + }, + }, + }) + + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toEqual([ + expect.objectContaining({ + aux: [ + { + _id: auxRow._id, + primaryDisplay: auxRow.name, + age: auxRow.age, + }, + ], + }), + ]) + }) + + it("enriches squashed fields", async () => { + const auxTable = await createAuxTable({ + user: { + name: "user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + constraints: { presence: true }, + }, + }) + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + ]) + + const user = config.getUser() + const auxRow = await config.api.row.save(auxTable._id!, { + name: generator.name(), + user: user._id, + }) + await config.api.row.save(table._id!, { + title: generator.word(), + aux: [auxRow], + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + title: { visible: true }, + aux: { + visible: true, + columns: { + name: { visible: true, readonly: true }, + user: { visible: true, readonly: true }, + }, + }, + }, + }) + + const response = await config.api.viewV2.search(view.id) + + expect(response.rows).toEqual([ + expect.objectContaining({ + aux: [ + { + _id: auxRow._id, + primaryDisplay: auxRow.name, + name: auxRow.name, + user: { + _id: user._id, + email: user.email, + firstName: user.firstName, + lastName: user.lastName, + primaryDisplay: user.email, + }, + }, + ], + }), + ]) + }) + }) + + !isLucene && + describe("calculations", () => { + let table: Table + let rows: Row[] + + beforeAll(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + rows = await Promise.all( + Array.from({ length: 10 }, () => + config.api.row.save(table._id!, { + quantity: generator.natural({ min: 1, max: 10 }), + price: generator.natural({ min: 1, max: 10 }), + }) + ) + ) + }) + + it("should be able to search by calculations", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + type: ViewV2Type.CALCULATION, + name: generator.guid(), + schema: { + "Quantity Sum": { + visible: true, + calculationType: CalculationType.SUM, + field: "quantity", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + "Quantity Sum": rows.reduce( + (acc, r) => acc + r.quantity, + 0 + ), + }), + ]) + ) + + // Calculation views do not return rows that can be linked back to + // the source table, and so should not have an _id field. + for (const row of response.rows) { + expect("_id" in row).toBe(false) + } + }) + + it("should be able to group by a basic field", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, schema: { quantity: { - type: FieldType.NUMBER, - name: "quantity", + visible: true, + field: "quantity", }, - price: { + "Total Price": { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + const priceByQuantity: Record = {} + for (const row of rows) { + priceByQuantity[row.quantity] ??= 0 + priceByQuantity[row.quantity] += row.price + } + + for (const row of response.rows) { + expect(row["Total Price"]).toEqual( + priceByQuantity[row.quantity] + ) + } + }) + + it.each([ + CalculationType.COUNT, + CalculationType.SUM, + CalculationType.AVG, + CalculationType.MIN, + CalculationType.MAX, + ])("should be able to calculate $type", async type => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + aggregate: { + visible: true, + calculationType: type, + field: "price", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + function calculate( + type: CalculationType, + numbers: number[] + ): number { + switch (type) { + case CalculationType.COUNT: + return numbers.length + case CalculationType.SUM: + return numbers.reduce((a, b) => a + b, 0) + case CalculationType.AVG: + return numbers.reduce((a, b) => a + b, 0) / numbers.length + case CalculationType.MIN: + return Math.min(...numbers) + case CalculationType.MAX: + return Math.max(...numbers) + } + } + + const prices = rows.map(row => row.price) + const expected = calculate(type, prices) + const actual = response.rows[0].aggregate + + if (type === CalculationType.AVG) { + // The average calculation can introduce floating point rounding + // errors, so we need to compare to within a small margin of + // error. + expect(actual).toBeCloseTo(expected) + } else { + expect(actual).toEqual(expected) + } + }) + + it("should be able to do a COUNT(DISTINCT)", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + count: { + visible: true, + calculationType: CalculationType.COUNT, + distinct: true, + field: "name", + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "John", + }, + { + name: "John", + }, + { + name: "Sue", + }, + ], + }) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(1) + expect(rows[0].count).toEqual(2) + }) + + it("should not be able to COUNT(DISTINCT ...) against a non-existent field", async () => { + await config.api.viewV2.create( + { + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + count: { + visible: true, + calculationType: CalculationType.COUNT, + distinct: true, + field: "does not exist oh no", + }, + }, + }, + { + status: 400, + body: { + message: + 'Calculation field "count" references field "does not exist oh no" which does not exist in the table schema', + }, + } + ) + }) + + it("should be able to filter rows on the view itself", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.EQUAL, + field: "quantity", + value: 1, + }, + ], + }, + ], + }, + schema: { + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + quantity: 1, + price: 1, + }, + { + quantity: 1, + price: 2, + }, + { + quantity: 2, + price: 10, + }, + ], + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: {}, + }) + expect(rows).toHaveLength(1) + expect(rows[0].sum).toEqual(3) + }) + + it("should be able to filter on group by fields", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + quantity: { visible: true }, + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + quantity: 1, + price: 1, + }, + { + quantity: 1, + price: 2, + }, + { + quantity: 2, + price: 10, + }, + ], + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: { + equal: { + quantity: 1, + }, + }, + }) + + expect(rows).toHaveLength(1) + expect(rows[0].sum).toEqual(3) + }) + + it("should be able to sort by group by field", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + quantity: { visible: true }, + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + quantity: 1, + price: 1, + }, + { + quantity: 1, + price: 2, + }, + { + quantity: 2, + price: 10, + }, + ], + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: {}, + sort: "quantity", + sortOrder: SortOrder.DESCENDING, + }) + + expect(rows).toEqual([ + expect.objectContaining({ quantity: 2, sum: 10 }), + expect.objectContaining({ quantity: 1, sum: 3 }), + ]) + }) + + it("should be able to sort by a calculation", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + quantity: 1, + price: 1, + }, + { + quantity: 1, + price: 2, + }, + { + quantity: 2, + price: 10, + }, + ], + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + quantity: { visible: true }, + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: {}, + sort: "sum", + sortOrder: SortOrder.DESCENDING, + }) + + expect(rows).toEqual([ + expect.objectContaining({ quantity: 2, sum: 10 }), + expect.objectContaining({ quantity: 1, sum: 3 }), + ]) + }) + }) + + !isLucene && + it("should not need required fields to be present", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + age: { + name: "age", type: FieldType.NUMBER, - name: "price", }, }, }) ) - rows = await Promise.all( - Array.from({ length: 10 }, () => - config.api.row.save(table._id!, { - quantity: generator.natural({ min: 1, max: 10 }), - price: generator.natural({ min: 1, max: 10 }), - }) - ) - ) - }) + await Promise.all([ + config.api.row.save(table._id!, { name: "Steve", age: 30 }), + config.api.row.save(table._id!, { name: "Jane", age: 31 }), + ]) - it("should be able to search by calculations", async () => { const view = await config.api.viewV2.create({ tableId: table._id!, - type: ViewV2Type.CALCULATION, name: generator.guid(), + type: ViewV2Type.CALCULATION, schema: { - "Quantity Sum": { + sum: { visible: true, calculationType: CalculationType.SUM, - field: "quantity", + field: "age", }, }, }) @@ -3091,506 +3886,407 @@ describe.each([ }) expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0), - }), - ]) - ) - - // Calculation views do not return rows that can be linked back to - // the source table, and so should not have an _id field. - for (const row of response.rows) { - expect("_id" in row).toBe(false) - } + expect(response.rows[0].sum).toEqual(61) }) - it("should be able to group by a basic field", async () => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - quantity: { - visible: true, - field: "quantity", - }, - "Total Price": { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: {}, - }) - - const priceByQuantity: Record = {} - for (const row of rows) { - priceByQuantity[row.quantity] ??= 0 - priceByQuantity[row.quantity] += row.price - } - - for (const row of response.rows) { - expect(row["Total Price"]).toEqual(priceByQuantity[row.quantity]) - } - }) - - it.each([ - CalculationType.COUNT, - CalculationType.SUM, - CalculationType.AVG, - CalculationType.MIN, - CalculationType.MAX, - ])("should be able to calculate $type", async type => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - aggregate: { - visible: true, - calculationType: type, - field: "price", - }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: {}, - }) - - function calculate( - type: CalculationType, - numbers: number[] - ): number { - switch (type) { - case CalculationType.COUNT: - return numbers.length - case CalculationType.SUM: - return numbers.reduce((a, b) => a + b, 0) - case CalculationType.AVG: - return numbers.reduce((a, b) => a + b, 0) / numbers.length - case CalculationType.MIN: - return Math.min(...numbers) - case CalculationType.MAX: - return Math.max(...numbers) - } - } - - const prices = rows.map(row => row.price) - const expected = calculate(type, prices) - const actual = response.rows[0].aggregate - - if (type === CalculationType.AVG) { - // The average calculation can introduce floating point rounding - // errors, so we need to compare to within a small margin of - // error. - expect(actual).toBeCloseTo(expected) - } else { - expect(actual).toEqual(expected) - } - }) - - it("should be able to do a COUNT(DISTINCT)", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - count: { - visible: true, - calculationType: CalculationType.COUNT, - distinct: true, - field: "name", - }, - }, - }) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - name: "John", - }, - { - name: "John", - }, - { - name: "Sue", - }, - ], - }) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(1) - expect(rows[0].count).toEqual(2) - }) - - it("should not be able to COUNT(DISTINCT ...) against a non-existent field", async () => { - await config.api.viewV2.create( - { - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - count: { - visible: true, - calculationType: CalculationType.COUNT, - distinct: true, - field: "does not exist oh no", - }, - }, - }, - { - status: 400, - body: { - message: - 'Calculation field "count" references field "does not exist oh no" which does not exist in the table schema', - }, - } - ) - }) - - it("should be able to filter rows on the view itself", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - query: { - equal: { - quantity: 1, - }, - }, - schema: { - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - quantity: 1, - price: 1, - }, - { - quantity: 1, - price: 2, - }, - { - quantity: 2, - price: 10, - }, - ], - }) - - const { rows } = await config.api.viewV2.search(view.id, { - query: {}, - }) - expect(rows).toHaveLength(1) - expect(rows[0].sum).toEqual(3) - }) - - it("should be able to filter on group by fields", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - quantity: { visible: true }, - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - quantity: 1, - price: 1, - }, - { - quantity: 1, - price: 2, - }, - { - quantity: 2, - price: 10, - }, - ], - }) - - const { rows } = await config.api.viewV2.search(view.id, { - query: { - equal: { - quantity: 1, - }, - }, - }) - - expect(rows).toHaveLength(1) - expect(rows[0].sum).toEqual(3) - }) - - it("should be able to sort by group by field", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - quantity: { visible: true }, - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - quantity: 1, - price: 1, - }, - { - quantity: 1, - price: 2, - }, - { - quantity: 2, - price: 10, - }, - ], - }) - - const { rows } = await config.api.viewV2.search(view.id, { - query: {}, - sort: "quantity", - sortOrder: SortOrder.DESCENDING, - }) - - expect(rows).toEqual([ - expect.objectContaining({ quantity: 2, sum: 10 }), - expect.objectContaining({ quantity: 1, sum: 3 }), - ]) - }) - - it("should be able to sort by a calculation", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - quantity: 1, - price: 1, - }, - { - quantity: 1, - price: 2, - }, - { - quantity: 2, - price: 10, - }, - ], - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - quantity: { visible: true }, - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - const { rows } = await config.api.viewV2.search(view.id, { - query: {}, - sort: "sum", - sortOrder: SortOrder.DESCENDING, - }) - - expect(rows).toEqual([ - expect.objectContaining({ quantity: 2, sum: 10 }), - expect.objectContaining({ quantity: 1, sum: 3 }), - ]) - }) - }) - - !isLucene && - it("should not need required fields to be present", async () => { + it("should be able to filter on a single user field in both the view query and search query", async () => { const table = await config.api.table.save( saveTableRequest({ schema: { - name: { - name: "name", - type: FieldType.STRING, - constraints: { - presence: true, - }, - }, - age: { - name: "age", - type: FieldType.NUMBER, + user: { + name: "user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, }, }, }) ) - await Promise.all([ - config.api.row.save(table._id!, { name: "Steve", age: 30 }), - config.api.row.save(table._id!, { name: "Jane", age: 31 }), - ]) + await config.api.row.save(table._id!, { + user: config.getUser()._id, + }) const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "age", - }, + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.EQUAL, + field: "user", + value: "{{ [user].[_id] }}", + }, + ], + }, + ], }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: {}, - }) - - expect(response.rows).toHaveLength(1) - expect(response.rows[0].sum).toEqual(61) - }) - - it("should be able to filter on a single user field in both the view query and search query", async () => { - const table = await config.api.table.save( - saveTableRequest({ schema: { user: { - name: "user", - type: FieldType.BB_REFERENCE_SINGLE, - subtype: BBReferenceFieldSubType.USER, + visible: true, }, }, }) - ) - await config.api.row.save(table._id!, { - user: config.getUser()._id, + const { rows } = await config.api.viewV2.search(view.id, { + query: { + equal: { + user: "{{ [user].[_id] }}", + }, + }, + }) + + expect(rows).toHaveLength(1) + expect(rows[0].user._id).toEqual(config.getUser()._id) }) - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - query: { - equal: { - user: "{{ [user].[_id] }}", - }, - }, - schema: { - user: { - visible: true, - }, - }, - }) + describe("search operators", () => { + let table: Table + beforeEach(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + string: { name: "string", type: FieldType.STRING }, + longform: { name: "longform", type: FieldType.LONGFORM }, + options: { + name: "options", + type: FieldType.OPTIONS, + constraints: { inclusion: ["a", "b", "c"] }, + }, + array: { + name: "array", + type: FieldType.ARRAY, + constraints: { + type: JsonFieldSubType.ARRAY, + inclusion: ["a", "b", "c"], + }, + }, + number: { name: "number", type: FieldType.NUMBER }, + bigint: { name: "bigint", type: FieldType.BIGINT }, + datetime: { name: "datetime", type: FieldType.DATETIME }, + boolean: { name: "boolean", type: FieldType.BOOLEAN }, + user: { + name: "user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + }, + users: { + name: "users", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + constraints: { + type: JsonFieldSubType.ARRAY, + }, + }, + }, + }) + ) + }) - const { rows } = await config.api.viewV2.search(view.id, { - query: { - equal: { - user: "{{ [user].[_id] }}", - }, - }, - }) + interface TestCase { + name: string + query: UISearchFilter | (() => UISearchFilter) + insert: Row[] | (() => Row[]) + expected: Row[] | (() => Row[]) + searchOpts?: Partial + } - expect(rows).toHaveLength(1) - expect(rows[0].user._id).toEqual(config.getUser()._id) + function simpleQuery(...filters: LegacyFilter[]): UISearchFilter { + return { groups: [{ filters }] } + } + + const testCases: TestCase[] = [ + { + name: "empty query return all", + insert: [{ string: "foo" }], + query: { + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + }, + expected: [{ string: "foo" }], + }, + { + name: "empty query return none", + insert: [{ string: "foo" }], + query: { + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + }, + expected: [], + }, + { + name: "simple string search", + insert: [{ string: "foo" }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "string", + value: "foo", + }), + expected: [{ string: "foo" }], + }, + { + name: "non matching string search", + insert: [{ string: "foo" }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "string", + value: "bar", + }), + expected: [], + }, + { + name: "allOr", + insert: [{ string: "bar" }, { string: "foo" }], + query: simpleQuery( + { + operator: BasicOperator.EQUAL, + field: "string", + value: "foo", + }, + { + operator: BasicOperator.EQUAL, + field: "string", + value: "bar", + }, + { + operator: "allOr", + } + ), + searchOpts: { + sort: "string", + sortOrder: SortOrder.ASCENDING, + }, + expected: [{ string: "bar" }, { string: "foo" }], + }, + { + name: "can find rows with fuzzy search", + insert: [{ string: "foo" }, { string: "bar" }], + query: simpleQuery({ + operator: BasicOperator.FUZZY, + field: "string", + value: "fo", + }), + expected: [{ string: "foo" }], + }, + { + name: "can find nothing with fuzzy search", + insert: [{ string: "foo" }, { string: "bar" }], + query: simpleQuery({ + operator: BasicOperator.FUZZY, + field: "string", + value: "baz", + }), + expected: [], + }, + { + name: "can find numeric rows", + insert: [{ number: 1 }, { number: 2 }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "number", + value: 1, + }), + expected: [{ number: 1 }], + }, + { + name: "can find numeric values with rangeHigh", + insert: [{ number: 1 }, { number: 2 }, { number: 3 }], + query: simpleQuery({ + operator: "rangeHigh", + field: "number", + value: 2, + }), + searchOpts: { + sort: "number", + sortOrder: SortOrder.ASCENDING, + }, + expected: [{ number: 1 }, { number: 2 }], + }, + { + name: "can find numeric values with rangeLow", + insert: [{ number: 1 }, { number: 2 }, { number: 3 }], + query: simpleQuery({ + operator: "rangeLow", + field: "number", + value: 2, + }), + searchOpts: { + sort: "number", + sortOrder: SortOrder.ASCENDING, + }, + expected: [{ number: 2 }, { number: 3 }], + }, + { + name: "can find numeric values with full range", + insert: [{ number: 1 }, { number: 2 }, { number: 3 }], + query: simpleQuery( + { + operator: "rangeHigh", + field: "number", + value: 2, + }, + { + operator: "rangeLow", + field: "number", + value: 2, + } + ), + expected: [{ number: 2 }], + }, + { + name: "can find longform values", + insert: [{ longform: "foo" }, { longform: "bar" }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "longform", + value: "foo", + }), + expected: [{ longform: "foo" }], + }, + { + name: "can find options values", + insert: [{ options: "a" }, { options: "b" }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "options", + value: "a", + }), + expected: [{ options: "a" }], + }, + { + name: "can find array values", + insert: [ + // Number field here is just to guarantee order. + { number: 1, array: ["a"] }, + { number: 2, array: ["b"] }, + { number: 3, array: ["a", "c"] }, + ], + query: simpleQuery({ + operator: ArrayOperator.CONTAINS, + field: "array", + value: "a", + }), + searchOpts: { + sort: "number", + sortOrder: SortOrder.ASCENDING, + }, + expected: [{ array: ["a"] }, { array: ["a", "c"] }], + }, + { + name: "can find bigint values", + insert: [{ bigint: "1" }, { bigint: "2" }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "bigint", + type: FieldType.BIGINT, + value: "1", + }), + expected: [{ bigint: "1" }], + }, + { + name: "can find datetime values", + insert: [ + { datetime: "2021-01-01T00:00:00.000Z" }, + { datetime: "2021-01-02T00:00:00.000Z" }, + ], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "datetime", + type: FieldType.DATETIME, + value: "2021-01-01", + }), + expected: [{ datetime: "2021-01-01T00:00:00.000Z" }], + }, + { + name: "can find boolean values", + insert: [{ boolean: true }, { boolean: false }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "boolean", + value: true, + }), + expected: [{ boolean: true }], + }, + { + name: "can find user values", + insert: () => [{ user: config.getUser() }], + query: () => + simpleQuery({ + operator: BasicOperator.EQUAL, + field: "user", + value: config.getUser()._id, + }), + expected: () => [ + { + user: expect.objectContaining({ _id: config.getUser()._id }), + }, + ], + }, + { + name: "can find users values", + insert: () => [{ users: [config.getUser()] }], + query: () => + simpleQuery({ + operator: ArrayOperator.CONTAINS, + field: "users", + value: [config.getUser()._id], + }), + expected: () => [ + { + users: [ + expect.objectContaining({ _id: config.getUser()._id }), + ], + }, + ], + }, + ] + + it.each(testCases)( + "$name", + async ({ query, insert, expected, searchOpts }) => { + // Some values can't be specified outside of a test (e.g. getting + // config.getUser(), it won't be initialised), so we use functions + // in those cases. + if (typeof insert === "function") { + insert = insert() + } + if (typeof expected === "function") { + expected = expected() + } + if (typeof query === "function") { + query = query() + } + + await config.api.row.bulkImport(table._id!, { rows: insert }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + queryUI: query, + schema: { + string: { visible: true }, + longform: { visible: true }, + options: { visible: true }, + array: { visible: true }, + number: { visible: true }, + bigint: { visible: true }, + datetime: { visible: true }, + boolean: { visible: true }, + user: { visible: true }, + users: { visible: true }, + }, + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: {}, + ...searchOpts, + }) + expect(rows).toEqual( + expected.map(r => expect.objectContaining(r)) + ) + } + ) + }) }) - }) describe("permissions", () => { beforeEach(async () => { diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts index 02a1a2d060..862d8c30c5 100644 --- a/packages/server/src/api/routes/utils/validators.ts +++ b/packages/server/src/api/routes/utils/validators.ts @@ -8,6 +8,7 @@ import { SearchFilters, Table, WebhookActionType, + BuiltinPermissionID, } from "@budibase/types" import Joi, { CustomValidator } from "joi" import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core" @@ -214,8 +215,8 @@ export function roleValidator() { }).optional(), // this is the base permission ID (for now a built in) permissionId: Joi.string() - .valid(...Object.values(permissions.BuiltinPermissionID)) - .required(), + .valid(...Object.values(BuiltinPermissionID)) + .optional(), permissions: Joi.object() .pattern( /.*/, diff --git a/packages/server/src/sdk/app/automations/utils.ts b/packages/server/src/sdk/app/automations/utils.ts index f5ae8bf948..79e70923f1 100644 --- a/packages/server/src/sdk/app/automations/utils.ts +++ b/packages/server/src/sdk/app/automations/utils.ts @@ -1,56 +1,7 @@ -import { - Automation, - AutomationActionStepId, - AutomationBuilderData, -} from "@budibase/types" -import { sdk as coreSdk } from "@budibase/shared-core" -import sdk from "../../../sdk" +import { Automation, AutomationActionStepId } from "@budibase/types" export function checkForCollectStep(automation: Automation) { return automation.definition.steps.some( (step: any) => step.stepId === AutomationActionStepId.COLLECT ) } - -export async function getBuilderData( - automations: Automation[] -): Promise> { - const tableNameCache: Record = {} - async function getTableName(tableId: string) { - if (!tableNameCache[tableId]) { - const table = await sdk.tables.getTable(tableId) - tableNameCache[tableId] = table.name - } - - return tableNameCache[tableId] - } - - const result: Record = {} - for (const automation of automations) { - const isRowAction = coreSdk.automations.isRowAction(automation) - if (!isRowAction) { - result[automation._id!] = { displayName: automation.name } - continue - } - - const { tableId, rowActionId } = automation.definition.trigger.inputs - if (!tableId || !rowActionId) { - continue - } - - const tableName = await getTableName(tableId) - const rowActionName = automation.name - result[automation._id!] = { - displayName: rowActionName, - triggerInfo: { - type: "Automation trigger", - table: { id: tableId, name: tableName }, - rowAction: { - id: rowActionId, - name: rowActionName, - }, - }, - } - } - return result -} diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index f80c1c1f8a..567e7d5cc8 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -6,6 +6,7 @@ import { Row, RowSearchParams, SearchFilterKey, + SearchFilters, SearchResponse, SortOrder, Table, @@ -90,17 +91,18 @@ export async function search( options = searchInputMapping(table, options) if (options.viewId) { - // Delete extraneous search params that cannot be overridden - delete options.query.onEmptyFilter - const view = source as ViewV2 + // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let viewQuery = await enrichSearchContext(view.query || {}, context) - viewQuery = dataFilters.buildQueryLegacy(viewQuery) || {} + let viewQuery = (await enrichSearchContext(view.query || {}, context)) as + | SearchFilters + | LegacyFilter[] + if (Array.isArray(viewQuery)) { + viewQuery = dataFilters.buildQuery(viewQuery) + } viewQuery = checkFilters(table, viewQuery) - delete viewQuery?.onEmptyFilter const sqsEnabled = await features.flags.isEnabled(FeatureFlag.SQS) const supportsLogicalOperators = @@ -113,13 +115,12 @@ export async function search( ? view.query : [] - delete options.query.onEmptyFilter + const { filters } = dataFilters.splitFiltersArray(queryFilters) // Extract existing fields - const existingFields = - queryFilters - ?.filter(filter => filter.field) - .map(filter => db.removeKeyNumbering(filter.field)) || [] + const existingFields = filters.map(filter => + db.removeKeyNumbering(filter.field) + ) // Carry over filters for unused fields Object.keys(options.query).forEach(key => { diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 7b3d6913cf..a8ad606647 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -16,6 +16,8 @@ import { } from "@budibase/types" import datasources from "../datasources" import sdk from "../../../sdk" +import { ensureQueryUISet } from "../views/utils" +import { isV2 } from "../views" export async function processTable(table: Table): Promise { if (!table) { @@ -23,6 +25,14 @@ export async function processTable(table: Table): Promise
{ } table = { ...table } + if (table.views) { + for (const [key, view] of Object.entries(table.views)) { + if (!isV2(view)) { + continue + } + table.views[key] = ensureQueryUISet(view) + } + } if (table._id && isExternalTableID(table._id)) { // Old created external tables via Budibase might have a missing field name breaking some UI such as filters if (table.schema["id"] && !table.schema["id"].name) { diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts index b69ac0b9eb..bee153a910 100644 --- a/packages/server/src/sdk/app/views/external.ts +++ b/packages/server/src/sdk/app/views/external.ts @@ -5,6 +5,7 @@ import sdk from "../../../sdk" import * as utils from "../../../db/utils" import { enrichSchema, isV2 } from "." import { breakExternalTableId } from "../../../integrations/utils" +import { ensureQuerySet, ensureQueryUISet } from "./utils" export async function get(viewId: string): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) @@ -18,7 +19,7 @@ export async function get(viewId: string): Promise { if (!found) { throw new Error("No view found") } - return found + return ensureQueryUISet(found) } export async function getEnriched(viewId: string): Promise { @@ -33,19 +34,22 @@ export async function getEnriched(viewId: string): Promise { if (!found) { throw new Error("No view found") } - return await enrichSchema(found, table.schema) + return await enrichSchema(ensureQueryUISet(found), table.schema) } export async function create( tableId: string, viewRequest: Omit ): Promise { - const view: ViewV2 = { + let view: ViewV2 = { ...viewRequest, id: utils.generateViewID(tableId), version: 2, } + view = ensureQuerySet(view) + view = ensureQueryUISet(view) + const db = context.getAppDB() const { datasourceId, tableName } = breakExternalTableId(tableId) @@ -56,7 +60,10 @@ export async function create( return view } -export async function update(tableId: string, view: ViewV2): Promise { +export async function update( + tableId: string, + view: Readonly +): Promise { const db = context.getAppDB() const { datasourceId, tableName } = breakExternalTableId(tableId) @@ -74,6 +81,9 @@ export async function update(tableId: string, view: ViewV2): Promise { throw new HTTPError(`Cannot update view type after creation`, 400) } + view = ensureQuerySet(view) + view = ensureQueryUISet(view) + delete views[existingView.name] views[view.name] = view await db.put(ds) diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts index 96b41bffe8..63807bcfd4 100644 --- a/packages/server/src/sdk/app/views/internal.ts +++ b/packages/server/src/sdk/app/views/internal.ts @@ -4,6 +4,7 @@ import { context, HTTPError } from "@budibase/backend-core" import sdk from "../../../sdk" import * as utils from "../../../db/utils" import { enrichSchema, isV2 } from "." +import { ensureQuerySet, ensureQueryUISet } from "./utils" export async function get(viewId: string): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) @@ -13,7 +14,7 @@ export async function get(viewId: string): Promise { if (!found) { throw new Error("No view found") } - return found + return ensureQueryUISet(found) } export async function getEnriched(viewId: string): Promise { @@ -24,19 +25,22 @@ export async function getEnriched(viewId: string): Promise { if (!found) { throw new Error("No view found") } - return await enrichSchema(found, table.schema) + return await enrichSchema(ensureQueryUISet(found), table.schema) } export async function create( tableId: string, viewRequest: Omit ): Promise { - const view: ViewV2 = { + let view: ViewV2 = { ...viewRequest, id: utils.generateViewID(tableId), version: 2, } + view = ensureQuerySet(view) + view = ensureQueryUISet(view) + const db = context.getAppDB() const table = await sdk.tables.getTable(tableId) @@ -47,7 +51,10 @@ export async function create( return view } -export async function update(tableId: string, view: ViewV2): Promise { +export async function update( + tableId: string, + view: Readonly +): Promise { const db = context.getAppDB() const table = await sdk.tables.getTable(tableId) table.views ??= {} @@ -63,6 +70,9 @@ export async function update(tableId: string, view: ViewV2): Promise { throw new HTTPError(`Cannot update view type after creation`, 400) } + view = ensureQuerySet(view) + view = ensureQueryUISet(view) + delete table.views[existingView.name] table.views[view.name] = view await db.put(table) diff --git a/packages/server/src/sdk/app/views/utils.ts b/packages/server/src/sdk/app/views/utils.ts new file mode 100644 index 0000000000..8ca5fd1098 --- /dev/null +++ b/packages/server/src/sdk/app/views/utils.ts @@ -0,0 +1,43 @@ +import { ViewV2 } from "@budibase/types" +import { utils, dataFilters } from "@budibase/shared-core" +import { cloneDeep, isPlainObject } from "lodash" +import { HTTPError } from "@budibase/backend-core" + +function isEmptyObject(obj: any) { + return obj && isPlainObject(obj) && Object.keys(obj).length === 0 +} + +export function ensureQueryUISet(viewArg: Readonly): ViewV2 { + const view = cloneDeep(viewArg) + if (!view.queryUI && view.query && !isEmptyObject(view.query)) { + if (!Array.isArray(view.query)) { + // In practice this should not happen. `view.query`, at the time this code + // goes into the codebase, only contains LegacyFilter[] in production. + // We're changing it in the change that this comment is part of to also + // include SearchFilters objects. These are created when we receive an + // update to a ViewV2 that contains a queryUI and not a query field. We + // can convert UISearchFilter (the type of queryUI) to SearchFilters, + // but not LegacyFilter[], they are incompatible due to UISearchFilter + // and SearchFilters being recursive types. + // + // So despite the type saying that `view.query` is a LegacyFilter[] | + // SearchFilters, it will never be a SearchFilters when a `view.queryUI` + // is specified, making it "safe" to throw an error here. + throw new HTTPError("view is missing queryUI field", 400) + } + + view.queryUI = utils.processSearchFilters(view.query) + } + return view +} + +export function ensureQuerySet(viewArg: Readonly): ViewV2 { + const view = cloneDeep(viewArg) + // We consider queryUI to be the source of truth, so we don't check for the + // presence of query here. We will overwrite it regardless of whether it is + // present or not. + if (view.queryUI && !isEmptyObject(view.queryUI)) { + view.query = dataFilters.buildQuery(view.queryUI) + } + return view +} diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index b63d6d1a78..49046d9eda 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -1,4 +1,4 @@ -import { permissions, roles, utils } from "@budibase/backend-core" +import { roles, utils } from "@budibase/backend-core" import { createHomeScreen } from "../../constants/screens" import { EMPTY_LAYOUT } from "../../constants/layouts" import { cloneDeep } from "lodash/fp" @@ -33,6 +33,7 @@ import { TableSourceType, Webhook, WebhookActionType, + BuiltinPermissionID, } from "@budibase/types" import { LoopInput } from "../../definitions/automations" import { merge } from "lodash" @@ -515,7 +516,7 @@ export function basicRole(): Role { return { name: `NewRole_${utils.newid()}`, inherits: roles.BUILTIN_ROLE_IDS.BASIC, - permissionId: permissions.BuiltinPermissionID.READ_ONLY, + permissionId: BuiltinPermissionID.WRITE, permissions: {}, version: "name", } diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 6885c4bc79..003e6b5082 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -19,8 +19,12 @@ import { RangeOperator, LogicalOperator, isLogicalSearchOperator, - SearchFilterGroup, - FilterGroupLogicalOperator, + UISearchFilter, + UILogicalOperator, + isBasicSearchOperator, + isArraySearchOperator, + isRangeSearchOperator, + SearchFilter, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" @@ -310,308 +314,194 @@ export class ColumnSplitter { * @param filter the builder filter structure */ -const buildCondition = (expression: LegacyFilter) => { - // Filter body - let query: SearchFilters = { - string: {}, - fuzzy: {}, - range: {}, - equal: {}, - notEqual: {}, - empty: {}, - notEmpty: {}, - contains: {}, - notContains: {}, - oneOf: {}, - containsAny: {}, - } - let { operator, field, type, value, externalType, onEmptyFilter } = expression - - if (!operator || !field) { +function buildCondition(filter: undefined): undefined +function buildCondition(filter: SearchFilter): SearchFilters +function buildCondition(filter?: SearchFilter): SearchFilters | undefined { + if (!filter) { return } - const queryOperator = operator as SearchFilterOperator - const isHbs = - typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 - // Parse all values into correct types - if (operator === "allOr") { - query.allOr = true - return - } - if (onEmptyFilter) { - query.onEmptyFilter = onEmptyFilter - return - } + const query: SearchFilters = {} + const { operator, field, type, externalType } = filter + let { value } = filter // Default the value for noValue fields to ensure they are correctly added // to the final query - if (queryOperator === "empty" || queryOperator === "notEmpty") { + if (operator === "empty" || operator === "notEmpty") { value = null } - if ( - type === "datetime" && - !isHbs && - queryOperator !== "empty" && - queryOperator !== "notEmpty" + const isHbs = + typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 + + // Parsing value depending on what the type is. + switch (type) { + case FieldType.DATETIME: + if (!isHbs && operator !== "empty" && operator !== "notEmpty") { + if (!value) { + return + } + value = new Date(value).toISOString() + } + break + case FieldType.NUMBER: + if (typeof value === "string" && !isHbs) { + if (operator === "oneOf") { + value = value.split(",").map(parseFloat) + } else { + value = parseFloat(value) + } + } + break + case FieldType.BOOLEAN: + value = `${value}`.toLowerCase() === "true" + break + case FieldType.ARRAY: + if ( + ["contains", "notContains", "containsAny"].includes( + operator.toLocaleString() + ) && + typeof value === "string" + ) { + value = value.split(",") + } + break + } + + if (isRangeSearchOperator(operator)) { + const key = externalType as keyof typeof SqlNumberTypeRangeMap + const limits = SqlNumberTypeRangeMap[key] || { + min: Number.MIN_SAFE_INTEGER, + max: Number.MAX_SAFE_INTEGER, + } + + query[operator] ??= {} + query[operator][field] = { + low: type === "number" ? limits.min : "0000-00-00T00:00:00.000Z", + high: type === "number" ? limits.max : "9999-00-00T00:00:00.000Z", + } + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range ??= {} + query.range[field] = { + ...query.range[field], + high: value, + } + } else if (operator === "rangeLow" && value != null && value !== "") { + query.range ??= {} + query.range[field] = { + ...query.range[field], + low: value, + } + } else if ( + isBasicSearchOperator(operator) || + isArraySearchOperator(operator) || + isRangeSearchOperator(operator) ) { - // Ensure date value is a valid date and parse into correct format - if (!value) { - return - } - try { - value = new Date(value).toISOString() - } catch (error) { - return - } - } - if (type === "number" && typeof value === "string" && !isHbs) { - if (queryOperator === "oneOf") { - value = value.split(",").map(item => parseFloat(item)) - } else { - value = parseFloat(value) - } - } - if (type === "boolean") { - value = `${value}`?.toLowerCase() === "true" - } - if ( - ["contains", "notContains", "containsAny"].includes( - operator.toLocaleString() - ) && - type === "array" && - typeof value === "string" - ) { - value = value.split(",") - } - if (operator.toLocaleString().startsWith("range") && query.range) { - const minint = - SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] - ?.min || Number.MIN_SAFE_INTEGER - const maxint = - SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] - ?.max || Number.MAX_SAFE_INTEGER - if (!query.range[field]) { - query.range[field] = { - low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", - high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", - } - } - if (operator === "rangeLow" && value != null && value !== "") { - query.range[field] = { - ...query.range[field], - low: value, - } - } else if (operator === "rangeHigh" && value != null && value !== "") { - query.range[field] = { - ...query.range[field], - high: value, - } - } - } else if (isLogicalSearchOperator(queryOperator)) { - // TODO - } else if (query[queryOperator] && operator !== "onEmptyFilter") { if (type === "boolean") { - // Transform boolean filters to cope with null. - // "equals false" needs to be "not equals true" - // "not equals false" needs to be "equals true" - if (queryOperator === "equal" && value === false) { + // TODO(samwho): I suspect this boolean transformation isn't needed anymore, + // write some tests to confirm. + + // Transform boolean filters to cope with null. "equals false" needs to + // be "not equals true" "not equals false" needs to be "equals true" + if (operator === "equal" && value === false) { query.notEqual = query.notEqual || {} query.notEqual[field] = true - } else if (queryOperator === "notEqual" && value === false) { + } else if (operator === "notEqual" && value === false) { query.equal = query.equal || {} query.equal[field] = true } else { - query[queryOperator] ??= {} - query[queryOperator]![field] = value + query[operator] ??= {} + query[operator][field] = value } } else { - query[queryOperator] ??= {} - query[queryOperator]![field] = value + query[operator] ??= {} + query[operator][field] = value } + } else { + throw new Error(`Unsupported operator: ${operator}`) } return query } -export const buildQueryLegacy = ( - filter?: LegacyFilter[] | SearchFilters -): SearchFilters | undefined => { - // this is of type SearchFilters or is undefined - if (!Array.isArray(filter)) { - return filter +export interface LegacyFilterSplit { + allOr?: boolean + onEmptyFilter?: EmptyFilterOption + filters: SearchFilter[] +} + +export function splitFiltersArray(filters: LegacyFilter[]) { + const split: LegacyFilterSplit = { + filters: [], } - let query: SearchFilters = { - string: {}, - fuzzy: {}, - range: {}, - equal: {}, - notEqual: {}, - empty: {}, - notEmpty: {}, - contains: {}, - notContains: {}, - oneOf: {}, - containsAny: {}, + for (const filter of filters) { + if ("operator" in filter && filter.operator === "allOr") { + split.allOr = true + } else if ("onEmptyFilter" in filter) { + split.onEmptyFilter = filter.onEmptyFilter + } else { + split.filters.push(filter) + } } - if (!Array.isArray(filter)) { - return query - } - - filter.forEach(expression => { - let { operator, field, type, value, externalType, onEmptyFilter } = - expression - const queryOperator = operator as SearchFilterOperator - const isHbs = - typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 - // Parse all values into correct types - if (operator === "allOr") { - query.allOr = true - return - } - if (onEmptyFilter) { - query.onEmptyFilter = onEmptyFilter - return - } - if ( - type === "datetime" && - !isHbs && - queryOperator !== "empty" && - queryOperator !== "notEmpty" - ) { - // Ensure date value is a valid date and parse into correct format - if (!value) { - return - } - try { - value = new Date(value).toISOString() - } catch (error) { - return - } - } - if (type === "number" && typeof value === "string" && !isHbs) { - if (queryOperator === "oneOf") { - value = value.split(",").map(item => parseFloat(item)) - } else { - value = parseFloat(value) - } - } - if (type === "boolean") { - value = `${value}`?.toLowerCase() === "true" - } - if ( - ["contains", "notContains", "containsAny"].includes( - operator.toLocaleString() - ) && - type === "array" && - typeof value === "string" - ) { - value = value.split(",") - } - if (operator.toLocaleString().startsWith("range") && query.range) { - const minint = - SqlNumberTypeRangeMap[ - externalType as keyof typeof SqlNumberTypeRangeMap - ]?.min || Number.MIN_SAFE_INTEGER - const maxint = - SqlNumberTypeRangeMap[ - externalType as keyof typeof SqlNumberTypeRangeMap - ]?.max || Number.MAX_SAFE_INTEGER - if (!query.range[field]) { - query.range[field] = { - low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", - high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", - } - } - if (operator === "rangeLow" && value != null && value !== "") { - query.range[field] = { - ...query.range[field], - low: value, - } - } else if (operator === "rangeHigh" && value != null && value !== "") { - query.range[field] = { - ...query.range[field], - high: value, - } - } - } else if (isLogicalSearchOperator(queryOperator)) { - // ignore - } else if (query[queryOperator] && operator !== "onEmptyFilter") { - if (type === "boolean") { - // Transform boolean filters to cope with null. - // "equals false" needs to be "not equals true" - // "not equals false" needs to be "equals true" - if (queryOperator === "equal" && value === false) { - query.notEqual = query.notEqual || {} - query.notEqual[field] = true - } else if (queryOperator === "notEqual" && value === false) { - query.equal = query.equal || {} - query.equal[field] = true - } else { - query[queryOperator] ??= {} - query[queryOperator]![field] = value - } - } else { - query[queryOperator] ??= {} - query[queryOperator]![field] = value - } - } - }) - return query + return split } /** - * Converts a **SearchFilterGroup** filter definition into a grouped + * Converts a **UISearchFilter** filter definition into a grouped * search query of type **SearchFilters** * * Legacy support remains for the old **SearchFilter[]** format. * These will be migrated to an appropriate **SearchFilters** object, if encountered - * - * @param filter - * - * @returns {SearchFilters} */ - -export const buildQuery = ( - filter?: SearchFilterGroup | LegacyFilter[] -): SearchFilters | undefined => { - const parsedFilter: SearchFilterGroup | undefined = - processSearchFilters(filter) - - if (!parsedFilter) { - return +export function buildQuery( + filter?: UISearchFilter | LegacyFilter[] +): SearchFilters { + if (!filter) { + return {} } - const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } = - { - [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND, - [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR, + if (Array.isArray(filter)) { + filter = processSearchFilters(filter) + if (!filter) { + return {} } - - const globalOnEmpty = parsedFilter.onEmptyFilter - ? parsedFilter.onEmptyFilter - : null - - const globalOperator: LogicalOperator = - operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] - - return { - ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), - [globalOperator]: { - conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { - return { - [operatorMap[group.logicalOperator]]: { - conditions: group.filters - ?.map(x => buildCondition(x)) - .filter(filter => filter), - }, - } - }), - }, } + + const operator = logicalOperatorFromUI( + filter.logicalOperator || UILogicalOperator.ALL + ) + + const query: SearchFilters = {} + if (filter.onEmptyFilter) { + query.onEmptyFilter = filter.onEmptyFilter + } else { + query.onEmptyFilter = EmptyFilterOption.RETURN_ALL + } + + query[operator] = { + conditions: (filter.groups || []).map(group => { + const { allOr, onEmptyFilter, filters } = splitFiltersArray( + group.filters || [] + ) + if (onEmptyFilter) { + query.onEmptyFilter = onEmptyFilter + } + const operator = allOr ? LogicalOperator.OR : LogicalOperator.AND + return { + [operator]: { conditions: filters.map(buildCondition).filter(f => f) }, + } + }), + } + + return query +} + +function logicalOperatorFromUI(operator: UILogicalOperator): LogicalOperator { + return operator === UILogicalOperator.ALL + ? LogicalOperator.AND + : LogicalOperator.OR } // The frontend can send single values for array fields sometimes, so to handle diff --git a/packages/shared-core/src/helpers/tests/roles.spec.ts b/packages/shared-core/src/helpers/tests/roles.spec.ts index 051a196a5b..003b37f8f2 100644 --- a/packages/shared-core/src/helpers/tests/roles.spec.ts +++ b/packages/shared-core/src/helpers/tests/roles.spec.ts @@ -1,5 +1,5 @@ import { checkForRoleInheritanceLoops } from "../roles" -import { Role } from "@budibase/types" +import { BuiltinPermissionID, Role } from "@budibase/types" /** * This unit test exists as this utility will be used in the frontend and backend, confirmation @@ -19,7 +19,7 @@ function role(id: string, inherits: string | string[]): TestRole { _id: id, inherits: inherits, name: "ROLE", - permissionId: "PERMISSION", + permissionId: BuiltinPermissionID.WRITE, permissions: {}, // not needed for this test } allRoles.push(role) diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index bead5961f0..0e49db9c7c 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -1,18 +1,28 @@ import { LegacyFilter, - SearchFilterGroup, - FilterGroupLogicalOperator, + UISearchFilter, + UILogicalOperator, SearchFilters, BasicOperator, ArrayOperator, isLogicalSearchOperator, + SearchFilter, + EmptyFilterOption, } from "@budibase/types" import * as Constants from "./constants" -import { removeKeyNumbering } from "./filters" +import { removeKeyNumbering, splitFiltersArray } from "./filters" +import _ from "lodash" -// an array of keys from filter type to properties that are in the type -// this can then be converted using .fromEntries to an object -type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][] +const FILTER_ALLOWED_KEYS: (keyof SearchFilter)[] = [ + "field", + "operator", + "value", + "type", + "externalType", + "valueType", + "noValue", + "formulaType", +] export function unreachable( value: never, @@ -125,100 +135,32 @@ export function isSupportedUserSearch(query: SearchFilters) { return false } } + return true } -/** - * Processes the filter config. Filters are migrated from - * SearchFilter[] to SearchFilterGroup - * - * If filters is not an array, the migration is skipped - * - * @param {LegacyFilter[] | SearchFilterGroup} filters - */ -export const processSearchFilters = ( - filters: LegacyFilter[] | SearchFilterGroup | undefined -): SearchFilterGroup | undefined => { - if (!filters) { - return +export function processSearchFilters( + filterArray?: LegacyFilter[] +): Required | undefined { + if (!filterArray || filterArray.length === 0) { + return undefined } - - // Base search config. - const defaultCfg: SearchFilterGroup = { - logicalOperator: FilterGroupLogicalOperator.ALL, - groups: [], + const { allOr, onEmptyFilter, filters } = splitFiltersArray(filterArray) + return { + logicalOperator: UILogicalOperator.ALL, + onEmptyFilter: onEmptyFilter || EmptyFilterOption.RETURN_ALL, + groups: [ + { + logicalOperator: allOr ? UILogicalOperator.ANY : UILogicalOperator.ALL, + filters: filters.map(filter => { + const trimmedFilter = _.pick( + filter, + FILTER_ALLOWED_KEYS + ) as SearchFilter + trimmedFilter.field = removeKeyNumbering(trimmedFilter.field) + return trimmedFilter + }), + }, + ], } - - const filterAllowedKeys = [ - "field", - "operator", - "value", - "type", - "externalType", - "valueType", - "noValue", - "formulaType", - ] - - if (Array.isArray(filters)) { - let baseGroup: SearchFilterGroup = { - filters: [], - logicalOperator: FilterGroupLogicalOperator.ALL, - } - - return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => { - // Sort the properties for easier debugging - const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[]) - .sort((a, b) => { - return a.localeCompare(b) - }) - .filter(key => filter[key]) - - if (filterPropertyKeys.length == 1) { - const key = filterPropertyKeys[0], - value = filter[key] - // Global - if (key === "onEmptyFilter") { - // unset otherwise - acc.onEmptyFilter = value - } else if (key === "operator" && value === "allOr") { - // Group 1 logical operator - baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY - } - - return acc - } - - const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce( - (acc: AllowedFilters, key) => { - const value = filter[key] - if (filterAllowedKeys.includes(key)) { - if (key === "field") { - acc.push([key, removeKeyNumbering(value)]) - } else { - acc.push([key, value]) - } - } - return acc - }, - [] - ) - - const migratedFilter: LegacyFilter = Object.fromEntries( - allowedFilterSettings - ) as LegacyFilter - - baseGroup.filters!.push(migratedFilter) - - if (!acc.groups || !acc.groups.length) { - // init the base group - acc.groups = [baseGroup] - } - - return acc - }, defaultCfg) - } else if (!filters?.groups) { - return - } - return filters } diff --git a/packages/types/src/api/web/automation.ts b/packages/types/src/api/web/automation.ts index 6c810d5e78..06080fc667 100644 --- a/packages/types/src/api/web/automation.ts +++ b/packages/types/src/api/web/automation.ts @@ -3,22 +3,6 @@ import { Automation } from "../../documents" export interface DeleteAutomationResponse extends DocumentDestroyResponse {} -export interface AutomationBuilderData { - displayName: string - triggerInfo?: { - type: string - table: { - id: string - name: string - } - rowAction: { - id: string - name: string - } - } -} - export interface FetchAutomationResponse { automations: Automation[] - builderData?: Record // The key will be the automationId } diff --git a/packages/types/src/api/web/role.ts b/packages/types/src/api/web/role.ts index df439e84e7..f1c573c849 100644 --- a/packages/types/src/api/web/role.ts +++ b/packages/types/src/api/web/role.ts @@ -1,12 +1,12 @@ import { Role, RoleUIMetadata } from "../../documents" -import { PermissionLevel } from "../../sdk" +import { PermissionLevel, BuiltinPermissionID } from "../../sdk" export interface SaveRoleRequest { _id?: string _rev?: string name: string inherits?: string | string[] - permissionId: string + permissionId?: BuiltinPermissionID permissions?: Record version?: string uiMetadata?: RoleUIMetadata diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index 23c599027e..7a6920fb60 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -1,23 +1,59 @@ import { FieldType } from "../../documents" import { EmptyFilterOption, - FilterGroupLogicalOperator, - SearchFilters, + UILogicalOperator, + BasicOperator, + RangeOperator, + ArrayOperator, } from "../../sdk" -export type LegacyFilter = { - operator: keyof SearchFilters | "rangeLow" | "rangeHigh" - onEmptyFilter?: EmptyFilterOption - field: string - type?: FieldType - value: any - externalType?: string +type AllOr = { + operator: "allOr" } -// this is a type purely used by the UI +type OnEmptyFilter = { + onEmptyFilter: EmptyFilterOption +} + +// TODO(samwho): this could be broken down further +export type SearchFilter = { + operator: + | BasicOperator + | RangeOperator + | ArrayOperator + | "rangeLow" + | "rangeHigh" + // Field name will often have a numerical prefix when coming from the frontend, + // use the ColumnSplitter class to remove it. + field: string + value: any + type?: FieldType + externalType?: string + noValue?: boolean + valueType?: string + formulaType?: string +} + +// Prior to v2, this is the type the frontend sent us when filters were +// involved. We convert this to a SearchFilters before use with the search SDK. +export type LegacyFilter = AllOr | OnEmptyFilter | SearchFilter + export type SearchFilterGroup = { - logicalOperator: FilterGroupLogicalOperator - onEmptyFilter?: EmptyFilterOption + logicalOperator?: UILogicalOperator groups?: SearchFilterGroup[] filters?: LegacyFilter[] } + +// As of v3, this is the format that the frontend always sends when search +// filters are involved. We convert this to SearchFilters before use with the +// search SDK. +// +// The reason we migrated was that we started to support "logical operators" in +// tests and SearchFilters because a recursive data structure. LegacyFilter[] +// wasn't able to support these sorts of recursive structures, so we changed the +// format. +export type UISearchFilter = { + logicalOperator?: UILogicalOperator + onEmptyFilter?: EmptyFilterOption + groups?: SearchFilterGroup[] +} diff --git a/packages/types/src/documents/app/role.ts b/packages/types/src/documents/app/role.ts index 22f4ab9cd3..b624a5c617 100644 --- a/packages/types/src/documents/app/role.ts +++ b/packages/types/src/documents/app/role.ts @@ -1,5 +1,5 @@ import { Document } from "../document" -import { PermissionLevel } from "../../sdk" +import { PermissionLevel, BuiltinPermissionID } from "../../sdk" export interface RoleUIMetadata { displayName?: string @@ -8,7 +8,7 @@ export interface RoleUIMetadata { } export interface Role extends Document { - permissionId: string + permissionId: BuiltinPermissionID inherits?: string | string[] permissions: Record version?: string diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 1b2372da85..3ae120fcca 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,4 +1,4 @@ -import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" +import { LegacyFilter, UISearchFilter, SortOrder, SortType } from "../../api" import { UIFieldMetadata } from "./table" import { Document } from "../document" import { DBView, SearchFilters } from "../../sdk" @@ -92,7 +92,7 @@ export interface ViewV2 { tableId: string query?: LegacyFilter[] | SearchFilters // duplicate to store UI information about filters - queryUI?: SearchFilterGroup + queryUI?: UISearchFilter sort?: { field: string order?: SortOrder diff --git a/packages/types/src/sdk/permissions.ts b/packages/types/src/sdk/permissions.ts index 638de97f48..bb36af4170 100644 --- a/packages/types/src/sdk/permissions.ts +++ b/packages/types/src/sdk/permissions.ts @@ -1,3 +1,5 @@ +// used in resource permissions - permissions can be at one of these levels +// endpoints will set what type of permission they require (e.g. searching requires READ) export enum PermissionLevel { READ = "read", WRITE = "write", @@ -5,6 +7,15 @@ export enum PermissionLevel { ADMIN = "admin", } +// used within the role, specifies base permissions +export enum BuiltinPermissionID { + PUBLIC = "public", + READ_ONLY = "read_only", + WRITE = "write", + ADMIN = "admin", + POWER = "power", +} + // these are the global types, that govern the underlying default behaviour export enum PermissionType { APP = "app", diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index d41bb0fb99..a2d4b4760f 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -32,7 +32,19 @@ export enum LogicalOperator { export function isLogicalSearchOperator( value: string ): value is LogicalOperator { - return value === LogicalOperator.AND || value === LogicalOperator.OR + return Object.values(LogicalOperator).includes(value as LogicalOperator) +} + +export function isBasicSearchOperator(value: string): value is BasicOperator { + return Object.values(BasicOperator).includes(value as BasicOperator) +} + +export function isArraySearchOperator(value: string): value is ArrayOperator { + return Object.values(ArrayOperator).includes(value as ArrayOperator) +} + +export function isRangeSearchOperator(value: string): value is RangeOperator { + return Object.values(RangeOperator).includes(value as RangeOperator) } export type SearchFilterOperator = @@ -191,7 +203,7 @@ export enum EmptyFilterOption { RETURN_NONE = "none", } -export enum FilterGroupLogicalOperator { +export enum UILogicalOperator { ALL = "all", ANY = "any", } diff --git a/packages/worker/src/api/routes/global/tests/roles.spec.ts b/packages/worker/src/api/routes/global/tests/roles.spec.ts index 28977d8521..4e7acb2741 100644 --- a/packages/worker/src/api/routes/global/tests/roles.spec.ts +++ b/packages/worker/src/api/routes/global/tests/roles.spec.ts @@ -1,6 +1,6 @@ import { structures, TestConfiguration } from "../../../../tests" -import { context, db, permissions, roles } from "@budibase/backend-core" -import { App, Database } from "@budibase/types" +import { context, db, roles } from "@budibase/backend-core" +import { App, Database, BuiltinPermissionID } from "@budibase/types" jest.mock("@budibase/backend-core", () => { const core = jest.requireActual("@budibase/backend-core") @@ -44,7 +44,7 @@ describe("/api/global/roles", () => { const role = new roles.Role( db.generateRoleID(ROLE_NAME), ROLE_NAME, - permissions.BuiltinPermissionID.READ_ONLY, + BuiltinPermissionID.READ_ONLY, { displayName: roles.BUILTIN_ROLE_IDS.BASIC } )