Merge branch 'v3-ui' of github.com:Budibase/budibase into view-calculation-ui

packages/backend-core/src/security/permissions.ts

@@ -1,4 +1,8 @@
-import { PermissionLevel, PermissionType } from "@budibase/types"
+import {
+  PermissionLevel,
+  PermissionType,
+  BuiltinPermissionID,
+} from "@budibase/types"
 import flatten from "lodash/flatten"
 import cloneDeep from "lodash/fp/cloneDeep"
 
@@ -57,14 +61,6 @@ export function getAllowedLevels(userPermLevel: PermissionLevel): string[] {
   }
 }
 
-export enum BuiltinPermissionID {
-  PUBLIC = "public",
-  READ_ONLY = "read_only",
-  WRITE = "write",
-  ADMIN = "admin",
-  POWER = "power",
-}
-
 export const BUILTIN_PERMISSIONS: {
   [key in keyof typeof BuiltinPermissionID]: {
     _id: (typeof BuiltinPermissionID)[key]

packages/backend-core/src/security/roles.ts

@@ -1,5 +1,4 @@
 import semver from "semver"
-import { BuiltinPermissionID, PermissionLevel } from "./permissions"
 import {
   prefixRoleID,
   getRoleParams,
@@ -14,6 +13,8 @@ import {
   RoleUIMetadata,
   Database,
   App,
+  BuiltinPermissionID,
+  PermissionLevel,
 } from "@budibase/types"
 import cloneDeep from "lodash/fp/cloneDeep"
 import { RoleColor, helpers } from "@budibase/shared-core"
@@ -50,7 +51,7 @@ export class Role implements RoleDoc {
   _id: string
   _rev?: string
   name: string
-  permissionId: string
+  permissionId: BuiltinPermissionID
   inherits?: string | string[]
   version?: string
   permissions: Record<string, PermissionLevel[]> = {}
@@ -59,7 +60,7 @@ export class Role implements RoleDoc {
   constructor(
     id: string,
     name: string,
-    permissionId: string,
+    permissionId: BuiltinPermissionID,
     uiMetadata?: RoleUIMetadata
   ) {
     this._id = id

packages/backend-core/src/security/tests/permissions.spec.ts

@@ -1,6 +1,7 @@
 import cloneDeep from "lodash/cloneDeep"
 import * as permissions from "../permissions"
 import { BUILTIN_ROLE_IDS } from "../roles"
+import { BuiltinPermissionID } from "@budibase/types"
 
 describe("levelToNumber", () => {
   it("should return 0 for EXECUTE", () => {
@@ -77,7 +78,7 @@ describe("doesHaveBasePermission", () => {
     const rolesHierarchy = [
       {
         roleId: BUILTIN_ROLE_IDS.ADMIN,
-        permissionId: permissions.BuiltinPermissionID.ADMIN,
+        permissionId: BuiltinPermissionID.ADMIN,
       },
     ]
     expect(
@@ -91,7 +92,7 @@ describe("doesHaveBasePermission", () => {
     const rolesHierarchy = [
       {
         roleId: BUILTIN_ROLE_IDS.PUBLIC,
-        permissionId: permissions.BuiltinPermissionID.PUBLIC,
+        permissionId: BuiltinPermissionID.PUBLIC,
       },
     ]
     expect(
@@ -129,7 +130,7 @@ describe("getBuiltinPermissions", () => {
 describe("getBuiltinPermissionByID", () => {
   it("returns correct permission object for valid ID", () => {
     const expectedPermission = {
-      _id: permissions.BuiltinPermissionID.PUBLIC,
+      _id: BuiltinPermissionID.PUBLIC,
       name: "Public",
       permissions: [
         new permissions.Permission(

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -3,7 +3,7 @@
     automationStore,
     selectedAutomation,
     permissions,
-    selectedAutomationDisplayData,
+    tables,
   } from "stores/builder"
   import {
     Icon,
@@ -17,6 +17,7 @@
     AbsTooltip,
     InlineAlert,
   } from "@budibase/bbui"
+  import { sdk } from "@budibase/shared-core"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
   import ActionModal from "./ActionModal.svelte"
@@ -51,7 +52,12 @@
   $: isAppAction && setPermissions(role)
   $: isAppAction && getPermissions(automationId)
 
-  $: triggerInfo = $selectedAutomationDisplayData?.triggerInfo
+  $: triggerInfo = sdk.automations.isRowAction($selectedAutomation) && {
+    title: "Automation trigger",
+    tableName: $tables.list.find(
+      x => x._id === $selectedAutomation.definition.trigger.inputs?.tableId
+    )?.name,
+  }
 
   async function setPermissions(role) {
     if (!role || !automationId) {
@@ -187,10 +193,10 @@
           {block}
           {webhookModal}
         />
-        {#if isTrigger && triggerInfo}
+        {#if triggerInfo}
           <InlineAlert
-            header={triggerInfo.type}
-            message={`This trigger is tied to the "${triggerInfo.rowAction.name}" row action in your ${triggerInfo.table.name} table`}
+            header={triggerInfo.title}
+            message={`This trigger is tied to your "${triggerInfo.tableName}" table`}
           />
         {/if}
         {#if lastStep}

packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte

@@ -112,7 +112,7 @@
   iconColor={automation.disabled
     ? "var(--spectrum-global-color-gray-600)"
     : "var(--spectrum-global-color-gray-900)"}
-  text={automation.displayName}
+  text={automation.name}
   selected={automation._id === $selectedAutomation?._id}
   hovering={automation._id === $contextMenuStore.id}
   on:click={() => automationStore.actions.select(automation._id)}

packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte

@@ -25,15 +25,9 @@
         automation.name.toLowerCase().includes(searchString.toLowerCase())
       )
     })
-    .map(automation => ({
-      ...automation,
-      displayName:
-        $automationStore.automationDisplayData[automation._id]?.displayName ||
-        automation.name,
-    }))
     .sort((a, b) => {
-      const lowerA = a.displayName.toLowerCase()
-      const lowerB = b.displayName.toLowerCase()
+      const lowerA = a.name.toLowerCase()
+      const lowerB = b.name.toLowerCase()
       return lowerA > lowerB ? 1 : -1
     })
 

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -798,7 +798,9 @@
         break
       }
     }
-    return utils.processSearchFilters(filters)
+    return Array.isArray(filters)
+      ? utils.processSearchFilters(filters)
+      : filters
   }
 
   function saveFilters(key) {

packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte

@@ -5,7 +5,6 @@
   import { getUserBindings } from "dataBinding"
   import { makePropSafe } from "@budibase/string-templates"
   import { search } from "@budibase/frontend-core"
-  import { utils } from "@budibase/shared-core"
   import { tables } from "stores/builder"
 
   export let schema
@@ -17,19 +16,16 @@
 
   let drawer
 
-  $: localFilters = utils.processSearchFilters(filters)
-
+  $: localFilters = filters
   $: schemaFields = search.getFields(
     $tables.list,
     Object.values(schema || {}),
     { allowLinks: true }
   )
-
   $: filterCount =
     localFilters?.groups?.reduce((acc, group) => {
       return (acc += group.filters.filter(filter => filter.field).length)
     }, 0) || 0
-
   $: bindings = [
     {
       type: "context",
@@ -61,7 +57,7 @@
   title="Filtering"
   on:drawerHide
   on:drawerShow={() => {
-    localFilters = utils.processSearchFilters(filters)
+    localFilters = filters
   }}
   forceModal
 >

packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js

@@ -79,6 +79,8 @@ describe("Export Modal", () => {
       props: propsCfg,
     })
 
+    expect(propsCfg.filters[0].field).toBe("1:Cost")
+
     expect(screen.getByTestId("filters-applied")).toBeVisible()
     expect(screen.getByTestId("filters-applied").textContent).toBe(
       "Filters applied"

packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte

@@ -92,6 +92,7 @@
   disabled={error ||
     !name ||
     (rows.length && (!allValid || displayColumn == null))}
+  size="M"
 >
   <Input
     thin

packages/builder/src/stores/builder/automations.js

@@ -23,7 +23,6 @@ const initialAutomationState = {
     ACTION: {},
   },
   selectedAutomationId: null,
-  automationDisplayData: {},
 }
 
 // If this functions, remove the actions elements
@@ -83,7 +82,7 @@ const automationActions = store => ({
   },
   fetch: async () => {
     const [automationResponse, definitions] = await Promise.all([
-      API.getAutomations({ enrich: true }),
+      API.getAutomations(),
       API.getAutomationDefinitions(),
     ])
     store.update(state => {
@@ -91,7 +90,6 @@ const automationActions = store => ({
       state.automations.sort((a, b) => {
         return a.name < b.name ? -1 : 1
       })
-      state.automationDisplayData = automationResponse.builderData
       state.blockDefinitions = getFinalDefinitions(
         definitions.trigger,
         definitions.action
@@ -153,8 +151,6 @@ const automationActions = store => ({
         state.selectedAutomationId = state.automations[0]?._id || null
       }
 
-      // Clear out automationDisplayData for the automation
-      delete state.automationDisplayData[automation._id]
       return state
     })
   },
@@ -432,13 +428,3 @@ export const selectedAutomation = derived(automationStore, $automationStore => {
     x => x._id === $automationStore.selectedAutomationId
   )
 })
-
-export const selectedAutomationDisplayData = derived(
-  [automationStore, selectedAutomation],
-  ([$automationStore, $selectedAutomation]) => {
-    if (!$selectedAutomation?._id) {
-      return null
-    }
-    return $automationStore.automationDisplayData[$selectedAutomation._id]
-  }
-)

packages/builder/src/stores/builder/components.js

@@ -207,7 +207,6 @@ export class ComponentStore extends BudiStore {
     )
     for (let setting of filterableTypes || []) {
       const isLegacy = Array.isArray(enrichedComponent[setting.key])
-
       if (isLegacy) {
         const processedSetting = utils.processSearchFilters(
           enrichedComponent[setting.key]

packages/builder/src/stores/builder/index.js

@@ -11,7 +11,6 @@ import {
   automationStore,
   selectedAutomation,
   automationHistoryStore,
-  selectedAutomationDisplayData,
 } from "./automations.js"
 import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js"
 import { deploymentStore } from "./deployments.js"
@@ -46,7 +45,6 @@ export {
   previewStore,
   automationStore,
   selectedAutomation,
-  selectedAutomationDisplayData,
   automationHistoryStore,
   sortedScreens,
   userStore,

packages/builder/src/stores/builder/viewsV2.js

@@ -1,30 +1,6 @@
 import { writable, derived, get } from "svelte/store"
 import { tables } from "./tables"
 import { API } from "api"
-import { dataFilters } from "@budibase/shared-core"
-
-function convertToSearchFilters(view) {
-  // convert from SearchFilterGroup type
-  if (view?.query) {
-    return {
-      ...view,
-      queryUI: view.query,
-      query: dataFilters.buildQuery(view.query),
-    }
-  }
-  return view
-}
-
-function convertToSearchFilterGroup(view) {
-  if (view?.queryUI) {
-    return {
-      ...view,
-      query: view.queryUI,
-      queryUI: undefined,
-    }
-  }
-  return view
-}
 
 export function createViewsV2Store() {
   const store = writable({
@@ -36,7 +12,7 @@ export function createViewsV2Store() {
       const views = Object.values(table?.views || {}).filter(view => {
         return view.version === 2
       })
-      list = list.concat(views.map(view => convertToSearchFilterGroup(view)))
+      list = list.concat(views)
     })
     return {
       ...$store,
@@ -58,7 +34,6 @@ export function createViewsV2Store() {
   }
 
   const create = async view => {
-    view = convertToSearchFilters(view)
     const savedViewResponse = await API.viewV2.create(view)
     const savedView = savedViewResponse.data
     replaceView(savedView.id, savedView)
@@ -66,7 +41,6 @@ export function createViewsV2Store() {
   }
 
   const save = async view => {
-    view = convertToSearchFilters(view)
     const res = await API.viewV2.update(view)
     const savedView = res?.data
     replaceView(view.id, savedView)
@@ -77,7 +51,6 @@ export function createViewsV2Store() {
     if (!viewId) {
       return
     }
-    view = convertToSearchFilterGroup(view)
     const existingView = get(derivedStore).list.find(view => view.id === viewId)
     const tableIndex = get(tables).list.findIndex(table => {
       return table._id === view?.tableId || table._id === existingView?.tableId

packages/frontend-core/src/api/automations.js

@@ -26,14 +26,9 @@ export const buildAutomationEndpoints = API => ({
   /**
    * Gets a list of all automations.
    */
-  getAutomations: async ({ enrich }) => {
-    const params = new URLSearchParams()
-    if (enrich) {
-      params.set("enrich", true)
-    }
-
+  getAutomations: async () => {
     return await API.get({
-      url: `/api/automations?${params.toString()}`,
+      url: "/api/automations",
     })
   },
 

packages/frontend-core/src/components/CoreFilterBuilder.svelte

@@ -10,12 +10,13 @@
   } from "@budibase/bbui"
   import {
     FieldType,
-    FilterGroupLogicalOperator,
+    UILogicalOperator,
     EmptyFilterOption,
   } from "@budibase/types"
   import { QueryUtils, Constants } from "@budibase/frontend-core"
   import { getContext, createEventDispatcher } from "svelte"
   import FilterField from "./FilterField.svelte"
+  import { utils } from "@budibase/shared-core"
 
   const dispatch = createEventDispatcher()
   const {
@@ -39,8 +40,7 @@
   export let toReadable
   export let toRuntime
 
-  $: editableFilters = filters ? Helpers.cloneDeep(filters) : null
-
+  $: editableFilters = migrateFilters(filters)
   $: {
     if (
       tables.find(
@@ -54,6 +54,16 @@
     }
   }
 
+  // We still may need to migrate this even though the backend does it automatically now
+  // for query definitions. This is because we might be editing saved filter definitions
+  // from old screens, which will still be of type LegacyFilter[].
+  const migrateFilters = filters => {
+    if (Array.isArray(filters)) {
+      return utils.processSearchFilters(filters)
+    }
+    return Helpers.cloneDeep(filters)
+  }
+
   const filterOperatorOptions = Object.values(FilterOperator).map(entry => {
     return { value: entry, label: Helpers.capitalise(entry) }
   })
@@ -222,7 +232,7 @@
     } else if (addGroup) {
       if (!editable?.groups?.length) {
         editable = {
-          logicalOperator: FilterGroupLogicalOperator.ALL,
+          logicalOperator: UILogicalOperator.ALL,
           onEmptyFilter: EmptyFilterOption.RETURN_NONE,
           groups: [],
         }

packages/frontend-core/src/components/grid/stores/datasources/viewV2.js

@@ -1,26 +1,13 @@
 import { get } from "svelte/store"
-import { dataFilters } from "@budibase/shared-core"
 import { SortOrder } from "@budibase/types"
 
-function convertToSearchFilters(view) {
-  // convert from SearchFilterGroup type
-  if (view?.query) {
-    return {
-      ...view,
-      queryUI: view.query,
-      query: dataFilters.buildQuery(view.query),
-    }
-  }
-  return view
-}
-
 const SuppressErrors = true
 
 export const createActions = context => {
   const { API, datasource, columns } = context
 
   const saveDefinition = async newDefinition => {
-    await API.viewV2.update(convertToSearchFilters(newDefinition))
+    await API.viewV2.update(newDefinition)
   }
 
   const saveRow = async row => {
@@ -139,7 +126,7 @@ export const initialise = context => {
         }
         // Only override filter state if we don't have an initial filter
         if (!get(initialFilter)) {
-          filter.set($definition.queryUI || $definition.query)
+          filter.set($definition.queryUI)
         }
       })
     )
@@ -198,7 +185,7 @@ export const initialise = context => {
         if (JSON.stringify($filter) !== JSON.stringify($view.queryUI)) {
           await datasource.actions.saveDefinition({
             ...$view,
-            query: $filter,
+            queryUI: $filter,
           })
 
           // Refresh data since view definition changed

packages/frontend-core/src/components/grid/stores/filter.js

@@ -1,5 +1,5 @@
 import { get, derived } from "svelte/store"
-import { FieldType, FilterGroupLogicalOperator } from "@budibase/types"
+import { FieldType, UILogicalOperator } from "@budibase/types"
 import { memo } from "../../../utils/memo"
 
 export const createStores = context => {
@@ -25,10 +25,10 @@ export const deriveStores = context => {
         return $filter
       }
       let allFilters = {
-        logicalOperator: FilterGroupLogicalOperator.ALL,
+        logicalOperator: UILogicalOperator.ALL,
         groups: [
           {
-            logicalOperator: FilterGroupLogicalOperator.ALL,
+            logicalOperator: UILogicalOperator.ALL,
             filters: $inlineFilters,
           },
         ],

packages/server/src/api/controllers/automation.ts

@@ -76,16 +76,8 @@ export async function update(ctx: UserCtx) {
 }
 
 export async function fetch(ctx: UserCtx<void, FetchAutomationResponse>) {
-  const query: { enrich?: string } = ctx.request.query || {}
-  const enrich = query.enrich === "true"
-
   const automations = await sdk.automations.fetch()
   ctx.body = { automations }
-  if (enrich) {
-    ctx.body.builderData = await sdk.automations.utils.getBuilderData(
-      automations
-    )
-  }
 }
 
 export async function find(ctx: UserCtx) {

packages/server/src/api/controllers/role.ts

@@ -18,7 +18,7 @@ import {
   UserCtx,
   UserMetadata,
   DocumentType,
-  PermissionLevel,
+  BuiltinPermissionID,
 } from "@budibase/types"
 import { RoleColor, sdk as sharedSdk, helpers } from "@budibase/shared-core"
 import sdk from "../../sdk"
@@ -134,7 +134,13 @@ export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
   }
   // assume write permission level for newly created roles
   if (isCreate && !permissionId) {
-    permissionId = PermissionLevel.WRITE
+    permissionId = BuiltinPermissionID.WRITE
+  } else if (!permissionId && dbRole?.permissionId) {
+    permissionId = dbRole.permissionId
+  }
+
+  if (!permissionId) {
+    ctx.throw(400, "Role requires permissionId to be specified.")
   }
 
   const role = new roles.Role(_id, name, permissionId, {

packages/server/src/api/routes/tests/application.spec.ts

@@ -16,7 +16,7 @@ import * as setup from "./utilities"
 import { AppStatus } from "../../../db/utils"
 import { events, utils, context, features } from "@budibase/backend-core"
 import env from "../../../environment"
-import { type App } from "@budibase/types"
+import { type App, BuiltinPermissionID } from "@budibase/types"
 import tk from "timekeeper"
 import * as uuid from "uuid"
 import { structures } from "@budibase/backend-core/tests"
@@ -80,7 +80,7 @@ describe("/applications", () => {
       const role = await config.api.roles.save({
         name: "Test",
         inherits: "PUBLIC",
-        permissionId: "read_only",
+        permissionId: BuiltinPermissionID.READ_ONLY,
         version: "name",
       })
 
@@ -112,7 +112,7 @@ describe("/applications", () => {
       const role = await config.api.roles.save({
         name: roleName,
         inherits: "PUBLIC",
-        permissionId: "read_only",
+        permissionId: BuiltinPermissionID.READ_ONLY,
         version: "name",
       })
 

packages/server/src/api/routes/tests/permissions.spec.ts

@@ -1,5 +1,12 @@
 import { roles } from "@budibase/backend-core"
-import { Document, PermissionLevel, Role, Row, Table } from "@budibase/types"
+import {
+  BuiltinPermissionID,
+  Document,
+  PermissionLevel,
+  Role,
+  Row,
+  Table,
+} from "@budibase/types"
 import * as setup from "./utilities"
 import { generator, mocks } from "@budibase/backend-core/tests"
 
@@ -304,7 +311,7 @@ describe("/permission", () => {
       role1 = await config.api.roles.save(
         {
           name: "test_1",
-          permissionId: PermissionLevel.WRITE,
+          permissionId: BuiltinPermissionID.WRITE,
           inherits: BUILTIN_ROLE_IDS.BASIC,
         },
         { status: 200 }
@@ -312,7 +319,7 @@ describe("/permission", () => {
       role2 = await config.api.roles.save(
         {
           name: "test_2",
-          permissionId: PermissionLevel.WRITE,
+          permissionId: BuiltinPermissionID.WRITE,
           inherits: BUILTIN_ROLE_IDS.BASIC,
         },
         { status: 200 }
@@ -345,7 +352,7 @@ describe("/permission", () => {
     it("should be able to fetch two tables, with different roles, using multi-inheritance", async () => {
       const role3 = await config.api.roles.save({
         name: "role3",
-        permissionId: PermissionLevel.WRITE,
+        permissionId: BuiltinPermissionID.WRITE,
         inherits: [role1._id!, role2._id!],
       })
 

packages/server/src/api/routes/tests/role.spec.ts

@@ -1,15 +1,9 @@
-import {
-  roles,
-  events,
-  permissions,
-  db as dbCore,
-} from "@budibase/backend-core"
+import { roles, events, db as dbCore } from "@budibase/backend-core"
 import * as setup from "./utilities"
-import { PermissionLevel } from "@budibase/types"
+import { PermissionLevel, BuiltinPermissionID } from "@budibase/types"
 
 const { basicRole } = setup.structures
 const { BUILTIN_ROLE_IDS } = roles
-const { BuiltinPermissionID } = permissions
 
 const LOOP_ERROR = "Role inheritance contains a loop, this is not supported"
 
@@ -58,6 +52,19 @@ describe("/roles", () => {
       })
       expect(res.inherits).toEqual([BUILTIN_ROLE_IDS.BASIC])
     })
+
+    it("save role without permissionId", async () => {
+      const res = await config.api.roles.save(
+        {
+          ...basicRole(),
+          permissionId: undefined,
+        },
+        {
+          status: 200,
+        }
+      )
+      expect(res.permissionId).toEqual(PermissionLevel.WRITE)
+    })
   })
 
   describe("update", () => {
@@ -149,7 +156,7 @@ describe("/roles", () => {
         _id: id1,
         name: id1,
         permissions: {},
-        permissionId: "write",
+        permissionId: BuiltinPermissionID.WRITE,
         version: "name",
         inherits: ["POWER"],
       })
@@ -157,7 +164,7 @@ describe("/roles", () => {
         _id: id2,
         permissions: {},
         name: id2,
-        permissionId: "write",
+        permissionId: BuiltinPermissionID.WRITE,
         version: "name",
         inherits: [id1],
       })
@@ -176,10 +183,25 @@ describe("/roles", () => {
         inherits: [BUILTIN_ROLE_IDS.ADMIN],
       })
       // remove the roles so that it will default back to DB roles, then save again
-      delete res.inherits
-      const updatedRes = await config.api.roles.save(res)
+      const updatedRes = await config.api.roles.save({
+        ...res,
+        inherits: undefined,
+      })
       expect(updatedRes.inherits).toEqual([BUILTIN_ROLE_IDS.ADMIN])
     })
+
+    it("handle updating a role, without its permissionId", async () => {
+      const res = await config.api.roles.save({
+        ...basicRole(),
+        permissionId: BuiltinPermissionID.READ_ONLY,
+      })
+      // permission ID can be removed during update
+      const updatedRes = await config.api.roles.save({
+        ...res,
+        permissionId: undefined,
+      })
+      expect(updatedRes.permissionId).toEqual(BuiltinPermissionID.READ_ONLY)
+    })
   })
 
   describe("fetch", () => {
@@ -210,9 +232,7 @@ describe("/roles", () => {
       const customRoleFetched = res.find(r => r._id === customRole.name)
       expect(customRoleFetched).toBeDefined()
       expect(customRoleFetched!.inherits).toEqual(BUILTIN_ROLE_IDS.BASIC)
-      expect(customRoleFetched!.permissionId).toEqual(
-        BuiltinPermissionID.READ_ONLY
-      )
+      expect(customRoleFetched!.permissionId).toEqual(BuiltinPermissionID.WRITE)
     })
 
     it("should be able to get the role with a permission added", async () => {
@@ -316,7 +336,7 @@ describe("/roles", () => {
       await config.api.roles.save({
         name: customRoleName,
         inherits: roles.BUILTIN_ROLE_IDS.BASIC,
-        permissionId: permissions.BuiltinPermissionID.READ_ONLY,
+        permissionId: BuiltinPermissionID.READ_ONLY,
         version: "name",
       })
       await config.withHeaders(
@@ -356,19 +376,19 @@ describe("/roles", () => {
       const { _id: roleId1 } = await config.api.roles.save({
         name: role1,
         inherits: roles.BUILTIN_ROLE_IDS.BASIC,
-        permissionId: permissions.BuiltinPermissionID.WRITE,
+        permissionId: BuiltinPermissionID.WRITE,
         version: "name",
       })
       const { _id: roleId2 } = await config.api.roles.save({
         name: role2,
         inherits: roles.BUILTIN_ROLE_IDS.POWER,
-        permissionId: permissions.BuiltinPermissionID.POWER,
+        permissionId: BuiltinPermissionID.POWER,
         version: "name",
       })
       await config.api.roles.save({
         name: role3,
         inherits: [roleId1!, roleId2!],
-        permissionId: permissions.BuiltinPermissionID.READ_ONLY,
+        permissionId: BuiltinPermissionID.READ_ONLY,
         version: "name",
       })
       const headers = await config.roleHeaders({

packages/server/src/api/routes/tests/screen.spec.ts

@@ -1,7 +1,7 @@
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import { events, roles } from "@budibase/backend-core"
-import { Screen, PermissionLevel, Role } from "@budibase/types"
+import { Screen, Role, BuiltinPermissionID } from "@budibase/types"
 
 const { basicScreen } = setup.structures
 
@@ -40,17 +40,17 @@ describe("/screens", () => {
       role1 = await config.api.roles.save({
         name: "role1",
         inherits: roles.BUILTIN_ROLE_IDS.BASIC,
-        permissionId: PermissionLevel.WRITE,
+        permissionId: BuiltinPermissionID.WRITE,
       })
       role2 = await config.api.roles.save({
         name: "role2",
         inherits: roles.BUILTIN_ROLE_IDS.BASIC,
-        permissionId: PermissionLevel.WRITE,
+        permissionId: BuiltinPermissionID.WRITE,
       })
       multiRole = await config.api.roles.save({
         name: "multiRole",
         inherits: [role1._id!, role2._id!],
-        permissionId: PermissionLevel.WRITE,
+        permissionId: BuiltinPermissionID.WRITE,
       })
       screen1 = await config.api.screen.save(
         {

packages/server/src/api/routes/tests/search.spec.ts

@@ -1688,7 +1688,7 @@ describe.each([
         })
       })
 
-    describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
+    describe("arrays", () => {
       beforeAll(async () => {
         tableOrViewId = await createTableOrView({
           numbers: {

packages/server/src/api/routes/utils/validators.ts

@@ -8,6 +8,7 @@ import {
   SearchFilters,
   Table,
   WebhookActionType,
+  BuiltinPermissionID,
 } from "@budibase/types"
 import Joi, { CustomValidator } from "joi"
 import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core"
@@ -214,8 +215,8 @@ export function roleValidator() {
       }).optional(),
       // this is the base permission ID (for now a built in)
       permissionId: Joi.string()
-        .valid(...Object.values(permissions.BuiltinPermissionID))
-        .required(),
+        .valid(...Object.values(BuiltinPermissionID))
+        .optional(),
       permissions: Joi.object()
         .pattern(
           /.*/,

packages/server/src/sdk/app/automations/utils.ts

@@ -1,56 +1,7 @@
-import {
-  Automation,
-  AutomationActionStepId,
-  AutomationBuilderData,
-} from "@budibase/types"
-import { sdk as coreSdk } from "@budibase/shared-core"
-import sdk from "../../../sdk"
+import { Automation, AutomationActionStepId } from "@budibase/types"
 
 export function checkForCollectStep(automation: Automation) {
   return automation.definition.steps.some(
     (step: any) => step.stepId === AutomationActionStepId.COLLECT
   )
 }
-
-export async function getBuilderData(
-  automations: Automation[]
-): Promise<Record<string, AutomationBuilderData>> {
-  const tableNameCache: Record<string, string> = {}
-  async function getTableName(tableId: string) {
-    if (!tableNameCache[tableId]) {
-      const table = await sdk.tables.getTable(tableId)
-      tableNameCache[tableId] = table.name
-    }
-
-    return tableNameCache[tableId]
-  }
-
-  const result: Record<string, AutomationBuilderData> = {}
-  for (const automation of automations) {
-    const isRowAction = coreSdk.automations.isRowAction(automation)
-    if (!isRowAction) {
-      result[automation._id!] = { displayName: automation.name }
-      continue
-    }
-
-    const { tableId, rowActionId } = automation.definition.trigger.inputs
-    if (!tableId || !rowActionId) {
-      continue
-    }
-
-    const tableName = await getTableName(tableId)
-    const rowActionName = automation.name
-    result[automation._id!] = {
-      displayName: rowActionName,
-      triggerInfo: {
-        type: "Automation trigger",
-        table: { id: tableId, name: tableName },
-        rowAction: {
-          id: rowActionId,
-          name: rowActionName,
-        },
-      },
-    }
-  }
-  return result
-}

packages/server/src/sdk/app/rows/search.ts

@@ -6,6 +6,7 @@ import {
   Row,
   RowSearchParams,
   SearchFilterKey,
+  SearchFilters,
   SearchResponse,
   SortOrder,
   Table,
@@ -90,17 +91,18 @@ export async function search(
     options = searchInputMapping(table, options)
 
     if (options.viewId) {
-      // Delete extraneous search params that cannot be overridden
-      delete options.query.onEmptyFilter
-
       const view = source as ViewV2
+
       // Enrich saved query with ephemeral query params.
       // We prevent searching on any fields that are saved as part of the query, as
       // that could let users find rows they should not be allowed to access.
-      let viewQuery = await enrichSearchContext(view.query || {}, context)
-      viewQuery = dataFilters.buildQueryLegacy(viewQuery) || {}
+      let viewQuery = (await enrichSearchContext(view.query || {}, context)) as
+        | SearchFilters
+        | LegacyFilter[]
+      if (Array.isArray(viewQuery)) {
+        viewQuery = dataFilters.buildQuery(viewQuery)
+      }
       viewQuery = checkFilters(table, viewQuery)
-      delete viewQuery?.onEmptyFilter
 
       const sqsEnabled = await features.flags.isEnabled(FeatureFlag.SQS)
       const supportsLogicalOperators =
@@ -113,13 +115,12 @@ export async function search(
           ? view.query
           : []
 
-        delete options.query.onEmptyFilter
+        const { filters } = dataFilters.splitFiltersArray(queryFilters)
 
         // Extract existing fields
-        const existingFields =
-          queryFilters
-            ?.filter(filter => filter.field)
-            .map(filter => db.removeKeyNumbering(filter.field)) || []
+        const existingFields = filters.map(filter =>
+          db.removeKeyNumbering(filter.field)
+        )
 
         // Carry over filters for unused fields
         Object.keys(options.query).forEach(key => {

packages/server/src/sdk/app/tables/getters.ts

@@ -16,6 +16,8 @@ import {
 } from "@budibase/types"
 import datasources from "../datasources"
 import sdk from "../../../sdk"
+import { ensureQueryUISet } from "../views/utils"
+import { isV2 } from "../views"
 
 export async function processTable(table: Table): Promise<Table> {
   if (!table) {
@@ -23,6 +25,14 @@ export async function processTable(table: Table): Promise<Table> {
   }
 
   table = { ...table }
+  if (table.views) {
+    for (const [key, view] of Object.entries(table.views)) {
+      if (!isV2(view)) {
+        continue
+      }
+      table.views[key] = ensureQueryUISet(view)
+    }
+  }
   if (table._id && isExternalTableID(table._id)) {
     // Old created external tables via Budibase might have a missing field name breaking some UI such as filters
     if (table.schema["id"] && !table.schema["id"].name) {

packages/server/src/sdk/app/views/external.ts

@@ -5,6 +5,7 @@ import sdk from "../../../sdk"
 import * as utils from "../../../db/utils"
 import { enrichSchema, isV2 } from "."
 import { breakExternalTableId } from "../../../integrations/utils"
+import { ensureQuerySet, ensureQueryUISet } from "./utils"
 
 export async function get(viewId: string): Promise<ViewV2> {
   const { tableId } = utils.extractViewInfoFromID(viewId)
@@ -18,7 +19,7 @@ export async function get(viewId: string): Promise<ViewV2> {
   if (!found) {
     throw new Error("No view found")
   }
-  return found
+  return ensureQueryUISet(found)
 }
 
 export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
@@ -33,19 +34,22 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
   if (!found) {
     throw new Error("No view found")
   }
-  return await enrichSchema(found, table.schema)
+  return await enrichSchema(ensureQueryUISet(found), table.schema)
 }
 
 export async function create(
   tableId: string,
   viewRequest: Omit<ViewV2, "id" | "version">
 ): Promise<ViewV2> {
-  const view: ViewV2 = {
+  let view: ViewV2 = {
     ...viewRequest,
     id: utils.generateViewID(tableId),
     version: 2,
   }
 
+  view = ensureQuerySet(view)
+  view = ensureQueryUISet(view)
+
   const db = context.getAppDB()
 
   const { datasourceId, tableName } = breakExternalTableId(tableId)
@@ -56,7 +60,10 @@ export async function create(
   return view
 }
 
-export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
+export async function update(
+  tableId: string,
+  view: Readonly<ViewV2>
+): Promise<ViewV2> {
   const db = context.getAppDB()
 
   const { datasourceId, tableName } = breakExternalTableId(tableId)
@@ -74,6 +81,9 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
     throw new HTTPError(`Cannot update view type after creation`, 400)
   }
 
+  view = ensureQuerySet(view)
+  view = ensureQueryUISet(view)
+
   delete views[existingView.name]
   views[view.name] = view
   await db.put(ds)

packages/server/src/sdk/app/views/internal.ts

@@ -4,6 +4,7 @@ import { context, HTTPError } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import * as utils from "../../../db/utils"
 import { enrichSchema, isV2 } from "."
+import { ensureQuerySet, ensureQueryUISet } from "./utils"
 
 export async function get(viewId: string): Promise<ViewV2> {
   const { tableId } = utils.extractViewInfoFromID(viewId)
@@ -13,7 +14,7 @@ export async function get(viewId: string): Promise<ViewV2> {
   if (!found) {
     throw new Error("No view found")
   }
-  return found
+  return ensureQueryUISet(found)
 }
 
 export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
@@ -24,19 +25,22 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
   if (!found) {
     throw new Error("No view found")
   }
-  return await enrichSchema(found, table.schema)
+  return await enrichSchema(ensureQueryUISet(found), table.schema)
 }
 
 export async function create(
   tableId: string,
   viewRequest: Omit<ViewV2, "id" | "version">
 ): Promise<ViewV2> {
-  const view: ViewV2 = {
+  let view: ViewV2 = {
     ...viewRequest,
     id: utils.generateViewID(tableId),
     version: 2,
   }
 
+  view = ensureQuerySet(view)
+  view = ensureQueryUISet(view)
+
   const db = context.getAppDB()
 
   const table = await sdk.tables.getTable(tableId)
@@ -47,7 +51,10 @@ export async function create(
   return view
 }
 
-export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
+export async function update(
+  tableId: string,
+  view: Readonly<ViewV2>
+): Promise<ViewV2> {
   const db = context.getAppDB()
   const table = await sdk.tables.getTable(tableId)
   table.views ??= {}
@@ -63,6 +70,9 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
     throw new HTTPError(`Cannot update view type after creation`, 400)
   }
 
+  view = ensureQuerySet(view)
+  view = ensureQueryUISet(view)
+
   delete table.views[existingView.name]
   table.views[view.name] = view
   await db.put(table)

packages/server/src/sdk/app/views/utils.ts

@@ -0,0 +1,43 @@
+import { ViewV2 } from "@budibase/types"
+import { utils, dataFilters } from "@budibase/shared-core"
+import { cloneDeep, isPlainObject } from "lodash"
+import { HTTPError } from "@budibase/backend-core"
+
+function isEmptyObject(obj: any) {
+  return obj && isPlainObject(obj) && Object.keys(obj).length === 0
+}
+
+export function ensureQueryUISet(viewArg: Readonly<ViewV2>): ViewV2 {
+  const view = cloneDeep<ViewV2>(viewArg)
+  if (!view.queryUI && view.query && !isEmptyObject(view.query)) {
+    if (!Array.isArray(view.query)) {
+      // In practice this should not happen. `view.query`, at the time this code
+      // goes into the codebase, only contains LegacyFilter[] in production.
+      // We're changing it in the change that this comment is part of to also
+      // include SearchFilters objects. These are created when we receive an
+      // update to a ViewV2 that contains a queryUI and not a query field. We
+      // can convert UISearchFilter (the type of queryUI) to SearchFilters,
+      // but not LegacyFilter[], they are incompatible due to UISearchFilter
+      // and SearchFilters being recursive types.
+      //
+      // So despite the type saying that `view.query` is a LegacyFilter[] |
+      // SearchFilters, it will never be a SearchFilters when a `view.queryUI`
+      // is specified, making it "safe" to throw an error here.
+      throw new HTTPError("view is missing queryUI field", 400)
+    }
+
+    view.queryUI = utils.processSearchFilters(view.query)
+  }
+  return view
+}
+
+export function ensureQuerySet(viewArg: Readonly<ViewV2>): ViewV2 {
+  const view = cloneDeep<ViewV2>(viewArg)
+  // We consider queryUI to be the source of truth, so we don't check for the
+  // presence of query here. We will overwrite it regardless of whether it is
+  // present or not.
+  if (view.queryUI && !isEmptyObject(view.queryUI)) {
+    view.query = dataFilters.buildQuery(view.queryUI)
+  }
+  return view
+}

packages/server/src/tests/utilities/structures.ts

@@ -1,4 +1,4 @@
-import { permissions, roles, utils } from "@budibase/backend-core"
+import { roles, utils } from "@budibase/backend-core"
 import { createHomeScreen } from "../../constants/screens"
 import { EMPTY_LAYOUT } from "../../constants/layouts"
 import { cloneDeep } from "lodash/fp"
@@ -33,6 +33,7 @@ import {
   TableSourceType,
   Webhook,
   WebhookActionType,
+  BuiltinPermissionID,
 } from "@budibase/types"
 import { LoopInput } from "../../definitions/automations"
 import { merge } from "lodash"
@@ -515,7 +516,7 @@ export function basicRole(): Role {
   return {
     name: `NewRole_${utils.newid()}`,
     inherits: roles.BUILTIN_ROLE_IDS.BASIC,
-    permissionId: permissions.BuiltinPermissionID.READ_ONLY,
+    permissionId: BuiltinPermissionID.WRITE,
     permissions: {},
     version: "name",
   }

packages/shared-core/src/filters.ts

@@ -19,8 +19,12 @@ import {
   RangeOperator,
   LogicalOperator,
   isLogicalSearchOperator,
-  SearchFilterGroup,
-  FilterGroupLogicalOperator,
+  UISearchFilter,
+  UILogicalOperator,
+  isBasicSearchOperator,
+  isArraySearchOperator,
+  isRangeSearchOperator,
+  SearchFilter,
 } from "@budibase/types"
 import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
@@ -310,308 +314,194 @@ export class ColumnSplitter {
  * @param filter the builder filter structure
  */
 
-const buildCondition = (expression: LegacyFilter) => {
-  // Filter body
-  let query: SearchFilters = {
-    string: {},
-    fuzzy: {},
-    range: {},
-    equal: {},
-    notEqual: {},
-    empty: {},
-    notEmpty: {},
-    contains: {},
-    notContains: {},
-    oneOf: {},
-    containsAny: {},
-  }
-  let { operator, field, type, value, externalType, onEmptyFilter } = expression
-
-  if (!operator || !field) {
+function buildCondition(filter: undefined): undefined
+function buildCondition(filter: SearchFilter): SearchFilters
+function buildCondition(filter?: SearchFilter): SearchFilters | undefined {
+  if (!filter) {
     return
   }
 
-  const queryOperator = operator as SearchFilterOperator
-  const isHbs =
-    typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
-  // Parse all values into correct types
-  if (operator === "allOr") {
-    query.allOr = true
-    return
-  }
-  if (onEmptyFilter) {
-    query.onEmptyFilter = onEmptyFilter
-    return
-  }
+  const query: SearchFilters = {}
+  const { operator, field, type, externalType } = filter
+  let { value } = filter
 
   // Default the value for noValue fields to ensure they are correctly added
   // to the final query
-  if (queryOperator === "empty" || queryOperator === "notEmpty") {
+  if (operator === "empty" || operator === "notEmpty") {
     value = null
   }
 
-  if (
-    type === "datetime" &&
-    !isHbs &&
-    queryOperator !== "empty" &&
-    queryOperator !== "notEmpty"
+  const isHbs =
+    typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
+
+  // Parsing value depending on what the type is.
+  switch (type) {
+    case FieldType.DATETIME:
+      if (!isHbs && operator !== "empty" && operator !== "notEmpty") {
+        if (!value) {
+          return
+        }
+        value = new Date(value).toISOString()
+      }
+      break
+    case FieldType.NUMBER:
+      if (typeof value === "string" && !isHbs) {
+        if (operator === "oneOf") {
+          value = value.split(",").map(parseFloat)
+        } else {
+          value = parseFloat(value)
+        }
+      }
+      break
+    case FieldType.BOOLEAN:
+      value = `${value}`.toLowerCase() === "true"
+      break
+    case FieldType.ARRAY:
+      if (
+        ["contains", "notContains", "containsAny"].includes(
+          operator.toLocaleString()
+        ) &&
+        typeof value === "string"
+      ) {
+        value = value.split(",")
+      }
+      break
+  }
+
+  if (isRangeSearchOperator(operator)) {
+    const key = externalType as keyof typeof SqlNumberTypeRangeMap
+    const limits = SqlNumberTypeRangeMap[key] || {
+      min: Number.MIN_SAFE_INTEGER,
+      max: Number.MAX_SAFE_INTEGER,
+    }
+
+    query[operator] ??= {}
+    query[operator][field] = {
+      low: type === "number" ? limits.min : "0000-00-00T00:00:00.000Z",
+      high: type === "number" ? limits.max : "9999-00-00T00:00:00.000Z",
+    }
+  } else if (operator === "rangeHigh" && value != null && value !== "") {
+    query.range ??= {}
+    query.range[field] = {
+      ...query.range[field],
+      high: value,
+    }
+  } else if (operator === "rangeLow" && value != null && value !== "") {
+    query.range ??= {}
+    query.range[field] = {
+      ...query.range[field],
+      low: value,
+    }
+  } else if (
+    isBasicSearchOperator(operator) ||
+    isArraySearchOperator(operator) ||
+    isRangeSearchOperator(operator)
   ) {
-    // Ensure date value is a valid date and parse into correct format
-    if (!value) {
-      return
-    }
-    try {
-      value = new Date(value).toISOString()
-    } catch (error) {
-      return
-    }
-  }
-  if (type === "number" && typeof value === "string" && !isHbs) {
-    if (queryOperator === "oneOf") {
-      value = value.split(",").map(item => parseFloat(item))
-    } else {
-      value = parseFloat(value)
-    }
-  }
-  if (type === "boolean") {
-    value = `${value}`?.toLowerCase() === "true"
-  }
-  if (
-    ["contains", "notContains", "containsAny"].includes(
-      operator.toLocaleString()
-    ) &&
-    type === "array" &&
-    typeof value === "string"
-  ) {
-    value = value.split(",")
-  }
-  if (operator.toLocaleString().startsWith("range") && query.range) {
-    const minint =
-      SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
-        ?.min || Number.MIN_SAFE_INTEGER
-    const maxint =
-      SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
-        ?.max || Number.MAX_SAFE_INTEGER
-    if (!query.range[field]) {
-      query.range[field] = {
-        low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
-        high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
-      }
-    }
-    if (operator === "rangeLow" && value != null && value !== "") {
-      query.range[field] = {
-        ...query.range[field],
-        low: value,
-      }
-    } else if (operator === "rangeHigh" && value != null && value !== "") {
-      query.range[field] = {
-        ...query.range[field],
-        high: value,
-      }
-    }
-  } else if (isLogicalSearchOperator(queryOperator)) {
-    // TODO
-  } else if (query[queryOperator] && operator !== "onEmptyFilter") {
     if (type === "boolean") {
-      // Transform boolean filters to cope with null.
-      // "equals false" needs to be "not equals true"
-      // "not equals false" needs to be "equals true"
-      if (queryOperator === "equal" && value === false) {
+      // TODO(samwho): I suspect this boolean transformation isn't needed anymore,
+      // write some tests to confirm.
+
+      // Transform boolean filters to cope with null.  "equals false" needs to
+      // be "not equals true" "not equals false" needs to be "equals true"
+      if (operator === "equal" && value === false) {
         query.notEqual = query.notEqual || {}
         query.notEqual[field] = true
-      } else if (queryOperator === "notEqual" && value === false) {
+      } else if (operator === "notEqual" && value === false) {
         query.equal = query.equal || {}
         query.equal[field] = true
       } else {
-        query[queryOperator] ??= {}
-        query[queryOperator]![field] = value
+        query[operator] ??= {}
+        query[operator][field] = value
       }
     } else {
-      query[queryOperator] ??= {}
-      query[queryOperator]![field] = value
+      query[operator] ??= {}
+      query[operator][field] = value
     }
+  } else {
+    throw new Error(`Unsupported operator: ${operator}`)
   }
 
   return query
 }
 
-export const buildQueryLegacy = (
-  filter?: LegacyFilter[] | SearchFilters
-): SearchFilters | undefined => {
-  // this is of type SearchFilters or is undefined
-  if (!Array.isArray(filter)) {
-    return filter
+export interface LegacyFilterSplit {
+  allOr?: boolean
+  onEmptyFilter?: EmptyFilterOption
+  filters: SearchFilter[]
+}
+
+export function splitFiltersArray(filters: LegacyFilter[]) {
+  const split: LegacyFilterSplit = {
+    filters: [],
   }
 
-  let query: SearchFilters = {
-    string: {},
-    fuzzy: {},
-    range: {},
-    equal: {},
-    notEqual: {},
-    empty: {},
-    notEmpty: {},
-    contains: {},
-    notContains: {},
-    oneOf: {},
-    containsAny: {},
+  for (const filter of filters) {
+    if ("operator" in filter && filter.operator === "allOr") {
+      split.allOr = true
+    } else if ("onEmptyFilter" in filter) {
+      split.onEmptyFilter = filter.onEmptyFilter
+    } else {
+      split.filters.push(filter)
+    }
   }
 
-  if (!Array.isArray(filter)) {
-    return query
-  }
-
-  filter.forEach(expression => {
-    let { operator, field, type, value, externalType, onEmptyFilter } =
-      expression
-    const queryOperator = operator as SearchFilterOperator
-    const isHbs =
-      typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
-    // Parse all values into correct types
-    if (operator === "allOr") {
-      query.allOr = true
-      return
-    }
-    if (onEmptyFilter) {
-      query.onEmptyFilter = onEmptyFilter
-      return
-    }
-    if (
-      type === "datetime" &&
-      !isHbs &&
-      queryOperator !== "empty" &&
-      queryOperator !== "notEmpty"
-    ) {
-      // Ensure date value is a valid date and parse into correct format
-      if (!value) {
-        return
-      }
-      try {
-        value = new Date(value).toISOString()
-      } catch (error) {
-        return
-      }
-    }
-    if (type === "number" && typeof value === "string" && !isHbs) {
-      if (queryOperator === "oneOf") {
-        value = value.split(",").map(item => parseFloat(item))
-      } else {
-        value = parseFloat(value)
-      }
-    }
-    if (type === "boolean") {
-      value = `${value}`?.toLowerCase() === "true"
-    }
-    if (
-      ["contains", "notContains", "containsAny"].includes(
-        operator.toLocaleString()
-      ) &&
-      type === "array" &&
-      typeof value === "string"
-    ) {
-      value = value.split(",")
-    }
-    if (operator.toLocaleString().startsWith("range") && query.range) {
-      const minint =
-        SqlNumberTypeRangeMap[
-          externalType as keyof typeof SqlNumberTypeRangeMap
-        ]?.min || Number.MIN_SAFE_INTEGER
-      const maxint =
-        SqlNumberTypeRangeMap[
-          externalType as keyof typeof SqlNumberTypeRangeMap
-        ]?.max || Number.MAX_SAFE_INTEGER
-      if (!query.range[field]) {
-        query.range[field] = {
-          low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
-          high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
-        }
-      }
-      if (operator === "rangeLow" && value != null && value !== "") {
-        query.range[field] = {
-          ...query.range[field],
-          low: value,
-        }
-      } else if (operator === "rangeHigh" && value != null && value !== "") {
-        query.range[field] = {
-          ...query.range[field],
-          high: value,
-        }
-      }
-    } else if (isLogicalSearchOperator(queryOperator)) {
-      // ignore
-    } else if (query[queryOperator] && operator !== "onEmptyFilter") {
-      if (type === "boolean") {
-        // Transform boolean filters to cope with null.
-        // "equals false" needs to be "not equals true"
-        // "not equals false" needs to be "equals true"
-        if (queryOperator === "equal" && value === false) {
-          query.notEqual = query.notEqual || {}
-          query.notEqual[field] = true
-        } else if (queryOperator === "notEqual" && value === false) {
-          query.equal = query.equal || {}
-          query.equal[field] = true
-        } else {
-          query[queryOperator] ??= {}
-          query[queryOperator]![field] = value
-        }
-      } else {
-        query[queryOperator] ??= {}
-        query[queryOperator]![field] = value
-      }
-    }
-  })
-  return query
+  return split
 }
 
 /**
- * Converts a **SearchFilterGroup** filter definition into a grouped
+ * Converts a **UISearchFilter** filter definition into a grouped
  * search query of type **SearchFilters**
  *
  * Legacy support remains for the old **SearchFilter[]** format.
  * These will be migrated to an appropriate **SearchFilters** object, if encountered
- *
- * @param filter
- *
- * @returns {SearchFilters}
  */
-
-export const buildQuery = (
-  filter?: SearchFilterGroup | LegacyFilter[]
-): SearchFilters | undefined => {
-  const parsedFilter: SearchFilterGroup | undefined =
-    processSearchFilters(filter)
-
-  if (!parsedFilter) {
-    return
+export function buildQuery(
+  filter?: UISearchFilter | LegacyFilter[]
+): SearchFilters {
+  if (!filter) {
+    return {}
   }
 
-  const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } =
-    {
-      [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND,
-      [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR,
+  if (Array.isArray(filter)) {
+    filter = processSearchFilters(filter)
+    if (!filter) {
+      return {}
     }
-
-  const globalOnEmpty = parsedFilter.onEmptyFilter
-    ? parsedFilter.onEmptyFilter
-    : null
-
-  const globalOperator: LogicalOperator =
-    operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
-
-  return {
-    ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
-    [globalOperator]: {
-      conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
-        return {
-          [operatorMap[group.logicalOperator]]: {
-            conditions: group.filters
-              ?.map(x => buildCondition(x))
-              .filter(filter => filter),
-          },
-        }
-      }),
-    },
   }
+
+  const operator = logicalOperatorFromUI(
+    filter.logicalOperator || UILogicalOperator.ALL
+  )
+
+  const query: SearchFilters = {}
+  if (filter.onEmptyFilter) {
+    query.onEmptyFilter = filter.onEmptyFilter
+  } else {
+    query.onEmptyFilter = EmptyFilterOption.RETURN_ALL
+  }
+
+  query[operator] = {
+    conditions: (filter.groups || []).map(group => {
+      const { allOr, onEmptyFilter, filters } = splitFiltersArray(
+        group.filters || []
+      )
+      if (onEmptyFilter) {
+        query.onEmptyFilter = onEmptyFilter
+      }
+      const operator = allOr ? LogicalOperator.OR : LogicalOperator.AND
+      return {
+        [operator]: { conditions: filters.map(buildCondition).filter(f => f) },
+      }
+    }),
+  }
+
+  return query
+}
+
+function logicalOperatorFromUI(operator: UILogicalOperator): LogicalOperator {
+  return operator === UILogicalOperator.ALL
+    ? LogicalOperator.AND
+    : LogicalOperator.OR
 }
 
 // The frontend can send single values for array fields sometimes, so to handle

packages/shared-core/src/helpers/tests/roles.spec.ts

@@ -1,5 +1,5 @@
 import { checkForRoleInheritanceLoops } from "../roles"
-import { Role } from "@budibase/types"
+import { BuiltinPermissionID, Role } from "@budibase/types"
 
 /**
  * This unit test exists as this utility will be used in the frontend and backend, confirmation
@@ -19,7 +19,7 @@ function role(id: string, inherits: string | string[]): TestRole {
     _id: id,
     inherits: inherits,
     name: "ROLE",
-    permissionId: "PERMISSION",
+    permissionId: BuiltinPermissionID.WRITE,
     permissions: {}, // not needed for this test
   }
   allRoles.push(role)

packages/shared-core/src/utils.ts

@@ -1,18 +1,28 @@
 import {
   LegacyFilter,
-  SearchFilterGroup,
-  FilterGroupLogicalOperator,
+  UISearchFilter,
+  UILogicalOperator,
   SearchFilters,
   BasicOperator,
   ArrayOperator,
   isLogicalSearchOperator,
+  SearchFilter,
+  EmptyFilterOption,
 } from "@budibase/types"
 import * as Constants from "./constants"
-import { removeKeyNumbering } from "./filters"
+import { removeKeyNumbering, splitFiltersArray } from "./filters"
+import _ from "lodash"
 
-// an array of keys from filter type to properties that are in the type
-// this can then be converted using .fromEntries to an object
-type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][]
+const FILTER_ALLOWED_KEYS: (keyof SearchFilter)[] = [
+  "field",
+  "operator",
+  "value",
+  "type",
+  "externalType",
+  "valueType",
+  "noValue",
+  "formulaType",
+]
 
 export function unreachable(
   value: never,
@@ -125,100 +135,32 @@ export function isSupportedUserSearch(query: SearchFilters) {
       return false
     }
   }
+
   return true
 }
 
-/**
- * Processes the filter config. Filters are migrated from
- * SearchFilter[] to SearchFilterGroup
- *
- * If filters is not an array, the migration is skipped
- *
- * @param {LegacyFilter[] | SearchFilterGroup} filters
- */
-export const processSearchFilters = (
-  filters: LegacyFilter[] | SearchFilterGroup | undefined
-): SearchFilterGroup | undefined => {
-  if (!filters) {
-    return
+export function processSearchFilters(
+  filterArray?: LegacyFilter[]
+): Required<UISearchFilter> | undefined {
+  if (!filterArray || filterArray.length === 0) {
+    return undefined
   }
-
-  // Base search config.
-  const defaultCfg: SearchFilterGroup = {
-    logicalOperator: FilterGroupLogicalOperator.ALL,
-    groups: [],
+  const { allOr, onEmptyFilter, filters } = splitFiltersArray(filterArray)
+  return {
+    logicalOperator: UILogicalOperator.ALL,
+    onEmptyFilter: onEmptyFilter || EmptyFilterOption.RETURN_ALL,
+    groups: [
+      {
+        logicalOperator: allOr ? UILogicalOperator.ANY : UILogicalOperator.ALL,
+        filters: filters.map(filter => {
+          const trimmedFilter = _.pick(
+            filter,
+            FILTER_ALLOWED_KEYS
+          ) as SearchFilter
+          trimmedFilter.field = removeKeyNumbering(trimmedFilter.field)
+          return trimmedFilter
+        }),
+      },
+    ],
   }
-
-  const filterAllowedKeys = [
-    "field",
-    "operator",
-    "value",
-    "type",
-    "externalType",
-    "valueType",
-    "noValue",
-    "formulaType",
-  ]
-
-  if (Array.isArray(filters)) {
-    let baseGroup: SearchFilterGroup = {
-      filters: [],
-      logicalOperator: FilterGroupLogicalOperator.ALL,
-    }
-
-    return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
-      // Sort the properties for easier debugging
-      const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
-        .sort((a, b) => {
-          return a.localeCompare(b)
-        })
-        .filter(key => filter[key])
-
-      if (filterPropertyKeys.length == 1) {
-        const key = filterPropertyKeys[0],
-          value = filter[key]
-        // Global
-        if (key === "onEmptyFilter") {
-          // unset otherwise
-          acc.onEmptyFilter = value
-        } else if (key === "operator" && value === "allOr") {
-          // Group 1 logical operator
-          baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
-        }
-
-        return acc
-      }
-
-      const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
-        (acc: AllowedFilters, key) => {
-          const value = filter[key]
-          if (filterAllowedKeys.includes(key)) {
-            if (key === "field") {
-              acc.push([key, removeKeyNumbering(value)])
-            } else {
-              acc.push([key, value])
-            }
-          }
-          return acc
-        },
-        []
-      )
-
-      const migratedFilter: LegacyFilter = Object.fromEntries(
-        allowedFilterSettings
-      ) as LegacyFilter
-
-      baseGroup.filters!.push(migratedFilter)
-
-      if (!acc.groups || !acc.groups.length) {
-        // init the base group
-        acc.groups = [baseGroup]
-      }
-
-      return acc
-    }, defaultCfg)
-  } else if (!filters?.groups) {
-    return
-  }
-  return filters
 }

packages/types/src/api/web/automation.ts

@@ -3,22 +3,6 @@ import { Automation } from "../../documents"
 
 export interface DeleteAutomationResponse extends DocumentDestroyResponse {}
 
-export interface AutomationBuilderData {
-  displayName: string
-  triggerInfo?: {
-    type: string
-    table: {
-      id: string
-      name: string
-    }
-    rowAction: {
-      id: string
-      name: string
-    }
-  }
-}
-
 export interface FetchAutomationResponse {
   automations: Automation[]
-  builderData?: Record<string, AutomationBuilderData> // The key will be the automationId
 }

packages/types/src/api/web/role.ts

@@ -1,12 +1,12 @@
 import { Role, RoleUIMetadata } from "../../documents"
-import { PermissionLevel } from "../../sdk"
+import { PermissionLevel, BuiltinPermissionID } from "../../sdk"
 
 export interface SaveRoleRequest {
   _id?: string
   _rev?: string
   name: string
   inherits?: string | string[]
-  permissionId: string
+  permissionId?: BuiltinPermissionID
   permissions?: Record<string, PermissionLevel[]>
   version?: string
   uiMetadata?: RoleUIMetadata

packages/types/src/api/web/searchFilter.ts

@@ -1,23 +1,59 @@
 import { FieldType } from "../../documents"
 import {
   EmptyFilterOption,
-  FilterGroupLogicalOperator,
-  SearchFilters,
+  UILogicalOperator,
+  BasicOperator,
+  RangeOperator,
+  ArrayOperator,
 } from "../../sdk"
 
-export type LegacyFilter = {
-  operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
-  onEmptyFilter?: EmptyFilterOption
-  field: string
-  type?: FieldType
-  value: any
-  externalType?: string
+type AllOr = {
+  operator: "allOr"
 }
 
-// this is a type purely used by the UI
+type OnEmptyFilter = {
+  onEmptyFilter: EmptyFilterOption
+}
+
+// TODO(samwho): this could be broken down further
+export type SearchFilter = {
+  operator:
+    | BasicOperator
+    | RangeOperator
+    | ArrayOperator
+    | "rangeLow"
+    | "rangeHigh"
+  // Field name will often have a numerical prefix when coming from the frontend,
+  // use the ColumnSplitter class to remove it.
+  field: string
+  value: any
+  type?: FieldType
+  externalType?: string
+  noValue?: boolean
+  valueType?: string
+  formulaType?: string
+}
+
+// Prior to v2, this is the type the frontend sent us when filters were
+// involved. We convert this to a SearchFilters before use with the search SDK.
+export type LegacyFilter = AllOr | OnEmptyFilter | SearchFilter
+
 export type SearchFilterGroup = {
-  logicalOperator: FilterGroupLogicalOperator
-  onEmptyFilter?: EmptyFilterOption
+  logicalOperator?: UILogicalOperator
   groups?: SearchFilterGroup[]
   filters?: LegacyFilter[]
 }
+
+// As of v3, this is the format that the frontend always sends when search
+// filters are involved. We convert this to SearchFilters before use with the
+// search SDK.
+//
+// The reason we migrated was that we started to support "logical operators" in
+// tests and SearchFilters because a recursive data structure. LegacyFilter[]
+// wasn't able to support these sorts of recursive structures, so we changed the
+// format.
+export type UISearchFilter = {
+  logicalOperator?: UILogicalOperator
+  onEmptyFilter?: EmptyFilterOption
+  groups?: SearchFilterGroup[]
+}

packages/types/src/documents/app/role.ts

@@ -1,5 +1,5 @@
 import { Document } from "../document"
-import { PermissionLevel } from "../../sdk"
+import { PermissionLevel, BuiltinPermissionID } from "../../sdk"
 
 export interface RoleUIMetadata {
   displayName?: string
@@ -8,7 +8,7 @@ export interface RoleUIMetadata {
 }
 
 export interface Role extends Document {
-  permissionId: string
+  permissionId: BuiltinPermissionID
   inherits?: string | string[]
   permissions: Record<string, PermissionLevel[]>
   version?: string

packages/types/src/documents/app/view.ts

@@ -1,4 +1,4 @@
-import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
+import { LegacyFilter, UISearchFilter, SortOrder, SortType } from "../../api"
 import { UIFieldMetadata } from "./table"
 import { Document } from "../document"
 import { DBView, SearchFilters } from "../../sdk"
@@ -92,7 +92,7 @@ export interface ViewV2 {
   tableId: string
   query?: LegacyFilter[] | SearchFilters
   // duplicate to store UI information about filters
-  queryUI?: SearchFilterGroup
+  queryUI?: UISearchFilter
   sort?: {
     field: string
     order?: SortOrder

packages/types/src/sdk/permissions.ts

@@ -1,3 +1,5 @@
+// used in resource permissions - permissions can be at one of these levels
+// endpoints will set what type of permission they require (e.g. searching requires READ)
 export enum PermissionLevel {
   READ = "read",
   WRITE = "write",
@@ -5,6 +7,15 @@ export enum PermissionLevel {
   ADMIN = "admin",
 }
 
+// used within the role, specifies base permissions
+export enum BuiltinPermissionID {
+  PUBLIC = "public",
+  READ_ONLY = "read_only",
+  WRITE = "write",
+  ADMIN = "admin",
+  POWER = "power",
+}
+
 // these are the global types, that govern the underlying default behaviour
 export enum PermissionType {
   APP = "app",

packages/types/src/sdk/search.ts

@@ -32,7 +32,19 @@ export enum LogicalOperator {
 export function isLogicalSearchOperator(
   value: string
 ): value is LogicalOperator {
-  return value === LogicalOperator.AND || value === LogicalOperator.OR
+  return Object.values(LogicalOperator).includes(value as LogicalOperator)
+}
+
+export function isBasicSearchOperator(value: string): value is BasicOperator {
+  return Object.values(BasicOperator).includes(value as BasicOperator)
+}
+
+export function isArraySearchOperator(value: string): value is ArrayOperator {
+  return Object.values(ArrayOperator).includes(value as ArrayOperator)
+}
+
+export function isRangeSearchOperator(value: string): value is RangeOperator {
+  return Object.values(RangeOperator).includes(value as RangeOperator)
 }
 
 export type SearchFilterOperator =
@@ -191,7 +203,7 @@ export enum EmptyFilterOption {
   RETURN_NONE = "none",
 }
 
-export enum FilterGroupLogicalOperator {
+export enum UILogicalOperator {
   ALL = "all",
   ANY = "any",
 }

packages/worker/src/api/routes/global/tests/roles.spec.ts

@@ -1,6 +1,6 @@
 import { structures, TestConfiguration } from "../../../../tests"
-import { context, db, permissions, roles } from "@budibase/backend-core"
-import { App, Database } from "@budibase/types"
+import { context, db, roles } from "@budibase/backend-core"
+import { App, Database, BuiltinPermissionID } from "@budibase/types"
 
 jest.mock("@budibase/backend-core", () => {
   const core = jest.requireActual("@budibase/backend-core")
@@ -44,7 +44,7 @@ describe("/api/global/roles", () => {
   const role = new roles.Role(
     db.generateRoleID(ROLE_NAME),
     ROLE_NAME,
-    permissions.BuiltinPermissionID.READ_ONLY,
+    BuiltinPermissionID.READ_ONLY,
     { displayName: roles.BUILTIN_ROLE_IDS.BASIC }
   )