Merge branch 'design-section-feature-branch' of github.com:Budibase/budibase into new-left-panel

.github/workflows/budibase_ci.yml

@@ -18,6 +18,8 @@ env:
   BRANCH: ${{ github.event.pull_request.head.ref }}
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  NX_BASE_BRANCH: origin/${{ github.base_ref }}
+  USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}}
 
 jobs:
   lint:
@@ -25,20 +27,20 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
       - run: yarn lint
 
   build:
@@ -46,45 +48,66 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
+
       # Run build all the projects
-      - run: yarn build
+      - name: Build
+        run: |
+          yarn build
       # Check the types of the projects built via esbuild
-      - run: yarn check:types
+      - name: Check types
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn check:types --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn check:types
+          fi
 
   test-libraries:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
-      - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
+      - name: Test
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
+          fi
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
@@ -96,21 +119,31 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
+        with:
+          fetch-depth: 0
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
-      - run: yarn test --scope=@budibase/worker --scope=@budibase/server
+      - name: Test worker and server
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --scope=@budibase/worker --scope=@budibase/server
+          fi
+
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
@@ -119,42 +152,50 @@ jobs:
 
   test-pro:
     runs-on: ubuntu-latest
-    if: github.repository == 'Budibase/budibase'
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
+          fetch-depth: 0
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
-      - run: yarn test --scope=@budibase/pro
+      - name: Test
+        run: |
+          if ${{ env.USE_NX_AFFECTED }}; then
+            yarn test --scope=@budibase/pro --since=${{ env.NX_BASE_BRANCH }}
+          else
+            yarn test --scope=@budibase/pro
+          fi
 
   integration-test:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
-      - run: yarn build --projects=@budibase/server,@budibase/worker,@budibase/client
+      - name: Build packages
+        run: yarn build --scope @budibase/server --scope @budibase/worker --scope @budibase/client --scope @budibase/backend-core
       - name: Run tests
         run: |
           cd qa-core
@@ -166,13 +207,12 @@ jobs:
 
   check-pro-submodule:
     runs-on: ubuntu-latest
-    if: github.repository == 'Budibase/budibase'
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         with:
           submodules: true
-          fetch-depth: 0
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
 
       - name: Check pro commit

.github/workflows/check_unreleased_changes.yml

@@ -0,0 +1,29 @@
+name: check_unreleased_changes
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  check_unreleased:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check for unreleased changes
+        env:
+          REPO: "Budibase/budibase"
+          TOKEN: ${{ secrets.GITHUB_TOKEN }} 
+        run: |
+          RELEASE_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \
+              "https://api.github.com/repos/$REPO/releases/latest" | \
+              jq -r .published_at)
+          COMMIT_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \
+              "https://api.github.com/repos/$REPO/commits/master" | \
+              jq -r .commit.committer.date)
+          RELEASE_SECONDS=$(date --date="$RELEASE_TIMESTAMP" "+%s")
+          COMMIT_SECONDS=$(date --date="$COMMIT_TIMESTAMP" "+%s")
+          if (( COMMIT_SECONDS > RELEASE_SECONDS )); then
+              echo "There are unreleased changes. Please release these changes before merging."
+              exit 1
+          fi
+          echo "No unreleased changes detected."

.github/workflows/release-develop.yml

@@ -44,7 +44,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: 14.x
+          node-version: 18.x
 
       - run: yarn install --frozen-lockfile
       - name: Update versions

.nvmrc

@@ -1 +1 @@
-v14.20.1
+v18.17.0

.tool-versions

@@ -1,3 +1,3 @@
-nodejs 14.21.3
+nodejs 18.17.0
 python 3.10.0
-yarn 1.22.19
+yarn 1.22.19

.vscode/launch.json

@@ -1,3 +1,4 @@
+
 {
   // Use IntelliSense to learn about possible attributes.
   // Hover to view descriptions of existing attributes.
@@ -27,4 +28,4 @@
       "configurations": ["Budibase Server", "Budibase Worker"]
     }
   ]
-}
+}

docs/CONTRIBUTING.md

@@ -90,7 +90,7 @@ Component libraries are collections of components as well as the definition of t
 
 #### 1. Prerequisites
 
-- NodeJS version `14.x.x`
+- NodeJS version `18.x.x`
 - Python version `3.x`
 
 ### Using asdf (recommended)

hosting/single/Dockerfile

@@ -1,7 +1,7 @@
-FROM node:14-slim as build
+FROM node:18-slim as build
 
 # install node-gyp dependencies
-RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python3
 
 # add pin script
 WORKDIR /

jestTestcontainersConfigGenerator.js

@@ -1,9 +1,26 @@
 module.exports = () => {
   return {
-    dockerCompose: {
+    couchdb: {
-      composeFilePath: "../../hosting",
+      image: "budibase/couchdb",
-      composeFile: "docker-compose.test.yaml",
+      ports: [5984],
-      startupTimeout: 10000,
+      env: {
-    },
+        COUCHDB_PASSWORD: "budibase",
+        COUCHDB_USER: "budibase",
+      },
+      wait: {
+        type: "ports",
+        timeout: 10000,
+      }
+    }
   }
 }
+
+// module.exports = () => {
+//   return {
+//     dockerCompose: {
+//       composeFilePath: "../../hosting",
+//       composeFile: "docker-compose.test.yaml",
+//       startupTimeout: 10000,
+//     },
+//   }
+// }

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.9.30-alpha.0",
+  "version": "2.9.30-alpha.9",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -34,7 +34,7 @@
     "preinstall": "node scripts/syncProPackage.js",
     "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
     "bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'",
-    "build": "yarn nx run-many -t=build",
+    "build": "lerna run build --stream",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
     "check:types": "lerna run check:types",
     "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap",
@@ -109,7 +109,7 @@
     "@budibase/types": "0.0.0"
   },
   "engines": {
-    "node": ">=14.0.0 <15.0.0"
+    "node": ">=18.0.0 <19.0.0"
   },
   "dependencies": {}
 }

packages/backend-core/src/features/index.ts

@@ -1,5 +1,6 @@
 import env from "../environment"
 import * as context from "../context"
+export * from "./installation"
 
 /**
  * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.

packages/backend-core/src/features/installation.ts

@@ -0,0 +1,17 @@
+export function processFeatureEnvVar<T>(
+  fullList: string[],
+  featureList?: string
+) {
+  let list
+  if (!featureList) {
+    list = fullList
+  } else {
+    list = featureList.split(",")
+  }
+  for (let feature of list) {
+    if (!fullList.includes(feature)) {
+      throw new Error(`Feature: ${feature} is not an allowed option`)
+    }
+  }
+  return list as unknown as T[]
+}

packages/backend-core/src/index.ts

@@ -6,7 +6,8 @@ export * as roles from "./security/roles"
 export * as permissions from "./security/permissions"
 export * as accounts from "./accounts"
 export * as installation from "./installation"
-export * as featureFlags from "./featureFlags"
+export * as featureFlags from "./features"
+export * as features from "./features/installation"
 export * as sessions from "./security/sessions"
 export * as platform from "./platform"
 export * as auth from "./auth"

packages/backend-core/src/security/permissions.ts

@@ -78,7 +78,6 @@ export const BUILTIN_PERMISSIONS = {
     permissions: [
       new Permission(PermissionType.QUERY, PermissionLevel.READ),
       new Permission(PermissionType.TABLE, PermissionLevel.READ),
-      new Permission(PermissionType.VIEW, PermissionLevel.READ),
     ],
   },
   WRITE: {
@@ -87,7 +86,6 @@ export const BUILTIN_PERMISSIONS = {
     permissions: [
       new Permission(PermissionType.QUERY, PermissionLevel.WRITE),
       new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
-      new Permission(PermissionType.VIEW, PermissionLevel.READ),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
     ],
   },
@@ -98,7 +96,6 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
       new Permission(PermissionType.USER, PermissionLevel.READ),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
-      new Permission(PermissionType.VIEW, PermissionLevel.READ),
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
     ],
   },
@@ -109,7 +106,6 @@ export const BUILTIN_PERMISSIONS = {
       new Permission(PermissionType.TABLE, PermissionLevel.ADMIN),
       new Permission(PermissionType.USER, PermissionLevel.ADMIN),
       new Permission(PermissionType.AUTOMATION, PermissionLevel.ADMIN),
-      new Permission(PermissionType.VIEW, PermissionLevel.ADMIN),
       new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
       new Permission(PermissionType.QUERY, PermissionLevel.ADMIN),
     ],

packages/backend-core/src/users/db.ts

@@ -1,30 +1,30 @@
 import env from "../environment"
 import * as eventHelpers from "./events"
 import * as accounts from "../accounts"
+import * as accountSdk from "../accounts"
 import * as cache from "../cache"
-import { getIdentity, getTenantId, getGlobalDB } from "../context"
+import { getGlobalDB, getIdentity, getTenantId } from "../context"
 import * as dbUtils from "../db"
 import { EmailUnavailableError, HTTPError } from "../errors"
 import * as platform from "../platform"
 import * as sessions from "../security/sessions"
 import * as usersCore from "./users"
 import {
+  Account,
   AllDocsResponse,
   BulkUserCreated,
   BulkUserDeleted,
+  isSSOAccount,
+  isSSOUser,
   RowResponse,
   SaveUserOpts,
   User,
-  Account,
-  isSSOUser,
-  isSSOAccount,
   UserStatus,
 } from "@budibase/types"
-import * as accountSdk from "../accounts"
 import {
-  validateUniqueUser,
   getAccountHolderFromUserIds,
   isAdmin,
+  validateUniqueUser,
 } from "./utils"
 import { searchExistingEmails } from "./lookup"
 import { hash } from "../utils"
@@ -179,6 +179,14 @@ export class UserDB {
     return user
   }
 
+  static async bulkGet(userIds: string[]) {
+    return await usersCore.bulkGetGlobalUsersById(userIds)
+  }
+
+  static async bulkUpdate(users: User[]) {
+    return await usersCore.bulkUpdateGlobalUsers(users)
+  }
+
   static async save(user: User, opts: SaveUserOpts = {}): Promise<User> {
     // default booleans to true
     if (opts.hashPassword == null) {

packages/backend-core/tests/core/utilities/mocks/licenses.ts

@@ -86,6 +86,10 @@ export const useAuditLogs = () => {
   return useFeature(Feature.AUDIT_LOGS)
 }
 
+export const usePublicApiUserRoles = () => {
+  return useFeature(Feature.USER_ROLE_PUBLIC_API)
+}
+
 export const useScimIntegration = () => {
   return useFeature(Feature.SCIM)
 }

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -32,8 +32,8 @@ function getTestContainerSettings(
 ): string | null {
   const entry = Object.entries(global).find(
     ([k]) =>
-      k.includes(`_${serverName.toUpperCase()}`) &&
+      k.includes(`${serverName.toUpperCase()}`) &&
-      k.includes(`_${key.toUpperCase()}__`)
+      k.includes(`${key.toUpperCase()}`)
   )
   if (!entry) {
     return null
@@ -67,27 +67,14 @@ function getContainerInfo(containerName: string, port: number) {
 }
 
 function getCouchConfig() {
-  return getContainerInfo("couchdb-service", 5984)
+  return getContainerInfo("couchdb", 5984)
-}
-
-function getMinioConfig() {
-  return getContainerInfo("minio-service", 9000)
-}
-
-function getRedisConfig() {
-  return getContainerInfo("redis-service", 6379)
 }
 
 export function setupEnv(...envs: any[]) {
-  const couch = getCouchConfig(),
+  const couch = getCouchConfig()
-    minio = getCouchConfig(),
-    redis = getRedisConfig()
   const configs = [
     { key: "COUCH_DB_PORT", value: couch.port },
     { key: "COUCH_DB_URL", value: couch.url },
-    { key: "MINIO_PORT", value: minio.port },
-    { key: "MINIO_URL", value: minio.url },
-    { key: "REDIS_URL", value: redis.url },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -6,13 +6,15 @@
     Select,
     Toggle,
     RadioGroup,
+    Icon,
     DatePicker,
     Modal,
     notifications,
     OptionSelectDnD,
     Layout,
+    AbsTooltip,
   } from "@budibase/bbui"
-  import { createEventDispatcher, getContext } from "svelte"
+  import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/backend"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
@@ -47,6 +49,7 @@
 
   export let field
 
+  let mounted = false
   let fieldDefinitions = cloneDeep(FIELDS)
   let originalName
   let linkEditDisabled
@@ -413,16 +416,22 @@
     }
     return newError
   }
+
+  onMount(() => {
+    mounted = true
+  })
 </script>
 
 <Layout noPadding gap="S">
-  <Input
+  {#if mounted}
-    bind:value={editableColumn.name}
+    <Input
-    disabled={uneditable ||
+      autofocus
-      (linkEditDisabled && editableColumn.type === LINK_TYPE)}
+      bind:value={editableColumn.name}
-    error={errors?.name}
+      disabled={uneditable ||
-  />
+        (linkEditDisabled && editableColumn.type === LINK_TYPE)}
-
+      error={errors?.name}
+    />
+  {/if}
   <Select
     disabled={!typeEnabled}
     bind:value={editableColumn.type}
@@ -452,12 +461,17 @@
     />
   {:else if editableColumn.type === "longform"}
     <div>
-      <Label
+      <div class="tooltip-alignment">
-        size="M"
+        <Label size="M">Formatting</Label>
-        tooltip="Rich text includes support for images, links, tables, lists and more"
+        <AbsTooltip
-      >
+          position="top"
-        Formatting
+          type="info"
-      </Label>
+          text={"Rich text includes support for images, link"}
+        >
+          <Icon size="XS" name="InfoOutline" />
+        </AbsTooltip>
+      </div>
+
       <Toggle
         bind:value={editableColumn.useRichText}
         text="Enable rich text support (markdown)"
@@ -488,13 +502,18 @@
     </div>
     {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}
       <div>
-        <Label
+        <div>
-          tooltip={isCreating
+          <Label>Time zones</Label>
-            ? null
+          <AbsTooltip
-            : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
+            position="top"
-        >
+            type="info"
-          Time zones
+            text={isCreating
-        </Label>
+              ? null
+              : "We recommend not changing how timezones are handled for existing columns, as existing data will not be updated"}
+          >
+            <Icon size="XS" name="InfoOutline" />
+          </AbsTooltip>
+        </div>
         <Toggle
           bind:value={editableColumn.ignoreTimezones}
           text="Ignore time zones"
@@ -671,6 +690,12 @@
     align-items: center;
   }
 
+  .tooltip-alignment {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-xs);
+  }
+
   .label-length {
     flex-basis: 40%;
   }

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ChangeFormStep.svelte

@@ -1,10 +1,12 @@
 <script>
-  import { Select, Label, Stepper } from "@budibase/bbui"
+  import { Select, Label } from "@budibase/bbui"
   import { currentAsset, store } from "builderStore"
   import { getActionProviderComponents } from "builderStore/dataBinding"
   import { onMount } from "svelte"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
 
   export let parameters
+  export let bindings = []
 
   $: actionProviders = getActionProviderComponents(
     $currentAsset,
@@ -51,7 +53,11 @@
   <Select bind:value={parameters.type} options={typeOptions} />
   {#if parameters.type === "specific"}
     <Label small>Number</Label>
-    <Stepper bind:value={parameters.number} />
+    <DrawerBindableInput
+      {bindings}
+      value={parameters.number}
+      on:change={e => (parameters.number = e.detail)}
+    />
   {/if}
 </div>
 

packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte

@@ -17,7 +17,7 @@
   import { generate } from "shortid"
   import { LuceneUtils, Constants } from "@budibase/frontend-core"
   import { getFields } from "helpers/searchFields"
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, onMount } from "svelte"
 
   export let schemaFields
   export let filters = []
@@ -70,6 +70,15 @@
       })
   }
 
+  onMount(() => {
+    parseFilters(filters)
+    rawFilters.forEach(filter => {
+      filter.type =
+        schemaFields.find(field => field.name === filter.field)?.type ||
+        filter.type
+    })
+  })
+
   // Add field key prefixes and a special metadata filter object to indicate
   // how to handle filter behaviour
   const enrichFilters = (rawFilters, matchAny, onEmptyFilter) => {

packages/builder/src/pages/builder/app/[application]/settings/automation-history/_components/StatusRenderer.svelte

@@ -4,11 +4,13 @@
 
   $: isError = !value || value.toLowerCase() === "error"
   $: isStoppedError = value?.toLowerCase() === "stopped_error"
-  $: isStopped = value?.toLowerCase() === "stopped" || isStoppedError
+  $: isStopped = value?.toLowerCase() === "stopped"
-  $: info = getInfo(isError, isStopped)
+  $: info = getInfo(isError, isStopped, isStoppedError)
 
-  const getInfo = (error, stopped) => {
+  function getInfo(error, stopped, stoppedError) {
-    if (error) {
+    if (stoppedError) {
+      return { color: "red", message: "Stopped - Error" }
+    } else if (error) {
       return { color: "red", message: "Error" }
     } else if (stopped) {
       return { color: "yellow", message: "Stopped" }

packages/builder/src/pages/builder/app/[application]/settings/automation-history/index.svelte

@@ -22,7 +22,8 @@
 
   const ERROR = "error",
     SUCCESS = "success",
-    STOPPED = "stopped"
+    STOPPED = "stopped",
+    STOPPED_ERROR = "stopped_error"
   const sidePanel = getContext("side-panel")
 
   let pageInfo = createPaginationStore()
@@ -52,6 +53,7 @@
     { value: SUCCESS, label: "Success" },
     { value: ERROR, label: "Error" },
     { value: STOPPED, label: "Stopped" },
+    { value: STOPPED_ERROR, label: "Stopped - Error" },
   ]
 
   const runHistorySchema = {

packages/cli/tsconfig.build.json

@@ -8,22 +8,16 @@
     "esModuleInterop": true,
     "resolveJsonModule": true,
     "incremental": true,
-    "types": [ "node", "jest" ],
+    "types": ["node", "jest"],
     "outDir": "dist",
     "skipLibCheck": true,
     "paths": {
       "@budibase/types": ["../types/src"],
       "@budibase/backend-core": ["../backend-core/src"],
-      "@budibase/backend-core/*": ["../backend-core/*"]
+      "@budibase/backend-core/*": ["../backend-core/*"],
+      "@budibase/shared-core": ["../shared-core/src"]
     }
   },
-  "include": [
+  "include": ["src/**/*"],
-    "src/**/*"
+  "exclude": ["node_modules", "dist", "**/*.spec.ts", "**/*.spec.js"]
-  ],
-  "exclude": [
-    "node_modules",
-    "dist",
-    "**/*.spec.ts",
-    "**/*.spec.js"
-  ]
 }

packages/client/manifest.json

@@ -2408,6 +2408,13 @@
         "label": "Disabled",
         "key": "disabled",
         "defaultValue": false
+      },
+
+      {
+        "type": "text",
+        "label": "Initial form step",
+        "key": "initialFormStep",
+        "defaultValue": 1
       }
     ],
     "context": [
@@ -2445,6 +2452,7 @@
     "name": "Form Step",
     "icon": "AssetsAdded",
     "hasChildren": true,
+    "requiredAncestors": ["form"],
     "illegalChildren": ["section", "form", "formstep", "formblock"],
     "styles": ["size"],
     "size": {
@@ -2464,6 +2472,7 @@
   "fieldgroup": {
     "name": "Field Group",
     "icon": "Group",
+    "requiredAncestors": ["form"],
     "illegalChildren": ["section"],
     "styles": ["size"],
     "hasChildren": true,

packages/client/src/components/app/forms/Form.svelte

@@ -9,6 +9,7 @@
   export let size
   export let disabled = false
   export let actionType = "Create"
+  export let initialFormStep = 1
 
   // Not exposed as a builder setting. Used internally to disable validation
   // for fields rendered in things like search blocks.
@@ -21,10 +22,18 @@
   const context = getContext("context")
   const { API, fetchDatasourceSchema } = getContext("sdk")
 
+  const getInitialFormStep = () => {
+    const parsedFormStep = parseInt(initialFormStep)
+    if (isNaN(parsedFormStep)) {
+      return 1
+    }
+    return parsedFormStep
+  }
+
   let loaded = false
   let schema
   let table
-  let currentStep = writable(1)
+  let currentStep = writable(getInitialFormStep())
 
   $: fetchSchema(dataSource)
   $: schemaKey = generateSchemaKey(schema)

packages/client/src/components/app/forms/InnerForm.svelte

@@ -250,7 +250,7 @@
       } else if (type === "first") {
         currentStep.set(1)
       } else if (type === "specific" && number && !isNaN(number)) {
-        currentStep.set(number)
+        currentStep.set(parseInt(number))
       }
     },
     setStep: step => {

packages/frontend-core/src/components/grid/layout/NewColumnButton.svelte

@@ -32,7 +32,7 @@
 <Popover
   bind:open
   {anchor}
-  align="right"
+  align={$renderedColumns.length ? "right" : "left"}
   offset={0}
   popoverTarget={document.getElementById(`add-column-button`)}
   animate={false}

packages/pro

@@ -1 +1 @@
-Subproject commit 02626390cde905a248cb60729968667c9e49fae9
+Subproject commit 06a28b18a409cc12e9e8a5b69a094adcc6babd5a

packages/server/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14-slim
+FROM node:18-slim
 
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh"
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh"
@@ -18,7 +18,7 @@ ENV TOP_LEVEL_PATH=/
 
 # handle node-gyp
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends g++ make python
+    && apt-get install -y --no-install-recommends g++ make python3
 RUN yarn global add pm2
 
 # Install client for oracle datasource

packages/server/package.json

@@ -100,7 +100,7 @@
     "memorystream": "0.3.1",
     "mongodb": "5.7",
     "mssql": "9.1.1",
-    "mysql2": "2.3.3",
+    "mysql2": "3.5.2",
     "node-fetch": "2.6.7",
     "object-sizeof": "2.6.1",
     "open": "8.4.0",
@@ -179,5 +179,20 @@
   },
   "optionalDependencies": {
     "oracledb": "5.3.0"
+  },
+  "nx": {
+    "targets": {
+      "dev:builder": {
+        "dependsOn": [
+          {
+            "comment": "Required for pro usage when submodule not loaded",
+            "projects": [
+              "@budibase/backend-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
   }
 }

packages/server/scripts/test.sh

@@ -5,8 +5,9 @@ if [[ -n $CI ]]
 then
   # --runInBand performs better in ci where resources are limited
   export NODE_OPTIONS="--max-old-space-size=4096"
-  echo "jest --coverage --runInBand --forceExit --bail"
+  node ../../node_modules/jest/bin/jest.js --version
-  jest --coverage --runInBand --forceExit --bail
+  echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail"
+  jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail
 else
   # --maxWorkers performs better in development
   echo "jest --coverage --maxWorkers=2 --forceExit"

packages/server/specs/openapi.json

@@ -1521,7 +1521,7 @@
             "type": "boolean"
           },
           "builder": {
-            "description": "Describes if the user is a builder user or not.",
+            "description": "Describes if the user is a builder user or not. This field can only be set on a business or enterprise license.",
             "type": "object",
             "properties": {
               "global": {
@@ -1531,7 +1531,7 @@
             }
           },
           "admin": {
-            "description": "Describes if the user is an admin user or not.",
+            "description": "Describes if the user is an admin user or not. This field can only be set on a business or enterprise license.",
             "type": "object",
             "properties": {
               "global": {
@@ -1541,7 +1541,7 @@
             }
           },
           "roles": {
-            "description": "Contains the roles of the user per app (assuming they are not a builder user).",
+            "description": "Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license.",
             "type": "object",
             "additionalProperties": {
               "type": "string",
@@ -1588,7 +1588,7 @@
                 "type": "boolean"
               },
               "builder": {
-                "description": "Describes if the user is a builder user or not.",
+                "description": "Describes if the user is a builder user or not. This field can only be set on a business or enterprise license.",
                 "type": "object",
                 "properties": {
                   "global": {
@@ -1598,7 +1598,7 @@
                 }
               },
               "admin": {
-                "description": "Describes if the user is an admin user or not.",
+                "description": "Describes if the user is an admin user or not. This field can only be set on a business or enterprise license.",
                 "type": "object",
                 "properties": {
                   "global": {
@@ -1608,7 +1608,7 @@
                 }
               },
               "roles": {
-                "description": "Contains the roles of the user per app (assuming they are not a builder user).",
+                "description": "Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license.",
                 "type": "object",
                 "additionalProperties": {
                   "type": "string",
@@ -1667,7 +1667,7 @@
                   "type": "boolean"
                 },
                 "builder": {
-                  "description": "Describes if the user is a builder user or not.",
+                  "description": "Describes if the user is a builder user or not. This field can only be set on a business or enterprise license.",
                   "type": "object",
                   "properties": {
                     "global": {
@@ -1677,7 +1677,7 @@
                   }
                 },
                 "admin": {
-                  "description": "Describes if the user is an admin user or not.",
+                  "description": "Describes if the user is an admin user or not. This field can only be set on a business or enterprise license.",
                   "type": "object",
                   "properties": {
                     "global": {
@@ -1687,7 +1687,7 @@
                   }
                 },
                 "roles": {
-                  "description": "Contains the roles of the user per app (assuming they are not a builder user).",
+                  "description": "Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license.",
                   "type": "object",
                   "additionalProperties": {
                     "type": "string",
@@ -1833,6 +1833,137 @@
         "required": [
           "name"
         ]
+      },
+      "rolesAssign": {
+        "type": "object",
+        "properties": {
+          "appBuilder": {
+            "type": "object",
+            "properties": {
+              "appId": {
+                "description": "The app that the users should have app builder privileges granted for.",
+                "type": "string"
+              }
+            },
+            "description": "Allow setting users to builders per app.",
+            "required": [
+              "appId"
+            ]
+          },
+          "builder": {
+            "type": "boolean",
+            "description": "Add/remove global builder permissions from the list of users."
+          },
+          "admin": {
+            "type": "boolean",
+            "description": "Add/remove global admin permissions from the list of users."
+          },
+          "role": {
+            "type": "object",
+            "properties": {
+              "roleId": {
+                "description": "The role ID, such as BASIC, ADMIN or a custom role ID.",
+                "type": "string"
+              },
+              "appId": {
+                "description": "The app that the role relates to.",
+                "type": "string"
+              }
+            },
+            "description": "Add/remove a per-app role, such as BASIC, ADMIN etc.",
+            "required": [
+              "roleId",
+              "appId"
+            ]
+          },
+          "userIds": {
+            "description": "The user IDs to be updated to add/remove the specified roles.",
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        },
+        "required": [
+          "userIds"
+        ]
+      },
+      "rolesUnAssign": {
+        "type": "object",
+        "properties": {
+          "appBuilder": {
+            "type": "object",
+            "properties": {
+              "appId": {
+                "description": "The app that the users should have app builder privileges granted for.",
+                "type": "string"
+              }
+            },
+            "description": "Allow setting users to builders per app.",
+            "required": [
+              "appId"
+            ]
+          },
+          "builder": {
+            "type": "boolean",
+            "description": "Add/remove global builder permissions from the list of users."
+          },
+          "admin": {
+            "type": "boolean",
+            "description": "Add/remove global admin permissions from the list of users."
+          },
+          "role": {
+            "type": "object",
+            "properties": {
+              "roleId": {
+                "description": "The role ID, such as BASIC, ADMIN or a custom role ID.",
+                "type": "string"
+              },
+              "appId": {
+                "description": "The app that the role relates to.",
+                "type": "string"
+              }
+            },
+            "description": "Add/remove a per-app role, such as BASIC, ADMIN etc.",
+            "required": [
+              "roleId",
+              "appId"
+            ]
+          },
+          "userIds": {
+            "description": "The user IDs to be updated to add/remove the specified roles.",
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        },
+        "required": [
+          "userIds"
+        ]
+      },
+      "rolesOutput": {
+        "type": "object",
+        "properties": {
+          "data": {
+            "type": "object",
+            "properties": {
+              "userIds": {
+                "description": "The updated users' IDs",
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              }
+            },
+            "required": [
+              "userIds"
+            ]
+          }
+        },
+        "required": [
+          "data"
+        ]
       }
     }
   },
@@ -2186,6 +2317,70 @@
         }
       }
     },
+    "/roles/assign": {
+      "post": {
+        "operationId": "roleAssign",
+        "summary": "Assign a role to a list of users",
+        "description": "This is a business/enterprise only endpoint",
+        "tags": [
+          "roles"
+        ],
+        "requestBody": {
+          "required": true,
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/rolesAssign"
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "Returns a list of updated user IDs",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/rolesOutput"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "/roles/unassign": {
+      "post": {
+        "operationId": "roleUnAssign",
+        "summary": "Un-assign a role from a list of users",
+        "description": "This is a business/enterprise only endpoint",
+        "tags": [
+          "roles"
+        ],
+        "requestBody": {
+          "required": true,
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/rolesUnAssign"
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "Returns a list of updated user IDs",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/rolesOutput"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
     "/tables/{tableId}/rows": {
       "post": {
         "operationId": "rowCreate",

packages/server/specs/openapi.yaml

@@ -1297,7 +1297,8 @@ components:
             login.
           type: boolean
         builder:
-          description: Describes if the user is a builder user or not.
+          description: Describes if the user is a builder user or not. This field can only
+            be set on a business or enterprise license.
           type: object
           properties:
             global:
@@ -1305,7 +1306,8 @@ components:
                 system.
               type: boolean
         admin:
-          description: Describes if the user is an admin user or not.
+          description: Describes if the user is an admin user or not. This field can only
+            be set on a business or enterprise license.
           type: object
           properties:
             global:
@@ -1313,7 +1315,8 @@ components:
               type: boolean
         roles:
           description: Contains the roles of the user per app (assuming they are not a
-            builder user).
+            builder user). This field can only be set on a business or
+            enterprise license.
           type: object
           additionalProperties:
             type: string
@@ -1352,7 +1355,8 @@ components:
                 login.
               type: boolean
             builder:
-              description: Describes if the user is a builder user or not.
+              description: Describes if the user is a builder user or not. This field can only
+                be set on a business or enterprise license.
               type: object
               properties:
                 global:
@@ -1360,7 +1364,8 @@ components:
                     system.
                   type: boolean
             admin:
-              description: Describes if the user is an admin user or not.
+              description: Describes if the user is an admin user or not. This field can only
+                be set on a business or enterprise license.
               type: object
               properties:
                 global:
@@ -1368,7 +1373,8 @@ components:
                   type: boolean
             roles:
               description: Contains the roles of the user per app (assuming they are not a
-                builder user).
+                builder user). This field can only be set on a business or
+                enterprise license.
               type: object
               additionalProperties:
                 type: string
@@ -1415,7 +1421,8 @@ components:
                   login.
                 type: boolean
               builder:
-                description: Describes if the user is a builder user or not.
+                description: Describes if the user is a builder user or not. This field can only
+                  be set on a business or enterprise license.
                 type: object
                 properties:
                   global:
@@ -1423,7 +1430,8 @@ components:
                       system.
                     type: boolean
               admin:
-                description: Describes if the user is an admin user or not.
+                description: Describes if the user is an admin user or not. This field can only
+                  be set on a business or enterprise license.
                 type: object
                 properties:
                   global:
@@ -1431,7 +1439,8 @@ components:
                     type: boolean
               roles:
                 description: Contains the roles of the user per app (assuming they are not a
-                  builder user).
+                  builder user). This field can only be set on a business or
+                  enterprise license.
                 type: object
                 additionalProperties:
                   type: string
@@ -1547,6 +1556,99 @@ components:
             insensitive starts with match.
       required:
         - name
+    rolesAssign:
+      type: object
+      properties:
+        appBuilder:
+          type: object
+          properties:
+            appId:
+              description: The app that the users should have app builder privileges granted
+                for.
+              type: string
+          description: Allow setting users to builders per app.
+          required:
+            - appId
+        builder:
+          type: boolean
+          description: Add/remove global builder permissions from the list of users.
+        admin:
+          type: boolean
+          description: Add/remove global admin permissions from the list of users.
+        role:
+          type: object
+          properties:
+            roleId:
+              description: The role ID, such as BASIC, ADMIN or a custom role ID.
+              type: string
+            appId:
+              description: The app that the role relates to.
+              type: string
+          description: Add/remove a per-app role, such as BASIC, ADMIN etc.
+          required:
+            - roleId
+            - appId
+        userIds:
+          description: The user IDs to be updated to add/remove the specified roles.
+          type: array
+          items:
+            type: string
+      required:
+        - userIds
+    rolesUnAssign:
+      type: object
+      properties:
+        appBuilder:
+          type: object
+          properties:
+            appId:
+              description: The app that the users should have app builder privileges granted
+                for.
+              type: string
+          description: Allow setting users to builders per app.
+          required:
+            - appId
+        builder:
+          type: boolean
+          description: Add/remove global builder permissions from the list of users.
+        admin:
+          type: boolean
+          description: Add/remove global admin permissions from the list of users.
+        role:
+          type: object
+          properties:
+            roleId:
+              description: The role ID, such as BASIC, ADMIN or a custom role ID.
+              type: string
+            appId:
+              description: The app that the role relates to.
+              type: string
+          description: Add/remove a per-app role, such as BASIC, ADMIN etc.
+          required:
+            - roleId
+            - appId
+        userIds:
+          description: The user IDs to be updated to add/remove the specified roles.
+          type: array
+          items:
+            type: string
+      required:
+        - userIds
+    rolesOutput:
+      type: object
+      properties:
+        data:
+          type: object
+          properties:
+            userIds:
+              description: The updated users' IDs
+              type: array
+              items:
+                type: string
+          required:
+            - userIds
+      required:
+        - data
 security:
   - ApiKeyAuth: []
 paths:
@@ -1757,6 +1859,46 @@ paths:
               examples:
                 queries:
                   $ref: "#/components/examples/queries"
+  /roles/assign:
+    post:
+      operationId: roleAssign
+      summary: Assign a role to a list of users
+      description: This is a business/enterprise only endpoint
+      tags:
+        - roles
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/rolesAssign"
+      responses:
+        "200":
+          description: Returns a list of updated user IDs
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/rolesOutput"
+  /roles/unassign:
+    post:
+      operationId: roleUnAssign
+      summary: Un-assign a role from a list of users
+      description: This is a business/enterprise only endpoint
+      tags:
+        - roles
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/rolesUnAssign"
+      responses:
+        "200":
+          description: Returns a list of updated user IDs
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/rolesOutput"
   "/tables/{tableId}/rows":
     post:
       operationId: rowCreate

packages/server/specs/resources/index.ts

@@ -5,6 +5,7 @@ import query from "./query"
 import user from "./user"
 import metrics from "./metrics"
 import misc from "./misc"
+import roles from "./roles"
 
 export const examples = {
   ...application.getExamples(),
@@ -23,4 +24,5 @@ export const schemas = {
   ...query.getSchemas(),
   ...user.getSchemas(),
   ...misc.getSchemas(),
+  ...roles.getSchemas(),
 }

packages/server/specs/resources/roles.ts

@@ -0,0 +1,65 @@
+import { object } from "./utils"
+import Resource from "./utils/Resource"
+
+const roleSchema = object(
+  {
+    appBuilder: object(
+      {
+        appId: {
+          description:
+            "The app that the users should have app builder privileges granted for.",
+          type: "string",
+        },
+      },
+      { description: "Allow setting users to builders per app." }
+    ),
+    builder: {
+      type: "boolean",
+      description:
+        "Add/remove global builder permissions from the list of users.",
+    },
+    admin: {
+      type: "boolean",
+      description:
+        "Add/remove global admin permissions from the list of users.",
+    },
+    role: object(
+      {
+        roleId: {
+          description: "The role ID, such as BASIC, ADMIN or a custom role ID.",
+          type: "string",
+        },
+        appId: {
+          description: "The app that the role relates to.",
+          type: "string",
+        },
+      },
+      { description: "Add/remove a per-app role, such as BASIC, ADMIN etc." }
+    ),
+    userIds: {
+      description:
+        "The user IDs to be updated to add/remove the specified roles.",
+      type: "array",
+      items: {
+        type: "string",
+      },
+    },
+  },
+  { required: ["userIds"] }
+)
+
+export default new Resource().setSchemas({
+  rolesAssign: roleSchema,
+  rolesUnAssign: roleSchema,
+  rolesOutput: object({
+    data: object({
+      userIds: {
+        description: "The updated users' IDs",
+        type: "array",
+        items: {
+          type: "string",
+        },
+      },
+    }),
+  }),
+})

packages/server/specs/resources/user.ts

@@ -58,7 +58,8 @@ const userSchema = object(
       type: "boolean",
     },
     builder: {
-      description: "Describes if the user is a builder user or not.",
+      description:
+        "Describes if the user is a builder user or not. This field can only be set on a business or enterprise license.",
       type: "object",
       properties: {
         global: {
@@ -69,7 +70,8 @@ const userSchema = object(
       },
     },
     admin: {
-      description: "Describes if the user is an admin user or not.",
+      description:
+        "Describes if the user is an admin user or not. This field can only be set on a business or enterprise license.",
       type: "object",
       properties: {
         global: {
@@ -81,7 +83,7 @@ const userSchema = object(
     },
     roles: {
       description:
-        "Contains the roles of the user per app (assuming they are not a builder user).",
+        "Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license.",
       type: "object",
       additionalProperties: {
         type: "string",

packages/server/src/api/controllers/deploy/index.ts

@@ -77,18 +77,19 @@ async function initDeployedApp(prodAppId: any) {
     )
   ).rows.map((row: any) => row.doc)
   await clearMetadata()
-  console.log("You have " + automations.length + " automations")
+  const { count } = await disableAllCrons(prodAppId)
   const promises = []
-  console.log("Disabling prod crons..")
-  await disableAllCrons(prodAppId)
-  console.log("Prod Cron triggers disabled..")
-  console.log("Enabling cron triggers for deployed app..")
   for (let automation of automations) {
     promises.push(enableCronTrigger(prodAppId, automation))
   }
-  await Promise.all(promises)
+  const results = await Promise.all(promises)
-  console.log("Enabled cron triggers for deployed app..")
+  const enabledCount = results
-  // sync the automations back to the dev DB - since there is now cron
+    .map(result => result.enabled)
+    .filter(result => result).length
+  console.log(
+    `Cleared ${count} old CRON, enabled ${enabledCount} new CRON triggers for app deployment`
+  )
+  // sync the automations back to the dev DB - since there is now CRON
   // information attached
   await sdk.applications.syncApp(dbCore.getDevAppID(prodAppId), {
     automationOnly: true,

packages/server/src/api/controllers/public/applications.ts

@@ -3,6 +3,8 @@ import { search as stringSearch, addRev } from "./utils"
 import * as controller from "../application"
 import * as deployController from "../deploy"
 import { Application } from "../../../definitions/common"
+import { UserCtx } from "@budibase/types"
+import { Next } from "koa"
 
 function fixAppID(app: Application, params: any) {
   if (!params) {
@@ -14,7 +16,7 @@ function fixAppID(app: Application, params: any) {
   return app
 }
 
-async function setResponseApp(ctx: any) {
+async function setResponseApp(ctx: UserCtx) {
   const appId = ctx.body?.appId
   if (appId && (!ctx.params || !ctx.params.appId)) {
     ctx.params = { appId }
@@ -28,14 +30,14 @@ async function setResponseApp(ctx: any) {
   }
 }
 
-export async function search(ctx: any, next: any) {
+export async function search(ctx: UserCtx, next: Next) {
   const { name } = ctx.request.body
   const apps = await dbCore.getAllApps({ all: true })
   ctx.body = stringSearch(apps, name)
   await next()
 }
 
-export async function create(ctx: any, next: any) {
+export async function create(ctx: UserCtx, next: Next) {
   if (!ctx.request.body || !ctx.request.body.useTemplate) {
     ctx.request.body = {
       useTemplate: false,
@@ -47,14 +49,14 @@ export async function create(ctx: any, next: any) {
   await next()
 }
 
-export async function read(ctx: any, next: any) {
+export async function read(ctx: UserCtx, next: Next) {
   await context.doInAppContext(ctx.params.appId, async () => {
     await setResponseApp(ctx)
     await next()
   })
 }
 
-export async function update(ctx: any, next: any) {
+export async function update(ctx: UserCtx, next: Next) {
   ctx.request.body = await addRev(fixAppID(ctx.request.body, ctx.params))
   await context.doInAppContext(ctx.params.appId, async () => {
     await controller.update(ctx)
@@ -63,7 +65,7 @@ export async function update(ctx: any, next: any) {
   })
 }
 
-export async function destroy(ctx: any, next: any) {
+export async function destroy(ctx: UserCtx, next: Next) {
   await context.doInAppContext(ctx.params.appId, async () => {
     // get the app before deleting it
     await setResponseApp(ctx)
@@ -75,14 +77,14 @@ export async function destroy(ctx: any, next: any) {
   })
 }
 
-export async function unpublish(ctx: any, next: any) {
+export async function unpublish(ctx: UserCtx, next: Next) {
   await context.doInAppContext(ctx.params.appId, async () => {
     await controller.unpublish(ctx)
     await next()
   })
 }
 
-export async function publish(ctx: any, next: any) {
+export async function publish(ctx: UserCtx, next: Next) {
   await context.doInAppContext(ctx.params.appId, async () => {
     await deployController.publishApp(ctx)
     await next()

packages/server/src/api/controllers/public/mapping/types.ts

@@ -16,6 +16,10 @@ export type CreateRowParams = components["schemas"]["row"]
 export type User = components["schemas"]["userOutput"]["data"]
 export type CreateUserParams = components["schemas"]["user"]
 
+export type RoleAssignRequest = components["schemas"]["rolesAssign"]
+export type RoleUnAssignRequest = components["schemas"]["rolesUnAssign"]
+export type RoleAssignmentResponse = components["schemas"]["rolesOutput"]
+
 export type SearchInputParams =
   | components["schemas"]["nameSearch"]
   | components["schemas"]["rowSearch"]

packages/server/src/api/controllers/public/queries.ts

@@ -1,14 +1,16 @@
 import { search as stringSearch } from "./utils"
 import * as queryController from "../query"
+import { UserCtx } from "@budibase/types"
+import { Next } from "koa"
 
-export async function search(ctx: any, next: any) {
+export async function search(ctx: UserCtx, next: Next) {
   await queryController.fetch(ctx)
   const { name } = ctx.request.body
   ctx.body = stringSearch(ctx.body, name)
   await next()
 }
 
-export async function execute(ctx: any, next: any) {
+export async function execute(ctx: UserCtx, next: Next) {
   // don't wrap this, already returns "data"
   await queryController.executeV2(ctx)
   await next()

packages/server/src/api/controllers/public/roles.ts

@@ -0,0 +1,33 @@
+import { UserCtx } from "@budibase/types"
+import { Next } from "koa"
+import { sdk } from "@budibase/pro"
+import {
+  RoleAssignmentResponse,
+  RoleUnAssignRequest,
+  RoleAssignRequest,
+} from "./mapping/types"
+
+async function assign(
+  ctx: UserCtx<RoleAssignRequest, RoleAssignmentResponse>,
+  next: Next
+) {
+  const { userIds, ...assignmentProps } = ctx.request.body
+  await sdk.publicApi.roles.assign(userIds, assignmentProps)
+  ctx.body = { data: { userIds } }
+  await next()
+}
+
+async function unAssign(
+  ctx: UserCtx<RoleUnAssignRequest, RoleAssignmentResponse>,
+  next: Next
+) {
+  const { userIds, ...unAssignmentProps } = ctx.request.body
+  await sdk.publicApi.roles.unAssign(userIds, unAssignmentProps)
+  ctx.body = { data: { userIds } }
+  await next()
+}
+
+export default {
+  assign,
+  unAssign,
+}

packages/server/src/api/controllers/public/rows.ts

@@ -1,7 +1,8 @@
 import * as rowController from "../row"
 import { addRev } from "./utils"
-import { Row } from "@budibase/types"
+import { Row, UserCtx } from "@budibase/types"
 import { convertBookmark } from "../../../utilities"
+import { Next } from "koa"
 
 // makes sure that the user doesn't need to pass in the type, tableId or _id params for
 // the call to be correct
@@ -21,7 +22,7 @@ export function fixRow(row: Row, params: any) {
   return row
 }
 
-export async function search(ctx: any, next: any) {
+export async function search(ctx: UserCtx, next: Next) {
   let { sort, paginate, bookmark, limit, query } = ctx.request.body
   // update the body to the correct format of the internal search
   if (!sort) {
@@ -40,25 +41,25 @@ export async function search(ctx: any, next: any) {
   await next()
 }
 
-export async function create(ctx: any, next: any) {
+export async function create(ctx: UserCtx, next: Next) {
   ctx.request.body = fixRow(ctx.request.body, ctx.params)
   await rowController.save(ctx)
   await next()
 }
 
-export async function read(ctx: any, next: any) {
+export async function read(ctx: UserCtx, next: Next) {
   await rowController.fetchEnrichedRow(ctx)
   await next()
 }
 
-export async function update(ctx: any, next: any) {
+export async function update(ctx: UserCtx, next: Next) {
   const { tableId } = ctx.params
   ctx.request.body = await addRev(fixRow(ctx.request.body, ctx.params), tableId)
   await rowController.save(ctx)
   await next()
 }
 
-export async function destroy(ctx: any, next: any) {
+export async function destroy(ctx: UserCtx, next: Next) {
   const { tableId } = ctx.params
   // set the body as expected, with the _id and _rev fields
   ctx.request.body = await addRev(

packages/server/src/api/controllers/public/tables.ts

@@ -1,6 +1,7 @@
 import { search as stringSearch, addRev } from "./utils"
 import * as controller from "../table"
-import { Table } from "@budibase/types"
+import { Table, UserCtx } from "@budibase/types"
+import { Next } from "koa"
 
 function fixTable(table: Table, params: any) {
   if (!params || !table) {
@@ -15,24 +16,24 @@ function fixTable(table: Table, params: any) {
   return table
 }
 
-export async function search(ctx: any, next: any) {
+export async function search(ctx: UserCtx, next: Next) {
   const { name } = ctx.request.body
   await controller.fetch(ctx)
   ctx.body = stringSearch(ctx.body, name)
   await next()
 }
 
-export async function create(ctx: any, next: any) {
+export async function create(ctx: UserCtx, next: Next) {
   await controller.save(ctx)
   await next()
 }
 
-export async function read(ctx: any, next: any) {
+export async function read(ctx: UserCtx, next: Next) {
   await controller.find(ctx)
   await next()
 }
 
-export async function update(ctx: any, next: any) {
+export async function update(ctx: UserCtx, next: Next) {
   ctx.request.body = await addRev(
     fixTable(ctx.request.body, ctx.params),
     ctx.params.tableId
@@ -41,7 +42,7 @@ export async function update(ctx: any, next: any) {
   await next()
 }
 
-export async function destroy(ctx: any, next: any) {
+export async function destroy(ctx: UserCtx, next: Next) {
   await controller.destroy(ctx)
   ctx.body = ctx.table
   await next()

packages/server/src/api/controllers/public/users.ts

@@ -7,16 +7,18 @@ import {
 import { publicApiUserFix } from "../../../utilities/users"
 import { db as dbCore } from "@budibase/backend-core"
 import { search as stringSearch } from "./utils"
-import { BBContext, User } from "@budibase/types"
+import { UserCtx, User } from "@budibase/types"
+import { Next } from "koa"
+import { sdk } from "@budibase/pro"
 
-function isLoggedInUser(ctx: BBContext, user: User) {
+function isLoggedInUser(ctx: UserCtx, user: User) {
   const loggedInId = ctx.user?._id
   const globalUserId = dbCore.getGlobalIDFromUserMetadataID(loggedInId!)
   // check both just incase
   return globalUserId === user._id || loggedInId === user._id
 }
 
-function getUser(ctx: BBContext, userId?: string) {
+function getUser(ctx: UserCtx, userId?: string) {
   if (userId) {
     ctx.params = { userId }
   } else if (!ctx.params?.userId) {
@@ -25,42 +27,38 @@ function getUser(ctx: BBContext, userId?: string) {
   return readGlobalUser(ctx)
 }
 
-export async function search(ctx: BBContext, next: any) {
+export async function search(ctx: UserCtx, next: Next) {
   const { name } = ctx.request.body
   const users = await allGlobalUsers(ctx)
   ctx.body = stringSearch(users, name, "email")
   await next()
 }
 
-export async function create(ctx: BBContext, next: any) {
+export async function create(ctx: UserCtx, next: Next) {
-  const response = await saveGlobalUser(publicApiUserFix(ctx))
+  ctx = publicApiUserFix(await sdk.publicApi.users.roleCheck(ctx))
+  const response = await saveGlobalUser(ctx)
   ctx.body = await getUser(ctx, response._id)
   await next()
 }
 
-export async function read(ctx: BBContext, next: any) {
+export async function read(ctx: UserCtx, next: Next) {
   ctx.body = await readGlobalUser(ctx)
   await next()
 }
 
-export async function update(ctx: BBContext, next: any) {
+export async function update(ctx: UserCtx, next: Next) {
   const user = await readGlobalUser(ctx)
   ctx.request.body = {
     ...ctx.request.body,
     _rev: user._rev,
   }
-  // disallow updating your own role - always overwrite with DB roles
+  ctx = publicApiUserFix(await sdk.publicApi.users.roleCheck(ctx, user))
-  if (isLoggedInUser(ctx, user)) {
+  const response = await saveGlobalUser(ctx)
-    ctx.request.body.builder = user.builder
-    ctx.request.body.admin = user.admin
-    ctx.request.body.roles = user.roles
-  }
-  const response = await saveGlobalUser(publicApiUserFix(ctx))
   ctx.body = await getUser(ctx, response._id)
   await next()
 }
 
-export async function destroy(ctx: BBContext, next: any) {
+export async function destroy(ctx: UserCtx, next: Next) {
   const user = await getUser(ctx)
   // disallow deleting yourself
   if (isLoggedInUser(ctx, user)) {

packages/server/src/api/controllers/query/index.ts

@@ -127,7 +127,7 @@ export async function preview(ctx: any) {
   const query = ctx.request.body
   // preview may not have a queryId as it hasn't been saved, but if it does
   // this stops dynamic variables from calling the same query
-  const { fields, parameters, queryVerb, transformer, queryId } = query
+  const { fields, parameters, queryVerb, transformer, queryId, schema } = query
 
   const authConfigCtx: any = getAuthConfig(ctx)
 
@@ -140,6 +140,7 @@ export async function preview(ctx: any) {
       parameters,
       transformer,
       queryId,
+      schema,
       // have to pass down to the thread runner - can't put into context now
       environmentVariables: envVars,
       ctx: {
@@ -235,6 +236,7 @@ async function execute(
         user: ctx.user,
         auth: { ...authConfigCtx },
       },
+      schema: query.schema,
     }
     const runFn = () => Runner.run(inputs)
 

packages/server/src/api/controllers/row/external.ts

@@ -16,7 +16,8 @@ import {
   EmptyFilterOption,
 } from "@budibase/types"
 import sdk from "../../../sdk"
-import { hasFilters } from "@budibase/shared-core/src/filters"
+import * as utils from "./utils"
+import { dataFilters } from "@budibase/shared-core"
 
 export async function handleRequest(
   operation: Operation,
@@ -40,7 +41,7 @@ export async function handleRequest(
   }
 
   if (
-    !hasFilters(opts?.filters) &&
+    !dataFilters.hasFilters(opts?.filters) &&
     opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE
   ) {
     return []
@@ -52,7 +53,7 @@ export async function handleRequest(
 }
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   const { _id, ...rowData } = ctx.request.body
 
   const validateResult = await sdk.rows.utils.validate({
@@ -79,7 +80,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 
 export async function save(ctx: UserCtx) {
   const inputs = ctx.request.body
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   const validateResult = await sdk.rows.utils.validate({
     row: inputs,
     tableId,
@@ -107,12 +108,12 @@ export async function save(ctx: UserCtx) {
 
 export async function find(ctx: UserCtx) {
   const id = ctx.params.rowId
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   return sdk.rows.external.getRow(tableId, id)
 }
 
 export async function destroy(ctx: UserCtx) {
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   const _id = ctx.request.body._id
   const { row } = (await handleRequest(Operation.DELETE, tableId, {
     id: breakRowIdField(_id),
@@ -123,7 +124,7 @@ export async function destroy(ctx: UserCtx) {
 
 export async function bulkDestroy(ctx: UserCtx) {
   const { rows } = ctx.request.body
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
   for (let row of rows) {
     promises.push(
@@ -139,7 +140,7 @@ export async function bulkDestroy(ctx: UserCtx) {
 
 export async function fetchEnrichedRow(ctx: UserCtx) {
   const id = ctx.params.rowId
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   const { datasourceId, tableName } = breakExternalTableId(tableId)
   const datasource: Datasource = await sdk.datasources.get(datasourceId!)
   if (!tableName) {

packages/server/src/api/controllers/row/index.ts

@@ -11,6 +11,9 @@ import {
   Row,
   PatchRowRequest,
   PatchRowResponse,
+  SearchRowResponse,
+  SearchRowRequest,
+  SearchParams,
 } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
@@ -197,10 +200,10 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
   ctx.body = response
 }
 
-export async function search(ctx: any) {
+export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
   const tableId = utils.getTableId(ctx)
 
-  const searchParams = {
+  const searchParams: SearchParams = {
     ...ctx.request.body,
     tableId,
   }

packages/server/src/api/controllers/row/internal.ts

@@ -13,7 +13,7 @@ import {
 import { FieldTypes } from "../../../constants"
 import * as utils from "./utils"
 import { cloneDeep } from "lodash/fp"
-import { context, db as dbCore } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
 import {
   UserCtx,
@@ -26,8 +26,8 @@ import {
 import sdk from "../../../sdk"
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
+  const tableId = utils.getTableId(ctx)
   const inputs = ctx.request.body
-  const tableId = inputs.tableId
   const isUserTable = tableId === InternalTables.USER_METADATA
   let oldRow
   const dbTable = await sdk.tables.getTable(tableId)
@@ -94,7 +94,8 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 
 export async function save(ctx: UserCtx) {
   let inputs = ctx.request.body
-  inputs.tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
+  inputs.tableId = tableId
 
   if (!inputs._rev && !inputs._id) {
     inputs._id = generateRowID(inputs.tableId)
@@ -132,20 +133,22 @@ export async function save(ctx: UserCtx) {
 }
 
 export async function find(ctx: UserCtx) {
-  const db = dbCore.getDB(ctx.appId)
+  const tableId = utils.getTableId(ctx),
-  const table = await sdk.tables.getTable(ctx.params.tableId)
+    rowId = ctx.params.rowId
-  let row = await utils.findRow(ctx, ctx.params.tableId, ctx.params.rowId)
+  const table = await sdk.tables.getTable(tableId)
+  let row = await utils.findRow(ctx, tableId, rowId)
   row = await outputProcessing(table, row)
   return row
 }
 
 export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
+  const tableId = utils.getTableId(ctx)
   const { _id } = ctx.request.body
   let row = await db.get<Row>(_id)
   let _rev = ctx.request.body._rev || row._rev
 
-  if (row.tableId !== ctx.params.tableId) {
+  if (row.tableId !== tableId) {
     throw "Supplied tableId doesn't match the row's tableId"
   }
   const table = await sdk.tables.getTable(row.tableId)
@@ -163,7 +166,7 @@ export async function destroy(ctx: UserCtx) {
   await updateRelatedFormula(table, row)
 
   let response
-  if (ctx.params.tableId === InternalTables.USER_METADATA) {
+  if (tableId === InternalTables.USER_METADATA) {
     ctx.params = {
       id: _id,
     }
@@ -176,7 +179,7 @@ export async function destroy(ctx: UserCtx) {
 }
 
 export async function bulkDestroy(ctx: UserCtx) {
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   const table = await sdk.tables.getTable(tableId)
   let { rows } = ctx.request.body
 
@@ -216,7 +219,7 @@ export async function bulkDestroy(ctx: UserCtx) {
 
 export async function fetchEnrichedRow(ctx: UserCtx) {
   const db = context.getAppDB()
-  const tableId = ctx.params.tableId
+  const tableId = utils.getTableId(ctx)
   const rowId = ctx.params.rowId
   // need table to work out where links go in row
   let [table, row] = await Promise.all([

packages/server/src/api/controllers/row/utils.ts

@@ -45,13 +45,20 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
 }
 
 export function getTableId(ctx: Ctx) {
-  if (ctx.request.body && ctx.request.body.tableId) {
+  // top priority, use the URL first
-    return ctx.request.body.tableId
+  if (ctx.params?.sourceId) {
+    return ctx.params.sourceId
   }
-  if (ctx.params && ctx.params.tableId) {
+  // now check for old way of specifying table ID
+  if (ctx.params?.tableId) {
     return ctx.params.tableId
   }
-  if (ctx.params && ctx.params.viewName) {
+  // check body for a table ID
+  if (ctx.request.body?.tableId) {
+    return ctx.request.body.tableId
+  }
+  // now check if a specific view name
+  if (ctx.params?.viewName) {
     return ctx.params.viewName
   }
 }

packages/server/src/api/controllers/row/views.ts

@@ -1,14 +1,18 @@
 import { quotas } from "@budibase/pro"
 import {
   UserCtx,
-  SearchResponse,
-  SortOrder,
-  SortType,
   ViewV2,
+  SearchRowResponse,
+  SearchViewRowRequest,
+  RequiredKeys,
+  SearchParams,
 } from "@budibase/types"
+import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../../sdk"
 
-export async function searchView(ctx: UserCtx<void, SearchResponse>) {
+export async function searchView(
+  ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
+) {
   const { viewId } = ctx.params
 
   const view = await sdk.views.get(viewId)
@@ -29,49 +33,35 @@ export async function searchView(ctx: UserCtx<void, SearchResponse>) {
     undefined
 
   ctx.status = 200
-  const result = await quotas.addQuery(
+
-    () =>
+  const { body } = ctx.request
-      sdk.rows.search({
+  const query = dataFilters.buildLuceneQuery(view.query || [])
-        tableId: view.tableId,
+
-        query: view.query || {},
+  const searchOptions: RequiredKeys<SearchViewRowRequest> &
-        fields: viewFields,
+    RequiredKeys<Pick<SearchParams, "tableId" | "query" | "fields">> = {
-        ...getSortOptions(ctx, view),
+    tableId: view.tableId,
-      }),
+    query,
-    {
+    fields: viewFields,
-      datasourceId: view.tableId,
+    ...getSortOptions(body, view),
-    }
+    limit: body.limit,
-  )
+    bookmark: body.bookmark,
+    paginate: body.paginate,
+  }
+
+  const result = await quotas.addQuery(() => sdk.rows.search(searchOptions), {
+    datasourceId: view.tableId,
+  })
 
   result.rows.forEach(r => (r._viewId = view.id))
   ctx.body = result
 }
 
-function getSortOptions(
+function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
-  ctx: UserCtx,
+  if (request.sort) {
-  view: ViewV2
-):
-  | {
-      sort: string
-      sortOrder?: SortOrder
-      sortType?: SortType
-    }
-  | undefined {
-  const { sort_column, sort_order, sort_type } = ctx.query
-  if (Array.isArray(sort_column)) {
-    ctx.throw(400, "sort_column cannot be an array")
-  }
-  if (Array.isArray(sort_order)) {
-    ctx.throw(400, "sort_order cannot be an array")
-  }
-  if (Array.isArray(sort_type)) {
-    ctx.throw(400, "sort_type cannot be an array")
-  }
-
-  if (sort_column) {
     return {
-      sort: sort_column,
+      sort: request.sort,
-      sortOrder: sort_order as SortOrder,
+      sortOrder: request.sortOrder,
-      sortType: sort_type as SortType,
+      sortType: request.sortType,
     }
   }
   if (view.sort) {
@@ -82,5 +72,9 @@ function getSortOptions(
     }
   }
 
-  return
+  return {
+    sort: undefined,
+    sortOrder: undefined,
+    sortType: undefined,
+  }
 }

packages/server/src/api/controllers/table/external.ts

@@ -22,9 +22,12 @@ import {
   QueryJson,
   RelationshipType,
   RenameColumn,
+  SaveTableRequest,
+  SaveTableResponse,
   Table,
   TableRequest,
   UserCtx,
+  ViewV2,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { builderSocket } from "../../../websockets"
@@ -198,8 +201,8 @@ function isRelationshipSetup(column: FieldSchema) {
   return column.foreignKey || column.through
 }
 
-export async function save(ctx: UserCtx) {
+export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
-  const inputs: TableRequest = ctx.request.body
+  const inputs = ctx.request.body
   const renamed = inputs?._rename
   // can't do this right now
   delete inputs.rows
@@ -215,7 +218,7 @@ export async function save(ctx: UserCtx) {
     ...inputs,
   }
 
-  let oldTable
+  let oldTable: Table | undefined
   if (ctx.request.body && ctx.request.body._id) {
     oldTable = await sdk.tables.getTable(ctx.request.body._id)
   }
@@ -224,6 +227,17 @@ export async function save(ctx: UserCtx) {
     ctx.throw(400, "A column type has changed.")
   }
 
+  for (let view in tableToSave.views) {
+    const tableView = tableToSave.views[view]
+    if (!tableView || !sdk.views.isV2(tableView)) continue
+
+    tableToSave.views[view] = sdk.views.syncSchema(
+      oldTable!.views![view] as ViewV2,
+      tableToSave.schema,
+      renamed
+    )
+  }
+
   const db = context.getAppDB()
   const datasource = await sdk.datasources.get(datasourceId)
   if (!datasource.entities) {

packages/server/src/api/controllers/table/index.ts

@@ -9,6 +9,8 @@ import { isExternalTable, isSQL } from "../../../integrations/utils"
 import { events } from "@budibase/backend-core"
 import {
   FetchTablesResponse,
+  SaveTableResponse,
+  SaveTableRequest,
   Table,
   TableResponse,
   UserCtx,
@@ -60,7 +62,7 @@ export async function find(ctx: UserCtx<void, TableResponse>) {
   ctx.body = sdk.tables.enrichViewSchemas(table)
 }
 
-export async function save(ctx: UserCtx) {
+export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   const appId = ctx.appId
   const table = ctx.request.body
   const isImport = table.rows

packages/server/src/api/controllers/table/internal.ts

@@ -9,7 +9,15 @@ import {
   fixAutoColumnSubType,
 } from "../../../utilities/rowProcessor"
 import { runStaticFormulaChecks } from "./bulkFormula"
-import { Table } from "@budibase/types"
+import {
+  SaveTableRequest,
+  SaveTableResponse,
+  Table,
+  TableRequest,
+  UserCtx,
+  ViewStatisticsSchema,
+  ViewV2,
+} from "@budibase/types"
 import { quotas } from "@budibase/pro"
 import isEqual from "lodash/isEqual"
 import { cloneDeep } from "lodash/fp"
@@ -33,10 +41,10 @@ function checkAutoColumns(table: Table, oldTable?: Table) {
   return table
 }
 
-export async function save(ctx: any) {
+export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   const db = context.getAppDB()
   const { rows, ...rest } = ctx.request.body
-  let tableToSave = {
+  let tableToSave: TableRequest = {
     type: "table",
     _id: generateTableID(),
     views: {},
@@ -44,7 +52,7 @@ export async function save(ctx: any) {
   }
 
   // if the table obj had an _id then it will have been retrieved
-  let oldTable
+  let oldTable: Table | undefined
   if (ctx.request.body && ctx.request.body._id) {
     oldTable = await sdk.tables.getTable(ctx.request.body._id)
   }
@@ -80,7 +88,7 @@ export async function save(ctx: any) {
   let { _rename } = tableToSave
   /* istanbul ignore next */
   if (_rename && _rename.old === _rename.updated) {
-    _rename = null
+    _rename = undefined
     delete tableToSave._rename
   }
 
@@ -97,7 +105,20 @@ export async function save(ctx: any) {
     const tableView = tableToSave.views[view]
     if (!tableView) continue
 
-    if (tableView.schema.group || tableView.schema.field) continue
+    if (sdk.views.isV2(tableView)) {
+      tableToSave.views[view] = sdk.views.syncSchema(
+        oldTable!.views![view] as ViewV2,
+        tableToSave.schema,
+        _rename
+      )
+      continue
+    }
+
+    if (
+      (tableView.schema as ViewStatisticsSchema).group ||
+      tableView.schema.field
+    )
+      continue
     tableView.schema = tableToSave.schema
   }
 
@@ -112,7 +133,7 @@ export async function save(ctx: any) {
       tableToSave._rev = linkResp._rev
     }
   } catch (err) {
-    ctx.throw(400, err)
+    ctx.throw(400, err as string)
   }
 
   // don't perform any updates until relationships have been

packages/server/src/api/controllers/table/utils.ts

@@ -418,7 +418,7 @@ export function areSwitchableTypes(type1: any, type2: any) {
   return false
 }
 
-export function hasTypeChanged(table: any, oldTable: any) {
+export function hasTypeChanged(table: Table, oldTable: Table | undefined) {
   if (!oldTable) {
     return false
   }

packages/server/src/api/routes/public/roles.ts

@@ -0,0 +1,56 @@
+import controller from "../../controllers/public/roles"
+import Endpoint from "./utils/Endpoint"
+
+const write = []
+
+/**
+ * @openapi
+ * /roles/assign:
+ *   post:
+ *     operationId: roleAssign
+ *     summary: Assign a role to a list of users
+ *     description: This is a business/enterprise only endpoint
+ *     tags:
+ *       - roles
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/rolesAssign'
+ *     responses:
+ *       200:
+ *         description: Returns a list of updated user IDs
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/rolesOutput'
+ */
+write.push(new Endpoint("post", "/roles/assign", controller.assign))
+
+/**
+ * @openapi
+ * /roles/unassign:
+ *   post:
+ *     operationId: roleUnAssign
+ *     summary: Un-assign a role from a list of users
+ *     description: This is a business/enterprise only endpoint
+ *     tags:
+ *       - roles
+ *     requestBody:
+ *       required: true
+ *       content:
+ *         application/json:
+ *           schema:
+ *             $ref: '#/components/schemas/rolesUnAssign'
+ *     responses:
+ *       200:
+ *         description: Returns a list of updated user IDs
+ *         content:
+ *           application/json:
+ *             schema:
+ *               $ref: '#/components/schemas/rolesOutput'
+ */
+write.push(new Endpoint("post", "/roles/unassign", controller.unAssign))
+
+export default { write, read: [] }

packages/server/src/api/routes/public/tests/users.spec.js

@@ -1,38 +0,0 @@
-const setup = require("../../tests/utilities")
-const { generateMakeRequest } = require("./utils")
-
-const workerRequests = require("../../../../utilities/workerRequests")
-
-let config = setup.getConfig()
-let apiKey, globalUser, makeRequest
-
-beforeAll(async () => {
-  await config.init()
-  globalUser = await config.globalUser()
-  apiKey = await config.generateApiKey(globalUser._id)
-  makeRequest = generateMakeRequest(apiKey)
-  workerRequests.readGlobalUser.mockReturnValue(globalUser)
-})
-
-afterAll(setup.afterAll)
-
-describe("check user endpoints", () => {
-  it("should not allow a user to update their own roles", async () => {
-    const res = await makeRequest("put", `/users/${globalUser._id}`, {
-      ...globalUser,
-      roles: {
-        "app_1": "ADMIN",
-      }
-    })
-    expect(workerRequests.saveGlobalUser.mock.lastCall[0].body.data.roles["app_1"]).toBeUndefined()
-    expect(res.status).toBe(200)
-    expect(res.body.data.roles["app_1"]).toBeUndefined()
-  })
-
-  it("should not allow a user to delete themselves", async () => {
-    const res = await makeRequest("delete", `/users/${globalUser._id}`)
-    expect(res.status).toBe(405)
-    expect(workerRequests.deleteGlobalUser.mock.lastCall).toBeUndefined()
-  })
-})
-

packages/server/src/api/routes/public/tests/users.spec.ts

@@ -0,0 +1,126 @@
+import * as setup from "../../tests/utilities"
+import { generateMakeRequest, MakeRequestResponse } from "./utils"
+import { User } from "@budibase/types"
+import { mocks } from "@budibase/backend-core/tests"
+
+import * as workerRequests from "../../../../utilities/workerRequests"
+
+const mockedWorkerReq = jest.mocked(workerRequests)
+
+let config = setup.getConfig()
+let apiKey: string, globalUser: User, makeRequest: MakeRequestResponse
+
+beforeAll(async () => {
+  await config.init()
+  globalUser = await config.globalUser()
+  apiKey = await config.generateApiKey(globalUser._id)
+  makeRequest = generateMakeRequest(apiKey)
+  mockedWorkerReq.readGlobalUser.mockImplementation(() =>
+    Promise.resolve(globalUser)
+  )
+})
+
+afterAll(setup.afterAll)
+
+function base() {
+  return {
+    tenantId: config.getTenantId(),
+    firstName: "Test",
+    lastName: "Test",
+  }
+}
+
+function updateMock() {
+  mockedWorkerReq.readGlobalUser.mockImplementation(ctx => ctx.request.body)
+}
+
+describe("check user endpoints", () => {
+  it("should not allow a user to update their own roles", async () => {
+    const res = await makeRequest("put", `/users/${globalUser._id}`, {
+      ...globalUser,
+      roles: {
+        app_1: "ADMIN",
+      },
+    })
+    expect(
+      mockedWorkerReq.saveGlobalUser.mock.lastCall?.[0].body.data.roles["app_1"]
+    ).toBeUndefined()
+    expect(res.status).toBe(200)
+    expect(res.body.data.roles["app_1"]).toBeUndefined()
+  })
+
+  it("should not allow a user to delete themselves", async () => {
+    const res = await makeRequest("delete", `/users/${globalUser._id}`)
+    expect(res.status).toBe(405)
+    expect(mockedWorkerReq.deleteGlobalUser.mock.lastCall).toBeUndefined()
+  })
+})
+
+describe("no user role update in free", () => {
+  beforeAll(() => {
+    updateMock()
+  })
+
+  it("should not allow 'roles' to be updated", async () => {
+    const res = await makeRequest("post", "/users", {
+      ...base(),
+      roles: { app_a: "BASIC" },
+    })
+    expect(res.status).toBe(200)
+    expect(res.body.data.roles["app_a"]).toBeUndefined()
+  })
+
+  it("should not allow 'admin' to be updated", async () => {
+    const res = await makeRequest("post", "/users", {
+      ...base(),
+      admin: { global: true },
+    })
+    expect(res.status).toBe(200)
+    expect(res.body.data.admin).toBeUndefined()
+  })
+
+  it("should not allow 'builder' to be updated", async () => {
+    const res = await makeRequest("post", "/users", {
+      ...base(),
+      builder: { global: true },
+    })
+    expect(res.status).toBe(200)
+    expect(res.body.data.builder).toBeUndefined()
+  })
+})
+
+describe("no user role update in business", () => {
+  beforeAll(() => {
+    updateMock()
+    mocks.licenses.usePublicApiUserRoles()
+  })
+
+  it("should allow 'roles' to be updated", async () => {
+    const res = await makeRequest("post", "/users", {
+      ...base(),
+      roles: { app_a: "BASIC" },
+    })
+    expect(res.status).toBe(200)
+    expect(res.body.data.roles["app_a"]).toBe("BASIC")
+  })
+
+  it("should allow 'admin' to be updated", async () => {
+    mocks.licenses.usePublicApiUserRoles()
+    const res = await makeRequest("post", "/users", {
+      ...base(),
+      admin: { global: true },
+    })
+    expect(res.status).toBe(200)
+    expect(res.body.data.admin.global).toBe(true)
+  })
+
+  it("should allow 'builder' to be updated", async () => {
+    mocks.licenses.usePublicApiUserRoles()
+    const res = await makeRequest("post", "/users", {
+      ...base(),
+      builder: { global: true },
+    })
+    expect(res.status).toBe(200)
+    expect(res.body.data.builder.global).toBe(true)
+  })
+})

packages/server/src/api/routes/row.ts

@@ -4,16 +4,14 @@ import authorized from "../../middleware/authorized"
 import { paramResource, paramSubResource } from "../../middleware/resourceId"
 import { permissions } from "@budibase/backend-core"
 import { internalSearchValidator } from "./utils/validators"
-import noViewData from "../../middleware/noViewData"
 import trimViewRowInfo from "../../middleware/trimViewRowInfo"
-import * as utils from "../../db/utils"
 const { PermissionType, PermissionLevel } = permissions
 
 const router: Router = new Router()
 
 router
   /**
-   * @api {get} /api/:tableId/:rowId/enrich Get an enriched row
+   * @api {get} /api/:sourceId/:rowId/enrich Get an enriched row
    * @apiName Get an enriched row
    * @apiGroup rows
    * @apiPermission table read access
@@ -27,13 +25,13 @@ router
    * @apiSuccess {object} row The response body will be the enriched row.
    */
   .get(
-    "/api/:tableId/:rowId/enrich",
+    "/api/:sourceId/:rowId/enrich",
-    paramSubResource("tableId", "rowId"),
+    paramSubResource("sourceId", "rowId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.fetchEnrichedRow
   )
   /**
-   * @api {get} /api/:tableId/rows Get all rows in a table
+   * @api {get} /api/:sourceId/rows Get all rows in a table
    * @apiName Get all rows in a table
    * @apiGroup rows
    * @apiPermission table read access
@@ -42,37 +40,37 @@ router
    * due to its lack of support for pagination. With SQL tables this will retrieve up to a limit and then
    * will simply stop.
    *
-   * @apiParam {string} tableId The ID of the table to retrieve all rows within.
+   * @apiParam {string} sourceId The ID of the table to retrieve all rows within.
    *
    * @apiSuccess {object[]} rows The response body will be an array of all rows found.
    */
   .get(
-    "/api/:tableId/rows",
+    "/api/:sourceId/rows",
-    paramResource("tableId"),
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.fetch
   )
   /**
-   * @api {get} /api/:tableId/rows/:rowId Retrieve a single row
+   * @api {get} /api/:sourceId/rows/:rowId Retrieve a single row
    * @apiName Retrieve a single row
    * @apiGroup rows
    * @apiPermission table read access
    * @apiDescription This endpoint retrieves only the specified row. If you wish to retrieve
    * a row by anything other than its _id field, use the search endpoint.
    *
-   * @apiParam {string} tableId The ID of the table to retrieve a row from.
+   * @apiParam {string} sourceId The ID of the table to retrieve a row from.
    * @apiParam {string} rowId The ID of the row to retrieve.
    *
    * @apiSuccess {object} body The response body will be the row that was found.
    */
   .get(
-    "/api/:tableId/rows/:rowId",
+    "/api/:sourceId/rows/:rowId",
-    paramSubResource("tableId", "rowId"),
+    paramSubResource("sourceId", "rowId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.find
   )
   /**
-   * @api {post} /api/:tableId/search Search for rows in a table
+   * @api {post} /api/:sourceId/search Search for rows in a table
    * @apiName Search for rows in a table
    * @apiGroup rows
    * @apiPermission table read access
@@ -80,7 +78,7 @@ router
    * and data UI in the builder are built atop this. All filtering, sorting and pagination is
    * handled through this, for internal and external (datasource plus, e.g. SQL) tables.
    *
-   * @apiParam {string} tableId The ID of the table to retrieve rows from.
+   * @apiParam {string} sourceId The ID of the table to retrieve rows from.
    *
    * @apiParam (Body) {boolean} [paginate] If pagination is required then this should be set to true,
    * defaults to false.
@@ -135,22 +133,22 @@ router
    * page.
    */
   .post(
-    "/api/:tableId/search",
+    "/api/:sourceId/search",
     internalSearchValidator(),
-    paramResource("tableId"),
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.search
   )
   // DEPRECATED - this is an old API, but for backwards compat it needs to be
   // supported still
   .post(
-    "/api/search/:tableId/rows",
+    "/api/search/:sourceId/rows",
-    paramResource("tableId"),
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.search
   )
   /**
-   * @api {post} /api/:tableId/rows Creates a new row
+   * @api {post} /api/:sourceId/rows Creates a new row
    * @apiName Creates a new row
    * @apiGroup rows
    * @apiPermission table write access
@@ -159,7 +157,7 @@ router
    * links to one. Please note that "_id", "_rev" and "tableId" are fields that are
    * already used by Budibase tables and cannot be used for columns.
    *
-   * @apiParam {string} tableId The ID of the table to save a row to.
+   * @apiParam {string} sourceId The ID of the table to save a row to.
    *
    * @apiParam (Body) {string} [_id] If the row exists already then an ID for the row must be provided.
    * @apiParam (Body) {string} [_rev] If working with an existing row for an internal table its revision
@@ -174,14 +172,14 @@ router
    * @apiSuccess {object} body The contents of the row that was saved will be returned as well.
    */
   .post(
-    "/api/:tableId/rows",
+    "/api/:sourceId/rows",
-    paramResource("tableId"),
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
-    noViewData,
+    trimViewRowInfo,
     rowController.save
   )
   /**
-   * @api {patch} /api/:tableId/rows Updates a row
+   * @api {patch} /api/:sourceId/rows Updates a row
    * @apiName Update a row
    * @apiGroup rows
    * @apiPermission table write access
@@ -189,14 +187,14 @@ router
    * error if an _id isn't provided, it will only function for existing rows.
    */
   .patch(
-    "/api/:tableId/rows",
+    "/api/:sourceId/rows",
-    paramResource("tableId"),
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
-    noViewData,
+    trimViewRowInfo,
     rowController.patch
   )
   /**
-   * @api {post} /api/:tableId/rows/validate Validate inputs for a row
+   * @api {post} /api/:sourceId/rows/validate Validate inputs for a row
    * @apiName Validate inputs for a row
    * @apiGroup rows
    * @apiPermission table write access
@@ -204,7 +202,7 @@ router
    * given the table schema, this will iterate through all the constraints on the table and
    * check if the request body is valid.
    *
-   * @apiParam {string} tableId The ID of the table the row is to be validated for.
+   * @apiParam {string} sourceId The ID of the table the row is to be validated for.
    *
    * @apiParam (Body) {any} [any] Any fields provided in the request body will be tested
    * against the table schema and constraints.
@@ -216,20 +214,20 @@ router
    * the schema.
    */
   .post(
-    "/api/:tableId/rows/validate",
+    "/api/:sourceId/rows/validate",
-    paramResource("tableId"),
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
     rowController.validate
   )
   /**
-   * @api {delete} /api/:tableId/rows Delete rows
+   * @api {delete} /api/:sourceId/rows Delete rows
    * @apiName Delete rows
    * @apiGroup rows
    * @apiPermission table write access
    * @apiDescription This endpoint can delete a single row, or delete them in a bulk
    * fashion.
    *
-   * @apiParam {string} tableId The ID of the table the row is to be deleted from.
+   * @apiParam {string} sourceId The ID of the table the row is to be deleted from.
    *
    * @apiParam (Body) {object[]} [rows] If bulk deletion is desired then provide the rows in this
    * key of the request body that are to be deleted.
@@ -242,117 +240,37 @@ router
    * is the deleted row.
    */
   .delete(
-    "/api/:tableId/rows",
+    "/api/:sourceId/rows",
-    paramResource("tableId"),
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
+    trimViewRowInfo,
     rowController.destroy
   )
 
   /**
-   * @api {post} /api/:tableId/rows/exportRows Export Rows
+   * @api {post} /api/:sourceId/rows/exportRows Export Rows
    * @apiName Export rows
    * @apiGroup rows
    * @apiPermission table write access
    * @apiDescription This API can export a number of provided rows
    *
-   * @apiParam {string} tableId The ID of the table the row is to be deleted from.
+   * @apiParam {string} sourceId The ID of the table the row is to be deleted from.
    *
    * @apiParam (Body) {object[]} [rows] The row IDs which are to be exported
    *
    * @apiSuccess {object[]|object}
    */
   .post(
-    "/api/:tableId/rows/exportRows",
+    "/api/:sourceId/rows/exportRows",
-    paramResource("tableId"),
+    paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
     rowController.exportRows
   )
 
-router
+router.post(
-  .get(
+  "/api/v2/views/:viewId/search",
-    "/api/v2/views/:viewId/search",
+  authorized(PermissionType.TABLE, PermissionLevel.READ),
-    authorized(PermissionType.VIEW, PermissionLevel.READ),
+  rowController.views.searchView
-    rowController.views.searchView
+)
-  )
-  /**
-   * @api {post} /api/:tableId/rows Creates a new row
-   * @apiName Creates a new row
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This API will create a new row based on the supplied body. If the
-   * body includes an "_id" field then it will update an existing row if the field
-   * links to one. Please note that "_id", "_rev" and "tableId" are fields that are
-   * already used by Budibase tables and cannot be used for columns.
-   *
-   * @apiParam {string} tableId The ID of the table to save a row to.
-   *
-   * @apiParam (Body) {string} [_id] If the row exists already then an ID for the row must be provided.
-   * @apiParam (Body) {string} [_rev] If working with an existing row for an internal table its revision
-   * must also be provided.
-   * @apiParam (Body) {string} _viewId The ID of the view should be specified in the row body itself.
-   * @apiParam (Body) {string} tableId The ID of the table should also be specified in the row body itself.
-   * @apiParam (Body) {any} [any] Any field supplied in the body will be assessed to see if it matches
-   * a column in the specified table. All other fields will be dropped and not stored.
-   *
-   * @apiSuccess {string} _id The ID of the row that was just saved, if it was just created this
-   * is the rows new ID.
-   * @apiSuccess {string} [_rev] If saving to an internal table a revision will also be returned.
-   * @apiSuccess {object} body The contents of the row that was saved will be returned as well.
-   */
-  .post(
-    "/api/v2/views/:viewId/rows",
-    paramResource("viewId"),
-    authorized(PermissionType.VIEW, PermissionLevel.WRITE),
-    trimViewRowInfo,
-    rowController.save
-  )
-  /**
-   * @api {patch} /api/v2/views/:viewId/rows/:rowId Updates a row
-   * @apiName Update a row
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This endpoint is identical to the row creation endpoint but instead it will
-   * error if an _id isn't provided, it will only function for existing rows.
-   */
-  .patch(
-    "/api/v2/views/:viewId/rows/:rowId",
-    paramResource("viewId"),
-    authorized(PermissionType.VIEW, PermissionLevel.WRITE),
-    trimViewRowInfo,
-    rowController.patch
-  )
-  /**
-   * @api {delete} /api/v2/views/:viewId/rows Delete rows for a view
-   * @apiName Delete rows for a view
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This endpoint can delete a single row, or delete them in a bulk
-   * fashion.
-   *
-   * @apiParam {string} tableId The ID of the table the row is to be deleted from.
-   *
-   * @apiParam (Body) {object[]} [rows] If bulk deletion is desired then provide the rows in this
-   * key of the request body that are to be deleted.
-   * @apiParam (Body) {string} [_id] If deleting a single row then provide its ID in this field.
-   * @apiParam (Body) {string} [_rev] If deleting a single row from an internal table then provide its
-   * revision here.
-   *
-   * @apiSuccess {object[]|object} body If deleting bulk then the response body will be an array
-   * of the deleted rows, if deleting a single row then the body will contain a "row" property which
-   * is the deleted row.
-   */
-  .delete(
-    "/api/v2/views/:viewId/rows",
-    paramResource("viewId"),
-    authorized(PermissionType.VIEW, PermissionLevel.WRITE),
-    // This is required as the implementation relies on the table id
-    (ctx, next) => {
-      ctx.params.tableId = utils.extractViewInfoFromID(
-        ctx.params.viewId
-      ).tableId
-      return next()
-    },
-    rowController.destroy
-  )
 
 export default router

packages/server/src/api/routes/tests/row.spec.ts

@@ -16,16 +16,12 @@ import {
   FieldType,
   SortType,
   SortOrder,
-  DeleteRow,
 } from "@budibase/types"
 import {
   expectAnyInternalColsAttributes,
   generator,
   structures,
 } from "@budibase/backend-core/tests"
-import trimViewRowInfoMiddleware from "../../../middleware/trimViewRowInfo"
-import noViewDataMiddleware from "../../../middleware/noViewData"
-import router from "../row"
 
 describe("/rows", () => {
   let request = setup.getRequest()
@@ -394,26 +390,6 @@ describe("/rows", () => {
       expect(saved.arrayFieldArrayStrKnown).toEqual(["One"])
       expect(saved.optsFieldStrKnown).toEqual("Alpha")
     })
-
-    it("should throw an error when creating a table row with view id data", async () => {
-      const res = await request
-        .post(`/api/${row.tableId}/rows`)
-        .send({ ...row, _viewId: generator.guid() })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(400)
-      expect(res.body.message).toEqual(
-        "Table row endpoints cannot contain view info"
-      )
-    })
-
-    it("should setup the noViewData middleware", async () => {
-      const route = router.stack.find(
-        r => r.methods.includes("POST") && r.path === "/api/:tableId/rows"
-      )
-      expect(route).toBeDefined()
-      expect(route?.stack).toContainEqual(noViewDataMiddleware)
-    })
   })
 
   describe("patch", () => {
@@ -463,33 +439,6 @@ describe("/rows", () => {
       await assertRowUsage(rowUsage)
       await assertQueryUsage(queryUsage)
     })
-
-    it("should throw an error when creating a table row with view id data", async () => {
-      const existing = await config.createRow()
-
-      const res = await config.api.row.patch(
-        table._id!,
-        {
-          ...existing,
-          _id: existing._id!,
-          _rev: existing._rev!,
-          tableId: table._id!,
-          _viewId: generator.guid(),
-        },
-        { expectStatus: 400 }
-      )
-      expect(res.body.message).toEqual(
-        "Table row endpoints cannot contain view info"
-      )
-    })
-
-    it("should setup the noViewData middleware", async () => {
-      const route = router.stack.find(
-        r => r.methods.includes("PATCH") && r.path === "/api/:tableId/rows"
-      )
-      expect(route).toBeDefined()
-      expect(route?.stack).toContainEqual(noViewDataMiddleware)
-    })
   })
 
   describe("destroy", () => {
@@ -758,7 +707,7 @@ describe("/rows", () => {
       })
       // the environment needs configured for this
       await setup.switchToSelfHosted(async () => {
-        context.doInAppContext(config.getAppId(), async () => {
+        return context.doInAppContext(config.getAppId(), async () => {
           const enriched = await outputProcessing(table, [row])
           expect((enriched as Row[])[0].attachment[0].url).toBe(
             `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
@@ -813,252 +762,6 @@ describe("/rows", () => {
     })
   })
 
-  describe("view search", () => {
-    function userTable(): Table {
-      return {
-        name: "user",
-        type: "user",
-        schema: {
-          name: {
-            type: FieldType.STRING,
-            name: "name",
-            constraints: { type: "string" },
-          },
-          age: {
-            type: FieldType.NUMBER,
-            name: "age",
-            constraints: {},
-          },
-        },
-      }
-    }
-
-    it("returns table rows from view", async () => {
-      const table = await config.createTable(userTable())
-      const rows = []
-      for (let i = 0; i < 10; i++) {
-        rows.push(await config.createRow({ tableId: table._id }))
-      }
-
-      const createViewResponse = await config.api.viewV2.create()
-      const response = await config.api.viewV2.search(createViewResponse.id)
-
-      expect(response.body.rows).toHaveLength(10)
-      expect(response.body).toEqual({
-        rows: expect.arrayContaining(rows.map(expect.objectContaining)),
-      })
-    })
-
-    it("searching respects the view filters", async () => {
-      const table = await config.createTable(userTable())
-      const expectedRows = []
-      for (let i = 0; i < 10; i++)
-        await config.createRow({
-          tableId: table._id,
-          name: generator.name(),
-          age: generator.integer({ min: 10, max: 30 }),
-        })
-
-      for (let i = 0; i < 5; i++)
-        expectedRows.push(
-          await config.createRow({
-            tableId: table._id,
-            name: generator.name(),
-            age: 40,
-          })
-        )
-
-      const createViewResponse = await config.api.viewV2.create({
-        query: { equal: { age: 40 } },
-      })
-
-      const response = await config.api.viewV2.search(createViewResponse.id)
-
-      expect(response.body.rows).toHaveLength(5)
-      expect(response.body).toEqual({
-        rows: expect.arrayContaining(expectedRows.map(expect.objectContaining)),
-      })
-    })
-
-    const sortTestOptions: [
-      {
-        field: string
-        order?: SortOrder
-        type?: SortType
-      },
-      string[]
-    ][] = [
-      [
-        {
-          field: "name",
-          order: SortOrder.ASCENDING,
-          type: SortType.STRING,
-        },
-        ["Alice", "Bob", "Charly", "Danny"],
-      ],
-      [
-        {
-          field: "name",
-        },
-        ["Alice", "Bob", "Charly", "Danny"],
-      ],
-      [
-        {
-          field: "name",
-          order: SortOrder.DESCENDING,
-        },
-        ["Danny", "Charly", "Bob", "Alice"],
-      ],
-      [
-        {
-          field: "name",
-          order: SortOrder.DESCENDING,
-          type: SortType.STRING,
-        },
-        ["Danny", "Charly", "Bob", "Alice"],
-      ],
-      [
-        {
-          field: "age",
-          order: SortOrder.ASCENDING,
-          type: SortType.number,
-        },
-        ["Danny", "Alice", "Charly", "Bob"],
-      ],
-      [
-        {
-          field: "age",
-          order: SortOrder.ASCENDING,
-        },
-        ["Danny", "Alice", "Charly", "Bob"],
-      ],
-      [
-        {
-          field: "age",
-          order: SortOrder.DESCENDING,
-        },
-        ["Bob", "Charly", "Alice", "Danny"],
-      ],
-      [
-        {
-          field: "age",
-          order: SortOrder.DESCENDING,
-          type: SortType.number,
-        },
-        ["Bob", "Charly", "Alice", "Danny"],
-      ],
-    ]
-
-    it.each(sortTestOptions)(
-      "allow sorting (%s)",
-      async (sortParams, expected) => {
-        await config.createTable(userTable())
-        const users = [
-          { name: "Alice", age: 25 },
-          { name: "Bob", age: 30 },
-          { name: "Charly", age: 27 },
-          { name: "Danny", age: 15 },
-        ]
-        for (const user of users) {
-          await config.createRow({
-            tableId: config.table!._id,
-            ...user,
-          })
-        }
-
-        const createViewResponse = await config.api.viewV2.create({
-          sort: sortParams,
-        })
-
-        const response = await config.api.viewV2.search(createViewResponse.id)
-
-        expect(response.body.rows).toHaveLength(4)
-        expect(response.body).toEqual({
-          rows: expected.map(name => expect.objectContaining({ name })),
-        })
-      }
-    )
-
-    it.each(sortTestOptions)(
-      "allow override the default view sorting (%s)",
-      async (sortParams, expected) => {
-        await config.createTable(userTable())
-        const users = [
-          { name: "Alice", age: 25 },
-          { name: "Bob", age: 30 },
-          { name: "Charly", age: 27 },
-          { name: "Danny", age: 15 },
-        ]
-        for (const user of users) {
-          await config.createRow({
-            tableId: config.table!._id,
-            ...user,
-          })
-        }
-
-        const createViewResponse = await config.api.viewV2.create({
-          sort: {
-            field: "name",
-            order: SortOrder.ASCENDING,
-            type: SortType.STRING,
-          },
-        })
-
-        const response = await config.api.viewV2.search(createViewResponse.id, {
-          sort: {
-            column: sortParams.field,
-            order: sortParams.order,
-            type: sortParams.type,
-          },
-        })
-
-        expect(response.body.rows).toHaveLength(4)
-        expect(response.body).toEqual({
-          rows: expected.map(name => expect.objectContaining({ name })),
-        })
-      }
-    )
-
-    it("when schema is defined, defined columns and row attributes are returned", async () => {
-      const table = await config.createTable(userTable())
-      const rows = []
-      for (let i = 0; i < 10; i++) {
-        rows.push(
-          await config.createRow({
-            tableId: table._id,
-            name: generator.name(),
-            age: generator.age(),
-          })
-        )
-      }
-
-      const view = await config.api.viewV2.create({
-        schema: { name: {} },
-      })
-      const response = await config.api.viewV2.search(view.id)
-
-      expect(response.body.rows).toHaveLength(10)
-      expect(response.body.rows).toEqual(
-        expect.arrayContaining(
-          rows.map(r => ({
-            ...expectAnyInternalColsAttributes,
-            _viewId: view.id,
-            name: r.name,
-          }))
-        )
-      )
-    })
-
-    it("views without data can be returned", async () => {
-      const table = await config.createTable(userTable())
-
-      const createViewResponse = await config.api.viewV2.create()
-      const response = await config.api.viewV2.search(createViewResponse.id)
-
-      expect(response.body.rows).toHaveLength(0)
-    })
-  })
-
   describe("view 2.0", () => {
     function userTable(): Table {
       return {
@@ -1110,7 +813,7 @@ describe("/rows", () => {
         })
 
         const data = randomRowData()
-        const newRow = await config.api.viewV2.row.create(view.id, {
+        const newRow = await config.api.row.save(view.id, {
           tableId: config.table!._id,
           _viewId: view.id,
           ...data,
@@ -1132,16 +835,6 @@ describe("/rows", () => {
         expect(row.body.age).toBeUndefined()
         expect(row.body.jobTitle).toBeUndefined()
       })
-
-      it("should setup the trimViewRowInfo middleware", async () => {
-        const route = router.stack.find(
-          r =>
-            r.methods.includes("POST") &&
-            r.path === "/api/v2/views/:viewId/rows"
-        )
-        expect(route).toBeDefined()
-        expect(route?.stack).toContainEqual(trimViewRowInfoMiddleware)
-      })
     })
 
     describe("patch", () => {
@@ -1156,13 +849,13 @@ describe("/rows", () => {
           },
         })
 
-        const newRow = await config.api.viewV2.row.create(view.id, {
+        const newRow = await config.api.row.save(view.id, {
           tableId,
           _viewId: view.id,
           ...randomRowData(),
         })
         const newData = randomRowData()
-        await config.api.viewV2.row.update(view.id, newRow._id!, {
+        await config.api.row.patch(view.id, {
           tableId,
           _viewId: view.id,
           _id: newRow._id!,
@@ -1185,16 +878,6 @@ describe("/rows", () => {
         expect(row.body.age).toBeUndefined()
         expect(row.body.jobTitle).toBeUndefined()
       })
-
-      it("should setup the trimViewRowInfo middleware", async () => {
-        const route = router.stack.find(
-          r =>
-            r.methods.includes("PATCH") &&
-            r.path === "/api/v2/views/:viewId/rows/:rowId"
-        )
-        expect(route).toBeDefined()
-        expect(route?.stack).toContainEqual(trimViewRowInfoMiddleware)
-      })
     })
 
     describe("destroy", () => {
@@ -1213,10 +896,7 @@ describe("/rows", () => {
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
-        const body: DeleteRow = {
+        await config.api.row.delete(view.id, [createdRow])
-          _id: createdRow._id!,
-        }
-        await config.api.viewV2.row.delete(view.id, body)
 
         await assertRowUsage(rowUsage - 1)
         await assertQueryUsage(queryUsage + 1)
@@ -1245,9 +925,7 @@ describe("/rows", () => {
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
-        await config.api.viewV2.row.delete(view.id, {
+        await config.api.row.delete(view.id, [rows[0], rows[2]])
-          rows: [rows[0], rows[2]],
-        })
 
         await assertRowUsage(rowUsage - 2)
         await assertQueryUsage(queryUsage + 1)
@@ -1261,5 +939,327 @@ describe("/rows", () => {
         await config.api.row.get(tableId, rows[1]._id!, { expectStatus: 200 })
       })
     })
+
+    describe("view search", () => {
+      function userTable(): Table {
+        return {
+          name: "user",
+          type: "user",
+          schema: {
+            name: {
+              type: FieldType.STRING,
+              name: "name",
+              constraints: { type: "string" },
+            },
+            age: {
+              type: FieldType.NUMBER,
+              name: "age",
+              constraints: {},
+            },
+          },
+        }
+      }
+
+      it("returns table rows from view", async () => {
+        const table = await config.createTable(userTable())
+        const rows = []
+        for (let i = 0; i < 10; i++) {
+          rows.push(await config.createRow({ tableId: table._id }))
+        }
+
+        const createViewResponse = await config.api.viewV2.create()
+        const response = await config.api.viewV2.search(createViewResponse.id)
+
+        expect(response.body.rows).toHaveLength(10)
+        expect(response.body).toEqual({
+          rows: expect.arrayContaining(rows.map(expect.objectContaining)),
+        })
+      })
+
+      it("searching respects the view filters", async () => {
+        const table = await config.createTable(userTable())
+        const expectedRows = []
+        for (let i = 0; i < 10; i++)
+          await config.createRow({
+            tableId: table._id,
+            name: generator.name(),
+            age: generator.integer({ min: 10, max: 30 }),
+          })
+
+        for (let i = 0; i < 5; i++)
+          expectedRows.push(
+            await config.createRow({
+              tableId: table._id,
+              name: generator.name(),
+              age: 40,
+            })
+          )
+
+        const createViewResponse = await config.api.viewV2.create({
+          query: [{ operator: "equal", field: "age", value: 40 }],
+        })
+
+        const response = await config.api.viewV2.search(createViewResponse.id)
+
+        expect(response.body.rows).toHaveLength(5)
+        expect(response.body).toEqual({
+          rows: expect.arrayContaining(
+            expectedRows.map(expect.objectContaining)
+          ),
+        })
+      })
+
+      const sortTestOptions: [
+        {
+          field: string
+          order?: SortOrder
+          type?: SortType
+        },
+        string[]
+      ][] = [
+        [
+          {
+            field: "name",
+            order: SortOrder.ASCENDING,
+            type: SortType.STRING,
+          },
+          ["Alice", "Bob", "Charly", "Danny"],
+        ],
+        [
+          {
+            field: "name",
+          },
+          ["Alice", "Bob", "Charly", "Danny"],
+        ],
+        [
+          {
+            field: "name",
+            order: SortOrder.DESCENDING,
+          },
+          ["Danny", "Charly", "Bob", "Alice"],
+        ],
+        [
+          {
+            field: "name",
+            order: SortOrder.DESCENDING,
+            type: SortType.STRING,
+          },
+          ["Danny", "Charly", "Bob", "Alice"],
+        ],
+        [
+          {
+            field: "age",
+            order: SortOrder.ASCENDING,
+            type: SortType.number,
+          },
+          ["Danny", "Alice", "Charly", "Bob"],
+        ],
+        [
+          {
+            field: "age",
+            order: SortOrder.ASCENDING,
+          },
+          ["Danny", "Alice", "Charly", "Bob"],
+        ],
+        [
+          {
+            field: "age",
+            order: SortOrder.DESCENDING,
+          },
+          ["Bob", "Charly", "Alice", "Danny"],
+        ],
+        [
+          {
+            field: "age",
+            order: SortOrder.DESCENDING,
+            type: SortType.number,
+          },
+          ["Bob", "Charly", "Alice", "Danny"],
+        ],
+      ]
+
+      it.each(sortTestOptions)(
+        "allow sorting (%s)",
+        async (sortParams, expected) => {
+          await config.createTable(userTable())
+          const users = [
+            { name: "Alice", age: 25 },
+            { name: "Bob", age: 30 },
+            { name: "Charly", age: 27 },
+            { name: "Danny", age: 15 },
+          ]
+          for (const user of users) {
+            await config.createRow({
+              tableId: config.table!._id,
+              ...user,
+            })
+          }
+
+          const createViewResponse = await config.api.viewV2.create({
+            sort: sortParams,
+          })
+
+          const response = await config.api.viewV2.search(createViewResponse.id)
+
+          expect(response.body.rows).toHaveLength(4)
+          expect(response.body).toEqual({
+            rows: expected.map(name => expect.objectContaining({ name })),
+          })
+        }
+      )
+
+      it.each(sortTestOptions)(
+        "allow override the default view sorting (%s)",
+        async (sortParams, expected) => {
+          await config.createTable(userTable())
+          const users = [
+            { name: "Alice", age: 25 },
+            { name: "Bob", age: 30 },
+            { name: "Charly", age: 27 },
+            { name: "Danny", age: 15 },
+          ]
+          for (const user of users) {
+            await config.createRow({
+              tableId: config.table!._id,
+              ...user,
+            })
+          }
+
+          const createViewResponse = await config.api.viewV2.create({
+            sort: {
+              field: "name",
+              order: SortOrder.ASCENDING,
+              type: SortType.STRING,
+            },
+          })
+
+          const response = await config.api.viewV2.search(
+            createViewResponse.id,
+            {
+              sort: sortParams.field,
+              sortOrder: sortParams.order,
+              sortType: sortParams.type,
+            }
+          )
+
+          expect(response.body.rows).toHaveLength(4)
+          expect(response.body).toEqual({
+            rows: expected.map(name => expect.objectContaining({ name })),
+          })
+        }
+      )
+
+      it("when schema is defined, defined columns and row attributes are returned", async () => {
+        const table = await config.createTable(userTable())
+        const rows = []
+        for (let i = 0; i < 10; i++) {
+          rows.push(
+            await config.createRow({
+              tableId: table._id,
+              name: generator.name(),
+              age: generator.age(),
+            })
+          )
+        }
+
+        const view = await config.api.viewV2.create({
+          schema: { name: {} },
+        })
+        const response = await config.api.viewV2.search(view.id)
+
+        expect(response.body.rows).toHaveLength(10)
+        expect(response.body.rows).toEqual(
+          expect.arrayContaining(
+            rows.map(r => ({
+              ...expectAnyInternalColsAttributes,
+              _viewId: view.id,
+              name: r.name,
+            }))
+          )
+        )
+      })
+
+      it("views without data can be returned", async () => {
+        const table = await config.createTable(userTable())
+
+        const createViewResponse = await config.api.viewV2.create()
+        const response = await config.api.viewV2.search(createViewResponse.id)
+
+        expect(response.body.rows).toHaveLength(0)
+      })
+
+      it("respects the limit parameter", async () => {
+        const table = await config.createTable(userTable())
+        const rows = []
+        for (let i = 0; i < 10; i++) {
+          rows.push(await config.createRow({ tableId: table._id }))
+        }
+        const limit = generator.integer({ min: 1, max: 8 })
+
+        const createViewResponse = await config.api.viewV2.create()
+        const response = await config.api.viewV2.search(createViewResponse.id, {
+          limit,
+        })
+
+        expect(response.body.rows).toHaveLength(limit)
+      })
+
+      it("can handle pagination", async () => {
+        const table = await config.createTable(userTable())
+        const rows = []
+        for (let i = 0; i < 10; i++) {
+          rows.push(await config.createRow({ tableId: table._id }))
+        }
+        // rows.sort((a, b) => (a._id! > b._id! ? 1 : -1))
+
+        const createViewResponse = await config.api.viewV2.create()
+        const allRows = (await config.api.viewV2.search(createViewResponse.id))
+          .body.rows
+
+        const firstPageResponse = await config.api.viewV2.search(
+          createViewResponse.id,
+          {
+            paginate: true,
+            limit: 4,
+          }
+        )
+        expect(firstPageResponse.body).toEqual({
+          rows: expect.arrayContaining(allRows.slice(0, 4)),
+          totalRows: 10,
+          hasNextPage: true,
+          bookmark: expect.any(String),
+        })
+
+        const secondPageResponse = await config.api.viewV2.search(
+          createViewResponse.id,
+          {
+            paginate: true,
+            limit: 4,
+            bookmark: firstPageResponse.body.bookmark,
+          }
+        )
+        expect(secondPageResponse.body).toEqual({
+          rows: expect.arrayContaining(allRows.slice(4, 8)),
+          totalRows: 10,
+          hasNextPage: true,
+          bookmark: expect.any(String),
+        })
+
+        const lastPageResponse = await config.api.viewV2.search(
+          createViewResponse.id,
+          {
+            paginate: true,
+            limit: 4,
+            bookmark: secondPageResponse.body.bookmark,
+          }
+        )
+        expect(lastPageResponse.body).toEqual({
+          rows: expect.arrayContaining(allRows.slice(8)),
+          totalRows: 10,
+          hasNextPage: false,
+          bookmark: expect.any(String),
+        })
+      })
+    })
   })
 })

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -62,7 +62,7 @@ describe("/v2/views", () => {
         name: generator.name(),
         tableId: config.table!._id!,
         primaryDisplay: generator.word(),
-        query: { allOr: false, equal: { field: "value" } },
+        query: [{ operator: "equal", field: "field", value: "value" }],
         sort: {
           field: "fieldToSort",
           order: SortOrder.DESCENDING,
@@ -190,7 +190,7 @@ describe("/v2/views", () => {
       const tableId = config.table!._id!
       await config.api.viewV2.update({
         ...view,
-        query: { equal: { newField: "thatValue" } },
+        query: [{ operator: "equal", field: "newField", value: "thatValue" }],
       })
 
       expect(await config.api.table.get(tableId)).toEqual({
@@ -198,7 +198,9 @@ describe("/v2/views", () => {
         views: {
           [view.name]: {
             ...view,
-            query: { equal: { newField: "thatValue" } },
+            query: [
+              { operator: "equal", field: "newField", value: "thatValue" },
+            ],
             schema: expect.anything(),
           },
         },
@@ -216,7 +218,13 @@ describe("/v2/views", () => {
         tableId,
         name: view.name,
         primaryDisplay: generator.word(),
-        query: { equal: { [generator.word()]: generator.word() } },
+        query: [
+          {
+            operator: "equal",
+            field: generator.word(),
+            value: generator.word(),
+          },
+        ],
         sort: {
           field: generator.word(),
           order: SortOrder.DESCENDING,
@@ -285,7 +293,7 @@ describe("/v2/views", () => {
         {
           ...view,
           tableId: generator.guid(),
-          query: { equal: { newField: "thatValue" } },
+          query: [{ operator: "equal", field: "newField", value: "thatValue" }],
         },
         { expectStatus: 404 }
       )

packages/server/src/api/routes/view.ts

@@ -34,7 +34,7 @@ router
     "/api/views/:viewName",
     paramResource("viewName"),
     authorized(
-      permissions.PermissionType.VIEW,
+      permissions.PermissionType.TABLE,
       permissions.PermissionLevel.READ
     ),
     rowController.fetchView

packages/server/src/app.ts

@@ -1,120 +1,43 @@
+import Sentry from "@sentry/node"
+
 if (process.env.DD_APM_ENABLED) {
   require("./ddApm")
 }
 
 // need to load environment first
 import env from "./environment"
-
-import { ExtendableContext } from "koa"
 import * as db from "./db"
 db.init()
-import Koa from "koa"
-import koaBody from "koa-body"
-import http from "http"
-import * as api from "./api"
-import * as automations from "./automations"
-import { Thread } from "./threads"
-import * as redis from "./utilities/redis"
 import { ServiceType } from "@budibase/types"
-import {
+import { env as coreEnv } from "@budibase/backend-core"
-  events,
-  logging,
-  middleware,
-  timers,
-  env as coreEnv,
-} from "@budibase/backend-core"
 coreEnv._set("SERVICE_TYPE", ServiceType.APPS)
+import { apiEnabled } from "./features"
+import createKoaApp from "./koa"
+import Koa from "koa"
+import { Server } from "http"
 import { startup } from "./startup"
-const Sentry = require("@sentry/node")
-const destroyable = require("server-destroy")
-const { userAgent } = require("koa-useragent")
 
-const app = new Koa()
+let app: Koa, server: Server
 
-let mbNumber = parseInt(env.HTTP_MB_LIMIT || "10")
+async function start() {
-if (!mbNumber || isNaN(mbNumber)) {
+  // if API disabled, could run automations instead
-  mbNumber = 10
+  if (apiEnabled()) {
-}
+    const koa = createKoaApp()
-// set up top level koa middleware
+    app = koa.app
-app.use(
+    server = koa.server
-  koaBody({
-    multipart: true,
-    formLimit: `${mbNumber}mb`,
-    jsonLimit: `${mbNumber}mb`,
-    textLimit: `${mbNumber}mb`,
-    // @ts-ignore
-    enableTypes: ["json", "form", "text"],
-    parsedMethods: ["POST", "PUT", "PATCH", "DELETE"],
-  })
-)
-
-app.use(middleware.correlation)
-app.use(middleware.pino)
-app.use(userAgent)
-
-if (env.isProd()) {
-  env._set("NODE_ENV", "production")
-  Sentry.init()
-
-  app.on("error", (err: any, ctx: ExtendableContext) => {
-    Sentry.withScope(function (scope: any) {
-      scope.addEventProcessor(function (event: any) {
-        return Sentry.Handlers.parseRequest(event, ctx.request)
-      })
-      Sentry.captureException(err)
-    })
-  })
-}
-
-const server = http.createServer(app.callback())
-destroyable(server)
-
-let shuttingDown = false,
-  errCode = 0
-
-server.on("close", async () => {
-  // already in process
-  if (shuttingDown) {
-    return
   }
-  shuttingDown = true
+  // startup includes automation runner - if enabled
-  console.log("Server Closed")
-  timers.cleanup()
-  await automations.shutdown()
-  await redis.shutdown()
-  events.shutdown()
-  await Thread.shutdown()
-  api.shutdown()
-  if (!env.isTest()) {
-    process.exit(errCode)
-  }
-})
-
-export default server.listen(env.PORT || 0, async () => {
   await startup(app, server)
-})
+  if (env.isProd()) {
-
+    env._set("NODE_ENV", "production")
-const shutdown = () => {
+    Sentry.init()
-  server.close()
+  }
-  // @ts-ignore
-  server.destroy()
 }
 
-process.on("uncaughtException", err => {
+start().catch(err => {
-  // @ts-ignore
+  console.error(`Failed server startup - ${err.message}`)
-  // don't worry about this error, comes from zlib isn't important
-  if (err && err["code"] === "ERR_INVALID_CHAR") {
-    return
-  }
-  errCode = -1
-  logging.logAlert("Uncaught exception.", err)
-  shutdown()
 })
 
-process.on("SIGTERM", () => {
+export function getServer() {
-  shutdown()
+  return server
-})
+}
-
-process.on("SIGINT", () => {
-  shutdown()
-})

packages/server/src/automations/index.ts

@@ -2,6 +2,7 @@ import { processEvent } from "./utils"
 import { automationQueue } from "./bullboard"
 import { rebootTrigger } from "./triggers"
 import BullQueue from "bull"
+import { automationsEnabled } from "../features"
 
 export { automationQueue } from "./bullboard"
 export { shutdown } from "./bullboard"
@@ -12,6 +13,9 @@ export { BUILTIN_ACTION_DEFINITIONS, getActionDefinitions } from "./actions"
  * This module is built purely to kick off the worker farm and manage the inputs/outputs
  */
 export async function init() {
+  if (!automationsEnabled()) {
+    return
+  }
   // this promise will not complete
   const promise = automationQueue.process(async job => {
     await processEvent(job)

packages/server/src/automations/tests/filter.spec.ts

@@ -1,11 +1,18 @@
-const setup = require("./utilities")
+import * as setup from "./utilities"
-const { FilterConditions } = require("../steps/filter")
+import { FilterConditions } from "../steps/filter"
 
 describe("test the filter logic", () => {
-  async function checkFilter(field, condition, value, pass = true) {
+  async function checkFilter(
-    let res = await setup.runStep(setup.actions.FILTER.stepId,
+    field: any,
-      { field, condition, value }
+    condition: string,
-    )
+    value: any,
+    pass = true
+  ) {
+    let res = await setup.runStep(setup.actions.FILTER.stepId, {
+      field,
+      condition,
+      value,
+    })
     expect(res.result).toEqual(pass)
     expect(res.success).toEqual(true)
   }
@@ -36,9 +43,9 @@ describe("test the filter logic", () => {
 
   it("check date coercion", async () => {
     await checkFilter(
-      (new Date()).toISOString(),
+      new Date().toISOString(),
       FilterConditions.GREATER_THAN,
-      (new Date(-10000)).toISOString(),
+      new Date(-10000).toISOString(),
       true
     )
   })

packages/server/src/automations/utils.ts

@@ -15,9 +15,13 @@ import {
   WebhookActionType,
 } from "@budibase/types"
 import sdk from "../sdk"
+import { automationsEnabled } from "../features"
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
-const Runner = new Thread(ThreadType.AUTOMATION)
+let Runner: Thread
+if (automationsEnabled()) {
+  Runner = new Thread(ThreadType.AUTOMATION)
+}
 
 function loggingArgs(
   job: AutomationJob,
@@ -130,7 +134,8 @@ export async function disableAllCrons(appId: any) {
       }
     }
   }
-  return Promise.all(promises)
+  const results = await Promise.all(promises)
+  return { count: results.length / 2 }
 }
 
 export async function disableCronById(jobId: number | string) {
@@ -169,6 +174,7 @@ export async function enableCronTrigger(appId: any, automation: Automation) {
   const needsCreated =
     !sdk.automations.isReboot(automation) &&
     !sdk.automations.disabled(automation)
+  let enabled = false
 
   // need to create cron job
   if (validCron && needsCreated) {
@@ -191,8 +197,9 @@ export async function enableCronTrigger(appId: any, automation: Automation) {
       automation._id = response.id
       automation._rev = response.rev
     })
+    enabled = true
   }
-  return automation
+  return { enabled, automation }
 }
 
 /**

packages/server/src/db/utils.ts

@@ -1,5 +1,7 @@
 import newid from "./newid"
 import { db as dbCore } from "@budibase/backend-core"
+import { DocumentType, VirtualDocumentType } from "@budibase/types"
+export { DocumentType, VirtualDocumentType } from "@budibase/types"
 
 type Optional = string | null
 
@@ -19,7 +21,6 @@ export const BudibaseInternalDB = {
 
 export const SEPARATOR = dbCore.SEPARATOR
 export const StaticDatabases = dbCore.StaticDatabases
-export const DocumentType = dbCore.DocumentType
 export const APP_PREFIX = dbCore.APP_PREFIX
 export const APP_DEV_PREFIX = dbCore.APP_DEV_PREFIX
 export const isDevAppID = dbCore.isDevAppID
@@ -284,10 +285,22 @@ export function getMultiIDParams(ids: string[]) {
  * @returns {string} The new view ID which the view doc can be stored under.
  */
 export function generateViewID(tableId: string) {
-  return `${tableId}${SEPARATOR}${newid()}`
+  return `${
+    VirtualDocumentType.VIEW
+  }${SEPARATOR}${tableId}${SEPARATOR}${newid()}`
+}
+
+export function isViewID(viewId: string) {
+  return viewId?.split(SEPARATOR)[0] === VirtualDocumentType.VIEW
 }
 
 export function extractViewInfoFromID(viewId: string) {
+  if (!isViewID(viewId)) {
+    throw new Error("Unable to extract table ID, is not a view ID")
+  }
+  const split = viewId.split(SEPARATOR)
+  split.shift()
+  viewId = split.join(SEPARATOR)
   const regex = new RegExp(`^(?<tableId>.+)${SEPARATOR}([^${SEPARATOR}]+)$`)
   const res = regex.exec(viewId)
   return {

packages/server/src/definitions/openapi.ts

@@ -34,6 +34,14 @@ export interface paths {
     /** Based on query properties (currently only name) search for queries. */
     post: operations["querySearch"];
   };
+  "/roles/assign": {
+    /** This is a business/enterprise only endpoint */
+    post: operations["roleAssign"];
+  };
+  "/roles/unassign": {
+    /** This is a business/enterprise only endpoint */
+    post: operations["roleUnAssign"];
+  };
   "/tables/{tableId}/rows": {
     /** Creates a row within the specified table. */
     post: operations["rowCreate"];
@@ -256,7 +264,8 @@ export interface components {
                 | "auto"
                 | "json"
                 | "internal"
-                | "barcodeqr";
+                | "barcodeqr"
+                | "bigint";
               /** @description A constraint can be applied to the column which will be validated against when a row is saved. */
               constraints?: {
                 /** @enum {string} */
@@ -362,7 +371,8 @@ export interface components {
                   | "auto"
                   | "json"
                   | "internal"
-                  | "barcodeqr";
+                  | "barcodeqr"
+                  | "bigint";
                 /** @description A constraint can be applied to the column which will be validated against when a row is saved. */
                 constraints?: {
                   /** @enum {string} */
@@ -470,7 +480,8 @@ export interface components {
                   | "auto"
                   | "json"
                   | "internal"
-                  | "barcodeqr";
+                  | "barcodeqr"
+                  | "bigint";
                 /** @description A constraint can be applied to the column which will be validated against when a row is saved. */
                 constraints?: {
                   /** @enum {string} */
@@ -577,17 +588,17 @@ export interface components {
       lastName?: string;
       /** @description If set to true forces the user to reset their password on first login. */
       forceResetPassword?: boolean;
-      /** @description Describes if the user is a builder user or not. */
+      /** @description Describes if the user is a builder user or not. This field can only be set on a business or enterprise license. */
       builder?: {
         /** @description If set to true the user will be able to build any app in the system. */
         global?: boolean;
       };
-      /** @description Describes if the user is an admin user or not. */
+      /** @description Describes if the user is an admin user or not. This field can only be set on a business or enterprise license. */
       admin?: {
         /** @description If set to true the user will be able to administrate the system. */
         global?: boolean;
       };
-      /** @description Contains the roles of the user per app (assuming they are not a builder user). */
+      /** @description Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license. */
       roles: { [key: string]: string };
     };
     userOutput: {
@@ -607,17 +618,17 @@ export interface components {
         lastName?: string;
         /** @description If set to true forces the user to reset their password on first login. */
         forceResetPassword?: boolean;
-        /** @description Describes if the user is a builder user or not. */
+        /** @description Describes if the user is a builder user or not. This field can only be set on a business or enterprise license. */
         builder?: {
           /** @description If set to true the user will be able to build any app in the system. */
           global?: boolean;
         };
-        /** @description Describes if the user is an admin user or not. */
+        /** @description Describes if the user is an admin user or not. This field can only be set on a business or enterprise license. */
         admin?: {
           /** @description If set to true the user will be able to administrate the system. */
           global?: boolean;
         };
-        /** @description Contains the roles of the user per app (assuming they are not a builder user). */
+        /** @description Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license. */
         roles: { [key: string]: string };
         /** @description The ID of the user. */
         _id: string;
@@ -640,17 +651,17 @@ export interface components {
         lastName?: string;
         /** @description If set to true forces the user to reset their password on first login. */
         forceResetPassword?: boolean;
-        /** @description Describes if the user is a builder user or not. */
+        /** @description Describes if the user is a builder user or not. This field can only be set on a business or enterprise license. */
         builder?: {
           /** @description If set to true the user will be able to build any app in the system. */
           global?: boolean;
         };
-        /** @description Describes if the user is an admin user or not. */
+        /** @description Describes if the user is an admin user or not. This field can only be set on a business or enterprise license. */
         admin?: {
           /** @description If set to true the user will be able to administrate the system. */
           global?: boolean;
         };
-        /** @description Contains the roles of the user per app (assuming they are not a builder user). */
+        /** @description Contains the roles of the user per app (assuming they are not a builder user). This field can only be set on a business or enterprise license. */
         roles: { [key: string]: string };
         /** @description The ID of the user. */
         _id: string;
@@ -712,6 +723,52 @@ export interface components {
       /** @description The name to be used when searching - this will be used in a case insensitive starts with match. */
       name: string;
     };
+    rolesAssign: {
+      /** @description Allow setting users to builders per app. */
+      appBuilder?: {
+        /** @description The app that the users should have app builder privileges granted for. */
+        appId: string;
+      };
+      /** @description Add/remove global builder permissions from the list of users. */
+      builder?: boolean;
+      /** @description Add/remove global admin permissions from the list of users. */
+      admin?: boolean;
+      /** @description Add/remove a per-app role, such as BASIC, ADMIN etc. */
+      role?: {
+        /** @description The role ID, such as BASIC, ADMIN or a custom role ID. */
+        roleId: string;
+        /** @description The app that the role relates to. */
+        appId: string;
+      };
+      /** @description The user IDs to be updated to add/remove the specified roles. */
+      userIds: string[];
+    };
+    rolesUnAssign: {
+      /** @description Allow setting users to builders per app. */
+      appBuilder?: {
+        /** @description The app that the users should have app builder privileges granted for. */
+        appId: string;
+      };
+      /** @description Add/remove global builder permissions from the list of users. */
+      builder?: boolean;
+      /** @description Add/remove global admin permissions from the list of users. */
+      admin?: boolean;
+      /** @description Add/remove a per-app role, such as BASIC, ADMIN etc. */
+      role?: {
+        /** @description The role ID, such as BASIC, ADMIN or a custom role ID. */
+        roleId: string;
+        /** @description The app that the role relates to. */
+        appId: string;
+      };
+      /** @description The user IDs to be updated to add/remove the specified roles. */
+      userIds: string[];
+    };
+    rolesOutput: {
+      data: {
+        /** @description The updated users' IDs */
+        userIds: string[];
+      };
+    };
   };
   parameters: {
     /** @description The ID of the table which this request is targeting. */
@@ -907,6 +964,38 @@ export interface operations {
       };
     };
   };
+  /** This is a business/enterprise only endpoint */
+  roleAssign: {
+    responses: {
+      /** Returns a list of updated user IDs */
+      200: {
+        content: {
+          "application/json": components["schemas"]["rolesOutput"];
+        };
+      };
+    };
+    requestBody: {
+      content: {
+        "application/json": components["schemas"]["rolesAssign"];
+      };
+    };
+  };
+  /** This is a business/enterprise only endpoint */
+  roleUnAssign: {
+    responses: {
+      /** Returns a list of updated user IDs */
+      200: {
+        content: {
+          "application/json": components["schemas"]["rolesOutput"];
+        };
+      };
+    };
+    requestBody: {
+      content: {
+        "application/json": components["schemas"]["rolesUnAssign"];
+      };
+    };
+  };
   /** Creates a row within the specified table. */
   rowCreate: {
     parameters: {

packages/server/src/environment.ts

@@ -38,6 +38,8 @@ function parseIntSafe(number?: string) {
 }
 
 const environment = {
+  // features
+  APP_FEATURES: process.env.APP_FEATURES,
   // important - prefer app port to generic port
   PORT: process.env.APP_PORT || process.env.PORT,
   COUCH_DB_URL: process.env.COUCH_DB_URL,

packages/server/src/features.ts

@@ -0,0 +1,24 @@
+import { features } from "@budibase/backend-core"
+import env from "./environment"
+
+enum AppFeature {
+  API = "api",
+  AUTOMATIONS = "automations",
+}
+
+const featureList = features.processFeatureEnvVar<AppFeature>(
+  Object.values(AppFeature),
+  env.APP_FEATURES
+)
+
+export function isFeatureEnabled(feature: AppFeature) {
+  return featureList.includes(feature)
+}
+
+export function automationsEnabled() {
+  return featureList.includes(AppFeature.AUTOMATIONS)
+}
+
+export function apiEnabled() {
+  return featureList.includes(AppFeature.API)
+}

packages/server/src/integrations/mysql.ts

@@ -93,6 +93,21 @@ const SCHEMA: Integration = {
   },
 }
 
+const defaultTypeCasting = function (field: any, next: any) {
+  if (
+    field.type == "DATETIME" ||
+    field.type === "DATE" ||
+    field.type === "TIMESTAMP" ||
+    field.type === "LONGLONG"
+  ) {
+    return field.string()
+  }
+  if (field.type === "BIT" && field.length === 1) {
+    return field.buffer()?.[0]
+  }
+  return next()
+}
+
 export function bindingTypeCoerce(bindings: any[]) {
   for (let i = 0; i < bindings.length; i++) {
     const binding = bindings[i]
@@ -147,21 +162,8 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     delete config.rejectUnauthorized
     this.config = {
       ...config,
+      typeCast: defaultTypeCasting,
       multipleStatements: true,
-      typeCast: function (field: any, next: any) {
-        if (
-          field.type == "DATETIME" ||
-          field.type === "DATE" ||
-          field.type === "TIMESTAMP" ||
-          field.type === "LONGLONG"
-        ) {
-          return field.string()
-        }
-        if (field.type === "BIT" && field.length === 1) {
-          return field.buffer()?.[0]
-        }
-        return next()
-      },
     }
   }
 
@@ -194,6 +196,37 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     return `concat(${parts.join(", ")})`
   }
 
+  defineTypeCastingFromSchema(schema: {
+    [key: string]: { name: string; type: string }
+  }): void {
+    if (!schema) {
+      return
+    }
+    this.config.typeCast = function (field: any, next: any) {
+      if (schema[field.name]?.name === field.name) {
+        if (["LONGLONG", "NEWDECIMAL", "DECIMAL"].includes(field.type)) {
+          if (schema[field.name]?.type === "number") {
+            const value = field.string()
+            return value ? Number(value) : null
+          } else {
+            return field.string()
+          }
+        }
+      }
+      if (
+        field.type == "DATETIME" ||
+        field.type === "DATE" ||
+        field.type === "TIMESTAMP"
+      ) {
+        return field.string()
+      }
+      if (field.type === "BIT" && field.length === 1) {
+        return field.buffer()?.[0]
+      }
+      return next()
+    }
+  }
+
   async connect() {
     this.client = await mysql.createConnection(this.config)
   }
@@ -204,7 +237,10 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
 
   async internalQuery(
     query: SqlQuery,
-    opts: { connect?: boolean; disableCoercion?: boolean } = {
+    opts: {
+      connect?: boolean
+      disableCoercion?: boolean
+    } = {
       connect: true,
       disableCoercion: false,
     }

packages/server/src/koa.ts

@@ -0,0 +1,102 @@
+import env from "./environment"
+import { ExtendableContext } from "koa"
+import Koa from "koa"
+import koaBody from "koa-body"
+import http from "http"
+import * as api from "./api"
+import * as automations from "./automations"
+import { Thread } from "./threads"
+import * as redis from "./utilities/redis"
+import { events, logging, middleware, timers } from "@budibase/backend-core"
+const Sentry = require("@sentry/node")
+const destroyable = require("server-destroy")
+const { userAgent } = require("koa-useragent")
+
+export default function createKoaApp() {
+  const app = new Koa()
+
+  let mbNumber = parseInt(env.HTTP_MB_LIMIT || "10")
+  if (!mbNumber || isNaN(mbNumber)) {
+    mbNumber = 10
+  }
+  // set up top level koa middleware
+  app.use(
+    koaBody({
+      multipart: true,
+      formLimit: `${mbNumber}mb`,
+      jsonLimit: `${mbNumber}mb`,
+      textLimit: `${mbNumber}mb`,
+      // @ts-ignore
+      enableTypes: ["json", "form", "text"],
+      parsedMethods: ["POST", "PUT", "PATCH", "DELETE"],
+    })
+  )
+
+  app.use(middleware.correlation)
+  app.use(middleware.pino)
+  app.use(userAgent)
+
+  if (env.isProd()) {
+    app.on("error", (err: any, ctx: ExtendableContext) => {
+      Sentry.withScope(function (scope: any) {
+        scope.addEventProcessor(function (event: any) {
+          return Sentry.Handlers.parseRequest(event, ctx.request)
+        })
+        Sentry.captureException(err)
+      })
+    })
+  }
+
+  const server = http.createServer(app.callback())
+  destroyable(server)
+
+  let shuttingDown = false,
+    errCode = 0
+
+  server.on("close", async () => {
+    // already in process
+    if (shuttingDown) {
+      return
+    }
+    shuttingDown = true
+    console.log("Server Closed")
+    timers.cleanup()
+    await automations.shutdown()
+    await redis.shutdown()
+    events.shutdown()
+    await Thread.shutdown()
+    api.shutdown()
+    if (!env.isTest()) {
+      process.exit(errCode)
+    }
+  })
+
+  const listener = server.listen(env.PORT || 0)
+
+  const shutdown = () => {
+    server.close()
+    // @ts-ignore
+    server.destroy()
+  }
+
+  process.on("uncaughtException", err => {
+    // @ts-ignore
+    // don't worry about this error, comes from zlib isn't important
+    if (err && err["code"] === "ERR_INVALID_CHAR") {
+      return
+    }
+    errCode = -1
+    logging.logAlert("Uncaught exception.", err)
+    shutdown()
+  })
+
+  process.on("SIGTERM", () => {
+    shutdown()
+  })
+
+  process.on("SIGINT", () => {
+    shutdown()
+  })
+
+  return { app, server: listener }
+}

packages/server/src/middleware/noViewData.ts

@@ -1,9 +0,0 @@
-import { Ctx, Row } from "@budibase/types"
-
-export default async (ctx: Ctx<Row>, next: any) => {
-  if (ctx.request.body._viewId) {
-    return ctx.throw(400, "Table row endpoints cannot contain view info")
-  }
-
-  return next()
-}

packages/server/src/middleware/tests/noViewData.spec.ts

@@ -1,83 +0,0 @@
-import { generator } from "@budibase/backend-core/tests"
-import { BBRequest, FieldType, Row, Table } from "@budibase/types"
-import { Next } from "koa"
-import * as utils from "../../db/utils"
-import noViewDataMiddleware from "../noViewData"
-
-class TestConfiguration {
-  next: Next
-  throw: jest.Mock<(status: number, message: string) => never>
-  middleware: typeof noViewDataMiddleware
-  params: Record<string, any>
-  request?: Pick<BBRequest<Row>, "body">
-
-  constructor() {
-    this.next = jest.fn()
-    this.throw = jest.fn()
-    this.params = {}
-
-    this.middleware = noViewDataMiddleware
-  }
-
-  executeMiddleware(ctxRequestBody: Row) {
-    this.request = {
-      body: ctxRequestBody,
-    }
-    return this.middleware(
-      {
-        request: this.request as any,
-        throw: this.throw as any,
-        params: this.params,
-      } as any,
-      this.next
-    )
-  }
-
-  afterEach() {
-    jest.clearAllMocks()
-  }
-}
-
-describe("noViewData middleware", () => {
-  let config: TestConfiguration
-
-  beforeEach(() => {
-    config = new TestConfiguration()
-  })
-
-  afterEach(() => {
-    config.afterEach()
-  })
-
-  const getRandomData = () => ({
-    _id: generator.guid(),
-    name: generator.name(),
-    age: generator.age(),
-    address: generator.address(),
-  })
-
-  it("it should pass without view id data", async () => {
-    const data = getRandomData()
-    await config.executeMiddleware({
-      ...data,
-    })
-
-    expect(config.next).toBeCalledTimes(1)
-    expect(config.throw).not.toBeCalled()
-  })
-
-  it("it should throw an error if _viewid is provided", async () => {
-    const data = getRandomData()
-    await config.executeMiddleware({
-      _viewId: generator.guid(),
-      ...data,
-    })
-
-    expect(config.throw).toBeCalledTimes(1)
-    expect(config.throw).toBeCalledWith(
-      400,
-      "Table row endpoints cannot contain view info"
-    )
-    expect(config.next).not.toBeCalled()
-  })
-})

packages/server/src/middleware/tests/trimViewRowInfo.spec.ts

@@ -117,7 +117,7 @@ describe("trimViewRowInfo middleware", () => {
     })
 
     expect(config.request?.body).toEqual(data)
-    expect(config.params.tableId).toEqual(table._id)
+    expect(config.params.sourceId).toEqual(table._id)
 
     expect(config.next).toBeCalledTimes(1)
     expect(config.throw).not.toBeCalled()
@@ -143,32 +143,9 @@ describe("trimViewRowInfo middleware", () => {
       name: data.name,
       address: data.address,
     })
-    expect(config.params.tableId).toEqual(table._id)
+    expect(config.params.sourceId).toEqual(table._id)
 
     expect(config.next).toBeCalledTimes(1)
     expect(config.throw).not.toBeCalled()
   })
-
-  it("it should throw an error if no viewid is provided on the body", async () => {
-    const data = getRandomData()
-    await config.executeMiddleware(viewId, {
-      ...data,
-    })
-
-    expect(config.throw).toBeCalledTimes(1)
-    expect(config.throw).toBeCalledWith(400, "_viewId is required")
-    expect(config.next).not.toBeCalled()
-  })
-
-  it("it should throw an error if no viewid is provided on the parameters", async () => {
-    const data = getRandomData()
-    await config.executeMiddleware(undefined as any, {
-      _viewId: viewId,
-      ...data,
-    })
-
-    expect(config.throw).toBeCalledTimes(1)
-    expect(config.throw).toBeCalledWith(400, "viewId path is required")
-    expect(config.next).not.toBeCalled()
-  })
 })

packages/server/src/middleware/trimViewRowInfo.ts

@@ -3,26 +3,35 @@ import * as utils from "../db/utils"
 import sdk from "../sdk"
 import { db } from "@budibase/backend-core"
 import { Next } from "koa"
+import { getTableId } from "../api/controllers/row/utils"
 
 export default async (ctx: Ctx<Row>, next: Next) => {
   const { body } = ctx.request
-  const { _viewId: viewId } = body
+  let { _viewId: viewId } = body
+
+  const possibleViewId = getTableId(ctx)
+  if (utils.isViewID(possibleViewId)) {
+    viewId = possibleViewId
+  }
+
+  // nothing to do, it is not a view (just a table ID)
   if (!viewId) {
-    return ctx.throw(400, "_viewId is required")
+    return next()
   }
 
-  if (!ctx.params.viewId) {
+  const { tableId } = utils.extractViewInfoFromID(viewId)
-    return ctx.throw(400, "viewId path is required")
+
+  // don't need to trim delete requests
+  if (ctx?.method?.toLowerCase() !== "delete") {
+    const { _viewId, ...trimmedView } = await trimViewFields(
+      viewId,
+      tableId,
+      body
+    )
+    ctx.request.body = trimmedView
   }
 
-  const { tableId } = utils.extractViewInfoFromID(ctx.params.viewId)
+  ctx.params.sourceId = tableId
-  const { _viewId, ...trimmedView } = await trimViewFields(
-    viewId,
-    tableId,
-    body
-  )
-  ctx.request.body = trimmedView
-  ctx.params.tableId = tableId
 
   return next()
 }

packages/server/src/sdk/app/rows/search.ts

@@ -1,23 +1,9 @@
-import { SearchFilters, SortOrder, SortType } from "@budibase/types"
+import { SearchFilters, SearchParams } from "@budibase/types"
 import { isExternalTable } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"
 import { Format } from "../../../api/controllers/view/exporters"
 
-export interface SearchParams {
-  tableId: string
-  paginate?: boolean
-  query: SearchFilters
-  bookmark?: string
-  limit?: number
-  sort?: string
-  sortOrder?: SortOrder
-  sortType?: SortType
-  version?: string
-  disableEscaping?: boolean
-  fields?: string[]
-}
-
 export interface ViewParams {
   calculation: string
   group: string

packages/server/src/sdk/app/rows/search/external.ts

@@ -6,6 +6,7 @@ import {
   IncludeRelationship,
   Row,
   SearchFilters,
+  SearchParams,
 } from "@budibase/types"
 import * as exporters from "../../../../api/controllers/view/exporters"
 import sdk from "../../../../sdk"
@@ -13,7 +14,7 @@ import { handleRequest } from "../../../../api/controllers/row/external"
 import { breakExternalTableId } from "../../../../integrations/utils"
 import { cleanExportRows } from "../utils"
 import { utils } from "@budibase/shared-core"
-import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
+import { ExportRowsParams, ExportRowsResult } from "../search"
 import { HTTPError, db } from "@budibase/backend-core"
 import pick from "lodash/pick"
 

packages/server/src/sdk/app/rows/search/internal.ts

@@ -12,7 +12,7 @@ import {
 } from "../../../../db/utils"
 import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
-import { Database, Row, Table } from "@budibase/types"
+import { Database, Row, Table, SearchParams } from "@budibase/types"
 import { cleanExportRows } from "../utils"
 import {
   Format,
@@ -28,7 +28,7 @@ import {
   getFromMemoryDoc,
 } from "../../../../api/controllers/view/utils"
 import sdk from "../../../../sdk"
-import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
+import { ExportRowsParams, ExportRowsResult } from "../search"
 import pick from "lodash/pick"
 
 export async function search(options: SearchParams) {

packages/server/src/sdk/app/rows/search/tests/external.spec.ts

@@ -1,14 +1,15 @@
 import { GenericContainer } from "testcontainers"
+
 import {
   Datasource,
-  EmptyFilterOption,
   FieldType,
   Row,
   SourceName,
   Table,
+  SearchParams,
 } from "@budibase/types"
+
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
-import { SearchParams } from "../../search"
 import { search } from "../external"
 import {
   expectAnyExternalColsAttributes,
@@ -122,22 +123,6 @@ describe.skip("external", () => {
       })
     })
 
-    it("empty filters search returns no data", async () => {
-      await config.doInContext(config.appId, async () => {
-        const tableId = config.table!._id!
-
-        const searchParams: SearchParams = {
-          tableId,
-          query: {
-            onEmptyFilter: EmptyFilterOption.RETURN_NONE,
-          },
-        }
-        const result = await search(searchParams)
-
-        expect(result.rows).toHaveLength(0)
-      })
-    })
-
     it("querying by fields will always return data attribute columns", async () => {
       await config.doInContext(config.appId, async () => {
         const tableId = config.table!._id!

packages/server/src/sdk/app/rows/search/tests/internal.spec.ts

@@ -1,6 +1,5 @@
-import { FieldType, Row, Table } from "@budibase/types"
+import { FieldType, Row, Table, SearchParams } from "@budibase/types"
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
-import { SearchParams } from "../../search"
 import { search } from "../internal"
 import {
   expectAnyInternalColsAttributes,

packages/server/src/sdk/app/views/index.ts

@@ -1,17 +1,20 @@
-import { HTTPError, context } from "@budibase/backend-core"
+import {
-import { FieldSchema, TableSchema, View, ViewV2 } from "@budibase/types"
+  FieldSchema,
+  RenameColumn,
+  TableSchema,
+  View,
+  ViewV2,
+} from "@budibase/types"
+import { context, HTTPError } from "@budibase/backend-core"
 
 import sdk from "../../../sdk"
 import * as utils from "../../../db/utils"
-import merge from "lodash/merge"
 
 export async function get(viewId: string): Promise<ViewV2 | undefined> {
   const { tableId } = utils.extractViewInfoFromID(viewId)
   const table = await sdk.tables.getTable(tableId)
   const views = Object.values(table.views!)
-  const view = views.find(v => isV2(v) && v.id === viewId) as ViewV2 | undefined
+  return views.find(v => isV2(v) && v.id === viewId) as ViewV2 | undefined
-
-  return view
 }
 
 export async function create(
@@ -106,3 +109,37 @@ export function enrichSchema(view: View | ViewV2, tableSchema: TableSchema) {
     schema: schema,
   }
 }
+
+export function syncSchema(
+  view: ViewV2,
+  schema: TableSchema,
+  renameColumn: RenameColumn | undefined
+): ViewV2 {
+  if (renameColumn) {
+    if (view.columns) {
+      view.columns[view.columns.indexOf(renameColumn.old)] =
+        renameColumn.updated
+    }
+    if (view.schemaUI) {
+      view.schemaUI[renameColumn.updated] = view.schemaUI[renameColumn.old]
+      delete view.schemaUI[renameColumn.old]
+    }
+  }
+
+  if (view.schemaUI) {
+    for (const fieldName of Object.keys(view.schemaUI)) {
+      if (!schema[fieldName]) {
+        delete view.schemaUI[fieldName]
+      }
+    }
+    for (const fieldName of Object.keys(schema)) {
+      if (!view.schemaUI[fieldName]) {
+        view.schemaUI[fieldName] = { visible: false }
+      }
+    }
+  }
+
+  view.columns = view.columns?.filter(x => schema[x])
+
+  return view
+}

packages/server/src/sdk/app/views/tests/views.spec.ts

@@ -1,53 +1,54 @@
-import { FieldType, Table, ViewV2 } from "@budibase/types"
+import _ from "lodash"
+import { FieldType, Table, TableSchema, ViewV2 } from "@budibase/types"
 import { generator } from "@budibase/backend-core/tests"
-import { enrichSchema } from ".."
+import { enrichSchema, syncSchema } from ".."
 
 describe("table sdk", () => {
-  describe("enrichViewSchemas", () => {
+  const basicTable: Table = {
-    const basicTable: Table = {
+    _id: generator.guid(),
-      _id: generator.guid(),
+    name: "TestTable",
-      name: "TestTable",
+    type: "table",
-      type: "table",
+    schema: {
-      schema: {
+      name: {
-        name: {
+        type: FieldType.STRING,
-          type: FieldType.STRING,
+        name: "name",
-          name: "name",
+        visible: true,
-          visible: true,
+        width: 80,
-          width: 80,
+        order: 2,
-          order: 2,
+        constraints: {
-          constraints: {
+          type: "string",
-            type: "string",
-          },
-        },
-        description: {
-          type: FieldType.STRING,
-          name: "description",
-          visible: true,
-          width: 200,
-          constraints: {
-            type: "string",
-          },
-        },
-        id: {
-          type: FieldType.NUMBER,
-          name: "id",
-          visible: true,
-          order: 1,
-          constraints: {
-            type: "number",
-          },
-        },
-        hiddenField: {
-          type: FieldType.STRING,
-          name: "hiddenField",
-          visible: false,
-          constraints: {
-            type: "string",
-          },
         },
       },
-    }
+      description: {
+        type: FieldType.STRING,
+        name: "description",
+        visible: true,
+        width: 200,
+        constraints: {
+          type: "string",
+        },
+      },
+      id: {
+        type: FieldType.NUMBER,
+        name: "id",
+        visible: true,
+        order: 1,
+        constraints: {
+          type: "number",
+        },
+      },
+      hiddenField: {
+        type: FieldType.STRING,
+        name: "hiddenField",
+        visible: false,
+        constraints: {
+          type: "string",
+        },
+      },
+    },
+  }
 
+  describe("enrichViewSchemas", () => {
     it("should fetch the default schema if not overriden", async () => {
       const tableId = basicTable._id!
       const view: ViewV2 = {
@@ -280,4 +281,294 @@ describe("table sdk", () => {
       )
     })
   })
+
+  describe("syncSchema", () => {
+    const basicView: ViewV2 = {
+      version: 2,
+      id: generator.guid(),
+      name: generator.guid(),
+      tableId: basicTable._id!,
+    }
+
+    describe("view without schema", () => {
+      it("no table schema changes will not amend the view", () => {
+        const view: ViewV2 = {
+          ...basicView,
+          columns: ["name", "id", "description"],
+        }
+        const result = syncSchema(
+          _.cloneDeep(view),
+          basicTable.schema,
+          undefined
+        )
+        expect(result).toEqual(view)
+      })
+
+      it("adding new columns will not change the view schema", () => {
+        const view: ViewV2 = {
+          ...basicView,
+          columns: ["name", "id", "description"],
+        }
+
+        const newTableSchema = {
+          ...basicTable.schema,
+          newField1: {
+            type: FieldType.STRING,
+            name: "newField1",
+            visible: true,
+          },
+          newField2: {
+            type: FieldType.NUMBER,
+            name: "newField2",
+            visible: false,
+          },
+        }
+
+        const result = syncSchema(_.cloneDeep(view), newTableSchema, undefined)
+        expect(result).toEqual({
+          ...view,
+          schemaUI: undefined,
+        })
+      })
+
+      it("deleting columns will not change the view schema", () => {
+        const view: ViewV2 = {
+          ...basicView,
+          columns: ["name", "id", "description"],
+        }
+        const { name, description, ...newTableSchema } = basicTable.schema
+
+        const result = syncSchema(_.cloneDeep(view), newTableSchema, undefined)
+        expect(result).toEqual({
+          ...view,
+          columns: ["id"],
+          schemaUI: undefined,
+        })
+      })
+
+      it("renaming mapped columns will update the view column mapping", () => {
+        const view: ViewV2 = {
+          ...basicView,
+          columns: ["name", "id", "description"],
+        }
+        const { description, ...newTableSchema } = {
+          ...basicTable.schema,
+          updatedDescription: {
+            ...basicTable.schema.description,
+            name: "updatedDescription",
+          },
+        } as TableSchema
+
+        const result = syncSchema(_.cloneDeep(view), newTableSchema, {
+          old: "description",
+          updated: "updatedDescription",
+        })
+        expect(result).toEqual({
+          ...view,
+          columns: ["name", "id", "updatedDescription"],
+          schemaUI: undefined,
+        })
+      })
+    })
+
+    describe("view with schema", () => {
+      it("no table schema changes will not amend the view", () => {
+        const view: ViewV2 = {
+          ...basicView,
+          columns: ["name", "id", "description"],
+          schemaUI: {
+            name: { visible: true, width: 100 },
+            id: { visible: true, width: 20 },
+            description: { visible: false },
+            hiddenField: { visible: false },
+          },
+        }
+        const result = syncSchema(
+          _.cloneDeep(view),
+          basicTable.schema,
+          undefined
+        )
+        expect(result).toEqual(view)
+      })
+
+      it("adding new columns will add them as not visible to the view", () => {
+        const view: ViewV2 = {
+          ...basicView,
+          columns: ["name", "id", "description"],
+          schemaUI: {
+            name: { visible: true, width: 100 },
+            id: { visible: true, width: 20 },
+            description: { visible: false },
+            hiddenField: { visible: false },
+          },
+        }
+
+        const newTableSchema = {
+          ...basicTable.schema,
+          newField1: {
+            type: FieldType.STRING,
+            name: "newField1",
+            visible: true,
+          },
+          newField2: {
+            type: FieldType.NUMBER,
+            name: "newField2",
+            visible: false,
+          },
+        }
+
+        const result = syncSchema(_.cloneDeep(view), newTableSchema, undefined)
+        expect(result).toEqual({
+          ...view,
+          schemaUI: {
+            ...view.schemaUI,
+            newField1: { visible: false },
+            newField2: { visible: false },
+          },
+        })
+      })
+
+      it("deleting columns will remove them from the UI", () => {
+        const view: ViewV2 = {
+          ...basicView,
+          columns: ["name", "id", "description"],
+          schemaUI: {
+            name: { visible: true, width: 100 },
+            id: { visible: true, width: 20 },
+            description: { visible: false },
+            hiddenField: { visible: false },
+          },
+        }
+        const { name, description, ...newTableSchema } = basicTable.schema
+
+        const result = syncSchema(_.cloneDeep(view), newTableSchema, undefined)
+        expect(result).toEqual({
+          ...view,
+          columns: ["id"],
+          schemaUI: {
+            ...view.schemaUI,
+            name: undefined,
+            description: undefined,
+          },
+        })
+      })
+
+      it("can handle additions and deletions at the same them UI", () => {
+        const view: ViewV2 = {
+          ...basicView,
+          columns: ["name", "id", "description"],
+          schemaUI: {
+            name: { visible: true, width: 100 },
+            id: { visible: true, width: 20 },
+            description: { visible: false },
+            hiddenField: { visible: false },
+          },
+        }
+        const { name, description, ...newTableSchema } = {
+          ...basicTable.schema,
+          newField1: {
+            type: FieldType.STRING,
+            name: "newField1",
+            visible: true,
+          },
+        } as TableSchema
+
+        const result = syncSchema(_.cloneDeep(view), newTableSchema, undefined)
+        expect(result).toEqual({
+          ...view,
+          columns: ["id"],
+          schemaUI: {
+            ...view.schemaUI,
+            name: undefined,
+            description: undefined,
+            newField1: { visible: false },
+          },
+        })
+      })
+
+      it("renaming mapped columns will update the view column mapping and it's schema", () => {
+        const view: ViewV2 = {
+          ...basicView,
+          columns: ["name", "id", "description"],
+          schemaUI: {
+            name: { visible: true },
+            id: { visible: true },
+            description: { visible: true, width: 150, icon: "ic-any" },
+            hiddenField: { visible: false },
+          },
+        }
+        const { description, ...newTableSchema } = {
+          ...basicTable.schema,
+          updatedDescription: {
+            ...basicTable.schema.description,
+            name: "updatedDescription",
+          },
+        } as TableSchema
+
+        const result = syncSchema(_.cloneDeep(view), newTableSchema, {
+          old: "description",
+          updated: "updatedDescription",
+        })
+        expect(result).toEqual({
+          ...view,
+          columns: ["name", "id", "updatedDescription"],
+          schemaUI: {
+            ...view.schemaUI,
+            description: undefined,
+            updatedDescription: { visible: true, width: 150, icon: "ic-any" },
+          },
+        })
+      })
+
+      it("changing no UI schema will not affect the view", () => {
+        const view: ViewV2 = {
+          ...basicView,
+          columns: ["name", "id", "description"],
+          schemaUI: {
+            name: { visible: true, width: 100 },
+            id: { visible: true, width: 20 },
+            description: { visible: false },
+            hiddenField: { visible: false },
+          },
+        }
+        const result = syncSchema(
+          _.cloneDeep(view),
+          {
+            ...basicTable.schema,
+            id: {
+              ...basicTable.schema.id,
+              type: FieldType.NUMBER,
+            },
+          },
+          undefined
+        )
+        expect(result).toEqual(view)
+      })
+
+      it("changing table column UI fields will not affect the view schema", () => {
+        const view: ViewV2 = {
+          ...basicView,
+          columns: ["name", "id", "description"],
+          schemaUI: {
+            name: { visible: true, width: 100 },
+            id: { visible: true, width: 20 },
+            description: { visible: false },
+            hiddenField: { visible: false },
+          },
+        }
+        const result = syncSchema(
+          _.cloneDeep(view),
+          {
+            ...basicTable.schema,
+            id: {
+              ...basicTable.schema.id,
+              visible: !basicTable.schema.id.visible,
+            },
+          },
+          undefined
+        )
+        expect(result).toEqual(view)
+      })
+    })
+  })
 })

packages/server/src/startup.ts

@@ -17,6 +17,7 @@ import * as pro from "@budibase/pro"
 import * as api from "./api"
 import sdk from "./sdk"
 import { initialise as initialiseWebsockets } from "./websockets"
+import { automationsEnabled } from "./features"
 
 let STARTUP_RAN = false
 
@@ -97,7 +98,9 @@ export async function startup(app?: any, server?: any) {
   // configure events to use the pro audit log write
   // can't integrate directly into backend-core due to cyclic issues
   queuePromises.push(events.processors.init(pro.sdk.auditLogs.write))
-  queuePromises.push(automations.init())
+  if (automationsEnabled()) {
+    queuePromises.push(automations.init())
+  }
   queuePromises.push(initPro())
   if (app) {
     // bring routes online as final step once everything ready

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -87,7 +87,7 @@ class TestConfiguration {
     if (openServer) {
       // use a random port because it doesn't matter
       env.PORT = "0"
-      this.server = require("../../app").default
+      this.server = require("../../app").getServer()
       // we need the request for logging in, involves cookies, hard to fake
       this.request = supertest(this.server)
       this.started = true
@@ -178,7 +178,7 @@ class TestConfiguration {
     if (this.server) {
       this.server.close()
     } else {
-      require("../../app").default.close()
+      require("../../app").getServer().close()
     }
     if (this.allApps) {
       cleanup(this.allApps.map(app => app.appId))

packages/server/src/tests/utilities/api/row.ts

@@ -1,4 +1,4 @@
-import { PatchRowRequest } from "@budibase/types"
+import { PatchRowRequest, SaveRowRequest, Row } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -8,12 +8,12 @@ export class RowAPI extends TestAPI {
   }
 
   get = async (
-    tableId: string,
+    sourceId: string,
     rowId: string,
     { expectStatus } = { expectStatus: 200 }
   ) => {
     const request = this.request
-      .get(`/api/${tableId}/rows/${rowId}`)
+      .get(`/api/${sourceId}/rows/${rowId}`)
       .set(this.config.defaultHeaders())
       .expect(expectStatus)
     if (expectStatus !== 404) {
@@ -22,16 +22,43 @@ export class RowAPI extends TestAPI {
     return request
   }
 
+  save = async (
+    sourceId: string,
+    row: SaveRowRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Row> => {
+    const resp = await this.request
+      .post(`/api/${sourceId}/rows`)
+      .send(row)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return resp.body as Row
+  }
+
   patch = async (
-    tableId: string,
+    sourceId: string,
     row: PatchRowRequest,
     { expectStatus } = { expectStatus: 200 }
   ) => {
     return this.request
-      .patch(`/api/${tableId}/rows`)
+      .patch(`/api/${sourceId}/rows`)
       .send(row)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(expectStatus)
   }
+
+  delete = async (
+    sourceId: string,
+    rows: Row[],
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    return this.request
+      .delete(`/api/${sourceId}/rows`)
+      .send({ rows })
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+  }
 }

packages/server/src/tests/utilities/api/viewV2.ts

@@ -1,13 +1,8 @@
 import {
   CreateViewRequest,
-  SortOrder,
-  SortType,
   UpdateViewRequest,
-  DeleteRowRequest,
-  PatchRowRequest,
-  PatchRowResponse,
-  Row,
   ViewV2,
+  SearchViewRowRequest,
 } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
@@ -81,75 +76,14 @@ export class ViewV2API extends TestAPI {
 
   search = async (
     viewId: string,
-    options?: {
+    params?: SearchViewRowRequest,
-      sort: {
-        column: string
-        order?: SortOrder
-        type?: SortType
-      }
-    },
     { expectStatus } = { expectStatus: 200 }
   ) => {
-    const qs: [string, any][] = []
-    if (options?.sort.column) {
-      qs.push(["sort_column", options.sort.column])
-    }
-    if (options?.sort.order) {
-      qs.push(["sort_order", options.sort.order])
-    }
-    if (options?.sort.type) {
-      qs.push(["sort_type", options.sort.type])
-    }
-    let url = `/api/v2/views/${viewId}/search`
-    if (qs.length) {
-      url += "?" + qs.map(q => q.join("=")).join("&")
-    }
     return this.request
-      .get(url)
+      .post(`/api/v2/views/${viewId}/search`)
+      .send(params)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(expectStatus)
   }
-
-  row = {
-    create: async (
-      viewId: string,
-      row: Row,
-      { expectStatus } = { expectStatus: 200 }
-    ): Promise<Row> => {
-      const result = await this.request
-        .post(`/api/v2/views/${viewId}/rows`)
-        .send(row)
-        .set(this.config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(expectStatus)
-      return result.body as Row
-    },
-    update: async (
-      viewId: string,
-      rowId: string,
-      row: PatchRowRequest,
-      { expectStatus } = { expectStatus: 200 }
-    ): Promise<PatchRowResponse> => {
-      const result = await this.request
-        .patch(`/api/v2/views/${viewId}/rows/${rowId}`)
-        .send(row)
-        .set(this.config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(expectStatus)
-      return result.body as PatchRowResponse
-    },
-    delete: async (
-      viewId: string,
-      body: DeleteRowRequest,
-      { expectStatus } = { expectStatus: 200 }
-    ): Promise<any> => {
-      const result = await this.request
-        .delete(`/api/v2/views/${viewId}/rows`)
-        .send(body)
-        .set(this.config.defaultHeaders())
-        .expect(expectStatus)
-      return result.body
-    },
-  }
 }

packages/server/src/threads/automation.ts

@@ -20,6 +20,7 @@ import {
   AutomationMetadata,
   AutomationStatus,
   AutomationStep,
+  AutomationStepStatus,
 } from "@budibase/types"
 import {
   AutomationContext,
@@ -452,7 +453,10 @@ class Orchestrator {
         this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
           id: step.id,
           stepId: step.stepId,
-          outputs: { status: AutomationStatus.NO_ITERATIONS, success: true },
+          outputs: {
+            status: AutomationStepStatus.NO_ITERATIONS,
+            success: true,
+          },
           inputs: {},
         })
 

packages/server/src/threads/definitions.ts

@@ -11,6 +11,12 @@ export interface QueryEvent {
   queryId: string
   environmentVariables?: Record<string, string>
   ctx?: any
+  schema?: {
+    [key: string]: {
+      name: string
+      type: string
+    }
+  }
 }
 
 export interface QueryVariable {

packages/server/src/threads/query.ts

@@ -8,6 +8,7 @@ import { context, cache, auth } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import sdk from "../sdk"
 import { cloneDeep } from "lodash/fp"
+import { SourceName } from "@budibase/types"
 
 import { isSQL } from "../integrations/utils"
 import { interpolateSQL } from "../integrations/queries/sql"
@@ -28,6 +29,7 @@ class QueryRunner {
   hasRerun: boolean
   hasRefreshedOAuth: boolean
   hasDynamicVariables: boolean
+  schema: any
 
   constructor(input: QueryEvent, flags = { noRecursiveQuery: false }) {
     this.datasource = input.datasource
@@ -37,6 +39,7 @@ class QueryRunner {
     this.pagination = input.pagination
     this.transformer = input.transformer
     this.queryId = input.queryId
+    this.schema = input.schema
     this.noRecursiveQuery = flags.noRecursiveQuery
     this.cachedVariables = []
     // Additional context items for enrichment
@@ -51,7 +54,7 @@ class QueryRunner {
   }
 
   async execute(): Promise<any> {
-    let { datasource, fields, queryVerb, transformer } = this
+    let { datasource, fields, queryVerb, transformer, schema } = this
     let datasourceClone = cloneDeep(datasource)
     let fieldsClone = cloneDeep(fields)
 
@@ -70,6 +73,9 @@ class QueryRunner {
 
     const integration = new Integration(datasourceClone.config)
 
+    // define the type casting from the schema
+    integration.defineTypeCastingFromSchema?.(schema)
+
     // pre-query, make sure datasource variables are added to parameters
     const parameters = await this.addDatasourceVariables()
 

packages/server/src/utilities/security.ts

@@ -1,5 +1,5 @@
 import { permissions, roles } from "@budibase/backend-core"
-import { DocumentType } from "../db/utils"
+import { DocumentType, VirtualDocumentType } from "../db/utils"
 
 export const CURRENTLY_SUPPORTED_LEVELS: string[] = [
   permissions.PermissionLevel.WRITE,
@@ -11,9 +11,10 @@ export function getPermissionType(resourceId: string) {
   const docType = Object.values(DocumentType).filter(docType =>
     resourceId.startsWith(docType)
   )[0]
-  switch (docType) {
+  switch (docType as DocumentType | VirtualDocumentType) {
     case DocumentType.TABLE:
     case DocumentType.ROW:
+    case VirtualDocumentType.VIEW:
       return permissions.PermissionType.TABLE
     case DocumentType.AUTOMATION:
       return permissions.PermissionType.AUTOMATION
@@ -22,9 +23,6 @@ export function getPermissionType(resourceId: string) {
     case DocumentType.QUERY:
     case DocumentType.DATASOURCE:
       return permissions.PermissionType.QUERY
-    default:
-      // views don't have an ID, will end up here
-      return permissions.PermissionType.VIEW
   }
 }
 

packages/server/src/utilities/users.ts

@@ -1,9 +1,9 @@
 import { InternalTables } from "../db/utils"
 import { getGlobalUser } from "./global"
-import { context, db as dbCore, roles } from "@budibase/backend-core"
+import { context, roles } from "@budibase/backend-core"
-import { BBContext } from "@budibase/types"
+import { UserCtx } from "@budibase/types"
 
-export async function getFullUser(ctx: BBContext, userId: string) {
+export async function getFullUser(ctx: UserCtx, userId: string) {
   const global = await getGlobalUser(userId)
   let metadata: any = {}
 
@@ -29,21 +29,12 @@ export async function getFullUser(ctx: BBContext, userId: string) {
   }
 }
 
-export function publicApiUserFix(ctx: BBContext) {
+export function publicApiUserFix(ctx: UserCtx) {
   if (!ctx.request.body) {
     return ctx
   }
   if (!ctx.request.body._id && ctx.params.userId) {
     ctx.request.body._id = ctx.params.userId
   }
-  if (!ctx.request.body.roles) {
-    ctx.request.body.roles = {}
-  } else {
-    const newRoles: { [key: string]: any } = {}
-    for (let [appId, role] of Object.entries(ctx.request.body.roles)) {
-      newRoles[dbCore.getProdAppID(appId)] = role
-    }
-    ctx.request.body.roles = newRoles
-  }
   return ctx
 }

packages/shared-core/package.json

@@ -32,7 +32,18 @@
             "target": "build"
           }
         ]
+      },
+      "dev:builder": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
       }
+
     }
   }
 }

packages/shared-core/tsconfig.build.json

@@ -1,7 +1,7 @@
 {
   "compilerOptions": {
     "target": "es6",
-    "moduleResolution": "node",
+    "module": "commonjs",
     "lib": ["es2020"],
     "strict": true,
     "noImplicitAny": true,

packages/shared-core/tsconfig.json

@@ -1,4 +1,10 @@
 {
   "extends": "./tsconfig.build.json",
+  "compilerOptions": {
+    "baseUrl": "..",
+    "rootDir": "src",
+    "composite": true,
+    "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo"
+  },
   "exclude": ["node_modules", "dist"]
 }

packages/types/src/api/web/app/rows.ts

@@ -1,5 +1,8 @@
+import { SearchParams } from "../../../sdk"
 import { Row } from "../../../documents"
 
+export interface SaveRowRequest extends Row {}
+
 export interface PatchRowRequest extends Row {
   _id: string
   _rev: string
@@ -8,6 +11,14 @@ export interface PatchRowRequest extends Row {
 
 export interface PatchRowResponse extends Row {}
 
-export interface SearchResponse {
+export interface SearchRowRequest extends Omit<SearchParams, "tableId"> {}
+
+export interface SearchViewRowRequest
+  extends Pick<
+    SearchRowRequest,
+    "sort" | "sortOrder" | "sortType" | "limit" | "bookmark" | "paginate"
+  > {}
+
+export interface SearchRowResponse {
   rows: any[]
 }

packages/types/src/api/web/app/table.ts

@@ -1,4 +1,10 @@
-import { Table, TableSchema, View, ViewV2 } from "../../../documents"
+import {
+  Table,
+  TableRequest,
+  TableSchema,
+  View,
+  ViewV2,
+} from "../../../documents"
 
 interface ViewV2Response extends ViewV2 {
   schema: TableSchema
@@ -11,3 +17,7 @@ export interface TableResponse extends Table {
 }
 
 export type FetchTablesResponse = TableResponse[]
+
+export interface SaveTableRequest extends TableRequest {}
+
+export type SaveTableResponse = Table

packages/types/src/documents/app/automation.ts

@@ -179,12 +179,15 @@ export interface AutomationTrigger extends AutomationTriggerSchema {
   id: string
 }
 
+export enum AutomationStepStatus {
+  NO_ITERATIONS = "no_iterations",
+}
+
 export enum AutomationStatus {
   SUCCESS = "success",
   ERROR = "error",
   STOPPED = "stopped",
   STOPPED_ERROR = "stopped_error",
-  NO_ITERATIONS = "no_iterations",
 }
 
 export interface AutomationResults {

packages/types/src/documents/app/view.ts

@@ -1,6 +1,5 @@
-import { SortOrder, SortType } from "../../api"
+import { SearchFilter, SortOrder, SortType } from "../../api"
-import { SearchFilters } from "../../sdk"
+import { UIFieldMetadata } from "./table"
-import { TableSchema, UIFieldMetadata } from "./table"
 
 export interface View {
   name: string
@@ -20,7 +19,7 @@ export interface ViewV2 {
   name: string
   primaryDisplay?: string
   tableId: string
-  query?: SearchFilters
+  query?: SearchFilter[]
   sort?: {
     field: string
     order?: SortOrder