Merge branch 'master' into fix/show-formblock-description-in-view-mode

.github/workflows/release-master.yml

@@ -36,6 +36,7 @@ jobs:
       - uses: actions/setup-node@v1
         with:
           node-version: 18.x
+          cache: yarn
 
       - run: yarn install --frozen-lockfile
       - name: Update versions
@@ -63,14 +64,64 @@ jobs:
           echo "Using tag $version"
           echo "version=$version" >> "$GITHUB_OUTPUT"
 
-      - name: Build/release Docker images
+      - name: Setup Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Docker login
         run: |
           docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build:docker
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }}
+
+      - name: Build worker docker
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
+          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
+          file: ./packages/worker/Dockerfile.v2
+          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
+          cache-to: type=inline
+        env:
+          IMAGE_NAME: budibase/worker
+          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
+          BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
+
+      - name: Build server docker
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
+          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
+          file: ./packages/server/Dockerfile.v2
+          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
+          cache-to: type=inline
+        env:
+          IMAGE_NAME: budibase/apps
+          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
+          BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
+
+      - name: Build proxy docker
+        uses: docker/build-push-action@v5
+        with:
+          context: ./hosting/proxy
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
+          file: ./hosting/proxy/Dockerfile
+          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
+          cache-to: type=inline
+        env:
+          IMAGE_NAME: budibase/proxy
+          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
 
   release-helm-chart:
     needs: [release-images]

.github/workflows/release-singleimage.yml

@@ -67,7 +67,7 @@ jobs:
           push: true
           platforms: linux/amd64,linux/arm64
           tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
+          file: ./hosting/single/Dockerfile.v2
       - name: Tag and release Budibase Azure App Service docker image
         uses: docker/build-push-action@v2
         with:
@@ -76,4 +76,4 @@ jobs:
           platforms: linux/amd64
           build-args: TARGETBUILD=aas
           tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
+          file: ./hosting/single/Dockerfile.v2

README.md

@@ -126,13 +126,6 @@ You can learn more about the Budibase API at the following places:
 
 - [Build an app with Budibase and Next.js](https://budibase.com/blog/building-a-crud-app-with-budibase-and-next.js/)
 
-<p align="center">
-  <img alt="Budibase data" src="https://res.cloudinary.com/daog6scxm/image/upload/v1647858558/Feb%20release/Start_building_with_Budibase_s_API_3_rhlzhv.png">
-</p>
-<br /><br />
-
-<br /><br /><br />
-
 ## 🏁 Get started
 
 Deploy Budibase self-hosted in your existing infrastructure, using Docker, Kubernetes, and Digital Ocean.

hosting/scripts/linux/release-to-docker-hub.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-tag=$1
-
-if [[ ! "$tag" ]]; then
-	echo "No tag present. You must pass a tag to this script"
-	exit 1
-fi
-
-echo "Tagging images with tag: $tag"
-
-docker tag proxy-service budibase/proxy:$tag
-docker tag app-service budibase/apps:$tag
-docker tag worker-service budibase/worker:$tag
-
-docker push --all-tags budibase/apps 
-docker push --all-tags budibase/worker
-docker push --all-tags budibase/proxy

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.11.45",
+  "version": "2.12.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -54,10 +54,6 @@
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
     "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
     "build:specs": "lerna run --stream specs",
-    "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
-    "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
-    "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
-    "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",

packages/backend-core/__mocks__/aws-sdk.ts

@@ -3,6 +3,7 @@ const mockS3 = {
   deleteObject: jest.fn().mockReturnThis(),
   deleteObjects: jest.fn().mockReturnThis(),
   createBucket: jest.fn().mockReturnThis(),
+  getObject: jest.fn().mockReturnThis(),
   listObject: jest.fn().mockReturnThis(),
   getSignedUrl: jest.fn((operation: string, params: any) => {
     return `http://s3.example.com/${params.Bucket}/${params.Key}`

packages/backend-core/package.json

@@ -21,7 +21,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/nano": "10.1.2",
+    "@budibase/nano": "10.1.3",
     "@budibase/pouchdb-replication-stream": "1.2.10",
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",

packages/backend-core/src/cache/writethrough.ts

@@ -119,8 +119,8 @@ export class Writethrough {
     this.writeRateMs = writeRateMs
   }
 
-  async put(doc: any) {
+  async put(doc: any, writeRateMs: number = this.writeRateMs) {
-    return put(this.db, doc, this.writeRateMs)
+    return put(this.db, doc, writeRateMs)
   }
 
   async get(id: string) {

packages/backend-core/src/db/constants.ts

@@ -8,3 +8,7 @@ export const CONSTANT_INTERNAL_ROW_COLS = [
 ] as const
 
 export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const
+
+export function isInternalColumnName(name: string): boolean {
+  return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name)
+}

packages/backend-core/src/docIds/params.ts

@@ -6,6 +6,7 @@ import {
   ViewName,
 } from "../constants"
 import { getProdAppID } from "./conversions"
+import { DatabaseQueryOpts } from "@budibase/types"
 
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this
@@ -22,8 +23,8 @@ import { getProdAppID } from "./conversions"
 export function getDocParams(
   docType: string,
   docId?: string | null,
-  otherProps: any = {}
+  otherProps: Partial<DatabaseQueryOpts> = {}
-) {
+): DatabaseQueryOpts {
   if (docId == null) {
     docId = ""
   }
@@ -45,8 +46,8 @@ export function getDocParams(
 export function getRowParams(
   tableId?: string | null,
   rowId?: string | null,
-  otherProps = {}
+  otherProps: Partial<DatabaseQueryOpts> = {}
-) {
+): DatabaseQueryOpts {
   if (tableId == null) {
     return getDocParams(DocumentType.ROW, null, otherProps)
   }
@@ -88,7 +89,10 @@ export const isDatasourceId = (id: string) => {
 /**
  * Gets parameters for retrieving workspaces.
  */
-export function getWorkspaceParams(id = "", otherProps = {}) {
+export function getWorkspaceParams(
+  id = "",
+  otherProps: Partial<DatabaseQueryOpts> = {}
+): DatabaseQueryOpts {
   return {
     ...otherProps,
     startkey: `${DocumentType.WORKSPACE}${SEPARATOR}${id}`,
@@ -99,7 +103,10 @@ export function getWorkspaceParams(id = "", otherProps = {}) {
 /**
  * Gets parameters for retrieving users.
  */
-export function getGlobalUserParams(globalId: any, otherProps: any = {}) {
+export function getGlobalUserParams(
+  globalId: any,
+  otherProps: Partial<DatabaseQueryOpts> = {}
+): DatabaseQueryOpts {
   if (!globalId) {
     globalId = ""
   }
@@ -117,11 +124,17 @@ export function getGlobalUserParams(globalId: any, otherProps: any = {}) {
 /**
  * Gets parameters for retrieving users, this is a utility function for the getDocParams function.
  */
-export function getUserMetadataParams(userId?: string | null, otherProps = {}) {
+export function getUserMetadataParams(
+  userId?: string | null,
+  otherProps: Partial<DatabaseQueryOpts> = {}
+): DatabaseQueryOpts {
   return getRowParams(InternalTable.USER_METADATA, userId, otherProps)
 }
 
-export function getUsersByAppParams(appId: any, otherProps: any = {}) {
+export function getUsersByAppParams(
+  appId: any,
+  otherProps: Partial<DatabaseQueryOpts> = {}
+): DatabaseQueryOpts {
   const prodAppId = getProdAppID(appId)
   return {
     ...otherProps,

packages/backend-core/src/security/roles.ts

@@ -122,7 +122,9 @@ export async function roleToNumber(id?: string) {
   if (isBuiltin(id)) {
     return builtinRoleToNumber(id)
   }
-  const hierarchy = (await getUserRoleHierarchy(id)) as RoleDoc[]
+  const hierarchy = (await getUserRoleHierarchy(id, {
+    defaultPublic: true,
+  })) as RoleDoc[]
   for (let role of hierarchy) {
     if (isBuiltin(role?.inherits)) {
       return builtinRoleToNumber(role.inherits) + 1
@@ -192,12 +194,15 @@ export async function getRole(
 /**
  * Simple function to get all the roles based on the top level user role ID.
  */
-async function getAllUserRoles(userRoleId?: string): Promise<RoleDoc[]> {
+async function getAllUserRoles(
+  userRoleId?: string,
+  opts?: { defaultPublic?: boolean }
+): Promise<RoleDoc[]> {
   // admins have access to all roles
   if (userRoleId === BUILTIN_IDS.ADMIN) {
     return getAllRoles()
   }
-  let currentRole = await getRole(userRoleId)
+  let currentRole = await getRole(userRoleId, opts)
   let roles = currentRole ? [currentRole] : []
   let roleIds = [userRoleId]
   // get all the inherited roles
@@ -226,12 +231,16 @@ export async function getUserRoleIdHierarchy(
  * Returns an ordered array of the user's inherited role IDs, this can be used
  * to determine if a user can access something that requires a specific role.
  * @param userRoleId The user's role ID, this can be found in their access token.
+ * @param opts optional - if want to default to public use this.
  * @returns returns an ordered array of the roles, with the first being their
  * highest level of access and the last being the lowest level.
  */
-export async function getUserRoleHierarchy(userRoleId?: string) {
+export async function getUserRoleHierarchy(
+  userRoleId?: string,
+  opts?: { defaultPublic?: boolean }
+) {
   // special case, if they don't have a role then they are a public user
-  return getAllUserRoles(userRoleId)
+  return getAllUserRoles(userRoleId, opts)
 }
 
 // this function checks that the provided permissions are in an array format

packages/backend-core/src/users/db.ts

@@ -25,12 +25,17 @@ import {
 import {
   getAccountHolderFromUserIds,
   isAdmin,
+  isCreator,
   validateUniqueUser,
 } from "./utils"
 import { searchExistingEmails } from "./lookup"
 import { hash } from "../utils"
 
-type QuotaUpdateFn = (change: number, cb?: () => Promise<any>) => Promise<any>
+type QuotaUpdateFn = (
+  change: number,
+  creatorsChange: number,
+  cb?: () => Promise<any>
+) => Promise<any>
 type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise<any>
 type FeatureFn = () => Promise<Boolean>
 type GroupGetFn = (ids: string[]) => Promise<UserGroup[]>
@@ -160,13 +165,9 @@ export class UserDB {
   }
 
   static async getUsersByAppAccess(opts: { appId?: string; limit?: number }) {
-    const params: any = {
-      include_docs: true,
-      limit: opts.limit || 50,
-    }
     let response: User[] = await usersCore.searchGlobalUsersByAppAccess(
       opts.appId,
-      params
+      { limit: opts.limit || 50 }
     )
     return response
   }
@@ -245,7 +246,8 @@ export class UserDB {
     }
 
     const change = dbUser ? 0 : 1 // no change if there is existing user
-    return UserDB.quotas.addUsers(change, async () => {
+    const creatorsChange = isCreator(dbUser) !== isCreator(user) ? 1 : 0
+    return UserDB.quotas.addUsers(change, creatorsChange, async () => {
       await validateUniqueUser(email, tenantId)
 
       let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser)
@@ -307,6 +309,7 @@ export class UserDB {
 
     let usersToSave: any[] = []
     let newUsers: any[] = []
+    let newCreators: any[] = []
 
     const emails = newUsersRequested.map((user: User) => user.email)
     const existingEmails = await searchExistingEmails(emails)
@@ -327,10 +330,16 @@ export class UserDB {
       }
       newUser.userGroups = groups
       newUsers.push(newUser)
+      if (isCreator(newUser)) {
+        newCreators.push(newUser)
+      }
     }
 
     const account = await accountSdk.getAccountByTenantId(tenantId)
-    return UserDB.quotas.addUsers(newUsers.length, async () => {
+    return UserDB.quotas.addUsers(
+      newUsers.length,
+      newCreators.length,
+      async () => {
         // create the promises array that will be called by bulkDocs
         newUsers.forEach((user: any) => {
           usersToSave.push(
@@ -379,7 +388,8 @@ export class UserDB {
           successful: saved,
           unsuccessful,
         }
-    })
+      }
+    )
   }
 
   static async bulkDelete(userIds: string[]): Promise<BulkUserDeleted> {
@@ -419,11 +429,12 @@ export class UserDB {
       _deleted: true,
     }))
     const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete)
+    const creatorsToDelete = usersToDelete.filter(isCreator)
 
-    await UserDB.quotas.removeUsers(toDelete.length)
     for (let user of usersToDelete) {
       await bulkDeleteProcessing(user)
     }
+    await UserDB.quotas.removeUsers(toDelete.length, creatorsToDelete.length)
 
     // Build Response
     // index users by id
@@ -472,7 +483,8 @@ export class UserDB {
 
     await db.remove(userId, dbUser._rev)
 
-    await UserDB.quotas.removeUsers(1)
+    const creatorsToDelete = isCreator(dbUser) ? 1 : 0
+    await UserDB.quotas.removeUsers(1, creatorsToDelete)
     await eventHelpers.handleDeleteEvents(dbUser)
     await cache.user.invalidateUser(userId)
     await sessions.invalidateSessions(userId, { reason: "deletion" })

packages/backend-core/src/users/users.ts

@@ -14,12 +14,13 @@ import {
 } from "../db"
 import {
   BulkDocsResponse,
-  ContextUser,
   SearchQuery,
   SearchQueryOperators,
   SearchUsersRequest,
   User,
+  ContextUser,
   DatabaseQueryOpts,
+  CouchFindOptions,
 } from "@budibase/types"
 import { getGlobalDB } from "../context"
 import * as context from "../context"
@@ -140,7 +141,7 @@ export const getGlobalUserByEmail = async (
 
 export const searchGlobalUsersByApp = async (
   appId: any,
-  opts: any,
+  opts: DatabaseQueryOpts,
   getOpts?: GetOpts
 ) => {
   if (typeof appId !== "string") {
@@ -166,7 +167,10 @@ export const searchGlobalUsersByApp = async (
   Return any user who potentially has access to the application
   Admins, developers and app users with the explicitly role.
 */
-export const searchGlobalUsersByAppAccess = async (appId: any, opts: any) => {
+export const searchGlobalUsersByAppAccess = async (
+  appId: any,
+  opts?: { limit?: number }
+) => {
   const roleSelector = `roles.${appId}`
 
   let orQuery: any[] = [
@@ -187,7 +191,7 @@ export const searchGlobalUsersByAppAccess = async (appId: any, opts: any) => {
     orQuery.push(roleCheck)
   }
 
-  let searchOptions = {
+  let searchOptions: CouchFindOptions = {
     selector: {
       $or: orQuery,
       _id: {
@@ -198,7 +202,7 @@ export const searchGlobalUsersByAppAccess = async (appId: any, opts: any) => {
   }
 
   const resp = await directCouchFind(context.getGlobalDBName(), searchOptions)
-  return resp?.rows
+  return resp.rows
 }
 
 export const getGlobalUserByAppPage = (appId: string, user: User) => {
@@ -245,7 +249,8 @@ export const paginatedUsers = async ({
   limit,
 }: SearchUsersRequest = {}) => {
   const db = getGlobalDB()
-  const pageLimit = limit ? limit + 1 : PAGE_LIMIT + 1
+  const pageSize = limit ?? PAGE_LIMIT
+  const pageLimit = pageSize + 1
   // get one extra document, to have the next page
   const opts: DatabaseQueryOpts = {
     include_docs: true,
@@ -272,7 +277,7 @@ export const paginatedUsers = async ({
     const response = await db.allDocs(getGlobalUserParams(null, opts))
     userList = response.rows.map((row: any) => row.doc)
   }
-  return pagination(userList, pageLimit, {
+  return pagination(userList, pageSize, {
     paginate: true,
     property,
     getKey,

packages/backend-core/tests/core/users/users.spec.js

@@ -0,0 +1,54 @@
+const _ = require('lodash/fp')
+const {structures} = require("../../../tests")
+
+jest.mock("../../../src/context")
+jest.mock("../../../src/db")
+
+const context = require("../../../src/context")
+const db = require("../../../src/db")
+
+const {getCreatorCount} = require('../../../src/users/users')
+
+describe("Users", () => {
+
+  let getGlobalDBMock
+  let getGlobalUserParamsMock
+  let paginationMock
+
+  beforeEach(() => {
+    jest.resetAllMocks()
+
+    getGlobalDBMock = jest.spyOn(context, "getGlobalDB")
+    getGlobalUserParamsMock = jest.spyOn(db, "getGlobalUserParams")
+    paginationMock = jest.spyOn(db, "pagination")
+  })
+
+  it("Retrieves the number of creators", async () => {
+    const getUsers = (offset, limit, creators = false) => {
+      const range = _.range(offset, limit)
+      const opts = creators ? {builder: {global: true}} : undefined
+      return range.map(() => structures.users.user(opts))
+    }
+    const page1Data = getUsers(0, 8)
+    const page2Data = getUsers(8, 12, true)
+    getGlobalDBMock.mockImplementation(() => ({
+      name   : "fake-db",
+      allDocs: () => ({
+        rows: [...page1Data, ...page2Data]
+      })
+    }))
+    paginationMock.mockImplementationOnce(() => ({
+      data: page1Data,
+      hasNextPage: true,
+      nextPage: "1"
+    }))
+    paginationMock.mockImplementation(() => ({
+      data: page2Data,
+      hasNextPage: false,
+      nextPage: undefined
+    }))
+    const creatorsCount = await getCreatorCount()
+    expect(creatorsCount).toBe(4)
+    expect(paginationMock).toHaveBeenCalledTimes(2)
+  })
+})

packages/backend-core/tests/core/utilities/mocks/date.ts

@@ -1,2 +1,3 @@
 export const MOCK_DATE = new Date("2020-01-01T00:00:00.000Z")
+
 export const MOCK_DATE_TIMESTAMP = 1577836800000

packages/backend-core/tests/core/utilities/structures/licenses.ts

@@ -123,6 +123,10 @@ export function customer(): Customer {
 export function subscription(): Subscription {
   return {
     amount: 10000,
+    amounts: {
+      user: 10000,
+      creator: 0,
+    },
     cancelAt: undefined,
     currency: "usd",
     currentPeriodEnd: 0,
@@ -131,6 +135,10 @@ export function subscription(): Subscription {
     duration: PriceDuration.MONTHLY,
     pastDueAt: undefined,
     quantity: 0,
+    quantities: {
+      user: 0,
+      creator: 0,
+    },
     status: "active",
   }
 }

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -159,8 +159,10 @@
           {#if selectedImage.size}
             <div class="filesize">
               {#if selectedImage.size <= BYTES_IN_MB}
-                {`${selectedImage.size / BYTES_IN_KB} KB`}
+                {`${(selectedImage.size / BYTES_IN_KB).toFixed(1)} KB`}
-              {:else}{`${selectedImage.size / BYTES_IN_MB} MB`}{/if}
+              {:else}{`${(selectedImage.size / BYTES_IN_MB).toFixed(
+                  1
+                )} MB`}{/if}
             </div>
           {/if}
           {#if !disabled}
@@ -203,8 +205,8 @@
             {#if file.size}
               <div class="filesize">
                 {#if file.size <= BYTES_IN_MB}
-                  {`${file.size / BYTES_IN_KB} KB`}
+                  {`${(file.size / BYTES_IN_KB).toFixed(1)} KB`}
-                {:else}{`${file.size / BYTES_IN_MB} MB`}{/if}
+                {:else}{`${(file.size / BYTES_IN_MB).toFixed(1)} MB`}{/if}
               </div>
             {/if}
             {#if !disabled}

packages/builder/src/builderStore/store/frontend.js

@@ -580,7 +580,7 @@ export const getFrontendStore = () => {
         let table = validTables.find(table => {
           return (
             table.sourceId !== BUDIBASE_INTERNAL_DB_ID &&
-            table.type === DB_TYPE_INTERNAL
+            table.sourceType === DB_TYPE_INTERNAL
           )
         })
         if (table) {
@@ -591,7 +591,7 @@ export const getFrontendStore = () => {
         table = validTables.find(table => {
           return (
             table.sourceId === BUDIBASE_INTERNAL_DB_ID &&
-            table.type === DB_TYPE_INTERNAL
+            table.sourceType === DB_TYPE_INTERNAL
           )
         })
         if (table) {
@@ -599,7 +599,7 @@ export const getFrontendStore = () => {
         }
 
         // Finally try an external table
-        return validTables.find(table => table.type === DB_TYPE_EXTERNAL)
+        return validTables.find(table => table.sourceType === DB_TYPE_EXTERNAL)
       },
       enrichEmptySettings: (component, opts) => {
         if (!component?._component) {

packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte

@@ -16,7 +16,6 @@
   $: linkedTable = $tables.list.find(table => table._id === linkedTableId)
   $: schema = linkedTable?.schema
   $: table = $tables.list.find(table => table._id === tableId)
-  $: type = table?.type
   $: fetchData(tableId, rowId)
   $: {
     let rowLabel = row?.[table?.primaryDisplay]
@@ -41,5 +40,5 @@
 </script>
 
 {#if row && row._id === rowId}
-  <Table {title} {schema} {data} {type} />
+  <Table {title} {schema} {data} />
 {/if}

packages/builder/src/components/backend/DataTable/TableDataTable.svelte

@@ -16,6 +16,7 @@
   import GridRelationshipButton from "components/backend/DataTable/buttons/grid/GridRelationshipButton.svelte"
   import GridEditColumnModal from "components/backend/DataTable/modals/grid/GridEditColumnModal.svelte"
   import GridUsersTableButton from "components/backend/DataTable/modals/grid/GridUsersTableButton.svelte"
+  import { DB_TYPE_EXTERNAL } from "constants/backend"
 
   const userSchemaOverrides = {
     firstName: { displayName: "First name", disabled: true },
@@ -27,7 +28,7 @@
 
   $: id = $tables.selected?._id
   $: isUsersTable = id === TableNames.USERS
-  $: isInternal = $tables.selected?.type !== "external"
+  $: isInternal = $tables.selected?.sourceType !== DB_TYPE_EXTERNAL
   $: gridDatasource = {
     type: "table",
     tableId: id,
@@ -46,10 +47,7 @@
     tables.replaceTable(id, e.detail)
 
     // We need to refresh datasources when an external table changes.
-    // Type "external" may exist - sometimes type is "table" and sometimes it
+    if (e.detail?.sourceType === DB_TYPE_EXTERNAL) {
-    // is "external" - it has different meanings in different endpoints.
-    // If we check both these then we hopefully catch all external tables.
-    if (e.detail?.type === "external" || e.detail?.sql) {
       await datasources.fetch()
     }
   }

packages/builder/src/components/backend/DataTable/ViewDataTable.svelte

@@ -17,7 +17,6 @@
   let hideAutocolumns = true
   let data = []
   let loading = false
-  let type = "internal"
 
   $: name = view.name
   $: schema = view.schema
@@ -66,7 +65,6 @@
   tableId={view.tableId}
   {data}
   {loading}
-  {type}
   rowCount={10}
   allowEditing={false}
   bind:hideAutocolumns

packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte

@@ -10,6 +10,6 @@
 <ImportButton
   {disabled}
   tableId={$datasource?.tableId}
-  tableType={$definition?.type}
+  tableType={$definition?.sourceType}
   on:importrows={rows.actions.refreshData}
 />

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -26,6 +26,7 @@
     ALLOWABLE_NUMBER_TYPES,
     SWITCHABLE_TYPES,
     PrettyRelationshipDefinitions,
+    DB_TYPE_EXTERNAL,
   } from "constants/backend"
   import { getAutoColumnInformation, buildAutoColumn } from "builderStore/utils"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
@@ -254,10 +255,11 @@
     !uneditable &&
     editableColumn?.type !== AUTO_TYPE &&
     !editableColumn.autocolumn
-  $: external = table.type === "external"
+  $: externalTable = table.sourceType === DB_TYPE_EXTERNAL
   // in the case of internal tables the sourceId will just be undefined
   $: tableOptions = $tables.list.filter(
-    opt => opt.type === table.type && table.sourceId === opt.sourceId
+    opt =>
+      opt.sourceType === table.sourceType && table.sourceId === opt.sourceId
   )
   $: typeEnabled =
     !originalName ||
@@ -409,7 +411,7 @@
       editableColumn.type === FieldType.BB_REFERENCE &&
       editableColumn.subtype === FieldSubtype.USERS
 
-    if (!external) {
+    if (!externalTable) {
       return [
         FIELDS.STRING,
         FIELDS.BARCODEQR,
@@ -441,7 +443,7 @@
         isUsers ? FIELDS.USERS : FIELDS.USER,
       ]
       // no-sql or a spreadsheet
-      if (!external || table.sql) {
+      if (!externalTable || table.sql) {
         fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
       }
       return fields
@@ -486,7 +488,7 @@
       })
     }
     const newError = {}
-    if (!external && fieldInfo.name?.startsWith("_")) {
+    if (!externalTable && fieldInfo.name?.startsWith("_")) {
       newError.name = `Column name cannot start with an underscore.`
     } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) {
       newError.name = `Illegal character; must be alpha-numeric.`
@@ -498,7 +500,7 @@
       newError.name = `Column name already in use.`
     }
 
-    if (fieldInfo.type == "auto" && !fieldInfo.subtype) {
+    if (fieldInfo.type === "auto" && !fieldInfo.subtype) {
       newError.subtype = `Auto Column requires a type`
     }
 

packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte

@@ -1,6 +1,6 @@
 <script>
   import { Select, Toggle, Multiselect } from "@budibase/bbui"
-  import { FIELDS } from "constants/backend"
+  import { DB_TYPE_INTERNAL, FIELDS } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
 
@@ -169,7 +169,7 @@
       </div>
     {/each}
   </div>
-  {#if tableType === "internal"}
+  {#if tableType === DB_TYPE_INTERNAL}
     <br />
     <Toggle
       bind:value={updateExistingRows}

packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte

@@ -8,6 +8,7 @@
   import {
     BUDIBASE_INTERNAL_DB_ID,
     BUDIBASE_DATASOURCE_TYPE,
+    DB_TYPE_INTERNAL,
   } from "constants/backend"
 
   $: tableNames = $tables.list.map(table => table.name)
@@ -55,8 +56,9 @@
       name,
       schema: { ...schema },
       rows,
-      type: "internal",
+      type: "table",
       sourceId: targetDatasourceId,
+      sourceType: DB_TYPE_INTERNAL,
     }
 
     // Only set primary display if defined

packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte

@@ -13,6 +13,7 @@
     notifications,
   } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import { DB_TYPE_EXTERNAL } from "constants/backend"
 
   export let table
 
@@ -27,8 +28,8 @@
   let willBeDeleted
   let deleteTableName
 
-  $: external = table?.type === "external"
+  $: externalTable = table?.sourceType === DB_TYPE_EXTERNAL
-  $: allowDeletion = !external || table?.created
+  $: allowDeletion = !externalTable || table?.created
 
   function showDeleteModal() {
     templateScreens = $store.screens.filter(
@@ -48,7 +49,7 @@
       for (let screen of templateScreens) {
         await store.actions.screens.delete(screen)
       }
-      if (table.type === "external") {
+      if (table.sourceType === DB_TYPE_EXTERNAL) {
         await datasources.fetch()
       }
       notifications.success("Table deleted")
@@ -91,7 +92,7 @@
     <div slot="control" class="icon">
       <Icon s hoverable name="MoreSmallList" />
     </div>
-    {#if !external}
+    {#if !externalTable}
       <MenuItem icon="Edit" on:click={editorModal.show}>Edit</MenuItem>
     {/if}
     <MenuItem icon="Delete" on:click={showDeleteModal}>Delete</MenuItem>

packages/builder/src/components/common/Dropzone.svelte

@@ -23,7 +23,7 @@
     try {
       return await API.uploadBuilderAttachment(data)
     } catch (error) {
-      notifications.error("Failed to upload attachment")
+      notifications.error(error.message || "Failed to upload attachment")
       return []
     }
   }

packages/builder/src/components/common/RoleSelect.svelte

@@ -39,7 +39,15 @@
     allowCreator
   ) => {
     if (allowedRoles?.length) {
-      return roles.filter(role => allowedRoles.includes(role._id))
+      const filteredRoles = roles.filter(role =>
+        allowedRoles.includes(role._id)
+      )
+      return [
+        ...filteredRoles,
+        ...(allowedRoles.includes(Constants.Roles.CREATOR)
+          ? [{ _id: Constants.Roles.CREATOR, name: "Creator", enabled: false }]
+          : []),
+      ]
     }
     let newRoles = [...roles]
 
@@ -129,8 +137,9 @@
     getOptionColour={getColor}
     getOptionIcon={getIcon}
     isOptionEnabled={option =>
-      option._id !== Constants.Roles.CREATOR ||
+      (option._id !== Constants.Roles.CREATOR ||
-      $licensing.perAppBuildersEnabled}
+        $licensing.perAppBuildersEnabled) &&
+      option.enabled !== false}
     {placeholder}
     {error}
   />

packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte

@@ -23,7 +23,7 @@
 </script>
 
 <div class="table">
-  <Table {schema} data={rowsCopy} type="external" allowEditing={false} />
+  <Table {schema} data={rowsCopy} allowEditing={false} />
 </div>
 
 <style>

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -516,6 +516,13 @@
     }
     return null
   }
+
+  const parseRole = user => {
+    if (user.isAdminOrGlobalBuilder) {
+      return Constants.Roles.CREATOR
+    }
+    return user.role
+  }
 </script>
 
 <svelte:window on:keydown={handleKeyDown} />
@@ -725,7 +732,7 @@
                     <RoleSelect
                       footer={getRoleFooter(user)}
                       placeholder={false}
-                      value={user.role}
+                      value={parseRole(user)}
                       allowRemove={user.role && !user.group}
                       allowPublic={false}
                       allowCreator={true}
@@ -744,7 +751,7 @@
                       autoWidth
                       align="right"
                       allowedRoles={user.isAdminOrGlobalBuilder
-                        ? [Constants.Roles.ADMIN]
+                        ? [Constants.Roles.CREATOR]
                         : null}
                     />
                   </div>

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Tables/CreateExternalTableModal.svelte

@@ -2,6 +2,7 @@
   import { ModalContent, Body, Input, notifications } from "@budibase/bbui"
   import { tables, datasources } from "stores/backend"
   import { goto } from "@roxi/routify"
+  import { DB_TYPE_EXTERNAL } from "constants/backend"
 
   export let datasource
 
@@ -16,9 +17,10 @@
   function buildDefaultTable(tableName, datasourceId) {
     return {
       name: tableName,
-      type: "external",
+      type: "table",
       primary: ["id"],
       sourceId: datasourceId,
+      sourceType: DB_TYPE_EXTERNAL,
       schema: {
         id: {
           autocolumn: true,

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/index.svelte

@@ -5,7 +5,7 @@
   import { tables, datasources } from "stores/backend"
   import { goto } from "@roxi/routify"
   import { onMount } from "svelte"
-  import { BUDIBASE_INTERNAL_DB_ID } from "constants/backend"
+  import { BUDIBASE_INTERNAL_DB_ID, DB_TYPE_EXTERNAL } from "constants/backend"
   import { TableNames } from "constants"
   import { store } from "builderStore"
 
@@ -14,7 +14,7 @@
   $: store.actions.websocket.selectResource(BUDIBASE_INTERNAL_DB_ID)
   $: internalTablesBySourceId = $tables.list.filter(
     table =>
-      table.type !== "external" &&
+      table.sourceType !== DB_TYPE_EXTERNAL &&
       table.sourceId === BUDIBASE_INTERNAL_DB_ID &&
       table._id !== TableNames.USERS
   )

packages/builder/src/pages/builder/app/[application]/data/datasource/datasource_internal_bb_default/index.svelte

@@ -4,7 +4,7 @@
   import ICONS from "components/backend/DatasourceNavigator/icons"
   import { tables, datasources } from "stores/backend"
   import { goto } from "@roxi/routify"
-  import { DEFAULT_BB_DATASOURCE_ID } from "constants/backend"
+  import { DEFAULT_BB_DATASOURCE_ID, DB_TYPE_EXTERNAL } from "constants/backend"
   import { onMount } from "svelte"
   import { store } from "builderStore"
 
@@ -13,7 +13,8 @@
   $: store.actions.websocket.selectResource(DEFAULT_BB_DATASOURCE_ID)
   $: internalTablesBySourceId = $tables.list.filter(
     table =>
-      table.type !== "external" && table.sourceId === DEFAULT_BB_DATASOURCE_ID
+      table.sourceType !== DB_TYPE_EXTERNAL &&
+      table.sourceId === DEFAULT_BB_DATASOURCE_ID
   )
 
   onMount(() => {

packages/builder/src/pages/builder/portal/users/users/index.svelte

@@ -3,7 +3,6 @@
     Heading,
     Body,
     Button,
-    ButtonGroup,
     Table,
     Layout,
     Modal,
@@ -46,6 +45,10 @@
     datasource: {
       type: "user",
     },
+    options: {
+      paginate: true,
+      limit: 10,
+    },
   })
 
   let groupsLoaded = !$licensing.groupsEnabled || $groups?.length
@@ -65,10 +68,12 @@
     { column: "role", component: RoleTableRenderer },
   ]
   let userData = []
+  let invitesLoaded = false
+  let pendingInvites = []
+  let parsedInvites = []
 
   $: isOwner = $auth.accountPortalAccess && $admin.cloud
   $: readonly = !sdk.users.isAdmin($auth.user) || $features.isScimEnabled
-
   $: debouncedUpdateFetch(searchEmail)
   $: schema = {
     email: {
@@ -88,16 +93,6 @@
       width: "1fr",
     },
   }
-
-  const getPendingSchema = tblSchema => {
-    if (!tblSchema) {
-      return {}
-    }
-    let pendingSchema = JSON.parse(JSON.stringify(tblSchema))
-    pendingSchema.email.displayName = "Pending Invites"
-    return pendingSchema
-  }
-
   $: pendingSchema = getPendingSchema(schema)
   $: userData = []
   $: inviteUsersResponse = { successful: [], unsuccessful: [] }
@@ -121,9 +116,15 @@
       }
     })
   }
-  let invitesLoaded = false
+
-  let pendingInvites = []
+  const getPendingSchema = tblSchema => {
-  let parsedInvites = []
+    if (!tblSchema) {
+      return {}
+    }
+    let pendingSchema = JSON.parse(JSON.stringify(tblSchema))
+    pendingSchema.email.displayName = "Pending Invites"
+    return pendingSchema
+  }
 
   const invitesToSchema = invites => {
     return invites.map(invite => {
@@ -143,8 +144,10 @@
   const updateFetch = email => {
     fetch.update({
       query: {
+        string: {
           email,
         },
+      },
     })
   }
   const debouncedUpdateFetch = Utils.debounce(updateFetch, 250)
@@ -296,7 +299,7 @@
   {/if}
   <div class="controls">
     {#if !readonly}
-      <ButtonGroup>
+      <div class="buttons">
         <Button
           disabled={readonly}
           on:click={$licensing.userLimitReached
@@ -315,7 +318,7 @@
         >
           Import
         </Button>
-      </ButtonGroup>
+      </div>
     {:else}
       <ScimBanner />
     {/if}
@@ -390,12 +393,15 @@
 </Modal>
 
 <style>
+  .buttons {
+    display: flex;
+    gap: 10px;
+  }
   .pagination {
     display: flex;
     flex-direction: row;
     justify-content: flex-end;
   }
-
   .controls {
     display: flex;
     flex-direction: row;
@@ -403,7 +409,6 @@
     align-items: center;
     gap: var(--spacing-xl);
   }
-
   .controls-right {
     display: flex;
     flex-direction: row;
@@ -411,7 +416,6 @@
     align-items: center;
     gap: var(--spacing-xl);
   }
-
   .controls-right :global(.spectrum-Search) {
     width: 200px;
   }

packages/client/src/utils/buttonActions.js

@@ -103,7 +103,6 @@ const fetchRowHandler = async action => {
 
 const deleteRowHandler = async action => {
   const { tableId, rowId: rowConfig, notificationOverride } = action.parameters
-
   if (tableId && rowConfig) {
     try {
       let requestConfig
@@ -129,9 +128,11 @@ const deleteRowHandler = async action => {
         requestConfig = [parsedRowConfig]
       } else if (Array.isArray(parsedRowConfig)) {
         requestConfig = parsedRowConfig
+      } else if (Number.isInteger(parsedRowConfig)) {
+        requestConfig = [String(parsedRowConfig)]
       }
 
-      if (!requestConfig.length) {
+      if (!requestConfig && !parsedRowConfig) {
         notificationStore.actions.warning("No valid rows were supplied")
         return false
       }

packages/frontend-core/src/api/tables.js

@@ -140,4 +140,13 @@ export const buildTableEndpoints = API => ({
       },
     })
   },
+  migrateColumn: async ({ tableId, oldColumn, newColumn }) => {
+    return await API.post({
+      url: `/api/tables/${tableId}/migrate`,
+      body: {
+        oldColumn,
+        newColumn,
+      },
+    })
+  },
 })

packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte

@@ -55,7 +55,7 @@
     try {
       return await API.uploadBuilderAttachment(data)
     } catch (error) {
-      $notifications.error("Failed to upload attachment")
+      $notifications.error(error.message || "Failed to upload attachment")
       return []
     }
   }

packages/frontend-core/src/components/grid/cells/HeaderCell.svelte

@@ -1,11 +1,20 @@
 <script>
   import { getContext, onMount, tick } from "svelte"
   import { canBeDisplayColumn, canBeSortColumn } from "@budibase/shared-core"
-  import { Icon, Popover, Menu, MenuItem, clickOutside } from "@budibase/bbui"
+  import {
+    Icon,
+    Popover,
+    Menu,
+    MenuItem,
+    clickOutside,
+    Modal,
+  } from "@budibase/bbui"
   import GridCell from "./GridCell.svelte"
   import { getColumnIcon } from "../lib/utils"
+  import MigrationModal from "../controls/MigrationModal.svelte"
   import { debounce } from "../../../utils/utils"
   import { FieldType, FormulaTypes } from "@budibase/types"
+  import { TableNames } from "../../../constants"
 
   export let column
   export let idx
@@ -45,6 +54,7 @@
   let editIsOpen = false
   let timeout
   let popover
+  let migrationModal
   let searchValue
   let input
 
@@ -189,6 +199,11 @@
     })
   }
 
+  const openMigrationModal = () => {
+    migrationModal.show()
+    open = false
+  }
+
   const startSearching = async () => {
     $focusedCellId = null
     searchValue = ""
@@ -224,6 +239,10 @@
   onMount(() => subscribe("close-edit-column", cancelEdit))
 </script>
 
+<Modal bind:this={migrationModal}>
+  <MigrationModal {column} />
+</Modal>
+
 <div
   class="header-cell"
   class:open
@@ -363,6 +382,11 @@
       >
         Hide column
       </MenuItem>
+      {#if $config.canEditColumns && column.schema.type === "link" && column.schema.tableId === TableNames.USERS}
+        <MenuItem icon="User" on:click={openMigrationModal}>
+          Migrate to user column
+        </MenuItem>
+      {/if}
     </Menu>
   {/if}
 </Popover>

packages/frontend-core/src/components/grid/controls/MigrationModal.svelte

@@ -0,0 +1,73 @@
+<script>
+  import {
+    ModalContent,
+    notifications,
+    Input,
+    InlineAlert,
+  } from "@budibase/bbui"
+  import { getContext } from "svelte"
+  import { ValidColumnNameRegex } from "@budibase/shared-core"
+  import { FieldSubtype, FieldType, RelationshipType } from "@budibase/types"
+
+  const { API, definition, rows } = getContext("grid")
+
+  export let column
+
+  let newColumnName = `${column.schema.name} migrated`
+  $: error = checkNewColumnName(newColumnName)
+
+  const checkNewColumnName = newColumnName => {
+    if (newColumnName === "") {
+      return "Column name can't be empty."
+    }
+    if (newColumnName in $definition.schema) {
+      return "New column name can't be the same as an existing column name."
+    }
+    if (newColumnName.match(ValidColumnNameRegex) === null) {
+      return "Illegal character; must be alpha-numeric."
+    }
+  }
+
+  const migrateUserColumn = async () => {
+    let subtype = FieldSubtype.USERS
+    if (column.schema.relationshipType === RelationshipType.ONE_TO_MANY) {
+      subtype = FieldSubtype.USER
+    }
+
+    try {
+      await API.migrateColumn({
+        tableId: $definition._id,
+        oldColumn: column.schema,
+        newColumn: {
+          name: newColumnName,
+          type: FieldType.BB_REFERENCE,
+          subtype,
+        },
+      })
+      notifications.success("Column migrated")
+    } catch (e) {
+      notifications.error(`Failed to migrate: ${e.message}`)
+    }
+    await rows.actions.refreshData()
+  }
+</script>
+
+<ModalContent
+  title="Migrate column"
+  confirmText="Continue"
+  cancelText="Cancel"
+  onConfirm={migrateUserColumn}
+  disabled={error !== undefined}
+  size="M"
+>
+  This operation will kick off a migration of the column "{column.schema.name}"
+  to a new column, with the name provided - this operation may take a moment to
+  complete.
+
+  <InlineAlert
+    type="error"
+    header="Are you sure?"
+    message="This will leave bindings which utilised the user relationship column in a state where they will need to be updated to use the new column instead."
+  />
+  <Input bind:value={newColumnName} label="New column name" {error} />
+</ModalContent>

packages/pro

@@ -1 +1 @@
-Subproject commit 5ed0ee2aca9d754d80cd46bae412b24621afa47e
+Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8

packages/server/Dockerfile

@@ -38,7 +38,7 @@ RUN apt update && apt upgrade -y \
 
 COPY package.json .
 COPY dist/yarn.lock .
-RUN yarn install --production=true \
+RUN yarn install --production=true --network-timeout 1000000 \
     # Remove unneeded data from file system to reduce image size
     && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \
     && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp

packages/server/__mocks__/aws-sdk.ts

@@ -70,6 +70,13 @@ module AwsMock {
         Contents: {},
       })
     )
+
+    // @ts-ignore
+    this.getObject = jest.fn(
+      response({
+        Body: "",
+      })
+    )
   }
 
   aws.DynamoDB = { DocumentClient }

packages/server/package.json

@@ -18,7 +18,6 @@
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "build:docker": "yarn build && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",

packages/server/scripts/integrations/postgres/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.8"
 services:
   db:
     container_name: postgres
-    image: postgres:15
+    image: postgres:15-bullseye
     restart: unless-stopped
     environment:
       POSTGRES_USER: root

packages/server/src/api/controllers/application.ts

@@ -32,11 +32,8 @@ import {
   tenancy,
   users,
 } from "@budibase/backend-core"
-import { USERS_TABLE_SCHEMA } from "../../constants"
+import { USERS_TABLE_SCHEMA, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
-import {
+import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
-  buildDefaultDocs,
-  DEFAULT_BB_DATASOURCE_ID,
-} from "../../db/defaultData/datasource_bb_default"
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
 import { stringToReadStream } from "../../utilities"
 import { doesUserHaveLock } from "../../utilities/redis"

packages/server/src/api/controllers/datasource.ts

@@ -12,7 +12,6 @@ import {
   CreateDatasourceResponse,
   Datasource,
   DatasourcePlus,
-  ExternalTable,
   FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
   IntegrationBase,
@@ -59,7 +58,7 @@ async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
   const connector = (await getConnector(datasource)) as DatasourcePlus
   return await connector.buildSchema(
     datasource._id!,
-    datasource.entities! as Record<string, ExternalTable>
+    datasource.entities! as Record<string, Table>
   )
 }
 

packages/server/src/api/controllers/public/utils.ts

@@ -1,12 +1,12 @@
 import { context } from "@budibase/backend-core"
-import { isExternalTable } from "../../../integrations/utils"
+import { isExternalTableID } from "../../../integrations/utils"
 import { APP_PREFIX, DocumentType } from "../../../db/utils"
 
 export async function addRev(
   body: { _id?: string; _rev?: string },
   tableId?: string
 ) {
-  if (!body._id || (tableId && isExternalTable(tableId))) {
+  if (!body._id || (tableId && isExternalTableID(tableId))) {
     return body
   }
   let id = body._id

packages/server/src/api/controllers/row/index.ts

@@ -1,7 +1,7 @@
 import { quotas } from "@budibase/pro"
 import * as internal from "./internal"
 import * as external from "./external"
-import { isExternalTable } from "../../../integrations/utils"
+import { isExternalTableID } from "../../../integrations/utils"
 import {
   Ctx,
   UserCtx,
@@ -30,7 +30,7 @@ import { Format } from "../view/exporters"
 export * as views from "./views"
 
 function pickApi(tableId: any) {
-  if (isExternalTable(tableId)) {
+  if (isExternalTableID(tableId)) {
     return external
   }
   return internal
@@ -227,7 +227,7 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
 export async function validate(ctx: Ctx<Row, ValidateResponse>) {
   const tableId = utils.getTableId(ctx)
   // external tables are hard to validate currently
-  if (isExternalTable(tableId)) {
+  if (isExternalTableID(tableId)) {
     ctx.body = { valid: true, errors: {} }
   } else {
     ctx.body = await sdk.rows.utils.validate({

packages/server/src/api/controllers/static/index.ts

@@ -1,3 +1,5 @@
+import { ValidFileExtensions } from "@budibase/shared-core"
+
 require("svelte/register")
 
 import { join } from "../../../utilities/centralPath"
@@ -11,34 +13,21 @@ import {
 } from "../../../utilities/fileSystem"
 import env from "../../../environment"
 import { DocumentType } from "../../../db/utils"
-import { context, objectStore, utils, configs } from "@budibase/backend-core"
+import {
+  context,
+  objectStore,
+  utils,
+  configs,
+  BadRequestError,
+} from "@budibase/backend-core"
 import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
-import { App, Ctx } from "@budibase/types"
+import { App, Ctx, ProcessAttachmentResponse, Upload } from "@budibase/types"
 
 const send = require("koa-send")
 
-async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
-  const response = await objectStore.upload({
-    bucket,
-    metadata,
-    filename: s3Key,
-    path: file.path,
-    type: file.type,
-  })
-
-  // don't store a URL, work this out on the way out as the URL could change
-  return {
-    size: file.size,
-    name: file.name,
-    url: objectStore.getAppFileUrl(s3Key),
-    extension: [...file.name.split(".")].pop(),
-    key: response.Key,
-  }
-}
-
 export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
 
@@ -72,23 +61,58 @@ export const serveBuilder = async function (ctx: Ctx) {
   await send(ctx, ctx.file, { root: builderPath })
 }
 
-export const uploadFile = async function (ctx: Ctx) {
+export const uploadFile = async function (
+  ctx: Ctx<{}, ProcessAttachmentResponse>
+) {
   const file = ctx.request?.files?.file
+  if (!file) {
+    throw new BadRequestError("No file provided")
+  }
+
   let files = file && Array.isArray(file) ? Array.from(file) : [file]
 
-  const uploads = files.map(async (file: any) => {
+  ctx.body = await Promise.all(
-    const fileExtension = [...file.name.split(".")].pop()
+    files.map(async file => {
+      if (!file.name) {
+        throw new BadRequestError(
+          "Attempted to upload a file without a filename"
+        )
+      }
+
+      const extension = [...file.name.split(".")].pop()
+      if (!extension) {
+        throw new BadRequestError(
+          `File "${file.name}" has no extension, an extension is required to upload a file`
+        )
+      }
+
+      if (!env.SELF_HOSTED && !ValidFileExtensions.includes(extension)) {
+        throw new BadRequestError(
+          `File "${file.name}" has an invalid extension: "${extension}"`
+        )
+      }
+
       // filenames converted to UUIDs so they are unique
-    const processedFileName = `${uuid.v4()}.${fileExtension}`
+      const processedFileName = `${uuid.v4()}.${extension}`
 
-    return prepareUpload({
+      const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
-      file,
+
-      s3Key: `${context.getProdAppId()}/attachments/${processedFileName}`,
+      const response = await objectStore.upload({
         bucket: ObjectStoreBuckets.APPS,
-    })
+        filename: s3Key,
+        path: file.path,
+        type: file.type,
       })
 
-  ctx.body = await Promise.all(uploads)
+      return {
+        size: file.size,
+        name: file.name,
+        url: objectStore.getAppFileUrl(s3Key),
+        extension,
+        key: response.Key,
+      }
+    })
+  )
 }
 
 export const deleteObjects = async function (ctx: Ctx) {

packages/server/src/api/controllers/table/index.ts

@@ -5,18 +5,27 @@ import {
   isSchema,
   validate as validateSchema,
 } from "../../../utilities/schema"
-import { isExternalTable, isSQL } from "../../../integrations/utils"
+import {
+  isExternalTable,
+  isExternalTableID,
+  isSQL,
+} from "../../../integrations/utils"
 import { events } from "@budibase/backend-core"
 import {
   BulkImportRequest,
   BulkImportResponse,
+  DocumentType,
   FetchTablesResponse,
+  MigrateRequest,
+  MigrateResponse,
+  Row,
   SaveTableRequest,
   SaveTableResponse,
   Table,
   TableResponse,
+  TableSourceType,
   UserCtx,
-  Row,
+  SEPARATOR,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
@@ -24,12 +33,10 @@ import { builderSocket } from "../../../websockets"
 import { cloneDeep, isEqual } from "lodash"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
-  if (table && !tableId) {
+  if (table && isExternalTable(table)) {
-    tableId = table._id
-  }
-  if (table && table.type === "external") {
     return external
-  } else if (tableId && isExternalTable(tableId)) {
+  }
+  if (tableId && isExternalTableID(tableId)) {
     return external
   }
   return internal
@@ -46,8 +53,8 @@ export async function fetch(ctx: UserCtx<void, FetchTablesResponse>) {
     if (entities) {
       return Object.values(entities).map<Table>((entity: Table) => ({
         ...entity,
-        type: "external",
+        sourceType: TableSourceType.EXTERNAL,
-        sourceId: datasource._id,
+        sourceId: datasource._id!,
         sql: isSQL(datasource),
       }))
     } else {
@@ -158,3 +165,19 @@ export async function validateExistingTableImport(ctx: UserCtx) {
     ctx.status = 422
   }
 }
+
+export async function migrate(ctx: UserCtx<MigrateRequest, MigrateResponse>) {
+  const { oldColumn, newColumn } = ctx.request.body
+  let tableId = ctx.params.tableId as string
+  const table = await sdk.tables.getTable(tableId)
+  let result = await sdk.tables.migrate(table, oldColumn, newColumn)
+
+  for (let table of result.tablesUpdated) {
+    builderSocket?.emitTableUpdate(ctx, table, {
+      includeOriginator: true,
+    })
+  }
+
+  ctx.status = 200
+  ctx.body = { message: `Column ${oldColumn.name} migrated.` }
+}

packages/server/src/api/controllers/table/internal.ts

@@ -7,6 +7,7 @@ import {
   SaveTableRequest,
   SaveTableResponse,
   Table,
+  TableSourceType,
   UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
@@ -16,10 +17,11 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   let tableToSave: Table & {
     _rename?: RenameColumn
   } = {
-    type: "table",
     _id: generateTableID(),
-    views: {},
     ...rest,
+    type: "table",
+    sourceType: TableSourceType.INTERNAL,
+    views: {},
   }
   const renaming = tableToSave._rename
   delete tableToSave._rename

packages/server/src/api/routes/row.ts

@@ -11,128 +11,24 @@ const { PermissionType, PermissionLevel } = permissions
 const router: Router = new Router()
 
 router
-  /**
-   * @api {get} /api/:sourceId/:rowId/enrich Get an enriched row
-   * @apiName Get an enriched row
-   * @apiGroup rows
-   * @apiPermission table read access
-   * @apiDescription This API is only useful when dealing with rows that have relationships.
-   * Normally when a row is a returned from the API relationships will only have the structure
-   * `{ primaryDisplay: "name", _id: ... }` but this call will return the full related rows
-   * for each relationship instead.
-   *
-   * @apiParam {string} rowId The ID of the row which is to be retrieved and enriched.
-   *
-   * @apiSuccess {object} row The response body will be the enriched row.
-   */
   .get(
     "/api/:sourceId/:rowId/enrich",
     paramSubResource("sourceId", "rowId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.fetchEnrichedRow
   )
-  /**
-   * @api {get} /api/:sourceId/rows Get all rows in a table
-   * @apiName Get all rows in a table
-   * @apiGroup rows
-   * @apiPermission table read access
-   * @apiDescription This is a deprecated endpoint that should not be used anymore, instead use the search endpoint.
-   * This endpoint gets all of the rows within the specified table - it is not heavily used
-   * due to its lack of support for pagination. With SQL tables this will retrieve up to a limit and then
-   * will simply stop.
-   *
-   * @apiParam {string} sourceId The ID of the table to retrieve all rows within.
-   *
-   * @apiSuccess {object[]} rows The response body will be an array of all rows found.
-   */
   .get(
     "/api/:sourceId/rows",
     paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.fetch
   )
-  /**
-   * @api {get} /api/:sourceId/rows/:rowId Retrieve a single row
-   * @apiName Retrieve a single row
-   * @apiGroup rows
-   * @apiPermission table read access
-   * @apiDescription This endpoint retrieves only the specified row. If you wish to retrieve
-   * a row by anything other than its _id field, use the search endpoint.
-   *
-   * @apiParam {string} sourceId The ID of the table to retrieve a row from.
-   * @apiParam {string} rowId The ID of the row to retrieve.
-   *
-   * @apiSuccess {object} body The response body will be the row that was found.
-   */
   .get(
     "/api/:sourceId/rows/:rowId",
     paramSubResource("sourceId", "rowId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.find
   )
-  /**
-   * @api {post} /api/:sourceId/search Search for rows in a table
-   * @apiName Search for rows in a table
-   * @apiGroup rows
-   * @apiPermission table read access
-   * @apiDescription This is the primary method of accessing rows in Budibase, the data provider
-   * and data UI in the builder are built atop this. All filtering, sorting and pagination is
-   * handled through this, for internal and external (datasource plus, e.g. SQL) tables.
-   *
-   * @apiParam {string} sourceId The ID of the table to retrieve rows from.
-   *
-   * @apiParam (Body) {boolean} [paginate] If pagination is required then this should be set to true,
-   * defaults to false.
-   * @apiParam (Body) {object} [query] This contains a set of filters which should be applied, if none
-   * specified then the request will be unfiltered. An example with all of the possible query
-   * options has been supplied below.
-   * @apiParam (Body) {number} [limit] This sets a limit for the number of rows that will be returned,
-   * this will be implemented at the database level if supported for performance reasons. This
-   * is useful when paginating to set exactly how many rows per page.
-   * @apiParam (Body) {string} [bookmark] If pagination is enabled then a bookmark will be returned
-   * with each successful search request, this should be supplied back to get the next page.
-   * @apiParam (Body) {object} [sort] If sort is desired this should contain the name of the column to
-   * sort on.
-   * @apiParam (Body) {string} [sortOrder] If sort is enabled then this can be either "descending" or
-   * "ascending" as required.
-   * @apiParam (Body) {string} [sortType] If sort is enabled then you must specify the type of search
-   * being used, either "string" or "number". This is only used for internal tables.
-   *
-   * @apiParamExample {json} Example:
-   * {
-   *  "tableId": "ta_70260ff0b85c467ca74364aefc46f26d",
-   *  "query": {
-   *    "string": {},
-   *    "fuzzy": {},
-   *    "range": {
-   *      "columnName": {
-   *        "high": 20,
-   *        "low": 10,
-   *      }
-   *    },
-   *    "equal": {
-   *      "columnName": "someValue"
-   *    },
-   *    "notEqual": {},
-   *    "empty": {},
-   *    "notEmpty": {},
-   *    "oneOf": {
-   *      "columnName": ["value"]
-   *    }
-   *  },
-   *  "limit": 10,
-   *  "sort": "name",
-   *  "sortOrder": "descending",
-   *  "sortType": "string",
-   *  "paginate": true
-   * }
-   *
-   * @apiSuccess {object[]} rows An array of rows that was found based on the supplied parameters.
-   * @apiSuccess {boolean} hasNextPage If pagination was enabled then this specifies whether or
-   * not there is another page after this request.
-   * @apiSuccess {string} bookmark The bookmark to be sent with the next request to get the next
-   * page.
-   */
   .post(
     "/api/:sourceId/search",
     internalSearchValidator(),
@@ -148,30 +44,6 @@ router
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.search
   )
-  /**
-   * @api {post} /api/:sourceId/rows Creates a new row
-   * @apiName Creates a new row
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This API will create a new row based on the supplied body. If the
-   * body includes an "_id" field then it will update an existing row if the field
-   * links to one. Please note that "_id", "_rev" and "tableId" are fields that are
-   * already used by Budibase tables and cannot be used for columns.
-   *
-   * @apiParam {string} sourceId The ID of the table to save a row to.
-   *
-   * @apiParam (Body) {string} [_id] If the row exists already then an ID for the row must be provided.
-   * @apiParam (Body) {string} [_rev] If working with an existing row for an internal table its revision
-   * must also be provided.
-   * @apiParam (Body) {string} tableId The ID of the table should also be specified in the row body itself.
-   * @apiParam (Body) {any} [any] Any field supplied in the body will be assessed to see if it matches
-   * a column in the specified table. All other fields will be dropped and not stored.
-   *
-   * @apiSuccess {string} _id The ID of the row that was just saved, if it was just created this
-   * is the rows new ID.
-   * @apiSuccess {string} [_rev] If saving to an internal table a revision will also be returned.
-   * @apiSuccess {object} body The contents of the row that was saved will be returned as well.
-   */
   .post(
     "/api/:sourceId/rows",
     paramResource("sourceId"),
@@ -179,14 +51,6 @@ router
     trimViewRowInfo,
     rowController.save
   )
-  /**
-   * @api {patch} /api/:sourceId/rows Updates a row
-   * @apiName Update a row
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This endpoint is identical to the row creation endpoint but instead it will
-   * error if an _id isn't provided, it will only function for existing rows.
-   */
   .patch(
     "/api/:sourceId/rows",
     paramResource("sourceId"),
@@ -194,52 +58,12 @@ router
     trimViewRowInfo,
     rowController.patch
   )
-  /**
-   * @api {post} /api/:sourceId/rows/validate Validate inputs for a row
-   * @apiName Validate inputs for a row
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription When attempting to save a row you may want to check if the row is valid
-   * given the table schema, this will iterate through all the constraints on the table and
-   * check if the request body is valid.
-   *
-   * @apiParam {string} sourceId The ID of the table the row is to be validated for.
-   *
-   * @apiParam (Body) {any} [any] Any fields provided in the request body will be tested
-   * against the table schema and constraints.
-   *
-   * @apiSuccess {boolean} valid If inputs provided are acceptable within the table schema this
-   * will be true, if it is not then then errors property will be populated.
-   * @apiSuccess {object} [errors] A key value map of information about fields on the input
-   * which do not match the table schema. The key name will be the column names that have breached
-   * the schema.
-   */
   .post(
     "/api/:sourceId/rows/validate",
     paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
     rowController.validate
   )
-  /**
-   * @api {delete} /api/:sourceId/rows Delete rows
-   * @apiName Delete rows
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This endpoint can delete a single row, or delete them in a bulk
-   * fashion.
-   *
-   * @apiParam {string} sourceId The ID of the table the row is to be deleted from.
-   *
-   * @apiParam (Body) {object[]} [rows] If bulk deletion is desired then provide the rows in this
-   * key of the request body that are to be deleted.
-   * @apiParam (Body) {string} [_id] If deleting a single row then provide its ID in this field.
-   * @apiParam (Body) {string} [_rev] If deleting a single row from an internal table then provide its
-   * revision here.
-   *
-   * @apiSuccess {object[]|object} body If deleting bulk then the response body will be an array
-   * of the deleted rows, if deleting a single row then the body will contain a "row" property which
-   * is the deleted row.
-   */
   .delete(
     "/api/:sourceId/rows",
     paramResource("sourceId"),
@@ -247,20 +71,6 @@ router
     trimViewRowInfo,
     rowController.destroy
   )
-
-  /**
-   * @api {post} /api/:sourceId/rows/exportRows Export Rows
-   * @apiName Export rows
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This API can export a number of provided rows
-   *
-   * @apiParam {string} sourceId The ID of the table the row is to be deleted from.
-   *
-   * @apiParam (Body) {object[]} [rows] The row IDs which are to be exported
-   *
-   * @apiSuccess {object[]|object}
-   */
   .post(
     "/api/:sourceId/rows/exportRows",
     paramResource("sourceId"),

packages/server/src/api/routes/table.ts

@@ -9,99 +9,13 @@ const { BUILDER, PermissionLevel, PermissionType } = permissions
 const router: Router = new Router()
 
 router
-  /**
-   * @api {get} /api/tables Fetch all tables
-   * @apiName Fetch all tables
-   * @apiGroup tables
-   * @apiPermission table read access
-   * @apiDescription This endpoint retrieves all of the tables which have been created in
-   * an app. This includes all of the external and internal tables; to tell the difference
-   * between these look for the "type" property on each table, either being "internal" or "external".
-   *
-   * @apiSuccess {object[]} body The response body will be the list of tables that was found - as
-   * this does not take any parameters the only error scenario is no access.
-   */
   .get("/api/tables", authorized(BUILDER), tableController.fetch)
-  /**
-   * @api {get} /api/tables/:id Fetch a single table
-   * @apiName Fetch a single table
-   * @apiGroup tables
-   * @apiPermission table read access
-   * @apiDescription Retrieves a single table this could be be internal or external based on
-   * the provided table ID.
-   *
-   * @apiParam {string} id The ID of the table which is to be retrieved.
-   *
-   * @apiSuccess {object[]} body The response body will be the table that was found.
-   */
   .get(
     "/api/tables/:tableId",
     paramResource("tableId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ, { schema: true }),
     tableController.find
   )
-  /**
-   * @api {post} /api/tables Save a table
-   * @apiName Save a table
-   * @apiGroup tables
-   * @apiPermission builder
-   * @apiDescription Create or update a table with this endpoint, this will function for both internal
-   * external tables.
-   *
-   * @apiParam (Body) {string} [_id] If updating an existing table then the ID of the table must be specified.
-   * @apiParam (Body) {string} [_rev] If updating an existing internal table then the revision must also be specified.
-   * @apiParam (Body) {string} type] This should either be "internal" or "external" depending on the table type -
-   * this will default to internal.
-   * @apiParam (Body) {string} [sourceId] If creating an external table then this should be set to the datasource ID. If
-   * building an internal table this does not need to be set, although it will be returned as "bb_internal".
-   * @apiParam (Body) {string} name The name of the table, this will be used in the UI. To rename the table simply
-   * supply the table structure to this endpoint with the name changed.
-   * @apiParam (Body) {object} schema A key value object which has all of the columns in the table as the keys in this
-   * object. For each column a "type" and "constraints" must be specified, with some types requiring further information.
-   * More information about the schema structure can be found in the Typescript definitions.
-   * @apiParam (Body) {string} [primaryDisplay] The name of the column which should be used when displaying rows
-   * from this table as relationships.
-   * @apiParam (Body) {object[]} [indexes] Specifies the search indexes - this is deprecated behaviour with the introduction
-   * of lucene indexes. This functionality is only available for internal tables.
-   * @apiParam (Body) {object} [_rename] If a column is to be renamed then the "old" column name should be set in this
-   * structure, and the "updated", new column name should also be supplied. The schema should also be updated, this field
-   * lets the server know that a field hasn't just been deleted, that the data has moved to a new name, this will fix
-   * the rows in the table. This functionality is only available for internal tables.
-   * @apiParam (Body) {object[]} [rows] When creating a table using a compatible data source, an array of objects to be imported into the new table can be provided.
-   *
-   * @apiParamExample {json} Example:
-   * {
-   *   "_id": "ta_05541307fa0f4044abee071ca2a82119",
-   *   "_rev": "10-0fbe4e78f69b255d79f1017e2eeef807",
-   *   "type": "internal",
-   *   "views": {},
-   *   "name": "tableName",
-   *   "schema": {
-   *     "column": {
-   *       "type": "string",
-   *       "constraints": {
-   *         "type": "string",
-   *         "length": {
-   *           "maximum": null
-   *         },
-   *         "presence": false
-   *       },
-   *       "name": "column"
-   *     },
-   *   },
-   *   "primaryDisplay": "column",
-   *   "indexes": [],
-   *   "sourceId": "bb_internal",
-   *   "_rename": {
-   *     "old": "columnName",
-   *     "updated": "newColumnName",
-   *   },
-   *   "rows": []
-   * }
-   *
-   * @apiSuccess {object} table The response body will contain the table structure after being cleaned up and
-   * saved to the database.
-   */
   .post(
     "/api/tables",
     // allows control over updating a table
@@ -125,41 +39,12 @@ router
     authorized(BUILDER),
     tableController.validateExistingTableImport
   )
-  /**
-   * @api {post} /api/tables/:tableId/:revId Delete a table
-   * @apiName Delete a table
-   * @apiGroup tables
-   * @apiPermission builder
-   * @apiDescription This endpoint will delete a table and all of its associated data, for this reason it is
-   * quite dangerous - it will work for internal and external tables.
-   *
-   * @apiParam {string} tableId The ID of the table which is to be deleted.
-   * @apiParam {string} [revId] If deleting an internal table then the revision must also be supplied (_rev), for
-   * external tables this can simply be set to anything, e.g. "external".
-   *
-   * @apiSuccess {string} message A message stating that the table was deleted successfully.
-   */
   .delete(
     "/api/tables/:tableId/:revId",
     paramResource("tableId"),
     authorized(BUILDER),
     tableController.destroy
   )
-  /**
-   * @api {post} /api/tables/:tableId/:revId Import CSV to existing table
-   * @apiName Import CSV to existing table
-   * @apiGroup tables
-   * @apiPermission builder
-   * @apiDescription This endpoint will import data to existing tables, internal or external. It is used in combination
-   * with the CSV validation endpoint. Take the output of the CSV validation endpoint and pass it to this endpoint to
-   * import the data; please note this will only import fields that already exist on the table/match the type.
-   *
-   * @apiParam {string} tableId The ID of the table which the data should be imported to.
-   *
-   * @apiParam (Body) {object[]} rows An array of objects representing the rows to be imported, key-value pairs not matching the table schema will be ignored.
-   *
-   * @apiSuccess {string} message A message stating that the data was imported successfully.
-   */
   .post(
     "/api/tables/:tableId/import",
     paramResource("tableId"),
@@ -167,4 +52,11 @@ router
     tableController.bulkImport
   )
 
+  .post(
+    "/api/tables/:tableId/migrate",
+    paramResource("tableId"),
+    authorized(BUILDER),
+    tableController.migrate
+  )
+
 export default router

packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap

@@ -7,7 +7,7 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = `
     "entities": [
       {
         "_id": "ta_users",
-        "_rev": "1-2375e1bc58aeec664dc1b1f04ad43e44",
+        "_rev": "1-73b7912e6cbdd3d696febc60f3715844",
         "createdAt": "2020-01-01T00:00:00.000Z",
         "name": "Users",
         "primaryDisplay": "email",
@@ -21,7 +21,6 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = `
               "presence": true,
               "type": "string",
             },
-            "fieldName": "email",
             "name": "email",
             "type": "string",
           },
@@ -30,7 +29,6 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = `
               "presence": false,
               "type": "string",
             },
-            "fieldName": "firstName",
             "name": "firstName",
             "type": "string",
           },
@@ -39,7 +37,6 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = `
               "presence": false,
               "type": "string",
             },
-            "fieldName": "lastName",
             "name": "lastName",
             "type": "string",
           },
@@ -54,7 +51,6 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = `
               "presence": false,
               "type": "string",
             },
-            "fieldName": "roleId",
             "name": "roleId",
             "type": "options",
           },
@@ -67,11 +63,12 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = `
               "presence": false,
               "type": "string",
             },
-            "fieldName": "status",
             "name": "status",
             "type": "options",
           },
         },
+        "sourceId": "bb_internal",
+        "sourceType": "internal",
         "type": "table",
         "updatedAt": "2020-01-01T00:00:00.000Z",
         "views": {},

packages/server/src/api/routes/tests/attachment.spec.ts

@@ -0,0 +1,49 @@
+import * as setup from "./utilities"
+import { APIError } from "@budibase/types"
+
+describe("/api/applications/:appId/sync", () => {
+  let config = setup.getConfig()
+
+  afterAll(setup.afterAll)
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  describe("/api/attachments/process", () => {
+    it("should accept an image file upload", async () => {
+      let resp = await config.api.attachment.process(
+        "1px.jpg",
+        Buffer.from([0])
+      )
+      expect(resp.length).toBe(1)
+
+      let upload = resp[0]
+      expect(upload.url.endsWith(".jpg")).toBe(true)
+      expect(upload.extension).toBe("jpg")
+      expect(upload.size).toBe(1)
+      expect(upload.name).toBe("1px.jpg")
+    })
+
+    it("should reject an upload with a malicious file extension", async () => {
+      await config.withEnv({ SELF_HOSTED: undefined }, async () => {
+        let resp = (await config.api.attachment.process(
+          "ohno.exe",
+          Buffer.from([0]),
+          { expectStatus: 400 }
+        )) as unknown as APIError
+        expect(resp.message).toContain("invalid extension")
+      })
+    })
+
+    it("should reject an upload with no file", async () => {
+      let resp = (await config.api.attachment.process(
+        undefined as any,
+        undefined as any,
+        {
+          expectStatus: 400,
+        }
+      )) as unknown as APIError
+      expect(resp.message).toContain("No file provided")
+    })
+  })
+})

packages/server/src/api/routes/tests/backup.spec.ts

@@ -5,6 +5,8 @@ import sdk from "../../../sdk"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import { mocks } from "@budibase/backend-core/tests"
 
+mocks.licenses.useBackups()
+
 describe("/backups", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -12,16 +14,17 @@ describe("/backups", () => {
   afterAll(setup.afterAll)
 
   beforeEach(async () => {
+    tk.reset()
     await config.init()
   })
 
-  describe("exportAppDump", () => {
+  describe("/api/backups/export", () => {
     it("should be able to export app", async () => {
-      const res = await request
+      const { body, headers } = await config.api.backup.exportBasicBackup(
-        .post(`/api/backups/export?appId=${config.getAppId()}`)
+        config.getAppId()!
-        .set(config.defaultHeaders())
+      )
-        .expect(200)
+      expect(body instanceof Buffer).toBe(true)
-      expect(res.headers["content-type"]).toEqual("application/gzip")
+      expect(headers["content-type"]).toEqual("application/gzip")
       expect(events.app.exported).toBeCalledTimes(1)
     })
 
@@ -36,11 +39,11 @@ describe("/backups", () => {
     it("should infer the app name from the app", async () => {
       tk.freeze(mocks.date.MOCK_DATE)
 
-      const res = await request
+      const { headers } = await config.api.backup.exportBasicBackup(
-        .post(`/api/backups/export?appId=${config.getAppId()}`)
+        config.getAppId()!
-        .set(config.defaultHeaders())
+      )
 
-      expect(res.headers["content-disposition"]).toEqual(
+      expect(headers["content-disposition"]).toEqual(
         `attachment; filename="${
           config.getApp()!.name
         }-export-${mocks.date.MOCK_DATE.getTime()}.tar.gz"`
@@ -48,6 +51,21 @@ describe("/backups", () => {
     })
   })
 
+  describe("/api/backups/import", () => {
+    it("should be able to import an app", async () => {
+      const appId = config.getAppId()!
+      const automation = await config.createAutomation()
+      await config.createAutomationLog(automation, appId)
+      await config.createScreen()
+      const exportRes = await config.api.backup.createBackup(appId)
+      expect(exportRes.backupId).toBeDefined()
+      const importRes = await config.api.backup.importBackup(
+        appId,
+        exportRes.backupId
+      )
+    })
+  })
+
   describe("calculateBackupStats", () => {
     it("should be able to calculate the backup statistics", async () => {
       await config.createAutomation()

packages/server/src/api/routes/tests/row.spec.ts

@@ -10,6 +10,7 @@ import {
   FieldSchema,
   FieldType,
   FieldTypeSubtypes,
+  INTERNAL_TABLE_SOURCE_ID,
   MonthlyQuotaName,
   PermissionLevel,
   QuotaUsageType,
@@ -21,6 +22,7 @@ import {
   SortType,
   StaticQuotaName,
   Table,
+  TableSourceType,
 } from "@budibase/types"
 import {
   expectAnyExternalColsAttributes,
@@ -65,6 +67,8 @@ describe.each([
       type: "table",
       primary: ["id"],
       primaryDisplay: "name",
+      sourceType: TableSourceType.INTERNAL,
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         id: {
           type: FieldType.AUTO,
@@ -134,9 +138,22 @@ describe.each([
       }
     : undefined
 
+  async function createTable(
+    cfg: Omit<SaveTableRequest, "sourceId" | "sourceType">,
+    opts?: { skipReassigning: boolean }
+  ) {
+    let table
+    if (dsProvider) {
+      table = await config.createExternalTable(cfg, opts)
+    } else {
+      table = await config.createTable(cfg, opts)
+    }
+    return table
+  }
+
   beforeAll(async () => {
     const tableConfig = generateTableConfig()
-    const table = await config.createTable(tableConfig)
+    let table = await createTable(tableConfig)
     tableId = table._id!
   })
 
@@ -165,7 +182,7 @@ describe.each([
       const queryUsage = await getQueryUsage()
 
       const tableConfig = generateTableConfig()
-      const newTable = await config.createTable(
+      const newTable = await createTable(
         {
           ...tableConfig,
           name: "TestTableAuto",
@@ -242,7 +259,7 @@ describe.each([
     })
 
     it("should list all rows for given tableId", async () => {
-      const table = await config.createTable(generateTableConfig(), {
+      const table = await createTable(generateTableConfig(), {
         skipReassigning: true,
       })
       const tableId = table._id!
@@ -323,7 +340,7 @@ describe.each([
             inclusion: ["Alpha", "Beta", "Gamma"],
           },
         }
-        const table = await config.createTable({
+        const table = await createTable({
           name: "TestTable2",
           type: "table",
           schema: {
@@ -438,7 +455,8 @@ describe.each([
 
   describe("view save", () => {
     it("views have extra data trimmed", async () => {
-      const table = await config.createTable({
+      const table = await createTable({
+        type: "table",
         name: "orders",
         primary: ["OrderID"],
         schema: {
@@ -494,7 +512,7 @@ describe.each([
   describe("patch", () => {
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
-      table = await config.createTable(tableConfig)
+      table = await createTable(tableConfig)
     })
 
     it("should update only the fields that are supplied", async () => {
@@ -548,7 +566,7 @@ describe.each([
   describe("destroy", () => {
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
-      table = await config.createTable(tableConfig)
+      table = await createTable(tableConfig)
     })
 
     it("should be able to delete a row", async () => {
@@ -566,7 +584,7 @@ describe.each([
   describe("validate", () => {
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
-      table = await config.createTable(tableConfig)
+      table = await createTable(tableConfig)
     })
 
     it("should return no errors on valid row", async () => {
@@ -603,7 +621,7 @@ describe.each([
   describe("bulkDelete", () => {
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
-      table = await config.createTable(tableConfig)
+      table = await createTable(tableConfig)
     })
 
     it("should be able to delete a bulk set of rows", async () => {
@@ -687,7 +705,7 @@ describe.each([
     describe("fetchView", () => {
       beforeEach(async () => {
         const tableConfig = generateTableConfig()
-        table = await config.createTable(tableConfig)
+        table = await createTable(tableConfig)
       })
 
       it("should be able to fetch tables contents via 'view'", async () => {
@@ -735,7 +753,7 @@ describe.each([
   describe("fetchEnrichedRows", () => {
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
-      table = await config.createTable(tableConfig)
+      table = await createTable(tableConfig)
     })
 
     it("should allow enriching some linked rows", async () => {
@@ -808,7 +826,7 @@ describe.each([
     describe("attachments", () => {
       beforeAll(async () => {
         const tableConfig = generateTableConfig()
-        table = await config.createTable(tableConfig)
+        table = await createTable(tableConfig)
       })
 
       it("should allow enriching attachment rows", async () => {
@@ -839,7 +857,7 @@ describe.each([
   describe("exportData", () => {
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
-      table = await config.createTable(tableConfig)
+      table = await createTable(tableConfig)
     })
 
     it("should allow exporting all columns", async () => {
@@ -880,6 +898,8 @@ describe.each([
     async function userTable(): Promise<Table> {
       return {
         name: `users_${generator.word()}`,
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
         type: "table",
         primary: ["id"],
         schema: {
@@ -925,7 +945,7 @@ describe.each([
 
     describe("create", () => {
       it("should persist a new row with only the provided view fields", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
         const view = await config.createView({
           schema: {
             name: { visible: true },
@@ -960,7 +980,7 @@ describe.each([
 
     describe("patch", () => {
       it("should update only the view fields for a row", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
         const tableId = table._id!
         const view = await config.createView({
           schema: {
@@ -1001,7 +1021,7 @@ describe.each([
 
     describe("destroy", () => {
       it("should be able to delete a row", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
         const tableId = table._id!
         const view = await config.createView({
           schema: {
@@ -1025,7 +1045,7 @@ describe.each([
       })
 
       it("should be able to delete multiple rows", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
         const tableId = table._id!
         const view = await config.createView({
           schema: {
@@ -1062,6 +1082,8 @@ describe.each([
       async function userTable(): Promise<Table> {
         return {
           name: `users_${generator.word()}`,
+          sourceId: INTERNAL_TABLE_SOURCE_ID,
+          sourceType: TableSourceType.INTERNAL,
           type: "table",
           primary: ["id"],
           schema: {
@@ -1088,7 +1110,7 @@ describe.each([
       }
 
       it("returns empty rows from view when no schema is passed", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
         const rows = await Promise.all(
           Array.from({ length: 10 }, () =>
             config.api.row.save(table._id!, { tableId: table._id })
@@ -1119,7 +1141,7 @@ describe.each([
       })
 
       it("searching respects the view filters", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
 
         await Promise.all(
           Array.from({ length: 10 }, () =>
@@ -1243,7 +1265,7 @@ describe.each([
 
       describe("sorting", () => {
         beforeAll(async () => {
-          const table = await config.createTable(await userTable())
+          const table = await createTable(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1310,7 +1332,7 @@ describe.each([
       })
 
       it("when schema is defined, defined columns and row attributes are returned", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
         const rows = await Promise.all(
           Array.from({ length: 10 }, () =>
             config.api.row.save(table._id!, {
@@ -1341,7 +1363,7 @@ describe.each([
       })
 
       it("views without data can be returned", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await createTable(await userTable())
 
         const createViewResponse = await config.createView()
         const response = await config.api.viewV2.search(createViewResponse.id)
@@ -1350,7 +1372,7 @@ describe.each([
       })
 
       it("respects the limit parameter", async () => {
-        await config.createTable(await userTable())
+        await createTable(await userTable())
         await Promise.all(Array.from({ length: 10 }, () => config.createRow()))
 
         const limit = generator.integer({ min: 1, max: 8 })
@@ -1365,7 +1387,7 @@ describe.each([
       })
 
       it("can handle pagination", async () => {
-        await config.createTable(await userTable())
+        await createTable(await userTable())
         await Promise.all(Array.from({ length: 10 }, () => config.createRow()))
 
         const createViewResponse = await config.createView()
@@ -1443,7 +1465,7 @@ describe.each([
         let tableId: string
 
         beforeAll(async () => {
-          await config.createTable(await userTable())
+          await createTable(await userTable())
           await Promise.all(
             Array.from({ length: 10 }, () => config.createRow())
           )
@@ -1521,13 +1543,13 @@ describe.each([
   let o2mTable: Table
   let m2mTable: Table
   beforeAll(async () => {
-    o2mTable = await config.createTable(
+    o2mTable = await createTable(
       { ...generateTableConfig(), name: "o2m" },
       {
         skipReassigning: true,
       }
     )
-    m2mTable = await config.createTable(
+    m2mTable = await createTable(
       { ...generateTableConfig(), name: "m2m" },
       {
         skipReassigning: true,
@@ -1597,9 +1619,9 @@ describe.each([
       const tableConfig = generateTableConfig()
 
       if (config.datasource) {
-        tableConfig.sourceId = config.datasource._id
+        tableConfig.sourceId = config.datasource._id!
         if (config.datasource.plus) {
-          tableConfig.type = "external"
+          tableConfig.sourceType = TableSourceType.EXTERNAL
         }
       }
       const table = await config.api.table.create({

packages/server/src/api/routes/tests/static.spec.js

@@ -5,11 +5,15 @@ describe("/static", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
   let app
+  let cleanupEnv
 
-  afterAll(setup.afterAll)
+  afterAll(() => {
+    setup.afterAll()
+    cleanupEnv()
+  })
 
   beforeAll(async () => {
-    config.modeSelf()
+    cleanupEnv = config.setEnv({ SELF_HOSTED: "true" })
     app = await config.init()
   })
 

packages/server/src/api/routes/tests/table.spec.ts

@@ -1,16 +1,24 @@
-import { events, context } from "@budibase/backend-core"
+import { context, events } from "@budibase/backend-core"
 import {
-  FieldType,
-  SaveTableRequest,
-  RelationshipType,
-  Table,
-  ViewCalculation,
   AutoFieldSubTypes,
+  FieldSubtype,
+  FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
+  InternalTable,
+  RelationshipType,
+  Row,
+  SaveTableRequest,
+  Table,
+  TableSourceType,
+  User,
+  ViewCalculation,
 } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
-const { basicTable } = setup.structures
 import sdk from "../../../sdk"
+import uuid from "uuid"
+
+const { basicTable } = setup.structures
 
 describe("/tables", () => {
   let request = setup.getRequest()
@@ -239,7 +247,8 @@ describe("/tables", () => {
         .expect(200)
       const fetchedTable = res.body[0]
       expect(fetchedTable.name).toEqual(testTable.name)
-      expect(fetchedTable.type).toEqual("internal")
+      expect(fetchedTable.type).toEqual("table")
+      expect(fetchedTable.sourceType).toEqual("internal")
     })
 
     it("should apply authorization to endpoint", async () => {
@@ -417,4 +426,281 @@ describe("/tables", () => {
       })
     })
   })
+
+  describe("migrate", () => {
+    let users: User[]
+    beforeAll(async () => {
+      users = await Promise.all([
+        config.createUser({ email: `${uuid.v4()}@example.com` }),
+        config.createUser({ email: `${uuid.v4()}@example.com` }),
+        config.createUser({ email: `${uuid.v4()}@example.com` }),
+      ])
+    })
+
+    it("should successfully migrate a one-to-many user relationship to a user column", async () => {
+      const table = await config.api.table.create({
+        name: "table",
+        type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
+        schema: {
+          "user relationship": {
+            type: FieldType.LINK,
+            fieldName: "test",
+            name: "user relationship",
+            constraints: {
+              type: "array",
+              presence: false,
+            },
+            relationshipType: RelationshipType.ONE_TO_MANY,
+            tableId: InternalTable.USER_METADATA,
+          },
+        },
+      })
+
+      const rows = await Promise.all(
+        users.map(u =>
+          config.api.row.save(table._id!, { "user relationship": [u] })
+        )
+      )
+
+      await config.api.table.migrate(table._id!, {
+        oldColumn: table.schema["user relationship"],
+        newColumn: {
+          name: "user column",
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldSubtype.USER,
+        },
+      })
+
+      const migratedTable = await config.api.table.get(table._id!)
+      expect(migratedTable.schema["user column"]).toBeDefined()
+      expect(migratedTable.schema["user relationship"]).not.toBeDefined()
+
+      const migratedRows = await config.api.row.fetch(table._id!)
+
+      rows.sort((a, b) => a._id!.localeCompare(b._id!))
+      migratedRows.sort((a, b) => a._id!.localeCompare(b._id!))
+
+      for (const [i, row] of rows.entries()) {
+        const migratedRow = migratedRows[i]
+        expect(migratedRow["user column"]).toBeDefined()
+        expect(migratedRow["user relationship"]).not.toBeDefined()
+        expect(row["user relationship"][0]._id).toEqual(
+          migratedRow["user column"][0]._id
+        )
+      }
+    })
+
+    it("should successfully migrate a many-to-many user relationship to a users column", async () => {
+      const table = await config.api.table.create({
+        name: "table",
+        type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
+        schema: {
+          "user relationship": {
+            type: FieldType.LINK,
+            fieldName: "test",
+            name: "user relationship",
+            constraints: {
+              type: "array",
+              presence: false,
+            },
+            relationshipType: RelationshipType.MANY_TO_MANY,
+            tableId: InternalTable.USER_METADATA,
+          },
+        },
+      })
+
+      const row1 = await config.api.row.save(table._id!, {
+        "user relationship": [users[0], users[1]],
+      })
+
+      const row2 = await config.api.row.save(table._id!, {
+        "user relationship": [users[1], users[2]],
+      })
+
+      await config.api.table.migrate(table._id!, {
+        oldColumn: table.schema["user relationship"],
+        newColumn: {
+          name: "user column",
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldSubtype.USERS,
+        },
+      })
+
+      const migratedTable = await config.api.table.get(table._id!)
+      expect(migratedTable.schema["user column"]).toBeDefined()
+      expect(migratedTable.schema["user relationship"]).not.toBeDefined()
+
+      const row1Migrated = (await config.api.row.get(table._id!, row1._id!))
+        .body as Row
+      expect(row1Migrated["user relationship"]).not.toBeDefined()
+      expect(row1Migrated["user column"].map((r: Row) => r._id)).toEqual(
+        expect.arrayContaining([users[0]._id, users[1]._id])
+      )
+
+      const row2Migrated = (await config.api.row.get(table._id!, row2._id!))
+        .body as Row
+      expect(row2Migrated["user relationship"]).not.toBeDefined()
+      expect(row2Migrated["user column"].map((r: Row) => r._id)).toEqual(
+        expect.arrayContaining([users[1]._id, users[2]._id])
+      )
+    })
+
+    it("should successfully migrate a many-to-one user relationship to a users column", async () => {
+      const table = await config.api.table.create({
+        name: "table",
+        type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
+        schema: {
+          "user relationship": {
+            type: FieldType.LINK,
+            fieldName: "test",
+            name: "user relationship",
+            constraints: {
+              type: "array",
+              presence: false,
+            },
+            relationshipType: RelationshipType.MANY_TO_ONE,
+            tableId: InternalTable.USER_METADATA,
+          },
+        },
+      })
+
+      const row1 = await config.api.row.save(table._id!, {
+        "user relationship": [users[0], users[1]],
+      })
+
+      const row2 = await config.api.row.save(table._id!, {
+        "user relationship": [users[2]],
+      })
+
+      await config.api.table.migrate(table._id!, {
+        oldColumn: table.schema["user relationship"],
+        newColumn: {
+          name: "user column",
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldSubtype.USERS,
+        },
+      })
+
+      const migratedTable = await config.api.table.get(table._id!)
+      expect(migratedTable.schema["user column"]).toBeDefined()
+      expect(migratedTable.schema["user relationship"]).not.toBeDefined()
+
+      const row1Migrated = (await config.api.row.get(table._id!, row1._id!))
+        .body as Row
+      expect(row1Migrated["user relationship"]).not.toBeDefined()
+      expect(row1Migrated["user column"].map((r: Row) => r._id)).toEqual(
+        expect.arrayContaining([users[0]._id, users[1]._id])
+      )
+
+      const row2Migrated = (await config.api.row.get(table._id!, row2._id!))
+        .body as Row
+      expect(row2Migrated["user relationship"]).not.toBeDefined()
+      expect(row2Migrated["user column"].map((r: Row) => r._id)).toEqual([
+        users[2]._id,
+      ])
+    })
+
+    describe("unhappy paths", () => {
+      let table: Table
+      beforeAll(async () => {
+        table = await config.api.table.create({
+          name: "table",
+          type: "table",
+          sourceId: INTERNAL_TABLE_SOURCE_ID,
+          sourceType: TableSourceType.INTERNAL,
+          schema: {
+            "user relationship": {
+              type: FieldType.LINK,
+              fieldName: "test",
+              name: "user relationship",
+              constraints: {
+                type: "array",
+                presence: false,
+              },
+              relationshipType: RelationshipType.MANY_TO_ONE,
+              tableId: InternalTable.USER_METADATA,
+            },
+            num: {
+              type: FieldType.NUMBER,
+              name: "num",
+              constraints: {
+                type: "number",
+                presence: false,
+              },
+            },
+          },
+        })
+      })
+
+      it("should fail if the new column name is blank", async () => {
+        await config.api.table.migrate(
+          table._id!,
+          {
+            oldColumn: table.schema["user relationship"],
+            newColumn: {
+              name: "",
+              type: FieldType.BB_REFERENCE,
+              subtype: FieldSubtype.USERS,
+            },
+          },
+          { expectStatus: 400 }
+        )
+      })
+
+      it("should fail if the new column name is a reserved name", async () => {
+        await config.api.table.migrate(
+          table._id!,
+          {
+            oldColumn: table.schema["user relationship"],
+            newColumn: {
+              name: "_id",
+              type: FieldType.BB_REFERENCE,
+              subtype: FieldSubtype.USERS,
+            },
+          },
+          { expectStatus: 400 }
+        )
+      })
+
+      it("should fail if the new column name is the same as an existing column", async () => {
+        await config.api.table.migrate(
+          table._id!,
+          {
+            oldColumn: table.schema["user relationship"],
+            newColumn: {
+              name: "num",
+              type: FieldType.BB_REFERENCE,
+              subtype: FieldSubtype.USERS,
+            },
+          },
+          { expectStatus: 400 }
+        )
+      })
+
+      it("should fail if the old column name isn't a column in the table", async () => {
+        await config.api.table.migrate(
+          table._id!,
+          {
+            oldColumn: {
+              name: "not a column",
+              type: FieldType.BB_REFERENCE,
+              subtype: FieldSubtype.USERS,
+            },
+            newColumn: {
+              name: "new column",
+              type: FieldType.BB_REFERENCE,
+              subtype: FieldSubtype.USERS,
+            },
+          },
+          { expectStatus: 400 }
+        )
+      })
+    })
+  })
 })

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -3,10 +3,12 @@ import {
   CreateViewRequest,
   FieldSchema,
   FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
   SearchQueryOperators,
   SortOrder,
   SortType,
   Table,
+  TableSourceType,
   UIFieldMetadata,
   UpdateViewRequest,
   ViewV2,
@@ -18,6 +20,8 @@ function priceTable(): Table {
   return {
     name: "table",
     type: "table",
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
+    sourceType: TableSourceType.INTERNAL,
     schema: {
       Price: {
         type: FieldType.NUMBER,
@@ -54,10 +58,10 @@ describe.each([
         },
       })
 
-      return config.createTable({
+      return config.createExternalTable({
         ...priceTable(),
         sourceId: datasource._id,
-        type: "external",
+        sourceType: TableSourceType.EXTERNAL,
       })
     },
   ],

packages/server/src/api/routes/tests/webhook.spec.ts

@@ -8,11 +8,15 @@ describe("/webhooks", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
   let webhook: Webhook
+  let cleanupEnv: () => void
 
-  afterAll(setup.afterAll)
+  afterAll(() => {
+    setup.afterAll()
+    cleanupEnv()
+  })
 
   const setupTest = async () => {
-    config.modeSelf()
+    cleanupEnv = config.setEnv({ SELF_HOSTED: "true" })
     await config.init()
     const autoConfig = basicAutomation()
     autoConfig.definition.trigger.schema = {

packages/server/src/constants/index.ts

@@ -1,5 +1,11 @@
-import { objectStore, roles, constants } from "@budibase/backend-core"
+import { constants, objectStore, roles } from "@budibase/backend-core"
-import { FieldType as FieldTypes } from "@budibase/types"
+import {
+  FieldType as FieldTypes,
+  INTERNAL_TABLE_SOURCE_ID,
+  Table,
+  TableSourceType,
+} from "@budibase/types"
+
 export {
   FieldType as FieldTypes,
   RelationshipType,
@@ -70,9 +76,11 @@ export enum SortDirection {
   DESCENDING = "DESCENDING",
 }
 
-export const USERS_TABLE_SCHEMA = {
+export const USERS_TABLE_SCHEMA: Table = {
   _id: "ta_users",
   type: "table",
+  sourceId: INTERNAL_TABLE_SOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   views: {},
   name: "Users",
   // TODO: ADMIN PANEL - when implemented this doesn't need to be carried out
@@ -87,12 +95,10 @@ export const USERS_TABLE_SCHEMA = {
         },
         presence: true,
       },
-      fieldName: "email",
       name: "email",
     },
     firstName: {
       name: "firstName",
-      fieldName: "firstName",
       type: FieldTypes.STRING,
       constraints: {
         type: FieldTypes.STRING,
@@ -101,7 +107,6 @@ export const USERS_TABLE_SCHEMA = {
     },
     lastName: {
       name: "lastName",
-      fieldName: "lastName",
       type: FieldTypes.STRING,
       constraints: {
         type: FieldTypes.STRING,
@@ -109,7 +114,6 @@ export const USERS_TABLE_SCHEMA = {
       },
     },
     roleId: {
-      fieldName: "roleId",
       name: "roleId",
       type: FieldTypes.OPTIONS,
       constraints: {
@@ -119,7 +123,6 @@ export const USERS_TABLE_SCHEMA = {
       },
     },
     status: {
-      fieldName: "status",
       name: "status",
       type: FieldTypes.OPTIONS,
       constraints: {
@@ -169,3 +172,8 @@ export enum AutomationErrors {
 export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets
 export const MAX_AUTOMATION_RECURRING_ERRORS = 5
 export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber"
+export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
+export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
+export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
+export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
+export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"

packages/server/src/db/defaultData/datasource_bb_default.ts

@@ -1,4 +1,12 @@
-import { FieldTypes, AutoFieldSubTypes } from "../../constants"
+import {
+  AutoFieldSubTypes,
+  FieldTypes,
+  DEFAULT_BB_DATASOURCE_ID,
+  DEFAULT_INVENTORY_TABLE_ID,
+  DEFAULT_EMPLOYEE_TABLE_ID,
+  DEFAULT_EXPENSES_TABLE_ID,
+  DEFAULT_JOBS_TABLE_ID,
+} from "../../constants"
 import { importToRows } from "../../api/controllers/table/utils"
 import { cloneDeep } from "lodash/fp"
 import LinkDocument from "../linkedRows/LinkDocument"
@@ -8,19 +16,14 @@ import { jobsImport } from "./jobsImport"
 import { expensesImport } from "./expensesImport"
 import { db as dbCore } from "@budibase/backend-core"
 import {
-  Table,
-  Row,
-  RelationshipType,
   FieldType,
+  RelationshipType,
+  Row,
+  Table,
   TableSchema,
+  TableSourceType,
 } from "@budibase/types"
 
-export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
-export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
-export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
-export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
-export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
-
 const defaultDatasource = {
   _id: DEFAULT_BB_DATASOURCE_ID,
   type: dbCore.BUDIBASE_DATASOURCE_TYPE,
@@ -89,9 +92,10 @@ const AUTO_COLUMNS: TableSchema = {
 
 export const DEFAULT_INVENTORY_TABLE_SCHEMA: Table = {
   _id: DEFAULT_INVENTORY_TABLE_ID,
-  type: "internal",
+  type: "table",
   views: {},
   sourceId: DEFAULT_BB_DATASOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   primaryDisplay: "Item Name",
   name: "Inventory",
   schema: {
@@ -198,10 +202,11 @@ export const DEFAULT_INVENTORY_TABLE_SCHEMA: Table = {
 
 export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = {
   _id: DEFAULT_EMPLOYEE_TABLE_ID,
-  type: "internal",
+  type: "table",
   views: {},
   name: "Employees",
   sourceId: DEFAULT_BB_DATASOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   primaryDisplay: "First Name",
   schema: {
     "First Name": {
@@ -346,9 +351,10 @@ export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = {
 
 export const DEFAULT_JOBS_TABLE_SCHEMA: Table = {
   _id: DEFAULT_JOBS_TABLE_ID,
-  type: "internal",
+  type: "table",
   name: "Jobs",
   sourceId: DEFAULT_BB_DATASOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   primaryDisplay: "Job ID",
   schema: {
     "Job ID": {
@@ -503,10 +509,11 @@ export const DEFAULT_JOBS_TABLE_SCHEMA: Table = {
 
 export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = {
   _id: DEFAULT_EXPENSES_TABLE_ID,
-  type: "internal",
+  type: "table",
   views: {},
   name: "Expenses",
   sourceId: DEFAULT_BB_DATASOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   primaryDisplay: "Expense ID",
   schema: {
     "Expense ID": {

packages/server/src/db/linkedRows/linkUtils.ts

@@ -2,7 +2,12 @@ import { ViewName, getQueryIndex, isRelationshipColumn } from "../utils"
 import { FieldTypes } from "../../constants"
 import { createLinkView } from "../views/staticViews"
 import { context, logging } from "@budibase/backend-core"
-import { LinkDocument, LinkDocumentValue, Table } from "@budibase/types"
+import {
+  DatabaseQueryOpts,
+  LinkDocument,
+  LinkDocumentValue,
+  Table,
+} from "@budibase/types"
 
 export { createLinkView } from "../views/staticViews"
 
@@ -36,13 +41,13 @@ export async function getLinkDocuments(args: {
 }): Promise<LinkDocumentValue[] | LinkDocument[]> {
   const { tableId, rowId, fieldName, includeDocs } = args
   const db = context.getAppDB()
-  let params: any
+  let params: DatabaseQueryOpts
   if (rowId) {
     params = { key: [tableId, rowId] }
   }
   // only table is known
   else {
-    params = { startKey: [tableId], endKey: [tableId, {}] }
+    params = { startkey: [tableId], endkey: [tableId, {}] }
   }
   if (includeDocs) {
     params.include_docs = true

packages/server/src/db/utils.ts

@@ -5,6 +5,7 @@ import {
   FieldSchema,
   RelationshipFieldMetadata,
   VirtualDocumentType,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import { FieldTypes } from "../constants"
 export { DocumentType, VirtualDocumentType } from "@budibase/types"
@@ -18,7 +19,7 @@ export const enum AppStatus {
 }
 
 export const BudibaseInternalDB = {
-  _id: "bb_internal",
+  _id: INTERNAL_TABLE_SOURCE_ID,
   type: dbCore.BUDIBASE_DATASOURCE_TYPE,
   name: "Budibase DB",
   source: "BUDIBASE",

packages/server/src/environment.ts

@@ -75,7 +75,6 @@ const environment = {
   },
   isTest: coreEnv.isTest,
   isJest: coreEnv.isJest,
-
   isDev: coreEnv.isDev,
   isProd: () => {
     return !coreEnv.isDev()

packages/server/src/integration-test/postgres.spec.ts

@@ -1,6 +1,4 @@
 import fetch from "node-fetch"
-// @ts-ignore
-fetch.mockSearch()
 import {
   generateMakeRequest,
   MakeRequestResponse,
@@ -13,12 +11,15 @@ import {
   RelationshipType,
   Row,
   Table,
+  TableSourceType,
 } from "@budibase/types"
 import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
 import { databaseTestProviders } from "../integrations/tests/utils"
 import { Client } from "pg"
+// @ts-ignore
+fetch.mockSearch()
 
 const config = setup.getConfig()!
 
@@ -52,7 +53,7 @@ describe("postgres integrations", () => {
     async function createAuxTable(prefix: string) {
       return await config.createTable({
         name: `${prefix}_${generator.word({ length: 6 })}`,
-        type: "external",
+        type: "table",
         primary: ["id"],
         primaryDisplay: "title",
         schema: {
@@ -67,6 +68,7 @@ describe("postgres integrations", () => {
           },
         },
         sourceId: postgresDatasource._id,
+        sourceType: TableSourceType.EXTERNAL,
       })
     }
 
@@ -88,7 +90,7 @@ describe("postgres integrations", () => {
 
     primaryPostgresTable = await config.createTable({
       name: `p_${generator.word({ length: 6 })}`,
-      type: "external",
+      type: "table",
       primary: ["id"],
       schema: {
         id: {
@@ -143,6 +145,7 @@ describe("postgres integrations", () => {
         },
       },
       sourceId: postgresDatasource._id,
+      sourceType: TableSourceType.EXTERNAL,
     })
   })
 
@@ -249,7 +252,7 @@ describe("postgres integrations", () => {
   async function createDefaultPgTable() {
     return await config.createTable({
       name: generator.word({ length: 10 }),
-      type: "external",
+      type: "table",
       primary: ["id"],
       schema: {
         id: {
@@ -259,6 +262,7 @@ describe("postgres integrations", () => {
         },
       },
       sourceId: postgresDatasource._id,
+      sourceType: TableSourceType.EXTERNAL,
     })
   }
 

packages/server/src/integrations/googlesheets.ts

@@ -10,11 +10,12 @@ import {
   QueryJson,
   QueryType,
   Row,
+  Schema,
   SearchFilters,
   SortJson,
-  ExternalTable,
+  Table,
   TableRequest,
-  Schema,
+  TableSourceType,
 } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
 import {
@@ -262,11 +263,13 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     id?: string
   ) {
     // base table
-    const table: ExternalTable = {
+    const table: Table = {
+      type: "table",
       name: title,
       primary: [GOOGLE_SHEETS_PRIMARY_KEY],
       schema: {},
       sourceId: datasourceId,
+      sourceType: TableSourceType.EXTERNAL,
     }
     if (id) {
       table._id = id
@@ -283,7 +286,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
     // not fully configured yet
     if (!this.config.auth) {
@@ -291,7 +294,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     }
     await this.connect()
     const sheets = this.client.sheetsByIndex
-    const tables: Record<string, ExternalTable> = {}
+    const tables: Record<string, Table> = {}
     let errors: Record<string, string> = {}
     await utils.parallelForeach(
       sheets,

packages/server/src/integrations/microsoftSqlServer.ts

@@ -2,7 +2,7 @@ import {
   DatasourceFieldType,
   Integration,
   Operation,
-  ExternalTable,
+  Table,
   TableSchema,
   QueryJson,
   QueryType,
@@ -12,6 +12,7 @@ import {
   ConnectionInfo,
   SourceName,
   Schema,
+  TableSourceType,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -380,7 +381,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
    */
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
     await this.connect()
     let tableInfo: MSSQLTablesResponse[] = await this.runSQL(this.TABLES_SQL)
@@ -394,7 +395,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
       .map((record: any) => record.TABLE_NAME)
       .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
 
-    const tables: Record<string, ExternalTable> = {}
+    const tables: Record<string, Table> = {}
     for (let tableName of tableNames) {
       // get the column definition (type)
       const definition = await this.runSQL(
@@ -439,7 +440,9 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
       }
       tables[tableName] = {
         _id: buildExternalTableId(datasourceId, tableName),
+        type: "table",
         sourceId: datasourceId,
+        sourceType: TableSourceType.EXTERNAL,
         primary: primaryKeys,
         name: tableName,
         schema,

packages/server/src/integrations/mysql.ts

@@ -4,13 +4,14 @@ import {
   QueryType,
   QueryJson,
   SqlQuery,
-  ExternalTable,
+  Table,
   TableSchema,
   DatasourcePlus,
   DatasourceFeature,
   ConnectionInfo,
   SourceName,
   Schema,
+  TableSourceType,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -278,9 +279,9 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
 
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
-    const tables: { [key: string]: ExternalTable } = {}
+    const tables: { [key: string]: Table } = {}
     await this.connect()
 
     try {
@@ -317,8 +318,10 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
         }
         if (!tables[tableName]) {
           tables[tableName] = {
+            type: "table",
             _id: buildExternalTableId(datasourceId, tableName),
             sourceId: datasourceId,
+            sourceType: TableSourceType.EXTERNAL,
             primary: primaryKeys,
             name: tableName,
             schema,

packages/server/src/integrations/oracle.ts

@@ -5,11 +5,12 @@ import {
   QueryJson,
   QueryType,
   SqlQuery,
-  ExternalTable,
+  Table,
   DatasourcePlus,
   DatasourceFeature,
   ConnectionInfo,
   Schema,
+  TableSourceType,
 } from "@budibase/types"
 import {
   buildExternalTableId,
@@ -263,25 +264,27 @@ class OracleIntegration extends Sql implements DatasourcePlus {
    */
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
     const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
       sql: this.COLUMNS_SQL,
     })
     const oracleTables = this.mapColumns(columnsResponse)
 
-    const tables: { [key: string]: ExternalTable } = {}
+    const tables: { [key: string]: Table } = {}
 
     // iterate each table
     Object.values(oracleTables).forEach(oracleTable => {
       let table = tables[oracleTable.name]
       if (!table) {
         table = {
+          type: "table",
           _id: buildExternalTableId(datasourceId, oracleTable.name),
           primary: [],
           name: oracleTable.name,
           schema: {},
           sourceId: datasourceId,
+          sourceType: TableSourceType.EXTERNAL,
         }
         tables[oracleTable.name] = table
       }

packages/server/src/integrations/postgres.ts

@@ -5,12 +5,13 @@ import {
   QueryType,
   QueryJson,
   SqlQuery,
-  ExternalTable,
+  Table,
   DatasourcePlus,
   DatasourceFeature,
   ConnectionInfo,
   SourceName,
   Schema,
+  TableSourceType,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -273,7 +274,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
    */
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
     let tableKeys: { [key: string]: string[] } = {}
     await this.openConnection()
@@ -300,7 +301,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
       const columnsResponse: { rows: PostgresColumn[] } =
         await this.client.query(this.COLUMNS_SQL)
 
-      const tables: { [key: string]: ExternalTable } = {}
+      const tables: { [key: string]: Table } = {}
 
       for (let column of columnsResponse.rows) {
         const tableName: string = column.table_name
@@ -309,11 +310,13 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
         // table key doesn't exist yet
         if (!tables[tableName] || !tables[tableName].schema) {
           tables[tableName] = {
+            type: "table",
             _id: buildExternalTableId(datasourceId, tableName),
             primary: tableKeys[tableName] || [],
             name: tableName,
             schema: {},
             sourceId: datasourceId,
+            sourceType: TableSourceType.EXTERNAL,
           }
         }
 

packages/server/src/integrations/tests/googlesheets.spec.ts

@@ -30,18 +30,24 @@ GoogleSpreadsheet.mockImplementation(() => mockGoogleIntegration)
 import { structures } from "@budibase/backend-core/tests"
 import TestConfiguration from "../../tests/utilities/TestConfiguration"
 import GoogleSheetsIntegration from "../googlesheets"
-import { FieldType, Table, TableSchema } from "@budibase/types"
+import { FieldType, Table, TableSchema, TableSourceType } from "@budibase/types"
+import { generateDatasourceID } from "../../db/utils"
 
 describe("Google Sheets Integration", () => {
   let integration: any,
     config = new TestConfiguration()
+  let cleanupEnv: () => void
 
   beforeAll(() => {
-    config.setGoogleAuth("test")
+    cleanupEnv = config.setEnv({
+      GOOGLE_CLIENT_ID: "test",
+      GOOGLE_CLIENT_SECRET: "test",
+    })
   })
 
   afterAll(async () => {
-    await config.end()
+    cleanupEnv()
+    config.end()
   })
 
   beforeEach(async () => {
@@ -60,7 +66,10 @@ describe("Google Sheets Integration", () => {
 
   function createBasicTable(name: string, columns: string[]): Table {
     return {
+      type: "table",
       name,
+      sourceId: generateDatasourceID(),
+      sourceType: TableSourceType.EXTERNAL,
       schema: {
         ...columns.reduce((p, c) => {
           p[c] = {

packages/server/src/integrations/tests/utils/postgres.ts

@@ -1,11 +1,17 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import env from "../../../environment"
 
 let container: StartedTestContainer | undefined
 
+const isMac = process.platform === "darwin"
+
 export async function getDsConfig(): Promise<Datasource> {
+  try {
     if (!container) {
-    container = await new GenericContainer("postgres")
+      // postgres 15-bullseye safer bet on Linux
+      const version = isMac ? undefined : "15-bullseye"
+      container = await new GenericContainer("postgres", version)
         .withExposedPorts(5432)
         .withEnv("POSTGRES_PASSWORD", "password")
         .withWaitStrategy(
@@ -15,7 +21,6 @@ export async function getDsConfig(): Promise<Datasource> {
         )
         .start()
     }
-
     const host = container.getContainerIpAddress()
     const port = container.getMappedPort(5432)
 
@@ -35,6 +40,9 @@ export async function getDsConfig(): Promise<Datasource> {
         ca: false,
       },
     }
+  } catch (err) {
+    throw new Error("**UNABLE TO CREATE TO POSTGRES CONTAINER**")
+  }
 }
 
 export async function stopContainer() {

packages/server/src/integrations/utils.ts

@@ -4,10 +4,14 @@ import {
   SearchFilters,
   Datasource,
   FieldType,
-  ExternalTable,
+  TableSourceType,
 } from "@budibase/types"
 import { DocumentType, SEPARATOR } from "../db/utils"
-import { InvalidColumns, NoEmptyFilterStrings } from "../constants"
+import {
+  InvalidColumns,
+  NoEmptyFilterStrings,
+  DEFAULT_BB_DATASOURCE_ID,
+} from "../constants"
 import { helpers } from "@budibase/shared-core"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
@@ -83,10 +87,29 @@ export enum SqlClient {
   ORACLE = "oracledb",
 }
 
-export function isExternalTable(tableId: string) {
+export function isExternalTableID(tableId: string) {
   return tableId.includes(DocumentType.DATASOURCE)
 }
 
+export function isInternalTableID(tableId: string) {
+  return !isExternalTableID(tableId)
+}
+
+export function isExternalTable(table: Table) {
+  if (
+    table?.sourceId &&
+    table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) &&
+    table?.sourceId !== DEFAULT_BB_DATASOURCE_ID
+  ) {
+    return true
+  } else if (table?.sourceType === TableSourceType.EXTERNAL) {
+    return true
+  } else if (table?._id && isExternalTableID(table._id)) {
+    return true
+  }
+  return false
+}
+
 export function buildExternalTableId(datasourceId: string, tableName: string) {
   // encode spaces
   if (tableName.includes(" ")) {
@@ -297,9 +320,9 @@ function copyExistingPropsOver(
  * @param entities The old list of tables, if there was any to look for definitions in.
  */
 export function finaliseExternalTables(
-  tables: Record<string, ExternalTable>,
+  tables: Record<string, Table>,
-  entities: Record<string, ExternalTable>
+  entities: Record<string, Table>
-): Record<string, ExternalTable> {
+): Record<string, Table> {
   let finalTables: Record<string, Table> = {}
   const tableIds = Object.values(tables).map(table => table._id!)
   for (let [name, table] of Object.entries(tables)) {
@@ -312,7 +335,7 @@ export function finaliseExternalTables(
 }
 
 export function checkExternalTables(
-  tables: Record<string, ExternalTable>
+  tables: Record<string, Table>
 ): Record<string, string> {
   const invalidColumns = Object.values(InvalidColumns) as string[]
   const errors: Record<string, string> = {}

packages/server/src/middleware/tests/trimViewRowInfo.spec.ts

@@ -1,5 +1,12 @@
 import { generator } from "@budibase/backend-core/tests"
-import { BBRequest, FieldType, Row, Table } from "@budibase/types"
+import {
+  BBRequest,
+  FieldType,
+  Row,
+  Table,
+  INTERNAL_TABLE_SOURCE_ID,
+  TableSourceType,
+} from "@budibase/types"
 import * as utils from "../../db/utils"
 import trimViewRowInfoMiddleware from "../trimViewRowInfo"
 
@@ -73,6 +80,8 @@ describe("trimViewRowInfo middleware", () => {
   const table: Table = {
     _id: tableId,
     name: generator.word(),
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
+    sourceType: TableSourceType.INTERNAL,
     type: "table",
     schema: {
       name: {

packages/server/src/middleware/trimViewRowInfo.ts

@@ -1,7 +1,6 @@
 import { Ctx, Row } from "@budibase/types"
 import * as utils from "../db/utils"
 import sdk from "../sdk"
-import { db } from "@budibase/backend-core"
 import { Next } from "koa"
 import { getTableId } from "../api/controllers/row/utils"
 

packages/server/src/migrations/functions/backfill/global/configs.ts

@@ -11,10 +11,11 @@ import {
   isOIDCConfig,
   isSettingsConfig,
   ConfigType,
+  DatabaseQueryOpts,
 } from "@budibase/types"
 import env from "./../../../../environment"
 
-export const getConfigParams = () => {
+export function getConfigParams(): DatabaseQueryOpts {
   return {
     include_docs: true,
     startkey: `${DocumentType.CONFIG}${SEPARATOR}`,

packages/server/src/sdk/app/backups/exports.ts

@@ -26,7 +26,6 @@ export interface DBDumpOpts {
 export interface ExportOpts extends DBDumpOpts {
   tar?: boolean
   excludeRows?: boolean
-  excludeLogs?: boolean
   encryptPassword?: string
 }
 
@@ -83,14 +82,15 @@ export async function exportDB(
   })
 }
 
-function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) {
+function defineFilter(excludeRows?: boolean) {
-  const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX]
+  const ids = [
+    USER_METDATA_PREFIX,
+    LINK_USER_METADATA_PREFIX,
+    AUTOMATION_LOG_PREFIX,
+  ]
   if (excludeRows) {
     ids.push(TABLE_ROW_PREFIX)
   }
-  if (excludeLogs) {
-    ids.push(AUTOMATION_LOG_PREFIX)
-  }
   return (doc: any) =>
     !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr)
 }
@@ -118,7 +118,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
         fs.writeFileSync(join(tmpPath, path), contents)
       }
     }
-    // get all of the files
+    // get all the files
     else {
       tmpPath = await objectStore.retrieveDirectory(
         ObjectStoreBuckets.APPS,
@@ -141,7 +141,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   // enforce an export of app DB to the tmp path
   const dbPath = join(tmpPath, DB_EXPORT_FILE)
   await exportDB(appId, {
-    filter: defineFilter(config?.excludeRows, config?.excludeLogs),
+    filter: defineFilter(config?.excludeRows),
     exportPath: dbPath,
   })
 
@@ -191,7 +191,6 @@ export async function streamExportApp({
 }) {
   const tmpPath = await exportApp(appId, {
     excludeRows,
-    excludeLogs: true,
     tar: true,
     encryptPassword,
   })

packages/server/src/sdk/app/links/index.ts

@@ -0,0 +1,5 @@
+import * as links from "./links"
+
+export default {
+  ...links,
+}

packages/server/src/sdk/app/links/links.ts

@@ -0,0 +1,39 @@
+import { context } from "@budibase/backend-core"
+import { isTableId } from "@budibase/backend-core/src/docIds"
+import {
+  DatabaseQueryOpts,
+  LinkDocument,
+  LinkDocumentValue,
+} from "@budibase/types"
+import { ViewName, getQueryIndex } from "../../../../src/db/utils"
+
+export async function fetch(tableId: string): Promise<LinkDocumentValue[]> {
+  if (!isTableId(tableId)) {
+    throw new Error(`Invalid tableId: ${tableId}`)
+  }
+
+  const db = context.getAppDB()
+  const params: DatabaseQueryOpts = {
+    startkey: [tableId],
+    endkey: [tableId, {}],
+  }
+  const linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows
+  return linkRows.map(row => row.value as LinkDocumentValue)
+}
+
+export async function fetchWithDocument(
+  tableId: string
+): Promise<LinkDocument[]> {
+  if (!isTableId(tableId)) {
+    throw new Error(`Invalid tableId: ${tableId}`)
+  }
+
+  const db = context.getAppDB()
+  const params: DatabaseQueryOpts = {
+    startkey: [tableId],
+    endkey: [tableId, {}],
+    include_docs: true,
+  }
+  const linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows
+  return linkRows.map(row => row.doc as LinkDocument)
+}

packages/server/src/sdk/app/rows/external.ts

@@ -1,4 +1,4 @@
-import { IncludeRelationship, Operation, Row } from "@budibase/types"
+import { IncludeRelationship, Operation } from "@budibase/types"
 import { handleRequest } from "../../../api/controllers/row/external"
 import { breakRowIdField } from "../../../integrations/utils"
 

packages/server/src/sdk/app/rows/search.ts

@@ -1,5 +1,5 @@
-import { SearchFilters, SearchParams, Row } from "@budibase/types"
+import { Row, SearchFilters, SearchParams } from "@budibase/types"
-import { isExternalTable } from "../../../integrations/utils"
+import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"
 import { Format } from "../../../api/controllers/view/exporters"
@@ -12,7 +12,7 @@ export interface ViewParams {
 }
 
 function pickApi(tableId: any) {
-  if (isExternalTable(tableId)) {
+  if (isExternalTableID(tableId)) {
     return external
   }
   return internal
@@ -49,6 +49,10 @@ export async function fetch(tableId: string): Promise<Row[]> {
   return pickApi(tableId).fetch(tableId)
 }
 
+export async function fetchRaw(tableId: string): Promise<Row[]> {
+  return pickApi(tableId).fetchRaw(tableId)
+}
+
 export async function fetchView(
   tableId: string,
   viewName: string,

packages/server/src/sdk/app/rows/search/external.ts

@@ -186,6 +186,12 @@ export async function fetch(tableId: string): Promise<Row[]> {
   })
 }
 
+export async function fetchRaw(tableId: string): Promise<Row[]> {
+  return await handleRequest<Operation.READ>(Operation.READ, tableId, {
+    includeSqlRelationships: IncludeRelationship.INCLUDE,
+  })
+}
+
 export async function fetchView(viewName: string) {
   // there are no views in external datasources, shouldn't ever be called
   // for now just fetch

packages/server/src/sdk/app/rows/search/internal.ts

@@ -140,14 +140,13 @@ export async function exportRows(
 }
 
 export async function fetch(tableId: string): Promise<Row[]> {
-  const db = context.getAppDB()
-
   const table = await sdk.tables.getTable(tableId)
-  const rows = await getRawTableData(db, tableId)
+  const rows = await fetchRaw(tableId)
   return await outputProcessing(table, rows)
 }
 
-async function getRawTableData(db: Database, tableId: string) {
+export async function fetchRaw(tableId: string): Promise<Row[]> {
+  const db = context.getAppDB()
   let rows
   if (tableId === InternalTables.USER_METADATA) {
     rows = await sdk.users.fetchMetadata()
@@ -182,7 +181,7 @@ export async function fetchView(
     })
   } else {
     const tableId = viewInfo.meta.tableId
-    const data = await getRawTableData(db, tableId)
+    const data = await fetchRaw(tableId)
     response = await inMemoryViews.runView(
       viewInfo,
       calculation as string,
@@ -198,11 +197,7 @@ export async function fetchView(
     try {
       table = await sdk.tables.getTable(viewInfo.meta.tableId)
     } catch (err) {
-      /* istanbul ignore next */
+      throw new Error("Unable to retrieve view table.")
-      table = {
-        name: "",
-        schema: {},
-      }
     }
     rows = await outputProcessing(table, response.rows)
   }

packages/server/src/sdk/app/rows/search/tests/external.spec.ts

@@ -7,6 +7,7 @@ import {
   SourceName,
   Table,
   SearchParams,
+  TableSourceType,
 } from "@budibase/types"
 
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
@@ -15,6 +16,7 @@ import {
   expectAnyExternalColsAttributes,
   generator,
 } from "@budibase/backend-core/tests"
+import datasource from "../../../../../api/routes/datasource"
 
 jest.unmock("mysql2/promise")
 
@@ -23,36 +25,7 @@ jest.setTimeout(30000)
 describe.skip("external", () => {
   const config = new TestConfiguration()
 
-  let externalDatasource: Datasource
+  let externalDatasource: Datasource, tableData: Table
-
-  const tableData: Table = {
-    name: generator.word(),
-    type: "external",
-    primary: ["id"],
-    schema: {
-      id: {
-        name: "id",
-        type: FieldType.AUTO,
-        autocolumn: true,
-      },
-      name: {
-        name: "name",
-        type: FieldType.STRING,
-      },
-      surname: {
-        name: "surname",
-        type: FieldType.STRING,
-      },
-      age: {
-        name: "age",
-        type: FieldType.NUMBER,
-      },
-      address: {
-        name: "address",
-        type: FieldType.STRING,
-      },
-    },
-  }
 
   beforeAll(async () => {
     const container = await new GenericContainer("mysql")
@@ -84,12 +57,43 @@ describe.skip("external", () => {
         },
       },
     })
+
+    tableData = {
+      name: generator.word(),
+      type: "table",
+      primary: ["id"],
+      sourceId: externalDatasource._id!,
+      sourceType: TableSourceType.EXTERNAL,
+      schema: {
+        id: {
+          name: "id",
+          type: FieldType.AUTO,
+          autocolumn: true,
+        },
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+        },
+        surname: {
+          name: "surname",
+          type: FieldType.STRING,
+        },
+        age: {
+          name: "age",
+          type: FieldType.NUMBER,
+        },
+        address: {
+          name: "address",
+          type: FieldType.STRING,
+        },
+      },
+    }
   })
 
   describe("search", () => {
     const rows: Row[] = []
     beforeAll(async () => {
-      const table = await config.createTable({
+      const table = await config.createExternalTable({
         ...tableData,
         sourceId: externalDatasource._id,
       })

packages/server/src/sdk/app/rows/search/tests/internal.spec.ts

@@ -1,4 +1,11 @@
-import { FieldType, Row, Table, SearchParams } from "@budibase/types"
+import {
+  FieldType,
+  Row,
+  Table,
+  SearchParams,
+  INTERNAL_TABLE_SOURCE_ID,
+  TableSourceType,
+} from "@budibase/types"
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
 import { search } from "../internal"
 import {
@@ -12,6 +19,8 @@ describe("internal", () => {
   const tableData: Table = {
     name: generator.word(),
     type: "table",
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
+    sourceType: TableSourceType.INTERNAL,
     schema: {
       name: {
         name: "name",

packages/server/src/sdk/app/rows/search/tests/utils.spec.ts

@@ -3,14 +3,19 @@ import { db as dbCore } from "@budibase/backend-core"
 import {
   FieldType,
   FieldTypeSubtypes,
-  Table,
+  INTERNAL_TABLE_SOURCE_ID,
   SearchParams,
+  Table,
+  TableSourceType,
 } from "@budibase/types"
 
 const tableId = "ta_a"
 const tableWithUserCol: Table = {
+  type: "table",
   _id: tableId,
   name: "table",
+  sourceId: INTERNAL_TABLE_SOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   schema: {
     user: {
       name: "user",
@@ -21,8 +26,11 @@ const tableWithUserCol: Table = {
 }
 
 const tableWithUsersCol: Table = {
+  type: "table",
   _id: tableId,
   name: "table",
+  sourceId: INTERNAL_TABLE_SOURCE_ID,
+  sourceType: TableSourceType.INTERNAL,
   schema: {
     user: {
       name: "user",

packages/server/src/sdk/app/tables/external/index.ts

@@ -35,10 +35,10 @@ export async function save(
   opts?: { tableId?: string; renaming?: RenameColumn }
 ) {
   let tableToSave: TableRequest = {
+    ...update,
     type: "table",
     _id: buildExternalTableId(datasourceId, update.name),
     sourceId: datasourceId,
-    ...update,
   }
 
   const tableId = opts?.tableId || update._id

packages/server/src/sdk/app/tables/external/utils.ts

@@ -6,6 +6,7 @@ import {
   RelationshipFieldMetadata,
   RelationshipType,
   Table,
+  TableSourceType,
 } from "@budibase/types"
 import { FieldTypes } from "../../../../constants"
 import {
@@ -76,12 +77,16 @@ export function generateManyLinkSchema(
   const primary = table.name + table.primary[0]
   const relatedPrimary = relatedTable.name + relatedTable.primary[0]
   const jcTblName = generateJunctionTableName(column, table, relatedTable)
+  const datasourceId = datasource._id!
   // first create the new table
-  const junctionTable = {
+  const junctionTable: Table = {
-    _id: buildExternalTableId(datasource._id!, jcTblName),
+    type: "table",
+    _id: buildExternalTableId(datasourceId, jcTblName),
     name: jcTblName,
     primary: [primary, relatedPrimary],
     constrained: [primary, relatedPrimary],
+    sourceId: datasourceId,
+    sourceType: TableSourceType.EXTERNAL,
     schema: {
       [primary]: foreignKeyStructure(primary, {
         toTable: table.name,

packages/server/src/sdk/app/tables/getters.ts

@@ -1,30 +1,47 @@
 import { context } from "@budibase/backend-core"
-import {
+import { getMultiIDParams, getTableParams } from "../../../db/utils"
-  BudibaseInternalDB,
-  getMultiIDParams,
-  getTableParams,
-} from "../../../db/utils"
 import {
   breakExternalTableId,
-  isExternalTable,
+  isExternalTableID,
   isSQL,
 } from "../../../integrations/utils"
 import {
-  AllDocsResponse,
   Database,
+  INTERNAL_TABLE_SOURCE_ID,
   Table,
   TableResponse,
+  TableSourceType,
   TableViewsResponse,
 } from "@budibase/types"
 import datasources from "../datasources"
 import sdk from "../../../sdk"
 
-function processInternalTables(docs: AllDocsResponse<Table[]>): Table[] {
+export function processTable(table: Table): Table {
-  return docs.rows.map((tableDoc: any) => ({
+  if (table._id && isExternalTableID(table._id)) {
-    ...tableDoc.doc,
+    return {
-    type: "internal",
+      ...table,
-    sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id,
+      type: "table",
-  }))
+      sourceType: TableSourceType.EXTERNAL,
+    }
+  } else {
+    return {
+      ...table,
+      type: "table",
+      sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
+    }
+  }
+}
+
+export function processTables(tables: Table[]): Table[] {
+  return tables.map(table => processTable(table))
+}
+
+function processEntities(tables: Record<string, Table>) {
+  for (let key of Object.keys(tables)) {
+    tables[key] = processTable(tables[key])
+  }
+  return tables
 }
 
 export async function getAllInternalTables(db?: Database): Promise<Table[]> {
@@ -36,7 +53,7 @@ export async function getAllInternalTables(db?: Database): Promise<Table[]> {
       include_docs: true,
     })
   )
-  return processInternalTables(internalTables)
+  return processTables(internalTables.rows.map(row => row.doc!))
 }
 
 async function getAllExternalTables(): Promise<Table[]> {
@@ -48,7 +65,7 @@ async function getAllExternalTables(): Promise<Table[]> {
       final = final.concat(Object.values(entities))
     }
   }
-  return final
+  return processTables(final)
 }
 
 export async function getExternalTable(
@@ -56,19 +73,21 @@ export async function getExternalTable(
   tableName: string
 ): Promise<Table> {
   const entities = await getExternalTablesInDatasource(datasourceId)
-  return entities[tableName]
+  return processTable(entities[tableName])
 }
 
 export async function getTable(tableId: string): Promise<Table> {
   const db = context.getAppDB()
-  if (isExternalTable(tableId)) {
+  let output: Table
+  if (isExternalTableID(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
     const datasource = await datasources.get(datasourceId!)
     const table = await getExternalTable(datasourceId!, tableName!)
-    return { ...table, sql: isSQL(datasource) }
+    output = { ...table, sql: isSQL(datasource) }
   } else {
-    return db.get(tableId)
+    output = await db.get<Table>(tableId)
   }
+  return processTable(output)
 }
 
 export async function getAllTables() {
@@ -76,7 +95,7 @@ export async function getAllTables() {
     getAllInternalTables(),
     getAllExternalTables(),
   ])
-  return [...internal, ...external]
+  return processTables([...internal, ...external])
 }
 
 export async function getExternalTablesInDatasource(
@@ -86,12 +105,14 @@ export async function getExternalTablesInDatasource(
   if (!datasource || !datasource.entities) {
     throw new Error("Datasource is not configured fully.")
   }
-  return datasource.entities
+  return processEntities(datasource.entities)
 }
 
 export async function getTables(tableIds: string[]): Promise<Table[]> {
-  const externalTableIds = tableIds.filter(tableId => isExternalTable(tableId)),
+  const externalTableIds = tableIds.filter(tableId =>
-    internalTableIds = tableIds.filter(tableId => !isExternalTable(tableId))
+      isExternalTableID(tableId)
+    ),
+    internalTableIds = tableIds.filter(tableId => !isExternalTableID(tableId))
   let tables: Table[] = []
   if (externalTableIds.length) {
     const externalTables = await getAllExternalTables()
@@ -106,9 +127,9 @@ export async function getTables(tableIds: string[]): Promise<Table[]> {
     const internalTableDocs = await db.allDocs<Table[]>(
       getMultiIDParams(internalTableIds)
     )
-    tables = tables.concat(processInternalTables(internalTableDocs))
+    tables = tables.concat(internalTableDocs.rows.map(row => row.doc!))
   }
-  return tables
+  return processTables(tables)
 }
 
 export function enrichViewSchemas(table: Table): TableResponse {

packages/server/src/sdk/app/tables/index.ts

@@ -2,10 +2,12 @@ import { populateExternalTableSchemas } from "./validation"
 import * as getters from "./getters"
 import * as updates from "./update"
 import * as utils from "./utils"
+import { migrate } from "./migration"
 
 export default {
   populateExternalTableSchemas,
   ...updates,
   ...getters,
   ...utils,
+  migrate,
 }

packages/server/src/sdk/app/tables/migration.ts

@@ -0,0 +1,194 @@
+import { BadRequestError, context, db as dbCore } from "@budibase/backend-core"
+import {
+  BBReferenceFieldMetadata,
+  FieldSchema,
+  FieldSubtype,
+  InternalTable,
+  isBBReferenceField,
+  isRelationshipField,
+  LinkDocument,
+  RelationshipFieldMetadata,
+  RelationshipType,
+  Row,
+  Table,
+} from "@budibase/types"
+import sdk from "../../../sdk"
+import { isExternalTableID } from "../../../integrations/utils"
+import { EventType, updateLinks } from "../../../db/linkedRows"
+import { cloneDeep } from "lodash"
+import { isInternalColumnName } from "@budibase/backend-core/src/db"
+
+export interface MigrationResult {
+  tablesUpdated: Table[]
+}
+
+export async function migrate(
+  table: Table,
+  oldColumn: FieldSchema,
+  newColumn: FieldSchema
+): Promise<MigrationResult> {
+  if (newColumn.name in table.schema) {
+    throw new BadRequestError(`Column "${newColumn.name}" already exists`)
+  }
+
+  if (newColumn.name === "") {
+    throw new BadRequestError(`Column name cannot be empty`)
+  }
+
+  if (isInternalColumnName(newColumn.name)) {
+    throw new BadRequestError(`Column name cannot be a reserved column name`)
+  }
+
+  table.schema[newColumn.name] = newColumn
+  table = await sdk.tables.saveTable(table)
+
+  let migrator = getColumnMigrator(table, oldColumn, newColumn)
+  try {
+    return await migrator.doMigration()
+  } catch (e) {
+    // If the migration fails then we need to roll back the table schema
+    // change.
+    delete table.schema[newColumn.name]
+    await sdk.tables.saveTable(table)
+    throw e
+  }
+}
+
+interface ColumnMigrator {
+  doMigration(): Promise<MigrationResult>
+}
+
+function getColumnMigrator(
+  table: Table,
+  oldColumn: FieldSchema,
+  newColumn: FieldSchema
+): ColumnMigrator {
+  // For now, we're only supporting migrations of user relationships to user
+  // columns in internal tables. In the future, we may want to support other
+  // migrations but for now return an error if we aren't migrating a user
+  // relationship.
+  if (isExternalTableID(table._id!)) {
+    throw new BadRequestError("External tables cannot be migrated")
+  }
+
+  if (!(oldColumn.name in table.schema)) {
+    throw new BadRequestError(`Column "${oldColumn.name}" does not exist`)
+  }
+
+  if (!isBBReferenceField(newColumn)) {
+    throw new BadRequestError(`Column "${newColumn.name}" is not a user column`)
+  }
+
+  if (newColumn.subtype !== "user" && newColumn.subtype !== "users") {
+    throw new BadRequestError(`Column "${newColumn.name}" is not a user column`)
+  }
+
+  if (!isRelationshipField(oldColumn)) {
+    throw new BadRequestError(
+      `Column "${oldColumn.name}" is not a user relationship`
+    )
+  }
+
+  if (oldColumn.tableId !== InternalTable.USER_METADATA) {
+    throw new BadRequestError(
+      `Column "${oldColumn.name}" is not a user relationship`
+    )
+  }
+
+  if (oldColumn.relationshipType === RelationshipType.ONE_TO_MANY) {
+    if (newColumn.subtype !== FieldSubtype.USER) {
+      throw new BadRequestError(
+        `Column "${oldColumn.name}" is a one-to-many column but "${newColumn.name}" is not a single user column`
+      )
+    }
+    return new SingleUserColumnMigrator(table, oldColumn, newColumn)
+  }
+  if (
+    oldColumn.relationshipType === RelationshipType.MANY_TO_MANY ||
+    oldColumn.relationshipType === RelationshipType.MANY_TO_ONE
+  ) {
+    if (newColumn.subtype !== FieldSubtype.USERS) {
+      throw new BadRequestError(
+        `Column "${oldColumn.name}" is a ${oldColumn.relationshipType} column but "${newColumn.name}" is not a multi user column`
+      )
+    }
+    return new MultiUserColumnMigrator(table, oldColumn, newColumn)
+  }
+
+  throw new BadRequestError(`Unknown migration type`)
+}
+
+abstract class UserColumnMigrator implements ColumnMigrator {
+  constructor(
+    protected table: Table,
+    protected oldColumn: RelationshipFieldMetadata,
+    protected newColumn: BBReferenceFieldMetadata
+  ) {}
+
+  abstract updateRow(row: Row, link: LinkDocument): void
+
+  async doMigration(): Promise<MigrationResult> {
+    let oldTable = cloneDeep(this.table)
+    let rows = await sdk.rows.fetchRaw(this.table._id!)
+    let rowsById = rows.reduce((acc, row) => {
+      acc[row._id!] = row
+      return acc
+    }, {} as Record<string, Row>)
+
+    let links = await sdk.links.fetchWithDocument(this.table._id!)
+    for (let link of links) {
+      if (
+        link.doc1.tableId !== this.table._id ||
+        link.doc1.fieldName !== this.oldColumn.name ||
+        link.doc2.tableId !== InternalTable.USER_METADATA
+      ) {
+        continue
+      }
+
+      let row = rowsById[link.doc1.rowId]
+      if (!row) {
+        // This can happen if the row has been deleted but the link hasn't,
+        // which was a state that was found during the initial testing of this
+        // feature. Not sure exactly what can cause it, but best to be safe.
+        continue
+      }
+
+      this.updateRow(row, link)
+    }
+
+    let db = context.getAppDB()
+    await db.bulkDocs(rows)
+
+    delete this.table.schema[this.oldColumn.name]
+    this.table = await sdk.tables.saveTable(this.table)
+    await updateLinks({
+      eventType: EventType.TABLE_UPDATED,
+      table: this.table,
+      oldTable,
+    })
+
+    let otherTable = await sdk.tables.getTable(this.oldColumn.tableId)
+    return {
+      tablesUpdated: [this.table, otherTable],
+    }
+  }
+}
+
+class SingleUserColumnMigrator extends UserColumnMigrator {
+  updateRow(row: Row, link: LinkDocument): void {
+    row[this.newColumn.name] = dbCore.getGlobalIDFromUserMetadataID(
+      link.doc2.rowId
+    )
+  }
+}
+
+class MultiUserColumnMigrator extends UserColumnMigrator {
+  updateRow(row: Row, link: LinkDocument): void {
+    if (!row[this.newColumn.name]) {
+      row[this.newColumn.name] = []
+    }
+    row[this.newColumn.name].push(
+      dbCore.getGlobalIDFromUserMetadataID(link.doc2.rowId)
+    )
+  }
+}

packages/server/src/sdk/app/tables/tests/tables.spec.ts

@@ -1,4 +1,10 @@
-import { FieldType, Table, ViewV2 } from "@budibase/types"
+import {
+  FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
+  Table,
+  TableSourceType,
+  ViewV2,
+} from "@budibase/types"
 import { generator } from "@budibase/backend-core/tests"
 import sdk from "../../.."
 
@@ -13,6 +19,8 @@ describe("table sdk", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
       schema: {
         name: {
           type: FieldType.STRING,

packages/server/src/sdk/app/tables/tests/validation.spec.ts

@@ -1,73 +1,92 @@
 import { populateExternalTableSchemas } from "../validation"
 import { cloneDeep } from "lodash/fp"
-import { AutoReason, Datasource, Table } from "@budibase/types"
+import {
+  AutoReason,
+  Datasource,
+  FieldType,
+  RelationshipType,
+  SourceName,
+  Table,
+  TableSourceType,
+} from "@budibase/types"
 import { isEqual } from "lodash"
+import { generateDatasourceID } from "../../../../db/utils"
 
-const SCHEMA = {
+const datasourceId = generateDatasourceID()
+
+const SCHEMA: Datasource = {
+  source: SourceName.POSTGRES,
+  type: "datasource",
+  _id: datasourceId,
   entities: {
     client: {
+      type: "table",
       _id: "tableA",
       name: "client",
       primary: ["idC"],
       primaryDisplay: "Name",
+      sourceId: datasourceId,
+      sourceType: TableSourceType.EXTERNAL,
       schema: {
         idC: {
           autocolumn: true,
           externalType: "int unsigned",
           name: "idC",
-          type: "number",
+          type: FieldType.NUMBER,
         },
         Name: {
           autocolumn: false,
           externalType: "varchar(255)",
           name: "Name",
-          type: "string",
+          type: FieldType.STRING,
         },
         project: {
           fieldName: "idC",
           foreignKey: "idC",
           main: true,
           name: "project",
-          relationshipType: "many-to-one",
+          relationshipType: RelationshipType.MANY_TO_ONE,
           tableId: "tableB",
-          type: "link",
+          type: FieldType.LINK,
         },
       },
     },
     project: {
+      type: "table",
       _id: "tableB",
       name: "project",
       primary: ["idP"],
       primaryDisplay: "Name",
+      sourceId: datasourceId,
+      sourceType: TableSourceType.EXTERNAL,
       schema: {
         idC: {
           externalType: "int unsigned",
           name: "idC",
-          type: "number",
+          type: FieldType.NUMBER,
         },
         idP: {
           autocolumn: true,
           externalType: "int unsigned",
           name: "idProject",
-          type: "number",
+          type: FieldType.NUMBER,
         },
         Name: {
           autocolumn: false,
           externalType: "varchar(255)",
           name: "Name",
-          type: "string",
+          type: FieldType.STRING,
         },
         client: {
           fieldName: "idC",
           foreignKey: "idC",
           name: "client",
-          relationshipType: "one-to-many",
+          relationshipType: RelationshipType.ONE_TO_MANY,
           tableId: "tableA",
-          type: "link",
+          type: FieldType.LINK,
         },
       },
       sql: true,
-      type: "table",
     },
   },
 }
@@ -95,12 +114,12 @@ describe("validation and update of external table schemas", () => {
   function noOtherTableChanges(response: any) {
     checkOtherColumns(
       response.entities!.client!,
-      SCHEMA.entities.client as Table,
+      SCHEMA.entities!.client,
       OTHER_CLIENT_COLS
     )
     checkOtherColumns(
       response.entities!.project!,
-      SCHEMA.entities.project as Table,
+      SCHEMA.entities!.project,
       OTHER_PROJECT_COLS
     )
   }

packages/server/src/sdk/app/tables/update.ts

@@ -1,23 +1,30 @@
 import { Table, RenameColumn } from "@budibase/types"
-import { isExternalTable } from "../../../integrations/utils"
+import { isExternalTableID } from "../../../integrations/utils"
 import sdk from "../../index"
 import { context } from "@budibase/backend-core"
 import { isExternal } from "./utils"
+import { DocumentInsertResponse } from "@budibase/nano"
 
 import * as external from "./external"
 import * as internal from "./internal"
+import { cloneDeep } from "lodash"
 export * as external from "./external"
 export * as internal from "./internal"
 
-export async function saveTable(table: Table) {
+export async function saveTable(table: Table): Promise<Table> {
   const db = context.getAppDB()
-  if (isExternalTable(table._id!)) {
+  let resp: DocumentInsertResponse
+  if (isExternalTableID(table._id!)) {
     const datasource = await sdk.datasources.get(table.sourceId!)
     datasource.entities![table.name] = table
-    await db.put(datasource)
+    resp = await db.put(datasource)
   } else {
-    await db.put(table)
+    resp = await db.put(table)
   }
+
+  let tableClone = cloneDeep(table)
+  tableClone._rev = resp.rev
+  return tableClone
 }
 
 export async function update(table: Table, renaming?: RenameColumn) {

packages/server/src/sdk/app/tables/utils.ts

@@ -1,10 +1,10 @@
-import { Table } from "@budibase/types"
+import { Table, TableSourceType } from "@budibase/types"
-import { isExternalTable } from "../../../integrations/utils"
+import { isExternalTableID } from "../../../integrations/utils"
 
 export function isExternal(opts: { table?: Table; tableId?: string }): boolean {
-  if (opts.table && opts.table.type === "external") {
+  if (opts.table && opts.table.sourceType === TableSourceType.EXTERNAL) {
     return true
-  } else if (opts.tableId && isExternalTable(opts.tableId)) {
+  } else if (opts.tableId && isExternalTableID(opts.tableId)) {
     return true
   }
   return false

packages/server/src/sdk/app/views/index.ts

@@ -4,13 +4,13 @@ import { cloneDeep } from "lodash"
 
 import sdk from "../../../sdk"
 import * as utils from "../../../db/utils"
-import { isExternalTable } from "../../../integrations/utils"
+import { isExternalTableID } from "../../../integrations/utils"
 
 import * as internal from "./internal"
 import * as external from "./external"
 
 function pickApi(tableId: any) {
-  if (isExternalTable(tableId)) {
+  if (isExternalTableID(tableId)) {
     return external
   }
   return internal

packages/server/src/sdk/app/views/tests/views.spec.ts

@@ -2,8 +2,10 @@ import _ from "lodash"
 import {
   FieldSchema,
   FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
   Table,
   TableSchema,
+  TableSourceType,
   ViewV2,
 } from "@budibase/types"
 import { generator } from "@budibase/backend-core/tests"
@@ -14,6 +16,8 @@ describe("table sdk", () => {
     _id: generator.guid(),
     name: "TestTable",
     type: "table",
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
+    sourceType: TableSourceType.INTERNAL,
     schema: {
       name: {
         type: FieldType.STRING,