Merge branch 'master' into master

.github/workflows/release-master.yml

@@ -36,6 +36,7 @@ jobs:
       - uses: actions/setup-node@v1
         with:
           node-version: 18.x
+          cache: yarn
 
       - run: yarn install --frozen-lockfile
       - name: Update versions
@@ -63,14 +64,64 @@ jobs:
           echo "Using tag $version"
           echo "version=$version" >> "$GITHUB_OUTPUT"
 
-      - name: Build/release Docker images
+      - name: Setup Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Docker login
         run: |
           docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build:docker
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }}
+
+      - name: Build worker docker
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
+          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
+          file: ./packages/worker/Dockerfile.v2
+          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
+          cache-to: type=inline
+        env:
+          IMAGE_NAME: budibase/worker
+          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
+          BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
+
+      - name: Build server docker
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
+          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
+          file: ./packages/server/Dockerfile.v2
+          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
+          cache-to: type=inline
+        env:
+          IMAGE_NAME: budibase/apps
+          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
+          BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
+
+      - name: Build proxy docker
+        uses: docker/build-push-action@v5
+        with:
+          context: ./hosting/proxy
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
+          file: ./hosting/proxy/Dockerfile
+          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
+          cache-to: type=inline
+        env:
+          IMAGE_NAME: budibase/proxy
+          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
 
   release-helm-chart:
     needs: [release-images]

.github/workflows/release-singleimage.yml

@@ -67,7 +67,7 @@ jobs:
           push: true
           platforms: linux/amd64,linux/arm64
           tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
+          file: ./hosting/single/Dockerfile.v2
       - name: Tag and release Budibase Azure App Service docker image
         uses: docker/build-push-action@v2
         with:
@@ -76,4 +76,4 @@ jobs:
           platforms: linux/amd64
           build-args: TARGETBUILD=aas
           tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
+          file: ./hosting/single/Dockerfile.v2

README.md

@@ -126,13 +126,6 @@ You can learn more about the Budibase API at the following places:
 
 - [Build an app with Budibase and Next.js](https://budibase.com/blog/building-a-crud-app-with-budibase-and-next.js/)
 
-<p align="center">
-  <img alt="Budibase data" src="https://res.cloudinary.com/daog6scxm/image/upload/v1647858558/Feb%20release/Start_building_with_Budibase_s_API_3_rhlzhv.png">
-</p>
-<br /><br />
-
-<br /><br /><br />
-
 ## 🏁 Get started
 
 Deploy Budibase self-hosted in your existing infrastructure, using Docker, Kubernetes, and Digital Ocean.

hosting/scripts/linux/release-to-docker-hub.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-tag=$1
-
-if [[ ! "$tag" ]]; then
-	echo "No tag present. You must pass a tag to this script"
-	exit 1
-fi
-
-echo "Tagging images with tag: $tag"
-
-docker tag proxy-service budibase/proxy:$tag
-docker tag app-service budibase/apps:$tag
-docker tag worker-service budibase/worker:$tag
-
-docker push --all-tags budibase/apps 
-docker push --all-tags budibase/worker
-docker push --all-tags budibase/proxy

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.11.45",
+  "version": "2.12.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -54,10 +54,6 @@
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
     "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
     "build:specs": "lerna run --stream specs",
-    "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
-    "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
-    "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
-    "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",

packages/backend-core/src/cache/writethrough.ts

@@ -119,8 +119,8 @@ export class Writethrough {
     this.writeRateMs = writeRateMs
   }
 
-  async put(doc: any) {
+  async put(doc: any, writeRateMs: number = this.writeRateMs) {
-    return put(this.db, doc, this.writeRateMs)
+    return put(this.db, doc, writeRateMs)
   }
 
   async get(id: string) {

packages/backend-core/src/security/roles.ts

@@ -122,7 +122,9 @@ export async function roleToNumber(id?: string) {
   if (isBuiltin(id)) {
     return builtinRoleToNumber(id)
   }
-  const hierarchy = (await getUserRoleHierarchy(id)) as RoleDoc[]
+  const hierarchy = (await getUserRoleHierarchy(id, {
+    defaultPublic: true,
+  })) as RoleDoc[]
   for (let role of hierarchy) {
     if (isBuiltin(role?.inherits)) {
       return builtinRoleToNumber(role.inherits) + 1
@@ -192,12 +194,15 @@ export async function getRole(
 /**
  * Simple function to get all the roles based on the top level user role ID.
  */
-async function getAllUserRoles(userRoleId?: string): Promise<RoleDoc[]> {
+async function getAllUserRoles(
+  userRoleId?: string,
+  opts?: { defaultPublic?: boolean }
+): Promise<RoleDoc[]> {
   // admins have access to all roles
   if (userRoleId === BUILTIN_IDS.ADMIN) {
     return getAllRoles()
   }
-  let currentRole = await getRole(userRoleId)
+  let currentRole = await getRole(userRoleId, opts)
   let roles = currentRole ? [currentRole] : []
   let roleIds = [userRoleId]
   // get all the inherited roles
@@ -226,12 +231,16 @@ export async function getUserRoleIdHierarchy(
  * Returns an ordered array of the user's inherited role IDs, this can be used
  * to determine if a user can access something that requires a specific role.
  * @param userRoleId The user's role ID, this can be found in their access token.
+ * @param opts optional - if want to default to public use this.
  * @returns returns an ordered array of the roles, with the first being their
  * highest level of access and the last being the lowest level.
  */
-export async function getUserRoleHierarchy(userRoleId?: string) {
+export async function getUserRoleHierarchy(
+  userRoleId?: string,
+  opts?: { defaultPublic?: boolean }
+) {
   // special case, if they don't have a role then they are a public user
-  return getAllUserRoles(userRoleId)
+  return getAllUserRoles(userRoleId, opts)
 }
 
 // this function checks that the provided permissions are in an array format

packages/backend-core/src/users/db.ts

@@ -25,12 +25,17 @@ import {
 import {
   getAccountHolderFromUserIds,
   isAdmin,
+  isCreator,
   validateUniqueUser,
 } from "./utils"
 import { searchExistingEmails } from "./lookup"
 import { hash } from "../utils"
 
-type QuotaUpdateFn = (change: number, cb?: () => Promise<any>) => Promise<any>
+type QuotaUpdateFn = (
+  change: number,
+  creatorsChange: number,
+  cb?: () => Promise<any>
+) => Promise<any>
 type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise<any>
 type FeatureFn = () => Promise<Boolean>
 type GroupGetFn = (ids: string[]) => Promise<UserGroup[]>
@@ -245,7 +250,8 @@ export class UserDB {
     }
 
     const change = dbUser ? 0 : 1 // no change if there is existing user
-    return UserDB.quotas.addUsers(change, async () => {
+    const creatorsChange = isCreator(dbUser) !== isCreator(user) ? 1 : 0
+    return UserDB.quotas.addUsers(change, creatorsChange, async () => {
       await validateUniqueUser(email, tenantId)
 
       let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser)
@@ -307,6 +313,7 @@ export class UserDB {
 
     let usersToSave: any[] = []
     let newUsers: any[] = []
+    let newCreators: any[] = []
 
     const emails = newUsersRequested.map((user: User) => user.email)
     const existingEmails = await searchExistingEmails(emails)
@@ -327,59 +334,66 @@ export class UserDB {
       }
       newUser.userGroups = groups
       newUsers.push(newUser)
+      if (isCreator(newUser)) {
+        newCreators.push(newUser)
+      }
     }
 
     const account = await accountSdk.getAccountByTenantId(tenantId)
-    return UserDB.quotas.addUsers(newUsers.length, async () => {
+    return UserDB.quotas.addUsers(
-      // create the promises array that will be called by bulkDocs
+      newUsers.length,
-      newUsers.forEach((user: any) => {
+      newCreators.length,
-        usersToSave.push(
+      async () => {
-          UserDB.buildUser(
+        // create the promises array that will be called by bulkDocs
-            user,
+        newUsers.forEach((user: any) => {
-            {
+          usersToSave.push(
-              hashPassword: true,
+            UserDB.buildUser(
-              requirePassword: user.requirePassword,
+              user,
-            },
+              {
-            tenantId,
+                hashPassword: true,
-            undefined, // no dbUser
+                requirePassword: user.requirePassword,
-            account
+              },
+              tenantId,
+              undefined, // no dbUser
+              account
+            )
           )
-        )
+        })
-      })
 
-      const usersToBulkSave = await Promise.all(usersToSave)
+        const usersToBulkSave = await Promise.all(usersToSave)
-      await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
+        await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
 
-      // Post-processing of bulk added users, e.g. events and cache operations
+        // Post-processing of bulk added users, e.g. events and cache operations
-      for (const user of usersToBulkSave) {
+        for (const user of usersToBulkSave) {
-        // TODO: Refactor to bulk insert users into the info db
+          // TODO: Refactor to bulk insert users into the info db
-        // instead of relying on looping tenant creation
+          // instead of relying on looping tenant creation
-        await platform.users.addUser(tenantId, user._id, user.email)
+          await platform.users.addUser(tenantId, user._id, user.email)
-        await eventHelpers.handleSaveEvents(user, undefined)
+          await eventHelpers.handleSaveEvents(user, undefined)
-      }
+        }
+
+        const saved = usersToBulkSave.map(user => {
+          return {
+            _id: user._id,
+            email: user.email,
+          }
+        })
+
+        // now update the groups
+        if (Array.isArray(saved) && groups) {
+          const groupPromises = []
+          const createdUserIds = saved.map(user => user._id)
+          for (let groupId of groups) {
+            groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds))
+          }
+          await Promise.all(groupPromises)
+        }
 
-      const saved = usersToBulkSave.map(user => {
         return {
-          _id: user._id,
+          successful: saved,
-          email: user.email,
+          unsuccessful,
         }
-      })
-
-      // now update the groups
-      if (Array.isArray(saved) && groups) {
-        const groupPromises = []
-        const createdUserIds = saved.map(user => user._id)
-        for (let groupId of groups) {
-          groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds))
-        }
-        await Promise.all(groupPromises)
       }
-
+    )
-      return {
-        successful: saved,
-        unsuccessful,
-      }
-    })
   }
 
   static async bulkDelete(userIds: string[]): Promise<BulkUserDeleted> {
@@ -419,11 +433,12 @@ export class UserDB {
       _deleted: true,
     }))
     const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete)
+    const creatorsToDelete = usersToDelete.filter(isCreator)
 
-    await UserDB.quotas.removeUsers(toDelete.length)
     for (let user of usersToDelete) {
       await bulkDeleteProcessing(user)
     }
+    await UserDB.quotas.removeUsers(toDelete.length, creatorsToDelete.length)
 
     // Build Response
     // index users by id
@@ -472,7 +487,8 @@ export class UserDB {
 
     await db.remove(userId, dbUser._rev)
 
-    await UserDB.quotas.removeUsers(1)
+    const creatorsToDelete = isCreator(dbUser) ? 1 : 0
+    await UserDB.quotas.removeUsers(1, creatorsToDelete)
     await eventHelpers.handleDeleteEvents(dbUser)
     await cache.user.invalidateUser(userId)
     await sessions.invalidateSessions(userId, { reason: "deletion" })

packages/backend-core/src/users/users.ts

@@ -14,11 +14,11 @@ import {
 } from "../db"
 import {
   BulkDocsResponse,
-  ContextUser,
   SearchQuery,
   SearchQueryOperators,
   SearchUsersRequest,
   User,
+  ContextUser,
   DatabaseQueryOpts,
 } from "@budibase/types"
 import { getGlobalDB } from "../context"

packages/backend-core/tests/core/users/users.spec.js

@@ -0,0 +1,54 @@
+const _ = require('lodash/fp')
+const {structures} = require("../../../tests")
+
+jest.mock("../../../src/context")
+jest.mock("../../../src/db")
+
+const context = require("../../../src/context")
+const db = require("../../../src/db")
+
+const {getCreatorCount} = require('../../../src/users/users')
+
+describe("Users", () => {
+
+  let getGlobalDBMock
+  let getGlobalUserParamsMock
+  let paginationMock
+
+  beforeEach(() => {
+    jest.resetAllMocks()
+
+    getGlobalDBMock = jest.spyOn(context, "getGlobalDB")
+    getGlobalUserParamsMock = jest.spyOn(db, "getGlobalUserParams")
+    paginationMock = jest.spyOn(db, "pagination")
+  })
+
+  it("Retrieves the number of creators", async () => {
+    const getUsers = (offset, limit, creators = false) => {
+      const range = _.range(offset, limit)
+      const opts = creators ? {builder: {global: true}} : undefined
+      return range.map(() => structures.users.user(opts))
+    }
+    const page1Data = getUsers(0, 8)
+    const page2Data = getUsers(8, 12, true)
+    getGlobalDBMock.mockImplementation(() => ({
+      name   : "fake-db",
+      allDocs: () => ({
+        rows: [...page1Data, ...page2Data]
+      })
+    }))
+    paginationMock.mockImplementationOnce(() => ({
+      data: page1Data,
+      hasNextPage: true,
+      nextPage: "1"
+    }))
+    paginationMock.mockImplementation(() => ({
+      data: page2Data,
+      hasNextPage: false,
+      nextPage: undefined
+    }))
+    const creatorsCount = await getCreatorCount()
+    expect(creatorsCount).toBe(4)
+    expect(paginationMock).toHaveBeenCalledTimes(2)
+  })
+})

packages/backend-core/tests/core/utilities/structures/licenses.ts

@@ -123,6 +123,10 @@ export function customer(): Customer {
 export function subscription(): Subscription {
   return {
     amount: 10000,
+    amounts: {
+      user: 10000,
+      creator: 0,
+    },
     cancelAt: undefined,
     currency: "usd",
     currentPeriodEnd: 0,
@@ -131,6 +135,10 @@ export function subscription(): Subscription {
     duration: PriceDuration.MONTHLY,
     pastDueAt: undefined,
     quantity: 0,
+    quantities: {
+      user: 0,
+      creator: 0,
+    },
     status: "active",
   }
 }

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -159,8 +159,10 @@
           {#if selectedImage.size}
             <div class="filesize">
               {#if selectedImage.size <= BYTES_IN_MB}
-                {`${selectedImage.size / BYTES_IN_KB} KB`}
+                {`${(selectedImage.size / BYTES_IN_KB).toFixed(1)} KB`}
-              {:else}{`${selectedImage.size / BYTES_IN_MB} MB`}{/if}
+              {:else}{`${(selectedImage.size / BYTES_IN_MB).toFixed(
+                  1
+                )} MB`}{/if}
             </div>
           {/if}
           {#if !disabled}
@@ -203,8 +205,8 @@
             {#if file.size}
               <div class="filesize">
                 {#if file.size <= BYTES_IN_MB}
-                  {`${file.size / BYTES_IN_KB} KB`}
+                  {`${(file.size / BYTES_IN_KB).toFixed(1)} KB`}
-                {:else}{`${file.size / BYTES_IN_MB} MB`}{/if}
+                {:else}{`${(file.size / BYTES_IN_MB).toFixed(1)} MB`}{/if}
               </div>
             {/if}
             {#if !disabled}

packages/builder/src/components/common/Dropzone.svelte

@@ -23,7 +23,7 @@
     try {
       return await API.uploadBuilderAttachment(data)
     } catch (error) {
-      notifications.error("Failed to upload attachment")
+      notifications.error(error.message || "Failed to upload attachment")
       return []
     }
   }

packages/builder/src/components/common/RoleSelect.svelte

@@ -39,7 +39,15 @@
     allowCreator
   ) => {
     if (allowedRoles?.length) {
-      return roles.filter(role => allowedRoles.includes(role._id))
+      const filteredRoles = roles.filter(role =>
+        allowedRoles.includes(role._id)
+      )
+      return [
+        ...filteredRoles,
+        ...(allowedRoles.includes(Constants.Roles.CREATOR)
+          ? [{ _id: Constants.Roles.CREATOR, name: "Creator", enabled: false }]
+          : []),
+      ]
     }
     let newRoles = [...roles]
 
@@ -129,8 +137,9 @@
     getOptionColour={getColor}
     getOptionIcon={getIcon}
     isOptionEnabled={option =>
-      option._id !== Constants.Roles.CREATOR ||
+      (option._id !== Constants.Roles.CREATOR ||
-      $licensing.perAppBuildersEnabled}
+        $licensing.perAppBuildersEnabled) &&
+      option.enabled !== false}
     {placeholder}
     {error}
   />

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -516,6 +516,13 @@
     }
     return null
   }
+
+  const parseRole = user => {
+    if (user.isAdminOrGlobalBuilder) {
+      return Constants.Roles.CREATOR
+    }
+    return user.role
+  }
 </script>
 
 <svelte:window on:keydown={handleKeyDown} />
@@ -725,7 +732,7 @@
                     <RoleSelect
                       footer={getRoleFooter(user)}
                       placeholder={false}
-                      value={user.role}
+                      value={parseRole(user)}
                       allowRemove={user.role && !user.group}
                       allowPublic={false}
                       allowCreator={true}
@@ -744,7 +751,7 @@
                       autoWidth
                       align="right"
                       allowedRoles={user.isAdminOrGlobalBuilder
-                        ? [Constants.Roles.ADMIN]
+                        ? [Constants.Roles.CREATOR]
                         : null}
                     />
                   </div>

packages/client/src/utils/buttonActions.js

@@ -103,7 +103,6 @@ const fetchRowHandler = async action => {
 
 const deleteRowHandler = async action => {
   const { tableId, rowId: rowConfig, notificationOverride } = action.parameters
-
   if (tableId && rowConfig) {
     try {
       let requestConfig
@@ -129,9 +128,11 @@ const deleteRowHandler = async action => {
         requestConfig = [parsedRowConfig]
       } else if (Array.isArray(parsedRowConfig)) {
         requestConfig = parsedRowConfig
+      } else if (Number.isInteger(parsedRowConfig)) {
+        requestConfig = [String(parsedRowConfig)]
       }
 
-      if (!requestConfig.length) {
+      if (!requestConfig && !parsedRowConfig) {
         notificationStore.actions.warning("No valid rows were supplied")
         return false
       }

packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte

@@ -55,7 +55,7 @@
     try {
       return await API.uploadBuilderAttachment(data)
     } catch (error) {
-      $notifications.error("Failed to upload attachment")
+      $notifications.error(error.message || "Failed to upload attachment")
       return []
     }
   }

packages/pro

@@ -1 +1 @@
-Subproject commit d24c0dc3a30014cbe61860252aa48104cad36376
+Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8

packages/server/Dockerfile

@@ -38,7 +38,7 @@ RUN apt update && apt upgrade -y \
 
 COPY package.json .
 COPY dist/yarn.lock .
-RUN yarn install --production=true \
+RUN yarn install --production=true --network-timeout 1000000 \
     # Remove unneeded data from file system to reduce image size
     && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \
     && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp

packages/server/package.json

@@ -18,7 +18,6 @@
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "build:docker": "yarn build && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",

packages/server/src/api/controllers/static/index.ts

@@ -1,3 +1,5 @@
+import { ValidFileExtensions } from "@budibase/shared-core"
+
 require("svelte/register")
 
 import { join } from "../../../utilities/centralPath"
@@ -11,34 +13,21 @@ import {
 } from "../../../utilities/fileSystem"
 import env from "../../../environment"
 import { DocumentType } from "../../../db/utils"
-import { context, objectStore, utils, configs } from "@budibase/backend-core"
+import {
+  context,
+  objectStore,
+  utils,
+  configs,
+  BadRequestError,
+} from "@budibase/backend-core"
 import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
-import { App, Ctx } from "@budibase/types"
+import { App, Ctx, ProcessAttachmentResponse, Upload } from "@budibase/types"
 
 const send = require("koa-send")
 
-async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
-  const response = await objectStore.upload({
-    bucket,
-    metadata,
-    filename: s3Key,
-    path: file.path,
-    type: file.type,
-  })
-
-  // don't store a URL, work this out on the way out as the URL could change
-  return {
-    size: file.size,
-    name: file.name,
-    url: objectStore.getAppFileUrl(s3Key),
-    extension: [...file.name.split(".")].pop(),
-    key: response.Key,
-  }
-}
-
 export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
 
@@ -72,23 +61,58 @@ export const serveBuilder = async function (ctx: Ctx) {
   await send(ctx, ctx.file, { root: builderPath })
 }
 
-export const uploadFile = async function (ctx: Ctx) {
+export const uploadFile = async function (
+  ctx: Ctx<{}, ProcessAttachmentResponse>
+) {
   const file = ctx.request?.files?.file
+  if (!file) {
+    throw new BadRequestError("No file provided")
+  }
+
   let files = file && Array.isArray(file) ? Array.from(file) : [file]
 
-  const uploads = files.map(async (file: any) => {
+  ctx.body = await Promise.all(
-    const fileExtension = [...file.name.split(".")].pop()
+    files.map(async file => {
-    // filenames converted to UUIDs so they are unique
+      if (!file.name) {
-    const processedFileName = `${uuid.v4()}.${fileExtension}`
+        throw new BadRequestError(
+          "Attempted to upload a file without a filename"
+        )
+      }
 
-    return prepareUpload({
+      const extension = [...file.name.split(".")].pop()
-      file,
+      if (!extension) {
-      s3Key: `${context.getProdAppId()}/attachments/${processedFileName}`,
+        throw new BadRequestError(
-      bucket: ObjectStoreBuckets.APPS,
+          `File "${file.name}" has no extension, an extension is required to upload a file`
+        )
+      }
+
+      if (!env.SELF_HOSTED && !ValidFileExtensions.includes(extension)) {
+        throw new BadRequestError(
+          `File "${file.name}" has an invalid extension: "${extension}"`
+        )
+      }
+
+      // filenames converted to UUIDs so they are unique
+      const processedFileName = `${uuid.v4()}.${extension}`
+
+      const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}`
+
+      const response = await objectStore.upload({
+        bucket: ObjectStoreBuckets.APPS,
+        filename: s3Key,
+        path: file.path,
+        type: file.type,
+      })
+
+      return {
+        size: file.size,
+        name: file.name,
+        url: objectStore.getAppFileUrl(s3Key),
+        extension,
+        key: response.Key,
+      }
     })
-  })
+  )
-
-  ctx.body = await Promise.all(uploads)
 }
 
 export const deleteObjects = async function (ctx: Ctx) {

packages/server/src/api/routes/row.ts

@@ -11,128 +11,24 @@ const { PermissionType, PermissionLevel } = permissions
 const router: Router = new Router()
 
 router
-  /**
-   * @api {get} /api/:sourceId/:rowId/enrich Get an enriched row
-   * @apiName Get an enriched row
-   * @apiGroup rows
-   * @apiPermission table read access
-   * @apiDescription This API is only useful when dealing with rows that have relationships.
-   * Normally when a row is a returned from the API relationships will only have the structure
-   * `{ primaryDisplay: "name", _id: ... }` but this call will return the full related rows
-   * for each relationship instead.
-   *
-   * @apiParam {string} rowId The ID of the row which is to be retrieved and enriched.
-   *
-   * @apiSuccess {object} row The response body will be the enriched row.
-   */
   .get(
     "/api/:sourceId/:rowId/enrich",
     paramSubResource("sourceId", "rowId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.fetchEnrichedRow
   )
-  /**
-   * @api {get} /api/:sourceId/rows Get all rows in a table
-   * @apiName Get all rows in a table
-   * @apiGroup rows
-   * @apiPermission table read access
-   * @apiDescription This is a deprecated endpoint that should not be used anymore, instead use the search endpoint.
-   * This endpoint gets all of the rows within the specified table - it is not heavily used
-   * due to its lack of support for pagination. With SQL tables this will retrieve up to a limit and then
-   * will simply stop.
-   *
-   * @apiParam {string} sourceId The ID of the table to retrieve all rows within.
-   *
-   * @apiSuccess {object[]} rows The response body will be an array of all rows found.
-   */
   .get(
     "/api/:sourceId/rows",
     paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.fetch
   )
-  /**
-   * @api {get} /api/:sourceId/rows/:rowId Retrieve a single row
-   * @apiName Retrieve a single row
-   * @apiGroup rows
-   * @apiPermission table read access
-   * @apiDescription This endpoint retrieves only the specified row. If you wish to retrieve
-   * a row by anything other than its _id field, use the search endpoint.
-   *
-   * @apiParam {string} sourceId The ID of the table to retrieve a row from.
-   * @apiParam {string} rowId The ID of the row to retrieve.
-   *
-   * @apiSuccess {object} body The response body will be the row that was found.
-   */
   .get(
     "/api/:sourceId/rows/:rowId",
     paramSubResource("sourceId", "rowId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.find
   )
-  /**
-   * @api {post} /api/:sourceId/search Search for rows in a table
-   * @apiName Search for rows in a table
-   * @apiGroup rows
-   * @apiPermission table read access
-   * @apiDescription This is the primary method of accessing rows in Budibase, the data provider
-   * and data UI in the builder are built atop this. All filtering, sorting and pagination is
-   * handled through this, for internal and external (datasource plus, e.g. SQL) tables.
-   *
-   * @apiParam {string} sourceId The ID of the table to retrieve rows from.
-   *
-   * @apiParam (Body) {boolean} [paginate] If pagination is required then this should be set to true,
-   * defaults to false.
-   * @apiParam (Body) {object} [query] This contains a set of filters which should be applied, if none
-   * specified then the request will be unfiltered. An example with all of the possible query
-   * options has been supplied below.
-   * @apiParam (Body) {number} [limit] This sets a limit for the number of rows that will be returned,
-   * this will be implemented at the database level if supported for performance reasons. This
-   * is useful when paginating to set exactly how many rows per page.
-   * @apiParam (Body) {string} [bookmark] If pagination is enabled then a bookmark will be returned
-   * with each successful search request, this should be supplied back to get the next page.
-   * @apiParam (Body) {object} [sort] If sort is desired this should contain the name of the column to
-   * sort on.
-   * @apiParam (Body) {string} [sortOrder] If sort is enabled then this can be either "descending" or
-   * "ascending" as required.
-   * @apiParam (Body) {string} [sortType] If sort is enabled then you must specify the type of search
-   * being used, either "string" or "number". This is only used for internal tables.
-   *
-   * @apiParamExample {json} Example:
-   * {
-   *  "tableId": "ta_70260ff0b85c467ca74364aefc46f26d",
-   *  "query": {
-   *    "string": {},
-   *    "fuzzy": {},
-   *    "range": {
-   *      "columnName": {
-   *        "high": 20,
-   *        "low": 10,
-   *      }
-   *    },
-   *    "equal": {
-   *      "columnName": "someValue"
-   *    },
-   *    "notEqual": {},
-   *    "empty": {},
-   *    "notEmpty": {},
-   *    "oneOf": {
-   *      "columnName": ["value"]
-   *    }
-   *  },
-   *  "limit": 10,
-   *  "sort": "name",
-   *  "sortOrder": "descending",
-   *  "sortType": "string",
-   *  "paginate": true
-   * }
-   *
-   * @apiSuccess {object[]} rows An array of rows that was found based on the supplied parameters.
-   * @apiSuccess {boolean} hasNextPage If pagination was enabled then this specifies whether or
-   * not there is another page after this request.
-   * @apiSuccess {string} bookmark The bookmark to be sent with the next request to get the next
-   * page.
-   */
   .post(
     "/api/:sourceId/search",
     internalSearchValidator(),
@@ -148,30 +44,6 @@ router
     authorized(PermissionType.TABLE, PermissionLevel.READ),
     rowController.search
   )
-  /**
-   * @api {post} /api/:sourceId/rows Creates a new row
-   * @apiName Creates a new row
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This API will create a new row based on the supplied body. If the
-   * body includes an "_id" field then it will update an existing row if the field
-   * links to one. Please note that "_id", "_rev" and "tableId" are fields that are
-   * already used by Budibase tables and cannot be used for columns.
-   *
-   * @apiParam {string} sourceId The ID of the table to save a row to.
-   *
-   * @apiParam (Body) {string} [_id] If the row exists already then an ID for the row must be provided.
-   * @apiParam (Body) {string} [_rev] If working with an existing row for an internal table its revision
-   * must also be provided.
-   * @apiParam (Body) {string} tableId The ID of the table should also be specified in the row body itself.
-   * @apiParam (Body) {any} [any] Any field supplied in the body will be assessed to see if it matches
-   * a column in the specified table. All other fields will be dropped and not stored.
-   *
-   * @apiSuccess {string} _id The ID of the row that was just saved, if it was just created this
-   * is the rows new ID.
-   * @apiSuccess {string} [_rev] If saving to an internal table a revision will also be returned.
-   * @apiSuccess {object} body The contents of the row that was saved will be returned as well.
-   */
   .post(
     "/api/:sourceId/rows",
     paramResource("sourceId"),
@@ -179,14 +51,6 @@ router
     trimViewRowInfo,
     rowController.save
   )
-  /**
-   * @api {patch} /api/:sourceId/rows Updates a row
-   * @apiName Update a row
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This endpoint is identical to the row creation endpoint but instead it will
-   * error if an _id isn't provided, it will only function for existing rows.
-   */
   .patch(
     "/api/:sourceId/rows",
     paramResource("sourceId"),
@@ -194,52 +58,12 @@ router
     trimViewRowInfo,
     rowController.patch
   )
-  /**
-   * @api {post} /api/:sourceId/rows/validate Validate inputs for a row
-   * @apiName Validate inputs for a row
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription When attempting to save a row you may want to check if the row is valid
-   * given the table schema, this will iterate through all the constraints on the table and
-   * check if the request body is valid.
-   *
-   * @apiParam {string} sourceId The ID of the table the row is to be validated for.
-   *
-   * @apiParam (Body) {any} [any] Any fields provided in the request body will be tested
-   * against the table schema and constraints.
-   *
-   * @apiSuccess {boolean} valid If inputs provided are acceptable within the table schema this
-   * will be true, if it is not then then errors property will be populated.
-   * @apiSuccess {object} [errors] A key value map of information about fields on the input
-   * which do not match the table schema. The key name will be the column names that have breached
-   * the schema.
-   */
   .post(
     "/api/:sourceId/rows/validate",
     paramResource("sourceId"),
     authorized(PermissionType.TABLE, PermissionLevel.WRITE),
     rowController.validate
   )
-  /**
-   * @api {delete} /api/:sourceId/rows Delete rows
-   * @apiName Delete rows
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This endpoint can delete a single row, or delete them in a bulk
-   * fashion.
-   *
-   * @apiParam {string} sourceId The ID of the table the row is to be deleted from.
-   *
-   * @apiParam (Body) {object[]} [rows] If bulk deletion is desired then provide the rows in this
-   * key of the request body that are to be deleted.
-   * @apiParam (Body) {string} [_id] If deleting a single row then provide its ID in this field.
-   * @apiParam (Body) {string} [_rev] If deleting a single row from an internal table then provide its
-   * revision here.
-   *
-   * @apiSuccess {object[]|object} body If deleting bulk then the response body will be an array
-   * of the deleted rows, if deleting a single row then the body will contain a "row" property which
-   * is the deleted row.
-   */
   .delete(
     "/api/:sourceId/rows",
     paramResource("sourceId"),
@@ -247,20 +71,6 @@ router
     trimViewRowInfo,
     rowController.destroy
   )
-
-  /**
-   * @api {post} /api/:sourceId/rows/exportRows Export Rows
-   * @apiName Export rows
-   * @apiGroup rows
-   * @apiPermission table write access
-   * @apiDescription This API can export a number of provided rows
-   *
-   * @apiParam {string} sourceId The ID of the table the row is to be deleted from.
-   *
-   * @apiParam (Body) {object[]} [rows] The row IDs which are to be exported
-   *
-   * @apiSuccess {object[]|object}
-   */
   .post(
     "/api/:sourceId/rows/exportRows",
     paramResource("sourceId"),

packages/server/src/api/routes/table.ts

@@ -9,99 +9,13 @@ const { BUILDER, PermissionLevel, PermissionType } = permissions
 const router: Router = new Router()
 
 router
-  /**
-   * @api {get} /api/tables Fetch all tables
-   * @apiName Fetch all tables
-   * @apiGroup tables
-   * @apiPermission table read access
-   * @apiDescription This endpoint retrieves all of the tables which have been created in
-   * an app. This includes all of the external and internal tables; to tell the difference
-   * between these look for the "type" property on each table, either being "internal" or "external".
-   *
-   * @apiSuccess {object[]} body The response body will be the list of tables that was found - as
-   * this does not take any parameters the only error scenario is no access.
-   */
   .get("/api/tables", authorized(BUILDER), tableController.fetch)
-  /**
-   * @api {get} /api/tables/:id Fetch a single table
-   * @apiName Fetch a single table
-   * @apiGroup tables
-   * @apiPermission table read access
-   * @apiDescription Retrieves a single table this could be be internal or external based on
-   * the provided table ID.
-   *
-   * @apiParam {string} id The ID of the table which is to be retrieved.
-   *
-   * @apiSuccess {object[]} body The response body will be the table that was found.
-   */
   .get(
     "/api/tables/:tableId",
     paramResource("tableId"),
     authorized(PermissionType.TABLE, PermissionLevel.READ, { schema: true }),
     tableController.find
   )
-  /**
-   * @api {post} /api/tables Save a table
-   * @apiName Save a table
-   * @apiGroup tables
-   * @apiPermission builder
-   * @apiDescription Create or update a table with this endpoint, this will function for both internal
-   * external tables.
-   *
-   * @apiParam (Body) {string} [_id] If updating an existing table then the ID of the table must be specified.
-   * @apiParam (Body) {string} [_rev] If updating an existing internal table then the revision must also be specified.
-   * @apiParam (Body) {string} type] This should either be "internal" or "external" depending on the table type -
-   * this will default to internal.
-   * @apiParam (Body) {string} [sourceId] If creating an external table then this should be set to the datasource ID. If
-   * building an internal table this does not need to be set, although it will be returned as "bb_internal".
-   * @apiParam (Body) {string} name The name of the table, this will be used in the UI. To rename the table simply
-   * supply the table structure to this endpoint with the name changed.
-   * @apiParam (Body) {object} schema A key value object which has all of the columns in the table as the keys in this
-   * object. For each column a "type" and "constraints" must be specified, with some types requiring further information.
-   * More information about the schema structure can be found in the Typescript definitions.
-   * @apiParam (Body) {string} [primaryDisplay] The name of the column which should be used when displaying rows
-   * from this table as relationships.
-   * @apiParam (Body) {object[]} [indexes] Specifies the search indexes - this is deprecated behaviour with the introduction
-   * of lucene indexes. This functionality is only available for internal tables.
-   * @apiParam (Body) {object} [_rename] If a column is to be renamed then the "old" column name should be set in this
-   * structure, and the "updated", new column name should also be supplied. The schema should also be updated, this field
-   * lets the server know that a field hasn't just been deleted, that the data has moved to a new name, this will fix
-   * the rows in the table. This functionality is only available for internal tables.
-   * @apiParam (Body) {object[]} [rows] When creating a table using a compatible data source, an array of objects to be imported into the new table can be provided.
-   *
-   * @apiParamExample {json} Example:
-   * {
-   *   "_id": "ta_05541307fa0f4044abee071ca2a82119",
-   *   "_rev": "10-0fbe4e78f69b255d79f1017e2eeef807",
-   *   "type": "internal",
-   *   "views": {},
-   *   "name": "tableName",
-   *   "schema": {
-   *     "column": {
-   *       "type": "string",
-   *       "constraints": {
-   *         "type": "string",
-   *         "length": {
-   *           "maximum": null
-   *         },
-   *         "presence": false
-   *       },
-   *       "name": "column"
-   *     },
-   *   },
-   *   "primaryDisplay": "column",
-   *   "indexes": [],
-   *   "sourceId": "bb_internal",
-   *   "_rename": {
-   *     "old": "columnName",
-   *     "updated": "newColumnName",
-   *   },
-   *   "rows": []
-   * }
-   *
-   * @apiSuccess {object} table The response body will contain the table structure after being cleaned up and
-   * saved to the database.
-   */
   .post(
     "/api/tables",
     // allows control over updating a table
@@ -125,41 +39,12 @@ router
     authorized(BUILDER),
     tableController.validateExistingTableImport
   )
-  /**
-   * @api {post} /api/tables/:tableId/:revId Delete a table
-   * @apiName Delete a table
-   * @apiGroup tables
-   * @apiPermission builder
-   * @apiDescription This endpoint will delete a table and all of its associated data, for this reason it is
-   * quite dangerous - it will work for internal and external tables.
-   *
-   * @apiParam {string} tableId The ID of the table which is to be deleted.
-   * @apiParam {string} [revId] If deleting an internal table then the revision must also be supplied (_rev), for
-   * external tables this can simply be set to anything, e.g. "external".
-   *
-   * @apiSuccess {string} message A message stating that the table was deleted successfully.
-   */
   .delete(
     "/api/tables/:tableId/:revId",
     paramResource("tableId"),
     authorized(BUILDER),
     tableController.destroy
   )
-  /**
-   * @api {post} /api/tables/:tableId/:revId Import CSV to existing table
-   * @apiName Import CSV to existing table
-   * @apiGroup tables
-   * @apiPermission builder
-   * @apiDescription This endpoint will import data to existing tables, internal or external. It is used in combination
-   * with the CSV validation endpoint. Take the output of the CSV validation endpoint and pass it to this endpoint to
-   * import the data; please note this will only import fields that already exist on the table/match the type.
-   *
-   * @apiParam {string} tableId The ID of the table which the data should be imported to.
-   *
-   * @apiParam (Body) {object[]} rows An array of objects representing the rows to be imported, key-value pairs not matching the table schema will be ignored.
-   *
-   * @apiSuccess {string} message A message stating that the data was imported successfully.
-   */
   .post(
     "/api/tables/:tableId/import",
     paramResource("tableId"),

packages/server/src/api/routes/tests/attachment.spec.ts

@@ -0,0 +1,49 @@
+import * as setup from "./utilities"
+import { APIError } from "@budibase/types"
+
+describe("/api/applications/:appId/sync", () => {
+  let config = setup.getConfig()
+
+  afterAll(setup.afterAll)
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  describe("/api/attachments/process", () => {
+    it("should accept an image file upload", async () => {
+      let resp = await config.api.attachment.process(
+        "1px.jpg",
+        Buffer.from([0])
+      )
+      expect(resp.length).toBe(1)
+
+      let upload = resp[0]
+      expect(upload.url.endsWith(".jpg")).toBe(true)
+      expect(upload.extension).toBe("jpg")
+      expect(upload.size).toBe(1)
+      expect(upload.name).toBe("1px.jpg")
+    })
+
+    it("should reject an upload with a malicious file extension", async () => {
+      await config.withEnv({ SELF_HOSTED: undefined }, async () => {
+        let resp = (await config.api.attachment.process(
+          "ohno.exe",
+          Buffer.from([0]),
+          { expectStatus: 400 }
+        )) as unknown as APIError
+        expect(resp.message).toContain("invalid extension")
+      })
+    })
+
+    it("should reject an upload with no file", async () => {
+      let resp = (await config.api.attachment.process(
+        undefined as any,
+        undefined as any,
+        {
+          expectStatus: 400,
+        }
+      )) as unknown as APIError
+      expect(resp.message).toContain("No file provided")
+    })
+  })
+})

packages/server/src/api/routes/tests/static.spec.js

@@ -5,11 +5,15 @@ describe("/static", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
   let app
+  let cleanupEnv
 
-  afterAll(setup.afterAll)
+  afterAll(() => {
+    setup.afterAll()
+    cleanupEnv()
+  })
 
   beforeAll(async () => {
-    config.modeSelf()
+    cleanupEnv = config.setEnv({ SELF_HOSTED: "true" })
     app = await config.init()
   })
 

packages/server/src/api/routes/tests/webhook.spec.ts

@@ -8,11 +8,15 @@ describe("/webhooks", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
   let webhook: Webhook
+  let cleanupEnv: () => void
 
-  afterAll(setup.afterAll)
+  afterAll(() => {
+    setup.afterAll()
+    cleanupEnv()
+  })
 
   const setupTest = async () => {
-    config.modeSelf()
+    cleanupEnv = config.setEnv({ SELF_HOSTED: "true" })
     await config.init()
     const autoConfig = basicAutomation()
     autoConfig.definition.trigger.schema = {

packages/server/src/integrations/tests/googlesheets.spec.ts

@@ -35,13 +35,18 @@ import { FieldType, Table, TableSchema } from "@budibase/types"
 describe("Google Sheets Integration", () => {
   let integration: any,
     config = new TestConfiguration()
+  let cleanupEnv: () => void
 
   beforeAll(() => {
-    config.setGoogleAuth("test")
+    cleanupEnv = config.setEnv({
+      GOOGLE_CLIENT_ID: "test",
+      GOOGLE_CLIENT_SECRET: "test",
+    })
   })
 
   afterAll(async () => {
-    await config.end()
+    cleanupEnv()
+    config.end()
   })
 
   beforeEach(async () => {

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -58,6 +58,7 @@ import {
 } from "@budibase/types"
 
 import API from "./api"
+import { cloneDeep } from "lodash"
 
 type DefaultUserValues = {
   globalUserId: string
@@ -188,30 +189,38 @@ class TestConfiguration {
     }
   }
 
-  // MODES
+  async withEnv(newEnvVars: Partial<typeof env>, f: () => Promise<void>) {
-  setMultiTenancy = (value: boolean) => {
+    let cleanup = this.setEnv(newEnvVars)
-    env._set("MULTI_TENANCY", value)
+    try {
-    coreEnv._set("MULTI_TENANCY", value)
+      await f()
+    } finally {
+      cleanup()
+    }
   }
 
-  setSelfHosted = (value: boolean) => {
+  /*
-    env._set("SELF_HOSTED", value)
+   * Sets the environment variables to the given values and returns a function
-    coreEnv._set("SELF_HOSTED", value)
+   * that can be called to reset the environment variables to their original values.
-  }
+   */
+  setEnv(newEnvVars: Partial<typeof env>): () => void {
+    const oldEnv = cloneDeep(env)
+    const oldCoreEnv = cloneDeep(coreEnv)
 
-  setGoogleAuth = (value: string) => {
+    let key: keyof typeof newEnvVars
-    env._set("GOOGLE_CLIENT_ID", value)
+    for (key in newEnvVars) {
-    env._set("GOOGLE_CLIENT_SECRET", value)
+      env._set(key, newEnvVars[key])
-    coreEnv._set("GOOGLE_CLIENT_ID", value)
+      coreEnv._set(key, newEnvVars[key])
-    coreEnv._set("GOOGLE_CLIENT_SECRET", value)
+    }
-  }
 
-  modeCloud = () => {
+    return () => {
-    this.setSelfHosted(false)
+      for (const [key, value] of Object.entries(oldEnv)) {
-  }
+        env._set(key, value)
+      }
 
-  modeSelf = () => {
+      for (const [key, value] of Object.entries(oldCoreEnv)) {
-    this.setSelfHosted(true)
+        coreEnv._set(key, value)
+      }
+    }
   }
 
   // UTILS

packages/server/src/tests/utilities/api/attachment.ts

@@ -0,0 +1,35 @@
+import {
+  APIError,
+  Datasource,
+  ProcessAttachmentResponse,
+} from "@budibase/types"
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+import fs from "fs"
+
+export class AttachmentAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  process = async (
+    name: string,
+    file: Buffer | fs.ReadStream | string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<ProcessAttachmentResponse> => {
+    const result = await this.request
+      .post(`/api/attachments/process`)
+      .attach("file", file, name)
+      .set(this.config.defaultHeaders())
+
+    if (result.statusCode !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          result.statusCode
+        }, body: ${JSON.stringify(result.body)}`
+      )
+    }
+
+    return result.body
+  }
+}

packages/server/src/tests/utilities/api/index.ts

@@ -7,6 +7,7 @@ import { DatasourceAPI } from "./datasource"
 import { LegacyViewAPI } from "./legacyView"
 import { ScreenAPI } from "./screen"
 import { ApplicationAPI } from "./application"
+import { AttachmentAPI } from "./attachment"
 
 export default class API {
   table: TableAPI
@@ -17,6 +18,7 @@ export default class API {
   datasource: DatasourceAPI
   screen: ScreenAPI
   application: ApplicationAPI
+  attachment: AttachmentAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
@@ -27,5 +29,6 @@ export default class API {
     this.datasource = new DatasourceAPI(config)
     this.screen = new ScreenAPI(config)
     this.application = new ApplicationAPI(config)
+    this.attachment = new AttachmentAPI(config)
   }
 }

packages/server/src/utilities/rowProcessor/index.ts

@@ -241,7 +241,7 @@ export async function outputProcessing<T extends Row[] | Row>(
           continue
         }
         row[property].forEach((attachment: RowAttachment) => {
-          attachment.url = objectStore.getAppFileUrl(attachment.key)
+          attachment.url ??= objectStore.getAppFileUrl(attachment.key)
         })
       }
     } else if (

packages/shared-core/src/constants.ts

@@ -96,3 +96,45 @@ export enum BuilderSocketEvent {
 export const SocketSessionTTL = 60
 export const ValidQueryNameRegex = /^[^()]*$/
 export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g
+export const ValidFileExtensions = [
+  "avif",
+  "css",
+  "csv",
+  "docx",
+  "drawio",
+  "editorconfig",
+  "edl",
+  "enc",
+  "export",
+  "geojson",
+  "gif",
+  "htm",
+  "html",
+  "ics",
+  "iqy",
+  "jfif",
+  "jpeg",
+  "jpg",
+  "json",
+  "log",
+  "md",
+  "mid",
+  "odt",
+  "pdf",
+  "png",
+  "ris",
+  "rtf",
+  "svg",
+  "tex",
+  "toml",
+  "twig",
+  "txt",
+  "url",
+  "wav",
+  "webp",
+  "xls",
+  "xlsx",
+  "xml",
+  "yaml",
+  "yml",
+]

packages/types/src/api/web/app/attachment.ts

@@ -0,0 +1,9 @@
+export interface Upload {
+  size: number
+  name: string
+  url: string
+  extension: string
+  key: string
+}
+
+export type ProcessAttachmentResponse = Upload[]

packages/types/src/api/web/app/index.ts

@@ -5,3 +5,4 @@ export * from "./view"
 export * from "./rows"
 export * from "./table"
 export * from "./permission"
+export * from "./attachment"

packages/types/src/sdk/featureFlag.ts

@@ -1,5 +1,8 @@
 export enum FeatureFlag {
   LICENSING = "LICENSING",
+  // Feature IDs in Posthog
+  PER_CREATOR_PER_USER_PRICE = "18873",
+  PER_CREATOR_PER_USER_PRICE_ALERT = "18530",
 }
 
 export interface TenantFeatureFlags {

packages/types/src/sdk/licensing/billing.ts

@@ -5,10 +5,17 @@ export interface Customer {
   currency: string | null | undefined
 }
 
+export interface SubscriptionItems {
+  user: number | undefined
+  creator: number | undefined
+}
+
 export interface Subscription {
   amount: number
+  amounts: SubscriptionItems | undefined
   currency: string
   quantity: number
+  quantities: SubscriptionItems | undefined
   duration: PriceDuration
   cancelAt: number | null | undefined
   currentPeriodStart: number

packages/types/src/sdk/licensing/plan.ts

@@ -4,7 +4,9 @@ export enum PlanType {
   PRO = "pro",
   /** @deprecated */
   TEAM = "team",
+  /** @deprecated */
   PREMIUM = "premium",
+  PREMIUM_PLUS = "premium_plus",
   BUSINESS = "business",
   ENTERPRISE = "enterprise",
 }
@@ -26,10 +28,12 @@ export interface AvailablePrice {
   currency: string
   duration: PriceDuration
   priceId: string
+  type?: string
 }
 
 export enum PlanModel {
   PER_USER = "perUser",
+  PER_CREATOR_PER_USER = "per_creator_per_user",
   DAY_PASS = "dayPass",
 }
 

packages/worker/Dockerfile

@@ -14,7 +14,7 @@ RUN yarn global add pm2
 
 COPY package.json .
 COPY dist/yarn.lock .
-RUN yarn install --production=true
+RUN yarn install --production=true --network-timeout 1000000
 # Remove unneeded data from file system to reduce image size
 RUN apk del .gyp \
     && yarn cache clean

packages/worker/package.json

@@ -20,7 +20,6 @@
     "run:docker": "node dist/index.js",
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
-    "build:docker": "yarn build && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
     "dev:builder": "npm run dev:stack:init && nodemon",
     "dev:built": "yarn run dev:stack:init && yarn run run:docker",

packages/worker/src/api/routes/global/tests/groups.spec.ts

@@ -1,7 +1,7 @@
 import { events } from "@budibase/backend-core"
 import { generator } from "@budibase/backend-core/tests"
 import { structures, TestConfiguration, mocks } from "../../../../tests"
-import { UserGroup } from "@budibase/types"
+import { User, UserGroup } from "@budibase/types"
 
 mocks.licenses.useGroups()
 
@@ -231,4 +231,39 @@ describe("/api/global/groups", () => {
       })
     })
   })
+
+  describe("with global builder role", () => {
+    let builder: User
+    let group: UserGroup
+
+    beforeAll(async () => {
+      builder = await config.createUser({
+        builder: { global: true },
+        admin: { global: false },
+      })
+      await config.createSession(builder)
+
+      let resp = await config.api.groups.saveGroup(
+        structures.groups.UserGroup()
+      )
+      group = resp.body as UserGroup
+    })
+
+    it("find should return 200", async () => {
+      await config.withUser(builder, async () => {
+        await config.api.groups.searchUsers(group._id!, {
+          emailSearch: `user1`,
+        })
+      })
+    })
+
+    it("update should return 200", async () => {
+      await config.withUser(builder, async () => {
+        await config.api.groups.updateGroupUsers(group._id!, {
+          add: [builder._id!],
+          remove: [],
+        })
+      })
+    })
+  })
 })

packages/worker/src/tests/TestConfiguration.ts

@@ -190,6 +190,16 @@ class TestConfiguration {
     }
   }
 
+  async withUser(user: User, f: () => Promise<void>) {
+    const oldUser = this.user
+    this.user = user
+    try {
+      await f()
+    } finally {
+      this.user = oldUser
+    }
+  }
+
   authHeaders(user: User) {
     const authToken: AuthToken = {
       userId: user._id!,
@@ -257,9 +267,10 @@ class TestConfiguration {
     })
   }
 
-  async createUser(user?: User) {
+  async createUser(opts?: Partial<User>) {
-    if (!user) {
+    let user = structures.users.user()
-      user = structures.users.user()
+    if (user) {
+      user = { ...user, ...opts }
     }
     const response = await this._req(user, null, controllers.users.save)
     const body = response as SaveUserResponse

packages/worker/src/tests/structures/groups.ts

@@ -1,8 +1,8 @@
 import { generator } from "@budibase/backend-core/tests"
 import { db } from "@budibase/backend-core"
-import { UserGroupRoles } from "@budibase/types"
+import { UserGroup as UserGroupType, UserGroupRoles } from "@budibase/types"
 
-export const UserGroup = () => {
+export function UserGroup(): UserGroupType {
   const appsCount = generator.integer({ min: 0, max: 3 })
   const roles = Array.from({ length: appsCount }).reduce(
     (p: UserGroupRoles, v) => {
@@ -14,13 +14,11 @@ export const UserGroup = () => {
     {}
   )
 
-  let group = {
+  return {
-    apps: [],
     color: generator.color(),
     icon: generator.word(),
     name: generator.word(),
     roles: roles,
     users: [],
   }
-  return group
 }