Merge remote-tracking branch 'origin/master' into feature/component-name-setting-ux-update

.github/workflows/budibase_ci.yml

@@ -18,8 +18,7 @@ env:
   BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   NX_BASE_BRANCH: origin/${{ github.base_ref }}
-  USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}}
+  USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }}
-  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
 
 jobs:
   lint:

.github/workflows/deploy-qa.yml

@@ -0,0 +1,20 @@
+name: Deploy QA
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  trigger-deploy-to-qa-env:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: peter-evans/repository-dispatch@v2
+        env:
+          PAYLOAD_VERSION: ${{ github.sha }}
+          REF_NAME: ${{ github.ref_name}}
+        with:
+          repository: budibase/budibase-deploys
+          event-type: budicloud-qa-deploy
+          token: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/release-master.yml

@@ -123,6 +123,7 @@ jobs:
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
           PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
+          REF_NAME: ${{ github.ref_name}}
         with:
           repository: budibase/budibase-deploys
           event: budicloud-qa-deploy

.github/workflows/release-singleimage-test.yml

@@ -54,6 +54,7 @@ jobs:
           push: true
           pull: true
           platforms: linux/amd64,linux/arm64
+          build-args: BUDIBASE_VERSION=0.0.0+test
           tags: budibase/budibase-test:test
           file: ./hosting/single/Dockerfile.v2
           cache-from: type=registry,ref=budibase/budibase-test:test
@@ -64,6 +65,8 @@ jobs:
           context: .
           push: true
           platforms: linux/amd64
-          build-args: TARGETBUILD=aas
+          build-args: |
+            TARGETBUILD=aas
+            BUDIBASE_VERSION=0.0.0+test
           tags: budibase/budibase-test:aas
           file: ./hosting/single/Dockerfile.v2

hosting/couchdb/Dockerfile

@@ -5,7 +5,7 @@ ENV COUCHDB_PASSWORD admin
 EXPOSE 5984
 
 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
-    wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | sudo apt-key add - && \
+    wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \
     apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \
     apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \
     apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bullseye main' && \

hosting/docker-compose.build.yaml

@@ -7,6 +7,8 @@ services:
     build:
       context: ..
       dockerfile: packages/server/Dockerfile.v2
+      args:
+        - BUDIBASE_VERSION=0.0.0+dev-docker
     container_name: build-bbapps
     environment:
       SELF_HOSTED: 1
@@ -30,13 +32,13 @@ services:
     depends_on:
       - worker-service
       - redis-service
-  #    volumes:
-  #      - /some/path/to/plugins:/plugins
 
   worker-service:
     build:
       context: ..
       dockerfile: packages/worker/Dockerfile.v2
+      args:
+        - BUDIBASE_VERSION=0.0.0+dev-docker
     container_name: build-bbworker
     environment:
       SELF_HOSTED: 1

hosting/single/Dockerfile.v2

@@ -26,6 +26,7 @@ RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json
 # We will never want to sync pro, but the script is still required
 RUN echo '' > scripts/syncProPackage.js
 RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json
+RUN ./scripts/removeWorkspaceDependencies.sh package.json
 RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production
 
 # copy the actual code
@@ -117,6 +118,10 @@ EXPOSE 443
 EXPOSE 2222
 VOLUME /data
 
+ARG BUDIBASE_VERSION
+# Ensuring the version argument is sent
+RUN test -n "$BUDIBASE_VERSION"
+ENV BUDIBASE_VERSION=$BUDIBASE_VERSION
 
 HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh"
 

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.11.39",
+  "version": "2.11.44",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

nx.json

@@ -3,14 +3,16 @@
     "default": {
       "runner": "nx-cloud",
       "options": {
-        "cacheableOperations": ["build", "test", "check:types"],
+        "cacheableOperations": ["build", "test", "check:types"]
-        "accessToken": "MmM4OGYxNzItMDBlYy00ZmE3LTk4MTYtNmJhYWMyZjBjZTUyfHJlYWQ="
       }
     }
   },
   "targetDefaults": {
     "build": {
-      "inputs": ["{workspaceRoot}/scripts/build.js"]
+      "inputs": [
+        "{workspaceRoot}/scripts/build.js",
+        "{workspaceRoot}/lerna.json"
+      ]
     }
   }
 }

packages/backend-core/src/cache/writethrough.ts

@@ -119,8 +119,8 @@ export class Writethrough {
     this.writeRateMs = writeRateMs
   }
 
-  async put(doc: any, writeRateMs: number = this.writeRateMs) {
+  async put(doc: any) {
-    return put(this.db, doc, writeRateMs)
+    return put(this.db, doc, this.writeRateMs)
   }
 
   async get(id: string) {

packages/backend-core/src/environment.ts

@@ -75,12 +75,12 @@ function getPackageJsonFields(): {
     const content = readFileSync(packageJsonFile!, "utf-8")
     const parsedContent = JSON.parse(content)
     return {
-      VERSION: parsedContent.version,
+      VERSION: process.env.BUDIBASE_VERSION || parsedContent.version,
       SERVICE_NAME: parsedContent.name,
     }
   } catch {
     // throwing an error here is confusing/causes backend-core to be hard to import
-    return { VERSION: "", SERVICE_NAME: "" }
+    return { VERSION: process.env.BUDIBASE_VERSION || "", SERVICE_NAME: "" }
   }
 }
 

packages/backend-core/src/objectStore/buckets/app.ts

@@ -1,37 +1,50 @@
 import env from "../../environment"
 import * as objectStore from "../objectStore"
 import * as cloudfront from "../cloudfront"
+import qs from "querystring"
+import { DEFAULT_TENANT_ID, getTenantId } from "../../context"
+
+export function clientLibraryPath(appId: string) {
+  return `${objectStore.sanitizeKey(appId)}/budibase-client.js`
+}
 
 /**
- * In production the client library is stored in the object store, however in development
+ * Previously we used to serve the client library directly from Cloudfront, however
- * we use the symlinked version produced by lerna, located in node modules. We link to this
+ * due to issues with the domain we were unable to continue doing this - keeping
- * via a specific endpoint (under /api/assets/client).
+ * incase we are able to switch back to CDN path again in future.
- * @param appId In production we need the appId to look up the correct bucket, as the
- * version of the client lib may differ between apps.
- * @param version The version to retrieve.
- * @return The URL to be inserted into appPackage response or server rendered
- * app index file.
  */
-export const clientLibraryUrl = (appId: string, version: string) => {
+export function clientLibraryCDNUrl(appId: string, version: string) {
-  if (env.isProd()) {
+  let file = clientLibraryPath(appId)
-    let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js`
+  if (env.CLOUDFRONT_CDN) {
-    if (env.CLOUDFRONT_CDN) {
+    // append app version to bust the cache
-      // append app version to bust the cache
+    if (version) {
-      if (version) {
+      file += `?v=${version}`
-        file += `?v=${version}`
-      }
-      // don't need to use presigned for client with cloudfront
-      // file is public
-      return cloudfront.getUrl(file)
-    } else {
-      return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
     }
+    // don't need to use presigned for client with cloudfront
+    // file is public
+    return cloudfront.getUrl(file)
   } else {
-    return `/api/assets/client`
+    return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
   }
 }
 
-export const getAppFileUrl = (s3Key: string) => {
+export function clientLibraryUrl(appId: string, version: string) {
+  let tenantId, qsParams: { appId: string; version: string; tenantId?: string }
+  try {
+    tenantId = getTenantId()
+  } finally {
+    qsParams = {
+      appId,
+      version,
+    }
+  }
+  if (tenantId && tenantId !== DEFAULT_TENANT_ID) {
+    qsParams.tenantId = tenantId
+  }
+  return `/api/assets/client?${qs.encode(qsParams)}`
+}
+
+export function getAppFileUrl(s3Key: string) {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {

packages/backend-core/src/objectStore/buckets/plugins.ts

@@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types"
 
 // URLS
 
-export const enrichPluginURLs = (plugins: Plugin[]) => {
+export function enrichPluginURLs(plugins: Plugin[]) {
   if (!plugins || !plugins.length) {
     return []
   }
@@ -17,12 +17,12 @@ export const enrichPluginURLs = (plugins: Plugin[]) => {
   })
 }
 
-const getPluginJSUrl = (plugin: Plugin) => {
+function getPluginJSUrl(plugin: Plugin) {
   const s3Key = getPluginJSKey(plugin)
   return getPluginUrl(s3Key)
 }
 
-const getPluginIconUrl = (plugin: Plugin): string | undefined => {
+function getPluginIconUrl(plugin: Plugin): string | undefined {
   const s3Key = getPluginIconKey(plugin)
   if (!s3Key) {
     return
@@ -30,7 +30,7 @@ const getPluginIconUrl = (plugin: Plugin): string | undefined => {
   return getPluginUrl(s3Key)
 }
 
-const getPluginUrl = (s3Key: string) => {
+function getPluginUrl(s3Key: string) {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {
@@ -40,11 +40,11 @@ const getPluginUrl = (s3Key: string) => {
 
 // S3 KEYS
 
-export const getPluginJSKey = (plugin: Plugin) => {
+export function getPluginJSKey(plugin: Plugin) {
   return getPluginS3Key(plugin, "plugin.min.js")
 }
 
-export const getPluginIconKey = (plugin: Plugin) => {
+export function getPluginIconKey(plugin: Plugin) {
   // stored iconUrl is deprecated - hardcode to icon.svg in this case
   const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName
   if (!iconFileName) {
@@ -53,12 +53,12 @@ export const getPluginIconKey = (plugin: Plugin) => {
   return getPluginS3Key(plugin, iconFileName)
 }
 
-const getPluginS3Key = (plugin: Plugin, fileName: string) => {
+function getPluginS3Key(plugin: Plugin, fileName: string) {
   const s3Key = getPluginS3Dir(plugin.name)
   return `${s3Key}/${fileName}`
 }
 
-export const getPluginS3Dir = (pluginName: string) => {
+export function getPluginS3Dir(pluginName: string) {
   let s3Key = `${pluginName}`
   if (env.MULTI_TENANCY) {
     const tenantId = context.getTenantId()

packages/backend-core/src/objectStore/buckets/tests/app.spec.ts

@@ -1,5 +1,4 @@
 import * as app from "../app"
-import { getAppFileUrl } from "../app"
 import { testEnv } from "../../../../tests/extra"
 
 describe("app", () => {
@@ -7,6 +6,15 @@ describe("app", () => {
     testEnv.nodeJest()
   })
 
+  function baseCheck(url: string, tenantId?: string) {
+    expect(url).toContain("/api/assets/client")
+    if (tenantId) {
+      expect(url).toContain(`tenantId=${tenantId}`)
+    }
+    expect(url).toContain("appId=app_123")
+    expect(url).toContain("version=2.0.0")
+  }
+
   describe("clientLibraryUrl", () => {
     function getClientUrl() {
       return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0")
@@ -20,31 +28,19 @@ describe("app", () => {
       it("gets url in dev", () => {
         testEnv.nodeDev()
         const url = getClientUrl()
-        expect(url).toBe("/api/assets/client")
+        baseCheck(url)
-      })
-
-      it("gets url with embedded minio", () => {
-        testEnv.withMinio()
-        const url = getClientUrl()
-        expect(url).toBe(
-          "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-        )
       })
 
       it("gets url with custom S3", () => {
         testEnv.withS3()
         const url = getClientUrl()
-        expect(url).toBe(
+        baseCheck(url)
-          "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-        )
       })
 
       it("gets url with cloudfront + s3", () => {
         testEnv.withCloudfront()
         const url = getClientUrl()
-        expect(url).toBe(
+        baseCheck(url)
-          "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
-        )
       })
     })
 
@@ -57,7 +53,7 @@ describe("app", () => {
         testEnv.nodeDev()
         await testEnv.withTenant(tenantId => {
           const url = getClientUrl()
-          expect(url).toBe("/api/assets/client")
+          baseCheck(url, tenantId)
         })
       })
 
@@ -65,9 +61,7 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withMinio()
           const url = getClientUrl()
-          expect(url).toBe(
+          baseCheck(url, tenantId)
-            "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-          )
         })
       })
 
@@ -75,9 +69,7 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withS3()
           const url = getClientUrl()
-          expect(url).toBe(
+          baseCheck(url, tenantId)
-            "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
-          )
         })
       })
 
@@ -85,9 +77,7 @@ describe("app", () => {
         await testEnv.withTenant(tenantId => {
           testEnv.withCloudfront()
           const url = getClientUrl()
-          expect(url).toBe(
+          baseCheck(url, tenantId)
-            "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
-          )
         })
       })
     })

packages/backend-core/src/objectStore/objectStore.ts

@@ -1,6 +1,6 @@
 const sanitize = require("sanitize-s3-objectkey")
 import AWS from "aws-sdk"
-import stream from "stream"
+import stream, { Readable } from "stream"
 import fetch from "node-fetch"
 import tar from "tar-fs"
 import zlib from "zlib"
@@ -66,10 +66,10 @@ export function sanitizeBucket(input: string) {
  * @return an S3 object store object, check S3 Nodejs SDK for usage.
  * @constructor
  */
-export const ObjectStore = (
+export function ObjectStore(
   bucket: string,
   opts: { presigning: boolean } = { presigning: false }
-) => {
+) {
   const config: any = {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
@@ -104,7 +104,7 @@ export const ObjectStore = (
  * Given an object store and a bucket name this will make sure the bucket exists,
  * if it does not exist then it will create it.
  */
-export const makeSureBucketExists = async (client: any, bucketName: string) => {
+export async function makeSureBucketExists(client: any, bucketName: string) {
   bucketName = sanitizeBucket(bucketName)
   try {
     await client
@@ -139,13 +139,13 @@ export const makeSureBucketExists = async (client: any, bucketName: string) => {
  * Uploads the contents of a file given the required parameters, useful when
  * temp files in use (for example file uploaded as an attachment).
  */
-export const upload = async ({
+export async function upload({
   bucket: bucketName,
   filename,
   path,
   type,
   metadata,
-}: UploadParams) => {
+}: UploadParams) {
   const extension = filename.split(".").pop()
   const fileBytes = fs.readFileSync(path)
 
@@ -180,12 +180,12 @@ export const upload = async ({
  * Similar to the upload function but can be used to send a file stream
  * through to the object store.
  */
-export const streamUpload = async (
+export async function streamUpload(
   bucketName: string,
   filename: string,
   stream: any,
   extra = {}
-) => {
+) {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
 
@@ -215,7 +215,7 @@ export const streamUpload = async (
  * retrieves the contents of a file from the object store, if it is a known content type it
  * will be converted, otherwise it will be returned as a buffer stream.
  */
-export const retrieve = async (bucketName: string, filepath: string) => {
+export async function retrieve(bucketName: string, filepath: string) {
   const objectStore = ObjectStore(bucketName)
   const params = {
     Bucket: sanitizeBucket(bucketName),
@@ -230,7 +230,7 @@ export const retrieve = async (bucketName: string, filepath: string) => {
   }
 }
 
-export const listAllObjects = async (bucketName: string, path: string) => {
+export async function listAllObjects(bucketName: string, path: string) {
   const objectStore = ObjectStore(bucketName)
   const list = (params: ListParams = {}) => {
     return objectStore
@@ -261,11 +261,11 @@ export const listAllObjects = async (bucketName: string, path: string) => {
 /**
  * Generate a presigned url with a default TTL of 1 hour
  */
-export const getPresignedUrl = (
+export function getPresignedUrl(
   bucketName: string,
   key: string,
   durationSeconds: number = 3600
-) => {
+) {
   const objectStore = ObjectStore(bucketName, { presigning: true })
   const params = {
     Bucket: sanitizeBucket(bucketName),
@@ -291,7 +291,7 @@ export const getPresignedUrl = (
 /**
  * Same as retrieval function but puts to a temporary file.
  */
-export const retrieveToTmp = async (bucketName: string, filepath: string) => {
+export async function retrieveToTmp(bucketName: string, filepath: string) {
   bucketName = sanitizeBucket(bucketName)
   filepath = sanitizeKey(filepath)
   const data = await retrieve(bucketName, filepath)
@@ -300,7 +300,7 @@ export const retrieveToTmp = async (bucketName: string, filepath: string) => {
   return outputPath
 }
 
-export const retrieveDirectory = async (bucketName: string, path: string) => {
+export async function retrieveDirectory(bucketName: string, path: string) {
   let writePath = join(budibaseTempDir(), v4())
   fs.mkdirSync(writePath)
   const objects = await listAllObjects(bucketName, path)
@@ -324,7 +324,7 @@ export const retrieveDirectory = async (bucketName: string, path: string) => {
 /**
  * Delete a single file.
  */
-export const deleteFile = async (bucketName: string, filepath: string) => {
+export async function deleteFile(bucketName: string, filepath: string) {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
@@ -334,7 +334,7 @@ export const deleteFile = async (bucketName: string, filepath: string) => {
   return objectStore.deleteObject(params).promise()
 }
 
-export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
+export async function deleteFiles(bucketName: string, filepaths: string[]) {
   const objectStore = ObjectStore(bucketName)
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
@@ -349,10 +349,10 @@ export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
 /**
  * Delete a path, including everything within.
  */
-export const deleteFolder = async (
+export async function deleteFolder(
   bucketName: string,
   folder: string
-): Promise<any> => {
+): Promise<any> {
   bucketName = sanitizeBucket(bucketName)
   folder = sanitizeKey(folder)
   const client = ObjectStore(bucketName)
@@ -383,11 +383,11 @@ export const deleteFolder = async (
   }
 }
 
-export const uploadDirectory = async (
+export async function uploadDirectory(
   bucketName: string,
   localPath: string,
   bucketPath: string
-) => {
+) {
   bucketName = sanitizeBucket(bucketName)
   let uploads = []
   const files = fs.readdirSync(localPath, { withFileTypes: true })
@@ -404,11 +404,11 @@ export const uploadDirectory = async (
   return files
 }
 
-export const downloadTarballDirect = async (
+export async function downloadTarballDirect(
   url: string,
   path: string,
   headers = {}
-) => {
+) {
   path = sanitizeKey(path)
   const response = await fetch(url, { headers })
   if (!response.ok) {
@@ -418,11 +418,11 @@ export const downloadTarballDirect = async (
   await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path))
 }
 
-export const downloadTarball = async (
+export async function downloadTarball(
   url: string,
   bucketName: string,
   path: string
-) => {
+) {
   bucketName = sanitizeBucket(bucketName)
   path = sanitizeKey(path)
   const response = await fetch(url)
@@ -438,3 +438,17 @@ export const downloadTarball = async (
   // return the temporary path incase there is a use for it
   return tmpPath
 }
+
+export async function getReadStream(
+  bucketName: string,
+  path: string
+): Promise<Readable> {
+  bucketName = sanitizeBucket(bucketName)
+  path = sanitizeKey(path)
+  const client = ObjectStore(bucketName)
+  const params = {
+    Bucket: bucketName,
+    Key: path,
+  }
+  return client.getObject(params).createReadStream()
+}

packages/backend-core/src/users/db.ts

@@ -25,17 +25,12 @@ import {
 import {
   getAccountHolderFromUserIds,
   isAdmin,
-  isCreator,
   validateUniqueUser,
 } from "./utils"
 import { searchExistingEmails } from "./lookup"
 import { hash } from "../utils"
 
-type QuotaUpdateFn = (
+type QuotaUpdateFn = (change: number, cb?: () => Promise<any>) => Promise<any>
-  change: number,
-  creatorsChange: number,
-  cb?: () => Promise<any>
-) => Promise<any>
 type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise<any>
 type FeatureFn = () => Promise<Boolean>
 type GroupGetFn = (ids: string[]) => Promise<UserGroup[]>
@@ -164,14 +159,14 @@ export class UserDB {
     }
   }
 
-  static async getUsersByAppAccess(appId?: string) {
+  static async getUsersByAppAccess(opts: { appId?: string; limit?: number }) {
-    const opts: any = {
+    const params: any = {
       include_docs: true,
-      limit: 50,
+      limit: opts.limit || 50,
     }
     let response: User[] = await usersCore.searchGlobalUsersByAppAccess(
-      appId,
+      opts.appId,
-      opts
+      params
     )
     return response
   }
@@ -250,8 +245,7 @@ export class UserDB {
     }
 
     const change = dbUser ? 0 : 1 // no change if there is existing user
-    const creatorsChange = isCreator(dbUser) !== isCreator(user) ? 1 : 0
+    return UserDB.quotas.addUsers(change, async () => {
-    return UserDB.quotas.addUsers(change, creatorsChange, async () => {
       await validateUniqueUser(email, tenantId)
 
       let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser)
@@ -313,7 +307,6 @@ export class UserDB {
 
     let usersToSave: any[] = []
     let newUsers: any[] = []
-    let newCreators: any[] = []
 
     const emails = newUsersRequested.map((user: User) => user.email)
     const existingEmails = await searchExistingEmails(emails)
@@ -334,66 +327,59 @@ export class UserDB {
       }
       newUser.userGroups = groups
       newUsers.push(newUser)
-      if (isCreator(newUser)) {
-        newCreators.push(newUser)
-      }
     }
 
     const account = await accountSdk.getAccountByTenantId(tenantId)
-    return UserDB.quotas.addUsers(
+    return UserDB.quotas.addUsers(newUsers.length, async () => {
-      newUsers.length,
+      // create the promises array that will be called by bulkDocs
-      newCreators.length,
+      newUsers.forEach((user: any) => {
-      async () => {
+        usersToSave.push(
-        // create the promises array that will be called by bulkDocs
+          UserDB.buildUser(
-        newUsers.forEach((user: any) => {
+            user,
-          usersToSave.push(
+            {
-            UserDB.buildUser(
+              hashPassword: true,
-              user,
+              requirePassword: user.requirePassword,
-              {
+            },
-                hashPassword: true,
+            tenantId,
-                requirePassword: user.requirePassword,
+            undefined, // no dbUser
-              },
+            account
-              tenantId,
-              undefined, // no dbUser
-              account
-            )
           )
-        })
+        )
+      })
 
-        const usersToBulkSave = await Promise.all(usersToSave)
+      const usersToBulkSave = await Promise.all(usersToSave)
-        await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
+      await usersCore.bulkUpdateGlobalUsers(usersToBulkSave)
 
-        // Post-processing of bulk added users, e.g. events and cache operations
+      // Post-processing of bulk added users, e.g. events and cache operations
-        for (const user of usersToBulkSave) {
+      for (const user of usersToBulkSave) {
-          // TODO: Refactor to bulk insert users into the info db
+        // TODO: Refactor to bulk insert users into the info db
-          // instead of relying on looping tenant creation
+        // instead of relying on looping tenant creation
-          await platform.users.addUser(tenantId, user._id, user.email)
+        await platform.users.addUser(tenantId, user._id, user.email)
-          await eventHelpers.handleSaveEvents(user, undefined)
+        await eventHelpers.handleSaveEvents(user, undefined)
-        }
-
-        const saved = usersToBulkSave.map(user => {
-          return {
-            _id: user._id,
-            email: user.email,
-          }
-        })
-
-        // now update the groups
-        if (Array.isArray(saved) && groups) {
-          const groupPromises = []
-          const createdUserIds = saved.map(user => user._id)
-          for (let groupId of groups) {
-            groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds))
-          }
-          await Promise.all(groupPromises)
-        }
-
-        return {
-          successful: saved,
-          unsuccessful,
-        }
       }
-    )
+
+      const saved = usersToBulkSave.map(user => {
+        return {
+          _id: user._id,
+          email: user.email,
+        }
+      })
+
+      // now update the groups
+      if (Array.isArray(saved) && groups) {
+        const groupPromises = []
+        const createdUserIds = saved.map(user => user._id)
+        for (let groupId of groups) {
+          groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds))
+        }
+        await Promise.all(groupPromises)
+      }
+
+      return {
+        successful: saved,
+        unsuccessful,
+      }
+    })
   }
 
   static async bulkDelete(userIds: string[]): Promise<BulkUserDeleted> {
@@ -433,12 +419,11 @@ export class UserDB {
       _deleted: true,
     }))
     const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete)
-    const creatorsToDelete = usersToDelete.filter(isCreator)
 
+    await UserDB.quotas.removeUsers(toDelete.length)
     for (let user of usersToDelete) {
       await bulkDeleteProcessing(user)
     }
-    await UserDB.quotas.removeUsers(toDelete.length, creatorsToDelete.length)
 
     // Build Response
     // index users by id
@@ -487,8 +472,7 @@ export class UserDB {
 
     await db.remove(userId, dbUser._rev)
 
-    const creatorsToDelete = isCreator(dbUser) ? 1 : 0
+    await UserDB.quotas.removeUsers(1)
-    await UserDB.quotas.removeUsers(1, creatorsToDelete)
     await eventHelpers.handleDeleteEvents(dbUser)
     await cache.user.invalidateUser(userId)
     await sessions.invalidateSessions(userId, { reason: "deletion" })

packages/backend-core/src/users/users.ts

@@ -14,11 +14,12 @@ import {
 } from "../db"
 import {
   BulkDocsResponse,
+  ContextUser,
   SearchQuery,
   SearchQueryOperators,
   SearchUsersRequest,
   User,
-  ContextUser,
+  DatabaseQueryOpts,
 } from "@budibase/types"
 import { getGlobalDB } from "../context"
 import * as context from "../context"
@@ -241,12 +242,14 @@ export const paginatedUsers = async ({
   bookmark,
   query,
   appId,
+  limit,
 }: SearchUsersRequest = {}) => {
   const db = getGlobalDB()
+  const pageLimit = limit ? limit + 1 : PAGE_LIMIT + 1
   // get one extra document, to have the next page
-  const opts: any = {
+  const opts: DatabaseQueryOpts = {
     include_docs: true,
-    limit: PAGE_LIMIT + 1,
+    limit: pageLimit,
   }
   // add a startkey if the page was specified (anchor)
   if (bookmark) {
@@ -269,7 +272,7 @@ export const paginatedUsers = async ({
     const response = await db.allDocs(getGlobalUserParams(null, opts))
     userList = response.rows.map((row: any) => row.doc)
   }
-  return pagination(userList, PAGE_LIMIT, {
+  return pagination(userList, pageLimit, {
     paginate: true,
     property,
     getKey,

packages/backend-core/tests/core/users/users.spec.js

@@ -1,54 +0,0 @@
-const _ = require('lodash/fp')
-const {structures} = require("../../../tests")
-
-jest.mock("../../../src/context")
-jest.mock("../../../src/db")
-
-const context = require("../../../src/context")
-const db = require("../../../src/db")
-
-const {getCreatorCount} = require('../../../src/users/users')
-
-describe("Users", () => {
-
-  let getGlobalDBMock
-  let getGlobalUserParamsMock
-  let paginationMock
-
-  beforeEach(() => {
-    jest.resetAllMocks()
-
-    getGlobalDBMock = jest.spyOn(context, "getGlobalDB")
-    getGlobalUserParamsMock = jest.spyOn(db, "getGlobalUserParams")
-    paginationMock = jest.spyOn(db, "pagination")
-  })
-
-  it("Retrieves the number of creators", async () => {
-    const getUsers = (offset, limit, creators = false) => {
-      const range = _.range(offset, limit)
-      const opts = creators ? {builder: {global: true}} : undefined
-      return range.map(() => structures.users.user(opts))
-    }
-    const page1Data = getUsers(0, 8)
-    const page2Data = getUsers(8, 12, true)
-    getGlobalDBMock.mockImplementation(() => ({
-      name   : "fake-db",
-      allDocs: () => ({
-        rows: [...page1Data, ...page2Data]
-      })
-    }))
-    paginationMock.mockImplementationOnce(() => ({
-      data: page1Data,
-      hasNextPage: true,
-      nextPage: "1"
-    }))
-    paginationMock.mockImplementation(() => ({
-      data: page2Data,
-      hasNextPage: false,
-      nextPage: undefined
-    }))
-    const creatorsCount = await getCreatorCount()
-    expect(creatorsCount).toBe(4)
-    expect(paginationMock).toHaveBeenCalledTimes(2)
-  })
-})

packages/backend-core/tests/core/utilities/structures/licenses.ts

@@ -123,10 +123,6 @@ export function customer(): Customer {
 export function subscription(): Subscription {
   return {
     amount: 10000,
-    amounts: {
-      user: 10000,
-      creator: 0,
-    },
     cancelAt: undefined,
     currency: "usd",
     currentPeriodEnd: 0,
@@ -135,10 +131,6 @@ export function subscription(): Subscription {
     duration: PriceDuration.MONTHLY,
     pastDueAt: undefined,
     quantity: 0,
-    quantities: {
-      user: 0,
-      creator: 0,
-    },
     status: "active",
   }
 }

packages/bbui/src/Table/Table.svelte

@@ -106,6 +106,13 @@
           name: fieldName,
         }
       }
+
+      // Delete numeric only widths as these are grid widths and should be
+      // ignored
+      const width = fixedSchema[fieldName].width
+      if (width != null && `${width}`.trim().match(/^[0-9]+$/)) {
+        delete fixedSchema[fieldName].width
+      }
     })
     return fixedSchema
   }

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -3,13 +3,10 @@
   import { goto, params } from "@roxi/routify"
   import { Table, Heading, Layout } from "@budibase/bbui"
   import Spinner from "components/common/Spinner.svelte"
-  import {
+  import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
-    TableNames,
-    UNEDITABLE_USER_FIELDS,
-    UNSORTABLE_TYPES,
-  } from "constants"
   import RoleCell from "./cells/RoleCell.svelte"
   import { createEventDispatcher } from "svelte"
+  import { canBeSortColumn } from "@budibase/shared-core"
 
   export let schema = {}
   export let data = []
@@ -32,12 +29,10 @@
   $: isUsersTable = tableId === TableNames.USERS
   $: data && resetSelectedRows()
   $: {
-    UNSORTABLE_TYPES.forEach(type => {
+    Object.values(schema || {}).forEach(col => {
-      Object.values(schema || {}).forEach(col => {
+      if (!canBeSortColumn(col.type)) {
-        if (col.type === type) {
+        col.sortable = false
-          col.sortable = false
+      }
-        }
-      })
     })
   }
   $: {

packages/builder/src/components/design/settings/controls/ColumnEditor/CellDrawer.svelte

@@ -16,7 +16,11 @@
 <DrawerContent>
   <div class="container">
     <Layout noPadding gap="S">
-      <Input bind:value={column.width} label="Width" placeholder="Auto" />
+      <Input
+        bind:value={column.width}
+        label="Width (must include a unit like px or %)"
+        placeholder="Auto"
+      />
       <Select
         label="Alignment"
         bind:value={column.align}

packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte

@@ -1,5 +1,9 @@
 <script>
-  import { getContextProviderComponents } from "builderStore/dataBinding"
+  import {
+    getContextProviderComponents,
+    readableToRuntimeBinding,
+    runtimeToReadableBinding,
+  } from "builderStore/dataBinding"
   import {
     Button,
     Popover,
@@ -9,6 +13,11 @@
     Heading,
     Drawer,
     DrawerContent,
+    Icon,
+    Modal,
+    ModalContent,
+    CoreDropzone,
+    notifications,
   } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { store, currentAsset } from "builderStore"
@@ -22,6 +31,8 @@
   import BindingBuilder from "components/integration/QueryBindingBuilder.svelte"
   import IntegrationQueryEditor from "components/integration/index.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
+  import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
+  import { API } from "api"
 
   export let value = {}
   export let otherSources
@@ -31,9 +42,13 @@
 
   const dispatch = createEventDispatcher()
   const arrayTypes = ["attachment", "array"]
+
   let anchorRight, dropdownRight
   let drawer
   let tmpQueryParams
+  let tmpCustomData
+  let customDataValid = true
+  let modal
 
   $: text = value?.label ?? "Choose an option"
   $: tables = $tablesStore.list.map(m => ({
@@ -125,6 +140,10 @@
         value: `{{ literal ${runtimeBinding} }}`,
       }
     })
+  $: custom = {
+    type: "custom",
+    label: "JSON / CSV",
+  }
 
   const handleSelected = selected => {
     dispatch("change", selected)
@@ -151,6 +170,11 @@
     drawer.show()
   }
 
+  const openCustomDrawer = () => {
+    tmpCustomData = runtimeToReadableBinding(bindings, value.data || "")
+    drawer.show()
+  }
+
   const getQueryValue = queries => {
     return queries.find(q => q._id === value._id) || value
   }
@@ -162,6 +186,35 @@
     })
     drawer.hide()
   }
+
+  const saveCustomData = () => {
+    handleSelected({
+      ...value,
+      data: readableToRuntimeBinding(bindings, tmpCustomData),
+    })
+    drawer.hide()
+  }
+
+  const promptForCSV = () => {
+    drawer.hide()
+    modal.show()
+  }
+
+  const handleCSV = async e => {
+    try {
+      const csv = await e.detail[0]?.text()
+      if (csv?.length) {
+        const js = await API.csvToJson(csv)
+        tmpCustomData = JSON.stringify(js)
+      }
+      modal.hide()
+      saveCustomData()
+    } catch (error) {
+      notifications.error("Failed to parse CSV")
+      modal.hide()
+      drawer.show()
+    }
+  }
 </script>
 
 <div class="container" bind:this={anchorRight}>
@@ -172,7 +225,9 @@
     on:click={dropdownRight.show}
   />
   {#if value?.type === "query"}
-    <i class="ri-settings-5-line" on:click={openQueryParamsDrawer} />
+    <div class="icon">
+      <Icon hoverable name="Settings" on:click={openQueryParamsDrawer} />
+    </div>
     <Drawer title={"Query Bindings"} bind:this={drawer}>
       <Button slot="buttons" cta on:click={saveQueryParams}>Save</Button>
       <DrawerContent slot="body">
@@ -198,6 +253,29 @@
       </DrawerContent>
     </Drawer>
   {/if}
+  {#if value?.type === "custom"}
+    <div class="icon">
+      <Icon hoverable name="Settings" on:click={openCustomDrawer} />
+    </div>
+    <Drawer title="Custom data" bind:this={drawer}>
+      <div slot="buttons" style="display:contents">
+        <Button primary on:click={promptForCSV}>Load CSV</Button>
+        <Button cta on:click={saveCustomData} disabled={!customDataValid}>
+          Save
+        </Button>
+      </div>
+      <div slot="description">Provide a JSON array to use as data</div>
+      <ClientBindingPanel
+        slot="body"
+        bind:valid={customDataValid}
+        value={tmpCustomData}
+        on:change={event => (tmpCustomData = event.detail)}
+        {bindings}
+        allowJS
+        allowHelpers
+      />
+    </Drawer>
+  {/if}
 </div>
 <Popover bind:this={dropdownRight} anchor={anchorRight}>
   <div class="dropdown">
@@ -285,20 +363,27 @@
         {/each}
       </ul>
     {/if}
-    {#if otherSources?.length}
+    <Divider />
-      <Divider />
+    <div class="title">
-      <div class="title">
+      <Heading size="XS">Other</Heading>
-        <Heading size="XS">Other</Heading>
+    </div>
-      </div>
+    <ul>
-      <ul>
+      <li on:click={() => handleSelected(custom)}>{custom.label}</li>
+      {#if otherSources?.length}
         {#each otherSources as source}
           <li on:click={() => handleSelected(source)}>{source.label}</li>
         {/each}
-      </ul>
+      {/if}
-    {/if}
+    </ul>
   </div>
 </Popover>
 
+<Modal bind:this={modal}>
+  <ModalContent title="Load CSV" showConfirmButton={false}>
+    <CoreDropzone compact extensions=".csv" on:change={handleCSV} />
+  </ModalContent>
+</Modal>
+
 <style>
   .container {
     display: flex;
@@ -340,16 +425,7 @@
     background-color: var(--spectrum-global-color-gray-200);
   }
 
-  i {
+  .icon {
-    margin-left: 5px;
+    margin-left: 8px;
-    display: flex;
-    align-items: center;
-    transition: all 0.2s;
-  }
-
-  i:hover {
-    transform: scale(1.1);
-    font-weight: 600;
-    cursor: pointer;
   }
 </style>

packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte

@@ -6,7 +6,7 @@
   } from "builderStore/dataBinding"
   import { currentAsset } from "builderStore"
   import { createEventDispatcher } from "svelte"
-  import { UNSORTABLE_TYPES } from "constants"
+  import { canBeSortColumn } from "@budibase/shared-core"
 
   export let componentInstance = {}
   export let value = ""
@@ -20,7 +20,7 @@
 
   const getSortableFields = schema => {
     return Object.entries(schema || {})
-      .filter(entry => !UNSORTABLE_TYPES.includes(entry[1].type))
+      .filter(entry => canBeSortColumn(entry[1].type))
       .map(entry => entry[0])
   }
 

packages/builder/src/constants/index.js

@@ -34,8 +34,6 @@ export const UNEDITABLE_USER_FIELDS = [
   "lastName",
 ]
 
-export const UNSORTABLE_TYPES = ["formula", "attachment", "array", "link"]
-
 export const LAYOUT_NAMES = {
   MASTER: {
     PRIVATE: "layout_private_master",

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -114,8 +114,9 @@
       query: {
         appId: query || !filterByAppAccess ? null : prodAppId,
         email: query,
-        paginated: query || !filterByAppAccess ? null : false,
       },
+      limit: 50,
+      paginate: query || !filterByAppAccess ? null : false,
     })
     await usersFetch.refresh()
 

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { isEmpty } from "lodash/fp"
+  import { helpers } from "@budibase/shared-core"
   import { DetailSummary, notifications } from "@budibase/bbui"
   import { store } from "builderStore"
   import PropertyControl from "components/design/settings/controls/PropertyControl.svelte"
@@ -69,41 +69,43 @@
   }
 
   const shouldDisplay = (instance, setting) => {
-    // Parse dependant settings
+    let dependsOn = setting.dependsOn
-    if (setting.dependsOn) {
+    if (dependsOn && !Array.isArray(dependsOn)) {
-      let dependantSetting = setting.dependsOn
+      dependsOn = [dependsOn]
-      let dependantValue = null
+    }
-      let invert = !!setting.dependsOn.invert
+    if (!dependsOn?.length) {
-      if (typeof setting.dependsOn === "object") {
+      return true
-        dependantSetting = setting.dependsOn.setting
+    }
-        dependantValue = setting.dependsOn.value
+
+    // Ensure all conditions are met
+    return dependsOn.every(condition => {
+      let dependantSetting = condition
+      let dependantValues = null
+      let invert = !!condition.invert
+      if (typeof condition === "object") {
+        dependantSetting = condition.setting
+        dependantValues = condition.value
       }
       if (!dependantSetting) {
         return false
       }
 
-      // If no specific value is depended upon, check if a value exists at all
+      // Ensure values is an array
-      // for the dependent setting
+      if (!Array.isArray(dependantValues)) {
-      if (dependantValue == null) {
+        dependantValues = [dependantValues]
-        const currentValue = instance[dependantSetting]
-        if (currentValue === false) {
-          return false
-        }
-        if (currentValue === true) {
-          return true
-        }
-        return !isEmpty(currentValue)
       }
 
-      // Otherwise check the value matches
+      // If inverting, we want to ensure that we don't have any matches.
-      if (invert) {
+      // If not inverting, we want to ensure that we do have any matches.
-        return instance[dependantSetting] !== dependantValue
+      const currentVal = helpers.deepGet(instance, dependantSetting)
-      } else {
+      const anyMatches = dependantValues.some(dependantVal => {
-        return instance[dependantSetting] === dependantValue
+        if (dependantVal == null) {
-      }
+          return currentVal != null && currentVal !== false && currentVal !== ""
-    }
+        }
-
+        return dependantVal === currentVal
-    return typeof setting.visible == "boolean" ? setting.visible : true
+      })
+      return anyMatches !== invert
+    })
   }
 
   const canRenderControl = (instance, setting, isScreen) => {

packages/builder/src/stores/backend/datasources.js

@@ -81,9 +81,9 @@ export function createDatasourcesStore() {
     }))
   }
 
-  const updateDatasource = response => {
+  const updateDatasource = (response, { ignoreErrors } = {}) => {
     const { datasource, errors } = response
-    if (errors && Object.keys(errors).length > 0) {
+    if (!ignoreErrors && errors && Object.keys(errors).length > 0) {
       throw new TableImportError(errors)
     }
     replaceDatasource(datasource._id, datasource)
@@ -137,7 +137,7 @@ export function createDatasourcesStore() {
       fetchSchema: integration.plus,
     })
 
-    return updateDatasource(response)
+    return updateDatasource(response, { ignoreErrors: true })
   }
 
   const update = async ({ integration, datasource }) => {

packages/client/manifest.json

@@ -5305,6 +5305,12 @@
         "key": "title",
         "nested": true
       },
+      {
+        "type": "text",
+        "label": "Description",
+        "key": "description",
+        "nested": true
+      },
       {
         "section": true,
         "dependsOn": {
@@ -5556,10 +5562,9 @@
       "width": 600,
       "height": 400
     },
-    "info": "Grid Blocks are only compatible with internal or SQL tables",
     "settings": [
       {
-        "type": "table",
+        "type": "dataSource",
         "label": "Data",
         "key": "table",
         "required": true
@@ -5568,18 +5573,35 @@
         "type": "columns/grid",
         "label": "Columns",
         "key": "columns",
-        "dependsOn": "table"
+        "dependsOn": [
+          "table",
+          {
+            "setting": "table.type",
+            "value": "custom",
+            "invert": true
+          }
+        ]
       },
       {
         "type": "filter",
         "label": "Filtering",
-        "key": "initialFilter"
+        "key": "initialFilter",
+        "dependsOn": {
+          "setting": "table.type",
+          "value": "custom",
+          "invert": true
+        }
       },
       {
         "type": "field/sortable",
         "label": "Sort column",
         "key": "initialSortColumn",
-        "placeholder": "Default"
+        "placeholder": "Default",
+        "dependsOn": {
+          "setting": "table.type",
+          "value": "custom",
+          "invert": true
+        }
       },
       {
         "type": "select",
@@ -5618,29 +5640,37 @@
             "label": "Clicked row",
             "key": "row"
           }
-        ],
+        ]
-        "dependsOn": {
-          "setting": "allowEditRows",
-          "value": false
-        }
       },
       {
         "type": "boolean",
         "label": "Add rows",
         "key": "allowAddRows",
-        "defaultValue": true
+        "defaultValue": true,
+        "dependsOn": {
+          "setting": "table.type",
+          "value": ["table", "viewV2"]
+        }
       },
       {
         "type": "boolean",
         "label": "Edit rows",
         "key": "allowEditRows",
-        "defaultValue": true
+        "defaultValue": true,
+        "dependsOn": {
+          "setting": "table.type",
+          "value": ["table", "viewV2"]
+        }
       },
       {
         "type": "boolean",
         "label": "Delete rows",
         "key": "allowDeleteRows",
-        "defaultValue": true
+        "defaultValue": true,
+        "dependsOn": {
+          "setting": "table.type",
+          "value": ["table", "viewV2"]
+        }
       },
       {
         "type": "boolean",

packages/client/src/components/app/DataProvider.svelte

@@ -81,6 +81,7 @@
       sortOrder: $fetch.sortOrder,
     },
     limit,
+    primaryDisplay: $fetch.definition?.primaryDisplay,
   }
 
   const createFetch = datasource => {

packages/client/src/components/app/GridBlock.svelte

@@ -4,6 +4,7 @@
   import { getContext } from "svelte"
   import { Grid } from "@budibase/frontend-core"
 
+  // table is actually any datasource, but called table for legacy compatibility
   export let table
   export let allowAddRows = true
   export let allowEditRows = true
@@ -21,7 +22,6 @@
 
   $: columnWhitelist = columns?.map(col => col.name)
   $: schemaOverrides = getSchemaOverrides(columns)
-  $: handleRowClick = allowEditRows ? undefined : onRowClick
 
   const getSchemaOverrides = columns => {
     let overrides = {}
@@ -58,7 +58,7 @@
     showControls={false}
     notifySuccess={notificationStore.actions.success}
     notifyError={notificationStore.actions.error}
-    on:rowclick={e => handleRowClick?.({ row: e.detail })}
+    on:rowclick={e => onRowClick?.({ row: e.detail })}
   />
 </div>
 

packages/client/src/components/app/blocks/form/FormBlock.svelte

@@ -12,6 +12,7 @@
   export let fields
   export let labelPosition
   export let title
+  export let description
   export let showDeleteButton
   export let showSaveButton
   export let saveButtonLabel
@@ -98,6 +99,7 @@
     fields: fieldsOrDefault,
     labelPosition,
     title,
+    description,
     saveButtonLabel: saveLabel,
     deleteButtonLabel: deleteLabel,
     schema,

packages/client/src/components/app/blocks/form/InnerFormBlock.svelte

@@ -11,6 +11,7 @@
   export let fields
   export let labelPosition
   export let title
+  export let description
   export let saveButtonLabel
   export let deleteButtonLabel
   export let schema
@@ -160,55 +161,71 @@
         <BlockComponent
           type="container"
           props={{
-            direction: "row",
+            direction: "column",
-            hAlign: "stretch",
+            gap: "S",
-            vAlign: "center",
-            gap: "M",
-            wrap: true,
           }}
           order={0}
         >
           <BlockComponent
-            type="heading"
+            type="container"
-            props={{ text: title || "" }}
+            props={{
+              direction: "row",
+              hAlign: "stretch",
+              vAlign: "center",
+              gap: "M",
+              wrap: true,
+            }}
             order={0}
-          />
+          >
-          {#if renderButtons}
             <BlockComponent
-              type="container"
+              type="heading"
-              props={{
+              props={{ text: title || "" }}
-                direction: "row",
+              order={0}
-                hAlign: "stretch",
+            />
-                vAlign: "center",
+            {#if renderButtons}
-                gap: "M",
+              <BlockComponent
-                wrap: true,
+                type="container"
-              }}
+                props={{
+                  direction: "row",
+                  hAlign: "stretch",
+                  vAlign: "center",
+                  gap: "M",
+                  wrap: true,
+                }}
+                order={1}
+              >
+                {#if renderDeleteButton}
+                  <BlockComponent
+                    type="button"
+                    props={{
+                      text: deleteButtonLabel,
+                      onClick: onDelete,
+                      quiet: true,
+                      type: "secondary",
+                    }}
+                    order={0}
+                  />
+                {/if}
+                {#if renderSaveButton}
+                  <BlockComponent
+                    type="button"
+                    props={{
+                      text: saveButtonLabel,
+                      onClick: onSave,
+                      type: "cta",
+                    }}
+                    order={1}
+                  />
+                {/if}
+              </BlockComponent>
+            {/if}
+          </BlockComponent>
+          {#if description}
+            <BlockComponent
+              type="text"
+              props={{ text: description }}
               order={1}
-            >
+            />
-              {#if renderDeleteButton}
-                <BlockComponent
-                  type="button"
-                  props={{
-                    text: deleteButtonLabel,
-                    onClick: onDelete,
-                    quiet: true,
-                    type: "secondary",
-                  }}
-                  order={0}
-                />
-              {/if}
-              {#if renderSaveButton}
-                <BlockComponent
-                  type="button"
-                  props={{
-                    text: saveButtonLabel,
-                    onClick: onSave,
-                    type: "cta",
-                  }}
-                  order={1}
-                />
-              {/if}
-            </BlockComponent>
           {/if}
         </BlockComponent>
       {/if}

packages/client/src/components/app/table/Table.svelte

@@ -2,8 +2,8 @@
   import { getContext } from "svelte"
   import { Table } from "@budibase/bbui"
   import SlotRenderer from "./SlotRenderer.svelte"
-  import { UnsortableTypes } from "../../../constants"
   import { onDestroy } from "svelte"
+  import { canBeSortColumn } from "@budibase/shared-core"
 
   export let dataProvider
   export let columns
@@ -32,7 +32,8 @@
   $: loading = dataProvider?.loading ?? false
   $: data = dataProvider?.rows || []
   $: fullSchema = dataProvider?.schema ?? {}
-  $: fields = getFields(fullSchema, columns, false)
+  $: primaryDisplay = dataProvider?.primaryDisplay
+  $: fields = getFields(fullSchema, columns, false, primaryDisplay)
   $: schema = getFilteredSchema(fullSchema, fields, hasChildren)
   $: setSorting = getAction(
     dataProvider?.id,
@@ -55,18 +56,13 @@
     }
   }
 
-  const getFields = (schema, customColumns, showAutoColumns) => {
+  const getFields = (
-    // Check for an invalid column selection
+    schema,
-    let invalid = false
+    customColumns,
-    customColumns?.forEach(column => {
+    showAutoColumns,
-      const columnName = typeof column === "string" ? column : column.name
+    primaryDisplay
-      if (schema[columnName] == null) {
+  ) => {
-        invalid = true
+    if (customColumns?.length) {
-      }
-    })
-
-    // Use column selection if it exists
-    if (!invalid && customColumns?.length) {
       return customColumns
     }
 
@@ -74,13 +70,38 @@
     let columns = []
     let autoColumns = []
     Object.entries(schema).forEach(([field, fieldSchema]) => {
+      if (fieldSchema.visible === false) {
+        return
+      }
       if (!fieldSchema?.autocolumn) {
         columns.push(field)
       } else if (showAutoColumns) {
         autoColumns.push(field)
       }
     })
-    return columns.concat(autoColumns)
+
+    // Sort columns to respect grid metadata
+    const allCols = columns.concat(autoColumns)
+    return allCols.sort((a, b) => {
+      if (a === primaryDisplay) {
+        return -1
+      }
+      if (b === primaryDisplay) {
+        return 1
+      }
+      const aOrder = schema[a].order
+      const bOrder = schema[b].order
+      if (aOrder === bOrder) {
+        return 0
+      }
+      if (aOrder == null) {
+        return 1
+      }
+      if (bOrder == null) {
+        return -1
+      }
+      return aOrder < bOrder ? -1 : 1
+    })
   }
 
   const getFilteredSchema = (schema, fields, hasChildren) => {
@@ -102,7 +123,7 @@
         return
       }
       newSchema[columnName] = schema[columnName]
-      if (UnsortableTypes.includes(schema[columnName].type)) {
+      if (!canBeSortColumn(schema[columnName].type)) {
         newSchema[columnName].sortable = false
       }
 

packages/client/src/constants.js

@@ -1,13 +1,5 @@
-import { FieldType as FieldTypes } from "@budibase/types"
 export { FieldType as FieldTypes } from "@budibase/types"
 
-export const UnsortableTypes = [
-  FieldTypes.FORMULA,
-  FieldTypes.ATTACHMENT,
-  FieldTypes.ARRAY,
-  FieldTypes.LINK,
-]
-
 export const ActionTypes = {
   ValidateForm: "ValidateForm",
   UpdateFieldValue: "UpdateFieldValue",

packages/frontend-core/src/components/grid/cells/DataCell.svelte

@@ -34,7 +34,7 @@
     column.schema.autocolumn ||
     column.schema.disabled ||
     column.schema.type === "formula" ||
-    (!$config.canEditRows && row._id)
+    (!$config.canEditRows && !row._isNewRow)
 
   // Register this cell API if the row is focused
   $: {

packages/frontend-core/src/components/grid/cells/HeaderCell.svelte

@@ -1,6 +1,6 @@
 <script>
   import { getContext, onMount, tick } from "svelte"
-  import { canBeDisplayColumn } from "@budibase/shared-core"
+  import { canBeDisplayColumn, canBeSortColumn } from "@budibase/shared-core"
   import { Icon, Popover, Menu, MenuItem, clickOutside } from "@budibase/bbui"
   import GridCell from "./GridCell.svelte"
   import { getColumnIcon } from "../lib/utils"
@@ -23,6 +23,7 @@
     columns,
     definition,
     datasource,
+    schema,
   } = getContext("grid")
 
   let anchor
@@ -119,16 +120,16 @@
     // Generate new name
     let newName = `${column.name} copy`
     let attempts = 2
-    while ($definition.schema[newName]) {
+    while ($schema[newName]) {
       newName = `${column.name} copy ${attempts++}`
     }
 
     // Save schema with new column
-    const existingColumnDefinition = $definition.schema[column.name]
+    const existingColumnDefinition = $schema[column.name]
     await datasource.actions.saveDefinition({
       ...$definition,
       schema: {
-        ...$definition.schema,
+        ...$schema,
         [newName]: {
           ...existingColumnDefinition,
           name: newName,
@@ -231,14 +232,16 @@
       <MenuItem
         icon="SortOrderUp"
         on:click={sortAscending}
-        disabled={column.name === $sort.column && $sort.order === "ascending"}
+        disabled={!canBeSortColumn(column.schema.type) ||
+          (column.name === $sort.column && $sort.order === "ascending")}
       >
         Sort {ascendingLabel}
       </MenuItem>
       <MenuItem
         icon="SortOrderDown"
         on:click={sortDescending}
-        disabled={column.name === $sort.column && $sort.order === "descending"}
+        disabled={!canBeSortColumn(column.schema.type) ||
+          (column.name === $sort.column && $sort.order === "descending")}
       >
         Sort {descendingLabel}
       </MenuItem>

packages/frontend-core/src/components/grid/controls/SortButton.svelte

@@ -1,6 +1,7 @@
 <script>
   import { getContext } from "svelte"
   import { ActionButton, Popover, Select } from "@budibase/bbui"
+  import { canBeSortColumn } from "@budibase/shared-core"
 
   const { sort, columns, stickyColumn } = getContext("grid")
 
@@ -19,7 +20,7 @@
         type: stickyColumn.schema?.type,
       })
     }
-    return [
+    options = [
       ...options,
       ...columns.map(col => ({
         label: col.label || col.name,
@@ -27,6 +28,7 @@
         type: col.schema?.type,
       })),
     ]
+    return options.filter(col => canBeSortColumn(col.type))
   }
 
   const getOrderOptions = (column, columnOptions) => {

packages/frontend-core/src/components/grid/layout/Grid.svelte

@@ -141,7 +141,14 @@
       </div>
     </div>
   {/if}
-  {#if $loaded}
+  {#if $error}
+    <div class="grid-error">
+      <div class="grid-error-title">There was a problem loading your grid</div>
+      <div class="grid-error-subtitle">
+        {$error}
+      </div>
+    </div>
+  {:else if $loaded}
     <div class="grid-data-outer" use:clickOutside={ui.actions.blur}>
       <div class="grid-data-inner">
         <StickyColumn>
@@ -171,13 +178,6 @@
         </div>
       </div>
     </div>
-  {:else if $error}
-    <div class="grid-error">
-      <div class="grid-error-title">There was a problem loading your grid</div>
-      <div class="grid-error-subtitle">
-        {$error}
-      </div>
-    </div>
   {/if}
   {#if $loading && !$error}
     <div in:fade|local={{ duration: 130 }} class="grid-loading">

packages/frontend-core/src/components/grid/layout/GridRow.svelte

@@ -18,6 +18,7 @@
     contentLines,
     isDragging,
     dispatch,
+    rows,
   } = getContext("grid")
 
   $: rowSelected = !!$selectedRows[row._id]
@@ -31,7 +32,7 @@
   on:focus
   on:mouseenter={$isDragging ? null : () => ($hoveredRowId = row._id)}
   on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
-  on:click={() => dispatch("rowclick", row)}
+  on:click={() => dispatch("rowclick", rows.actions.cleanRow(row))}
 >
   {#each $renderedColumns as column, columnIdx (column.name)}
     {@const cellId = `${row._id}-${column.name}`}

packages/frontend-core/src/components/grid/layout/NewRow.svelte

@@ -33,7 +33,7 @@
 
   let visible = false
   let isAdding = false
-  let newRow = {}
+  let newRow
   let offset = 0
 
   $: firstColumn = $stickyColumn || $renderedColumns[0]
@@ -58,7 +58,9 @@
 
     // Create row
     const newRowIndex = offset ? undefined : 0
-    const savedRow = await rows.actions.addRow(newRow, newRowIndex)
+    let rowToCreate = { ...newRow }
+    delete rowToCreate._isNewRow
+    const savedRow = await rows.actions.addRow(rowToCreate, newRowIndex)
     if (savedRow) {
       // Reset state
       clear()
@@ -109,7 +111,7 @@
     }
 
     // Update state and select initial cell
-    newRow = {}
+    newRow = { _isNewRow: true }
     visible = true
     $hoveredRowId = NewRowID
     if (firstColumn) {

packages/frontend-core/src/components/grid/layout/StickyColumn.svelte

@@ -74,7 +74,7 @@
           class="row"
           on:mouseenter={$isDragging ? null : () => ($hoveredRowId = row._id)}
           on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
-          on:click={() => dispatch("rowclick", row)}
+          on:click={() => dispatch("rowclick", rows.actions.cleanRow(row))}
         >
           <GutterCell {row} {rowFocused} {rowHovered} {rowSelected} />
           {#if $stickyColumn}

packages/frontend-core/src/components/grid/lib/utils.js

@@ -1,6 +1,6 @@
 export const getColor = (idx, opacity = 0.3) => {
   if (idx == null || idx === -1) {
-    return null
+    idx = 0
   }
   return `hsla(${((idx + 1) * 222) % 360}, 90%, 75%, ${opacity})`
 }

packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte

@@ -17,6 +17,7 @@
     focusedCellAPI,
     focusedRowId,
     notifications,
+    isDatasourcePlus,
   } = getContext("grid")
 
   $: style = makeStyle($menu)
@@ -75,7 +76,7 @@
       </MenuItem>
       <MenuItem
         icon="Copy"
-        disabled={isNewRow || !$focusedRow?._id}
+        disabled={isNewRow || !$focusedRow?._id || !$isDatasourcePlus}
         on:click={() => copyToClipboard($focusedRow?._id)}
         on:click={menu.actions.close}
       >

packages/frontend-core/src/components/grid/stores/columns.js

@@ -69,7 +69,7 @@ export const deriveStores = context => {
 }
 
 export const createActions = context => {
-  const { columns, stickyColumn, datasource, definition } = context
+  const { columns, stickyColumn, datasource, definition, schema } = context
 
   // Updates the datasources primary display column
   const changePrimaryDisplay = async column => {
@@ -101,7 +101,7 @@ export const createActions = context => {
     const $columns = get(columns)
     const $definition = get(definition)
     const $stickyColumn = get(stickyColumn)
-    const newSchema = cloneDeep($definition.schema)
+    let newSchema = cloneDeep(get(schema)) || {}
 
     // Build new updated datasource schema
     Object.keys(newSchema).forEach(column => {
@@ -142,26 +142,35 @@ export const createActions = context => {
 }
 
 export const initialise = context => {
-  const { definition, columns, stickyColumn, schema } = context
+  const { definition, columns, stickyColumn, enrichedSchema } = context
 
   // Merge new schema fields with existing schema in order to preserve widths
-  schema.subscribe($schema => {
+  enrichedSchema.subscribe($enrichedSchema => {
-    if (!$schema) {
+    if (!$enrichedSchema) {
       columns.set([])
       stickyColumn.set(null)
       return
     }
     const $definition = get(definition)
+    const $columns = get(columns)
+    const $stickyColumn = get(stickyColumn)
+
+    // Generate array of all columns to easily find pre-existing columns
+    let allColumns = $columns || []
+    if ($stickyColumn) {
+      allColumns.push($stickyColumn)
+    }
 
     // Find primary display
     let primaryDisplay
-    if ($definition.primaryDisplay && $schema[$definition.primaryDisplay]) {
+    const candidatePD = $definition.primaryDisplay || $stickyColumn?.name
-      primaryDisplay = $definition.primaryDisplay
+    if (candidatePD && $enrichedSchema[candidatePD]) {
+      primaryDisplay = candidatePD
     }
 
     // Get field list
     let fields = []
-    Object.keys($schema).forEach(field => {
+    Object.keys($enrichedSchema).forEach(field => {
       if (field !== primaryDisplay) {
         fields.push(field)
       }
@@ -170,14 +179,18 @@ export const initialise = context => {
     // Update columns, removing extraneous columns and adding missing ones
     columns.set(
       fields
-        .map(field => ({
+        .map(field => {
-          name: field,
+          const fieldSchema = $enrichedSchema[field]
-          label: $schema[field].displayName || field,
+          const oldColumn = allColumns?.find(x => x.name === field)
-          schema: $schema[field],
+          return {
-          width: $schema[field].width || DefaultColumnWidth,
+            name: field,
-          visible: $schema[field].visible ?? true,
+            label: fieldSchema.displayName || field,
-          order: $schema[field].order,
+            schema: fieldSchema,
-        }))
+            width: fieldSchema.width || oldColumn?.width || DefaultColumnWidth,
+            visible: fieldSchema.visible ?? true,
+            order: fieldSchema.order ?? oldColumn?.order,
+          }
+        })
         .sort((a, b) => {
           // Sort by order first
           const orderA = a.order
@@ -205,11 +218,13 @@ export const initialise = context => {
       stickyColumn.set(null)
       return
     }
+    const stickySchema = $enrichedSchema[primaryDisplay]
+    const oldStickyColumn = allColumns?.find(x => x.name === primaryDisplay)
     stickyColumn.set({
       name: primaryDisplay,
-      label: $schema[primaryDisplay].displayName || primaryDisplay,
+      label: stickySchema.displayName || primaryDisplay,
-      schema: $schema[primaryDisplay],
+      schema: stickySchema,
-      width: $schema[primaryDisplay].width || DefaultColumnWidth,
+      width: stickySchema.width || oldStickyColumn?.width || DefaultColumnWidth,
       visible: true,
       order: 0,
       left: GutterWidth,

packages/frontend-core/src/components/grid/stores/config.js

@@ -37,9 +37,10 @@ export const deriveStores = context => {
     [props, hasNonAutoColumn],
     ([$props, $hasNonAutoColumn]) => {
       let config = { ...$props }
+      const type = $props.datasource?.type
 
       // Disable some features if we're editing a view
-      if ($props.datasource?.type === "viewV2") {
+      if (type === "viewV2") {
         config.canEditColumns = false
       }
 
@@ -48,6 +49,16 @@ export const deriveStores = context => {
         config.canAddRows = false
       }
 
+      // Disable features for non DS+
+      if (!["table", "viewV2"].includes(type)) {
+        config.canAddRows = false
+        config.canEditRows = false
+        config.canDeleteRows = false
+        config.canExpandRows = false
+        config.canSaveSchema = false
+        config.canEditColumns = false
+      }
+
       return config
     }
   )

packages/frontend-core/src/components/grid/stores/datasource.js

@@ -1,4 +1,5 @@
 import { derived, get, writable } from "svelte/store"
+import { getDatasourceDefinition } from "../../../fetch"
 
 export const createStores = () => {
   const definition = writable(null)
@@ -9,21 +10,38 @@ export const createStores = () => {
 }
 
 export const deriveStores = context => {
-  const { definition, schemaOverrides, columnWhitelist } = context
+  const { definition, schemaOverrides, columnWhitelist, datasource } = context
 
-  const schema = derived(
+  const schema = derived(definition, $definition => {
-    [definition, schemaOverrides, columnWhitelist],
+    let schema = $definition?.schema
-    ([$definition, $schemaOverrides, $columnWhitelist]) => {
+    if (!schema) {
-      if (!$definition?.schema) {
+      return null
+    }
+
+    // Ensure schema is configured as objects.
+    // Certain datasources like queries use primitives.
+    Object.keys(schema || {}).forEach(key => {
+      if (typeof schema[key] !== "object") {
+        schema[key] = { type: schema[key] }
+      }
+    })
+
+    return schema
+  })
+
+  const enrichedSchema = derived(
+    [schema, schemaOverrides, columnWhitelist],
+    ([$schema, $schemaOverrides, $columnWhitelist]) => {
+      if (!$schema) {
         return null
       }
-      let newSchema = { ...$definition?.schema }
+      let enrichedSchema = { ...$schema }
 
       // Apply schema overrides
       Object.keys($schemaOverrides || {}).forEach(field => {
-        if (newSchema[field]) {
+        if (enrichedSchema[field]) {
-          newSchema[field] = {
+          enrichedSchema[field] = {
-            ...newSchema[field],
+            ...enrichedSchema[field],
             ...$schemaOverrides[field],
           }
         }
@@ -31,41 +49,64 @@ export const deriveStores = context => {
 
       // Apply whitelist if specified
       if ($columnWhitelist?.length) {
-        Object.keys(newSchema).forEach(key => {
+        Object.keys(enrichedSchema).forEach(key => {
           if (!$columnWhitelist.includes(key)) {
-            delete newSchema[key]
+            delete enrichedSchema[key]
           }
         })
       }
 
-      return newSchema
+      return enrichedSchema
     }
   )
 
+  const isDatasourcePlus = derived(datasource, $datasource => {
+    return ["table", "viewV2"].includes($datasource?.type)
+  })
+
   return {
     schema,
+    enrichedSchema,
+    isDatasourcePlus,
   }
 }
 
 export const createActions = context => {
-  const { datasource, definition, config, dispatch, table, viewV2 } = context
+  const {
+    API,
+    datasource,
+    definition,
+    config,
+    dispatch,
+    table,
+    viewV2,
+    nonPlus,
+  } = context
 
   // Gets the appropriate API for the configured datasource type
   const getAPI = () => {
     const $datasource = get(datasource)
-    switch ($datasource?.type) {
+    const type = $datasource?.type
+    if (!type) {
+      return null
+    }
+    switch (type) {
       case "table":
         return table
       case "viewV2":
         return viewV2
       default:
-        return null
+        return nonPlus
     }
   }
 
   // Refreshes the datasource definition
   const refreshDefinition = async () => {
-    return await getAPI()?.actions.refreshDefinition()
+    const def = await getDatasourceDefinition({
+      API,
+      datasource: get(datasource),
+    })
+    definition.set(def)
   }
 
   // Saves the datasource definition
@@ -113,6 +154,11 @@ export const createActions = context => {
     return getAPI()?.actions.canUseColumn(name)
   }
 
+  // Gets the default number of rows for a single page
+  const getFeatures = () => {
+    return getAPI()?.actions.getFeatures()
+  }
+
   return {
     datasource: {
       ...datasource,
@@ -125,6 +171,7 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
+        getFeatures,
       },
     },
   }

packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js

@@ -0,0 +1,124 @@
+import { get } from "svelte/store"
+
+export const createActions = context => {
+  const { columns, stickyColumn, table, viewV2 } = context
+
+  const saveDefinition = async () => {
+    throw "This datasource does not support updating the definition"
+  }
+
+  const saveRow = async () => {
+    throw "This datasource does not support saving rows"
+  }
+
+  const deleteRows = async () => {
+    throw "This datasource does not support deleting rows"
+  }
+
+  const getRow = () => {
+    throw "This datasource does not support fetching individual rows"
+  }
+
+  const isDatasourceValid = datasource => {
+    // There are many different types and shapes of datasource, so we only
+    // check that we aren't null
+    return (
+      !table.actions.isDatasourceValid(datasource) &&
+      !viewV2.actions.isDatasourceValid(datasource) &&
+      datasource?.type != null
+    )
+  }
+
+  const canUseColumn = name => {
+    const $columns = get(columns)
+    const $sticky = get(stickyColumn)
+    return $columns.some(col => col.name === name) || $sticky?.name === name
+  }
+
+  const getFeatures = () => {
+    // We don't support any features
+    return {}
+  }
+
+  return {
+    nonPlus: {
+      actions: {
+        saveDefinition,
+        addRow: saveRow,
+        updateRow: saveRow,
+        deleteRows,
+        getRow,
+        isDatasourceValid,
+        canUseColumn,
+        getFeatures,
+      },
+    },
+  }
+}
+
+// Small util to compare datasource definitions
+const isSameDatasource = (a, b) => {
+  return JSON.stringify(a) === JSON.stringify(b)
+}
+
+export const initialise = context => {
+  const {
+    datasource,
+    sort,
+    filter,
+    nonPlus,
+    initialFilter,
+    initialSortColumn,
+    initialSortOrder,
+    fetch,
+  } = context
+  // Keep a list of subscriptions so that we can clear them when the datasource
+  // config changes
+  let unsubscribers = []
+
+  // Observe datasource changes and apply logic for view V2 datasources
+  datasource.subscribe($datasource => {
+    // Clear previous subscriptions
+    unsubscribers?.forEach(unsubscribe => unsubscribe())
+    unsubscribers = []
+    if (!nonPlus.actions.isDatasourceValid($datasource)) {
+      return
+    }
+
+    // Wipe state
+    filter.set(get(initialFilter))
+    sort.set({
+      column: get(initialSortColumn),
+      order: get(initialSortOrder) || "ascending",
+    })
+
+    // Update fetch when filter changes
+    unsubscribers.push(
+      filter.subscribe($filter => {
+        // Ensure we're updating the correct fetch
+        const $fetch = get(fetch)
+        if (!isSameDatasource($fetch?.options?.datasource, $datasource)) {
+          return
+        }
+        $fetch.update({
+          filter: $filter,
+        })
+      })
+    )
+
+    // Update fetch when sorting changes
+    unsubscribers.push(
+      sort.subscribe($sort => {
+        // Ensure we're updating the correct fetch
+        const $fetch = get(fetch)
+        if (!isSameDatasource($fetch?.options?.datasource, $datasource)) {
+          return
+        }
+        $fetch.update({
+          sortOrder: $sort.order || "ascending",
+          sortColumn: $sort.column,
+        })
+      })
+    )
+  })
+}

packages/frontend-core/src/components/grid/stores/datasources/table.js

@@ -1,13 +1,10 @@
 import { get } from "svelte/store"
+import TableFetch from "../../../../fetch/TableFetch"
 
 const SuppressErrors = true
 
 export const createActions = context => {
-  const { definition, API, datasource, columns, stickyColumn } = context
+  const { API, datasource, columns, stickyColumn } = context
-
-  const refreshDefinition = async () => {
-    definition.set(await API.fetchTableDefinition(get(datasource).tableId))
-  }
 
   const saveDefinition = async newDefinition => {
     await API.saveTable(newDefinition)
@@ -49,10 +46,13 @@ export const createActions = context => {
     return $columns.some(col => col.name === name) || $sticky?.name === name
   }
 
+  const getFeatures = () => {
+    return new TableFetch({ API }).determineFeatureFlags()
+  }
+
   return {
     table: {
       actions: {
-        refreshDefinition,
         saveDefinition,
         addRow: saveRow,
         updateRow: saveRow,
@@ -60,6 +60,7 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
+        getFeatures,
       },
     },
   }

packages/frontend-core/src/components/grid/stores/datasources/viewV2.js

@@ -1,22 +1,10 @@
 import { get } from "svelte/store"
+import ViewV2Fetch from "../../../../fetch/ViewV2Fetch"
 
 const SuppressErrors = true
 
 export const createActions = context => {
-  const { definition, API, datasource, columns, stickyColumn } = context
+  const { API, datasource, columns, stickyColumn } = context
-
-  const refreshDefinition = async () => {
-    const $datasource = get(datasource)
-    if (!$datasource) {
-      definition.set(null)
-      return
-    }
-    const table = await API.fetchTableDefinition($datasource.tableId)
-    const view = Object.values(table?.views || {}).find(
-      view => view.id === $datasource.id
-    )
-    definition.set(view)
-  }
 
   const saveDefinition = async newDefinition => {
     await API.viewV2.update(newDefinition)
@@ -58,10 +46,13 @@ export const createActions = context => {
     )
   }
 
+  const getFeatures = () => {
+    return new ViewV2Fetch({ API }).determineFeatureFlags()
+  }
+
   return {
     viewV2: {
       actions: {
-        refreshDefinition,
         saveDefinition,
         addRow: saveRow,
         updateRow: saveRow,
@@ -69,6 +60,7 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
+        getFeatures,
       },
     },
   }

packages/frontend-core/src/components/grid/stores/index.js

@@ -15,9 +15,10 @@ import * as Config from "./config"
 import * as Sort from "./sort"
 import * as Filter from "./filter"
 import * as Notifications from "./notifications"
-import * as Table from "./table"
-import * as ViewV2 from "./viewV2"
 import * as Datasource from "./datasource"
+import * as Table from "./datasources/table"
+import * as ViewV2 from "./datasources/viewV2"
+import * as NonPlus from "./datasources/nonPlus"
 
 const DependencyOrderedStores = [
   Sort,
@@ -26,6 +27,7 @@ const DependencyOrderedStores = [
   Scroll,
   Table,
   ViewV2,
+  NonPlus,
   Datasource,
   Columns,
   Rows,

packages/frontend-core/src/components/grid/stores/rows.js

@@ -1,7 +1,8 @@
 import { writable, derived, get } from "svelte/store"
-import { fetchData } from "../../../fetch/fetchData"
+import { fetchData } from "../../../fetch"
 import { NewRowID, RowPageSize } from "../lib/constants"
 import { tick } from "svelte"
+import { Helpers } from "@budibase/bbui"
 
 export const createStores = () => {
   const rows = writable([])
@@ -76,11 +77,11 @@ export const createActions = context => {
     columns,
     rowChangeCache,
     inProgressChanges,
-    previousFocusedRowId,
     hasNextPage,
     error,
     notifications,
     fetch,
+    isDatasourcePlus,
   } = context
   const instanceLoaded = writable(false)
 
@@ -93,12 +94,14 @@ export const createActions = context => {
   datasource.subscribe(async $datasource => {
     // Unsub from previous fetch if one exists
     unsubscribe?.()
+    unsubscribe = null
     fetch.set(null)
     instanceLoaded.set(false)
     loading.set(true)
 
     // Abandon if we don't have a valid datasource
     if (!datasource.actions.isDatasourceValid($datasource)) {
+      error.set("Datasource is invalid")
       return
     }
 
@@ -108,6 +111,10 @@ export const createActions = context => {
     const $filter = get(filter)
     const $sort = get(sort)
 
+    // Determine how many rows to fetch per page
+    const features = datasource.actions.getFeatures()
+    const limit = features?.supportsPagination ? RowPageSize : null
+
     // Create new fetch model
     const newFetch = fetchData({
       API,
@@ -116,7 +123,7 @@ export const createActions = context => {
         filter: $filter,
         sortColumn: $sort.column,
         sortOrder: $sort.order,
-        limit: RowPageSize,
+        limit,
         paginate: true,
       },
     })
@@ -355,7 +362,7 @@ export const createActions = context => {
 
       // Update row
       const saved = await datasource.actions.updateRow({
-        ...row,
+        ...cleanRow(row),
         ...get(rowChangeCache)[rowId],
       })
 
@@ -411,8 +418,17 @@ export const createActions = context => {
     }
     let rowsToAppend = []
     let newRow
+    const $isDatasourcePlus = get(isDatasourcePlus)
     for (let i = 0; i < newRows.length; i++) {
       newRow = newRows[i]
+
+      // Ensure we have a unique _id.
+      // This means generating one for non DS+, overriting any that may already
+      // exist as we cannot allow duplicates.
+      if (!$isDatasourcePlus) {
+        newRow._id = Helpers.uuid()
+      }
+
       if (!rowCacheMap[newRow._id]) {
         rowCacheMap[newRow._id] = true
         rowsToAppend.push(newRow)
@@ -449,15 +465,16 @@ export const createActions = context => {
     return get(rowLookupMap)[id] != null
   }
 
-  // Wipe the row change cache when changing row
+  // Cleans a row by removing any internal grid metadata from it.
-  previousFocusedRowId.subscribe(id => {
+  // Call this before passing a row to any sort of external flow.
-    if (id && !get(inProgressChanges)[id]) {
+  const cleanRow = row => {
-      rowChangeCache.update(state => {
+    let clone = { ...row }
-        delete state[id]
+    delete clone.__idx
-        return state
+    if (!get(isDatasourcePlus)) {
-      })
+      delete clone._id
     }
-  })
+    return clone
+  }
 
   return {
     rows: {
@@ -474,7 +491,22 @@ export const createActions = context => {
         refreshRow,
         replaceRow,
         refreshData,
+        cleanRow,
       },
     },
   }
 }
+
+export const initialise = context => {
+  const { rowChangeCache, inProgressChanges, previousFocusedRowId } = context
+
+  // Wipe the row change cache when changing row
+  previousFocusedRowId.subscribe(id => {
+    if (id && !get(inProgressChanges)[id]) {
+      rowChangeCache.update(state => {
+        delete state[id]
+        return state
+      })
+    }
+  })
+}

packages/frontend-core/src/components/grid/stores/sort.js

@@ -17,7 +17,7 @@ export const createStores = context => {
 }
 
 export const initialise = context => {
-  const { sort, initialSortColumn, initialSortOrder, definition } = context
+  const { sort, initialSortColumn, initialSortOrder, schema } = context
 
   // Reset sort when initial sort props change
   initialSortColumn.subscribe(newSortColumn => {
@@ -28,15 +28,12 @@ export const initialise = context => {
   })
 
   // Derive if the current sort column exists in the schema
-  const sortColumnExists = derived(
+  const sortColumnExists = derived([sort, schema], ([$sort, $schema]) => {
-    [sort, definition],
+    if (!$sort?.column || !$schema) {
-    ([$sort, $definition]) => {
+      return true
-      if (!$sort?.column || !$definition) {
-        return true
-      }
-      return $definition.schema?.[$sort.column] != null
     }
-  )
+    return $schema[$sort.column] != null
+  })
 
   // Clear sort state if our sort column does not exist
   sortColumnExists.subscribe(exists => {

packages/frontend-core/src/fetch/CustomFetch.js

@@ -0,0 +1,145 @@
+import DataFetch from "./DataFetch.js"
+
+export default class CustomFetch extends DataFetch {
+  // Gets the correct Budibase type for a JS value
+  getType(value) {
+    if (value == null) {
+      return "string"
+    }
+    const type = typeof value
+    if (type === "object") {
+      if (Array.isArray(value)) {
+        // Use our custom array type to render badges
+        return "array"
+      }
+      // Use JSON for objects to ensure they are stringified
+      return "json"
+    } else if (!isNaN(value)) {
+      return "number"
+    } else {
+      return "string"
+    }
+  }
+
+  // Parses the custom data into an array format
+  parseCustomData(data) {
+    if (!data) {
+      return []
+    }
+
+    // Happy path - already an array
+    if (Array.isArray(data)) {
+      return data
+    }
+
+    // For strings, try JSON then fall back to attempting a CSV
+    if (typeof data === "string") {
+      try {
+        const js = JSON.parse(data)
+        return Array.isArray(js) ? js : [js]
+      } catch (error) {
+        // Ignore
+      }
+
+      // Try splitting by newlines first
+      if (data.includes("\n")) {
+        return data.split("\n").map(x => x.trim())
+      }
+
+      // Split by commas next
+      return data.split(",").map(x => x.trim())
+    }
+
+    // Other cases we just assume it's a single object and wrap it
+    return [data]
+  }
+
+  // Enriches the custom data to ensure the structure and format is usable
+  enrichCustomData(data) {
+    if (!data?.length) {
+      return []
+    }
+
+    // Filter out any invalid values
+    data = data.filter(x => x != null && x !== "" && !Array.isArray(x))
+
+    // Ensure all values are packed into objects
+    return data.map(value => {
+      if (typeof value === "object") {
+        return value
+      }
+
+      // Try parsing strings
+      if (typeof value === "string") {
+        const split = value.split(",").map(x => x.trim())
+        let obj = {}
+        for (let i = 0; i < split.length; i++) {
+          const suffix = i === 0 ? "" : ` ${i + 1}`
+          const key = `Value${suffix}`
+          obj[key] = split[i]
+        }
+        return obj
+      }
+
+      // For anything else, wrap in an object
+      return { Value: value }
+    })
+  }
+
+  // Extracts and parses the custom data from the datasource definition
+  getCustomData(datasource) {
+    return this.enrichCustomData(this.parseCustomData(datasource?.data))
+  }
+
+  async getDefinition(datasource) {
+    // Try and work out the schema from the array provided
+    let schema = {}
+    const data = this.getCustomData(datasource)
+    if (!data?.length) {
+      return { schema }
+    }
+
+    // Go through every object and extract all valid keys
+    for (let datum of data) {
+      for (let key of Object.keys(datum)) {
+        if (key === "_id") {
+          continue
+        }
+        if (!schema[key]) {
+          let type = this.getType(datum[key])
+          let constraints = {}
+
+          // Determine whether we should render text columns as options instead
+          if (type === "string") {
+            const uniqueValues = [...new Set(data.map(x => x[key]))]
+            const uniqueness = uniqueValues.length / data.length
+            if (uniqueness <= 0.8 && uniqueValues.length > 1) {
+              type = "options"
+              constraints.inclusion = uniqueValues
+            }
+          }
+
+          // Generate options for array columns
+          else if (type === "array") {
+            constraints.inclusion = [...new Set(data.map(x => x[key]).flat())]
+          }
+
+          schema[key] = {
+            type,
+            constraints,
+          }
+        }
+      }
+    }
+    return { schema }
+  }
+
+  async getData() {
+    const { datasource } = this.options
+    return {
+      rows: this.getCustomData(datasource),
+      hasNextPage: false,
+      cursor: null,
+    }
+  }
+}

packages/frontend-core/src/fetch/index.js

@@ -8,6 +8,7 @@ import FieldFetch from "./FieldFetch.js"
 import JSONArrayFetch from "./JSONArrayFetch.js"
 import UserFetch from "./UserFetch.js"
 import GroupUserFetch from "./GroupUserFetch.js"
+import CustomFetch from "./CustomFetch.js"
 
 const DataFetchMap = {
   table: TableFetch,
@@ -17,6 +18,7 @@ const DataFetchMap = {
   link: RelationshipFetch,
   user: UserFetch,
   groupUser: GroupUserFetch,
+  custom: CustomFetch,
 
   // Client specific datasource types
   provider: NestedProviderFetch,
@@ -24,7 +26,18 @@ const DataFetchMap = {
   jsonarray: JSONArrayFetch,
 }
 
+// Constructs a new fetch model for a certain datasource
 export const fetchData = ({ API, datasource, options }) => {
   const Fetch = DataFetchMap[datasource?.type] || TableFetch
   return new Fetch({ API, datasource, ...options })
 }
+
+// Fetches the definition of any type of datasource
+export const getDatasourceDefinition = async ({ API, datasource }) => {
+  const handler = DataFetchMap[datasource?.type]
+  if (!handler) {
+    return null
+  }
+  const instance = new handler({ API })
+  return await instance.getDefinition(datasource)
+}

packages/frontend-core/src/index.js

@@ -1,5 +1,5 @@
 export { createAPIClient } from "./api"
-export { fetchData } from "./fetch/fetchData"
+export { fetchData } from "./fetch"
 export { Utils } from "./utils"
 export * as Constants from "./constants"
 export * from "./stores"

packages/pro

@@ -1 +1 @@
-Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76
+Subproject commit d24c0dc3a30014cbe61860252aa48104cad36376

packages/server/Dockerfile.v2

@@ -67,6 +67,11 @@ COPY packages/server/docker_run.sh .
 COPY packages/server/builder/ builder/
 COPY packages/server/client/ client/
 
+ARG BUDIBASE_VERSION
+# Ensuring the version argument is sent
+RUN test -n "$BUDIBASE_VERSION"
+ENV BUDIBASE_VERSION=$BUDIBASE_VERSION
+
 EXPOSE 4001
 
 # have to add node environment production after install

packages/server/package.json

@@ -18,7 +18,7 @@
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
-    "build:docker": "yarn build && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "build:docker": "yarn build && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION",
     "run:docker": "node dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
     "dev:stack:up": "node scripts/dev/manage.js up",

packages/server/scripts/dev/manage.js

@@ -47,6 +47,7 @@ async function init() {
       TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
       HTTP_MIGRATIONS: "0",
       HTTP_LOGGING: "0",
+      VERSION: "0.0.0+local",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

packages/server/src/api/controllers/static/index.ts

@@ -16,7 +16,7 @@ import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
-import { App } from "@budibase/types"
+import { App, Ctx } from "@budibase/types"
 
 const send = require("koa-send")
 
@@ -39,7 +39,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
   }
 }
 
-export const toggleBetaUiFeature = async function (ctx: any) {
+export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
 
   if (ctx.cookies.get(cookieName)) {
@@ -67,16 +67,14 @@ export const toggleBetaUiFeature = async function (ctx: any) {
   }
 }
 
-export const serveBuilder = async function (ctx: any) {
+export const serveBuilder = async function (ctx: Ctx) {
   const builderPath = join(TOP_LEVEL_PATH, "builder")
   await send(ctx, ctx.file, { root: builderPath })
 }
 
-export const uploadFile = async function (ctx: any) {
+export const uploadFile = async function (ctx: Ctx) {
-  let files =
+  const file = ctx.request?.files?.file
-    ctx.request.files.file.length > 1
+  let files = file && Array.isArray(file) ? Array.from(file) : [file]
-      ? Array.from(ctx.request.files.file)
-      : [ctx.request.files.file]
 
   const uploads = files.map(async (file: any) => {
     const fileExtension = [...file.name.split(".")].pop()
@@ -93,14 +91,14 @@ export const uploadFile = async function (ctx: any) {
   ctx.body = await Promise.all(uploads)
 }
 
-export const deleteObjects = async function (ctx: any) {
+export const deleteObjects = async function (ctx: Ctx) {
   ctx.body = await objectStore.deleteFiles(
     ObjectStoreBuckets.APPS,
     ctx.request.body.keys
   )
 }
 
-export const serveApp = async function (ctx: any) {
+export const serveApp = async function (ctx: Ctx) {
   const bbHeaderEmbed =
     ctx.request.get("x-budibase-embed")?.toLowerCase() === "true"
 
@@ -181,7 +179,7 @@ export const serveApp = async function (ctx: any) {
   }
 }
 
-export const serveBuilderPreview = async function (ctx: any) {
+export const serveBuilderPreview = async function (ctx: Ctx) {
   const db = context.getAppDB({ skip_setup: true })
   const appInfo = await db.get<App>(DocumentType.APP_METADATA)
 
@@ -197,18 +195,30 @@ export const serveBuilderPreview = async function (ctx: any) {
   }
 }
 
-export const serveClientLibrary = async function (ctx: any) {
+export const serveClientLibrary = async function (ctx: Ctx) {
+  const appId = context.getAppId() || (ctx.request.query.appId as string)
   let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist")
-  // incase running from TS directly
+  if (!appId) {
-  if (env.isDev() && !fs.existsSync(rootPath)) {
+    ctx.throw(400, "No app ID provided - cannot fetch client library.")
-    rootPath = join(require.resolve("@budibase/client"), "..")
+  }
+  if (env.isProd()) {
+    ctx.body = await objectStore.getReadStream(
+      ObjectStoreBuckets.APPS,
+      objectStore.clientLibraryPath(appId!)
+    )
+    ctx.set("Content-Type", "application/javascript")
+  } else if (env.isDev()) {
+    // incase running from TS directly
+    const tsPath = join(require.resolve("@budibase/client"), "..")
+    return send(ctx, "budibase-client.js", {
+      root: !fs.existsSync(rootPath) ? tsPath : rootPath,
+    })
+  } else {
+    ctx.throw(500, "Unable to retrieve client library.")
   }
-  return send(ctx, "budibase-client.js", {
-    root: rootPath,
-  })
 }
 
-export const getSignedUploadURL = async function (ctx: any) {
+export const getSignedUploadURL = async function (ctx: Ctx) {
   // Ensure datasource is valid
   let datasource
   try {
@@ -247,7 +257,7 @@ export const getSignedUploadURL = async function (ctx: any) {
       const params = { Bucket: bucket, Key: key }
       signedUrl = s3.getSignedUrl("putObject", params)
       publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}`
-    } catch (error) {
+    } catch (error: any) {
       ctx.throw(400, error)
     }
   }

packages/server/src/api/routes/static.ts

@@ -27,15 +27,9 @@ router.param("file", async (file: any, ctx: any, next: any) => {
   return next()
 })
 
-// only used in development for retrieving the client library,
-// in production the client lib is always stored in the object store.
-if (env.isDev()) {
-  router.get("/api/assets/client", controller.serveClientLibrary)
-}
-
 router
-  // TODO: for now this builder endpoint is not authorized/secured, will need to be
   .get("/builder/:file*", controller.serveBuilder)
+  .get("/api/assets/client", controller.serveClientLibrary)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
   .post(
     "/api/attachments/delete",

packages/server/src/api/routes/tests/debug.spec.ts

@@ -41,7 +41,7 @@ describe("/component", () => {
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body).toEqual({
-        budibaseVersion: "0.0.0",
+        budibaseVersion: "0.0.0+jest",
         cpuArch: "arm64",
         cpuCores: 1,
         cpuInfo: "test",

packages/server/src/api/routes/tests/dev.spec.js

@@ -1,6 +1,6 @@
 const setup = require("./utilities")
 const { events } = require("@budibase/backend-core")
-const version = require("../../../../package.json").version
+
 
 describe("/dev", () => {
   let request = setup.getRequest()
@@ -32,9 +32,9 @@ describe("/dev", () => {
         .expect("Content-Type", /json/)
         .expect(200)
 
-      expect(res.body.version).toBe(version)
+      expect(res.body.version).toBe('0.0.0+jest')
       expect(events.installation.versionChecked).toBeCalledTimes(1)
-      expect(events.installation.versionChecked).toBeCalledWith(version)
+      expect(events.installation.versionChecked).toBeCalledWith('0.0.0+jest')
     })
   })
 })

packages/server/src/tests/jestEnv.ts

@@ -9,3 +9,4 @@ process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
 process.env.MOCK_REDIS = "1"
 process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.REDIS_PASSWORD = "budibase"
+process.env.BUDIBASE_VERSION = "0.0.0+jest"

packages/shared-core/src/table.ts

@@ -20,6 +20,30 @@ const allowDisplayColumnByType: Record<FieldType, boolean> = {
   [FieldType.BB_REFERENCE]: false,
 }
 
+const allowSortColumnByType: Record<FieldType, boolean> = {
+  [FieldType.STRING]: true,
+  [FieldType.LONGFORM]: true,
+  [FieldType.OPTIONS]: true,
+  [FieldType.NUMBER]: true,
+  [FieldType.DATETIME]: true,
+  [FieldType.AUTO]: true,
+  [FieldType.INTERNAL]: true,
+  [FieldType.BARCODEQR]: true,
+  [FieldType.BIGINT]: true,
+  [FieldType.BOOLEAN]: true,
+  [FieldType.JSON]: true,
+
+  [FieldType.FORMULA]: false,
+  [FieldType.ATTACHMENT]: false,
+  [FieldType.ARRAY]: false,
+  [FieldType.LINK]: false,
+  [FieldType.BB_REFERENCE]: false,
+}
+
 export function canBeDisplayColumn(type: FieldType): boolean {
   return !!allowDisplayColumnByType[type]
 }
+
+export function canBeSortColumn(type: FieldType): boolean {
+  return !!allowSortColumnByType[type]
+}

packages/types/src/api/web/user.ts

@@ -55,6 +55,7 @@ export interface SearchUsersRequest {
   bookmark?: string
   query?: SearchQuery
   appId?: string
+  limit?: number
   paginate?: boolean
 }
 

packages/types/src/sdk/featureFlag.ts

@@ -1,8 +1,5 @@
 export enum FeatureFlag {
   LICENSING = "LICENSING",
-  // Feature IDs in Posthog
-  PER_CREATOR_PER_USER_PRICE = "18873",
-  PER_CREATOR_PER_USER_PRICE_ALERT = "18530",
 }
 
 export interface TenantFeatureFlags {

packages/types/src/sdk/licensing/billing.ts

@@ -5,17 +5,10 @@ export interface Customer {
   currency: string | null | undefined
 }
 
-export interface SubscriptionItems {
-  user: number | undefined
-  creator: number | undefined
-}
-
 export interface Subscription {
   amount: number
-  amounts: SubscriptionItems | undefined
   currency: string
   quantity: number
-  quantities: SubscriptionItems | undefined
   duration: PriceDuration
   cancelAt: number | null | undefined
   currentPeriodStart: number

packages/types/src/sdk/licensing/plan.ts

@@ -4,9 +4,7 @@ export enum PlanType {
   PRO = "pro",
   /** @deprecated */
   TEAM = "team",
-  /** @deprecated */
   PREMIUM = "premium",
-  PREMIUM_PLUS = "premium_plus",
   BUSINESS = "business",
   ENTERPRISE = "enterprise",
 }
@@ -28,12 +26,10 @@ export interface AvailablePrice {
   currency: string
   duration: PriceDuration
   priceId: string
-  type?: string
 }
 
 export enum PlanModel {
   PER_USER = "perUser",
-  PER_CREATOR_PER_USER = "per_creator_per_user",
   DAY_PASS = "dayPass",
 }
 

packages/types/src/sdk/migrations.ts

@@ -46,7 +46,7 @@ export enum MigrationName {
   GLOBAL_INFO_SYNC_USERS = "global_info_sync_users",
   TABLE_SETTINGS_LINKS_TO_ACTIONS = "table_settings_links_to_actions",
   // increment this number to re-activate this migration
-  SYNC_QUOTAS = "sync_quotas_1",
+  SYNC_QUOTAS = "sync_quotas_2",
 }
 
 export interface MigrationDefinition {

packages/worker/Dockerfile.v2

@@ -50,4 +50,9 @@ ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
 ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
 ENV ACCOUNT_PORTAL_URL=https://account.budibase.app
 
+ARG BUDIBASE_VERSION
+# Ensuring the version argument is sent
+RUN test -n "$BUDIBASE_VERSION"
+ENV BUDIBASE_VERSION=$BUDIBASE_VERSION
+
 CMD ["./docker_run.sh"]

packages/worker/package.json

@@ -20,7 +20,7 @@
     "run:docker": "node dist/index.js",
     "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
     "run:docker:cluster": "pm2-runtime start pm2.config.js",
-    "build:docker": "yarn build && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "build:docker": "yarn build && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION",
     "dev:stack:init": "node ./scripts/dev/manage.js init",
     "dev:builder": "npm run dev:stack:init && nodemon",
     "dev:built": "yarn run dev:stack:init && yarn run run:docker",

packages/worker/scripts/dev/manage.js

@@ -31,6 +31,7 @@ async function init() {
       TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
       ENABLE_EMAIL_TEST_MODE: 1,
       HTTP_LOGGING: 0,
+      VERSION: "0.0.0+local",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

packages/worker/src/api/controllers/global/users.ts

@@ -189,7 +189,10 @@ export const destroy = async (ctx: any) => {
 
 export const getAppUsers = async (ctx: Ctx<SearchUsersRequest>) => {
   const body = ctx.request.body
-  const users = await userSdk.db.getUsersByAppAccess(body?.appId)
+  const users = await userSdk.db.getUsersByAppAccess({
+    appId: body.appId,
+    limit: body.limit,
+  })
 
   ctx.body = { data: users }
 }

packages/worker/src/api/routes/global/tests/users.spec.ts

@@ -569,9 +569,13 @@ describe("/api/global/users", () => {
         {
           query: { equal: { firstName: user.firstName } },
         },
-        501
+        { status: 501 }
       )
     })
+
+    it("should throw an error if public query performed", async () => {
+      await config.api.users.searchUsers({}, { status: 403, noHeaders: true })
+    })
   })
 
   describe("DELETE /api/global/users/:userId", () => {

packages/worker/src/api/routes/global/users.ts

@@ -72,7 +72,8 @@ router
   )
 
   .get("/api/global/users", auth.builderOrAdmin, controller.fetch)
-  .post("/api/global/users/search", auth.builderOrAdmin, controller.search)
+  // search can be used by any user now, to retrieve users for user column
+  .post("/api/global/users/search", controller.search)
   .delete("/api/global/users/:id", auth.adminOnly, controller.destroy)
   .get(
     "/api/global/users/count/:appId",

packages/worker/src/tests/api/users.ts

@@ -134,13 +134,19 @@ export class UserAPI extends TestAPI {
       .expect(status ? status : 200)
   }
 
-  searchUsers = ({ query }: { query?: SearchQuery }, status = 200) => {
+  searchUsers = (
-    return this.request
+    { query }: { query?: SearchQuery },
+    opts?: { status?: number; noHeaders?: boolean }
+  ) => {
+    const req = this.request
       .post("/api/global/users/search")
-      .set(this.config.defaultHeaders())
       .send({ query })
       .expect("Content-Type", /json/)
-      .expect(status ? status : 200)
+      .expect(opts?.status ? opts.status : 200)
+    if (!opts?.noHeaders) {
+      req.set(this.config.defaultHeaders())
+    }
+    return req
   }
 
   getUser = (userId: string, opts?: TestAPIOpts) => {

packages/worker/src/tests/jestEnv.ts

@@ -10,3 +10,4 @@ process.env.PLATFORM_URL = "http://localhost:10000"
 process.env.INTERNAL_API_KEY = "tet"
 process.env.DISABLE_ACCOUNT_PORTAL = "0"
 process.env.MOCK_REDIS = "1"
+process.env.BUDIBASE_VERSION = "0.0.0+jest"

qa-core/package.json

@@ -17,7 +17,7 @@
     "test:notify": "node scripts/testResultsWebhook",
     "test:cloud:prod": "yarn run test --testPathIgnorePatterns=\\.integration\\.",
     "test:cloud:qa": "yarn run test",
-    "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\.",
+    "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.license\\.",
     "serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci",
     "serve": "start-server-and-test dev:built http://localhost:4001/health",
     "dev:built": "cd ../ && yarn dev:built"

qa-core/src/account-api/api/AccountInternalAPI.ts

@@ -1,5 +1,5 @@
 import AccountInternalAPIClient from "./AccountInternalAPIClient"
-import { AccountAPI, LicenseAPI, AuthAPI } from "./apis"
+import { AccountAPI, LicenseAPI, AuthAPI, StripeAPI } from "./apis"
 import { State } from "../../types"
 
 export default class AccountInternalAPI {
@@ -8,11 +8,13 @@ export default class AccountInternalAPI {
   auth: AuthAPI
   accounts: AccountAPI
   licenses: LicenseAPI
+  stripe: StripeAPI
 
   constructor(state: State) {
     this.client = new AccountInternalAPIClient(state)
     this.auth = new AuthAPI(this.client)
     this.accounts = new AccountAPI(this.client)
     this.licenses = new LicenseAPI(this.client)
+    this.stripe = new StripeAPI(this.client)
   }
 }

qa-core/src/account-api/api/apis/LicenseAPI.ts

@@ -2,21 +2,19 @@ import AccountInternalAPIClient from "../AccountInternalAPIClient"
 import {
   Account,
   CreateOfflineLicenseRequest,
+  GetLicenseKeyResponse,
   GetOfflineLicenseResponse,
   UpdateLicenseRequest,
 } from "@budibase/types"
 import { Response } from "node-fetch"
 import BaseAPI from "./BaseAPI"
 import { APIRequestOpts } from "../../../types"
-
 export default class LicenseAPI extends BaseAPI {
   client: AccountInternalAPIClient
-
   constructor(client: AccountInternalAPIClient) {
     super()
     this.client = client
   }
-
   async updateLicense(
     accountId: string,
     body: UpdateLicenseRequest,
@@ -29,9 +27,7 @@ export default class LicenseAPI extends BaseAPI {
       })
     }, opts)
   }
-
   // TODO: Better approach for setting tenant id header
-
   async createOfflineLicense(
     accountId: string,
     tenantId: string,
@@ -51,7 +47,6 @@ export default class LicenseAPI extends BaseAPI {
     expect(response.status).toBe(opts.status ? opts.status : 201)
     return response
   }
-
   async getOfflineLicense(
     accountId: string,
     tenantId: string,
@@ -69,4 +64,74 @@ export default class LicenseAPI extends BaseAPI {
     expect(response.status).toBe(opts.status ? opts.status : 200)
     return [response, json]
   }
+  async getLicenseKey(
+    opts: { status?: number } = {}
+  ): Promise<[Response, GetLicenseKeyResponse]> {
+    const [response, json] = await this.client.get(`/api/license/key`)
+    expect(response.status).toBe(opts.status || 200)
+    return [response, json]
+  }
+  async activateLicense(
+    apiKey: string,
+    tenantId: string,
+    licenseKey: string,
+    opts: APIRequestOpts = { status: 200 }
+  ) {
+    return this.doRequest(() => {
+      return this.client.post(`/api/license/activate`, {
+        body: {
+          apiKey: apiKey,
+          tenantId: tenantId,
+          licenseKey: licenseKey,
+        },
+      })
+    }, opts)
+  }
+  async regenerateLicenseKey(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.post(`/api/license/key/regenerate`, {})
+    }, opts)
+  }
+
+  async getPlans(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.get(`/api/plans`)
+    }, opts)
+  }
+
+  async updatePlan(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.put(`/api/license/plan`)
+    }, opts)
+  }
+
+  async refreshAccountLicense(
+    accountId: string,
+    opts: { status?: number } = {}
+  ): Promise<Response> {
+    const [response, json] = await this.client.post(
+      `/api/accounts/${accountId}/license/refresh`,
+      {
+        internal: true,
+      }
+    )
+    expect(response.status).toBe(opts.status ? opts.status : 201)
+    return response
+  }
+
+  async getLicenseUsage(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.get(`/api/license/usage`)
+    }, opts)
+  }
+
+  async licenseUsageTriggered(
+    opts: { status?: number } = {}
+  ): Promise<Response> {
+    const [response, json] = await this.client.post(
+      `/api/license/usage/triggered`
+    )
+    expect(response.status).toBe(opts.status ? opts.status : 201)
+    return response
+  }
 }

qa-core/src/account-api/api/apis/StripeAPI.ts

@@ -0,0 +1,64 @@
+import AccountInternalAPIClient from "../AccountInternalAPIClient"
+import BaseAPI from "./BaseAPI"
+import { APIRequestOpts } from "../../../types"
+
+export default class StripeAPI extends BaseAPI {
+  client: AccountInternalAPIClient
+
+  constructor(client: AccountInternalAPIClient) {
+    super()
+    this.client = client
+  }
+
+  async createCheckoutSession(
+    priceId: string,
+    opts: APIRequestOpts = { status: 200 }
+  ) {
+    return this.doRequest(() => {
+      return this.client.post(`/api/stripe/checkout-session`, {
+        body: { priceId },
+      })
+    }, opts)
+  }
+
+  async checkoutSuccess(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.post(`/api/stripe/checkout-success`)
+    }, opts)
+  }
+
+  async createPortalSession(
+    stripeCustomerId: string,
+    opts: APIRequestOpts = { status: 200 }
+  ) {
+    return this.doRequest(() => {
+      return this.client.post(`/api/stripe/portal-session`, {
+        body: { stripeCustomerId },
+      })
+    }, opts)
+  }
+
+  async linkStripeCustomer(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.post(`/api/stripe/link`)
+    }, opts)
+  }
+
+  async getInvoices(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.get(`/api/stripe/invoices`)
+    }, opts)
+  }
+
+  async getUpcomingInvoice(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.get(`/api/stripe/upcoming-invoice`)
+    }, opts)
+  }
+
+  async getStripeCustomers(opts: APIRequestOpts = { status: 200 }) {
+    return this.doRequest(() => {
+      return this.client.get(`/api/stripe/customers`)
+    }, opts)
+  }
+}

qa-core/src/account-api/api/apis/index.ts

@@ -1,3 +1,4 @@
 export { default as AuthAPI } from "./AuthAPI"
 export { default as AccountAPI } from "./AccountAPI"
 export { default as LicenseAPI } from "./LicenseAPI"
+export { default as StripeAPI } from "./StripeAPI"

qa-core/src/account-api/tests/licensing/license.activate.spec.ts

@@ -0,0 +1,68 @@
+import TestConfiguration from "../../config/TestConfiguration"
+import * as fixures from "../../fixtures"
+import { Feature, Hosting } from "@budibase/types"
+
+describe("license activation", () => {
+  const config = new TestConfiguration()
+
+  beforeAll(async () => {
+    await config.beforeAll()
+  })
+
+  afterAll(async () => {
+    await config.afterAll()
+  })
+
+  it("creates, activates and deletes online license - self host", async () => {
+    // Remove existing license key
+    await config.internalApi.license.deleteLicenseKey()
+
+    // Verify license key not found
+    await config.internalApi.license.getLicenseKey({ status: 404 })
+
+    // Create self host account
+    const createAccountRequest = fixures.accounts.generateAccount({
+      hosting: Hosting.SELF,
+    })
+    const [createAccountRes, account] =
+      await config.accountsApi.accounts.create(createAccountRequest, {
+        autoVerify: true,
+      })
+
+    let licenseKey: string = " "
+    await config.doInNewState(async () => {
+      await config.loginAsAccount(createAccountRequest)
+      // Retrieve license key
+      const [res, body] = await config.accountsApi.licenses.getLicenseKey()
+      licenseKey = body.licenseKey
+    })
+
+    const accountId = account.accountId!
+
+    // Update license to have paid feature
+    const [res, acc] = await config.accountsApi.licenses.updateLicense(
+      accountId,
+      {
+        overrides: {
+          features: [Feature.APP_BACKUPS],
+        },
+      }
+    )
+
+    // Activate license key
+    await config.internalApi.license.activateLicenseKey({ licenseKey })
+
+    // Verify license updated with new feature
+    await config.doInNewState(async () => {
+      await config.loginAsAccount(createAccountRequest)
+      const [selfRes, body] = await config.api.accounts.self()
+      expect(body.license.features[0]).toBe("appBackups")
+    })
+
+    // Remove license key
+    await config.internalApi.license.deleteLicenseKey()
+
+    // Verify license key not found
+    await config.internalApi.license.getLicenseKey({ status: 404 })
+  })
+})

qa-core/src/internal-api/api/apis/LicenseAPI.ts

@@ -1,17 +1,19 @@
 import { Response } from "node-fetch"
 import {
+  ActivateLicenseKeyRequest,
   ActivateOfflineLicenseTokenRequest,
+  GetLicenseKeyResponse,
   GetOfflineIdentifierResponse,
   GetOfflineLicenseTokenResponse,
 } from "@budibase/types"
 import BudibaseInternalAPIClient from "../BudibaseInternalAPIClient"
 import BaseAPI from "./BaseAPI"
+import { APIRequestOpts } from "../../../types"
 
 export default class LicenseAPI extends BaseAPI {
   constructor(client: BudibaseInternalAPIClient) {
     super(client)
   }
-
   async getOfflineLicenseToken(
     opts: { status?: number } = {}
   ): Promise<[Response, GetOfflineLicenseTokenResponse]> {
@@ -21,19 +23,16 @@ export default class LicenseAPI extends BaseAPI {
     )
     return [response, body]
   }
-
   async deleteOfflineLicenseToken(): Promise<[Response]> {
     const [response] = await this.del(`/global/license/offline`, 204)
     return [response]
   }
-
   async activateOfflineLicenseToken(
     body: ActivateOfflineLicenseTokenRequest
   ): Promise<[Response]> {
     const [response] = await this.post(`/global/license/offline`, body)
     return [response]
   }
-
   async getOfflineIdentifier(): Promise<
     [Response, GetOfflineIdentifierResponse]
   > {
@@ -42,4 +41,23 @@ export default class LicenseAPI extends BaseAPI {
     )
     return [response, body]
   }
+
+  async getLicenseKey(
+    opts: { status?: number } = {}
+  ): Promise<[Response, GetLicenseKeyResponse]> {
+    const [response, body] = await this.get(`/global/license/key`, opts.status)
+    return [response, body]
+  }
+
+  async activateLicenseKey(
+    body: ActivateLicenseKeyRequest
+  ): Promise<[Response]> {
+    const [response] = await this.post(`/global/license/key`, body)
+    return [response]
+  }
+
+  async deleteLicenseKey(): Promise<[Response]> {
+    const [response] = await this.del(`/global/license/key`, 204)
+    return [response]
+  }
 }

scripts/build-single-image.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 yarn build --scope @budibase/server --scope @budibase/worker
-docker build -f hosting/single/Dockerfile.v2 -t budibase:latest .
+version=$(./scripts/getCurrentVersion.sh)
+docker build -f hosting/single/Dockerfile.v2 -t budibase:latest --build-arg BUDIBASE_VERSION=$version .