Merge branch 'develop' into api-tests-generate-tenants

.github/ISSUE_TEMPLATE/bug_report.md

@@ -6,6 +6,8 @@ labels: bug
 assignees: ''
 
 ---
+## Checklist
+- [ ] I have searched budibase discussions and github issues to check if my issue already exists
 
 **Hosting**
 <!-- Delete as appropriate -->

.github/workflows/release-selfhost.yml

@@ -67,16 +67,24 @@ jobs:
         uses: azure/setup-helm@v1
         id: helm-install
 
+      # due to helm repo index issue: https://github.com/helm/helm/issues/7363
+      # we need to create new package in a different dir, merge the index and move the package back
       - name: Build and release helm chart
         run: | 
           git config user.name "Budibase Helm Bot"
           git config user.email "<>"
           git reset --hard
           git pull
-          helm package charts/budibase --version "$RELEASE_VERSION" --app-version "$RELEASE_VERSION"
+          mkdir sync
+          echo "Packaging chart to sync dir"
+          helm package charts/budibase --version "$RELEASE_VERSION" --app-version "$RELEASE_VERSION" --destination sync
+          echo "Packaging successful"
           git checkout gh-pages
-          mv *.tgz docs
+          echo "Indexing helm repo"
-          helm repo index docs
+          helm repo index --merge docs/index.yaml sync
+          mv -f sync/* docs
+          rm -rf sync
+          echo "Pushing new helm release"
           git add -A
           git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}"
           git push

.vscode/settings.json

@@ -17,6 +17,6 @@
         ]
     },
     "[typescript]": {
-        "editor.defaultFormatter": "vscode.typescript-language-features"
+        "editor.defaultFormatter": "esbenp.prettier-vscode"
     },
 }

charts/budibase/Chart.yaml

@@ -17,7 +17,7 @@ version: 0.0.0
 appVersion: 0.0.0
 dependencies:
   - name: couchdb
-    version: 3.6.1
+    version: 3.3.4
     repository: https://apache.github.io/couchdb-helm
     condition: services.couchdb.enabled
   - name: ingress-nginx

charts/budibase/templates/app-service-deployment.yaml

@@ -4,9 +4,6 @@ metadata:
   annotations:
     kompose.cmd: kompose convert
     kompose.version: 1.21.0 (992df58d8)
-{{ if .Values.globals.logAnnotations }}
-{{ toYaml .Values.globals.logAnnotations | indent 4 }}
-{{ end }}
   creationTimestamp: null
   labels:
     io.kompose.service: app-service
@@ -23,6 +20,9 @@ spec:
       annotations:
         kompose.cmd: kompose convert
         kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.apps.annotations }}
+{{- toYaml .Values.services.apps.annotations | indent 8 -}}
+{{ end }}
       creationTimestamp: null
       labels:
         io.kompose.service: app-service

charts/budibase/templates/proxy-service-deployment.yaml

@@ -4,9 +4,6 @@ metadata:
   annotations:
     kompose.cmd: kompose convert
     kompose.version: 1.21.0 (992df58d8)
-{{ if .Values.globals.logAnnotations }}
-{{ toYaml .Values.globals.logAnnotations | indent 4 }}
-{{ end }}
   creationTimestamp: null
   labels:
     app.kubernetes.io/name: budibase-proxy
@@ -23,6 +20,9 @@ spec:
       annotations:
         kompose.cmd: kompose convert
         kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.proxy.annotations }}
+{{- toYaml .Values.services.proxy.annotations | indent 8 -}}
+{{ end }}
       creationTimestamp: null
       labels:
         app.kubernetes.io/name: budibase-proxy

charts/budibase/templates/worker-service-deployment.yaml

@@ -4,9 +4,6 @@ metadata:
   annotations:
     kompose.cmd: kompose convert
     kompose.version: 1.21.0 (992df58d8)
-{{ if .Values.globals.logAnnotations }}
-{{ toYaml .Values.globals.logAnnotations | indent 4 }}
-{{ end }}
   creationTimestamp: null
   labels:
     io.kompose.service: worker-service
@@ -24,6 +21,9 @@ spec:
       annotations:
         kompose.cmd: kompose convert
         kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.worker.annotations }}
+{{- toYaml .Values.services.worker.annotations | indent 8 -}}
+{{ end }}
       creationTimestamp: null
       labels:
         io.kompose.service: worker-service

charts/budibase/values.yaml

@@ -22,12 +22,6 @@ serviceAccount:
 
 podAnnotations: {}
 
-# logAnnotations:
-#   co.elastic.logs/multiline.type: pattern
-#   co.elastic.logs/multiline.pattern: '^[[:space:]]'
-#   co.elastic.logs/multiline.negate: false
-#   co.elastic.logs/multiline.match: after
-
 podSecurityContext:
   {}
   # fsGroup: 2000
@@ -130,6 +124,10 @@ services:
       minio: 'http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}'
       couchdb: 'http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}'
     resources: {}
+#    annotations:
+#      co.elastic.logs/module: nginx
+#      co.elastic.logs/fileset.stdout: access
+#      co.elastic.logs/fileset.stderr: error
 
   apps:
     port: 4002
@@ -137,11 +135,20 @@ services:
     logLevel: info
     resources: {}
 #    nodeDebug: "" # set the value of NODE_DEBUG
-
+#    annotations:
+#      co.elastic.logs/multiline.type: pattern
+#      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+#      co.elastic.logs/multiline.negate: false
+#      co.elastic.logs/multiline.match: after
   worker:
     port: 4003
     replicaCount: 1
     resources: {}
+#    annotations:
+#      co.elastic.logs/multiline.type: pattern
+#      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+#      co.elastic.logs/multiline.negate: false
+#      co.elastic.logs/multiline.match: after
 
   couchdb:
     enabled: true

hosting/single/runner.sh

@@ -78,7 +78,7 @@ mkdir -p ${DATA_DIR}/search
 chown -R couchdb:couchdb ${DATA_DIR}/couch
 redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
 /opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
-/minio/minio server ${DATA_DIR}/minio > /dev/stdout 2>&1 &
+/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
 /docker-entrypoint.sh /opt/couchdb/bin/couchdb &
 /etc/init.d/nginx restart
 if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.2.10-alpha.12",
+  "version": "2.2.12-alpha.21",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -45,8 +45,8 @@
     "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
     "test": "lerna run test && yarn test:pro",
     "test:pro": "bash scripts/pro/test.sh",
-    "lint:eslint": "eslint packages",
+    "lint:eslint": "eslint packages && eslint qa-core",
-    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",
+    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
     "lint": "yarn run lint:eslint && yarn run lint:prettier",
     "lint:fix:eslint": "eslint --fix packages qa-core",
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
@@ -84,4 +84,4 @@
     "install:pro": "bash scripts/pro/install.sh",
     "dep:clean": "yarn clean && yarn bootstrap"
   }
-}
+}

packages/backend-core/jest.config.ts

@@ -6,6 +6,9 @@ const config: Config.InitialOptions = {
   setupFiles: ["./tests/jestSetup.ts"],
   collectCoverageFrom: ["src/**/*.{js,ts}"],
   coverageReporters: ["lcov", "json", "clover"],
+  transform: {
+    "^.+\\.ts?$": "@swc/jest",
+  },
 }
 
 if (!process.env.CI) {

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.2.10-alpha.12",
+  "version": "2.2.12-alpha.21",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -23,7 +23,7 @@
   },
   "dependencies": {
     "@budibase/nano": "10.1.1",
-    "@budibase/types": "2.2.10-alpha.12",
+    "@budibase/types": "2.2.12-alpha.21",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
@@ -31,6 +31,7 @@
     "bcrypt": "5.0.1",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
+    "correlation-id": "4.0.0",
     "dotenv": "16.0.1",
     "emitter-listener": "1.1.2",
     "ioredis": "4.28.0",
@@ -56,19 +57,23 @@
     "zlib": "1.0.5"
   },
   "devDependencies": {
+    "@swc/core": "^1.3.25",
+    "@swc/jest": "^0.2.24",
     "@types/chance": "1.1.3",
     "@types/ioredis": "4.28.0",
     "@types/jest": "27.5.1",
     "@types/koa": "2.13.4",
+    "@types/koa-pino-logger": "3.0.0",
     "@types/lodash": "4.14.180",
     "@types/node": "14.18.20",
     "@types/node-fetch": "2.6.1",
+    "@types/pino-http": "5.8.1",
     "@types/pouchdb": "6.4.0",
     "@types/redlock": "4.0.3",
     "@types/semver": "7.3.7",
     "@types/tar-fs": "2.0.1",
     "@types/uuid": "8.3.4",
-    "chance": "1.1.3",
+    "chance": "1.1.8",
     "ioredis-mock": "5.8.0",
     "jest": "28.1.1",
     "koa": "2.13.4",

packages/backend-core/src/cache/base/index.ts

@@ -7,7 +7,7 @@ function generateTenantKey(key: string) {
   return `${key}:${tenantId}`
 }
 
-export = class BaseCache {
+export default class BaseCache {
   client: Client | undefined
 
   constructor(client: Client | undefined = undefined) {

packages/backend-core/src/cache/generic.ts

@@ -1,6 +1,6 @@
 const BaseCache = require("./base")
 
-const GENERIC = new BaseCache()
+const GENERIC = new BaseCache.default()
 
 export enum CacheKey {
   CHECKLIST = "checklist",

packages/backend-core/src/cloud/api.ts

@@ -1,6 +1,7 @@
 import fetch from "node-fetch"
+import * as logging from "../logging"
 
-export = class API {
+export default class API {
   host: string
 
   constructor(host: string) {
@@ -22,6 +23,9 @@ export = class API {
 
     let json = options.headers["Content-Type"] === "application/json"
 
+    // add x-budibase-correlation-id header
+    logging.correlation.setHeader(options.headers)
+
     const requestOptions = {
       method: method,
       body: json ? JSON.stringify(options.body) : options.body,

packages/backend-core/src/constants/misc.ts

@@ -22,6 +22,7 @@ export enum Header {
   TENANT_ID = "x-budibase-tenant-id",
   TOKEN = "x-budibase-token",
   CSRF_TOKEN = "x-csrf-token",
+  CORRELATION_ID = "x-budibase-correlation-id",
 }
 
 export enum GlobalRole {

packages/backend-core/src/context/tests/index.spec.ts

@@ -1,7 +1,7 @@
 require("../../../tests")
 const context = require("../")
 const { DEFAULT_TENANT_ID } = require("../../constants")
-const env = require("../../environment")
+import env from "../../environment"
 
 describe("context", () => {
   describe("doInTenant", () => {
@@ -26,7 +26,7 @@ describe("context", () => {
 
       it("fails when no tenant id is set", () => {
         const test = () => {
-          let error
+          let error: any
           try {
             context.getTenantId()
           } catch (e) {
@@ -45,7 +45,7 @@ describe("context", () => {
 
       it("fails when no tenant db is set", () => {
         const test = () => {
-          let error
+          let error: any
           try {
             context.getGlobalDB()
           } catch (e) {

packages/backend-core/src/db/tests/utils.spec.ts

@@ -5,18 +5,13 @@ const {
   isDevAppID,
   isProdAppID,
 } = require("../conversions")
-const {
+const { generateAppID, getPlatformUrl, getScopedConfig } = require("../utils")
-  generateAppID,
-  getPlatformUrl,
-  getScopedConfig
-} = require("../utils")
 const tenancy = require("../../tenancy")
 const { Config, DEFAULT_TENANT_ID } = require("../../constants")
-const env = require("../../environment")
+import env from "../../environment"
 
 describe("utils", () => {
   describe("app ID manipulation", () => {
-
     function getID() {
       const appId = generateAppID()
       const split = appId.split("_")
@@ -28,42 +23,42 @@ describe("utils", () => {
     it("should be able to generate a new app ID", () => {
       expect(generateAppID().startsWith("app_")).toEqual(true)
     })
-  
+
     it("should be able to convert a production app ID to development", () => {
       const { appId, uuid } = getID()
       expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
     })
-  
+
     it("should be able to convert a development app ID to development", () => {
       const { devAppId, uuid } = getID()
       expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
     })
-  
+
     it("should be able to convert a development ID to a production", () => {
       const { devAppId, uuid } = getID()
       expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
     })
-  
+
     it("should be able to convert a production ID to production", () => {
       const { appId, uuid } = getID()
       expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
     })
-  
+
     it("should be able to confirm dev app ID is development", () => {
       const { devAppId } = getID()
       expect(isDevAppID(devAppId)).toEqual(true)
     })
-  
+
     it("should be able to confirm prod app ID is not development", () => {
       const { appId } = getID()
       expect(isDevAppID(appId)).toEqual(false)
     })
-  
+
     it("should be able to confirm prod app ID is prod", () => {
       const { appId } = getID()
       expect(isProdAppID(appId)).toEqual(true)
     })
-  
+
     it("should be able to confirm dev app ID is not prod", () => {
       const { devAppId } = getID()
       expect(isProdAppID(devAppId)).toEqual(false)
@@ -81,8 +76,8 @@ const setDbPlatformUrl = async () => {
     _id: "config_settings",
     type: Config.SETTINGS,
     config: {
-      platformUrl: DB_URL
+      platformUrl: DB_URL,
-    }
+    },
   })
 }
 
@@ -92,17 +87,16 @@ const clearSettingsConfig = async () => {
     try {
       const config = await db.get("config_settings")
       await db.remove("config_settings", config._rev)
-    } catch (e) {
+    } catch (e: any) {
       if (e.status !== 404) {
         throw e
       }
     }
   })
 }
- 
+
 describe("getPlatformUrl", () => {
   describe("self host", () => {
-
     beforeEach(async () => {
       env._set("SELF_HOST", 1)
       await clearSettingsConfig()
@@ -129,10 +123,9 @@ describe("getPlatformUrl", () => {
         const url = await getPlatformUrl()
         expect(url).toBe(DB_URL)
       })
-    })    
+    })
   })
 
-
   describe("cloud", () => {
     const TENANT_AWARE_URL = "http://default.env.com"
 
@@ -163,13 +156,12 @@ describe("getPlatformUrl", () => {
         const url = await getPlatformUrl()
         expect(url).toBe(TENANT_AWARE_URL)
       })
-    })    
+    })
   })
 })
 
 describe("getScopedConfig", () => {
   describe("settings config", () => {
-
     beforeEach(async () => {
       env._set("SELF_HOSTED", 1)
       env._set("PLATFORM_URL", "")

packages/backend-core/src/environment.ts

@@ -102,4 +102,4 @@ for (let [key, value] of Object.entries(environment)) {
   }
 }
 
-export = environment
+export default environment

packages/backend-core/src/errors/errors.ts

@@ -0,0 +1,37 @@
+import * as licensing from "./licensing"
+
+// combine all error codes into single object
+
+export const codes = {
+  ...licensing.codes,
+}
+
+// combine all error types
+export const types = [licensing.type]
+
+// combine all error contexts
+const context = {
+  ...licensing.context,
+}
+
+// derive a public error message using codes, types and any custom contexts
+export const getPublicError = (err: any) => {
+  let error
+  if (err.code || err.type) {
+    // add generic error information
+    error = {
+      code: err.code,
+      type: err.type,
+    }
+
+    if (err.code && context[err.code]) {
+      error = {
+        ...error,
+        // get any additional context from this error
+        ...context[err.code](err),
+      }
+    }
+  }
+
+  return error
+}

packages/backend-core/src/errors/index.ts

@@ -1,47 +1,3 @@
-import { HTTPError } from "./http"
+export * from "./errors"
-import { UsageLimitError, FeatureDisabledError } from "./licensing"
+export { UsageLimitError, FeatureDisabledError } from "./licensing"
-import * as licensing from "./licensing"
+export { HTTPError } from "./http"
-
-const codes = {
-  ...licensing.codes,
-}
-
-const types = [licensing.type]
-
-const context = {
-  ...licensing.context,
-}
-
-const getPublicError = (err: any) => {
-  let error
-  if (err.code || err.type) {
-    // add generic error information
-    error = {
-      code: err.code,
-      type: err.type,
-    }
-
-    if (err.code && context[err.code]) {
-      error = {
-        ...error,
-        // get any additional context from this error
-        ...context[err.code](err),
-      }
-    }
-  }
-
-  return error
-}
-
-const pkg = {
-  codes,
-  types,
-  errors: {
-    UsageLimitError,
-    FeatureDisabledError,
-    HTTPError,
-  },
-  getPublicError,
-}
-
-export = pkg

packages/backend-core/src/events/events.ts

@@ -1,6 +1,6 @@
 import { Event } from "@budibase/types"
 import { processors } from "./processors"
-import * as identification from "./identification"
+import identification from "./identification"
 import * as backfill from "./backfill"
 
 export const publishEvent = async (

packages/backend-core/src/events/identification.ts

@@ -33,7 +33,7 @@ const pkg = require("../../package.json")
  * - tenant
  * - installation
  */
-export const getCurrentIdentity = async (): Promise<Identity> => {
+const getCurrentIdentity = async (): Promise<Identity> => {
   let identityContext = identityCtx.getIdentity()
   const environment = getDeploymentEnvironment()
 
@@ -94,7 +94,7 @@ export const getCurrentIdentity = async (): Promise<Identity> => {
   }
 }
 
-export const identifyInstallationGroup = async (
+const identifyInstallationGroup = async (
   installId: string,
   timestamp?: string | number
 ): Promise<void> => {
@@ -118,7 +118,7 @@ export const identifyInstallationGroup = async (
   await identify({ ...group, id: `$${type}_${id}` }, timestamp)
 }
 
-export const identifyTenantGroup = async (
+const identifyTenantGroup = async (
   tenantId: string,
   account: Account | undefined,
   timestamp?: string | number
@@ -156,7 +156,7 @@ export const identifyTenantGroup = async (
   await identify({ ...group, id: `$${type}_${id}` }, timestamp)
 }
 
-export const identifyUser = async (
+const identifyUser = async (
   user: User,
   account: CloudAccount | undefined,
   timestamp?: string | number
@@ -191,7 +191,7 @@ export const identifyUser = async (
   await identify(identity, timestamp)
 }
 
-export const identifyAccount = async (account: Account) => {
+const identifyAccount = async (account: Account) => {
   let id = account.accountId
   const tenantId = account.tenantId
   let type = IdentityType.USER
@@ -224,17 +224,11 @@ export const identifyAccount = async (account: Account) => {
   await identify(identity)
 }
 
-export const identify = async (
+const identify = async (identity: Identity, timestamp?: string | number) => {
-  identity: Identity,
-  timestamp?: string | number
-) => {
   await processors.identify(identity, timestamp)
 }
 
-export const identifyGroup = async (
+const identifyGroup = async (group: Group, timestamp?: string | number) => {
-  group: Group,
-  timestamp?: string | number
-) => {
   await processors.identifyGroup(group, timestamp)
 }
 
@@ -250,7 +244,7 @@ const getHostingFromEnv = () => {
   return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD
 }
 
-export const getInstallationId = async () => {
+const getInstallationId = async () => {
   if (isAccountPortal()) {
     return "account-portal"
   }
@@ -300,3 +294,14 @@ const formatDistinctId = (id: string, type: IdentityType) => {
     return id
   }
 }
+
+export default {
+  getCurrentIdentity,
+  identifyInstallationGroup,
+  identifyTenantGroup,
+  identifyUser,
+  identifyAccount,
+  identify,
+  identifyGroup,
+  getInstallationId,
+}

packages/backend-core/src/events/index.ts

@@ -1,7 +1,7 @@
 export * from "./publishers"
 export * as processors from "./processors"
 export * as analytics from "./analytics"
-export * as identification from "./identification"
+export { default as identification } from "./identification"
 export * as backfillCache from "./backfill"
 
 import { processors } from "./processors"

packages/backend-core/src/events/processors/LoggingProcessor.ts

@@ -23,7 +23,7 @@ export default class LoggingProcessor implements EventProcessor {
       return
     }
     let timestampString = getTimestampString(timestamp)
-    let message = `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${timestampString} ${event} `
+    let message = `[audit] [identityType=${identity.type}] ${timestampString} ${event} `
     if (env.isDev()) {
       message = message + `[debug: [properties=${JSON.stringify(properties)}] ]`
     }

packages/backend-core/src/events/publishers/account.ts

@@ -7,23 +7,29 @@ import {
   AccountVerifiedEvent,
 } from "@budibase/types"
 
-export async function created(account: Account) {
+async function created(account: Account) {
   const properties: AccountCreatedEvent = {
     tenantId: account.tenantId,
   }
   await publishEvent(Event.ACCOUNT_CREATED, properties)
 }
 
-export async function deleted(account: Account) {
+async function deleted(account: Account) {
   const properties: AccountDeletedEvent = {
     tenantId: account.tenantId,
   }
   await publishEvent(Event.ACCOUNT_DELETED, properties)
 }
 
-export async function verified(account: Account) {
+async function verified(account: Account) {
   const properties: AccountVerifiedEvent = {
     tenantId: account.tenantId,
   }
   await publishEvent(Event.ACCOUNT_VERIFIED, properties)
 }
+
+export default {
+  created,
+  deleted,
+  verified,
+}

packages/backend-core/src/events/publishers/app.ts

@@ -15,7 +15,7 @@ import {
   AppExportedEvent,
 } from "@budibase/types"
 
-export const created = async (app: App, timestamp?: string | number) => {
+const created = async (app: App, timestamp?: string | number) => {
   const properties: AppCreatedEvent = {
     appId: app.appId,
     version: app.version,
@@ -23,7 +23,7 @@ export const created = async (app: App, timestamp?: string | number) => {
   await publishEvent(Event.APP_CREATED, properties, timestamp)
 }
 
-export async function updated(app: App) {
+async function updated(app: App) {
   const properties: AppUpdatedEvent = {
     appId: app.appId,
     version: app.version,
@@ -31,35 +31,35 @@ export async function updated(app: App) {
   await publishEvent(Event.APP_UPDATED, properties)
 }
 
-export async function deleted(app: App) {
+async function deleted(app: App) {
   const properties: AppDeletedEvent = {
     appId: app.appId,
   }
   await publishEvent(Event.APP_DELETED, properties)
 }
 
-export async function published(app: App, timestamp?: string | number) {
+async function published(app: App, timestamp?: string | number) {
   const properties: AppPublishedEvent = {
     appId: app.appId,
   }
   await publishEvent(Event.APP_PUBLISHED, properties, timestamp)
 }
 
-export async function unpublished(app: App) {
+async function unpublished(app: App) {
   const properties: AppUnpublishedEvent = {
     appId: app.appId,
   }
   await publishEvent(Event.APP_UNPUBLISHED, properties)
 }
 
-export async function fileImported(app: App) {
+async function fileImported(app: App) {
   const properties: AppFileImportedEvent = {
     appId: app.appId,
   }
   await publishEvent(Event.APP_FILE_IMPORTED, properties)
 }
 
-export async function templateImported(app: App, templateKey: string) {
+async function templateImported(app: App, templateKey: string) {
   const properties: AppTemplateImportedEvent = {
     appId: app.appId,
     templateKey,
@@ -67,7 +67,7 @@ export async function templateImported(app: App, templateKey: string) {
   await publishEvent(Event.APP_TEMPLATE_IMPORTED, properties)
 }
 
-export async function versionUpdated(
+async function versionUpdated(
   app: App,
   currentVersion: string,
   updatedToVersion: string
@@ -80,7 +80,7 @@ export async function versionUpdated(
   await publishEvent(Event.APP_VERSION_UPDATED, properties)
 }
 
-export async function versionReverted(
+async function versionReverted(
   app: App,
   currentVersion: string,
   revertedToVersion: string
@@ -93,16 +93,30 @@ export async function versionReverted(
   await publishEvent(Event.APP_VERSION_REVERTED, properties)
 }
 
-export async function reverted(app: App) {
+async function reverted(app: App) {
   const properties: AppRevertedEvent = {
     appId: app.appId,
   }
   await publishEvent(Event.APP_REVERTED, properties)
 }
 
-export async function exported(app: App) {
+async function exported(app: App) {
   const properties: AppExportedEvent = {
     appId: app.appId,
   }
   await publishEvent(Event.APP_EXPORTED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  published,
+  unpublished,
+  fileImported,
+  templateImported,
+  versionUpdated,
+  versionReverted,
+  reverted,
+  exported,
+}

packages/backend-core/src/events/publishers/auth.ts

@@ -12,7 +12,7 @@ import {
 } from "@budibase/types"
 import { identification } from ".."
 
-export async function login(source: LoginSource) {
+async function login(source: LoginSource) {
   const identity = await identification.getCurrentIdentity()
   const properties: LoginEvent = {
     userId: identity.id,
@@ -21,7 +21,7 @@ export async function login(source: LoginSource) {
   await publishEvent(Event.AUTH_LOGIN, properties)
 }
 
-export async function logout() {
+async function logout() {
   const identity = await identification.getCurrentIdentity()
   const properties: LogoutEvent = {
     userId: identity.id,
@@ -29,30 +29,39 @@ export async function logout() {
   await publishEvent(Event.AUTH_LOGOUT, properties)
 }
 
-export async function SSOCreated(type: SSOType, timestamp?: string | number) {
+async function SSOCreated(type: SSOType, timestamp?: string | number) {
   const properties: SSOCreatedEvent = {
     type,
   }
   await publishEvent(Event.AUTH_SSO_CREATED, properties, timestamp)
 }
 
-export async function SSOUpdated(type: SSOType) {
+async function SSOUpdated(type: SSOType) {
   const properties: SSOUpdatedEvent = {
     type,
   }
   await publishEvent(Event.AUTH_SSO_UPDATED, properties)
 }
 
-export async function SSOActivated(type: SSOType, timestamp?: string | number) {
+async function SSOActivated(type: SSOType, timestamp?: string | number) {
   const properties: SSOActivatedEvent = {
     type,
   }
   await publishEvent(Event.AUTH_SSO_ACTIVATED, properties, timestamp)
 }
 
-export async function SSODeactivated(type: SSOType) {
+async function SSODeactivated(type: SSOType) {
   const properties: SSODeactivatedEvent = {
     type,
   }
   await publishEvent(Event.AUTH_SSO_DEACTIVATED, properties)
 }
+
+export default {
+  login,
+  logout,
+  SSOCreated,
+  SSOUpdated,
+  SSOActivated,
+  SSODeactivated,
+}

packages/backend-core/src/events/publishers/automation.ts

@@ -12,10 +12,7 @@ import {
   AutomationsRunEvent,
 } from "@budibase/types"
 
-export async function created(
+async function created(automation: Automation, timestamp?: string | number) {
-  automation: Automation,
-  timestamp?: string | number
-) {
   const properties: AutomationCreatedEvent = {
     appId: automation.appId,
     automationId: automation._id as string,
@@ -25,7 +22,7 @@ export async function created(
   await publishEvent(Event.AUTOMATION_CREATED, properties, timestamp)
 }
 
-export async function triggerUpdated(automation: Automation) {
+async function triggerUpdated(automation: Automation) {
   const properties: AutomationTriggerUpdatedEvent = {
     appId: automation.appId,
     automationId: automation._id as string,
@@ -35,7 +32,7 @@ export async function triggerUpdated(automation: Automation) {
   await publishEvent(Event.AUTOMATION_TRIGGER_UPDATED, properties)
 }
 
-export async function deleted(automation: Automation) {
+async function deleted(automation: Automation) {
   const properties: AutomationDeletedEvent = {
     appId: automation.appId,
     automationId: automation._id as string,
@@ -45,7 +42,7 @@ export async function deleted(automation: Automation) {
   await publishEvent(Event.AUTOMATION_DELETED, properties)
 }
 
-export async function tested(automation: Automation) {
+async function tested(automation: Automation) {
   const properties: AutomationTestedEvent = {
     appId: automation.appId,
     automationId: automation._id as string,
@@ -55,14 +52,14 @@ export async function tested(automation: Automation) {
   await publishEvent(Event.AUTOMATION_TESTED, properties)
 }
 
-export const run = async (count: number, timestamp?: string | number) => {
+const run = async (count: number, timestamp?: string | number) => {
   const properties: AutomationsRunEvent = {
     count,
   }
   await publishEvent(Event.AUTOMATIONS_RUN, properties, timestamp)
 }
 
-export async function stepCreated(
+async function stepCreated(
   automation: Automation,
   step: AutomationStep,
   timestamp?: string | number
@@ -78,10 +75,7 @@ export async function stepCreated(
   await publishEvent(Event.AUTOMATION_STEP_CREATED, properties, timestamp)
 }
 
-export async function stepDeleted(
+async function stepDeleted(automation: Automation, step: AutomationStep) {
-  automation: Automation,
-  step: AutomationStep
-) {
   const properties: AutomationStepDeletedEvent = {
     appId: automation.appId,
     automationId: automation._id as string,
@@ -92,3 +86,13 @@ export async function stepDeleted(
   }
   await publishEvent(Event.AUTOMATION_STEP_DELETED, properties)
 }
+
+export default {
+  created,
+  triggerUpdated,
+  deleted,
+  tested,
+  run,
+  stepCreated,
+  stepDeleted,
+}

packages/backend-core/src/events/publishers/backfill.ts

@@ -8,18 +8,18 @@ import {
   InstallationBackfillSucceededEvent,
   InstallationBackfillFailedEvent,
 } from "@budibase/types"
-const env = require("../../environment")
+import env from "../../environment"
 
 const shouldSkip = !env.SELF_HOSTED && !env.isDev()
 
-export async function appSucceeded(properties: AppBackfillSucceededEvent) {
+async function appSucceeded(properties: AppBackfillSucceededEvent) {
   if (shouldSkip) {
     return
   }
   await publishEvent(Event.APP_BACKFILL_SUCCEEDED, properties)
 }
 
-export async function appFailed(error: any) {
+async function appFailed(error: any) {
   if (shouldSkip) {
     return
   }
@@ -29,16 +29,14 @@ export async function appFailed(error: any) {
   await publishEvent(Event.APP_BACKFILL_FAILED, properties)
 }
 
-export async function tenantSucceeded(
+async function tenantSucceeded(properties: TenantBackfillSucceededEvent) {
-  properties: TenantBackfillSucceededEvent
-) {
   if (shouldSkip) {
     return
   }
   await publishEvent(Event.TENANT_BACKFILL_SUCCEEDED, properties)
 }
 
-export async function tenantFailed(error: any) {
+async function tenantFailed(error: any) {
   if (shouldSkip) {
     return
   }
@@ -48,7 +46,7 @@ export async function tenantFailed(error: any) {
   await publishEvent(Event.TENANT_BACKFILL_FAILED, properties)
 }
 
-export async function installationSucceeded() {
+async function installationSucceeded() {
   if (shouldSkip) {
     return
   }
@@ -56,7 +54,7 @@ export async function installationSucceeded() {
   await publishEvent(Event.INSTALLATION_BACKFILL_SUCCEEDED, properties)
 }
 
-export async function installationFailed(error: any) {
+async function installationFailed(error: any) {
   if (shouldSkip) {
     return
   }
@@ -65,3 +63,12 @@ export async function installationFailed(error: any) {
   }
   await publishEvent(Event.INSTALLATION_BACKFILL_FAILED, properties)
 }
+
+export default {
+  appSucceeded,
+  appFailed,
+  tenantSucceeded,
+  tenantFailed,
+  installationSucceeded,
+  installationFailed,
+}

packages/backend-core/src/events/publishers/backup.ts

@@ -8,7 +8,7 @@ import {
 } from "@budibase/types"
 import { publishEvent } from "../events"
 
-export async function appBackupRestored(backup: AppBackup) {
+async function appBackupRestored(backup: AppBackup) {
   const properties: AppBackupRestoreEvent = {
     appId: backup.appId,
     restoreId: backup._id!,
@@ -18,7 +18,7 @@ export async function appBackupRestored(backup: AppBackup) {
   await publishEvent(Event.APP_BACKUP_RESTORED, properties)
 }
 
-export async function appBackupTriggered(
+async function appBackupTriggered(
   appId: string,
   backupId: string,
   type: AppBackupType,
@@ -32,3 +32,8 @@ export async function appBackupTriggered(
   }
   await publishEvent(Event.APP_BACKUP_TRIGGERED, properties)
 }
+
+export default {
+  appBackupRestored,
+  appBackupTriggered,
+}

packages/backend-core/src/events/publishers/datasource.ts

@@ -14,10 +14,7 @@ function isCustom(datasource: Datasource) {
   return !sources.includes(datasource.source)
 }
 
-export async function created(
+async function created(datasource: Datasource, timestamp?: string | number) {
-  datasource: Datasource,
-  timestamp?: string | number
-) {
   const properties: DatasourceCreatedEvent = {
     datasourceId: datasource._id as string,
     source: datasource.source,
@@ -26,7 +23,7 @@ export async function created(
   await publishEvent(Event.DATASOURCE_CREATED, properties, timestamp)
 }
 
-export async function updated(datasource: Datasource) {
+async function updated(datasource: Datasource) {
   const properties: DatasourceUpdatedEvent = {
     datasourceId: datasource._id as string,
     source: datasource.source,
@@ -35,7 +32,7 @@ export async function updated(datasource: Datasource) {
   await publishEvent(Event.DATASOURCE_UPDATED, properties)
 }
 
-export async function deleted(datasource: Datasource) {
+async function deleted(datasource: Datasource) {
   const properties: DatasourceDeletedEvent = {
     datasourceId: datasource._id as string,
     source: datasource.source,
@@ -43,3 +40,9 @@ export async function deleted(datasource: Datasource) {
   }
   await publishEvent(Event.DATASOURCE_DELETED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+}

packages/backend-core/src/events/publishers/email.ts

@@ -1,12 +1,17 @@
 import { publishEvent } from "../events"
 import { Event, SMTPCreatedEvent, SMTPUpdatedEvent } from "@budibase/types"
 
-export async function SMTPCreated(timestamp?: string | number) {
+async function SMTPCreated(timestamp?: string | number) {
   const properties: SMTPCreatedEvent = {}
   await publishEvent(Event.EMAIL_SMTP_CREATED, properties, timestamp)
 }
 
-export async function SMTPUpdated() {
+async function SMTPUpdated() {
   const properties: SMTPUpdatedEvent = {}
   await publishEvent(Event.EMAIL_SMTP_UPDATED, properties)
 }
+
+export default {
+  SMTPCreated,
+  SMTPUpdated,
+}

packages/backend-core/src/events/publishers/group.ts

@@ -11,28 +11,28 @@ import {
   UserGroupRoles,
 } from "@budibase/types"
 
-export async function created(group: UserGroup, timestamp?: number) {
+async function created(group: UserGroup, timestamp?: number) {
   const properties: GroupCreatedEvent = {
     groupId: group._id as string,
   }
   await publishEvent(Event.USER_GROUP_CREATED, properties, timestamp)
 }
 
-export async function updated(group: UserGroup) {
+async function updated(group: UserGroup) {
   const properties: GroupUpdatedEvent = {
     groupId: group._id as string,
   }
   await publishEvent(Event.USER_GROUP_UPDATED, properties)
 }
 
-export async function deleted(group: UserGroup) {
+async function deleted(group: UserGroup) {
   const properties: GroupDeletedEvent = {
     groupId: group._id as string,
   }
   await publishEvent(Event.USER_GROUP_DELETED, properties)
 }
 
-export async function usersAdded(count: number, group: UserGroup) {
+async function usersAdded(count: number, group: UserGroup) {
   const properties: GroupUsersAddedEvent = {
     count,
     groupId: group._id as string,
@@ -40,7 +40,7 @@ export async function usersAdded(count: number, group: UserGroup) {
   await publishEvent(Event.USER_GROUP_USERS_ADDED, properties)
 }
 
-export async function usersDeleted(count: number, group: UserGroup) {
+async function usersDeleted(count: number, group: UserGroup) {
   const properties: GroupUsersDeletedEvent = {
     count,
     groupId: group._id as string,
@@ -48,7 +48,7 @@ export async function usersDeleted(count: number, group: UserGroup) {
   await publishEvent(Event.USER_GROUP_USERS_REMOVED, properties)
 }
 
-export async function createdOnboarding(groupId: string) {
+async function createdOnboarding(groupId: string) {
   const properties: GroupAddedOnboardingEvent = {
     groupId: groupId,
     onboarding: true,
@@ -56,9 +56,19 @@ export async function createdOnboarding(groupId: string) {
   await publishEvent(Event.USER_GROUP_ONBOARDING, properties)
 }
 
-export async function permissionsEdited(roles: UserGroupRoles) {
+async function permissionsEdited(roles: UserGroupRoles) {
   const properties: UserGroupRoles = {
     ...roles,
   }
   await publishEvent(Event.USER_GROUP_PERMISSIONS_EDITED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  usersAdded,
+  usersDeleted,
+  createdOnboarding,
+  permissionsEdited,
+}

packages/backend-core/src/events/publishers/index.ts

@@ -1,22 +1,22 @@
-export * as account from "./account"
+export { default as account } from "./account"
-export * as app from "./app"
+export { default as app } from "./app"
-export * as auth from "./auth"
+export { default as auth } from "./auth"
-export * as automation from "./automation"
+export { default as automation } from "./automation"
-export * as datasource from "./datasource"
+export { default as datasource } from "./datasource"
-export * as email from "./email"
+export { default as email } from "./email"
-export * as license from "./license"
+export { default as license } from "./license"
-export * as layout from "./layout"
+export { default as layout } from "./layout"
-export * as org from "./org"
+export { default as org } from "./org"
-export * as query from "./query"
+export { default as query } from "./query"
-export * as role from "./role"
+export { default as role } from "./role"
-export * as screen from "./screen"
+export { default as screen } from "./screen"
-export * as rows from "./rows"
+export { default as rows } from "./rows"
-export * as table from "./table"
+export { default as table } from "./table"
-export * as serve from "./serve"
+export { default as serve } from "./serve"
-export * as user from "./user"
+export { default as user } from "./user"
-export * as view from "./view"
+export { default as view } from "./view"
-export * as installation from "./installation"
+export { default as installation } from "./installation"
-export * as backfill from "./backfill"
+export { default as backfill } from "./backfill"
-export * as group from "./group"
+export { default as group } from "./group"
-export * as plugin from "./plugin"
+export { default as plugin } from "./plugin"
-export * as backup from "./backup"
+export { default as backup } from "./backup"

packages/backend-core/src/events/publishers/installation.ts

@@ -1,14 +1,14 @@
 import { publishEvent } from "../events"
 import { Event, VersionCheckedEvent, VersionChangeEvent } from "@budibase/types"
 
-export async function versionChecked(version: string) {
+async function versionChecked(version: string) {
   const properties: VersionCheckedEvent = {
     currentVersion: version,
   }
   await publishEvent(Event.INSTALLATION_VERSION_CHECKED, properties)
 }
 
-export async function upgraded(from: string, to: string) {
+async function upgraded(from: string, to: string) {
   const properties: VersionChangeEvent = {
     from,
     to,
@@ -17,7 +17,7 @@ export async function upgraded(from: string, to: string) {
   await publishEvent(Event.INSTALLATION_VERSION_UPGRADED, properties)
 }
 
-export async function downgraded(from: string, to: string) {
+async function downgraded(from: string, to: string) {
   const properties: VersionChangeEvent = {
     from,
     to,
@@ -25,7 +25,14 @@ export async function downgraded(from: string, to: string) {
   await publishEvent(Event.INSTALLATION_VERSION_DOWNGRADED, properties)
 }
 
-export async function firstStartup() {
+async function firstStartup() {
   const properties = {}
   await publishEvent(Event.INSTALLATION_FIRST_STARTUP, properties)
 }
+
+export default {
+  versionChecked,
+  upgraded,
+  downgraded,
+  firstStartup,
+}

packages/backend-core/src/events/publishers/layout.ts

@@ -6,16 +6,21 @@ import {
   LayoutDeletedEvent,
 } from "@budibase/types"
 
-export async function created(layout: Layout, timestamp?: string | number) {
+async function created(layout: Layout, timestamp?: string | number) {
   const properties: LayoutCreatedEvent = {
     layoutId: layout._id as string,
   }
   await publishEvent(Event.LAYOUT_CREATED, properties, timestamp)
 }
 
-export async function deleted(layoutId: string) {
+async function deleted(layoutId: string) {
   const properties: LayoutDeletedEvent = {
     layoutId,
   }
   await publishEvent(Event.LAYOUT_DELETED, properties)
 }
+
+export default {
+  created,
+  deleted,
+}

packages/backend-core/src/events/publishers/license.ts

@@ -13,7 +13,7 @@ import {
   LicensePaymentRecoveredEvent,
 } from "@budibase/types"
 
-export async function tierChanged(account: Account, from: number, to: number) {
+async function tierChanged(account: Account, from: number, to: number) {
   const properties: LicenseTierChangedEvent = {
     accountId: account.accountId,
     to,
@@ -22,11 +22,7 @@ export async function tierChanged(account: Account, from: number, to: number) {
   await publishEvent(Event.LICENSE_TIER_CHANGED, properties)
 }
 
-export async function planChanged(
+async function planChanged(account: Account, from: PlanType, to: PlanType) {
-  account: Account,
-  from: PlanType,
-  to: PlanType
-) {
   const properties: LicensePlanChangedEvent = {
     accountId: account.accountId,
     to,
@@ -35,44 +31,55 @@ export async function planChanged(
   await publishEvent(Event.LICENSE_PLAN_CHANGED, properties)
 }
 
-export async function activated(account: Account) {
+async function activated(account: Account) {
   const properties: LicenseActivatedEvent = {
     accountId: account.accountId,
   }
   await publishEvent(Event.LICENSE_ACTIVATED, properties)
 }
 
-export async function checkoutOpened(account: Account) {
+async function checkoutOpened(account: Account) {
   const properties: LicenseCheckoutOpenedEvent = {
     accountId: account.accountId,
   }
   await publishEvent(Event.LICENSE_CHECKOUT_OPENED, properties)
 }
 
-export async function checkoutSuccess(account: Account) {
+async function checkoutSuccess(account: Account) {
   const properties: LicenseCheckoutSuccessEvent = {
     accountId: account.accountId,
   }
   await publishEvent(Event.LICENSE_CHECKOUT_SUCCESS, properties)
 }
 
-export async function portalOpened(account: Account) {
+async function portalOpened(account: Account) {
   const properties: LicensePortalOpenedEvent = {
     accountId: account.accountId,
   }
   await publishEvent(Event.LICENSE_PORTAL_OPENED, properties)
 }
 
-export async function paymentFailed(account: Account) {
+async function paymentFailed(account: Account) {
   const properties: LicensePaymentFailedEvent = {
     accountId: account.accountId,
   }
   await publishEvent(Event.LICENSE_PAYMENT_FAILED, properties)
 }
 
-export async function paymentRecovered(account: Account) {
+async function paymentRecovered(account: Account) {
   const properties: LicensePaymentRecoveredEvent = {
     accountId: account.accountId,
   }
   await publishEvent(Event.LICENSE_PAYMENT_RECOVERED, properties)
 }
+
+export default {
+  tierChanged,
+  planChanged,
+  activated,
+  checkoutOpened,
+  checkoutSuccess,
+  portalOpened,
+  paymentFailed,
+  paymentRecovered,
+}

packages/backend-core/src/events/publishers/org.ts

@@ -1,29 +1,37 @@
 import { publishEvent } from "../events"
 import { Event } from "@budibase/types"
 
-export async function nameUpdated(timestamp?: string | number) {
+async function nameUpdated(timestamp?: string | number) {
   const properties = {}
   await publishEvent(Event.ORG_NAME_UPDATED, properties, timestamp)
 }
 
-export async function logoUpdated(timestamp?: string | number) {
+async function logoUpdated(timestamp?: string | number) {
   const properties = {}
   await publishEvent(Event.ORG_LOGO_UPDATED, properties, timestamp)
 }
 
-export async function platformURLUpdated(timestamp?: string | number) {
+async function platformURLUpdated(timestamp?: string | number) {
   const properties = {}
   await publishEvent(Event.ORG_PLATFORM_URL_UPDATED, properties, timestamp)
 }
 
 // TODO
 
-export async function analyticsOptOut() {
+async function analyticsOptOut() {
   const properties = {}
   await publishEvent(Event.ANALYTICS_OPT_OUT, properties)
 }
 
-export async function analyticsOptIn() {
+async function analyticsOptIn() {
   const properties = {}
   await publishEvent(Event.ANALYTICS_OPT_OUT, properties)
 }
+
+export default {
+  nameUpdated,
+  logoUpdated,
+  platformURLUpdated,
+  analyticsOptOut,
+  analyticsOptIn,
+}

packages/backend-core/src/events/publishers/plugin.ts

@@ -7,7 +7,7 @@ import {
   PluginInitEvent,
 } from "@budibase/types"
 
-export async function init(plugin: Plugin) {
+async function init(plugin: Plugin) {
   const properties: PluginInitEvent = {
     type: plugin.schema.type,
     name: plugin.name,
@@ -17,7 +17,7 @@ export async function init(plugin: Plugin) {
   await publishEvent(Event.PLUGIN_INIT, properties)
 }
 
-export async function imported(plugin: Plugin) {
+async function imported(plugin: Plugin) {
   const properties: PluginImportedEvent = {
     pluginId: plugin._id as string,
     type: plugin.schema.type,
@@ -29,7 +29,7 @@ export async function imported(plugin: Plugin) {
   await publishEvent(Event.PLUGIN_IMPORTED, properties)
 }
 
-export async function deleted(plugin: Plugin) {
+async function deleted(plugin: Plugin) {
   const properties: PluginDeletedEvent = {
     pluginId: plugin._id as string,
     type: plugin.schema.type,
@@ -39,3 +39,9 @@ export async function deleted(plugin: Plugin) {
   }
   await publishEvent(Event.PLUGIN_DELETED, properties)
 }
+
+export default {
+  init,
+  imported,
+  deleted,
+}

packages/backend-core/src/events/publishers/query.ts

@@ -13,7 +13,7 @@ import {
 
 /* eslint-disable */
 
-export const created = async (
+const created = async (
   datasource: Datasource,
   query: Query,
   timestamp?: string | number
@@ -27,7 +27,7 @@ export const created = async (
   await publishEvent(Event.QUERY_CREATED, properties, timestamp)
 }
 
-export const updated = async (datasource: Datasource, query: Query) => {
+const updated = async (datasource: Datasource, query: Query) => {
   const properties: QueryUpdatedEvent = {
     queryId: query._id as string,
     datasourceId: datasource._id as string,
@@ -37,7 +37,7 @@ export const updated = async (datasource: Datasource, query: Query) => {
   await publishEvent(Event.QUERY_UPDATED, properties)
 }
 
-export const deleted = async (datasource: Datasource, query: Query) => {
+const deleted = async (datasource: Datasource, query: Query) => {
   const properties: QueryDeletedEvent = {
     queryId: query._id as string,
     datasourceId: datasource._id as string,
@@ -47,7 +47,7 @@ export const deleted = async (datasource: Datasource, query: Query) => {
   await publishEvent(Event.QUERY_DELETED, properties)
 }
 
-export const imported = async (
+const imported = async (
   datasource: Datasource,
   importSource: any,
   count: any
@@ -61,14 +61,14 @@ export const imported = async (
   await publishEvent(Event.QUERY_IMPORT, properties)
 }
 
-export const run = async (count: number, timestamp?: string | number) => {
+const run = async (count: number, timestamp?: string | number) => {
   const properties: QueriesRunEvent = {
     count,
   }
   await publishEvent(Event.QUERIES_RUN, properties, timestamp)
 }
 
-export const previewed = async (datasource: Datasource, query: Query) => {
+const previewed = async (datasource: Datasource, query: Query) => {
   const properties: QueryPreviewedEvent = {
     queryId: query._id,
     datasourceId: datasource._id as string,
@@ -77,3 +77,12 @@ export const previewed = async (datasource: Datasource, query: Query) => {
   }
   await publishEvent(Event.QUERY_PREVIEWED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  imported,
+  run,
+  previewed,
+}

packages/backend-core/src/events/publishers/role.ts

@@ -10,7 +10,7 @@ import {
   User,
 } from "@budibase/types"
 
-export async function created(role: Role, timestamp?: string | number) {
+async function created(role: Role, timestamp?: string | number) {
   const properties: RoleCreatedEvent = {
     roleId: role._id as string,
     permissionId: role.permissionId,
@@ -19,7 +19,7 @@ export async function created(role: Role, timestamp?: string | number) {
   await publishEvent(Event.ROLE_CREATED, properties, timestamp)
 }
 
-export async function updated(role: Role) {
+async function updated(role: Role) {
   const properties: RoleUpdatedEvent = {
     roleId: role._id as string,
     permissionId: role.permissionId,
@@ -28,7 +28,7 @@ export async function updated(role: Role) {
   await publishEvent(Event.ROLE_UPDATED, properties)
 }
 
-export async function deleted(role: Role) {
+async function deleted(role: Role) {
   const properties: RoleDeletedEvent = {
     roleId: role._id as string,
     permissionId: role.permissionId,
@@ -37,7 +37,7 @@ export async function deleted(role: Role) {
   await publishEvent(Event.ROLE_DELETED, properties)
 }
 
-export async function assigned(user: User, roleId: string, timestamp?: number) {
+async function assigned(user: User, roleId: string, timestamp?: number) {
   const properties: RoleAssignedEvent = {
     userId: user._id as string,
     roleId,
@@ -45,10 +45,18 @@ export async function assigned(user: User, roleId: string, timestamp?: number) {
   await publishEvent(Event.ROLE_ASSIGNED, properties, timestamp)
 }
 
-export async function unassigned(user: User, roleId: string) {
+async function unassigned(user: User, roleId: string) {
   const properties: RoleUnassignedEvent = {
     userId: user._id as string,
     roleId,
   }
   await publishEvent(Event.ROLE_UNASSIGNED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  assigned,
+  unassigned,
+}

packages/backend-core/src/events/publishers/rows.ts

@@ -3,28 +3,27 @@ import {
   Event,
   RowsImportedEvent,
   RowsCreatedEvent,
-  RowImportFormat,
   Table,
 } from "@budibase/types"
 
 /* eslint-disable */
 
-export const created = async (count: number, timestamp?: string | number) => {
+const created = async (count: number, timestamp?: string | number) => {
   const properties: RowsCreatedEvent = {
     count,
   }
   await publishEvent(Event.ROWS_CREATED, properties, timestamp)
 }
 
-export const imported = async (
+const imported = async (table: Table, count: number) => {
-  table: Table,
-  format: RowImportFormat,
-  count: number
-) => {
   const properties: RowsImportedEvent = {
     tableId: table._id as string,
-    format,
     count,
   }
   await publishEvent(Event.ROWS_IMPORTED, properties)
 }
+
+export default {
+  created,
+  imported,
+}

packages/backend-core/src/events/publishers/screen.ts

@@ -6,7 +6,7 @@ import {
   ScreenDeletedEvent,
 } from "@budibase/types"
 
-export async function created(screen: Screen, timestamp?: string | number) {
+async function created(screen: Screen, timestamp?: string | number) {
   const properties: ScreenCreatedEvent = {
     layoutId: screen.layoutId,
     screenId: screen._id as string,
@@ -15,7 +15,7 @@ export async function created(screen: Screen, timestamp?: string | number) {
   await publishEvent(Event.SCREEN_CREATED, properties, timestamp)
 }
 
-export async function deleted(screen: Screen) {
+async function deleted(screen: Screen) {
   const properties: ScreenDeletedEvent = {
     layoutId: screen.layoutId,
     screenId: screen._id as string,
@@ -23,3 +23,8 @@ export async function deleted(screen: Screen) {
   }
   await publishEvent(Event.SCREEN_DELETED, properties)
 }
+
+export default {
+  created,
+  deleted,
+}

packages/backend-core/src/events/publishers/serve.ts

@@ -7,14 +7,14 @@ import {
   AppServedEvent,
 } from "@budibase/types"
 
-export async function servedBuilder(timezone: string) {
+async function servedBuilder(timezone: string) {
   const properties: BuilderServedEvent = {
     timezone,
   }
   await publishEvent(Event.SERVED_BUILDER, properties)
 }
 
-export async function servedApp(app: App, timezone: string) {
+async function servedApp(app: App, timezone: string) {
   const properties: AppServedEvent = {
     appVersion: app.version,
     timezone,
@@ -22,7 +22,7 @@ export async function servedApp(app: App, timezone: string) {
   await publishEvent(Event.SERVED_APP, properties)
 }
 
-export async function servedAppPreview(app: App, timezone: string) {
+async function servedAppPreview(app: App, timezone: string) {
   const properties: AppPreviewServedEvent = {
     appId: app.appId,
     appVersion: app.version,
@@ -30,3 +30,9 @@ export async function servedAppPreview(app: App, timezone: string) {
   }
   await publishEvent(Event.SERVED_APP_PREVIEW, properties)
 }
+
+export default {
+  servedBuilder,
+  servedApp,
+  servedAppPreview,
+}

packages/backend-core/src/events/publishers/table.ts

@@ -2,7 +2,6 @@ import { publishEvent } from "../events"
 import {
   Event,
   TableExportFormat,
-  TableImportFormat,
   Table,
   TableCreatedEvent,
   TableUpdatedEvent,
@@ -11,28 +10,28 @@ import {
   TableImportedEvent,
 } from "@budibase/types"
 
-export async function created(table: Table, timestamp?: string | number) {
+async function created(table: Table, timestamp?: string | number) {
   const properties: TableCreatedEvent = {
     tableId: table._id as string,
   }
   await publishEvent(Event.TABLE_CREATED, properties, timestamp)
 }
 
-export async function updated(table: Table) {
+async function updated(table: Table) {
   const properties: TableUpdatedEvent = {
     tableId: table._id as string,
   }
   await publishEvent(Event.TABLE_UPDATED, properties)
 }
 
-export async function deleted(table: Table) {
+async function deleted(table: Table) {
   const properties: TableDeletedEvent = {
     tableId: table._id as string,
   }
   await publishEvent(Event.TABLE_DELETED, properties)
 }
 
-export async function exported(table: Table, format: TableExportFormat) {
+async function exported(table: Table, format: TableExportFormat) {
   const properties: TableExportedEvent = {
     tableId: table._id as string,
     format,
@@ -40,10 +39,17 @@ export async function exported(table: Table, format: TableExportFormat) {
   await publishEvent(Event.TABLE_EXPORTED, properties)
 }
 
-export async function imported(table: Table, format: TableImportFormat) {
+async function imported(table: Table) {
   const properties: TableImportedEvent = {
     tableId: table._id as string,
-    format,
   }
   await publishEvent(Event.TABLE_IMPORTED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  exported,
+  imported,
+}

packages/backend-core/src/events/publishers/user.ts

@@ -15,21 +15,21 @@ import {
   UserUpdatedEvent,
 } from "@budibase/types"
 
-export async function created(user: User, timestamp?: number) {
+async function created(user: User, timestamp?: number) {
   const properties: UserCreatedEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_CREATED, properties, timestamp)
 }
 
-export async function updated(user: User) {
+async function updated(user: User) {
   const properties: UserUpdatedEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_UPDATED, properties)
 }
 
-export async function deleted(user: User) {
+async function deleted(user: User) {
   const properties: UserDeletedEvent = {
     userId: user._id as string,
   }
@@ -38,7 +38,7 @@ export async function deleted(user: User) {
 
 // PERMISSIONS
 
-export async function permissionAdminAssigned(user: User, timestamp?: number) {
+async function permissionAdminAssigned(user: User, timestamp?: number) {
   const properties: UserPermissionAssignedEvent = {
     userId: user._id as string,
   }
@@ -49,17 +49,14 @@ export async function permissionAdminAssigned(user: User, timestamp?: number) {
   )
 }
 
-export async function permissionAdminRemoved(user: User) {
+async function permissionAdminRemoved(user: User) {
   const properties: UserPermissionRemovedEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_PERMISSION_ADMIN_REMOVED, properties)
 }
 
-export async function permissionBuilderAssigned(
+async function permissionBuilderAssigned(user: User, timestamp?: number) {
-  user: User,
-  timestamp?: number
-) {
   const properties: UserPermissionAssignedEvent = {
     userId: user._id as string,
   }
@@ -70,7 +67,7 @@ export async function permissionBuilderAssigned(
   )
 }
 
-export async function permissionBuilderRemoved(user: User) {
+async function permissionBuilderRemoved(user: User) {
   const properties: UserPermissionRemovedEvent = {
     userId: user._id as string,
   }
@@ -79,12 +76,12 @@ export async function permissionBuilderRemoved(user: User) {
 
 // INVITE
 
-export async function invited() {
+async function invited() {
   const properties: UserInvitedEvent = {}
   await publishEvent(Event.USER_INVITED, properties)
 }
 
-export async function inviteAccepted(user: User) {
+async function inviteAccepted(user: User) {
   const properties: UserInviteAcceptedEvent = {
     userId: user._id as string,
   }
@@ -93,30 +90,46 @@ export async function inviteAccepted(user: User) {
 
 // PASSWORD
 
-export async function passwordForceReset(user: User) {
+async function passwordForceReset(user: User) {
   const properties: UserPasswordForceResetEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_PASSWORD_FORCE_RESET, properties)
 }
 
-export async function passwordUpdated(user: User) {
+async function passwordUpdated(user: User) {
   const properties: UserPasswordUpdatedEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_PASSWORD_UPDATED, properties)
 }
 
-export async function passwordResetRequested(user: User) {
+async function passwordResetRequested(user: User) {
   const properties: UserPasswordResetRequestedEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_PASSWORD_RESET_REQUESTED, properties)
 }
 
-export async function passwordReset(user: User) {
+async function passwordReset(user: User) {
   const properties: UserPasswordResetEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_PASSWORD_RESET, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  permissionAdminAssigned,
+  permissionAdminRemoved,
+  permissionBuilderAssigned,
+  permissionBuilderRemoved,
+  invited,
+  inviteAccepted,
+  passwordForceReset,
+  passwordUpdated,
+  passwordResetRequested,
+  passwordReset,
+}

packages/backend-core/src/events/publishers/view.ts

@@ -19,28 +19,28 @@ import {
 
 /* eslint-disable */
 
-export async function created(view: View, timestamp?: string | number) {
+async function created(view: View, timestamp?: string | number) {
   const properties: ViewCreatedEvent = {
     tableId: view.tableId,
   }
   await publishEvent(Event.VIEW_CREATED, properties, timestamp)
 }
 
-export async function updated(view: View) {
+async function updated(view: View) {
   const properties: ViewUpdatedEvent = {
     tableId: view.tableId,
   }
   await publishEvent(Event.VIEW_UPDATED, properties)
 }
 
-export async function deleted(view: View) {
+async function deleted(view: View) {
   const properties: ViewDeletedEvent = {
     tableId: view.tableId,
   }
   await publishEvent(Event.VIEW_DELETED, properties)
 }
 
-export async function exported(table: Table, format: TableExportFormat) {
+async function exported(table: Table, format: TableExportFormat) {
   const properties: ViewExportedEvent = {
     tableId: table._id as string,
     format,
@@ -48,31 +48,28 @@ export async function exported(table: Table, format: TableExportFormat) {
   await publishEvent(Event.VIEW_EXPORTED, properties)
 }
 
-export async function filterCreated(view: View, timestamp?: string | number) {
+async function filterCreated(view: View, timestamp?: string | number) {
   const properties: ViewFilterCreatedEvent = {
     tableId: view.tableId,
   }
   await publishEvent(Event.VIEW_FILTER_CREATED, properties, timestamp)
 }
 
-export async function filterUpdated(view: View) {
+async function filterUpdated(view: View) {
   const properties: ViewFilterUpdatedEvent = {
     tableId: view.tableId,
   }
   await publishEvent(Event.VIEW_FILTER_UPDATED, properties)
 }
 
-export async function filterDeleted(view: View) {
+async function filterDeleted(view: View) {
   const properties: ViewFilterDeletedEvent = {
     tableId: view.tableId,
   }
   await publishEvent(Event.VIEW_FILTER_DELETED, properties)
 }
 
-export async function calculationCreated(
+async function calculationCreated(view: View, timestamp?: string | number) {
-  view: View,
-  timestamp?: string | number
-) {
   const properties: ViewCalculationCreatedEvent = {
     tableId: view.tableId,
     calculation: view.calculation as ViewCalculation,
@@ -80,7 +77,7 @@ export async function calculationCreated(
   await publishEvent(Event.VIEW_CALCULATION_CREATED, properties, timestamp)
 }
 
-export async function calculationUpdated(view: View) {
+async function calculationUpdated(view: View) {
   const properties: ViewCalculationUpdatedEvent = {
     tableId: view.tableId,
     calculation: view.calculation as ViewCalculation,
@@ -88,10 +85,23 @@ export async function calculationUpdated(view: View) {
   await publishEvent(Event.VIEW_CALCULATION_UPDATED, properties)
 }
 
-export async function calculationDeleted(existingView: View) {
+async function calculationDeleted(existingView: View) {
   const properties: ViewCalculationDeletedEvent = {
     tableId: existingView.tableId,
     calculation: existingView.calculation as ViewCalculation,
   }
   await publishEvent(Event.VIEW_CALCULATION_DELETED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  exported,
+  filterCreated,
+  filterUpdated,
+  filterDeleted,
+  calculationCreated,
+  calculationUpdated,
+  calculationDeleted,
+}

packages/backend-core/src/index.ts

@@ -1,68 +1,42 @@
-import errors from "./errors"
+export * as events from "./events"
-const errorClasses = errors.errors
+export * as migrations from "./migrations"
-import * as events from "./events"
+export * as users from "./users"
-import * as migrations from "./migrations"
+export * as roles from "./security/roles"
-import * as users from "./users"
+export * as permissions from "./security/permissions"
-import * as roles from "./security/roles"
+export * as accounts from "./cloud/accounts"
-import * as permissions from "./security/permissions"
+export * as installation from "./installation"
-import * as accounts from "./cloud/accounts"
+export * as tenancy from "./tenancy"
-import * as installation from "./installation"
+export * as featureFlags from "./featureFlags"
-import env from "./environment"
+export * as sessions from "./security/sessions"
-import * as tenancy from "./tenancy"
+export * as deprovisioning from "./context/deprovision"
-import * as featureFlags from "./featureFlags"
+export * as auth from "./auth"
-import * as sessions from "./security/sessions"
+export * as constants from "./constants"
-import * as deprovisioning from "./context/deprovision"
+export * as logging from "./logging"
-import * as auth from "./auth"
+export * as middleware from "./middleware"
-import * as constants from "./constants"
+export * as plugins from "./plugin"
-import * as logging from "./logging"
+export * as encryption from "./security/encryption"
-import * as pino from "./pino"
+export * as queue from "./queue"
-import * as middleware from "./middleware"
+export * as db from "./db"
-import * as plugins from "./plugin"
+export * as context from "./context"
-import * as encryption from "./security/encryption"
+export * as cache from "./cache"
-import * as queue from "./queue"
+export * as objectStore from "./objectStore"
-import * as db from "./db"
+export * as redis from "./redis"
-import * as context from "./context"
+export * as utils from "./utils"
-import * as cache from "./cache"
+export * as errors from "./errors"
-import * as objectStore from "./objectStore"
+export { default as env } from "./environment"
-import * as redis from "./redis"
-import * as utils from "./utils"
 
-const init = (opts: any = {}) => {
+// expose error classes directly
+export * from "./errors"
+
+// expose constants directly
+export * from "./constants"
+
+// expose inner locks from redis directly
+import * as redis from "./redis"
+export const locks = redis.redlock
+
+// expose package init function
+import * as db from "./db"
+export const init = (opts: any = {}) => {
   db.init(opts.db)
 }
-
-const core = {
-  init,
-  db,
-  ...constants,
-  redis,
-  locks: redis.redlock,
-  objectStore,
-  utils,
-  users,
-  cache,
-  auth,
-  constants,
-  migrations,
-  env,
-  accounts,
-  tenancy,
-  context,
-  featureFlags,
-  events,
-  sessions,
-  deprovisioning,
-  installation,
-  errors,
-  logging,
-  roles,
-  plugins,
-  ...pino,
-  ...errorClasses,
-  middleware,
-  encryption,
-  queue,
-  permissions,
-}
-
-export = core

packages/backend-core/src/logging.ts

@@ -1,3 +1,9 @@
+import { Header } from "./constants"
+import env from "./environment"
+const correlator = require("correlation-id")
+import { Options } from "pino-http"
+import { IncomingMessage } from "http"
+
 const NonErrors = ["AccountError"]
 
 function isSuppressed(e?: any) {
@@ -29,8 +35,26 @@ export function logWarn(message: string) {
   console.warn(`bb-warn: ${message}`)
 }
 
-export default {
+export function pinoSettings(): Options {
-  logAlert,
+  return {
-  logAlertWithInfo,
+    prettyPrint: {
-  logWarn,
+      levelFirst: true,
+    },
+    genReqId: correlator.getId,
+    level: env.LOG_LEVEL || "error",
+    autoLogging: {
+      ignore: (req: IncomingMessage) => !!req.url?.includes("/health"),
+    },
+  }
+}
+
+const setCorrelationHeader = (headers: any) => {
+  const correlationId = correlator.getId()
+  if (correlationId) {
+    headers[Header.CORRELATION_ID] = correlationId
+  }
+}
+
+export const correlation = {
+  setHeader: setCorrelationHeader,
 }

packages/backend-core/src/middleware/adminOnly.ts

@@ -1,6 +1,6 @@
 import { BBContext } from "@budibase/types"
 
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   if (
     !ctx.internal &&
     (!ctx.user || !ctx.user.admin || !ctx.user.admin.global)

packages/backend-core/src/middleware/auditLog.ts

@@ -1,6 +1,6 @@
 import { BBContext } from "@budibase/types"
 
-export = async (ctx: BBContext | any, next: any) => {
+export default async (ctx: BBContext | any, next: any) => {
   // Placeholder for audit log middleware
   return next()
 }

packages/backend-core/src/middleware/authenticated.ts

@@ -66,7 +66,7 @@ async function checkApiKey(apiKey: string, populateUser?: Function) {
  * The tenancy modules should not be used here and it should be assumed that the tenancy context
  * has not yet been populated.
  */
-export = function (
+export default function (
   noAuthPatterns: EndpointMatcher[] = [],
   opts: { publicAllowed?: boolean; populateUser?: Function } = {
     publicAllowed: false,

packages/backend-core/src/middleware/builderOnly.ts

@@ -1,6 +1,6 @@
 import { BBContext } from "@budibase/types"
 
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   if (
     !ctx.internal &&
     (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)

packages/backend-core/src/middleware/builderOrAdmin.ts

@@ -1,6 +1,6 @@
 import { BBContext } from "@budibase/types"
 
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   if (
     !ctx.internal &&
     (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) &&

packages/backend-core/src/middleware/csrf.ts

@@ -32,7 +32,7 @@ const INCLUDED_CONTENT_TYPES = [
  * https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html#synchronizer-token-pattern
  *
  */
-export = function (
+export default function (
   opts: { noCsrfPatterns: EndpointMatcher[] } = { noCsrfPatterns: [] }
 ) {
   const noCsrfOptions = buildMatcherRegex(opts.noCsrfPatterns)

packages/backend-core/src/middleware/index.ts

@@ -1,38 +1,19 @@
-import * as jwt from "./passport/jwt"
+export * as jwt from "./passport/jwt"
-import * as local from "./passport/local"
+export * as local from "./passport/local"
-import * as google from "./passport/google"
+export * as google from "./passport/google"
-import * as oidc from "./passport/oidc"
+export * as oidc from "./passport/oidc"
-import { authError, ssoCallbackUrl } from "./passport/utils"
-import authenticated from "./authenticated"
-import auditLog from "./auditLog"
-import tenancy from "./tenancy"
-import internalApi from "./internalApi"
 import * as datasourceGoogle from "./passport/datasource/google"
-import csrf from "./csrf"
+export const datasource = {
-import adminOnly from "./adminOnly"
+  google: datasourceGoogle,
-import builderOrAdmin from "./builderOrAdmin"
-import builderOnly from "./builderOnly"
-import * as joiValidator from "./joi-validator"
-
-const pkg = {
-  google,
-  oidc,
-  jwt,
-  local,
-  authenticated,
-  auditLog,
-  tenancy,
-  authError,
-  internalApi,
-  ssoCallbackUrl,
-  datasource: {
-    google: datasourceGoogle,
-  },
-  csrf,
-  adminOnly,
-  builderOnly,
-  builderOrAdmin,
-  joiValidator,
 }
-
+export { authError, ssoCallbackUrl } from "./passport/utils"
-export = pkg
+export { default as authenticated } from "./authenticated"
+export { default as auditLog } from "./auditLog"
+export { default as tenancy } from "./tenancy"
+export { default as internalApi } from "./internalApi"
+export { default as csrf } from "./csrf"
+export { default as adminOnly } from "./adminOnly"
+export { default as builderOrAdmin } from "./builderOrAdmin"
+export { default as builderOnly } from "./builderOnly"
+export { default as logging } from "./logging"
+export * as joiValidator from "./joi-validator"

packages/backend-core/src/middleware/internalApi.ts

@@ -5,7 +5,7 @@ import { BBContext } from "@budibase/types"
 /**
  * API Key only endpoint.
  */
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   const apiKey = ctx.request.headers[Header.API_KEY]
   if (apiKey !== env.INTERNAL_API_KEY) {
     ctx.throw(403, "Unauthorized")

packages/backend-core/src/middleware/logging.ts

@@ -0,0 +1,88 @@
+const correlator = require("correlation-id")
+import { Header } from "../constants"
+import { v4 as uuid } from "uuid"
+import * as context from "../context"
+
+const debug = console.warn
+const trace = console.trace
+const log = console.log
+const info = console.info
+const warn = console.warn
+const error = console.error
+
+const getTenantId = () => {
+  let tenantId
+  try {
+    tenantId = context.getTenantId()
+  } catch (e: any) {
+    // do nothing
+  }
+  return tenantId
+}
+
+const getAppId = () => {
+  let appId
+  try {
+    appId = context.getAppId()
+  } catch (e) {
+    // do nothing
+  }
+  return appId
+}
+
+const getIdentityId = () => {
+  let identityId
+  try {
+    const identity = context.getIdentity()
+    identityId = identity?._id
+  } catch (e) {
+    // do nothing
+  }
+  return identityId
+}
+
+const print = (fn: any, data: any[]) => {
+  let message = ""
+
+  const correlationId = correlator.getId()
+  if (correlationId) {
+    message = message + `[correlationId=${correlator.getId()}]`
+  }
+
+  const tenantId = getTenantId()
+  if (tenantId) {
+    message = message + ` [tenantId=${tenantId}]`
+  }
+
+  const appId = getAppId()
+  if (appId) {
+    message = message + ` [appId=${appId}]`
+  }
+
+  const identityId = getIdentityId()
+  if (identityId) {
+    message = message + ` [identityId=${identityId}]`
+  }
+
+  fn(message, data)
+}
+
+const logging = (ctx: any, next: any) => {
+  // use the provided correlation id header if present
+  let correlationId = ctx.headers[Header.CORRELATION_ID]
+  if (!correlationId) {
+    correlationId = uuid()
+  }
+
+  return correlator.withId(correlationId, () => {
+    console.debug = data => print(debug, data)
+    console.trace = data => print(trace, data)
+    console.log = data => print(log, data)
+    console.info = data => print(info, data)
+    console.warn = data => print(warn, data)
+    console.error = data => print(error, data)
+    return next()
+  })
+}
+
+export default logging

packages/backend-core/src/middleware/passport/oidc.ts

@@ -2,7 +2,6 @@ import fetch from "node-fetch"
 import { authenticateThirdParty, SaveUserFunction } from "./third-party-common"
 import { ssoCallbackUrl } from "./utils"
 import {
-  Config,
   ConfigType,
   OIDCInnerCfg,
   Database,

packages/backend-core/src/middleware/tenancy.ts

@@ -8,7 +8,7 @@ import {
   TenantResolutionStrategy,
 } from "@budibase/types"
 
-export = function (
+export default function (
   allowQueryStringPatterns: EndpointMatcher[],
   noTenancyPatterns: EndpointMatcher[],
   opts: { noTenancyRequired?: boolean } = { noTenancyRequired: false }

packages/backend-core/src/migrations/migrations.ts

@@ -88,7 +88,7 @@ export const runMigration = async (
 
     await doWithDB(dbName, async (db: any) => {
       try {
-        const doc = await exports.getMigrationsDoc(db)
+        const doc = await getMigrationsDoc(db)
 
         // the migration has already been run
         if (doc[migrationName]) {

packages/backend-core/src/objectStore/objectStore.ts

@@ -329,9 +329,9 @@ export const deleteFile = async (bucketName: string, filepath: string) => {
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
     Bucket: bucketName,
-    Key: filepath,
+    Key: sanitizeKey(filepath),
   }
-  return objectStore.deleteObject(params)
+  return objectStore.deleteObject(params).promise()
 }
 
 export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
@@ -340,7 +340,7 @@ export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
   const params = {
     Bucket: bucketName,
     Delete: {
-      Objects: filepaths.map((path: any) => ({ Key: path })),
+      Objects: filepaths.map((path: any) => ({ Key: sanitizeKey(path) })),
     },
   }
   return objectStore.deleteObjects(params).promise()

packages/backend-core/src/pino.ts

@@ -1,13 +0,0 @@
-import env from "./environment"
-
-export function pinoSettings() {
-  return {
-    prettyPrint: {
-      levelFirst: true,
-    },
-    level: env.LOG_LEVEL || "error",
-    autoLogging: {
-      ignore: (req: { url: string }) => req.url.includes("/health"),
-    },
-  }
-}

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -137,4 +137,4 @@ class InMemoryQueue {
   }
 }
 
-export = InMemoryQueue
+export default InMemoryQueue

packages/backend-core/src/redis/redis.ts

@@ -276,4 +276,4 @@ class RedisWrapper {
   }
 }
 
-export = RedisWrapper
+export default RedisWrapper

packages/backend-core/src/security/sessions.ts

@@ -1,7 +1,7 @@
 const redis = require("../redis/init")
 const { v4: uuidv4 } = require("uuid")
 const { logWarn } = require("../logging")
-const env = require("../environment")
+import env from "../environment"
 import {
   Session,
   ScannedSession,

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -1,7 +1,8 @@
 import { structures } from "../../../tests"
 import * as utils from "../../utils"
 import * as events from "../../events"
-import { DEFAULT_TENANT_ID } from "../../constants"
+import * as db from "../../db"
+import { DEFAULT_TENANT_ID, Header } from "../../constants"
 import { doInTenant } from "../../context"
 
 describe("utils", () => {
@@ -14,4 +15,95 @@ describe("utils", () => {
       })
     })
   })
+
+  describe("getAppIdFromCtx", () => {
+    it("gets appId from header", async () => {
+      const ctx = structures.koa.newContext()
+      const expected = db.generateAppID()
+      ctx.request.headers = {
+        [Header.APP_ID]: expected,
+      }
+
+      const actual = await utils.getAppIdFromCtx(ctx)
+      expect(actual).toBe(expected)
+    })
+
+    it("gets appId from body", async () => {
+      const ctx = structures.koa.newContext()
+      const expected = db.generateAppID()
+      ctx.request.body = {
+        appId: expected,
+      }
+
+      const actual = await utils.getAppIdFromCtx(ctx)
+      expect(actual).toBe(expected)
+    })
+
+    it("gets appId from path", async () => {
+      const ctx = structures.koa.newContext()
+      const expected = db.generateAppID()
+      ctx.path = `/apps/${expected}`
+
+      const actual = await utils.getAppIdFromCtx(ctx)
+      expect(actual).toBe(expected)
+    })
+
+    it("gets appId from url", async () => {
+      const ctx = structures.koa.newContext()
+      const expected = db.generateAppID()
+      const app = structures.apps.app(expected)
+
+      // set custom url
+      const appUrl = "custom-url"
+      app.url = `/${appUrl}`
+      ctx.path = `/app/${appUrl}`
+
+      // save the app
+      const database = db.getDB(expected)
+      await database.put(app)
+
+      const actual = await utils.getAppIdFromCtx(ctx)
+      expect(actual).toBe(expected)
+    })
+
+    it("doesn't get appId from url when previewing", async () => {
+      const ctx = structures.koa.newContext()
+      const appId = db.generateAppID()
+      const app = structures.apps.app(appId)
+
+      // set custom url
+      const appUrl = "preview"
+      app.url = `/${appUrl}`
+      ctx.path = `/app/${appUrl}`
+
+      // save the app
+      const database = db.getDB(appId)
+      await database.put(app)
+
+      const actual = await utils.getAppIdFromCtx(ctx)
+      expect(actual).toBe(undefined)
+    })
+
+    it("gets appId from referer", async () => {
+      const ctx = structures.koa.newContext()
+      const expected = db.generateAppID()
+      ctx.request.headers = {
+        referer: `http://test.com/builder/app/${expected}/design/screen_123/screens`,
+      }
+
+      const actual = await utils.getAppIdFromCtx(ctx)
+      expect(actual).toBe(expected)
+    })
+
+    it("doesn't get appId from referer when not builder", async () => {
+      const ctx = structures.koa.newContext()
+      const appId = db.generateAppID()
+      ctx.request.headers = {
+        referer: `http://test.com/foo/app/${appId}/bar`,
+      }
+
+      const actual = await utils.getAppIdFromCtx(ctx)
+      expect(actual).toBe(undefined)
+    })
+  })
 })

packages/backend-core/src/utils/utils.ts

@@ -25,13 +25,16 @@ const jwt = require("jsonwebtoken")
 const APP_PREFIX = DocumentType.APP + SEPARATOR
 const PROD_APP_PREFIX = "/app/"
 
+const BUILDER_PREVIEW_PATH = "/app/preview"
+const BUILDER_REFERER_PREFIX = "/builder/app/"
+
 function confirmAppId(possibleAppId: string | undefined) {
   return possibleAppId && possibleAppId.startsWith(APP_PREFIX)
     ? possibleAppId
     : undefined
 }
 
-async function resolveAppUrl(ctx: Ctx) {
+export async function resolveAppUrl(ctx: Ctx) {
   const appUrl = ctx.path.split("/")[2]
   let possibleAppUrl = `/${appUrl.toLowerCase()}`
 
@@ -75,7 +78,7 @@ export function isServingApp(ctx: Ctx) {
  */
 export async function getAppIdFromCtx(ctx: Ctx) {
   // look in headers
-  const options = [ctx.headers[Header.APP_ID]]
+  const options = [ctx.request.headers[Header.APP_ID]]
   let appId
   for (let option of options) {
     appId = confirmAppId(option as string)
@@ -95,15 +98,23 @@ export async function getAppIdFromCtx(ctx: Ctx) {
     appId = confirmAppId(pathId)
   }
 
-  // look in the referer
+  // lookup using custom url - prod apps only
-  const refererId = parseAppIdFromUrl(ctx.request.headers.referer)
+  // filter out the builder preview path which collides with the prod app path
-  if (!appId && refererId) {
+  // to ensure we don't load all apps excessively
-    appId = confirmAppId(refererId)
+  const isBuilderPreview = ctx.path.startsWith(BUILDER_PREVIEW_PATH)
+  const isViewingProdApp =
+    ctx.path.startsWith(PROD_APP_PREFIX) && !isBuilderPreview
+  if (!appId && isViewingProdApp) {
+    appId = confirmAppId(await resolveAppUrl(ctx))
   }
 
-  // look in the url - prod app
+  // look in the referer - builder only
-  if (!appId && ctx.path.startsWith(PROD_APP_PREFIX)) {
+  // make sure this is performed after prod app url resolution, in case the
-    appId = confirmAppId(await resolveAppUrl(ctx))
+  // referer header is present from a builder redirect
+  const referer = ctx.request.headers.referer
+  if (!appId && referer?.includes(BUILDER_REFERER_PREFIX)) {
+    const refererId = parseAppIdFromUrl(ctx.request.headers.referer)
+    appId = confirmAppId(refererId)
   }
 
   return appId

packages/backend-core/tests/utilities/mocks/accounts.ts

@@ -1,9 +1,13 @@
-export const getAccount = jest.fn()
+const mockGetAccount = jest.fn()
-export const getAccountByTenantId = jest.fn()
+const mockGetAccountByTenantId = jest.fn()
-export const getStatus = jest.fn()
+const mockGetStatus = jest.fn()
 
 jest.mock("../../../src/cloud/accounts", () => ({
-  getAccount,
+  getAccount: mockGetAccount,
-  getAccountByTenantId,
+  getAccountByTenantId: mockGetAccountByTenantId,
-  getStatus,
+  getStatus: mockGetStatus,
 }))
+
+export const getAccount = mockGetAccount
+export const getAccountByTenantId = mockGetAccountByTenantId
+export const getStatus = mockGetStatus

packages/backend-core/tests/utilities/mocks/events.ts

@@ -1,9 +1,8 @@
-const processors = require("../../../src/events/processors")
+import * as processors from "../../../src/events/processors"
+import * as events from "../../../src/events"
 
 jest.spyOn(processors.analyticsProcessor, "processEvent")
 
-const events = require("../../../src/events")
-
 jest.spyOn(events.identification, "identifyTenantGroup")
 jest.spyOn(events.identification, "identifyUser")
 

packages/backend-core/tests/utilities/structures/apps.ts

@@ -0,0 +1,21 @@
+import { generator } from "."
+import { App } from "@budibase/types"
+import { DEFAULT_TENANT_ID, DocumentType } from "../../../src/constants"
+
+export function app(id: string): App {
+  return {
+    _id: DocumentType.APP_METADATA,
+    appId: id,
+    type: "",
+    version: "0.0.1",
+    componentLibraries: [],
+    name: generator.name(),
+    url: `/custom-url`,
+    instance: {
+      _id: id,
+    },
+    tenantId: DEFAULT_TENANT_ID,
+    status: "",
+    template: undefined,
+  }
+}

packages/backend-core/tests/utilities/structures/index.ts

@@ -3,7 +3,8 @@ export * from "./common"
 import Chance from "chance"
 export const generator = new Chance()
 
-export * as koa from "./koa"
 export * as accounts from "./accounts"
+export * as apps from "./apps"
+export * as koa from "./koa"
 export * as licenses from "./licenses"
 export * as plugins from "./plugins"

packages/backend-core/tests/utilities/structures/koa.ts

@@ -5,9 +5,11 @@ export const newContext = (): BBContext => {
   const ctx = createMockContext()
   return {
     ...ctx,
+    path: "/",
     cookies: createMockCookies(),
     request: {
       ...ctx.request,
+      headers: {},
       body: {},
     },
   }

packages/backend-core/tsconfig.build.json

@@ -12,6 +12,7 @@
     "declaration": true,
     "types": [ "node", "jest" ],
     "outDir": "dist",
+    "skipLibCheck": true
   },
   "include": [
     "**/*.js",
@@ -22,6 +23,7 @@
     "node_modules",
     "dist",
     "**/*.spec.ts",
-    "**/*.spec.js"
+    "**/*.spec.js",
+    "__mocks__"
   ]
 }

packages/backend-core/tsconfig.json

@@ -12,6 +12,6 @@
   ],
   "exclude": [
     "node_modules",
-    "dist"
+    "dist",
   ]
 }

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.2.10-alpha.12",
+  "version": "2.2.12-alpha.21",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.2.10-alpha.12",
+    "@budibase/string-templates": "2.2.12-alpha.21",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",
     "@spectrum-css/avatar": "3.0.2",

packages/bbui/yarn.lock

@@ -1549,9 +1549,9 @@ json-parse-even-better-errors@^2.3.0:
   integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==
 
 json5@^1.0.1:
-  version "1.0.1"
+  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.1.tgz#779fb0018604fa854eacbf6252180d83543e3dbe"
+  resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593"
-  integrity sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==
+  integrity sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==
   dependencies:
     minimist "^1.2.0"
 
@@ -1662,16 +1662,11 @@ minimatch@^3.0.4:
   dependencies:
     brace-expansion "^1.1.7"
 
-minimist@^1.2.0:
+minimist@^1.2.0, minimist@^1.2.5:
   version "1.2.7"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18"
   integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==
 
-minimist@^1.2.5:
-  version "1.2.6"
-  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
-  integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
-
 mkdirp@~0.5.1:
   version "0.5.5"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def"

packages/builder/cypress/integration/adminAndManagement/accountPortals.spec.js

@@ -2,7 +2,7 @@ import filterTests from "../../support/filterTests"
 const interact = require('../../support/interact')
 
 filterTests(["smoke", "all"], () => {
-  context("Account Portals", () => {
+  xcontext("Account Portals", () => {
 
     const bbUserEmail = "bbuser@test.com"
 
@@ -44,7 +44,7 @@ filterTests(["smoke", "all"], () => {
       //cy.logoutNoAppGrid()
     })
 
-    it("should verify Standard Portal", () => {
+    xit("should verify Standard Portal", () => {
       // Development access should be disabled (Admin access is already disabled)
       cy.login()
       cy.setUserRole("bbuser", "App User")

packages/builder/cypress/integration/appPublishWorkflow.spec.js

@@ -3,100 +3,101 @@ import { APP_TABLE_APP_NAME, DEPLOY_SUCCESS_MODAL } from "../support/interact";
 const interact = require('../support/interact')
 
 filterTests(['all'], () => {
-  context("Publish Application Workflow", () => {
+  xcontext("Publish Application Workflow", () => {
     before(() => {
       cy.login()
       cy.deleteAllApps()
       cy.createApp("Cypress Tests", false)
     })
 
-    it("Should reflect the unpublished status correctly", () => {
+    xit("Should reflect the unpublished status correctly", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 })
 
       cy.get(interact.APP_TABLE_STATUS, { timeout: 3000 }).eq(0)
-      .within(() => {
+        .within(() => {
-        cy.contains("Unpublished")
+          cy.contains("Unpublished")
-        cy.get(interact.GLOBESTRIKE).should("exist")
+          cy.get(interact.GLOBESTRIKE).should("exist")
-      })
+        })
 
       cy.get(interact.APP_TABLE_ROW_ACTION).eq(0)
-      .within(() => {
+        .within(() => {
-        cy.get(interact.SPECTRUM_BUTTON_TEMPLATE).contains("Edit").click({ force: true })
+          cy.get(interact.SPECTRUM_BUTTON_TEMPLATE).contains("Edit").click({ force: true })
-      })
+        })
 
       cy.get(interact.DEPLOYMENT_TOP_NAV_GLOBESTRIKE).should("exist")
       cy.get(interact.DEPLOYMENT_TOP_GLOBE).should("not.exist")
     })
 
-    it("Should publish an application and correctly reflect that", () => {
+    xit("Should publish an application and correctly reflect that", () => {
       //Assuming the previous test was run and the unpublished app is open in edit mode.
-      cy.get(interact.TOPRIGHTNAV_BUTTON_SPECTRUM).contains("Publish").click({ force : true })
+      cy.get(interact.TOPRIGHTNAV_BUTTON_SPECTRUM).contains("Publish").click({ force: true })
 
       cy.get(interact.DEPLOY_APP_MODAL).should("be.visible")
-      .within(() => {
+        .within(() => {
-        cy.get(interact.SPECTRUM_BUTTON).contains("Publish").click({ force : true })
+          cy.get(interact.SPECTRUM_BUTTON).contains("Publish").click({ force: true })
-      });
+        });
 
       //Verify that the app url is presented correctly to the user
       cy.get(interact.DEPLOY_SUCCESS_MODAL, { timeout: 1000 })
-      .should("be.visible")
+        .should("be.visible")
-      .within(() => {
+        .within(() => {
-        let appUrl = Cypress.config().baseUrl + '/app/cypress-tests'
+          let appUrl = Cypress.config().baseUrl + '/app/cypress-tests'
-        cy.get(interact.DEPLOY_APP_URL_INPUT).should('have.value', appUrl)
+          cy.get(interact.DEPLOY_APP_URL_INPUT).should('have.value', appUrl)
-        cy.get(interact.SPECTRUM_BUTTON).contains("Done").click({ force: true })
+          cy.get(interact.SPECTRUM_BUTTON).contains("Done").click({ force: true })
-      })
+        })
 
       cy.visit(`${Cypress.config().baseUrl}/builder`)
 
       cy.get(interact.APP_TABLE_STATUS, { timeout: 3000 }).eq(0)
-      .within(() => {
+        .within(() => {
-        cy.contains("Published")
+          cy.contains("Published")
-        cy.get(interact.GLOBE).should("exist")
+          cy.get(interact.GLOBE).should("exist")
-      })
+        })
 
       cy.get(interact.APP_TABLE_ROW_ACTION).eq(0)
-      .within(() => {
+        .within(() => {
-        cy.get(interact.SPECTRUM_BUTTON).contains("Manage")
+          cy.get(interact.SPECTRUM_BUTTON).contains("Manage")
-        cy.get(interact.SPECTRUM_BUTTON).contains("Edit").click({ force: true })
+          cy.get(interact.SPECTRUM_BUTTON).contains("Edit").click({ force: true })
-      })
+        })
 
       cy.get(interact.DEPLOYMENT_TOP_GLOBE).should("exist").click({ force: true })
 
       cy.get(interact.PUBLISH_POPOVER_MENU).should("be.visible")
-      .within(() => {
+        .within(() => {
-        cy.get(interact.PUBLISH_POPOVER_ACTION).should("exist")
+          cy.get(interact.PUBLISH_POPOVER_ACTION).should("exist")
-        cy.get("button").contains("View app").should("exist")
+          cy.get("button").contains("View app").should("exist")
-        cy.get(interact.PUBLISH_POPOVER_MESSAGE).should("have.text", "Last published a few seconds ago")
+          cy.get(interact.PUBLISH_POPOVER_MESSAGE).should("have.text", "Last published a few seconds ago")
-      })
+        })
     })
 
-    it("Should unpublish an application using the link and reflect the status change", () => {
+    xit("Should unpublish an application using the link and reflect the status change", () => {
       //Assuming the previous test app exists and is published
 
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 })
 
       cy.get(interact.APP_TABLE_STATUS).eq(0)
-      .within(() => {
+        .within(() => {
-        cy.contains("Published")
+          cy.contains("Published")
-        cy.get("svg[aria-label='Globe']").should("exist")
+          cy.get("svg[aria-label='Globe']").should("exist")
-      })
+        })
 
       cy.get(interact.APP_TABLE).eq(0)
-      .within(() => {
+        .within(() => {
-        cy.get(interact.APP_TABLE_APP_NAME).click({ force: true })
+          cy.get(interact.APP_TABLE_APP_NAME).click({ force: true })
-      })
+        })
-      
+
       cy.get(interact.DEPLOYMENT_TOP_GLOBE).should("exist").click({ force: true })
-      
+
       cy.get("[data-cy='publish-popover-menu']")
-      .within(() => {
+        .within(() => {
-        cy.get(interact.PUBLISH_POPOVER_ACTION).click({ force: true })
+          cy.get(interact.PUBLISH_POPOVER_ACTION).click({ force: true })
-      })
+        })
 
       cy.get(interact.UNPUBLISH_MODAL).should("be.visible")
-      .within(() => {
+        .within(() => {
-        cy.get(interact.CONFIRM_WRAP_BUTTON).click({ force: true }
+          cy.get(interact.CONFIRM_WRAP_BUTTON).click({ force: true }
-      )})
+          )
+        })
 
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 6000 })
       cy.wait(500)

packages/builder/cypress/integration/createScreen.spec.js

@@ -2,7 +2,7 @@ import filterTests from "../support/filterTests"
 const interact = require('../support/interact')
 
 filterTests(["smoke", "all"], () => {
-  context("Screen Tests", () => {
+  xcontext("Screen Tests", () => {
     before(() => {
       cy.login()
       cy.createTestApp()
@@ -25,7 +25,7 @@ filterTests(["smoke", "all"], () => {
 
     it.skip("should delete all screens then create first screen via button", () => {
       cy.deleteAllScreens()
-      
+
       cy.contains("Create first screen").click()
       cy.get(interact.BODY, { timeout: 2000 }).should('contain', '/home')
     })
@@ -33,7 +33,7 @@ filterTests(["smoke", "all"], () => {
     it("Should create and filter screens by access level", () => {
       const accessLevels = ["Basic", "Admin", "Public", "Power"]
 
-      for (const access of accessLevels){
+      for (const access of accessLevels) {
         // Create screen with specified access level
         cy.createScreen(access, access)
         // Filter by access level and confirm screen visible
@@ -46,9 +46,9 @@ filterTests(["smoke", "all"], () => {
       // Filter by All screens - Confirm all screens visible
       cy.filterScreensAccessLevel("All screens")
       cy.get(interact.BODY).should('contain', accessLevels[0])
-      .and('contain', accessLevels[1])
+        .and('contain', accessLevels[1])
-      .and('contain', accessLevels[2])
+        .and('contain', accessLevels[2])
-      .and('contain', accessLevels[3])
+        .and('contain', accessLevels[3])
     })
   })
 })

packages/builder/cypress/integration/createView.spec.js

@@ -2,7 +2,7 @@ import filterTests from "../support/filterTests"
 const interact = require('../support/interact')
 
 filterTests(['smoke', 'all'], () => {
-  context("Create a View", () => {
+  xcontext("Create a View", () => {
     before(() => {
       cy.login()
       cy.createTestApp()
@@ -20,7 +20,7 @@ filterTests(['smoke', 'all'], () => {
       cy.addRow(["Teachers", 36, 3])
     })
 
-    it("creates a view", () => {
+    xit("creates a view", () => {
       cy.contains("Create view").click()
       cy.get(interact.MODAL_INNER_WRAPPER).within(() => {
         cy.get("input").type("Test View")
@@ -38,7 +38,7 @@ filterTests(['smoke', 'all'], () => {
       })
     })
 
-    it("filters the view by age over 10", () => {
+    xit("filters the view by age over 10", () => {
       cy.contains("Filter").click()
       cy.contains("Add Filter").click()
 
@@ -58,7 +58,7 @@ filterTests(['smoke', 'all'], () => {
       })
     })
 
-    it("creates a stats calculation view based on age", () => {
+    xit("creates a stats calculation view based on age", () => {
       cy.wait(1000)
       cy.contains("Calculate").click()
       cy.get(interact.MODAL_INNER_WRAPPER).within(() => {
@@ -95,7 +95,7 @@ filterTests(['smoke', 'all'], () => {
       })
     })
 
-    it("groups the view by group", () => {
+    xit("groups the view by group", () => {
       cy.contains("Group by").click()
       cy.get(interact.MODAL_INNER_WRAPPER).within(() => {
         cy.get(interact.SPECTRUM_PICKER_LABEL).eq(0).click()
@@ -127,7 +127,7 @@ filterTests(['smoke', 'all'], () => {
       })
     })
 
-    it("renames a view", () => {
+    xit("renames a view", () => {
       cy.contains(interact.NAV_ITEM, "Test View")
         .find(".actions .icon.open-popover")
         .click({ force: true })

packages/builder/cypress/integration/queryLevelTransformers.spec.js

@@ -13,9 +13,9 @@ filterTests(["smoke", "all"], () => {
       const datasource = "REST"
       const restUrl = "https://api.openbrewerydb.org/breweries"
       cy.selectExternalDatasource(datasource)
-      cy.createRestQuery("GET", restUrl, "/breweries")
+      cy.createRestQuery("GET", restUrl, "breweries")
       cy.reload()
-      cy.contains(".nav-item-content", "/breweries", { timeout: 20000 }).click()
+      cy.contains(".nav-item-content", "breweries", { timeout: 20000 }).click()
       cy.contains(interact.SPECTRUM_TABS_ITEM, "Transformer", { timeout: 5000 }).click({ force: true })
       // Get Transformer Function from file
       cy.readFile("cypress/support/queryLevelTransformerFunction.js").then(
@@ -44,9 +44,9 @@ filterTests(["smoke", "all"], () => {
       const datasource = "REST"
       const restUrl = "https://api.openbrewerydb.org/breweries"
       cy.selectExternalDatasource(datasource)
-      cy.createRestQuery("GET", restUrl, "/breweries")
+      cy.createRestQuery("GET", restUrl, "breweries")
       cy.reload()
-      cy.contains(".nav-item-content", "/breweries", { timeout: 2000 }).click()
+      cy.contains(".nav-item-content", "breweries", { timeout: 2000 }).click()
       cy.contains(interact.SPECTRUM_TABS_ITEM, "Transformer", { timeout: 5000 }).click({ force: true })
       // Get Transformer Function with Data from file
       cy.readFile(
@@ -75,9 +75,9 @@ filterTests(["smoke", "all"], () => {
       const datasource = "REST"
       const restUrl = "https://api.openbrewerydb.org/breweries"
       cy.selectExternalDatasource(datasource)
-      cy.createRestQuery("GET", restUrl, "/breweries")
+      cy.createRestQuery("GET", restUrl, "breweries")
       cy.reload()
-      cy.contains(".nav-item-content", "/breweries", { timeout: 2000 }).click()
+      cy.contains(".nav-item-content", "breweries", { timeout: 10000 }).click()
       cy.contains(interact.SPECTRUM_TABS_ITEM, "Transformer", { timeout: 5000 }).click({ force: true })
       // Clear the code box and add "test"
       cy.get(interact.CODEMIRROR_TEXTAREA)

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.2.10-alpha.12",
+  "version": "2.2.12-alpha.21",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -71,10 +71,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.2.10-alpha.12",
+    "@budibase/bbui": "2.2.12-alpha.21",
-    "@budibase/client": "2.2.10-alpha.12",
+    "@budibase/client": "2.2.12-alpha.21",
-    "@budibase/frontend-core": "2.2.10-alpha.12",
+    "@budibase/frontend-core": "2.2.12-alpha.21",
-    "@budibase/string-templates": "2.2.10-alpha.12",
+    "@budibase/string-templates": "2.2.12-alpha.21",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte

@@ -12,6 +12,10 @@
       name: "JSON",
       key: "json",
     },
+    {
+      name: "JSON with Schema",
+      key: "jsonWithSchema",
+    },
   ]
 
   export let view
@@ -24,7 +28,7 @@
         viewName: view,
         format: exportFormat,
       })
-      download(data, `export.${exportFormat}`)
+      download(data, `export.${exportFormat === "csv" ? "csv" : "json"}`)
     } catch (error) {
       notifications.error(`Unable to export ${exportFormat.toUpperCase()} data`)
     }

packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte

@@ -6,22 +6,22 @@
     Body,
     Layout,
   } from "@budibase/bbui"
-  import TableDataImport from "../../TableNavigator/TableDataImport.svelte"
+  import TableDataImport from "../../TableNavigator/ExistingTableDataImport.svelte"
   import { API } from "api"
   import { createEventDispatcher } from "svelte"
 
   const dispatch = createEventDispatcher()
 
   export let tableId
-  let dataImport
+  let rows = []
-
+  let allValid = false
-  $: valid = dataImport?.csvString != null && dataImport?.valid
+  let displayColumn = null
 
   async function importData() {
     try {
       await API.importTableData({
         tableId,
-        data: dataImport,
+        rows,
       })
       notifications.success("Rows successfully imported")
     } catch (error) {
@@ -37,14 +37,14 @@
   title="Import Data"
   confirmText="Import"
   onConfirm={importData}
-  disabled={!valid}
+  disabled={!allValid}
 >
   <Body size="S">
-    Import rows to an existing table from a CSV. Only columns from the CSV which
+    Import rows to an existing table from a CSV or JSON file. Only columns from
-    exist in the table will be imported.
+    the file which exist in the table will be imported.
   </Body>
   <Layout gap="XS" noPadding>
-    <Label grey extraSmall>CSV to import</Label>
+    <Label grey extraSmall>CSV or JSON file to import</Label>
-    <TableDataImport bind:dataImport bind:existingTableId={tableId} />
+    <TableDataImport {tableId} bind:rows bind:allValid bind:displayColumn />
   </Layout>
 </ModalContent>

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -340,9 +340,7 @@
   {:else if isManyToOne && toTable}
     <Select
       label={`Foreign Key (${toTable?.name})`}
-      options={Object.keys(toTable?.schema).filter(
+      options={Object.keys(toTable?.schema)}
-        field => toTable?.primary.indexOf(field) === -1
-      )}
       on:change={() => ($touched.foreign = true)}
       bind:error={errors.foreign}
       bind:value={fromRelationship.fieldName}

packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte

@@ -0,0 +1,251 @@
+<script>
+  import { Select } from "@budibase/bbui"
+  import { FIELDS } from "constants/backend"
+  import { API } from "api"
+  import { parseFile } from "./utils"
+
+  let error = null
+  let fileName = null
+  let fileType = null
+
+  let loading = false
+  let validation = {}
+  let validateHash = ""
+  let schema = null
+  let invalidColumns = []
+
+  export let tableId = null
+  export let rows = []
+  export let allValid = false
+
+  const typeOptions = [
+    {
+      label: "Text",
+      value: FIELDS.STRING.type,
+    },
+    {
+      label: "Number",
+      value: FIELDS.NUMBER.type,
+    },
+    {
+      label: "Date",
+      value: FIELDS.DATETIME.type,
+    },
+    {
+      label: "Options",
+      value: FIELDS.OPTIONS.type,
+    },
+    {
+      label: "Multi-select",
+      value: FIELDS.ARRAY.type,
+    },
+    {
+      label: "Barcode/QR",
+      value: FIELDS.BARCODEQR.type,
+    },
+    {
+      label: "Long Form Text",
+      value: FIELDS.LONGFORM.type,
+    },
+  ]
+
+  $: {
+    schema = fetchSchema(tableId)
+  }
+
+  async function fetchSchema(tableId) {
+    try {
+      const definition = await API.fetchTableDefinition(tableId)
+      schema = definition.schema
+    } catch (e) {
+      error = e
+    }
+  }
+
+  async function handleFile(e) {
+    loading = true
+    error = null
+    validation = {}
+
+    try {
+      const response = await parseFile(e)
+      rows = response.rows
+      fileName = response.fileName
+      fileType = response.fileType
+    } catch (e) {
+      loading = false
+      error = e
+    }
+  }
+
+  async function validate(rows) {
+    loading = true
+    error = null
+    validation = {}
+    allValid = false
+
+    try {
+      if (rows.length > 0) {
+        const response = await API.validateExistingTableImport({
+          rows,
+          tableId,
+        })
+
+        validation = response.schemaValidation
+        invalidColumns = response.invalidColumns
+        allValid = response.allValid
+      }
+    } catch (e) {
+      error = e.message
+    }
+
+    loading = false
+  }
+
+  $: {
+    // binding in consumer is causing double renders here
+    const newValidateHash = JSON.stringify(rows)
+
+    if (newValidateHash !== validateHash) {
+      validate(rows)
+    }
+
+    validateHash = newValidateHash
+  }
+</script>
+
+<div class="dropzone">
+  <input
+    disabled={!schema || loading}
+    id="file-upload"
+    accept="text/csv,application/json"
+    type="file"
+    on:change={handleFile}
+  />
+  <label for="file-upload" class:uploaded={rows.length > 0}>
+    {#if loading}
+      loading...
+    {:else if error}
+      error: {error}
+    {:else if fileName}
+      {fileName}
+    {:else}
+      Upload
+    {/if}
+  </label>
+</div>
+{#if fileName && Object.keys(validation).length === 0}
+  <p>No valid fields, try another file</p>
+{:else if rows.length > 0 && !error}
+  <div class="schema-fields">
+    {#each Object.keys(validation) as name}
+      <div class="field">
+        <span>{name}</span>
+        <Select
+          value={schema[name]?.type}
+          options={typeOptions}
+          placeholder={null}
+          getOptionLabel={option => option.label}
+          getOptionValue={option => option.value}
+          disabled
+        />
+        <span
+          class={loading || validation[name]
+            ? "fieldStatusSuccess"
+            : "fieldStatusFailure"}
+        >
+          {validation[name] ? "Success" : "Failure"}
+        </span>
+      </div>
+    {/each}
+  </div>
+  {#if invalidColumns.length > 0}
+    <p class="spectrum-FieldLabel spectrum-FieldLabel--sizeM">
+      The following columns are present in the data you wish to import, but do
+      not match the schema of this table and will be ignored.
+    </p>
+    <ul class="ignoredList">
+      {#each invalidColumns as column}
+        <li>{column}</li>
+      {/each}
+    </ul>
+  {/if}
+{/if}
+
+<style>
+  .dropzone {
+    text-align: center;
+    display: flex;
+    align-items: center;
+    flex-direction: column;
+    border-radius: 10px;
+    transition: all 0.3s;
+  }
+
+  input {
+    display: none;
+  }
+
+  label {
+    font-family: var(--font-sans);
+    cursor: pointer;
+    font-weight: 600;
+    box-sizing: border-box;
+    overflow: hidden;
+    border-radius: var(--border-radius-s);
+    color: var(--ink);
+    padding: var(--spacing-m) var(--spacing-l);
+    transition: all 0.2s ease 0s;
+    display: inline-flex;
+    text-rendering: optimizeLegibility;
+    min-width: auto;
+    outline: none;
+    font-feature-settings: "case" 1, "rlig" 1, "calt" 0;
+    -webkit-box-align: center;
+    user-select: none;
+    flex-shrink: 0;
+    align-items: center;
+    justify-content: center;
+    width: 100%;
+    background-color: var(--grey-2);
+    font-size: var(--font-size-xs);
+    line-height: normal;
+    border: var(--border-transparent);
+  }
+
+  .uploaded {
+    color: var(--blue);
+  }
+
+  .schema-fields {
+    margin-top: var(--spacing-xl);
+  }
+
+  .field {
+    display: grid;
+    grid-template-columns: 2fr 2fr 1fr auto;
+    margin-top: var(--spacing-m);
+    align-items: center;
+    grid-gap: var(--spacing-m);
+    font-size: var(--spectrum-global-dimension-font-size-75);
+  }
+
+  .fieldStatusSuccess {
+    color: var(--green);
+    justify-self: center;
+    font-weight: 600;
+  }
+
+  .fieldStatusFailure {
+    color: var(--red);
+    justify-self: center;
+    font-weight: 600;
+  }
+
+  .ignoredList {
+    margin: 0;
+    padding: 0;
+    list-style: none;
+    font-size: var(--spectrum-global-dimension-font-size-75);
+  }
+</style>

packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte

@@ -1,107 +1,21 @@
 <script>
-  import { Select, InlineAlert, notifications } from "@budibase/bbui"
+  import { Select } from "@budibase/bbui"
   import { FIELDS } from "constants/backend"
   import { API } from "api"
+  import { parseFile } from "./utils"
 
-  const BYTES_IN_MB = 1000000
+  let error = null
-  const FILE_SIZE_LIMIT = BYTES_IN_MB * 5
+  let fileName = null
+  let fileType = null
 
-  export let files = []
+  let loading = false
-  export let dataImport = {
+  let validation = {}
-    valid: true,
+  let validateHash = ""
-    schema: {},
-  }
-  export let existingTableId
 
-  let csvString = undefined
+  export let rows = []
-  let primaryDisplay = undefined
+  export let schema = {}
-  let schema = {}
+  export let allValid = false
-  let fields = []
+  export let displayColumn = null
-  let hasValidated = false
-
-  $: valid =
-    !schema ||
-    (fields.every(column => schema[column].success) &&
-      (!hasValidated || Object.keys(schema).length > 0))
-  $: dataImport = {
-    valid,
-    schema: buildTableSchema(schema),
-    csvString,
-    primaryDisplay,
-  }
-  $: noFieldsError = existingTableId
-    ? "No columns in CSV match existing table schema"
-    : "Could not find any columns to import"
-
-  function buildTableSchema(schema) {
-    const tableSchema = {}
-    for (let key in schema) {
-      const type = schema[key].type
-
-      if (type === "omit") continue
-
-      tableSchema[key] = {
-        name: key,
-        type,
-        constraints: FIELDS[type.toUpperCase()].constraints,
-      }
-    }
-    return tableSchema
-  }
-
-  async function validateCSV() {
-    try {
-      const parseResult = await API.validateTableCSV({
-        csvString,
-        schema: schema || {},
-        tableId: existingTableId,
-      })
-      schema = parseResult?.schema
-      fields = Object.keys(schema || {}).filter(
-        key => schema[key].type !== "omit"
-      )
-
-      // Check primary display is valid
-      if (!primaryDisplay || fields.indexOf(primaryDisplay) === -1) {
-        primaryDisplay = fields[0]
-      }
-
-      hasValidated = true
-    } catch (error) {
-      notifications.error("CSV Invalid, please try another CSV file")
-    }
-  }
-
-  async function handleFile(evt) {
-    const fileArray = Array.from(evt.target.files)
-    if (fileArray.some(file => file.size >= FILE_SIZE_LIMIT)) {
-      notifications.error(
-        `Files cannot exceed ${
-          FILE_SIZE_LIMIT / BYTES_IN_MB
-        }MB. Please try again with smaller files.`
-      )
-      return
-    }
-
-    // Read CSV as plain text to upload alongside schema
-    let reader = new FileReader()
-    reader.addEventListener("load", function (e) {
-      csvString = e.target.result
-      files = fileArray
-      validateCSV()
-    })
-    reader.readAsText(fileArray[0])
-  }
-
-  async function omitColumn(columnName) {
-    schema[columnName].type = "omit"
-    await validateCSV()
-  }
-
-  const handleTypeChange = column => evt => {
-    schema[column].type = evt.detail
-    validateCSV()
-  }
 
   const typeOptions = [
     {
@@ -133,54 +47,114 @@
       value: FIELDS.LONGFORM.type,
     },
   ]
+
+  async function handleFile(e) {
+    loading = true
+    error = null
+    validation = {}
+
+    try {
+      const response = await parseFile(e)
+      rows = response.rows
+      schema = response.schema
+      fileName = response.fileName
+      fileType = response.fileType
+    } catch (e) {
+      loading = false
+      error = e
+    }
+  }
+
+  async function validate(rows, schema) {
+    loading = true
+    error = null
+    validation = {}
+    allValid = false
+
+    try {
+      if (rows.length > 0) {
+        const response = await API.validateNewTableImport({ rows, schema })
+        validation = response.schemaValidation
+        allValid = response.allValid
+      }
+    } catch (e) {
+      error = e.message
+    }
+
+    loading = false
+  }
+
+  $: {
+    // binding in consumer is causing double renders here
+    const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
+
+    if (newValidateHash !== validateHash) {
+      validate(rows, schema)
+    }
+
+    validateHash = newValidateHash
+  }
 </script>
 
 <div class="dropzone">
-  <input id="file-upload" accept=".csv" type="file" on:change={handleFile} />
+  <input
-  <label for="file-upload" class:uploaded={files[0]}>
+    disabled={loading}
-    {#if files[0]}{files[0].name}{:else}Upload{/if}
+    id="file-upload"
+    accept="text/csv,application/json"
+    type="file"
+    on:change={handleFile}
+  />
+  <label for="file-upload" class:uploaded={rows.length > 0}>
+    {#if loading}
+      loading...
+    {:else if error}
+      error: {error}
+    {:else if fileName}
+      {fileName}
+    {:else}
+      Upload
+    {/if}
   </label>
 </div>
-{#if fields.length}
+{#if rows.length > 0 && !error}
   <div class="schema-fields">
-    {#each fields as columnName}
+    {#each Object.values(schema) as column}
       <div class="field">
-        <span>{columnName}</span>
+        <span>{column.name}</span>
         <Select
-          bind:value={schema[columnName].type}
+          bind:value={column.type}
-          on:change={handleTypeChange(columnName)}
+          on:change={e => (column.type = e.detail)}
           options={typeOptions}
           placeholder={null}
           getOptionLabel={option => option.label}
           getOptionValue={option => option.value}
-          disabled={!!existingTableId}
+          disabled={loading}
         />
-        <span class="field-status" class:error={!schema[columnName].success}>
+        <span
-          {schema[columnName].success ? "Success" : "Failure"}
+          class={loading || validation[column.name]
+            ? "fieldStatusSuccess"
+            : "fieldStatusFailure"}
+        >
+          {validation[column.name] ? "Success" : "Failure"}
         </span>
         <i
-          class="omit-button ri-close-circle-fill"
+          class={`omit-button ri-close-circle-fill ${
-          on:click={() => omitColumn(columnName)}
+            loading ? "omit-button-disabled" : ""
+          }`}
+          on:click={() => {
+            delete schema[column.name]
+            schema = schema
+          }}
         />
       </div>
     {/each}
   </div>
-  {#if !existingTableId}
+  <div class="display-column">
-    <div class="display-column">
+    <Select
-      <Select
+      label="Display Column"
-        label="Display Column"
+      bind:value={displayColumn}
-        bind:value={primaryDisplay}
+      options={Object.keys(schema)}
-        options={fields}
+      sort
-        sort
-      />
-    </div>
-  {/if}
-{:else if hasValidated}
-  <div>
-    <InlineAlert
-      header="Invalid CSV"
-      bind:message={noFieldsError}
-      type="error"
     />
   </div>
 {/if}
@@ -195,28 +169,10 @@
     transition: all 0.3s;
   }
 
-  .field-status {
+  input {
-    color: var(--green);
-    justify-self: center;
-    font-weight: 600;
-  }
-
-  .error {
-    color: var(--red);
-  }
-
-  .uploaded {
-    color: var(--blue);
-  }
-
-  input[type="file"] {
     display: none;
   }
 
-  .schema-fields {
-    margin-top: var(--spacing-xl);
-  }
-
   label {
     font-family: var(--font-sans);
     cursor: pointer;
@@ -244,11 +200,12 @@
     border: var(--border-transparent);
   }
 
-  .omit-button {
+  .uploaded {
-    font-size: 1.2em;
+    color: var(--blue);
-    color: var(--grey-7);
+  }
-    cursor: pointer;
+
-    justify-self: flex-end;
+  .schema-fields {
+    margin-top: var(--spacing-xl);
   }
 
   .field {
@@ -260,6 +217,30 @@
     font-size: var(--spectrum-global-dimension-font-size-75);
   }
 
+  .fieldStatusSuccess {
+    color: var(--green);
+    justify-self: center;
+    font-weight: 600;
+  }
+
+  .fieldStatusFailure {
+    color: var(--red);
+    justify-self: center;
+    font-weight: 600;
+  }
+
+  .omit-button {
+    font-size: 1.2em;
+    color: var(--grey-7);
+    cursor: pointer;
+    justify-self: flex-end;
+  }
+
+  .omit-button-disabled {
+    pointer-events: none;
+    opacity: 70%;
+  }
+
   .display-column {
     margin-top: var(--spacing-xl);
   }

packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte

@@ -29,18 +29,27 @@
     : BUDIBASE_INTERNAL_DB_ID
 
   export let name
-  let dataImport
   let error = ""
   let autoColumns = getAutoColumnInformation()
+  let schema = {}
+  let rows = []
+  let allValid = false
+  let displayColumn = null
 
-  function addAutoColumns(tableName, schema) {
+  function getAutoColumns() {
-    for (let [subtype, col] of Object.entries(autoColumns)) {
+    const selectedAutoColumns = {}
-      if (!col.enabled) {
+
-        continue
+    Object.entries(autoColumns).forEach(([subtype, column]) => {
+      if (column.enabled) {
+        selectedAutoColumns[column.name] = buildAutoColumn(
+          name,
+          column.name,
+          subtype
+        )
       }
-      schema[col.name] = buildAutoColumn(tableName, col.name, subtype)
+    })
-    }
+
-    return schema
+    return selectedAutoColumns
   }
 
   function checkValid(evt) {
@@ -55,15 +64,15 @@
   async function saveTable() {
     let newTable = {
       name,
-      schema: addAutoColumns(name, dataImport.schema || {}),
+      schema: { ...schema, ...getAutoColumns() },
-      dataImport,
+      rows,
       type: "internal",
       sourceId: targetDatasourceId,
     }
 
     // Only set primary display if defined
-    if (dataImport.primaryDisplay && dataImport.primaryDisplay.length) {
+    if (displayColumn && displayColumn.length) {
-      newTable.primaryDisplay = dataImport.primaryDisplay
+      newTable.primaryDisplay = displayColumn
     }
 
     // Create table
@@ -90,7 +99,7 @@
   title="Create Table"
   confirmText="Create"
   onConfirm={saveTable}
-  disabled={error || !name || (dataImport && !dataImport.valid)}
+  disabled={error || !name || !allValid}
 >
   <Input
     data-cy="table-name-input"
@@ -117,8 +126,10 @@
   </div>
   <div>
     <Layout gap="XS" noPadding>
-      <Label grey extraSmall>Create Table from CSV (Optional)</Label>
+      <Label grey extraSmall
-      <TableDataImport bind:dataImport />
+        >Create a Table from a CSV or JSON file (Optional)</Label
+      >
+      <TableDataImport bind:rows bind:schema bind:allValid bind:displayColumn />
     </Layout>
   </div>
 </ModalContent>

packages/builder/src/components/backend/TableNavigator/utils.js

@@ -0,0 +1,71 @@
+import { API } from "api"
+import { FIELDS } from "constants/backend"
+
+const BYTES_IN_MB = 1000000
+const FILE_SIZE_LIMIT = BYTES_IN_MB * 5
+
+const getDefaultSchema = rows => {
+  const newSchema = {}
+
+  rows.forEach(row => {
+    Object.keys(row).forEach(column => {
+      newSchema[column] = {
+        name: column,
+        type: "string",
+        constraints: FIELDS["STRING"].constraints,
+      }
+    })
+  })
+
+  return newSchema
+}
+
+export const parseFile = e => {
+  return new Promise((resolve, reject) => {
+    const file = Array.from(e.target.files)[0]
+
+    if (file.size >= FILE_SIZE_LIMIT) {
+      reject("file too large")
+      return
+    }
+
+    let reader = new FileReader()
+
+    const resolveRows = (rows, schema = null) => {
+      resolve({
+        rows,
+        schema: schema ?? getDefaultSchema(rows),
+        fileName: file.name,
+        fileType: file.type,
+      })
+    }
+
+    reader.addEventListener("load", function (e) {
+      const fileData = e.target.result
+
+      if (file.type === "text/csv") {
+        API.csvToJson(fileData)
+          .then(rows => {
+            resolveRows(rows)
+          })
+          .catch(() => {
+            reject("can't convert csv to json")
+          })
+      } else if (file.type === "application/json") {
+        const parsedFileData = JSON.parse(fileData)
+
+        if (Array.isArray(parsedFileData)) {
+          resolveRows(parsedFileData)
+        } else if (typeof parsedFileData === "object") {
+          resolveRows(parsedFileData.rows, parsedFileData.schema)
+        } else {
+          reject("invalid json format")
+        }
+      } else {
+        reject("invalid file type")
+      }
+    })
+
+    reader.readAsText(file)
+  })
+}

packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte

@@ -144,6 +144,11 @@
     drawer.show()
   }
 
+  const getQueryValue = queries => {
+    value = queries.find(q => q._id === value._id) || value
+    return value
+  }
+
   const saveQueryParams = () => {
     handleSelected({
       ...value,
@@ -175,7 +180,7 @@
           {/if}
           <IntegrationQueryEditor
             height={200}
-            query={value}
+            query={getQueryValue(queries)}
             schema={fetchQueryDefinition(value)}
             datasource={getQueryDatasource(value)}
             editable={false}

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -117,10 +117,17 @@
     )
   }
 
+  const cleanUrl = inputUrl =>
+    url
+      ?.replace(/(http)|(https)|[{}:]/g, "")
+      ?.replaceAll(".", "_")
+      ?.replaceAll("/", " ")
+      ?.trim() || inputUrl
+
   function checkQueryName(inputUrl = null) {
     if (query && (!query.name || query.flags.urlName)) {
       query.flags.urlName = true
-      query.name = url || inputUrl
+      query.name = cleanUrl(inputUrl)
     }
   }
 

packages/builder/src/components/integration/index.svelte

@@ -105,6 +105,7 @@
       {#if !query.fields.steps?.length}
         <div class="controls">
           <Button
+            disabled={!editable}
             secondary
             slot="buttons"
             on:click={() => {
@@ -131,6 +132,7 @@
                       {#if index > 0}
                         <ActionButton
                           quiet
+                          disabled={!editable}
                           on:click={() => {
                             updateEditorsOnSwap(index, index - 1)
                             const target = query.fields.steps[index - 1].key
@@ -144,6 +146,7 @@
                       {#if index < query.fields.steps.length - 1}
                         <ActionButton
                           quiet
+                          disabled={!editable}
                           on:click={() => {
                             updateEditorsOnSwap(index, index + 1)
                             const target = query.fields.steps[index + 1].key
@@ -156,6 +159,7 @@
                       {/if}
                     </div>
                     <ActionButton
+                      disabled={!editable}
                       on:click={() => {
                         updateEditorsOnDelete(index)
                         query.fields.steps.splice(index, 1)
@@ -169,6 +173,7 @@
                   <div class="fields">
                     <div class="block-field">
                       <Select
+                        disabled={!editable}
                         value={step.key}
                         options={schema.steps.map(s => s.key)}
                         on:change={({ detail }) => {
@@ -178,6 +183,7 @@
                       <Editor
                         bind:this={stepEditors[index]}
                         editorHeight={height / 2}
+                        readOnly={!editable}
                         mode="json"
                         value={typeof step.value === "string"
                           ? step.value
@@ -194,9 +200,11 @@
             <div class="separator" />
             {#if index === query.fields.steps.length - 1}
               <Icon
+                disabled={!editable}
                 hoverable
                 name="AddCircle"
                 size="S"
+                readOnly={!editable}
                 on:click={() => {
                   query.fields.steps = [
                     ...query.fields.steps,

packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte

@@ -26,7 +26,7 @@
     [PluginSource.FILE]: [opt("File Upload")],
   }
   let file
-  let source = PluginSource.URL
+  let source = PluginSource.GITHUB
   let dynamicValues = {}
 
   let validation

packages/builder/src/pages/builder/portal/overview/_components/AccessTab.svelte

@@ -68,7 +68,8 @@
 
   async function updateUserRole(role, user) {
     user.roles[fixedAppId] = role
-    await users.save(user)
+    const response = await users.save(user)
+    user._rev = response._rev
   }
 
   async function updateGroupRole(role, group) {

packages/builder/yarn.lock

@@ -4279,17 +4279,10 @@ json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1:
   resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
   integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=
 
-json5@^2.1.2:
+json5@^2.1.2, json5@^2.2.1:
-  version "2.2.0"
+  version "2.2.3"
-  resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.0.tgz#2dfefe720c6ba525d9ebd909950f0515316c89a3"
+  resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283"
-  integrity sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA==
+  integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==
-  dependencies:
-    minimist "^1.2.5"
-
-json5@^2.2.1:
-  version "2.2.1"
-  resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.1.tgz#655d50ed1e6f95ad1a3caababd2b0efda10b395c"
-  integrity sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==
 
 jsonfile@^4.0.0:
   version "4.0.0"
@@ -4320,7 +4313,7 @@ jsprim@^2.0.2:
 kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0:
   version "3.2.2"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
-  integrity sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=
+  integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==
   dependencies:
     is-buffer "^1.1.5"