Merge branch 'develop' of github.com:Budibase/budibase into feature/app-backups

hosting/proxy/10-listen-on-ipv6-by-default.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# vim:sw=4:ts=4:et
+
+set -e
+
+ME=$(basename $0)
+NGINX_CONF_FILE="/etc/nginx/nginx.conf"
+DEFAULT_CONF_FILE="/etc/nginx/conf.d/default.conf"
+
+# check if we have ipv6 available
+if [ ! -f "/proc/net/if_inet6" ]; then
+    # ipv6 not available so delete lines from nginx conf
+    if [ -f "$NGINX_CONF_FILE" ]; then
+        sed -i '/listen  \[::\]/d' $NGINX_CONF_FILE
+    fi
+    if [ -f "$DEFAULT_CONF_FILE" ]; then
+        sed -i '/listen  \[::\]/d' $DEFAULT_CONF_FILE
+    fi
+    echo "$ME: info: ipv6 not available so delete lines from nginx conf"
+else
+    echo "$ME: info: ipv6 is available so no need to delete lines from nginx conf"
+fi
+
+exit 0

hosting/proxy/Dockerfile

@@ -5,7 +5,7 @@ FROM nginx:latest
 # override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d
 ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
 COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template
-
+COPY 10-listen-on-ipv6-by-default.sh /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
 # Error handling
 COPY error.html /usr/share/nginx/html/error.html
 

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.0.30-alpha.0",
+  "version": "2.0.30-alpha.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.0.30-alpha.0",
+  "version": "2.0.30-alpha.3",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -20,7 +20,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/types": "2.0.30-alpha.0",
+    "@budibase/types": "2.0.30-alpha.3",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-sdk": "2.1030.0",
@@ -62,6 +62,7 @@
     ]
   },
   "devDependencies": {
+    "@types/chance": "1.1.3",
     "@types/jest": "27.5.1",
     "@types/koa": "2.0.52",
     "@types/lodash": "4.14.180",
@@ -72,6 +73,7 @@
     "@types/semver": "7.3.7",
     "@types/tar-fs": "2.0.1",
     "@types/uuid": "8.3.4",
+    "chance": "1.1.3",
     "ioredis-mock": "5.8.0",
     "jest": "27.5.1",
     "koa": "2.7.0",

packages/backend-core/src/index.ts

@@ -37,6 +37,7 @@ const core = {
   db,
   ...dbConstants,
   redis,
+  locks: redis.redlock,
   objectStore,
   utils,
   users,

packages/backend-core/src/pkg/redis.ts

@@ -3,9 +3,11 @@
 import Client from "../redis"
 import utils from "../redis/utils"
 import clients from "../redis/init"
+import * as redlock from "../redis/redlock"
 
 export = {
   Client,
   utils,
   clients,
+  redlock,
 }

packages/backend-core/src/redis/init.js

@@ -1,27 +1,23 @@
 const Client = require("./index")
 const utils = require("./utils")
-const { getRedlock } = require("./redlock")
 
-let userClient, sessionClient, appClient, cacheClient, writethroughClient
+let userClient,
-let migrationsRedlock
+  sessionClient,
-
+  appClient,
-// turn retry off so that only one instance can ever hold the lock
+  cacheClient,
-const migrationsRedlockConfig = { retryCount: 0 }
+  writethroughClient,
+  lockClient
 
 async function init() {
   userClient = await new Client(utils.Databases.USER_CACHE).init()
   sessionClient = await new Client(utils.Databases.SESSIONS).init()
   appClient = await new Client(utils.Databases.APP_METADATA).init()
   cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init()
+  lockClient = await new Client(utils.Databases.LOCKS).init()
   writethroughClient = await new Client(
     utils.Databases.WRITE_THROUGH,
     utils.SelectableDatabases.WRITE_THROUGH
   ).init()
-  // pass the underlying ioredis client to redlock
-  migrationsRedlock = getRedlock(
-    cacheClient.getClient(),
-    migrationsRedlockConfig
-  )
 }
 
 process.on("exit", async () => {
@@ -30,6 +26,7 @@ process.on("exit", async () => {
   if (appClient) await appClient.finish()
   if (cacheClient) await cacheClient.finish()
   if (writethroughClient) await writethroughClient.finish()
+  if (lockClient) await lockClient.finish()
 })
 
 module.exports = {
@@ -63,10 +60,10 @@ module.exports = {
     }
     return writethroughClient
   },
-  getMigrationsRedlock: async () => {
+  getLockClient: async () => {
-    if (!migrationsRedlock) {
+    if (!lockClient) {
       await init()
     }
-    return migrationsRedlock
+    return lockClient
   },
 }

packages/backend-core/src/redis/redlock.ts

@@ -1,14 +1,37 @@
-import Redlock from "redlock"
+import Redlock, { Options } from "redlock"
+import { getLockClient } from "./init"
+import { LockOptions, LockType } from "@budibase/types"
+import * as tenancy from "../tenancy"
 
-export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => {
+let noRetryRedlock: Redlock | undefined
-  return new Redlock([redisClient], {
+
+const getClient = async (type: LockType): Promise<Redlock> => {
+  switch (type) {
+    case LockType.TRY_ONCE: {
+      if (!noRetryRedlock) {
+        noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE)
+      }
+      return noRetryRedlock
+    }
+    default: {
+      throw new Error(`Could not get redlock client: ${type}`)
+    }
+  }
+}
+
+export const OPTIONS = {
+  TRY_ONCE: {
+    // immediately throws an error if the lock is already held
+    retryCount: 0,
+  },
+  DEFAULT: {
     // the expected clock drift; for more details
     // see http://redis.io/topics/distlock
     driftFactor: 0.01, // multiplied by lock ttl to determine drift time
 
     // the max number of times Redlock will attempt
     // to lock a resource before erroring
-    retryCount: opts.retryCount,
+    retryCount: 10,
 
     // the time in ms between attempts
     retryDelay: 200, // time in ms
@@ -16,6 +39,45 @@ export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => {
     // the max time in ms randomly added to retries
     // to improve performance under high contention
     // see https://www.awsarchitectureblog.com/2015/03/backoff.html
-    retryJitter: 200, // time in ms
+    retryJitter: 100, // time in ms
-  })
+  },
+}
+
+export const newRedlock = async (opts: Options = {}) => {
+  let options = { ...OPTIONS.DEFAULT, ...opts }
+  const redisWrapper = await getLockClient()
+  const client = redisWrapper.getClient()
+  return new Redlock([client], options)
+}
+
+export const doWithLock = async (opts: LockOptions, task: any) => {
+  const redlock = await getClient(opts.type)
+  let lock
+  try {
+    // aquire lock
+    let name: string = `${tenancy.getTenantId()}_${opts.name}`
+    if (opts.nameSuffix) {
+      name = name + `_${opts.nameSuffix}`
+    }
+    lock = await redlock.lock(name, opts.ttl)
+    // perform locked task
+    return task()
+  } catch (e: any) {
+    // lock limit exceeded
+    if (e.name === "LockError") {
+      if (opts.type === LockType.TRY_ONCE) {
+        // don't throw for try-once locks, they will always error
+        // due to retry count (0) exceeded
+        return
+      } else {
+        throw e
+      }
+    } else {
+      throw e
+    }
+  } finally {
+    if (lock) {
+      await lock.unlock()
+    }
+  }
 }

packages/backend-core/src/redis/utils.js

@@ -28,6 +28,7 @@ exports.Databases = {
   LICENSES: "license",
   GENERIC_CACHE: "data_cache",
   WRITE_THROUGH: "writeThrough",
+  LOCKS: "locks",
 }
 
 /**

packages/backend-core/tests/utilities/structures/accounts.ts

@@ -0,0 +1,23 @@
+import { generator, uuid } from "."
+import { AuthType, CloudAccount, Hosting } from "@budibase/types"
+import * as db from "../../../src/db/utils"
+
+export const cloudAccount = (): CloudAccount => {
+  return {
+    accountId: uuid(),
+    createdAt: Date.now(),
+    verified: true,
+    verificationSent: true,
+    tier: "",
+    email: generator.email(),
+    tenantId: generator.word(),
+    hosting: Hosting.CLOUD,
+    authType: AuthType.PASSWORD,
+    password: generator.word(),
+    tenantName: generator.word(),
+    name: generator.name(),
+    size: "10+",
+    profession: "Software Engineer",
+    budibaseUserId: db.generateGlobalUserID(),
+  }
+}

packages/backend-core/tests/utilities/structures/common.ts

@@ -0,0 +1 @@
+export { v4 as uuid } from "uuid"

packages/backend-core/tests/utilities/structures/index.ts

@@ -1 +1,8 @@
+export * from "./common"
+
+import Chance from "chance"
+export const generator = new Chance()
+
 export * as koa from "./koa"
+export * as accounts from "./accounts"
+export * as licenses from "./licenses"

packages/backend-core/tests/utilities/structures/licenses.ts

@@ -0,0 +1,18 @@
+import { AccountPlan, License, PlanType, Quotas } from "@budibase/types"
+
+const newPlan = (type: PlanType = PlanType.FREE): AccountPlan => {
+  return {
+    type,
+  }
+}
+
+export const newLicense = (opts: {
+  quotas: Quotas
+  planType?: PlanType
+}): License => {
+  return {
+    features: [],
+    quotas: opts.quotas,
+    plan: newPlan(opts.planType),
+  }
+}

packages/backend-core/yarn.lock

@@ -663,6 +663,11 @@
     "@types/connect" "*"
     "@types/node" "*"
 
+"@types/chance@1.1.3":
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/@types/chance/-/chance-1.1.3.tgz#d19fe9391288d60fdccd87632bfc9ab2b4523fea"
+  integrity sha512-X6c6ghhe4/sQh4XzcZWSFaTAUOda38GQHmq9BUanYkOE/EO7ZrkazwKmtsj3xzTjkLWmwULE++23g3d3CCWaWw==
+
 "@types/connect@*":
   version "3.4.35"
   resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1"
@@ -1555,6 +1560,11 @@ chalk@^4.0.0, chalk@^4.1.0:
     ansi-styles "^4.1.0"
     supports-color "^7.1.0"
 
+chance@1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.3.tgz#414f08634ee479c7a316b569050ea20751b82dd3"
+  integrity sha512-XeJsdoVAzDb1WRPRuMBesRSiWpW1uNTo5Fd7mYxPJsAfgX71+jfuCOHOdbyBz2uAUZ8TwKcXgWk3DMedFfJkbg==
+
 char-regex@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.0.30-alpha.0",
+  "version": "2.0.30-alpha.3",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "2.0.30-alpha.0",
+    "@budibase/string-templates": "2.0.30-alpha.3",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.0.30-alpha.0",
+  "version": "2.0.30-alpha.3",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -71,10 +71,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.0.30-alpha.0",
+    "@budibase/bbui": "2.0.30-alpha.3",
-    "@budibase/client": "2.0.30-alpha.0",
+    "@budibase/client": "2.0.30-alpha.3",
-    "@budibase/frontend-core": "2.0.30-alpha.0",
+    "@budibase/frontend-core": "2.0.30-alpha.3",
-    "@budibase/string-templates": "2.0.30-alpha.0",
+    "@budibase/string-templates": "2.0.30-alpha.3",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/pages/builder/portal/settings/upgrade.svelte

@@ -91,7 +91,6 @@
   })
 </script>
 
-{"is adming" + $auth.isAdmin}
 {#if $auth.isAdmin}
   <DeleteLicenseKeyModal
     bind:this={deleteLicenseKeyModal}

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.0.30-alpha.0",
+  "version": "2.0.30-alpha.3",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.0.30-alpha.0",
+    "@budibase/backend-core": "2.0.30-alpha.3",
-    "@budibase/string-templates": "2.0.30-alpha.0",
+    "@budibase/string-templates": "2.0.30-alpha.3",
-    "@budibase/types": "2.0.30-alpha.0",
+    "@budibase/types": "2.0.30-alpha.3",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.0.30-alpha.0",
+  "version": "2.0.30-alpha.3",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.0.30-alpha.0",
+    "@budibase/bbui": "2.0.30-alpha.3",
-    "@budibase/frontend-core": "2.0.30-alpha.0",
+    "@budibase/frontend-core": "2.0.30-alpha.3",
-    "@budibase/string-templates": "2.0.30-alpha.0",
+    "@budibase/string-templates": "2.0.30-alpha.3",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.0.30-alpha.0",
+  "version": "2.0.30-alpha.3",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.0.30-alpha.0",
+    "@budibase/bbui": "2.0.30-alpha.3",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.0.30-alpha.0",
+  "version": "2.0.30-alpha.3",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.0.30-alpha.0",
+  "version": "2.0.30-alpha.3",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.0.30-alpha.0",
+    "@budibase/backend-core": "2.0.30-alpha.3",
-    "@budibase/client": "2.0.30-alpha.0",
+    "@budibase/client": "2.0.30-alpha.3",
-    "@budibase/pro": "2.0.30-alpha.0",
+    "@budibase/pro": "2.0.30-alpha.3",
-    "@budibase/string-templates": "2.0.30-alpha.0",
+    "@budibase/string-templates": "2.0.30-alpha.3",
-    "@budibase/types": "2.0.30-alpha.0",
+    "@budibase/types": "2.0.30-alpha.3",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/controllers/deploy/index.ts

@@ -17,7 +17,6 @@ import {
   getProdAppDB,
   getDevAppDB,
 } from "@budibase/backend-core/context"
-import { quotas } from "@budibase/pro"
 import { events } from "@budibase/backend-core"
 
 // the max time we can wait for an invalidation to complete before considering it failed

packages/server/src/api/controllers/query/index.ts

@@ -56,6 +56,7 @@ const _import = async (ctx: any) => {
       config: {
         url: info.url,
         defaultHeaders: [],
+        rejectUnauthorized: true,
       },
       name: info.name,
     }

packages/server/src/integrations/rest.ts

@@ -78,16 +78,11 @@ const SCHEMA: Integration = {
       default: {},
     },
     rejectUnauthorized: {
+      display: "Reject Unauthorized",
       type: DatasourceFieldType.BOOLEAN,
       default: true,
       required: false,
     },
-    legacyHttpParser: {
-      display: "Legacy HTTP Support",
-      type: DatasourceFieldType.BOOLEAN,
-      required: false,
-      default: false,
-    },
   },
   query: {
     create: {
@@ -397,6 +392,7 @@ class RestIntegration implements IntegrationBase {
       })
     }
 
+    // Deprecated by rejectUnauthorized
     if (this.config.legacyHttpParser) {
       // https://github.com/nodejs/node/issues/43798
       input.extraHttpOptions = { insecureHTTPParser: true }

packages/server/src/integrations/tests/rest.spec.ts

@@ -1,5 +1,4 @@
-jest.mock("node-fetch", () =>
+const mockFetch = jest.fn(() => ({
-  jest.fn(() => ({
   headers: {
     raw: () => {
       return { "content-type": ["application/json"] }
@@ -10,8 +9,8 @@ jest.mock("node-fetch", () =>
     my_next_cursor: 123,
   })),
   text: jest.fn(),
-  }))
+}))
-)
+jest.mock("node-fetch", () => mockFetch)
 import fetch from "node-fetch"
 import { default as RestIntegration } from "../rest"
 const FormData = require("form-data")
@@ -572,4 +571,21 @@ describe("REST Integration", () => {
       })
     })
   })
+
+  it("Attaches custom agent when Reject Unauthorized option is false", async () => {
+    config = new TestConfiguration({
+      url: BASE_URL,
+      rejectUnauthorized: false,
+    })
+    await config.integration.read({})
+
+    const calls: any = mockFetch.mock.calls[0]
+    const url = calls[0]
+    expect(url).toBe(`${BASE_URL}/`)
+
+    const calledConfig = calls[1]
+    expect(calledConfig.method).toBe("GET")
+    expect(calledConfig.headers).toEqual({})
+    expect(calledConfig.agent.options.rejectUnauthorized).toBe(false)
+  })
 })

packages/server/src/migrations/index.ts

@@ -1,5 +1,11 @@
-import { migrations, redis } from "@budibase/backend-core"
+import { locks, migrations } from "@budibase/backend-core"
-import { Migration, MigrationOptions, MigrationName } from "@budibase/types"
+import {
+  Migration,
+  MigrationOptions,
+  MigrationName,
+  LockType,
+  LockName,
+} from "@budibase/types"
 import env from "../environment"
 
 // migration functions
@@ -86,33 +92,14 @@ export const migrate = async (options?: MigrationOptions) => {
 }
 
 const migrateWithLock = async (options?: MigrationOptions) => {
-  // get a new lock client
+  await locks.doWithLock(
-  const redlock = await redis.clients.getMigrationsRedlock()
+    {
-  // lock for 15 minutes
+      type: LockType.TRY_ONCE,
-  const ttl = 1000 * 60 * 15
+      name: LockName.MIGRATIONS,
-
+      ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
-  let migrationLock
+    },
-
+    async () => {
-  // acquire lock
-  try {
-    migrationLock = await redlock.lock("migrations", ttl)
-  } catch (e: any) {
-    if (e.name === "LockError") {
-      return
-    } else {
-      throw e
-    }
-  }
-
-  // run migrations
-  try {
       await migrations.runMigrations(MIGRATIONS, options)
-  } finally {
-    // release lock
-    try {
-      await migrationLock.unlock()
-    } catch (e) {
-      console.error("unable to release migration lock")
-    }
     }
+  )
 }

packages/server/yarn.lock

@@ -1094,12 +1094,12 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.0.30-alpha.0":
+"@budibase/backend-core@2.0.30-alpha.3":
-  version "2.0.30-alpha.0"
+  version "2.0.30-alpha.3"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.0.tgz#aa9277ec26da51f8396fe047ebab39a0dff69c93"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.3.tgz#42235b33676c3357d8cf8e181e619c4c92065fd0"
-  integrity sha512-6+KHAceEZv1SC0qSA4GTNQBAlF1wnr9jfl85D5Tq1O2caD4J7drkIz8D6e0/DVv8zccHT+aUA7hy7YhKxbHAUA==
+  integrity sha512-ixE28ddY8FqvCcaCkqsxUI5UjkFKxFu23MgNARJ8VSpj5NLq6d8qAR+xE0E2pJ58hm5r1KLn2yAZiNXZzu0dzg==
   dependencies:
-    "@budibase/types" "2.0.30-alpha.0"
+    "@budibase/types" "2.0.30-alpha.3"
     "@shopify/jest-koa-mocks" "5.0.1"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-sdk "2.1030.0"
@@ -1180,13 +1180,13 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/pro@2.0.30-alpha.0":
+"@budibase/pro@2.0.30-alpha.3":
-  version "2.0.30-alpha.0"
+  version "2.0.30-alpha.3"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.0.tgz#6f0d9e45bc458e0e423280355c1a339f3143da47"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.3.tgz#ad8f8aa5ced2b5200e05b98e392267b9f84c8177"
-  integrity sha512-8PFdiIHT7wUusa1Gi+KvcpLaJdBAtnAK6bfYGayasnrS9OIasfIBAMgzXRONMAGMvY7PZz84psZVimXSkW7EEA==
+  integrity sha512-Cw7XUGVVbHCxKo3tJFvG5EKUvO0zdOoU/baFArjMiNHAivlZvFVWCsxptIua+qtFkaUpjQneTHihFiQnY9g83A==
   dependencies:
-    "@budibase/backend-core" "2.0.30-alpha.0"
+    "@budibase/backend-core" "2.0.30-alpha.3"
-    "@budibase/types" "2.0.30-alpha.0"
+    "@budibase/types" "2.0.30-alpha.3"
     "@koa/router" "8.0.8"
     joi "17.6.0"
     node-fetch "^2.6.1"
@@ -1209,10 +1209,10 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/types@2.0.30-alpha.0":
+"@budibase/types@2.0.30-alpha.3":
-  version "2.0.30-alpha.0"
+  version "2.0.30-alpha.3"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.0.tgz#fb5e80870b00850601785f1102f64bd22339f4e9"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.3.tgz#cb55bcced75b711cc8a675284fbacaa8ebf1c0f2"
-  integrity sha512-i9c02crPzZPRzn+RXwn2f4H4WCLWhC7HlsiXH2w4Ngk7PJvtcR9Jn7mKBKVcXM/9PAK6d0vWIQidA8XuU0CpZA==
+  integrity sha512-rHeFVuNbSSE4fMnX6uyrM2r47m+neqFXlVNOkhHU9i7KoIcIZbEYInU8CjUFR2da3ruST9ajXjJ5UenX2+MnTg==
 
 "@bull-board/api@3.7.0":
   version "3.7.0"

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "2.0.30-alpha.0",
+  "version": "2.0.30-alpha.3",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/types",
-  "version": "2.0.30-alpha.0",
+  "version": "2.0.30-alpha.3",
   "description": "Budibase types",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/types/src/api/account/index.ts

@@ -1 +1,2 @@
 export * from "./user"
+export * from "./license"

packages/types/src/api/account/license.ts

@@ -0,0 +1,11 @@
+import { QuotaUsage } from "../../documents"
+
+export interface GetLicenseRequest {
+  quotaUsage: QuotaUsage
+}
+
+export interface QuotaTriggeredRequest {
+  percentage: number
+  name: string
+  resetDate?: string
+}

packages/types/src/documents/account/account.ts

@@ -1,4 +1,12 @@
-import { Feature, Hosting, PlanType, Quotas } from "../../sdk"
+import {
+  Feature,
+  Hosting,
+  MonthlyQuotaName,
+  PlanType,
+  Quotas,
+  StaticQuotaName,
+} from "../../sdk"
+import { MonthlyUsage, QuotaUsage, StaticUsage } from "../global"
 
 export interface CreateAccount {
   email: string
@@ -42,6 +50,7 @@ export interface Account extends CreateAccount {
   licenseKey?: string
   licenseKeyActivatedAt?: number
   licenseOverrides?: LicenseOverrides
+  quotaUsage?: QuotaUsage
 }
 
 export interface PasswordAccount extends Account {

packages/types/src/documents/global/quotas.ts

@@ -24,19 +24,34 @@ export interface UsageBreakdown {
   }
 }
 
-export type MonthlyUsage = {
+export type QuotaTriggers = {
+  [key: string]: string | undefined
+}
+
+export interface StaticUsage {
+  [StaticQuotaName.APPS]: number
+  [StaticQuotaName.PLUGINS]: number
+  [StaticQuotaName.USER_GROUPS]: number
+  [StaticQuotaName.ROWS]: number
+  triggers: {
+    [key in StaticQuotaName]?: QuotaTriggers
+  }
+}
+
+export interface MonthlyUsage {
   [MonthlyQuotaName.QUERIES]: number
   [MonthlyQuotaName.AUTOMATIONS]: number
   [MonthlyQuotaName.DAY_PASSES]: number
+  triggers: {
+    [key in MonthlyQuotaName]?: QuotaTriggers
+  }
   breakdown?: {
     [key in BreakdownQuotaName]?: UsageBreakdown
   }
 }
 
 export interface BaseQuotaUsage {
-  usageQuota: {
+  usageQuota: StaticUsage
-    [key in StaticQuotaName]: number
-  }
   monthly: {
     [key: string]: MonthlyUsage
   }
@@ -51,6 +66,13 @@ export interface QuotaUsage extends BaseQuotaUsage {
   }
 }
 
+export type SetUsageValues = {
+  total: number
+  app?: number
+  breakdown?: number
+  triggers?: QuotaTriggers
+}
+
 export type UsageValues = {
   total: number
   app?: number

packages/types/src/sdk/index.ts

@@ -7,3 +7,4 @@ export * from "./datasources"
 export * from "./search"
 export * from "./koa"
 export * from "./auth"
+export * from "./locks"

packages/types/src/sdk/licensing/quota.ts

@@ -61,26 +61,40 @@ export type PlanQuotas = {
   [PlanType.ENTERPRISE]: Quotas
 }
 
-export type Quotas = {
+export type MonthlyQuotas = {
-  [QuotaType.USAGE]: {
-    [QuotaUsageType.MONTHLY]: {
   [MonthlyQuotaName.QUERIES]: Quota
   [MonthlyQuotaName.AUTOMATIONS]: Quota
   [MonthlyQuotaName.DAY_PASSES]: Quota
-    }
+}
-    [QuotaUsageType.STATIC]: {
+
+export type StaticQuotas = {
   [StaticQuotaName.ROWS]: Quota
   [StaticQuotaName.APPS]: Quota
   [StaticQuotaName.USER_GROUPS]: Quota
   [StaticQuotaName.PLUGINS]: Quota
-    }
+}
-  }
+
-  [QuotaType.CONSTANT]: {
+export type ConstantQuotas = {
   [ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota
+}
+
+export type Quotas = {
+  [QuotaType.USAGE]: {
+    [QuotaUsageType.MONTHLY]: MonthlyQuotas
+    [QuotaUsageType.STATIC]: StaticQuotas
   }
+  [QuotaType.CONSTANT]: ConstantQuotas
 }
 
 export interface Quota {
   name: string
   value: number
+  /**
+   * Array of whole numbers (1-100) that dictate the percentage that this quota should trigger
+   * at in relation to the corresponding usage inside budibase.
+   *
+   * Triggering results in a budibase installation sending a request to account-portal,
+   * which can have subsequent effects such as sending emails to users.
+   */
+  triggers: number[]
 }

packages/types/src/sdk/locks.ts

@@ -0,0 +1,31 @@
+export enum LockType {
+  /**
+   * If this lock is already held the attempted operation will not be performed.
+   * No retries will take place and no error will be thrown.
+   */
+  TRY_ONCE = "try_once",
+}
+
+export enum LockName {
+  MIGRATIONS = "migrations",
+  TRIGGER_QUOTA = "trigger_quota",
+}
+
+export interface LockOptions {
+  /**
+   * The lock type determines which client to use
+   */
+  type: LockType
+  /**
+   * The name for the lock
+   */
+  name: LockName
+  /**
+   * The ttl to auto-expire the lock if not unlocked manually
+   */
+  ttl: number
+  /**
+   * The suffix to add to the lock name for additional uniqueness
+   */
+  nameSuffix?: string
+}

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.0.30-alpha.0",
+  "version": "2.0.30-alpha.3",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -36,10 +36,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "2.0.30-alpha.0",
+    "@budibase/backend-core": "2.0.30-alpha.3",
-    "@budibase/pro": "2.0.30-alpha.0",
+    "@budibase/pro": "2.0.30-alpha.3",
-    "@budibase/string-templates": "2.0.30-alpha.0",
+    "@budibase/string-templates": "2.0.30-alpha.3",
-    "@budibase/types": "2.0.30-alpha.0",
+    "@budibase/types": "2.0.30-alpha.3",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",

packages/worker/src/api/controllers/system/tenants.js

@@ -1,58 +0,0 @@
-const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db")
-const { getTenantId } = require("@budibase/backend-core/tenancy")
-const { deleteTenant } = require("@budibase/backend-core/deprovision")
-const { quotas } = require("@budibase/pro")
-
-exports.exists = async ctx => {
-  const tenantId = ctx.request.params
-  ctx.body = {
-    exists: await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => {
-      let exists = false
-      try {
-        const tenantsDoc = await db.get(
-          StaticDatabases.PLATFORM_INFO.docs.tenants
-        )
-        if (tenantsDoc) {
-          exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1
-        }
-      } catch (err) {
-        // if error it doesn't exist
-      }
-      return exists
-    }),
-  }
-}
-
-exports.fetch = async ctx => {
-  ctx.body = await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => {
-    let tenants = []
-    try {
-      const tenantsDoc = await db.get(
-        StaticDatabases.PLATFORM_INFO.docs.tenants
-      )
-      if (tenantsDoc) {
-        tenants = tenantsDoc.tenantIds
-      }
-    } catch (err) {
-      // if error it doesn't exist
-    }
-    return tenants
-  })
-}
-
-exports.delete = async ctx => {
-  const tenantId = getTenantId()
-
-  if (ctx.params.tenantId !== tenantId) {
-    ctx.throw(403, "Unauthorized")
-  }
-
-  try {
-    await deleteTenant(tenantId)
-    await quotas.bustCache()
-    ctx.status = 204
-  } catch (err) {
-    ctx.log.error(err)
-    throw err
-  }
-}

packages/worker/src/api/controllers/system/tenants.ts

@@ -0,0 +1,66 @@
+const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db")
+const { getTenantId } = require("@budibase/backend-core/tenancy")
+const { deleteTenant } = require("@budibase/backend-core/deprovision")
+import { quotas } from "@budibase/pro"
+
+export const exists = async (ctx: any) => {
+  const tenantId = ctx.request.params
+  ctx.body = {
+    exists: await doWithDB(
+      StaticDatabases.PLATFORM_INFO.name,
+      async (db: any) => {
+        let exists = false
+        try {
+          const tenantsDoc = await db.get(
+            StaticDatabases.PLATFORM_INFO.docs.tenants
+          )
+          if (tenantsDoc) {
+            exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1
+          }
+        } catch (err) {
+          // if error it doesn't exist
+        }
+        return exists
+      }
+    ),
+  }
+}
+
+export const fetch = async (ctx: any) => {
+  ctx.body = await doWithDB(
+    StaticDatabases.PLATFORM_INFO.name,
+    async (db: any) => {
+      let tenants = []
+      try {
+        const tenantsDoc = await db.get(
+          StaticDatabases.PLATFORM_INFO.docs.tenants
+        )
+        if (tenantsDoc) {
+          tenants = tenantsDoc.tenantIds
+        }
+      } catch (err) {
+        // if error it doesn't exist
+      }
+      return tenants
+    }
+  )
+}
+
+const _delete = async (ctx: any) => {
+  const tenantId = getTenantId()
+
+  if (ctx.params.tenantId !== tenantId) {
+    ctx.throw(403, "Unauthorized")
+  }
+
+  try {
+    await deleteTenant(tenantId)
+    await quotas.bustCache()
+    ctx.status = 204
+  } catch (err) {
+    ctx.log.error(err)
+    throw err
+  }
+}
+
+export { _delete as delete }

packages/worker/src/migrations/index.ts

@@ -1,5 +1,11 @@
-import { migrations, redis } from "@budibase/backend-core"
+import { migrations, locks } from "@budibase/backend-core"
-import { Migration, MigrationOptions, MigrationName } from "@budibase/types"
+import {
+  Migration,
+  MigrationOptions,
+  MigrationName,
+  LockType,
+  LockName,
+} from "@budibase/types"
 import env from "../environment"
 
 // migration functions
@@ -42,33 +48,14 @@ export const migrate = async (options?: MigrationOptions) => {
 }
 
 const migrateWithLock = async (options?: MigrationOptions) => {
-  // get a new lock client
+  await locks.doWithLock(
-  const redlock = await redis.clients.getMigrationsRedlock()
+    {
-  // lock for 15 minutes
+      type: LockType.TRY_ONCE,
-  const ttl = 1000 * 60 * 15
+      name: LockName.MIGRATIONS,
-
+      ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
-  let migrationLock
+    },
-
+    async () => {
-  // acquire lock
-  try {
-    migrationLock = await redlock.lock("migrations", ttl)
-  } catch (e: any) {
-    if (e.name === "LockError") {
-      return
-    } else {
-      throw e
-    }
-  }
-
-  // run migrations
-  try {
       await migrations.runMigrations(MIGRATIONS, options)
-  } finally {
-    // release lock
-    try {
-      await migrationLock.unlock()
-    } catch (e) {
-      console.error("unable to release migration lock")
-    }
     }
+  )
 }

packages/worker/src/sdk/users/events.ts

@@ -1,7 +1,6 @@
 import env from "../../environment"
 import { events, accounts, tenancy } from "@budibase/backend-core"
 import { User, UserRoles, CloudAccount } from "@budibase/types"
-import { users as pro } from "@budibase/pro"
 
 export const handleDeleteEvents = async (user: any) => {
   await events.user.deleted(user)

packages/worker/yarn.lock

@@ -291,12 +291,12 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.0.30-alpha.0":
+"@budibase/backend-core@2.0.30-alpha.3":
-  version "2.0.30-alpha.0"
+  version "2.0.30-alpha.3"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.0.tgz#aa9277ec26da51f8396fe047ebab39a0dff69c93"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.3.tgz#42235b33676c3357d8cf8e181e619c4c92065fd0"
-  integrity sha512-6+KHAceEZv1SC0qSA4GTNQBAlF1wnr9jfl85D5Tq1O2caD4J7drkIz8D6e0/DVv8zccHT+aUA7hy7YhKxbHAUA==
+  integrity sha512-ixE28ddY8FqvCcaCkqsxUI5UjkFKxFu23MgNARJ8VSpj5NLq6d8qAR+xE0E2pJ58hm5r1KLn2yAZiNXZzu0dzg==
   dependencies:
-    "@budibase/types" "2.0.30-alpha.0"
+    "@budibase/types" "2.0.30-alpha.3"
     "@shopify/jest-koa-mocks" "5.0.1"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-sdk "2.1030.0"
@@ -327,21 +327,21 @@
     uuid "8.3.2"
     zlib "1.0.5"
 
-"@budibase/pro@2.0.30-alpha.0":
+"@budibase/pro@2.0.30-alpha.3":
-  version "2.0.30-alpha.0"
+  version "2.0.30-alpha.3"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.0.tgz#6f0d9e45bc458e0e423280355c1a339f3143da47"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.3.tgz#ad8f8aa5ced2b5200e05b98e392267b9f84c8177"
-  integrity sha512-8PFdiIHT7wUusa1Gi+KvcpLaJdBAtnAK6bfYGayasnrS9OIasfIBAMgzXRONMAGMvY7PZz84psZVimXSkW7EEA==
+  integrity sha512-Cw7XUGVVbHCxKo3tJFvG5EKUvO0zdOoU/baFArjMiNHAivlZvFVWCsxptIua+qtFkaUpjQneTHihFiQnY9g83A==
   dependencies:
-    "@budibase/backend-core" "2.0.30-alpha.0"
+    "@budibase/backend-core" "2.0.30-alpha.3"
-    "@budibase/types" "2.0.30-alpha.0"
+    "@budibase/types" "2.0.30-alpha.3"
     "@koa/router" "8.0.8"
     joi "17.6.0"
     node-fetch "^2.6.1"
 
-"@budibase/types@2.0.30-alpha.0":
+"@budibase/types@2.0.30-alpha.3":
-  version "2.0.30-alpha.0"
+  version "2.0.30-alpha.3"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.0.tgz#fb5e80870b00850601785f1102f64bd22339f4e9"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.3.tgz#cb55bcced75b711cc8a675284fbacaa8ebf1c0f2"
-  integrity sha512-i9c02crPzZPRzn+RXwn2f4H4WCLWhC7HlsiXH2w4Ngk7PJvtcR9Jn7mKBKVcXM/9PAK6d0vWIQidA8XuU0CpZA==
+  integrity sha512-rHeFVuNbSSE4fMnX6uyrM2r47m+neqFXlVNOkhHU9i7KoIcIZbEYInU8CjUFR2da3ruST9ajXjJ5UenX2+MnTg==
 
 "@cspotcode/source-map-consumer@0.8.0":
   version "0.8.0"