Merge branch 'develop' into fix/sync-apps-reset-rows-quotas

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.27-alpha.8",
+  "version": "1.0.27-alpha.12",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/README.md

@@ -4,7 +4,7 @@ This library contains core functionality, like auth and security features
 which are shared between backend services.
 
 #### Note about top level JS files
-For the purposes of being able to do say `require("@budibase/auth/permissions")` we need to
+For the purposes of being able to do say `require("@budibase/backend-core/permissions")` we need to
 specify the exports at the top-level of the module.
 
 For these files they should be limited to a single `require` of the file that should

packages/backend-core/package.json

@@ -1,7 +1,7 @@
 {
-  "name": "@budibase/auth",
-  "version": "1.0.27-alpha.8",
-  "description": "Authentication middlewares for budibase builder and apps",
+  "name": "@budibase/backend-core",
+  "version": "1.0.27-alpha.12",
+  "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",
   "license": "GPL-3.0",

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.27-alpha.8",
+  "version": "1.0.27-alpha.12",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/yarn.lock

@@ -2076,9 +2076,9 @@ postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.27:
     supports-color "^6.1.0"
 
 postcss@^8.2.9:
-  version "8.2.10"
-  resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.2.10.tgz#ca7a042aa8aff494b334d0ff3e9e77079f6f702b"
-  integrity sha512-b/h7CPV7QEdrqIxtAf2j31U5ef05uBDuvoXv6L51Q4rcS1jdlXAVKJv+atCFdUXYl9dyTHGyoMzIepwowRJjFw==
+  version "8.2.13"
+  resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.2.13.tgz#dbe043e26e3c068e45113b1ed6375d2d37e2129f"
+  integrity sha512-FCE5xLH+hjbzRdpbRb1IMCvPv9yZx2QnDarBEYSN0N0HYk+TcXsEhwdFcFb+SRWOKzKGErhIEbBK2ogyLdTtfQ==
   dependencies:
     colorette "^1.2.2"
     nanoid "^3.1.22"

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.27-alpha.8",
+  "version": "1.0.27-alpha.12",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.27-alpha.8",
-    "@budibase/client": "^1.0.27-alpha.8",
+    "@budibase/bbui": "^1.0.27-alpha.12",
+    "@budibase/client": "^1.0.27-alpha.12",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^1.0.27-alpha.8",
+    "@budibase/string-templates": "^1.0.27-alpha.12",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -9,6 +9,7 @@
     Modal,
     Button,
     StatusLight,
+    ActionButton,
   } from "@budibase/bbui"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
@@ -27,7 +28,7 @@
   let blockComplete
 
   $: testResult = $automationStore.selectedAutomation.testResults?.steps.filter(
-    step => step.stepId === block.stepId
+    step => (block.id ? step.id === block.id : step.stepId === block.stepId)
   )
   $: isTrigger = block.type === "TRIGGER"
 
@@ -119,19 +120,13 @@
     <div class="blockSection">
       <Layout noPadding gap="S">
         <div class="splitHeader">
-          <div
-            on:click|stopPropagation={() => {
-              setupToggled = !setupToggled
-            }}
-            class="center-items"
+          <ActionButton
+            on:click={() => (setupToggled = !setupToggled)}
+            quiet
+            icon={setupToggled ? "ChevronDown" : "ChevronRight"}
           >
-            {#if setupToggled}
-              <Icon size="M" name="ChevronDown" />
-            {:else}
-              <Icon size="M" name="ChevronRight" />
-            {/if}
             <Detail size="S">Setup</Detail>
-          </div>
+          </ActionButton>
           {#if !isTrigger}
             <div on:click={() => deleteStep()}>
               <Icon name="DeleteOutline" />
@@ -187,6 +182,7 @@
   .splitHeader {
     display: flex;
     justify-content: space-between;
+    align-items: center;
   }
   .iconAlign {
     padding: 0 0 0 var(--spacing-m);

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -10,6 +10,7 @@
     ActionButton,
     Drawer,
     Modal,
+    Detail,
   } from "@budibase/bbui"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
 
@@ -37,6 +38,7 @@
   let drawer
   let tempFilters = lookForFilters(schemaProperties) || []
   let fillWidth = true
+  let codeBindingOpen = false
 
   $: stepId = block.stepId
   $: bindings = getAvailableBindings(
@@ -233,7 +235,16 @@
         <SchemaSetup on:change={e => onChange(e, key)} value={inputData[key]} />
       {:else if value.customType === "code"}
         <CodeEditorModal>
-          <pre>{JSON.stringify(bindings, null, 2)}</pre>
+          <ActionButton
+            on:click={() => (codeBindingOpen = !codeBindingOpen)}
+            quiet
+            icon={codeBindingOpen ? "ChevronDown" : "ChevronRight"}
+          >
+            <Detail size="S">Bindings</Detail>
+          </ActionButton>
+          {#if codeBindingOpen}
+            <pre>{JSON.stringify(bindings, null, 2)}</pre>
+          {/if}
           <Editor
             mode="javascript"
             on:change={e => {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.27-alpha.8",
+  "version": "1.0.27-alpha.12",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.27-alpha.8",
+  "version": "1.0.27-alpha.12",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.27-alpha.8",
+    "@budibase/bbui": "^1.0.27-alpha.12",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^1.0.27-alpha.8",
+    "@budibase/string-templates": "^1.0.27-alpha.12",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.27-alpha.8",
+  "version": "1.0.27-alpha.12",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/auth": "^1.0.27-alpha.8",
-    "@budibase/client": "^1.0.27-alpha.8",
-    "@budibase/string-templates": "^1.0.27-alpha.8",
+    "@budibase/backend-core": "^1.0.27-alpha.12",
+    "@budibase/client": "^1.0.27-alpha.12",
+    "@budibase/string-templates": "^1.0.27-alpha.12",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",

packages/server/scripts/replicateApp.js

@@ -7,7 +7,7 @@
 
 const CouchDB = require("../src/db")
 const { DocumentTypes } = require("../src/db/utils")
-const { getAllDbs } = require("@budibase/auth/db")
+const { getAllDbs } = require("@budibase/backend-core/db")
 
 const appName = process.argv[2].toLowerCase()
 const remoteUrl = process.argv[3]

packages/server/src/api/controllers/apikeys.js

@@ -1,5 +1,5 @@
-const { StaticDatabases } = require("@budibase/auth/db")
-const { getGlobalDB } = require("@budibase/auth/tenancy")
+const { StaticDatabases } = require("@budibase/backend-core/db")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 
 const KEYS_DOC = StaticDatabases.GLOBAL.docs.apiKeys
 

packages/server/src/api/controllers/application.js

@@ -19,7 +19,10 @@ const {
   DocumentTypes,
   AppStatus,
 } = require("../../db/utils")
-const { BUILTIN_ROLE_IDS, AccessController } = require("@budibase/auth/roles")
+const {
+  BUILTIN_ROLE_IDS,
+  AccessController,
+} = require("@budibase/backend-core/roles")
 const { BASE_LAYOUTS } = require("../../constants/layouts")
 const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
@@ -28,7 +31,7 @@ const {
   isDevAppID,
   getDeployedAppID,
   Replication,
-} = require("@budibase/auth/db")
+} = require("@budibase/backend-core/db")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
 const {
   getDeployedApps,
@@ -41,9 +44,9 @@ const {
   backupClientLibrary,
   revertClientLibrary,
 } = require("../../utilities/fileSystem/clientLibrary")
-const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy")
+const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
 const { syncGlobalUsers } = require("./user")
-const { app: appCache } = require("@budibase/auth/cache")
+const { app: appCache } = require("@budibase/backend-core/cache")
 const { cleanupAutomations } = require("../../automations/utils")
 
 const URL_REGEX_SLASH = /\/|\\/g

packages/server/src/api/controllers/auth.js

@@ -2,7 +2,7 @@ const CouchDB = require("../../db")
 const { outputProcessing } = require("../../utilities/rowProcessor")
 const { InternalTables } = require("../../db/utils")
 const { getFullUser } = require("../../utilities/users")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 
 exports.fetchSelf = async ctx => {
   const appId = ctx.appId

packages/server/src/api/controllers/cloud.js

@@ -1,5 +1,5 @@
 const env = require("../../environment")
-const { getAllApps } = require("@budibase/auth/db")
+const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../../db")
 const {
   exportDB,
@@ -7,7 +7,10 @@ const {
   readFileSync,
 } = require("../../utilities/fileSystem")
 const { stringToReadStream } = require("../../utilities")
-const { getGlobalDBName, getGlobalDB } = require("@budibase/auth/tenancy")
+const {
+  getGlobalDBName,
+  getGlobalDB,
+} = require("@budibase/backend-core/tenancy")
 const { create } = require("./application")
 const { getDocParams, DocumentTypes, isDevAppID } = require("../../db/utils")
 

packages/server/src/api/controllers/datasource.js

@@ -10,6 +10,7 @@ const {
 const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
 const { integrations } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
+const { invalidateDynamicVariables } = require("../../threads/utils")
 
 exports.fetch = async function (ctx) {
   const database = new CouchDB(ctx.appId)
@@ -57,10 +58,43 @@ exports.buildSchemaFromDb = async function (ctx) {
   ctx.body = response
 }
 
+/**
+ * Check for variables that have been updated or removed and invalidate them.
+ */
+const invalidateVariables = async (existingDatasource, updatedDatasource) => {
+  const existingVariables = existingDatasource.config.dynamicVariables
+  const updatedVariables = updatedDatasource.config.dynamicVariables
+  const toInvalidate = []
+
+  if (!existingVariables) {
+    return
+  }
+
+  if (!updatedVariables) {
+    // invalidate all
+    toInvalidate.push(...existingVariables)
+  } else {
+    // invaldate changed / removed
+    existingVariables.forEach(existing => {
+      const unchanged = updatedVariables.find(
+        updated =>
+          existing.name === updated.name &&
+          existing.queryId === updated.queryId &&
+          existing.value === updated.value
+      )
+      if (!unchanged) {
+        toInvalidate.push(existing)
+      }
+    })
+  }
+  await invalidateDynamicVariables(toInvalidate)
+}
+
 exports.update = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const datasourceId = ctx.params.datasourceId
   let datasource = await db.get(datasourceId)
+  await invalidateVariables(datasource, ctx.request.body)
   datasource = { ...datasource, ...ctx.request.body }
 
   const response = await db.put(datasource)

packages/server/src/api/controllers/deploy/index.js

@@ -1,12 +1,12 @@
 const CouchDB = require("../../../db")
 const Deployment = require("./Deployment")
-const { Replication, getDeployedAppID } = require("@budibase/auth/db")
+const { Replication, getDeployedAppID } = require("@budibase/backend-core/db")
 const { DocumentTypes, getAutomationParams } = require("../../../db/utils")
 const {
   disableAllCrons,
   enableCronTrigger,
 } = require("../../../automations/utils")
-const { app: appCache } = require("@budibase/auth/cache")
+const { app: appCache } = require("@budibase/backend-core/cache")
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000

packages/server/src/api/controllers/dev.js

@@ -4,9 +4,9 @@ const env = require("../../environment")
 const { checkSlashesInUrl } = require("../../utilities")
 const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
-const { Replication } = require("@budibase/auth").db
+const { Replication } = require("@budibase/backend-core").db
 const { DocumentTypes } = require("../../db/utils")
-const { app: appCache } = require("@budibase/auth/cache")
+const { app: appCache } = require("@budibase/backend-core/cache")
 
 async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params

packages/server/src/api/controllers/hosting.js

@@ -1,7 +1,7 @@
 const CouchDB = require("../../db")
 const { getDeployedApps } = require("../../utilities/workerRequests")
-const { getScopedConfig } = require("@budibase/auth/db")
-const { Configs } = require("@budibase/auth").constants
+const { getScopedConfig } = require("@budibase/backend-core/db")
+const { Configs } = require("@budibase/backend-core").constants
 const { checkSlashesInUrl } = require("../../utilities")
 
 exports.fetchUrls = async ctx => {

packages/server/src/api/controllers/permission.js

@@ -1,10 +1,10 @@
-const { getBuiltinPermissions } = require("@budibase/auth/permissions")
+const { getBuiltinPermissions } = require("@budibase/backend-core/permissions")
 const {
   isBuiltin,
   getDBRoleID,
   getExternalRoleID,
   getBuiltinRoles,
-} = require("@budibase/auth/roles")
+} = require("@budibase/backend-core/roles")
 const { getRoleParams } = require("../../db/utils")
 const CouchDB = require("../../db")
 const {

packages/server/src/api/controllers/role.js

@@ -4,7 +4,7 @@ const {
   getRole,
   isBuiltin,
   getAllRoles,
-} = require("@budibase/auth/roles")
+} = require("@budibase/backend-core/roles")
 const {
   generateRoleID,
   getUserMetadataParams,

packages/server/src/api/controllers/routing.js

@@ -2,7 +2,7 @@ const { getRoutingInfo } = require("../../utilities/routing")
 const {
   getUserRoleHierarchy,
   BUILTIN_ROLE_IDS,
-} = require("@budibase/auth/roles")
+} = require("@budibase/backend-core/roles")
 
 const URL_SEPARATOR = "/"
 

packages/server/src/api/controllers/row/internalSearch.js

@@ -1,6 +1,6 @@
 const { SearchIndexes } = require("../../../db/utils")
 const fetch = require("node-fetch")
-const { getCouchUrl } = require("@budibase/auth/db")
+const { getCouchUrl } = require("@budibase/backend-core/db")
 
 /**
  * Class to build lucene query URLs.

packages/server/src/api/controllers/screen.js

@@ -1,6 +1,6 @@
 const CouchDB = require("../../db")
 const { getScreenParams, generateScreenID } = require("../../db/utils")
-const { AccessController } = require("@budibase/auth/roles")
+const { AccessController } = require("@budibase/backend-core/roles")
 
 exports.fetch = async ctx => {
   const appId = ctx.appId

packages/server/src/api/controllers/user.js

@@ -8,10 +8,13 @@ const { InternalTables } = require("../../db/utils")
 const { getGlobalUsers, getRawGlobalUser } = require("../../utilities/global")
 const { getFullUser } = require("../../utilities/users")
 const { isEqual } = require("lodash")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
-const { getDevelopmentAppID, getDeployedAppIDs } = require("@budibase/auth/db")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const {
+  getDevelopmentAppID,
+  getDeployedAppIDs,
+} = require("@budibase/backend-core/db")
 const { doesDatabaseExist } = require("../../utilities")
-const { UserStatus } = require("@budibase/auth/constants")
+const { UserStatus } = require("@budibase/backend-core/constants")
 
 async function rawMetadata(db) {
   return (

packages/server/src/api/controllers/webhook.js

@@ -3,7 +3,7 @@ const { generateWebhookID, getWebhookParams } = require("../../db/utils")
 const toJsonSchema = require("to-json-schema")
 const validate = require("jsonschema").validate
 const triggers = require("../../automations/triggers")
-const { getDeployedAppID } = require("@budibase/auth/db")
+const { getDeployedAppID } = require("@budibase/backend-core/db")
 
 const AUTOMATION_DESCRIPTION = "Generated from Webhook Schema"
 

packages/server/src/api/index.js

@@ -4,7 +4,7 @@ const {
   auditLog,
   buildTenancyMiddleware,
   buildAppTenancyMiddleware,
-} = require("@budibase/auth").auth
+} = require("@budibase/backend-core").auth
 const currentApp = require("../middleware/currentapp")
 const compress = require("koa-compress")
 const zlib = require("zlib")

packages/server/src/api/routes/apikeys.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/apikeys")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/application.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/application")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 const usage = require("../../middleware/usageQuota")
 
 const router = Router()

packages/server/src/api/routes/automation.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const Joi = require("joi")
 const { bodyResource, paramResource } = require("../../middleware/resourceId")
 const {

packages/server/src/api/routes/backup.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/backup")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/cloud.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/cloud")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/component.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/component")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/datasource.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("@budibase/auth/permissions")
+} = require("@budibase/backend-core/permissions")
 const Joi = require("joi")
 const { DataSourceOperation } = require("../../constants")
 

packages/server/src/api/routes/deploy.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/deploy")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/dev.js

@@ -2,7 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/dev")
 const env = require("../../environment")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("@budibase/auth/permissions")
+const { BUILDER } = require("@budibase/backend-core/permissions")
 
 const router = Router()