Merge branch 'develop' of github.com:Budibase/budibase into dnd-improvements

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.14-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/context.js

@@ -6,6 +6,7 @@ const {
   updateAppId,
   doInAppContext,
   doInTenant,
+  doInContext,
 } = require("./src/context")
 
 const identity = require("./src/context/identity")
@@ -19,4 +20,5 @@ module.exports = {
   doInAppContext,
   doInTenant,
   identity,
+  doInContext,
 }

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.14-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -20,7 +20,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/types": "1.4.18-alpha.1",
+    "@budibase/types": "2.0.14-alpha.0",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-sdk": "2.1030.0",

packages/backend-core/src/context/index.ts

@@ -65,7 +65,16 @@ export const getTenantIDFromAppID = (appId: string) => {
   }
 }
 
-// used for automations, API endpoints should always be in context already
+export const doInContext = async (appId: string, task: any) => {
+  // gets the tenant ID from the app ID
+  const tenantId = getTenantIDFromAppID(appId)
+  return doInTenant(tenantId, async () => {
+    return doInAppContext(appId, async () => {
+      return task()
+    })
+  })
+}
+
 export const doInTenant = (tenantId: string | null, task: any) => {
   // make sure default always selected in single tenancy
   if (!env.MULTI_TENANCY) {

packages/backend-core/src/db/constants.ts

@@ -46,6 +46,9 @@ export enum DocumentType {
   AUTOMATION_LOG = "log_au",
   ACCOUNT_METADATA = "acc_metadata",
   PLUGIN = "plg",
+  TABLE = "ta",
+  DATASOURCE = "datasource",
+  DATASOURCE_PLUS = "datasource_plus",
 }
 
 export const StaticDatabases = {

packages/backend-core/src/db/conversions.js

@@ -36,6 +36,7 @@ exports.getDevelopmentAppID = appId => {
   const rest = split.join(APP_PREFIX)
   return `${APP_DEV_PREFIX}${rest}`
 }
+exports.getDevAppID = exports.getDevelopmentAppID
 
 /**
  * Convert a development app ID to a deployed app ID.

packages/backend-core/src/db/utils.ts

@@ -64,6 +64,28 @@ export function getQueryIndex(viewName: ViewName) {
   return `database/${viewName}`
 }
 
+/**
+ * Check if a given ID is that of a table.
+ * @returns {boolean}
+ */
+export const isTableId = (id: string) => {
+  // this includes datasource plus tables
+  return (
+    id &&
+    (id.startsWith(`${DocumentType.TABLE}${SEPARATOR}`) ||
+      id.startsWith(`${DocumentType.DATASOURCE_PLUS}${SEPARATOR}`))
+  )
+}
+
+/**
+ * Check if a given ID is that of a datasource or datasource plus.
+ * @returns {boolean}
+ */
+export const isDatasourceId = (id: string) => {
+  // this covers both datasources and datasource plus
+  return id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`)
+}
+
 /**
  * Generates a new workspace ID.
  * @returns {string} The new workspace ID which the workspace doc can be stored under.

packages/backend-core/src/migrations/definitions.ts

@@ -11,7 +11,7 @@ export const DEFINITIONS: MigrationDefinition[] = [
   },
   {
     type: MigrationType.GLOBAL,
-    name: MigrationName.QUOTAS_1,
+    name: MigrationName.SYNC_QUOTAS,
   },
   {
     type: MigrationType.APP,
@@ -33,8 +33,4 @@ export const DEFINITIONS: MigrationDefinition[] = [
     type: MigrationType.GLOBAL,
     name: MigrationName.GLOBAL_INFO_SYNC_USERS,
   },
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.PLUGIN_COUNT,
-  },
 ]

packages/backend-core/src/objectStore/index.ts

@@ -182,6 +182,11 @@ export const streamUpload = async (
       ...extra,
       ContentType: "application/javascript",
     }
+  } else if (filename?.endsWith(".svg")) {
+    extra = {
+      ...extra,
+      ContentType: "image",
+    }
   }
 
   const params = {

packages/backend-core/src/pkg/context.ts

@@ -8,6 +8,7 @@ import {
   updateAppId,
   doInAppContext,
   doInTenant,
+  doInContext,
 } from "../context"
 
 import * as identity from "../context/identity"
@@ -20,5 +21,6 @@ export = {
   updateAppId,
   doInAppContext,
   doInTenant,
+  doInContext,
   identity,
 }

packages/backend-core/src/plugin/utils.js

@@ -67,12 +67,8 @@ function validateDatasource(schema) {
       description: joi.string().required(),
       datasource: joi.object().pattern(joi.string(), fieldValidator).required(),
       query: joi
-        .object({
+        .object()
-          create: queryValidator,
+        .pattern(joi.string(), queryValidator)
-          read: queryValidator,
-          update: queryValidator,
-          delete: queryValidator,
-        })
         .unknown(true)
         .required(),
       extra: joi.object().pattern(

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.14-alpha.0",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "1.4.18-alpha.1",
+    "@budibase/string-templates": "2.0.14-alpha.0",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/builder/cypress/integration/adminAndManagement/userManagement.spec.js

@@ -20,7 +20,9 @@ filterTests(["smoke", "all"], () => {
       cy.get(".spectrum-Form-itemField").eq(3).should('contain', 'App User')
 
       // User should not have app access
-      cy.get(interact.LIST_ITEMS, { timeout: 500 }).should("contain", "No apps")
+      cy.get(".spectrum-Heading").contains("Apps").parent().within(() => {
+        cy.get(interact.LIST_ITEMS, { timeout: 500 }).should("contain", "This user has access to no apps")
+      })
     })
 
     if (Cypress.env("TEST_ENV")) {

packages/builder/cypress/integration/autoScreensUI.spec.js

@@ -54,6 +54,7 @@ filterTests(['smoke', 'all'], () => {
       cy.createDatasourceScreen([initialTable, secondTable])
       // Confirm screens have been auto generated
       // Previously generated tables are suffixed with numbers - as expected
+      cy.wait(1000)
       cy.get(interact.BODY).should('contain', 'cypress-tests-2')
         .and('contain', 'cypress-tests-2/:id')
         .and('contain', 'cypress-tests-2/new/row')

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.14-alpha.0",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -71,10 +71,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "1.4.18-alpha.1",
+    "@budibase/bbui": "2.0.14-alpha.0",
-    "@budibase/client": "1.4.18-alpha.1",
+    "@budibase/client": "2.0.14-alpha.0",
-    "@budibase/frontend-core": "1.4.18-alpha.1",
+    "@budibase/frontend-core": "2.0.14-alpha.0",
-    "@budibase/string-templates": "1.4.18-alpha.1",
+    "@budibase/string-templates": "2.0.14-alpha.0",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/componentUtils.js

@@ -143,7 +143,10 @@ export const getComponentSettings = componentType => {
   }
 
   // Ensure whole component name is used
-  if (!componentType.startsWith("@budibase")) {
+  if (
+    !componentType.startsWith("plugin/") &&
+    !componentType.startsWith("@budibase")
+  ) {
     componentType = `@budibase/standard-components/${componentType}`
   }
 

packages/builder/src/builderStore/dataBinding.js

@@ -243,18 +243,18 @@ export const getDatasourceForProvider = (asset, component) => {
     return null
   }
 
-  // There are different types of setting which can be a datasource, for
+  // For legacy compatibility, we need to be able to handle datasources that are
-  // example an actual datasource object, or a table ID string.
+  // just strings. These are not generated any more, so could be removed in
-  // Convert the datasource setting into a proper datasource object so that
+  // future.
-  // we can use it properly
+  // TODO: remove at some point
-  if (datasourceSetting.type === "table") {
+  const datasource = component[datasourceSetting?.key]
+  if (typeof datasource === "string") {
     return {
-      tableId: component[datasourceSetting?.key],
+      tableId: datasource,
       type: "table",
     }
-  } else {
-    return component[datasourceSetting?.key]
   }
+  return datasource
 }
 
 /**

packages/builder/src/builderStore/store/frontend.js

@@ -88,27 +88,12 @@ export const getFrontendStore = () => {
     initialise: async pkg => {
       const { layouts, screens, application, clientLibPath } = pkg
 
-      // Fetch component definitions.
+      await store.actions.components.refreshDefinitions(application.appId)
-      // Allow errors to propagate.
-      const components = await API.fetchComponentLibDefinitions(
-        application.appId
-      )
-
-      // Filter out custom component keys so we can flag them
-      const customComponents = Object.keys(components).filter(name =>
-        name.startsWith("plugin/")
-      )
 
       // Reset store state
       store.update(state => ({
         ...state,
         libraries: application.componentLibraries,
-        components,
-        customComponents,
-        clientFeatures: {
-          ...INITIAL_FRONTEND_STATE.clientFeatures,
-          ...components.features,
-        },
         name: application.name,
         description: application.description,
         appId: application.appId,
@@ -385,6 +370,29 @@ export const getFrontendStore = () => {
       },
     },
     components: {
+      refreshDefinitions: async appId => {
+        if (!appId) {
+          appId = get(store).appId
+        }
+
+        // Fetch definitions and filter out custom component definitions so we
+        // can flag them
+        const components = await API.fetchComponentLibDefinitions(appId)
+        const customComponents = Object.keys(components).filter(name =>
+          name.startsWith("plugin/")
+        )
+
+        // Update store
+        store.update(state => ({
+          ...state,
+          components,
+          customComponents,
+          clientFeatures: {
+            ...INITIAL_FRONTEND_STATE.clientFeatures,
+            ...components.features,
+          },
+        }))
+      },
       getDefinition: componentName => {
         if (!componentName) {
           return null
@@ -428,7 +436,7 @@ export const getFrontendStore = () => {
           _id: Helpers.uuid(),
           _component: definition.component,
           _styles: { normal: {}, hover: {}, active: {} },
-          _instanceName: `New ${definition.name}`,
+          _instanceName: `New ${definition.friendlyName || definition.name}`,
           ...cloneDeep(props),
           ...extras,
         }

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -13,7 +13,7 @@
     customQueryIconColor,
     customQueryText,
   } from "helpers/data/utils"
-  import { getIcon } from "./icons"
+  import IntegrationIcon from "./IntegrationIcon.svelte"
   import { notifications } from "@budibase/bbui"
 
   let openDataSources = []
@@ -123,10 +123,10 @@
         on:iconClick={() => toggleNode(datasource)}
       >
         <div class="datasource-icon" slot="icon">
-          <svelte:component
+          <IntegrationIcon
-            this={getIcon(datasource.source, datasource.schema)}
+            integrationType={datasource.source}
-            height="18"
+            schema={datasource.schema}
-            width="18"
+            size="18"
           />
         </div>
         {#if datasource._id !== BUDIBASE_INTERNAL_DB}

packages/builder/src/components/backend/DatasourceNavigator/IntegrationIcon.svelte

@@ -0,0 +1,32 @@
+<script>
+  import { getIcon } from "./icons"
+  import CustomSVG from "components/common/CustomSVG.svelte"
+  import { admin } from "stores/portal"
+
+  export let integrationType
+  export let schema
+  export let size = "18"
+
+  $: objectStoreUrl = $admin.cloud ? "https://cdn.budi.live" : ""
+  $: pluginsUrl = `${objectStoreUrl}/plugins`
+  $: iconInfo = getIcon(integrationType, schema)
+
+  async function getSvgFromUrl(info) {
+    const url = `${pluginsUrl}/${info.url}`
+    const resp = await fetch(url, {
+      headers: {
+        ["pragma"]: "no-cache",
+        ["cache-control"]: "no-cache",
+      },
+    })
+    return resp.text()
+  }
+</script>
+
+{#if iconInfo.icon}
+  <svelte:component this={iconInfo.icon} height={size} width={size} />
+{:else if iconInfo.url}
+  {#await getSvgFromUrl(iconInfo) then retrievedSvg}
+    <CustomSVG {size} svgHtml={retrievedSvg} />
+  {/await}
+{/if}

packages/builder/src/components/backend/DatasourceNavigator/_components/DatasourceCard.svelte

@@ -1,7 +1,7 @@
 <script>
   import { createEventDispatcher } from "svelte"
   import { Heading, Detail } from "@budibase/bbui"
-  import { getIcon } from "../icons"
+  import IntegrationIcon from "../IntegrationIcon.svelte"
 
   export let integration
   export let integrationType
@@ -16,11 +16,7 @@
   class="item hoverable"
 >
   <div class="item-body" class:with-type={!!schema.type}>
-    <svelte:component
+    <IntegrationIcon {integrationType} {schema} size="25" />
-      this={getIcon(integrationType, schema)}
-      height="20"
-      width="20"
-    />
     <div class="text">
       <Heading size="XXS">{schema.friendlyName}</Heading>
       {#if schema.type}

packages/builder/src/components/backend/DatasourceNavigator/icons/index.js

@@ -16,6 +16,8 @@ import Firebase from "./Firebase.svelte"
 import Redis from "./Redis.svelte"
 import Snowflake from "./Snowflake.svelte"
 import Custom from "./Custom.svelte"
+import { integrations } from "stores/backend"
+import { get } from "svelte/store"
 
 const ICONS = {
   BUDIBASE: Budibase,
@@ -41,9 +43,12 @@ const ICONS = {
 export default ICONS
 
 export function getIcon(integrationType, schema) {
-  if (schema?.custom || !ICONS[integrationType]) {
+  const integrationList = get(integrations)
-    return ICONS.CUSTOM
+  if (integrationList[integrationType]?.iconUrl) {
+    return { url: integrationList[integrationType].iconUrl }
+  } else if (schema?.custom || !ICONS[integrationType]) {
+    return { icon: ICONS.CUSTOM }
   } else {
-    return ICONS[integrationType]
+    return { icon: ICONS[integrationType] }
   }
 }

packages/builder/src/components/common/CustomSVG.svelte

@@ -0,0 +1,23 @@
+<script>
+  import { Helpers } from "@budibase/bbui"
+  export let size
+  export let svgHtml
+
+  function substituteSize(svg) {
+    if (svg.includes("height=")) {
+      svg = svg.replace(/height="[^"]+"/, `height="${size}"`)
+    }
+    if (svg.includes("width=")) {
+      svg = svg.replace(/width="[^"]+"/, `width="${size}"`)
+    }
+    if (svg.includes("id=")) {
+      const matches = svg.match(/id="([^"]+)"/g)
+      for (let match of matches) {
+        svg = svg.replace(new RegExp(match, "g"), Helpers.uuid())
+      }
+    }
+    return svg
+  }
+</script>
+
+{@html substituteSize(svgHtml)}

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -43,7 +43,7 @@
   let helpers = handlebarsCompletions()
   let getCaretPosition
   let search = ""
-  let initialValueJS = value?.startsWith("{{ js ")
+  let initialValueJS = typeof value === "string" && value?.startsWith("{{ js ")
   let mode = initialValueJS ? "JavaScript" : "Handlebars"
   let jsValue = initialValueJS ? value : null
   let hbsValue = initialValueJS ? null : value

packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte

@@ -20,6 +20,8 @@
   import { createEventDispatcher, onMount } from "svelte"
 
   const dispatch = createEventDispatcher()
+  const { OperatorOptions } = Constants
+  const { getValidOperatorsForType } = LuceneUtils
 
   export let schemaFields
   export let filters = []
@@ -45,7 +47,7 @@
       {
         id: generate(),
         field: null,
-        operator: Constants.OperatorOptions.Equals.value,
+        operator: OperatorOptions.Equals.value,
         value: null,
         valueType: "Value",
       },
@@ -66,49 +68,60 @@
     return schemaFields.find(field => field.name === filter.field)
   }
 
-  const onFieldChange = (expression, field) => {
+  const santizeTypes = filter => {
-    // Update the field types
+    // Update type based on field
-    expression.type = enrichedSchemaFields.find(x => x.name === field)?.type
+    const fieldSchema = enrichedSchemaFields.find(x => x.name === filter.field)
-    expression.externalType = getSchema(expression)?.externalType
+    filter.type = fieldSchema?.type
 
-    // Ensure a valid operator is set
+    // Update external type based on field
-    const validOperators = LuceneUtils.getValidOperatorsForType(
+    filter.externalType = getSchema(filter)?.externalType
-      expression.type
+  }
-    ).map(x => x.value)
+
-    if (!validOperators.includes(expression.operator)) {
+  const santizeOperator = filter => {
-      expression.operator =
+    // Ensure a valid operator is selected
-        validOperators[0] ?? Constants.OperatorOptions.Equals.value
+    const operators = getValidOperatorsForType(filter.type).map(x => x.value)
-      onOperatorChange(expression, expression.operator)
+    if (!operators.includes(filter.operator)) {
+      filter.operator = operators[0] ?? OperatorOptions.Equals.value
     }
 
-    // if changed to an array, change default value to empty array
+    // Update the noValue flag if the operator does not take a value
-    const idx = filters.findIndex(x => x.id === expression.id)
+    const noValueOptions = [
-    if (expression.type === "array") {
+      OperatorOptions.Empty.value,
-      filters[idx].value = []
+      OperatorOptions.NotEmpty.value,
-    } else {
+    ]
-      filters[idx].value = null
+    filter.noValue = noValueOptions.includes(filter.operator)
+  }
+
+  const santizeValue = filter => {
+    // Check if the operator allows a value at all
+    if (filter.noValue) {
+      filter.value = null
+      return
+    }
+
+    // Ensure array values are properly set and cleared
+    if (Array.isArray(filter.value)) {
+      if (filter.valueType !== "Value" || filter.type !== "array") {
+        filter.value = null
+      }
+    } else if (filter.type === "array" && filter.valueType === "Value") {
+      filter.value = []
     }
   }
 
-  const onOperatorChange = (expression, operator) => {
+  const onFieldChange = filter => {
-    const noValueOptions = [
+    santizeTypes(filter)
-      Constants.OperatorOptions.Empty.value,
+    santizeOperator(filter)
-      Constants.OperatorOptions.NotEmpty.value,
+    santizeValue(filter)
-    ]
+  }
-    expression.noValue = noValueOptions.includes(operator)
+
-    if (expression.noValue) {
+  const onOperatorChange = filter => {
-      expression.value = null
+    santizeOperator(filter)
-    }
+    santizeValue(filter)
-    if (
+  }
-      operator === Constants.OperatorOptions.In.value &&
+
-      !Array.isArray(expression.value)
+  const onValueTypeChange = filter => {
-    ) {
+    santizeValue(filter)
-      if (expression.value) {
-        expression.value = [expression.value]
-      } else {
-        expression.value = []
-      }
-    }
   }
 
   const getFieldOptions = field => {
@@ -153,23 +166,24 @@
               <Select
                 bind:value={filter.field}
                 options={fieldOptions}
-                on:change={e => onFieldChange(filter, e.detail)}
+                on:change={() => onFieldChange(filter)}
                 placeholder="Column"
               />
               <Select
                 disabled={!filter.field}
-                options={LuceneUtils.getValidOperatorsForType(filter.type)}
+                options={getValidOperatorsForType(filter.type)}
                 bind:value={filter.operator}
-                on:change={e => onOperatorChange(filter, e.detail)}
+                on:change={() => onOperatorChange(filter)}
                 placeholder={null}
               />
               <Select
                 disabled={filter.noValue || !filter.field}
                 options={valueTypeOptions}
                 bind:value={filter.valueType}
+                on:change={() => onValueTypeChange(filter)}
                 placeholder={null}
               />
-              {#if filter.valueType === "Binding"}
+              {#if filter.field && filter.valueType === "Binding"}
                 <DrawerBindableInput
                   disabled={filter.noValue}
                   title={`Value for "${filter.field}"`}
@@ -250,7 +264,7 @@
     column-gap: var(--spacing-l);
     row-gap: var(--spacing-s);
     align-items: center;
-    grid-template-columns: 1fr 120px 120px 1fr auto auto;
+    grid-template-columns: 1fr 150px 120px 1fr 16px 16px;
   }
 
   .filter-label {

packages/builder/src/components/design/settings/controls/TableSelect.svelte

@@ -1,26 +1,28 @@
 <script>
   import { Select } from "@budibase/bbui"
-  import { tables } from "stores/backend"
+  import { createEventDispatcher } from "svelte"
+  import { tables as tablesStore } from "stores/backend"
 
   export let value
+
+  const dispatch = createEventDispatcher()
+
+  $: tables = $tablesStore.list.map(m => ({
+    label: m.name,
+    tableId: m._id,
+    type: "table",
+  }))
+
+  const onChange = e => {
+    const dataSource = tables?.find(x => x.tableId === e.detail)
+    dispatch("change", dataSource)
+  }
 </script>
 
-<div>
+<Select
-  <Select extraThin secondary wide on:change {value}>
+  on:change={onChange}
-    <option value="">Choose a table</option>
+  value={value?.tableId}
-    {#each $tables.list as table}
+  options={tables}
-      <option value={table._id}>{table.name}</option>
+  getOptionValue={x => x.tableId}
-    {/each}
+  getOptionLabel={x => x.label}
-  </Select>
+/>
-</div>
-
-<style>
-  div {
-    flex: 1 1 auto;
-    display: flex;
-    flex-direction: row;
-  }
-  div :global(> *) {
-    flex: 1 1 auto;
-  }
-</style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte

@@ -198,6 +198,8 @@
             block: "center",
           })
         }
+      } else if (type === "reload-plugin") {
+        await store.actions.components.refreshDefinitions()
       } else {
         console.warn(`Client sent unknown event type: ${type}`)
       }

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/navigation/ComponentTree.svelte

@@ -18,6 +18,8 @@
 
   let closedNodes = {}
 
+  $: currentScreen = get(selectedScreen)
+
   $: filteredComponents = components?.filter(component => {
     return (
       !$store.componentToPaste?.isCut ||
@@ -68,9 +70,27 @@
     closedNodes = closedNodes
   }
 
-  const onDrop = async e => {
+  const onDrop = async (e, component) => {
     e.stopPropagation()
     try {
+      const compDef = store.actions.components.getDefinition(
+        $dndStore.source?._component
+      )
+      const compTypeName = compDef.name.toLowerCase()
+      const path = findComponentPath(currentScreen.props, component._id)
+
+      for (let pathComp of path) {
+        const pathCompDef = store.actions.components.getDefinition(
+          pathComp?._component
+        )
+        if (pathCompDef?.illegalChildren?.indexOf(compTypeName) > -1) {
+          notifications.warning(
+            `${compDef.name} cannot be a child of ${pathCompDef.name} (${pathComp._instanceName})`
+          )
+          return
+        }
+      }
+
       await dndStore.actions.drop()
     } catch (error) {
       console.error(error)
@@ -114,7 +134,9 @@
         on:dragstart={() => dndStore.actions.dragstart(component)}
         on:dragover={dragover(component, index)}
         on:iconClick={() => toggleNodeOpen(component._id)}
-        on:drop={onDrop}
+        on:drop={e => {
+          onDrop(e, component)
+        }}
         text={getComponentText(component)}
         icon={getComponentIcon(component)}
         withArrow={componentHasChildren(component)}

packages/builder/src/pages/builder/apps/index.svelte

@@ -133,7 +133,7 @@
             </Body>
           </Layout>
           <Divider />
-          {#if $licensing.usageMetrics.dayPasses >= 100}
+          {#if $licensing.usageMetrics?.dayPasses >= 100}
             <div>
               <Layout gap="S" justifyItems="center">
                 <img class="spaceman" alt="spaceman" src={Spaceman} />

packages/builder/src/pages/builder/portal/_layout.svelte

@@ -56,7 +56,7 @@
         {
           title: "Plugins",
           href: "/builder/portal/manage/plugins",
-          badge: "Beta",
+          badge: "New",
         },
 
         {

packages/builder/src/pages/builder/portal/manage/groups/[groupId].svelte

@@ -25,6 +25,7 @@
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import CreateEditGroupModal from "./_components/CreateEditGroupModal.svelte"
   import GroupIcon from "./_components/GroupIcon.svelte"
+  import AppAddModal from "./_components/AppAddModal.svelte"
 
   export let groupId
 
@@ -34,15 +35,14 @@
   let prevSearch = undefined
   let pageInfo = createPaginationStore()
   let loaded = false
-  let editModal
+  let editModal, deleteModal, appAddModal
-  let deleteModal
 
   $: page = $pageInfo.page
   $: fetchUsers(page, searchTerm)
   $: group = $groups.find(x => x._id === groupId)
   $: filtered = $users.data
   $: groupApps = $apps.filter(app =>
-    groups.actions.getGroupAppIds(group).includes(apps.getProdAppID(app.appId))
+    groups.actions.getGroupAppIds(group).includes(apps.getProdAppID(app.devId))
   )
   $: {
     if (loaded && !group?._id) {
@@ -182,7 +182,14 @@
     </Layout>
 
     <Layout noPadding gap="S">
-      <Heading size="S">Apps</Heading>
+      <div class="header">
+        <Heading size="S">Apps</Heading>
+        <div>
+          <Button on:click={appAddModal.show()} icon="ExperienceAdd" cta>
+            Add app
+          </Button>
+        </div>
+      </div>
       <List>
         {#if groupApps.length}
           {#each groupApps as app}
@@ -197,12 +204,24 @@
                 <StatusLight
                   square
                   color={RoleUtils.getRoleColour(
-                    group.roles[apps.getProdAppID(app.appId)]
+                    group.roles[apps.getProdAppID(app.devId)]
                   )}
                 >
-                  {getRoleLabel(app.appId)}
+                  {getRoleLabel(app.devId)}
                 </StatusLight>
               </div>
+              <Icon
+                on:click={e => {
+                  groups.actions.removeApp(
+                    groupId,
+                    apps.getProdAppID(app.devId)
+                  )
+                  e.stopPropagation()
+                }}
+                hoverable
+                size="S"
+                name="Close"
+              />
             </ListItem>
           {/each}
         {:else}
@@ -216,6 +235,11 @@
 <Modal bind:this={editModal}>
   <CreateEditGroupModal {group} {saveGroup} />
 </Modal>
+
+<Modal bind:this={appAddModal}>
+  <AppAddModal {group} />
+</Modal>
+
 <ConfirmDialog
   bind:this={deleteModal}
   title="Delete user group"

packages/builder/src/pages/builder/portal/manage/groups/_components/AppAddModal.svelte

@@ -0,0 +1,53 @@
+<script>
+  import { Body, ModalContent, Select } from "@budibase/bbui"
+  import { apps, groups } from "stores/portal"
+  import { roles } from "stores/backend"
+  import RoleSelect from "components/common/RoleSelect.svelte"
+
+  export let group
+
+  $: appOptions = $apps.map(app => ({
+    label: app.name,
+    value: app,
+  }))
+  $: confirmDisabled =
+    (!selectingRole && !selectedApp) || (selectingRole && !selectedRoleId)
+  let selectedApp, selectedRoleId
+  let selectingRole = false
+
+  async function appSelected() {
+    const prodAppId = apps.getProdAppID(selectedApp.devId)
+    if (!selectingRole) {
+      selectingRole = true
+      await roles.fetchByAppId(prodAppId)
+      // return false to stop closing modal
+      return false
+    } else {
+      await groups.actions.addApp(group._id, prodAppId, selectedRoleId)
+    }
+  }
+</script>
+
+<ModalContent
+  onConfirm={appSelected}
+  size="M"
+  title="Add app to group"
+  confirmText={selectingRole ? "Confirm" : "Next"}
+  showSecondaryButton={selectingRole}
+  secondaryButtonText="Back"
+  secondaryAction={() => (selectingRole = false)}
+  disabled={confirmDisabled}
+>
+  {#if !selectingRole}
+    <Body
+      >Select an app to assign roles for members of <i>"{group.name}"</i></Body
+    >
+    <Select bind:value={selectedApp} options={appOptions} />
+  {:else}
+    <Body
+      >Select the role that all members of "<i>{group.name}</i>" will have for
+      <i>"{selectedApp.name}"</i></Body
+    >
+    <RoleSelect allowPublic={false} bind:value={selectedRoleId} />
+  {/if}
+</ModalContent>

packages/builder/src/pages/builder/portal/manage/groups/index.svelte

@@ -27,7 +27,6 @@
     icon: "UserGroup",
     color: "var(--spectrum-global-color-blue-600)",
     users: [],
-    apps: [],
     roles: {},
   }
 
@@ -91,16 +90,14 @@
 
 <Layout noPadding gap="M">
   <Layout gap="XS" noPadding>
-    <Heading size="M">User groups</Heading>
+    <div class="title">
-    {#if !$licensing.groupsEnabled}
+      <Heading size="M">User groups</Heading>
-      <Tags>
+      {#if !$licensing.groupsEnabled}
-        <div class="tags">
+        <Tags>
-          <div class="tag">
+          <Tag icon="LockClosed">Pro plan</Tag>
-            <Tag icon="LockClosed">Pro plan</Tag>
+        </Tags>
-          </div>
+      {/if}
-        </div>
+    </div>
-      </Tags>
-    {/if}
     <Body>
       Easily assign and manage your users' access with user groups.
       {#if !$auth.accountPortalAccess && !$licensing.groupsEnabled && $admin.cloud}
@@ -124,6 +121,7 @@
       {:else}
         <Button
           newStyles
+          primary
           disabled={!$auth.accountPortalAccess && $admin.cloud}
           on:click={$licensing.goToUpgradePage()}
         >
@@ -141,18 +139,22 @@
         </Button>
       {/if}
     </ButtonGroup>
-    <div class="controls-right">
+    {#if $licensing.groupsEnabled}
-      <Search bind:value={searchString} placeholder="Search" />
+      <div class="controls-right">
-    </div>
+        <Search bind:value={searchString} placeholder="Search" />
+      </div>
+    {/if}
   </div>
-  <Table
+  {#if $licensing.groupsEnabled}
-    on:click={({ detail }) => $goto(`./${detail._id}`)}
+    <Table
-    {schema}
+      on:click={({ detail }) => $goto(`./${detail._id}`)}
-    data={filteredGroups}
+      {schema}
-    allowEditColumns={false}
+      data={filteredGroups}
-    allowEditRows={false}
+      allowEditColumns={false}
-    {customRenderers}
+      allowEditRows={false}
-  />
+      {customRenderers}
+    />
+  {/if}
 </Layout>
 
 <Modal bind:this={modal}>
@@ -176,8 +178,11 @@
   .controls-right :global(.spectrum-Search) {
     width: 200px;
   }
-  .tag {
+  .title {
-    margin-top: var(--spacing-xs);
+    display: flex;
-    margin-left: var(--spacing-m);
+    flex-direction: row;
+    align-items: center;
+    justify-content: flex-start;
+    gap: var(--spacing-m);
   }
 </style>

packages/builder/src/pages/builder/portal/manage/plugins/index.svelte

@@ -55,18 +55,20 @@
           Add plugin
         </Button>
       </div>
-      <div class="filters">
+      {#if filteredPlugins?.length}
-        <div class="select">
+        <div class="filters">
-          <Select
+          <div class="select">
-            bind:value={filter}
+            <Select
-            placeholder={null}
+              bind:value={filter}
-            options={filterOptions}
+              placeholder={null}
-            autoWidth
+              options={filterOptions}
-            quiet
+              autoWidth
-          />
+              quiet
+            />
+          </div>
+          <Search bind:value={searchTerm} placeholder="Search plugins" />
         </div>
-        <Search bind:value={searchTerm} placeholder="Search plugins" />
+      {/if}
-      </div>
     </div>
     {#if filteredPlugins?.length}
       <Layout noPadding gap="S">

packages/builder/src/stores/backend/roles.js

@@ -5,16 +5,24 @@ import { RoleUtils } from "@budibase/frontend-core"
 export function createRolesStore() {
   const { subscribe, update, set } = writable([])
 
+  function setRoles(roles) {
+    set(
+      roles.sort((a, b) => {
+        const priorityA = RoleUtils.getRolePriority(a._id)
+        const priorityB = RoleUtils.getRolePriority(b._id)
+        return priorityA > priorityB ? -1 : 1
+      })
+    )
+  }
+
   const actions = {
     fetch: async () => {
       const roles = await API.getRoles()
-      set(
+      setRoles(roles)
-        roles.sort((a, b) => {
+    },
-          const priorityA = RoleUtils.getRolePriority(a._id)
+    fetchByAppId: async appId => {
-          const priorityB = RoleUtils.getRolePriority(b._id)
+      const { roles } = await API.getRolesForApp(appId)
-          return priorityA > priorityB ? -1 : 1
+      setRoles(roles)
-        })
-      )
     },
     delete: async role => {
       await API.deleteRole({

packages/builder/src/stores/portal/apps.js

@@ -21,6 +21,8 @@ const getProdAppID = appId => {
   } else if (!appId.startsWith("app")) {
     rest = appId
     separator = "_"
+  } else {
+    return appId
   }
   return `app${separator}${rest}`
 }

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.14-alpha.0",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "1.4.18-alpha.1",
+    "@budibase/backend-core": "2.0.14-alpha.0",
-    "@budibase/string-templates": "1.4.18-alpha.1",
+    "@budibase/string-templates": "2.0.14-alpha.0",
-    "@budibase/types": "1.4.18-alpha.1",
+    "@budibase/types": "2.0.14-alpha.0",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",
@@ -36,6 +36,7 @@
     "docker-compose": "0.23.6",
     "dotenv": "16.0.1",
     "download": "8.0.0",
+    "find-free-port": "^2.0.0",
     "inquirer": "8.0.0",
     "joi": "17.6.0",
     "lookpath": "1.1.0",
@@ -45,7 +46,8 @@
     "pouchdb": "7.3.0",
     "pouchdb-replication-stream": "1.2.9",
     "randomstring": "1.1.5",
-    "tar": "6.1.11"
+    "tar": "6.1.11",
+    "yaml": "^2.1.1"
   },
   "devDependencies": {
     "copyfiles": "^2.4.1",

packages/cli/src/constants.js

@@ -21,3 +21,5 @@ exports.AnalyticsEvents = {
 }
 
 exports.POSTHOG_TOKEN = "phc_yGOn4i7jWKaCTapdGR6lfA4AvmuEQ2ijn5zAVSFYPlS"
+
+exports.GENERATED_USER_EMAIL = "admin@admin.com"

packages/cli/src/hosting/genUser.js

@@ -0,0 +1,22 @@
+const { success } = require("../utils")
+const { updateDockerComposeService } = require("./utils")
+const randomString = require("randomstring")
+const { GENERATED_USER_EMAIL } = require("../constants")
+
+exports.generateUser = async (password, silent) => {
+  const email = GENERATED_USER_EMAIL
+  if (!password) {
+    password = randomString.generate({ length: 6 })
+  }
+  updateDockerComposeService(service => {
+    service.environment["BB_ADMIN_USER_EMAIL"] = email
+    service.environment["BB_ADMIN_USER_PASSWORD"] = password
+  })
+  if (!silent) {
+    console.log(
+      success(
+        `User admin credentials configured, access with email: ${email} - password: ${password}`
+      )
+    )
+  }
+}

packages/cli/src/hosting/index.js

@@ -1,164 +1,18 @@
 const Command = require("../structures/Command")
-const { CommandWords, InitTypes, AnalyticsEvents } = require("../constants")
+const { CommandWords } = require("../constants")
-const { lookpath } = require("lookpath")
+const { init } = require("./init")
-const {
+const { start } = require("./start")
-  downloadFile,
+const { stop } = require("./stop")
-  logErrorToFile,
+const { status } = require("./status")
-  success,
+const { update } = require("./update")
-  info,
+const { generateUser } = require("./genUser")
-  parseEnv,
+const { watchPlugins } = require("./watch")
-} = require("../utils")
-const { confirmation } = require("../questions")
-const fs = require("fs")
-const compose = require("docker-compose")
-const makeEnv = require("./makeEnv")
-const axios = require("axios")
-const { captureEvent } = require("../events")
-
-const BUDIBASE_SERVICES = ["app-service", "worker-service", "proxy-service"]
-const ERROR_FILE = "docker-error.log"
-const FILE_URLS = [
-  "https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml",
-]
-const DO_USER_DATA_URL = "http://169.254.169.254/metadata/v1/user-data"
-
-async function downloadFiles() {
-  const promises = []
-  for (let url of FILE_URLS) {
-    const fileName = url.split("/").slice(-1)[0]
-    promises.push(downloadFile(url, `./${fileName}`))
-  }
-  await Promise.all(promises)
-}
-
-async function checkDockerConfigured() {
-  const error =
-    "docker/docker-compose has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
-  const docker = await lookpath("docker")
-  const compose = await lookpath("docker-compose")
-  if (!docker || !compose) {
-    throw error
-  }
-}
-
-function checkInitComplete() {
-  if (!fs.existsSync(makeEnv.filePath)) {
-    throw "Please run the hosting --init command before any other hosting command."
-  }
-}
-
-async function handleError(func) {
-  try {
-    await func()
-  } catch (err) {
-    if (err && err.err) {
-      logErrorToFile(ERROR_FILE, err.err)
-    }
-    throw `Failed to start - logs written to file: ${ERROR_FILE}`
-  }
-}
-
-async function init(type) {
-  const isQuick = type === InitTypes.QUICK || type === InitTypes.DIGITAL_OCEAN
-  await checkDockerConfigured()
-  if (!isQuick) {
-    const shouldContinue = await confirmation(
-      "This will create multiple files in current directory, should continue?"
-    )
-    if (!shouldContinue) {
-      console.log("Stopping.")
-      return
-    }
-  }
-  captureEvent(AnalyticsEvents.SelfHostInit, {
-    type,
-  })
-  await downloadFiles()
-  const config = isQuick ? makeEnv.QUICK_CONFIG : {}
-  if (type === InitTypes.DIGITAL_OCEAN) {
-    try {
-      const output = await axios.get(DO_USER_DATA_URL)
-      const response = parseEnv(output.data)
-      for (let [key, value] of Object.entries(makeEnv.ConfigMap)) {
-        if (response[key]) {
-          config[value] = response[key]
-        }
-      }
-    } catch (err) {
-      // don't need to handle error, just don't do anything
-    }
-  }
-  await makeEnv.make(config)
-}
-
-async function start() {
-  await checkDockerConfigured()
-  checkInitComplete()
-  console.log(
-    info(
-      "Starting services, this may take a moment - first time this may take a few minutes to download images."
-    )
-  )
-  const port = makeEnv.get("MAIN_PORT")
-  await handleError(async () => {
-    // need to log as it makes it more clear
-    await compose.upAll({ cwd: "./", log: true })
-  })
-  console.log(
-    success(
-      `Services started, please go to http://localhost:${port} for next steps.`
-    )
-  )
-}
-
-async function status() {
-  await checkDockerConfigured()
-  checkInitComplete()
-  console.log(info("Budibase status"))
-  await handleError(async () => {
-    const response = await compose.ps()
-    console.log(response.out)
-  })
-}
-
-async function stop() {
-  await checkDockerConfigured()
-  checkInitComplete()
-  console.log(info("Stopping services, this may take a moment."))
-  await handleError(async () => {
-    await compose.stop()
-  })
-  console.log(success("Services have been stopped successfully."))
-}
-
-async function update() {
-  await checkDockerConfigured()
-  checkInitComplete()
-  if (await confirmation("Do you wish to update you docker-compose.yaml?")) {
-    await downloadFiles()
-  }
-  await handleError(async () => {
-    const status = await compose.ps()
-    const parts = status.out.split("\n")
-    const isUp = parts[2] && parts[2].indexOf("Up") !== -1
-    if (isUp) {
-      console.log(info("Stopping services, this may take a moment."))
-      await compose.stop()
-    }
-    console.log(info("Beginning update, this may take a few minutes."))
-    await compose.pullMany(BUDIBASE_SERVICES, { log: true })
-    if (isUp) {
-      console.log(success("Update complete, restarting services..."))
-      await start()
-    }
-  })
-}
 
 const command = new Command(`${CommandWords.HOSTING}`)
   .addHelp("Controls self hosting on the Budibase platform.")
   .addSubOption(
     "--init [type]",
-    "Configure a self hosted platform in current directory, type can be unspecified or 'quick'.",
+    "Configure a self hosted platform in current directory, type can be unspecified, 'quick' or 'single'.",
     init
   )
   .addSubOption(
@@ -181,5 +35,16 @@ const command = new Command(`${CommandWords.HOSTING}`)
     "Update the Budibase images to the latest version.",
     update
   )
+  .addSubOption(
+    "--watch-plugin-dir [directory]",
+    "Add plugin directory watching to a Budibase install.",
+    watchPlugins
+  )
+  .addSubOption(
+    "--gen-user",
+    "Create an admin user automatically as part of first start.",
+    generateUser
+  )
+  .addSubOption("--single", "Specify this with init to use the single image.")
 
 exports.command = command

packages/cli/src/hosting/init.js

@@ -0,0 +1,75 @@
+const { InitTypes, AnalyticsEvents } = require("../constants")
+const { confirmation } = require("../questions")
+const { captureEvent } = require("../events")
+const makeFiles = require("./makeFiles")
+const axios = require("axios")
+const { parseEnv } = require("../utils")
+const { checkDockerConfigured, downloadFiles } = require("./utils")
+const { watchPlugins } = require("./watch")
+const { generateUser } = require("./genUser")
+
+const DO_USER_DATA_URL = "http://169.254.169.254/metadata/v1/user-data"
+
+async function getInitConfig(type, isQuick, port) {
+  const config = isQuick ? makeFiles.QUICK_CONFIG : {}
+  if (type === InitTypes.DIGITAL_OCEAN) {
+    try {
+      const output = await axios.get(DO_USER_DATA_URL)
+      const response = parseEnv(output.data)
+      for (let [key, value] of Object.entries(makeFiles.ConfigMap)) {
+        if (response[key]) {
+          config[value] = response[key]
+        }
+      }
+    } catch (err) {
+      // don't need to handle error, just don't do anything
+    }
+  }
+  // override port
+  if (port) {
+    config[makeFiles.ConfigMap.MAIN_PORT] = port
+  }
+  return config
+}
+
+exports.init = async opts => {
+  let type, isSingle, watchDir, genUser, port, silent
+  if (typeof opts === "string") {
+    type = opts
+  } else {
+    type = opts["init"]
+    isSingle = opts["single"]
+    watchDir = opts["watchPluginDir"]
+    genUser = opts["genUser"]
+    port = opts["port"]
+    silent = opts["silent"]
+  }
+  const isQuick = type === InitTypes.QUICK || type === InitTypes.DIGITAL_OCEAN
+  await checkDockerConfigured()
+  if (!isQuick) {
+    const shouldContinue = await confirmation(
+      "This will create multiple files in current directory, should continue?"
+    )
+    if (!shouldContinue) {
+      console.log("Stopping.")
+      return
+    }
+  }
+  captureEvent(AnalyticsEvents.SelfHostInit, {
+    type,
+  })
+  const config = await getInitConfig(type, isQuick, port)
+  if (!isSingle) {
+    await downloadFiles()
+    await makeFiles.makeEnv(config, silent)
+  } else {
+    await makeFiles.makeSingleCompose(config, silent)
+  }
+  if (watchDir) {
+    await watchPlugins(watchDir, silent)
+  }
+  if (genUser) {
+    const inputPassword = typeof genUser === "string" ? genUser : null
+    await generateUser(inputPassword, silent)
+  }
+}

packages/cli/src/hosting/makeEnv.js

@@ -1,66 +0,0 @@
-const { number } = require("../questions")
-const { success } = require("../utils")
-const fs = require("fs")
-const path = require("path")
-const randomString = require("randomstring")
-
-const FILE_PATH = path.resolve("./.env")
-
-function getContents(port) {
-  return `
-# Use the main port in the builder for your self hosting URL, e.g. localhost:10000
-MAIN_PORT=${port}
-
-# This section contains all secrets pertaining to the system
-JWT_SECRET=${randomString.generate()}
-MINIO_ACCESS_KEY=${randomString.generate()}
-MINIO_SECRET_KEY=${randomString.generate()}
-COUCH_DB_PASSWORD=${randomString.generate()}
-COUCH_DB_USER=${randomString.generate()}
-REDIS_PASSWORD=${randomString.generate()}
-INTERNAL_API_KEY=${randomString.generate()}
-
-# This section contains variables that do not need to be altered under normal circumstances
-APP_PORT=4002
-WORKER_PORT=4003
-MINIO_PORT=4004
-COUCH_DB_PORT=4005
-REDIS_PORT=6379
-WATCHTOWER_PORT=6161
-BUDIBASE_ENVIRONMENT=PRODUCTION`
-}
-
-module.exports.filePath = FILE_PATH
-module.exports.ConfigMap = {
-  MAIN_PORT: "port",
-}
-module.exports.QUICK_CONFIG = {
-  key: "budibase",
-  port: 10000,
-}
-
-module.exports.make = async (inputs = {}) => {
-  const hostingPort =
-    inputs.port ||
-    (await number(
-      "Please enter the port on which you want your installation to run: ",
-      10000
-    ))
-  const fileContents = getContents(hostingPort)
-  fs.writeFileSync(FILE_PATH, fileContents)
-  console.log(
-    success(
-      "Configuration has been written successfully - please check .env file for more details."
-    )
-  )
-}
-
-module.exports.get = property => {
-  const props = fs.readFileSync(FILE_PATH, "utf8").split(property)
-  if (props[0].charAt(0) === "=") {
-    property = props[0]
-  } else {
-    property = props[1]
-  }
-  return property.split("=")[1].split("\n")[0]
-}

packages/cli/src/hosting/makeFiles.js

@@ -0,0 +1,137 @@
+const { number } = require("../questions")
+const { success, stringifyToDotEnv } = require("../utils")
+const fs = require("fs")
+const path = require("path")
+const randomString = require("randomstring")
+const yaml = require("yaml")
+const { getAppService } = require("./utils")
+
+const SINGLE_IMAGE = "budibase/budibase:latest"
+const VOL_NAME = "budibase_data"
+const COMPOSE_PATH = path.resolve("./docker-compose.yaml")
+const ENV_PATH = path.resolve("./.env")
+
+function getSecrets(opts = { single: false }) {
+  const secrets = [
+    "JWT_SECRET",
+    "MINIO_ACCESS_KEY",
+    "MINIO_SECRET_KEY",
+    "REDIS_PASSWORD",
+    "INTERNAL_API_KEY",
+  ]
+  const obj = {}
+  secrets.forEach(secret => (obj[secret] = randomString.generate()))
+  // setup couch creds separately
+  if (opts && opts.single) {
+    obj["COUCHDB_USER"] = "admin"
+    obj["COUCHDB_PASSWORD"] = randomString.generate()
+  } else {
+    obj["COUCH_DB_USER"] = "admin"
+    obj["COUCH_DB_PASSWORD"] = randomString.generate()
+  }
+  return obj
+}
+
+function getSingleCompose(port) {
+  const singleComposeObj = {
+    version: "3",
+    services: {
+      budibase: {
+        restart: "unless-stopped",
+        image: SINGLE_IMAGE,
+        ports: [`${port}:80`],
+        environment: getSecrets({ single: true }),
+        volumes: [`${VOL_NAME}:/data`],
+      },
+    },
+    volumes: {
+      [VOL_NAME]: {
+        driver: "local",
+      },
+    },
+  }
+  return yaml.stringify(singleComposeObj)
+}
+
+function getEnv(port) {
+  const partOne = stringifyToDotEnv({
+    MAIN_PORT: port,
+  })
+  const partTwo = stringifyToDotEnv(getSecrets())
+  const partThree = stringifyToDotEnv({
+    APP_PORT: 4002,
+    WORKER_PORT: 4003,
+    MINIO_PORT: 4004,
+    COUCH_DB_PORT: 4005,
+    REDIS_PORT: 6379,
+    WATCHTOWER_PORT: 6161,
+    BUDIBASE_ENVIRONMENT: "PRODUCTION",
+  })
+  return [
+    "# Use the main port in the builder for your self hosting URL, e.g. localhost:10000",
+    partOne,
+    "# This section contains all secrets pertaining to the system",
+    partTwo,
+    "# This section contains variables that do not need to be altered under normal circumstances",
+    partThree,
+  ].join("\n")
+}
+
+exports.ENV_PATH = ENV_PATH
+exports.COMPOSE_PATH = COMPOSE_PATH
+
+module.exports.ConfigMap = {
+  MAIN_PORT: "port",
+}
+
+module.exports.QUICK_CONFIG = {
+  key: "budibase",
+  port: 10000,
+}
+
+async function make(path, contentsFn, inputs = {}, silent) {
+  const port =
+    inputs.port ||
+    (await number(
+      "Please enter the port on which you want your installation to run: ",
+      10000
+    ))
+  const fileContents = contentsFn(port)
+  fs.writeFileSync(path, fileContents)
+  if (!silent) {
+    console.log(
+      success(
+        `Configuration has been written successfully - please check ${path} for more details.`
+      )
+    )
+  }
+}
+
+module.exports.makeEnv = async (inputs = {}, silent) => {
+  return make(ENV_PATH, getEnv, inputs, silent)
+}
+
+module.exports.makeSingleCompose = async (inputs = {}, silent) => {
+  return make(COMPOSE_PATH, getSingleCompose, inputs, silent)
+}
+
+module.exports.getEnvProperty = property => {
+  const props = fs.readFileSync(ENV_PATH, "utf8").split(property)
+  if (props[0].charAt(0) === "=") {
+    property = props[0]
+  } else {
+    property = props[1]
+  }
+  return property.split("=")[1].split("\n")[0]
+}
+
+module.exports.getComposeProperty = property => {
+  const { service } = getAppService(COMPOSE_PATH)
+  if (property === "port" && Array.isArray(service.ports)) {
+    const port = service.ports[0]
+    return port.split(":")[0]
+  } else if (service.environment) {
+    return service.environment[property]
+  }
+  return null
+}

packages/cli/src/hosting/start.js

@@ -0,0 +1,34 @@
+const {
+  checkDockerConfigured,
+  checkInitComplete,
+  handleError,
+} = require("./utils")
+const { info, success } = require("../utils")
+const makeFiles = require("./makeFiles")
+const compose = require("docker-compose")
+const fs = require("fs")
+
+exports.start = async () => {
+  await checkDockerConfigured()
+  checkInitComplete()
+  console.log(
+    info(
+      "Starting services, this may take a moment - first time this may take a few minutes to download images."
+    )
+  )
+  let port
+  if (fs.existsSync(makeFiles.ENV_PATH)) {
+    port = makeFiles.getEnvProperty("MAIN_PORT")
+  } else {
+    port = makeFiles.getComposeProperty("port")
+  }
+  await handleError(async () => {
+    // need to log as it makes it more clear
+    await compose.upAll({ cwd: "./", log: true })
+  })
+  console.log(
+    success(
+      `Services started, please go to http://localhost:${port} for next steps.`
+    )
+  )
+}

packages/cli/src/hosting/status.js

@@ -0,0 +1,17 @@
+const {
+  checkDockerConfigured,
+  checkInitComplete,
+  handleError,
+} = require("./utils")
+const { info } = require("../utils")
+const compose = require("docker-compose")
+
+exports.status = async () => {
+  await checkDockerConfigured()
+  checkInitComplete()
+  console.log(info("Budibase status"))
+  await handleError(async () => {
+    const response = await compose.ps()
+    console.log(response.out)
+  })
+}

packages/cli/src/hosting/stop.js

@@ -0,0 +1,17 @@
+const {
+  checkDockerConfigured,
+  checkInitComplete,
+  handleError,
+} = require("./utils")
+const { info, success } = require("../utils")
+const compose = require("docker-compose")
+
+exports.stop = async () => {
+  await checkDockerConfigured()
+  checkInitComplete()
+  console.log(info("Stopping services, this may take a moment."))
+  await handleError(async () => {
+    await compose.stop()
+  })
+  console.log(success("Services have been stopped successfully."))
+}

packages/cli/src/hosting/update.js

@@ -0,0 +1,49 @@
+const {
+  checkDockerConfigured,
+  checkInitComplete,
+  downloadFiles,
+  handleError,
+  getServices,
+} = require("./utils")
+const { confirmation } = require("../questions")
+const compose = require("docker-compose")
+const { COMPOSE_PATH } = require("./makeFiles")
+const { info, success } = require("../utils")
+const { start } = require("./start")
+
+const BB_COMPOSE_SERVICES = ["app-service", "worker-service", "proxy-service"]
+const BB_SINGLE_SERVICE = ["budibase"]
+
+exports.update = async () => {
+  const { services } = getServices(COMPOSE_PATH)
+  const isSingle = Object.keys(services).length === 1
+  await checkDockerConfigured()
+  checkInitComplete()
+  if (
+    !isSingle &&
+    (await confirmation("Do you wish to update you docker-compose.yaml?"))
+  ) {
+    await downloadFiles()
+  }
+  await handleError(async () => {
+    const status = await compose.ps()
+    const parts = status.out.split("\n")
+    const isUp = parts[2] && parts[2].indexOf("Up") !== -1
+    if (isUp) {
+      console.log(info("Stopping services, this may take a moment."))
+      await compose.stop()
+    }
+    console.log(info("Beginning update, this may take a few minutes."))
+    let services
+    if (isSingle) {
+      services = BB_SINGLE_SERVICE
+    } else {
+      services = BB_COMPOSE_SERVICES
+    }
+    await compose.pullMany(services, { log: true })
+    if (isUp) {
+      console.log(success("Update complete, restarting services..."))
+      await start()
+    }
+  })
+}

packages/cli/src/hosting/utils.js

@@ -0,0 +1,87 @@
+const { lookpath } = require("lookpath")
+const fs = require("fs")
+const makeFiles = require("./makeFiles")
+const { logErrorToFile, downloadFile, error } = require("../utils")
+const yaml = require("yaml")
+
+const ERROR_FILE = "docker-error.log"
+const FILE_URLS = [
+  "https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml",
+]
+
+exports.downloadFiles = async () => {
+  const promises = []
+  for (let url of FILE_URLS) {
+    const fileName = url.split("/").slice(-1)[0]
+    promises.push(downloadFile(url, `./${fileName}`))
+  }
+  await Promise.all(promises)
+}
+
+exports.checkDockerConfigured = async () => {
+  const error =
+    "docker/docker-compose has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
+  const docker = await lookpath("docker")
+  const compose = await lookpath("docker-compose")
+  if (!docker || !compose) {
+    throw error
+  }
+}
+
+exports.checkInitComplete = () => {
+  if (
+    !fs.existsSync(makeFiles.ENV_PATH) &&
+    !fs.existsSync(makeFiles.COMPOSE_PATH)
+  ) {
+    throw "Please run the hosting --init command before any other hosting command."
+  }
+}
+
+exports.handleError = async func => {
+  try {
+    await func()
+  } catch (err) {
+    if (err && err.err) {
+      logErrorToFile(ERROR_FILE, err.err)
+    }
+    throw `Failed to start - logs written to file: ${ERROR_FILE}`
+  }
+}
+
+exports.getServices = path => {
+  const dockerYaml = fs.readFileSync(path, "utf8")
+  const parsedYaml = yaml.parse(dockerYaml)
+  return { yaml: parsedYaml, services: parsedYaml.services }
+}
+
+exports.getAppService = path => {
+  const { yaml, services } = exports.getServices(path),
+    serviceList = Object.keys(services)
+  let service
+  if (services["app-service"]) {
+    service = services["app-service"]
+  } else if (serviceList.length === 1) {
+    service = services[serviceList[0]]
+  }
+  return { yaml, service }
+}
+
+exports.updateDockerComposeService = updateFn => {
+  const opts = ["docker-compose.yaml", "docker-compose.yml"]
+  const dockerFilePath = opts.find(name => fs.existsSync(name))
+  if (!dockerFilePath) {
+    console.log(error("Unable to locate docker-compose YAML."))
+    return
+  }
+  const { yaml: parsedYaml, service } = exports.getAppService(dockerFilePath)
+  if (!service) {
+    console.log(
+      error(
+        "Unable to locate service within compose file, is it a valid Budibase configuration?"
+      )
+    )
+    return
+  }
+  updateFn(service)
+  fs.writeFileSync(dockerFilePath, yaml.stringify(parsedYaml))
+}

packages/cli/src/hosting/watch.js

@@ -0,0 +1,36 @@
+const { resolve } = require("path")
+const fs = require("fs")
+const { error, success } = require("../utils")
+const { updateDockerComposeService } = require("./utils")
+
+exports.watchPlugins = async (pluginPath, silent) => {
+  const PLUGIN_PATH = "/plugins"
+  // get absolute path
+  pluginPath = resolve(pluginPath)
+  if (!fs.existsSync(pluginPath)) {
+    console.log(
+      error(
+        `The directory "${pluginPath}" does not exist, please create and then try again.`
+      )
+    )
+    return
+  }
+  updateDockerComposeService(service => {
+    // set environment variable
+    service.environment["PLUGINS_DIR"] = PLUGIN_PATH
+    // add volumes to parsed yaml
+    if (!service.volumes) {
+      service.volumes = []
+    }
+    const found = service.volumes.find(vol => vol.includes(PLUGIN_PATH))
+    if (found) {
+      service.volumes.splice(service.volumes.indexOf(found), 1)
+    }
+    service.volumes.push(`${pluginPath}:${PLUGIN_PATH}`)
+  })
+  if (!silent) {
+    console.log(
+      success(`Docker compose configured to watch directory: ${pluginPath}`)
+    )
+  }
+}

packages/cli/src/plugins/index.js

@@ -1,5 +1,5 @@
 const Command = require("../structures/Command")
-const { CommandWords, AnalyticsEvents } = require("../constants")
+const { CommandWords, AnalyticsEvents, InitTypes } = require("../constants")
 const { getSkeleton, fleshOutSkeleton } = require("./skeleton")
 const questions = require("../questions")
 const fs = require("fs")
@@ -9,6 +9,10 @@ const { runPkgCommand } = require("../exec")
 const { join } = require("path")
 const { success, error, info, moveDirectory } = require("../utils")
 const { captureEvent } = require("../events")
+const fp = require("find-free-port")
+const { GENERATED_USER_EMAIL } = require("../constants")
+const { init: hostingInit } = require("../hosting/init")
+const { start: hostingStart } = require("../hosting/start")
 
 function checkInPlugin() {
   if (!fs.existsSync("package.json")) {
@@ -141,6 +145,29 @@ async function watch() {
   }
 }
 
+async function dev() {
+  const pluginDir = await questions.string("Directory to watch", "./")
+  const [port] = await fp(10000)
+  const password = "admin"
+  await hostingInit({
+    init: InitTypes.QUICK,
+    single: true,
+    watchPluginDir: pluginDir,
+    genUser: password,
+    port,
+    silent: true,
+  })
+  await hostingStart()
+  console.log(success(`Configuration has been written to docker-compose.yaml`))
+  console.log(
+    success("Development environment started successfully - connect at: ") +
+      info(`http://localhost:${port}`)
+  )
+  console.log(success("Use the following credentials to login:"))
+  console.log(success("Email: ") + info(GENERATED_USER_EMAIL))
+  console.log(success("Password: ") + info(password))
+}
+
 const command = new Command(`${CommandWords.PLUGIN}`)
   .addHelp(
     "Custom plugins for Budibase, init, build and verify your components and datasources with this tool."
@@ -160,5 +187,10 @@ const command = new Command(`${CommandWords.PLUGIN}`)
     "Automatically build any changes to your plugin.",
     watch
   )
+  .addSubOption(
+    "--dev",
+    "Run a development environment which automatically watches the current directory.",
+    dev
+  )
 
 exports.command = command

packages/cli/src/structures/Command.js

@@ -1,4 +1,9 @@
-const { getSubHelpDescription, getHelpDescription, error } = require("../utils")
+const {
+  getSubHelpDescription,
+  getHelpDescription,
+  error,
+  capitaliseFirstLetter,
+} = require("../utils")
 
 class Command {
   constructor(command, func = null) {
@@ -8,6 +13,15 @@ class Command {
     this.func = func
   }
 
+  convertToCommander(lookup) {
+    const parts = lookup.toLowerCase().split("-")
+    // camel case, separate out first
+    const first = parts.shift()
+    return [first]
+      .concat(parts.map(part => capitaliseFirstLetter(part)))
+      .join("")
+  }
+
   addHelp(help) {
     this.help = help
     return this
@@ -25,10 +39,7 @@ class Command {
       command = command.description(getHelpDescription(thisCmd.help))
     }
     for (let opt of thisCmd.opts) {
-      command = command.option(
+      command = command.option(opt.command, getSubHelpDescription(opt.help))
-        `${opt.command}`,
-        getSubHelpDescription(opt.help)
-      )
     }
     command.helpOption(
       "--help",
@@ -36,17 +47,25 @@ class Command {
     )
     command.action(async options => {
       try {
-        let executed = false
+        let executed = false,
+          found = false
         for (let opt of thisCmd.opts) {
-          const lookup = opt.command.split(" ")[0].replace("--", "")
+          let lookup = opt.command.split(" ")[0].replace("--", "")
-          if (!executed && options[lookup]) {
+          // need to handle how commander converts watch-plugin-dir to watchPluginDir
+          lookup = this.convertToCommander(lookup)
+          found = !executed && options[lookup]
+          if (found && opt.func) {
             const input =
               Object.keys(options).length > 1 ? options : options[lookup]
             await opt.func(input)
             executed = true
           }
         }
-        if (!executed) {
+        if (found && !executed) {
+          console.log(
+            error(`${Object.keys(options)[0]} is an option, not an operation.`)
+          )
+        } else if (!executed) {
           console.log(error(`Unknown ${this.command} option.`))
           command.help()
         }

packages/cli/src/structures/ConfigManager.js

@@ -33,11 +33,10 @@ class ConfigManager {
   }
 
   setValue(key, value) {
-    const updated = {
+    this.config = {
       ...this.config,
       [key]: value,
     }
-    this.config = updated
   }
 
   removeKey(key) {

packages/cli/src/utils.js

@@ -84,3 +84,15 @@ exports.moveDirectory = (oldPath, newPath) => {
   }
   fs.rmdirSync(oldPath)
 }
+
+exports.capitaliseFirstLetter = str => {
+  return str.charAt(0).toUpperCase() + str.slice(1)
+}
+
+exports.stringifyToDotEnv = json => {
+  let str = ""
+  for (let [key, value] of Object.entries(json)) {
+    str += `${key}=${value}\n`
+  }
+  return str
+}

packages/cli/yarn.lock

@@ -1113,6 +1113,11 @@ fill-range@^7.0.1:
   dependencies:
     to-regex-range "^5.0.1"
 
+find-free-port@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/find-free-port/-/find-free-port-2.0.0.tgz#4b22e5f6579eb1a38c41ac6bcb3efed1b6da9b1b"
+  integrity sha512-J1j8gfEVf5FN4PR5w5wrZZ7NYs2IvqsHcd03cAeQx3Ec/mo+lKceaVNhpsRKoZpZKbId88o8qh+dwUwzBV6WCg==
+
 find-replace@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/find-replace/-/find-replace-3.0.0.tgz#3e7e23d3b05167a76f770c9fbd5258b0def68c38"
@@ -3080,6 +3085,11 @@ yallist@^4.0.0:
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
   integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
 
+yaml@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.1.1.tgz#1e06fb4ca46e60d9da07e4f786ea370ed3c3cfec"
+  integrity sha512-o96x3OPo8GjWeSLF+wOAbrPfhFOGY0W00GNaxCDv+9hkcDJEnev1yh8S7pgHF0ik6zc8sQLuL8hjHjJULZp8bw==
+
 yargs-parser@^20.2.2:
   version "20.2.9"
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee"

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.14-alpha.0",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "1.4.18-alpha.1",
+    "@budibase/bbui": "2.0.14-alpha.0",
-    "@budibase/frontend-core": "1.4.18-alpha.1",
+    "@budibase/frontend-core": "2.0.14-alpha.0",
-    "@budibase/string-templates": "1.4.18-alpha.1",
+    "@budibase/string-templates": "2.0.14-alpha.0",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/client/src/components/ClientApp.svelte

@@ -16,6 +16,7 @@
     themeStore,
     appStore,
     devToolsStore,
+    environmentStore,
   } from "stores"
   import NotificationDisplay from "components/overlay/NotificationDisplay.svelte"
   import ConfirmationDisplay from "components/overlay/ConfirmationDisplay.svelte"
@@ -47,6 +48,8 @@
     !$builderStore.inBuilder &&
     $devToolsStore.enabled &&
     !$routeStore.queryParams?.peek
+  $: objectStoreUrl = $environmentStore.cloud ? "https://cdn.budi.live" : ""
+  $: pluginsUrl = `${objectStoreUrl}/plugins`
 
   // Handle no matching route
   $: {
@@ -92,7 +95,8 @@
 <svelte:head>
   {#if $builderStore.usedPlugins?.length}
     {#each $builderStore.usedPlugins as plugin (plugin.hash)}
-      <script src={`/plugins/${plugin.jsUrl}?r=${plugin.hash || ""}`}></script>
+      <script
+        src={`${pluginsUrl}/${plugin.jsUrl}?r=${plugin.hash || ""}`}></script>
     {/each}
   {/if}
 </svelte:head>

packages/client/src/index.js

@@ -1,5 +1,11 @@
 import ClientApp from "./components/ClientApp.svelte"
-import { componentStore, builderStore, appStore, devToolsStore } from "./stores"
+import {
+  componentStore,
+  builderStore,
+  appStore,
+  devToolsStore,
+  environmentStore,
+} from "./stores"
 import loadSpectrumIcons from "@budibase/bbui/spectrum-icons-rollup.js"
 import { get } from "svelte/store"
 import { initWebsocket } from "./websocket.js"
@@ -15,7 +21,7 @@ loadSpectrumIcons()
 
 let app
 
-const loadBudibase = () => {
+const loadBudibase = async () => {
   // Update builder store with any builder flags
   builderStore.set({
     inBuilder: !!window["##BUDIBASE_IN_BUILDER##"],
@@ -36,6 +42,9 @@ const loadBudibase = () => {
   // server rendered app HTML
   appStore.actions.setAppId(window["##BUDIBASE_APP_ID##"])
 
+  // Fetch environment info
+  await environmentStore.actions.fetchEnvironment()
+
   // Enable dev tools or not. We need to be using a dev app and not inside
   // the builder preview to enable them.
   const enableDevTools = !get(builderStore).inBuilder && get(appStore).isDevApp

packages/client/src/stores/builder.js

@@ -98,6 +98,9 @@ const createBuilderStore = () => {
           return state
         })
       }
+
+      // Notify the builder so we can reload component definitions
+      dispatchEvent("reload-plugin")
     },
   }
   return {

packages/client/src/stores/initialise.js

@@ -1,9 +1,7 @@
 import { routeStore } from "./routes"
 import { appStore } from "./app"
-import { environmentStore } from "./environment"
 
 export async function initialise() {
   await routeStore.actions.fetchRoutes()
   await appStore.actions.fetchAppDefinition()
-  await environmentStore.actions.fetchEnvironment()
 }

packages/client/src/websocket.js

@@ -1,13 +1,19 @@
-import { builderStore, environmentStore } from "./stores/index.js"
+import {
+  builderStore,
+  environmentStore,
+  notificationStore,
+} from "./stores/index.js"
 import { get } from "svelte/store"
 import { io } from "socket.io-client"
 
+let socket
+
 export const initWebsocket = () => {
   const { inBuilder, location } = get(builderStore)
   const { cloud } = get(environmentStore)
 
   // Only connect when we're inside the builder preview, for now
-  if (!inBuilder || !location || cloud) {
+  if (!inBuilder || !location || cloud || socket) {
     return
   }
 
@@ -16,20 +22,20 @@ export const initWebsocket = () => {
   const proto = tls ? "wss:" : "ws:"
   const host = location.hostname
   const port = location.port || (tls ? 443 : 80)
-  const socket = io(`${proto}//${host}:${port}`, {
+  socket = io(`${proto}//${host}:${port}`, {
     path: "/socket/client",
-    // Cap reconnection attempts to 10 (total of 95 seconds before giving up)
+    // Cap reconnection attempts to 3 (total of 15 seconds before giving up)
-    reconnectionAttempts: 10,
+    reconnectionAttempts: 3,
-    // Delay initial reconnection attempt by 5 seconds
+    // Delay reconnection attempt by 5 seconds
     reconnectionDelay: 5000,
-    // Then decrease to 10 second intervals
+    reconnectionDelayMax: 5000,
-    reconnectionDelayMax: 10000,
+    // Timeout after 4 seconds so we never stack requests
-    // Timeout after 5 seconds so we never stack requests
+    timeout: 4000,
-    timeout: 5000,
   })
 
   // Event handlers
   socket.on("plugin-update", data => {
     builderStore.actions.updateUsedPlugin(data.name, data.hash)
+    notificationStore.actions.info(`"${data.name}" plugin reloaded`)
   })
 }

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.14-alpha.0",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "1.4.18-alpha.1",
+    "@budibase/bbui": "2.0.14-alpha.0",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/frontend-core/src/api/roles.js

@@ -29,4 +29,13 @@ export const buildRoleEndpoints = API => ({
       url: "/api/roles",
     })
   },
+
+  /**
+   * Gets a list of roles within a specified app.
+   */
+  getRolesForApp: async appId => {
+    return await API.get({
+      url: `/api/global/roles/${appId}`,
+    })
+  },
 })

packages/frontend-core/src/api/rows.js

@@ -8,10 +8,9 @@ export const buildRowEndpoints = API => ({
     if (!tableId || !rowId) {
       return null
     }
-    const row = await API.get({
+    return await API.get({
       url: `/api/${tableId}/rows/${rowId}`,
     })
-    return (await API.enrichRows([row], tableId))[0]
   },
 
   /**

packages/frontend-core/src/constants.js

@@ -40,7 +40,7 @@ export const OperatorOptions = {
   },
   NotContains: {
     value: "notContains",
-    label: "Does Not Contain",
+    label: "Does not contain",
   },
   In: {
     value: "oneOf",

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.14-alpha.0",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.14-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "1.4.18-alpha.1",
+    "@budibase/backend-core": "2.0.14-alpha.0",
-    "@budibase/client": "1.4.18-alpha.1",
+    "@budibase/client": "2.0.14-alpha.0",
-    "@budibase/pro": "1.4.18-alpha.1",
+    "@budibase/pro": "2.0.14-alpha.0",
-    "@budibase/string-templates": "1.4.18-alpha.1",
+    "@budibase/string-templates": "2.0.14-alpha.0",
-    "@budibase/types": "1.4.18-alpha.1",
+    "@budibase/types": "2.0.14-alpha.0",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -146,7 +146,7 @@
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
-    "vm2": "3.9.6",
+    "vm2": "3.9.11",
     "worker-farm": "1.7.0",
     "xml2js": "0.4.23",
     "yargs": "13.2.4",

packages/server/src/api/controllers/application.ts

@@ -356,7 +356,7 @@ const appPostCreate = async (ctx: any, app: App) => {
   await creationEvents(ctx.request, app)
   // app import & template creation
   if (ctx.request.body.useTemplate === "true") {
-    const rows = await getUniqueRows([app.appId])
+    const { rows } = await getUniqueRows([app.appId])
     const rowCount = rows ? rows.length : 0
     if (rowCount) {
       try {
@@ -490,7 +490,7 @@ const destroyApp = async (ctx: any) => {
 }
 
 const preDestroyApp = async (ctx: any) => {
-  const rows = await getUniqueRows([ctx.params.appId])
+  const { rows } = await getUniqueRows([ctx.params.appId])
   ctx.rowCount = rows.length
 }
 

packages/server/src/api/controllers/query/index.ts

@@ -153,7 +153,10 @@ export async function preview(ctx: any) {
           auth: { ...authConfigCtx },
         },
       })
-    const { rows, keys, info, extra } = await quotas.addQuery(runFn)
+
+    const { rows, keys, info, extra } = await quotas.addQuery(runFn, {
+      datasourceId: datasource._id,
+    })
     const schemaFields: any = {}
     if (rows?.length > 0) {
       for (let key of [...new Set(keys)] as string[]) {
@@ -234,7 +237,9 @@ async function execute(
         },
       })
 
-    const { rows, pagination, extra } = await quotas.addQuery(runFn)
+    const { rows, pagination, extra } = await quotas.addQuery(runFn, {
+      datasourceId: datasource._id,
+    })
     if (opts && opts.rowsOnly) {
       ctx.body = rows
     } else {

packages/server/src/api/controllers/row/index.ts

@@ -31,8 +31,11 @@ export async function patch(ctx: any): Promise<any> {
     return save(ctx)
   }
   try {
-    const { row, table } = await quotas.addQuery(() =>
+    const { row, table } = await quotas.addQuery(
-      pickApi(tableId).patch(ctx)
+      () => pickApi(tableId).patch(ctx),
+      {
+        datasourceId: tableId,
+      }
     )
     ctx.status = 200
     ctx.eventEmitter &&
@@ -54,7 +57,9 @@ export const save = async (ctx: any) => {
   }
   try {
     const { row, table } = await quotas.addRow(() =>
-      quotas.addQuery(() => pickApi(tableId).save(ctx))
+      quotas.addQuery(() => pickApi(tableId).save(ctx), {
+        datasourceId: tableId,
+      })
     )
     ctx.status = 200
     ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:save`, appId, row, table)
@@ -68,7 +73,9 @@ export const save = async (ctx: any) => {
 export async function fetchView(ctx: any) {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx))
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx), {
+      datasourceId: tableId,
+    })
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -77,7 +84,9 @@ export async function fetchView(ctx: any) {
 export async function fetch(ctx: any) {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx))
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx), {
+      datasourceId: tableId,
+    })
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -86,7 +95,9 @@ export async function fetch(ctx: any) {
 export async function find(ctx: any) {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx))
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx), {
+      datasourceId: tableId,
+    })
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -98,8 +109,11 @@ export async function destroy(ctx: any) {
   const tableId = getTableId(ctx)
   let response, row
   if (inputs.rows) {
-    let { rows } = await quotas.addQuery(() =>
+    let { rows } = await quotas.addQuery(
-      pickApi(tableId).bulkDestroy(ctx)
+      () => pickApi(tableId).bulkDestroy(ctx),
+      {
+        datasourceId: tableId,
+      }
     )
     await quotas.removeRows(rows.length)
     response = rows
@@ -107,7 +121,9 @@ export async function destroy(ctx: any) {
       ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row)
     }
   } else {
-    let resp = await quotas.addQuery(() => pickApi(tableId).destroy(ctx))
+    let resp = await quotas.addQuery(() => pickApi(tableId).destroy(ctx), {
+      datasourceId: tableId,
+    })
     await quotas.removeRow()
     response = resp.response
     row = resp.row
@@ -123,7 +139,9 @@ export async function search(ctx: any) {
   const tableId = getTableId(ctx)
   try {
     ctx.status = 200
-    ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx))
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx), {
+      datasourceId: tableId,
+    })
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -141,8 +159,11 @@ export async function validate(ctx: any) {
 export async function fetchEnrichedRow(ctx: any) {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await quotas.addQuery(() =>
+    ctx.body = await quotas.addQuery(
-      pickApi(tableId).fetchEnrichedRow(ctx)
+      () => pickApi(tableId).fetchEnrichedRow(ctx),
+      {
+        datasourceId: tableId,
+      }
     )
   } catch (err) {
     ctx.throw(400, err)
@@ -152,7 +173,9 @@ export async function fetchEnrichedRow(ctx: any) {
 export const exportRows = async (ctx: any) => {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx))
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx), {
+      datasourceId: tableId,
+    })
   } catch (err) {
     ctx.throw(400, err)
   }

packages/server/src/api/controllers/table/internal.ts

@@ -145,7 +145,9 @@ export async function destroy(ctx: any) {
   await db.bulkDocs(
     rows.rows.map((row: any) => ({ ...row.doc, _deleted: true }))
   )
-  await quotas.removeRows(rows.rows.length)
+  await quotas.removeRows(rows.rows.length, {
+    tableId: ctx.params.tableId,
+  })
 
   // update linked rows
   await updateLinks({

packages/server/src/api/controllers/table/utils.ts

@@ -148,7 +148,9 @@ export async function handleDataImport(user: any, table: any, dataImport: any) {
     finalData.push(row)
   }
 
-  await quotas.addRows(finalData.length, () => db.bulkDocs(finalData))
+  await quotas.addRows(finalData.length, () => db.bulkDocs(finalData), {
+    tableId: table._id,
+  })
   await events.rows.imported(table, "csv", finalData.length)
   return table
 }

packages/server/src/api/routes/tests/row.spec.js

@@ -34,18 +34,13 @@ describe("/rows", () => {
       .expect(status)
 
   const getRowUsage = async () => {
-    return config.doInContext(null, () =>
+    const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS))
-      quotas.getCurrentUsageValue(QuotaUsageType.STATIC, StaticQuotaName.ROWS)
+    return total
-    )
   }
 
   const getQueryUsage = async () => {
-    return config.doInContext(null, () =>
+    const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.MONTHLY, MonthlyQuotaName.QUERIES))
-      quotas.getCurrentUsageValue(
+    return total
-        QuotaUsageType.MONTHLY,
-        MonthlyQuotaName.QUERIES
-      )
-    )
   }
 
   const assertRowUsage = async expected => {
@@ -60,26 +55,26 @@ describe("/rows", () => {
 
   describe("save, load, update", () => {
     it("returns a success message when the row is created", async () => {
-      // const rowUsage = await getRowUsage()
+      const rowUsage = await getRowUsage()
-      // const queryUsage = await getQueryUsage()
+      const queryUsage = await getQueryUsage()
-      //
+
-      // const res = await request
+      const res = await request
-      //   .post(`/api/${row.tableId}/rows`)
+        .post(`/api/${row.tableId}/rows`)
-      //   .send(row)
+        .send(row)
-      //   .set(config.defaultHeaders())
+        .set(config.defaultHeaders())
-      //   .expect('Content-Type', /json/)
+        .expect('Content-Type', /json/)
-      //   .expect(200)
+        .expect(200)
-      // expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`)
+      expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`)
-      // expect(res.body.name).toEqual("Test Contact")
+      expect(res.body.name).toEqual("Test Contact")
-      // expect(res.body._rev).toBeDefined()
+      expect(res.body._rev).toBeDefined()
-      // await assertRowUsage(rowUsage + 1)
+      await assertRowUsage(rowUsage + 1)
-      // await assertQueryUsage(queryUsage + 1)
+      await assertQueryUsage(queryUsage + 1)
     })
 
     it("updates a row successfully", async () => {
       const existing = await config.createRow()
-      // const rowUsage = await getRowUsage()
+      const rowUsage = await getRowUsage()
-      // const queryUsage = await getQueryUsage()
+      const queryUsage = await getQueryUsage()
 
       const res = await request
         .post(`/api/${table._id}/rows`)
@@ -97,8 +92,8 @@ describe("/rows", () => {
         `${table.name} updated successfully.`
       )
       expect(res.body.name).toEqual("Updated Name")
-      // await assertRowUsage(rowUsage)
+      await assertRowUsage(rowUsage)
-      // await assertQueryUsage(queryUsage + 1)
+      await assertQueryUsage(queryUsage + 1)
     })
 
     it("should load a row", async () => {

packages/server/src/automations/tests/automation.spec.js

@@ -29,16 +29,11 @@ describe("Run through some parts of the automations system", () => {
   afterAll(setup.afterAll)
 
   it("should be able to init in builder", async () => {
-    await triggers.externalTrigger(basicAutomation(), { a: 1 })
+    await triggers.externalTrigger(basicAutomation(), { a: 1, appId: "app_123" })
     await wait(100)
     expect(thread.execute).toHaveBeenCalled()
   })
 
-  it("should be able to init in prod", async () => {
-    await triggers.externalTrigger(basicAutomation(), { a: 1 })
-    await wait(100)
-  })
-
   it("should check coercion", async () => {
     const table = await config.createTable()
     const automation = basicAutomation()

packages/server/src/automations/utils.ts

@@ -13,7 +13,7 @@ import {
   getAppId,
   getProdAppDB,
 } from "@budibase/backend-core/context"
-import { tenancy } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import { Automation } from "@budibase/types"
 
@@ -28,12 +28,14 @@ const jobMessage = (job: any, message: string) => {
 
 export async function processEvent(job: any) {
   try {
+    const automationId = job.data.automation._id
     console.log(jobMessage(job, "running"))
     // need to actually await these so that an error can be captured properly
-    const tenantId = tenancy.getTenantIDFromAppID(job.data.event.appId)
+    return await context.doInContext(job.data.event.appId, async () => {
-    return await tenancy.doInTenant(tenantId, async () => {
       const runFn = () => Runner.run(job)
-      return quotas.addAutomation(runFn)
+      return quotas.addAutomation(runFn, {
+        automationId,
+      })
     })
   } catch (err) {
     const errJson = JSON.stringify(err)

packages/server/src/db/utils.js

@@ -34,8 +34,6 @@ const DocumentType = {
   INSTANCE: "inst",
   LAYOUT: "layout",
   SCREEN: "screen",
-  DATASOURCE: "datasource",
-  DATASOURCE_PLUS: "datasource_plus",
   QUERY: "query",
   DEPLOYMENTS: "deployments",
   METADATA: "metadata",

packages/server/src/integrations/index.ts

@@ -13,6 +13,7 @@ import googlesheets from "./googlesheets"
 import firebase from "./firebase"
 import redis from "./redis"
 import snowflake from "./snowflake"
+import oracle from "./oracle"
 import { getPlugins } from "../api/controllers/plugin"
 import { SourceName, Integration, PluginType } from "@budibase/types"
 import { getDatasourcePlugin } from "../utilities/fileSystem"
@@ -56,8 +57,11 @@ const INTEGRATIONS: { [key: string]: any } = {
 }
 
 // optionally add oracle integration if the oracle binary can be installed
-if (process.arch && !process.arch.startsWith("arm")) {
+if (
-  const oracle = require("./oracle")
+  process.arch &&
+  !process.arch.startsWith("arm") &&
+  oracle.integration.isInstalled()
+) {
   DEFINITIONS[SourceName.ORACLE] = oracle.schema
   INTEGRATIONS[SourceName.ORACLE] = oracle.integration
 }
@@ -78,6 +82,9 @@ module.exports = {
           ...plugin.schema["schema"],
           custom: true,
         }
+        if (plugin.iconUrl) {
+          pluginSchemas[sourceId].iconUrl = plugin.iconUrl
+        }
       }
     }
     return {

packages/server/src/integrations/oracle.ts

@@ -15,17 +15,22 @@ import {
   getSqlQuery,
   SqlClient,
 } from "./utils"
-import oracledb, {
+import Sql from "./base/sql"
+import { FieldTypes } from "../constants"
+import {
   BindParameters,
   Connection,
   ConnectionAttributes,
   ExecuteOptions,
   Result,
 } from "oracledb"
-import Sql from "./base/sql"
+let oracledb: any
-import { FieldTypes } from "../constants"
+try {
-
+  oracledb = require("oracledb")
-oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT
+  oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT
+} catch (err) {
+  console.log("ORACLEDB is not installed")
+}
 
 interface OracleConfig {
   host: string
@@ -183,6 +188,10 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     return parts.join(" || ")
   }
 
+  static isInstalled() {
+    return oracledb != null
+  }
+
   /**
    * Map the flat tabular columns and constraints data into a nested object
    */

packages/server/src/migrations/functions/backfill/global.ts

@@ -8,7 +8,7 @@ import {
   accounts,
   db as dbUtils,
 } from "@budibase/backend-core"
-import { QuotaUsage } from "@budibase/pro"
+import { QuotaUsage } from "@budibase/types"
 import {
   CloudAccount,
   App,

packages/server/src/migrations/functions/pluginCount.ts

@@ -1,12 +0,0 @@
-import { tenancy, logging } from "@budibase/backend-core"
-import { plugins } from "@budibase/pro"
-
-export const run = async () => {
-  try {
-    await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => {
-      await plugins.checkPluginQuotas()
-    })
-  } catch (err) {
-    logging.logAlert("Failed to update plugin quotas", err)
-  }
-}

packages/server/src/migrations/functions/syncQuotas.ts

@@ -1,20 +1,15 @@
 import { runQuotaMigration } from "./usageQuotas"
 import * as syncApps from "./usageQuotas/syncApps"
 import * as syncRows from "./usageQuotas/syncRows"
+import * as syncPlugins from "./usageQuotas/syncPlugins"
 
 /**
- * Date:
+ * Synchronise quotas to the state of the db.
- * January 2022
- *
- * Description:
- * Synchronise the app and row quotas to the state of the db after it was
- * discovered that the quota resets were still in place and the row quotas
- * weren't being decremented correctly.
  */
-
 export const run = async () => {
   await runQuotaMigration(async () => {
     await syncApps.run()
     await syncRows.run()
+    await syncPlugins.run()
   })
 }

packages/server/src/migrations/functions/tests/syncQuotas.spec.js

@@ -2,11 +2,13 @@ const TestConfig = require("../../../tests/utilities/TestConfiguration")
 
 const syncApps = jest.fn()
 const syncRows = jest.fn()
+const syncPlugins = jest.fn()
 
 jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) )
 jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) )
+jest.mock("../usageQuotas/syncPlugins", () => ({ run: syncPlugins }) )
 
-const migration = require("../quotas1")
+const migration = require("../syncQuotas")
 
 describe("run", () => {
   let config = new TestConfig(false)
@@ -17,9 +19,10 @@ describe("run", () => {
 
   afterAll(config.end)  
 
-  it("runs ", async () => {
+  it("run", async () => {
     await migration.run()
     expect(syncApps).toHaveBeenCalledTimes(1)
     expect(syncRows).toHaveBeenCalledTimes(1)
+    expect(syncPlugins).toHaveBeenCalledTimes(1)
   })
 })

packages/server/src/migrations/functions/usageQuotas/syncApps.ts

@@ -5,7 +5,6 @@ import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 
 export const run = async () => {
   // get app count
-  // @ts-ignore
   const devApps = await getAllApps({ dev: true })
   const appCount = devApps ? devApps.length : 0
 

packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts

@@ -0,0 +1,10 @@
+import { logging } from "@budibase/backend-core"
+import { plugins } from "@budibase/pro"
+
+export const run = async () => {
+  try {
+    await plugins.checkPluginQuotas()
+  } catch (err) {
+    logging.logAlert("Failed to update plugin quotas", err)
+  }
+}

packages/server/src/migrations/functions/usageQuotas/syncRows.ts

@@ -2,19 +2,28 @@ import { getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
-import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
+import { StaticQuotaName, QuotaUsageType } from "@budibase/types"
 
 export const run = async () => {
   // get all rows in all apps
-  // @ts-ignore
   const allApps = await getAllApps({ all: true })
-  // @ts-ignore
   const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
-  const rows = await getUniqueRows(appIds)
+  const { appRows } = await getUniqueRows(appIds)
-  const rowCount = rows ? rows.length : 0
+
+  // get the counts per app
+  const counts: { [key: string]: number } = {}
+  let rowCount = 0
+  Object.entries(appRows).forEach(([appId, rows]) => {
+    counts[appId] = rows.length
+    rowCount += rows.length
+  })
 
   // sync row count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
-  await quotas.setUsage(rowCount, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
+  await quotas.setUsagePerApp(
+    counts,
+    StaticQuotaName.ROWS,
+    QuotaUsageType.STATIC
+  )
 }

packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -2,6 +2,7 @@ import TestConfig from "../../../../tests/utilities/TestConfiguration"
 import * as syncRows from "../syncRows"
 import { quotas } from "@budibase/pro"
 import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
+const { getProdAppID } = require("@budibase/backend-core/db")
 
 describe("syncRows", () => {
   let config = new TestConfig(false)
@@ -22,10 +23,11 @@ describe("syncRows", () => {
       expect(usageDoc.usageQuota.rows).toEqual(300)
 
       // app 1
+      const app1 = config.app
       await config.createTable()
       await config.createRow()
       // app 2
-      await config.createApp("second-app")
+      const app2 = await config.createApp("second-app")
       await config.createTable()
       await config.createRow()
       await config.createRow()
@@ -36,6 +38,12 @@ describe("syncRows", () => {
       // assert the migration worked
       usageDoc = await quotas.getQuotaUsage()
       expect(usageDoc.usageQuota.rows).toEqual(3)
+      expect(usageDoc.apps?.[getProdAppID(app1.appId)].usageQuota.rows).toEqual(
+        1
+      )
+      expect(usageDoc.apps?.[getProdAppID(app2.appId)].usageQuota.rows).toEqual(
+        2
+      )
     })
   })
 })

packages/server/src/migrations/index.ts

@@ -4,11 +4,9 @@ import env from "../environment"
 
 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
-import * as quota1 from "./functions/quotas1"
+import * as syncQuotas from "./functions/syncQuotas"
 import * as appUrls from "./functions/appUrls"
 import * as backfill from "./functions/backfill"
-import * as pluginCount from "./functions/pluginCount"
-
 /**
  * Populate the migration function and additional configuration from
  * the static migration definitions.
@@ -26,10 +24,10 @@ export const buildMigrations = () => {
         })
         break
       }
-      case MigrationName.QUOTAS_1: {
+      case MigrationName.SYNC_QUOTAS: {
         serverMigrations.push({
           ...definition,
-          fn: quota1.run,
+          fn: syncQuotas.run,
         })
         break
       }
@@ -69,16 +67,6 @@ export const buildMigrations = () => {
         })
         break
       }
-      case MigrationName.PLUGIN_COUNT: {
-        if (env.SELF_HOSTED) {
-          serverMigrations.push({
-            ...definition,
-            fn: pluginCount.run,
-            silent: !!env.SELF_HOSTED,
-            preventRetry: false,
-          })
-        }
-      }
     }
   }
 

packages/server/src/migrations/tests/index.spec.ts

@@ -4,7 +4,6 @@ import {
   tenancy,
   DocumentType,
   context,
-  db,
 } from "@budibase/backend-core"
 import TestConfig from "../../tests/utilities/TestConfiguration"
 import structures from "../../tests/utilities/structures"

packages/server/src/utilities/usageQuota/rows.js

@@ -2,6 +2,7 @@ const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
 const {
   isDevAppID,
   getDevelopmentAppID,
+  getProdAppID,
   doWithDB,
 } = require("@budibase/backend-core/db")
 
@@ -52,7 +53,8 @@ const getAppRows = async appId => {
  * Rows duplicates may exist across apps due to data import so they are not filtered out.
  */
 exports.getUniqueRows = async appIds => {
-  let uniqueRows = []
+  let uniqueRows = [],
+    rowsByApp = {}
   const pairs = getAppPairs(appIds)
 
   for (let pair of Object.values(pairs)) {
@@ -73,8 +75,10 @@ exports.getUniqueRows = async appIds => {
     // this can't be done on all rows because app import results in
     // duplicate row ids across apps
     // the array pre-concat is important to avoid stack overflow
-    uniqueRows = uniqueRows.concat([...new Set(appRows)])
+    const prodId = getProdAppID(pair.devId || pair.prodId)
+    rowsByApp[prodId] = [...new Set(appRows)]
+    uniqueRows = uniqueRows.concat(rowsByApp[prodId])
   }
 
-  return uniqueRows
+  return { rows: uniqueRows, appRows: rowsByApp }
 }

packages/server/yarn.lock

@@ -1094,12 +1094,12 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.4.18-alpha.1":
+"@budibase/backend-core@2.0.14-alpha.0":
-  version "1.4.18-alpha.1"
+  version "2.0.14-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.4.18-alpha.1.tgz#4222d3927d2a37bf7e505e17533c15fb0f134525"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.14-alpha.0.tgz#e4115967c9e37147216376bbabd622a9a13403d4"
-  integrity sha512-lYySQUPt8wuIXh3LRiUTlr81KHFLiJWa0piP1UDa5viqzxPRZ/UhvsrPFw+avJ7dZl2FR314PN2enO3q9/Az/A==
+  integrity sha512-igWtifz/AFZx3kbQi7yO+dRDQX7mbY/ZCA2aRhmDIlhm3zky94XgFyG/7iPFmxh9jK+gpg1Sg3axY7vNDSX6+Q==
   dependencies:
-    "@budibase/types" "1.4.18-alpha.1"
+    "@budibase/types" "2.0.14-alpha.0"
     "@shopify/jest-koa-mocks" "5.0.1"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-sdk "2.1030.0"
@@ -1180,13 +1180,13 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/pro@1.4.18-alpha.1":
+"@budibase/pro@2.0.14-alpha.0":
-  version "1.4.18-alpha.1"
+  version "2.0.14-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.4.18-alpha.1.tgz#a00471e34d352f57e5bf56f2ae596ff6c1311021"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.14-alpha.0.tgz#e6cb571a0a757871e9ab65555470e2c9f4fc4403"
-  integrity sha512-UkE1hgMTzWOBa67czStcGyegZAlmf/TgyUjNB8OxeyzQ7jCjUHdVAijnMUMVCfAnq0gdewg+vAu2UFck0dNDGw==
+  integrity sha512-Qh0U89AfnIpBA9fE4xH8hXYp4HexYSoc6WDjlVuNI46IvGRlHaeBAsRkI8XYG8mx830fQqVeEIY1WuRUso7bOg==
   dependencies:
-    "@budibase/backend-core" "1.4.18-alpha.1"
+    "@budibase/backend-core" "2.0.14-alpha.0"
-    "@budibase/types" "1.4.18-alpha.1"
+    "@budibase/types" "2.0.14-alpha.0"
     "@koa/router" "8.0.8"
     joi "17.6.0"
     node-fetch "^2.6.1"
@@ -1209,10 +1209,10 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/types@1.4.18-alpha.1":
+"@budibase/types@2.0.14-alpha.0":
-  version "1.4.18-alpha.1"
+  version "2.0.14-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.4.18-alpha.1.tgz#aad43686afe861838f89e69ecf16ccee9e45f462"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.14-alpha.0.tgz#419ceefde9698b1918c1b41f90fc3010927acde7"
-  integrity sha512-5fWImsg5cZomUSNs+DK0ZWH5pd4pzHk6x0mgHBcaL+CpzO+ROs/W1JWSVJ6uWYUi9ohy6G1FKB0+4/F5yUaOcw==
+  integrity sha512-20+VfYR9oIui3PDExL+3Ld0XWkrbD74CfWHS8+dYiRmW/PqUkhAT0suwpNui5OsVUn1I+9Jw0wvbitpgT5u2VQ==
 
 "@bull-board/api@3.7.0":
   version "3.7.0"
@@ -14362,10 +14362,10 @@ verror@1.10.0:
     core-util-is "1.0.2"
     extsprintf "^1.2.0"
 
-vm2@3.9.6:
+vm2@3.9.11:
-  version "3.9.6"
+  version "3.9.11"
-  resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.6.tgz#2f9b2fd0d82802dcd872e1011869ba8ae6b74778"
+  resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.11.tgz#a880f510a606481719ec3f9803b940c5805a06fe"
-  integrity sha512-BF7euUjgO+ezsz2UKex9kO9M/PtDNOf+KEpiqNepZsgf1MT7JYfJEIvG8BoYhZMLAVjqevFJ0UmXNuETe8m5dQ==
+  integrity sha512-PFG8iJRSjvvBdisowQ7iVF580DXb1uCIiGaXgm7tynMR1uTBlv7UJlB1zdv5KJ+Tmq1f0Upnj3fayoEOPpCBKg==
   dependencies:
     acorn "^8.7.0"
     acorn-walk "^8.2.0"

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.14-alpha.0",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/string-templates/yarn.lock

@@ -4413,9 +4413,9 @@ vlq@^0.2.2:
   integrity sha512-DRibZL6DsNhIgYQ+wNdWDL2SL3bKPlVrRiBqV5yuMm++op8W4kGFtaQfCs4KEJn0wBZcHVHJ3eoywX8983k1ow==
 
 vm2@^3.9.4:
-  version "3.9.6"
+  version "3.9.11"
-  resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.6.tgz#2f9b2fd0d82802dcd872e1011869ba8ae6b74778"
+  resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.11.tgz#a880f510a606481719ec3f9803b940c5805a06fe"
-  integrity sha512-BF7euUjgO+ezsz2UKex9kO9M/PtDNOf+KEpiqNepZsgf1MT7JYfJEIvG8BoYhZMLAVjqevFJ0UmXNuETe8m5dQ==
+  integrity sha512-PFG8iJRSjvvBdisowQ7iVF580DXb1uCIiGaXgm7tynMR1uTBlv7UJlB1zdv5KJ+Tmq1f0Upnj3fayoEOPpCBKg==
   dependencies:
     acorn "^8.7.0"
     acorn-walk "^8.2.0"

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/types",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.14-alpha.0",
   "description": "Budibase types",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/types/src/documents/global/plugin.ts

@@ -21,6 +21,7 @@ export interface Plugin extends Document {
   name: string
   version: string
   jsUrl?: string
+  iconUrl?: string
   source: PluginSource
   package: { [key: string]: any }
   hash: string

packages/types/src/documents/global/quotas.ts

@@ -1,15 +1,58 @@
 import { MonthlyQuotaName, StaticQuotaName } from "../../sdk"
 
-export interface QuotaUsage {
+export enum BreakdownQuotaName {
-  _id: string
+  ROW_QUERIES = "rowQueries",
-  _rev?: string
+  DATASOURCE_QUERIES = "datasourceQueries",
-  quotaReset: string
+  AUTOMATIONS = "automations",
+}
+
+export const APP_QUOTA_NAMES = [
+  StaticQuotaName.ROWS,
+  MonthlyQuotaName.QUERIES,
+  MonthlyQuotaName.AUTOMATIONS,
+]
+
+export const BREAKDOWN_QUOTA_NAMES = [
+  MonthlyQuotaName.QUERIES,
+  MonthlyQuotaName.AUTOMATIONS,
+]
+
+export interface UsageBreakdown {
+  parent: MonthlyQuotaName
+  values: {
+    [key: string]: number
+  }
+}
+
+export type MonthlyUsage = {
+  [MonthlyQuotaName.QUERIES]: number
+  [MonthlyQuotaName.AUTOMATIONS]: number
+  [MonthlyQuotaName.DAY_PASSES]: number
+  breakdown?: {
+    [key in BreakdownQuotaName]?: UsageBreakdown
+  }
+}
+
+export interface BaseQuotaUsage {
   usageQuota: {
     [key in StaticQuotaName]: number
   }
   monthly: {
-    [key: string]: {
+    [key: string]: MonthlyUsage
-      [key in MonthlyQuotaName]: number
-    }
   }
 }
+
+export interface QuotaUsage extends BaseQuotaUsage {
+  _id: string
+  _rev?: string
+  quotaReset: string
+  apps?: {
+    [key: string]: BaseQuotaUsage
+  }
+}
+
+export type UsageValues = {
+  total: number
+  app?: number
+  breakdown?: number
+}

packages/types/src/sdk/datasources.ts

@@ -96,6 +96,7 @@ export interface Integration {
   description: string
   friendlyName: string
   type?: string
+  iconUrl?: string
   datasource: {}
   query: {
     [key: string]: QueryDefinition

packages/types/src/sdk/licensing/quota.ts

@@ -27,6 +27,7 @@ export enum ConstantQuotaName {
   AUTOMATION_LOG_RETENTION_DAYS = "automationLogRetentionDays",
 }
 
+export type MeteredQuotaName = StaticQuotaName | MonthlyQuotaName
 export type QuotaName = StaticQuotaName | MonthlyQuotaName | ConstantQuotaName
 
 export const isStaticQuota = (

packages/types/src/sdk/migrations.ts

@@ -39,14 +39,13 @@ export interface MigrationOptions {
 
 export enum MigrationName {
   USER_EMAIL_VIEW_CASING = "user_email_view_casing",
-  QUOTAS_1 = "quotas_1",
   APP_URLS = "app_urls",
   EVENT_APP_BACKFILL = "event_app_backfill",
   EVENT_GLOBAL_BACKFILL = "event_global_backfill",
   EVENT_INSTALLATION_BACKFILL = "event_installation_backfill",
   GLOBAL_INFO_SYNC_USERS = "global_info_sync_users",
-  PLATFORM_USERS_EMAIL_CASING = "platform_users_email_casing",
+  // increment this number to re-activate this migration
-  PLUGIN_COUNT = "plugin_count",
+  SYNC_QUOTAS = "sync_quotas_1",
 }
 
 export interface MigrationDefinition {

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.14-alpha.0",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -36,10 +36,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "1.4.18-alpha.1",
+    "@budibase/backend-core": "2.0.14-alpha.0",
-    "@budibase/pro": "1.4.18-alpha.1",
+    "@budibase/pro": "2.0.14-alpha.0",
-    "@budibase/string-templates": "1.4.18-alpha.1",
+    "@budibase/string-templates": "2.0.14-alpha.0",
-    "@budibase/types": "1.4.18-alpha.1",
+    "@budibase/types": "2.0.14-alpha.0",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",

packages/worker/src/api/controllers/global/roles.js

@@ -2,6 +2,7 @@ const { getAllRoles } = require("@budibase/backend-core/roles")
 const {
   getAllApps,
   getProdAppID,
+  getDevAppID,
   DocumentType,
 } = require("@budibase/backend-core/db")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
@@ -34,7 +35,7 @@ exports.fetch = async ctx => {
 
 exports.find = async ctx => {
   const appId = ctx.params.appId
-  await doInAppContext(appId, async () => {
+  await doInAppContext(getDevAppID(appId), async () => {
     const db = getAppDB()
     const app = await db.get(DocumentType.APP_METADATA)
     ctx.body = {

packages/worker/yarn.lock

@@ -291,12 +291,12 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.4.18-alpha.1":
+"@budibase/backend-core@2.0.14-alpha.0":
-  version "1.4.18-alpha.1"
+  version "2.0.14-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.4.18-alpha.1.tgz#4222d3927d2a37bf7e505e17533c15fb0f134525"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.14-alpha.0.tgz#e4115967c9e37147216376bbabd622a9a13403d4"
-  integrity sha512-lYySQUPt8wuIXh3LRiUTlr81KHFLiJWa0piP1UDa5viqzxPRZ/UhvsrPFw+avJ7dZl2FR314PN2enO3q9/Az/A==
+  integrity sha512-igWtifz/AFZx3kbQi7yO+dRDQX7mbY/ZCA2aRhmDIlhm3zky94XgFyG/7iPFmxh9jK+gpg1Sg3axY7vNDSX6+Q==
   dependencies:
-    "@budibase/types" "1.4.18-alpha.1"
+    "@budibase/types" "2.0.14-alpha.0"
     "@shopify/jest-koa-mocks" "5.0.1"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-sdk "2.1030.0"
@@ -327,21 +327,21 @@
     uuid "8.3.2"
     zlib "1.0.5"
 
-"@budibase/pro@1.4.18-alpha.1":
+"@budibase/pro@2.0.14-alpha.0":
-  version "1.4.18-alpha.1"
+  version "2.0.14-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.4.18-alpha.1.tgz#a00471e34d352f57e5bf56f2ae596ff6c1311021"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.14-alpha.0.tgz#e6cb571a0a757871e9ab65555470e2c9f4fc4403"
-  integrity sha512-UkE1hgMTzWOBa67czStcGyegZAlmf/TgyUjNB8OxeyzQ7jCjUHdVAijnMUMVCfAnq0gdewg+vAu2UFck0dNDGw==
+  integrity sha512-Qh0U89AfnIpBA9fE4xH8hXYp4HexYSoc6WDjlVuNI46IvGRlHaeBAsRkI8XYG8mx830fQqVeEIY1WuRUso7bOg==
   dependencies:
-    "@budibase/backend-core" "1.4.18-alpha.1"
+    "@budibase/backend-core" "2.0.14-alpha.0"
-    "@budibase/types" "1.4.18-alpha.1"
+    "@budibase/types" "2.0.14-alpha.0"
     "@koa/router" "8.0.8"
     joi "17.6.0"
     node-fetch "^2.6.1"
 
-"@budibase/types@1.4.18-alpha.1":
+"@budibase/types@2.0.14-alpha.0":
-  version "1.4.18-alpha.1"
+  version "2.0.14-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.4.18-alpha.1.tgz#aad43686afe861838f89e69ecf16ccee9e45f462"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.14-alpha.0.tgz#419ceefde9698b1918c1b41f90fc3010927acde7"
-  integrity sha512-5fWImsg5cZomUSNs+DK0ZWH5pd4pzHk6x0mgHBcaL+CpzO+ROs/W1JWSVJ6uWYUi9ohy6G1FKB0+4/F5yUaOcw==
+  integrity sha512-20+VfYR9oIui3PDExL+3Ld0XWkrbD74CfWHS8+dYiRmW/PqUkhAT0suwpNui5OsVUn1I+9Jw0wvbitpgT5u2VQ==
 
 "@cspotcode/source-map-consumer@0.8.0":
   version "0.8.0"