Merge remote-tracking branch 'origin/feat/automation-features' into feat/automation-features

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.14.6",
+  "version": "2.14.8",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit b23fb3b17961fb04badd9487913a683fcf26dbe6
+Subproject commit 319c8499e7c3d33fbb96cf4d73a922690709686c

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -19,6 +19,8 @@ import { WriteStream, ReadStream } from "fs"
 import { newid } from "../../docIds/newid"
 import { DDInstrumentedDatabase } from "../instrumentation"
 
+const DATABASE_NOT_FOUND = "Database does not exist."
+
 function buildNano(couchInfo: { url: string; cookie: string }) {
   return Nano({
     url: couchInfo.url,
@@ -31,6 +33,8 @@ function buildNano(couchInfo: { url: string; cookie: string }) {
   })
 }
 
+type DBCall<T> = () => Promise<T>
+
 export function DatabaseWithConnection(
   dbName: string,
   connection: string,
@@ -78,7 +82,11 @@ export class DatabaseImpl implements Database {
     return this.instanceNano || DatabaseImpl.nano
   }
 
-  async checkSetup() {
+  private getDb() {
+    return this.nano().db.use(this.name)
+  }
+
+  private async checkAndCreateDb() {
     let shouldCreate = !this.pouchOpts?.skip_setup
     // check exists in a lightweight fashion
     let exists = await this.exists()
@@ -95,14 +103,22 @@ export class DatabaseImpl implements Database {
         }
       }
     }
-    return this.nano().db.use(this.name)
+    return this.getDb()
   }
 
-  private async updateOutput(fnc: any) {
+  // this function fetches the DB and handles if DB creation is needed
+  private async performCall<T>(
+    call: (db: Nano.DocumentScope<any>) => Promise<DBCall<T>> | DBCall<T>
+  ): Promise<any> {
+    const db = this.getDb()
+    const fnc = await call(db)
     try {
       return await fnc()
     } catch (err: any) {
-      if (err.statusCode) {
+      if (err.statusCode === 404 && err.reason === DATABASE_NOT_FOUND) {
+        await this.checkAndCreateDb()
+        return await this.performCall(call)
+      } else if (err.statusCode) {
         err.status = err.statusCode
       }
       throw err
@@ -110,11 +126,12 @@ export class DatabaseImpl implements Database {
   }
 
   async get<T extends Document>(id?: string): Promise<T> {
-    const db = await this.checkSetup()
+    return this.performCall(db => {
-    if (!id) {
+      if (!id) {
-      throw new Error("Unable to get doc without a valid _id.")
+        throw new Error("Unable to get doc without a valid _id.")
-    }
+      }
-    return this.updateOutput(() => db.get(id))
+      return () => db.get(id)
+    })
   }
 
   async getMultiple<T extends Document>(
@@ -147,22 +164,23 @@ export class DatabaseImpl implements Database {
   }
 
   async remove(idOrDoc: string | Document, rev?: string) {
-    const db = await this.checkSetup()
+    return this.performCall(db => {
-    let _id: string
+      let _id: string
-    let _rev: string
+      let _rev: string
 
-    if (isDocument(idOrDoc)) {
+      if (isDocument(idOrDoc)) {
-      _id = idOrDoc._id!
+        _id = idOrDoc._id!
-      _rev = idOrDoc._rev!
+        _rev = idOrDoc._rev!
-    } else {
+      } else {
-      _id = idOrDoc
+        _id = idOrDoc
-      _rev = rev!
+        _rev = rev!
-    }
+      }
 
-    if (!_id || !_rev) {
+      if (!_id || !_rev) {
-      throw new Error("Unable to remove doc without a valid _id and _rev.")
+        throw new Error("Unable to remove doc without a valid _id and _rev.")
-    }
+      }
-    return this.updateOutput(() => db.destroy(_id, _rev))
+      return () => db.destroy(_id, _rev)
+    })
   }
 
   async post(document: AnyDocument, opts?: DatabasePutOpts) {
@@ -176,45 +194,49 @@ export class DatabaseImpl implements Database {
     if (!document._id) {
       throw new Error("Cannot store document without _id field.")
     }
-    const db = await this.checkSetup()
+    return this.performCall(async db => {
-    if (!document.createdAt) {
+      if (!document.createdAt) {
-      document.createdAt = new Date().toISOString()
+        document.createdAt = new Date().toISOString()
-    }
+      }
-    document.updatedAt = new Date().toISOString()
+      document.updatedAt = new Date().toISOString()
-    if (opts?.force && document._id) {
+      if (opts?.force && document._id) {
-      try {
+        try {
-        const existing = await this.get(document._id)
+          const existing = await this.get(document._id)
-        if (existing) {
+          if (existing) {
-          document._rev = existing._rev
+            document._rev = existing._rev
-        }
+          }
-      } catch (err: any) {
+        } catch (err: any) {
-        if (err.status !== 404) {
+          if (err.status !== 404) {
-          throw err
+            throw err
+          }
         }
       }
-    }
+      return () => db.insert(document)
-    return this.updateOutput(() => db.insert(document))
+    })
   }
 
   async bulkDocs(documents: AnyDocument[]) {
-    const db = await this.checkSetup()
+    return this.performCall(db => {
-    return this.updateOutput(() => db.bulk({ docs: documents }))
+      return () => db.bulk({ docs: documents })
+    })
   }
 
   async allDocs<T extends Document>(
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>> {
-    const db = await this.checkSetup()
+    return this.performCall(db => {
-    return this.updateOutput(() => db.list(params))
+      return () => db.list(params)
+    })
   }
 
   async query<T extends Document>(
     viewName: string,
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>> {
-    const db = await this.checkSetup()
+    return this.performCall(db => {
-    const [database, view] = viewName.split("/")
+      const [database, view] = viewName.split("/")
-    return this.updateOutput(() => db.view(database, view, params))
+      return () => db.view(database, view, params)
+    })
   }
 
   async destroy() {
@@ -231,8 +253,9 @@ export class DatabaseImpl implements Database {
   }
 
   async compact() {
-    const db = await this.checkSetup()
+    return this.performCall(db => {
-    return this.updateOutput(() => db.compact())
+      return () => db.compact()
+    })
   }
 
   // All below functions are in-frequently called, just utilise PouchDB

packages/backend-core/src/db/instrumentation.ts

@@ -31,13 +31,6 @@ export class DDInstrumentedDatabase implements Database {
     })
   }
 
-  checkSetup(): Promise<DocumentScope<any>> {
-    return tracer.trace("db.checkSetup", span => {
-      span?.addTags({ db_name: this.name })
-      return this.db.checkSetup()
-    })
-  }
-
   get<T extends Document>(id?: string | undefined): Promise<T> {
     return tracer.trace("db.get", span => {
       span?.addTags({ db_name: this.name, doc_id: id })

packages/bbui/src/Modal/ModalContent.svelte

@@ -40,7 +40,7 @@
     loading = false
   }
 
-  async function confirm() {
+  export async function confirm() {
     loading = true
     if (!onConfirm || (await onConfirm()) !== keepOpen) {
       hide()

packages/builder/src/components/design/ScreenDetailsModal.svelte

@@ -13,6 +13,7 @@
   const appPrefix = "/app"
   let touched = false
   let error
+  let modal
 
   $: appUrl = screenUrl
     ? `${window.location.origin}${appPrefix}${screenUrl}`
@@ -50,6 +51,7 @@
 </script>
 
 <ModalContent
+  bind:this={modal}
   size="M"
   title={"Screen details"}
   {confirmText}
@@ -58,15 +60,17 @@
   cancelText={"Back"}
   disabled={!screenUrl || error || !touched}
 >
-  <Input
+  <form on:submit|preventDefault={() => modal.confirm()}>
-    label="Enter a URL for the new screen"
+    <Input
-    {error}
+      label="Enter a URL for the new screen"
-    bind:value={screenUrl}
+      {error}
-    on:change={routeChanged}
+      bind:value={screenUrl}
-  />
+      on:change={routeChanged}
-  <div class="app-server" title={appUrl}>
+    />
-    {appUrl}
+    <div class="app-server" title={appUrl}>
-  </div>
+      {appUrl}
+    </div>
+  </form>
 </ModalContent>
 
 <style>

packages/builder/src/components/design/settings/controls/FormStepControls.svelte

@@ -23,7 +23,7 @@
         stepAction("addStep")
       }}
     >
-      Add Step
+      Add step
     </ActionButton>
   </div>
 {:else}

packages/client/manifest.json

@@ -6093,15 +6093,44 @@
             "key": "steps",
             "nested": true,
             "labelHidden": true,
-            "resetOn": [
+            "resetOn": ["dataSource", "actionType"],
-              "dataSource",
+            "defaultValue": [{}]
-              "actionType"
-            ],
-            "defaultValue": [
-              {}
-            ]
           }
         ]
+      },
+      {
+        "tag": "style",
+        "type": "select",
+        "label": "Size",
+        "key": "size",
+        "options": [
+          {
+            "label": "Medium",
+            "value": "spectrum--medium"
+          },
+          {
+            "label": "Large",
+            "value": "spectrum--large"
+          }
+        ],
+        "defaultValue": "spectrum--medium"
+      },
+      {
+        "tag": "style",
+        "type": "select",
+        "label": "Button position",
+        "key": "buttonPosition",
+        "options": [
+          {
+            "label": "Bottom",
+            "value": "bottom"
+          },
+          {
+            "label": "Top",
+            "value": "top"
+          }
+        ],
+        "defaultValue": "bottom"
       }
     ],
     "actions": [

packages/client/src/components/app/blocks/MultiStepFormblock.svelte

@@ -11,6 +11,8 @@
   export let noRowsMessage
   export let steps
   export let dataSource
+  export let buttonPosition = "bottom"
+  export let size
 
   const { fetchDatasourceSchema } = getContext("sdk")
   const component = getContext("component")
@@ -127,6 +129,7 @@
     type="form"
     context="form"
     props={{
+      size,
       dataSource,
       actionType: actionType === "Create" ? "Create" : "Update",
       readonly: actionType === "View",
@@ -154,8 +157,33 @@
             size: "shrink",
           }}
         >
-          <BlockComponent type="container" order={0}>
+          <BlockComponent
-            <BlockComponent type="heading" props={{ text: step.title }} />
+            type="container"
+            props={{
+              direction: "column",
+              gap: "S",
+            }}
+            order={0}
+          >
+            <BlockComponent
+              type="container"
+              props={{
+                direction: "row",
+                hAlign: "stretch",
+                vAlign: "center",
+                gap: "M",
+                wrap: true,
+              }}
+              order={0}
+            >
+              <BlockComponent type="heading" props={{ text: step.title }} />
+              {#if buttonPosition === "top"}
+                <BlockComponent
+                  type="buttongroup"
+                  props={{ buttons: step.buttons }}
+                />
+              {/if}
+            </BlockComponent>
           </BlockComponent>
           <BlockComponent type="text" props={{ text: step.desc }} order={1} />
           <BlockComponent type="container" order={2}>
@@ -176,16 +204,13 @@
               {/each}
             </div>
           </BlockComponent>
-          <BlockComponent
+          {#if buttonPosition === "bottom"}
-            type="buttongroup"
+            <BlockComponent
-            props={{ buttons: step.buttons }}
+              type="buttongroup"
-            styles={{
+              props={{ buttons: step.buttons }}
-              normal: {
+              order={3}
-                "margin-top": "16px",
+            />
-              },
+          {/if}
-            }}
-            order={3}
-          />
         </BlockComponent>
       </BlockComponent>
     {/each}

packages/string-templates/package.json

@@ -25,10 +25,10 @@
     "manifest": "node ./scripts/gen-collection-info.js"
   },
   "dependencies": {
-    "@budibase/handlebars-helpers": "^0.11.11",
+    "@budibase/handlebars-helpers": "^0.12.0",
     "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
-    "lodash": "4.17.21",
+    "lodash.clonedeep": "^4.5.0",
     "vm2": "^3.9.19"
   },
   "devDependencies": {

packages/string-templates/src/helpers/javascript.js

@@ -1,5 +1,5 @@
 const { atob } = require("../utilities")
-const { cloneDeep } = require("lodash/fp")
+const cloneDeep = require("lodash.clonedeep")
 const { LITERAL_MARKER } = require("../helpers/constants")
 const { getHelperList } = require("./list")
 

packages/types/src/sdk/db.ts

@@ -121,7 +121,6 @@ export interface Database {
   name: string
 
   exists(): Promise<boolean>
-  checkSetup(): Promise<Nano.DocumentScope<any>>
   get<T extends Document>(id?: string): Promise<T>
   getMultiple<T extends Document>(
     ids: string[],

packages/types/src/sdk/locks.ts

@@ -22,6 +22,7 @@ export enum LockName {
   QUOTA_USAGE_EVENT = "quota_usage_event",
   APP_MIGRATION = "app_migrations",
   PROCESS_AUTO_COLUMNS = "process_auto_columns",
+  PROCESS_USER_INVITE = "process_user_invite",
 }
 
 export type LockOptions = {

packages/worker/src/api/controllers/global/users.ts

@@ -12,6 +12,8 @@ import {
   InviteUserRequest,
   InviteUsersRequest,
   InviteUsersResponse,
+  LockName,
+  LockType,
   MigrationType,
   SaveUserResponse,
   SearchUsersRequest,
@@ -27,6 +29,7 @@ import {
   platform,
   tenancy,
   db,
+  locks,
 } from "@budibase/backend-core"
 import { checkAnyUserExists } from "../../../utilities/users"
 import { isEmailConfigured } from "../../../utilities/email"
@@ -380,51 +383,60 @@ export const inviteAccept = async (
 ) => {
   const { inviteCode, password, firstName, lastName } = ctx.request.body
   try {
-    // info is an extension of the user object that was stored by global
+    await locks.doWithLock(
-    const { email, info }: any = await cache.invite.getCode(inviteCode)
+      {
-    await cache.invite.deleteCode(inviteCode)
+        type: LockType.AUTO_EXTEND,
-    const user = await tenancy.doInTenant(info.tenantId, async () => {
+        name: LockName.PROCESS_USER_INVITE,
-      let request: any = {
+        resource: inviteCode,
-        firstName,
+        systemLock: true,
-        lastName,
+      },
-        password,
+      async () => {
-        email,
+        // info is an extension of the user object that was stored by global
-        admin: { global: info?.admin?.global || false },
+        const { email, info } = await cache.invite.getCode(inviteCode)
-        roles: info.apps,
+        const user = await tenancy.doInTenant(info.tenantId, async () => {
-        tenantId: info.tenantId,
+          let request: any = {
-      }
+            firstName,
-      let builder: { global: boolean; apps?: string[] } = {
+            lastName,
-        global: info?.builder?.global || false,
+            password,
-      }
+            email,
+            admin: { global: info?.admin?.global || false },
+            roles: info.apps,
+            tenantId: info.tenantId,
+          }
+          const builder: { global: boolean; apps?: string[] } = {
+            global: info?.builder?.global || false,
+          }
 
-      if (info?.builder?.apps) {
+          if (info?.builder?.apps) {
-        builder.apps = info.builder.apps
+            builder.apps = info.builder.apps
-        request.builder = builder
+            request.builder = builder
-      }
+          }
-      delete info.apps
+          delete info.apps
-      request = {
+          request = {
-        ...request,
+            ...request,
-        ...info,
+            ...info,
-      }
+          }
 
-      const saved = await userSdk.db.save(request)
+          const saved = await userSdk.db.save(request)
-      const db = tenancy.getGlobalDB()
+          await events.user.inviteAccepted(saved)
-      const user = await db.get<User>(saved._id)
+          return saved
-      await events.user.inviteAccepted(user)
+        })
-      return saved
-    })
 
-    ctx.body = {
+        await cache.invite.deleteCode(inviteCode)
-      _id: user._id!,
+
-      _rev: user._rev!,
+        ctx.body = {
-      email: user.email,
+          _id: user._id!,
-    }
+          _rev: user._rev!,
+          email: user.email,
+        }
+      }
+    )
   } catch (err: any) {
     if (err.code === ErrorCode.USAGE_LIMIT_EXCEEDED) {
       // explicitly re-throw limit exceeded errors
       ctx.throw(400, err)
     }
     console.warn("Error inviting user", err)
-    ctx.throw(400, "Unable to create new user, invitation invalid.")
+    ctx.throw(400, err || "Unable to create new user, invitation invalid.")
   }
 }

qa-core/package.json

@@ -24,6 +24,8 @@
   },
   "devDependencies": {
     "@budibase/types": "^2.3.17",
+    "@swc/core": "1.3.71",
+    "@swc/jest": "0.2.27",
     "@trendyol/jest-testcontainers": "2.1.1",
     "@types/jest": "29.5.3",
     "@types/node-fetch": "2.6.4",
@@ -32,8 +34,6 @@
     "jest": "29.7.0",
     "prettier": "2.7.1",
     "start-server-and-test": "1.14.0",
-    "@swc/core": "1.3.71",
-    "@swc/jest": "0.2.27",
     "timekeeper": "2.2.0",
     "ts-jest": "29.1.1",
     "ts-node": "10.8.1",
@@ -43,6 +43,7 @@
   "dependencies": {
     "@budibase/backend-core": "^2.3.17",
     "form-data": "^4.0.0",
-    "node-fetch": "2.6.7"
+    "node-fetch": "2.6.7",
+    "stripe": "^14.11.0"
   }
 }

qa-core/yarn.lock

@@ -951,6 +951,13 @@
   resolved "https://registry.npmjs.org/@types/node/-/node-18.15.1.tgz"
   integrity sha512-U2TWca8AeHSmbpi314QBESRk7oPjSZjDsR+c+H4ECC1l+kFgpZf8Ydhv3SJpPy51VyZHHqxlb6mTTqYNNRVAIw==
 
+"@types/node@>=8.1.0":
+  version "20.11.0"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.0.tgz#8e0b99e70c0c1ade1a86c4a282f7b7ef87c9552f"
+  integrity sha512-o9bjXmDNcF7GbM4CNQpmi+TutCgap/K3w1JyKgxAjqx41zp9qlIAVFi0IhCNsJcXolEqLWhbFbEeL0PvYm4pcQ==
+  dependencies:
+    undici-types "~5.26.4"
+
 "@types/stack-utils@^2.0.0":
   version "2.0.1"
   resolved "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz"
@@ -4549,6 +4556,14 @@ strip-json-comments@^3.1.1:
   resolved "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz"
   integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==
 
+stripe@^14.11.0:
+  version "14.11.0"
+  resolved "https://registry.yarnpkg.com/stripe/-/stripe-14.11.0.tgz#1df63c31bcff3b136457c2b7584f917509e8030c"
+  integrity sha512-NmFEkDC0PldP7CQtdPgKs5dVZA/pF+IepldbmB+Kk9B4d7EBkWnbANp0y+/zJcbRGul48s8hmQzeqNWUlWW0wg==
+  dependencies:
+    "@types/node" ">=8.1.0"
+    qs "^6.11.0"
+
 supports-color@^5.3.0:
   version "5.5.0"
   resolved "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz"
@@ -4807,6 +4822,11 @@ uid2@0.0.x:
   resolved "https://registry.npmjs.org/uid2/-/uid2-0.0.4.tgz"
   integrity sha512-IevTus0SbGwQzYh3+fRsAMTVVPOoIVufzacXcHPmdlle1jUpq7BRL+mw3dgeLanvGZdwwbWhRV6XrcFNdBmjWA==
 
+undici-types@~5.26.4:
+  version "5.26.5"
+  resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
+  integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
+
 universalify@^0.2.0:
   version "0.2.0"
   resolved "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz"

scripts/removeWorkspaceDependencies.sh

@@ -3,26 +3,14 @@
 packages_to_remove="@budibase/backend-core @budibase/bbui @budibase/builder @budibase/cli @budibase/client @budibase/frontend-core @budibase/pro @budibase/sdk @budibase/server @budibase/shared-core @budibase/string-templates @budibase/types @budibase/worker"
 
 package_json_path="$1"
-package_json=$(cat "$package_json_path")
 
 process_package() {
   pkg_path="$1"
-  package_json=$(cat "$pkg_path")
-  has_changes=false
 
   for package_name in $packages_to_remove; do
-    if echo "$package_json" | jq -e --arg package_name "$package_name" '.dependencies | has($package_name)' > /dev/null; then
+    jq "del(.dependencies[\"$package_name\"])" $pkg_path > tmp_file.json && mv tmp_file.json $pkg_path
-      package_json=$(echo "$package_json" | jq "del(.dependencies[\"$package_name\"])")
+    jq "del(.resolutions[\"$package_name\"])" $pkg_path > tmp_file.json && mv tmp_file.json $pkg_path
-      has_changes=true
-    fi
   done
-
-  if [ "$has_changes" = true ]; then
-    echo "$package_json" > "$pkg_path"
-  fi
 }
 
 process_package "$package_json_path"
-
-package_json=$(cat "$package_json_path")
-echo "$package_json" | jq "del(.resolutions)" > "$1"