Merge branch 'master' into budi-7105-builder-option-for-read-only-form-fields

.github/workflows/deploy-qa.yml

@@ -11,10 +11,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: peter-evans/repository-dispatch@v2
-        env:
-          PAYLOAD_VERSION: ${{ github.sha }}
-          REF_NAME: ${{ github.ref_name}}
         with:
           repository: budibase/budibase-deploys
           event-type: budicloud-qa-deploy
           token: ${{ secrets.GH_ACCESS_TOKEN }}
+          client-payload: |-
+            {
+              "PAYLOAD_VERSION": "${{ github.sha }}",
+              "REF_NAME": "${{ github.ref_name}}"
+            }

.github/workflows/release-master.yml

@@ -165,17 +165,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Get the current budibase release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
 
-      - uses: passeidireto/trigger-external-workflow-action@main
+      - uses: peter-evans/repository-dispatch@v2
-        env:
-          PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
-          REF_NAME: ${{ github.ref_name}}
         with:
           repository: budibase/budibase-deploys
           event: budicloud-qa-deploy
           github_pat: ${{ secrets.GH_ACCESS_TOKEN }}
+          client-payload: |-
+            {
+              "PAYLOAD_VERSION": "${{ github.ref_name }}",
+              "REF_NAME": "${{ github.ref_name}}"
+            }

.github/workflows/release-singleimage.yml

@@ -66,7 +66,7 @@ jobs:
           context: .
           push: true
           platforms: linux/amd64,linux/arm64
-          build-args: BUDIBASE_VERSION=$BUDIBASE_VERSION
+          build-args: BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
           tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }}
           file: ./hosting/single/Dockerfile.v2
         env:
@@ -79,7 +79,7 @@ jobs:
           platforms: linux/amd64
           build-args: |
             TARGETBUILD=aas
-            BUDIBASE_VERSION=$BUDIBASE_VERSION
+            BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
           tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }}
           file: ./hosting/single/Dockerfile.v2
         env:

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.12.4",
+  "version": "2.12.6",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -27,7 +27,7 @@
   "scripts": {
     "preinstall": "node scripts/syncProPackage.js",
     "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
-    "build": "lerna run build --stream",
+    "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
     "check:types": "lerna run check:types",
     "build:sdk": "lerna run --stream build:sdk",

packages/backend-core/src/index.ts

@@ -30,6 +30,7 @@ export * as timers from "./timers"
 export { default as env } from "./environment"
 export * as blacklist from "./blacklist"
 export * as docUpdates from "./docUpdates"
+export * from "./utils/Duration"
 export { SearchParams } from "./db"
 // Add context to tenancy for backwards compatibility
 // only do this for external usages to prevent internal

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -36,7 +36,7 @@ class InMemoryQueue {
    * @param opts This is not used by the in memory queue as there is no real use
    * case when in memory, but is the same API as Bull
    */
-  constructor(name: string, opts = null) {
+  constructor(name: string, opts?: any) {
     this._name = name
     this._opts = opts
     this._messages = []

packages/backend-core/src/queue/queue.ts

@@ -2,11 +2,18 @@ import env from "../environment"
 import { getRedisOptions } from "../redis/utils"
 import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
-import BullQueue from "bull"
+import BullQueue, { QueueOptions } from "bull"
 import { addListeners, StalledFn } from "./listeners"
+import { Duration } from "../utils"
 import * as timers from "../timers"
+import * as Redis from "ioredis"
 
-const CLEANUP_PERIOD_MS = 60 * 1000
+// the queue lock is held for 5 minutes
+const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
+// queue lock is refreshed every 30 seconds
+const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs()
+// cleanup the queue every 60 seconds
+const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs()
 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
 let cleanupInterval: NodeJS.Timeout
 
@@ -21,7 +28,14 @@ export function createQueue<T>(
   opts: { removeStalledCb?: StalledFn } = {}
 ): BullQueue.Queue<T> {
   const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
-  const queueConfig: any = redisProtocolUrl || { redis: redisOpts }
+  const queueConfig: QueueOptions = {
+    redis: redisProtocolUrl! || (redisOpts as Redis.RedisOptions),
+    settings: {
+      maxStalledCount: 0,
+      lockDuration: QUEUE_LOCK_MS,
+      lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
+    },
+  }
   let queue: any
   if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)

packages/backend-core/src/utils/Duration.ts

@@ -0,0 +1,49 @@
+export enum DurationType {
+  MILLISECONDS = "milliseconds",
+  SECONDS = "seconds",
+  MINUTES = "minutes",
+  HOURS = "hours",
+  DAYS = "days",
+}
+
+const conversion: Record<DurationType, number> = {
+  milliseconds: 1,
+  seconds: 1000,
+  minutes: 60 * 1000,
+  hours: 60 * 60 * 1000,
+  days: 24 * 60 * 60 * 1000,
+}
+
+export class Duration {
+  static convert(from: DurationType, to: DurationType, duration: number) {
+    const milliseconds = duration * conversion[from]
+    return milliseconds / conversion[to]
+  }
+
+  static from(from: DurationType, duration: number) {
+    return {
+      to: (to: DurationType) => {
+        return Duration.convert(from, to, duration)
+      },
+      toMs: () => {
+        return Duration.convert(from, DurationType.MILLISECONDS, duration)
+      },
+    }
+  }
+
+  static fromSeconds(duration: number) {
+    return Duration.from(DurationType.SECONDS, duration)
+  }
+
+  static fromMinutes(duration: number) {
+    return Duration.from(DurationType.MINUTES, duration)
+  }
+
+  static fromHours(duration: number) {
+    return Duration.from(DurationType.HOURS, duration)
+  }
+
+  static fromDays(duration: number) {
+    return Duration.from(DurationType.DAYS, duration)
+  }
+}

packages/backend-core/src/utils/index.ts

@@ -1,3 +1,4 @@
 export * from "./hashing"
 export * from "./utils"
 export * from "./stringUtils"
+export * from "./Duration"

packages/backend-core/src/utils/tests/Duration.spec.ts

@@ -0,0 +1,19 @@
+import { Duration, DurationType } from "../Duration"
+
+describe("duration", () => {
+  it("should convert minutes to milliseconds", () => {
+    expect(Duration.fromMinutes(5).toMs()).toBe(300000)
+  })
+
+  it("should convert seconds to milliseconds", () => {
+    expect(Duration.fromSeconds(30).toMs()).toBe(30000)
+  })
+
+  it("should convert days to milliseconds", () => {
+    expect(Duration.fromDays(1).toMs()).toBe(86400000)
+  })
+
+  it("should convert minutes to days", () => {
+    expect(Duration.fromMinutes(1440).to(DurationType.DAYS)).toBe(1)
+  })
+})

packages/builder/src/components/design/settings/componentSettings.js

@@ -1,5 +1,5 @@
 import { Checkbox, Select, RadioGroup, Stepper, Input } from "@budibase/bbui"
-import DataSourceSelect from "./controls/DataSourceSelect.svelte"
+import DataSourceSelect from "./controls/DataSourceSelect/DataSourceSelect.svelte"
 import S3DataSourceSelect from "./controls/S3DataSourceSelect.svelte"
 import DataProviderSelect from "./controls/DataProviderSelect.svelte"
 import ButtonActionEditor from "./controls/ButtonActionEditor/ButtonActionEditor.svelte"

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte

@@ -0,0 +1,55 @@
+<script>
+  import { Divider, Heading } from "@budibase/bbui"
+
+  export let dividerState
+  export let heading
+  export let dataSet
+  export let value
+  export let onSelect
+</script>
+
+{#if dividerState}
+  <Divider />
+{/if}
+{#if heading}
+  <div class="title">
+    <Heading size="XS">{heading}</Heading>
+  </div>
+{/if}
+<ul class="spectrum-Menu" role="listbox">
+  {#each dataSet as data}
+    <li
+      class="spectrum-Menu-item"
+      class:is-selected={value?.label === data.label &&
+        value?.type === data.type}
+      role="option"
+      aria-selected="true"
+      tabindex="0"
+      on:click={() => onSelect(data)}
+    >
+      <span class="spectrum-Menu-itemLabel">
+        {data.label}
+      </span>
+      <svg
+        class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"
+        focusable="false"
+        aria-hidden="true"
+      >
+        <use xlink:href="#spectrum-css-icon-Checkmark100" />
+      </svg>
+    </li>
+  {/each}
+</ul>
+
+<style>
+  .title {
+    padding: 0 var(--spacing-m) var(--spacing-s) var(--spacing-m);
+  }
+
+  ul {
+    list-style: none;
+    padding-left: 0px;
+    margin: 0px;
+    width: 100%;
+  }
+</style>

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte

@@ -7,10 +7,8 @@
   import {
     Button,
     Popover,
-    Divider,
     Select,
     Layout,
-    Heading,
     Drawer,
     DrawerContent,
     Icon,
@@ -32,6 +30,7 @@
   import IntegrationQueryEditor from "components/integration/index.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
+  import DataSourceCategory from "components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte"
   import { API } from "api"
 
   export let value = {}
@@ -279,102 +278,81 @@
 </div>
 <Popover bind:this={dropdownRight} anchor={anchorRight}>
   <div class="dropdown">
-    <div class="title">
+    <DataSourceCategory
-      <Heading size="XS">Tables</Heading>
+      heading="Tables"
-    </div>
+      dataSet={tables}
-    <ul>
+      {value}
-      {#each tables as table}
+      onSelect={handleSelected}
-        <li on:click={() => handleSelected(table)}>{table.label}</li>
+    />
-      {/each}
-    </ul>
     {#if views?.length}
-      <Divider />
+      <DataSourceCategory
-      <div class="title">
+        dividerState={true}
-        <Heading size="XS">Views</Heading>
+        heading="Views"
-      </div>
+        dataSet={views}
-      <ul>
+        {value}
-        {#each views as view}
+        onSelect={handleSelected}
-          <li on:click={() => handleSelected(view)}>{view.label}</li>
+      />
-        {/each}
-      </ul>
     {/if}
     {#if queries?.length}
-      <Divider />
+      <DataSourceCategory
-      <div class="title">
+        dividerState={true}
-        <Heading size="XS">Queries</Heading>
+        heading="Queries"
-      </div>
+        dataSet={queries}
-      <ul>
+        {value}
-        {#each queries as query}
+        onSelect={handleSelected}
-          <li
+      />
-            class:selected={value === query}
-            on:click={() => handleSelected(query)}
-          >
-            {query.label}
-          </li>
-        {/each}
-      </ul>
     {/if}
     {#if links?.length}
-      <Divider />
+      <DataSourceCategory
-      <div class="title">
+        dividerState={true}
-        <Heading size="XS">Relationships</Heading>
+        heading="Links"
-      </div>
+        dataSet={links}
-      <ul>
+        {value}
-        {#each links as link}
+        onSelect={handleSelected}
-          <li on:click={() => handleSelected(link)}>{link.label}</li>
+      />
-        {/each}
-      </ul>
     {/if}
     {#if fields?.length}
-      <Divider />
+      <DataSourceCategory
-      <div class="title">
+        dividerState={true}
-        <Heading size="XS">Fields</Heading>
+        heading="Fields"
-      </div>
+        dataSet={fields}
-      <ul>
+        {value}
-        {#each fields as field}
+        onSelect={handleSelected}
-          <li on:click={() => handleSelected(field)}>{field.label}</li>
+      />
-        {/each}
-      </ul>
     {/if}
     {#if jsonArrays?.length}
-      <Divider />
+      <DataSourceCategory
-      <div class="title">
+        dividerState={true}
-        <Heading size="XS">JSON Arrays</Heading>
+        heading="JSON Arrays"
-      </div>
+        dataSet={jsonArrays}
-      <ul>
+        {value}
-        {#each jsonArrays as field}
+        onSelect={handleSelected}
-          <li on:click={() => handleSelected(field)}>{field.label}</li>
+      />
-        {/each}
-      </ul>
     {/if}
     {#if showDataProviders && dataProviders?.length}
-      <Divider />
+      <DataSourceCategory
-      <div class="title">
+        dividerState={true}
-        <Heading size="XS">Data Providers</Heading>
+        heading="Data Providers"
-      </div>
+        dataSet={dataProviders}
-      <ul>
+        {value}
-        {#each dataProviders as provider}
+        onSelect={handleSelected}
-          <li
+      />
-            class:selected={value === provider}
+    {/if}
-            on:click={() => handleSelected(provider)}
+    <DataSourceCategory
-          >
+      dividerState={true}
-            {provider.label}
+      heading="Other"
-          </li>
+      dataSet={[custom]}
-        {/each}
+      {value}
-      </ul>
+      onSelect={handleSelected}
+    />
+    {#if otherSources?.length}
+      <DataSourceCategory
+        dividerState={false}
+        dataSet={otherSources}
+        {value}
+        onSelect={handleSelected}
+      />
     {/if}
-    <Divider />
-    <div class="title">
-      <Heading size="XS">Other</Heading>
-    </div>
-    <ul>
-      <li on:click={() => handleSelected(custom)}>{custom.label}</li>
-      {#if otherSources?.length}
-        {#each otherSources as source}
-          <li on:click={() => handleSelected(source)}>{source.label}</li>
-        {/each}
-      {/if}
-    </ul>
   </div>
 </Popover>
 
@@ -398,31 +376,6 @@
   .dropdown {
     padding: var(--spacing-m) 0;
     z-index: 99999999;
-    overflow-y: scroll;
-  }
-  .title {
-    padding: 0 var(--spacing-m) var(--spacing-s) var(--spacing-m);
-  }
-
-  ul {
-    list-style: none;
-    padding-left: 0px;
-    margin: 0px;
-  }
-
-  li {
-    cursor: pointer;
-    margin: 0px;
-    padding: var(--spacing-s) var(--spacing-m);
-    font-size: var(--font-size-m);
-  }
-
-  .selected {
-    color: var(--spectrum-global-color-blue-600);
-  }
-
-  li:hover {
-    background-color: var(--spectrum-global-color-gray-200);
   }
 
   .icon {

packages/builder/src/components/design/settings/controls/SchemaSelect.svelte

@@ -1,5 +1,5 @@
 <script>
-  import DataSourceSelect from "./DataSourceSelect.svelte"
+  import DataSourceSelect from "./DataSourceSelect/DataSourceSelect.svelte"
 
   const otherSources = [{ name: "Custom", label: "Custom" }]
 </script>

pull_request_template.md

@@ -1,18 +1,16 @@
 ## Description
+
 _Describe the problem or feature in addition to a link to the relevant github issues._
 
 Addresses:
+
 - `<Enter the Link to the issue(s) this PR addresses>`
 - ...more if required
 
 ## App Export
+
 - If possible, attach an app export file along with your request template to make QA testing easier, with minimal setup.
 
 ## Screenshots
+
 _If a UI facing feature, a short video of the happy path, and some screenshots of the new functionality._
-
-## Documentation
-- [ ] I have reviewed the budibase documentatation to verify if this feature requires any changes. If changes or new docs are required I have written them.
-
-
-