diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml index d850d289ff..0580ac1c3a 100644 --- a/.github/workflows/deploy-qa.yml +++ b/.github/workflows/deploy-qa.yml @@ -11,10 +11,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: peter-evans/repository-dispatch@v2 - env: - PAYLOAD_VERSION: ${{ github.sha }} - REF_NAME: ${{ github.ref_name}} with: repository: budibase/budibase-deploys event-type: budicloud-qa-deploy token: ${{ secrets.GH_ACCESS_TOKEN }} + client-payload: |- + { + "PAYLOAD_VERSION": "${{ github.sha }}", + "REF_NAME": "${{ github.ref_name}}" + } diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index df25182cd6..7648ae32e7 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -165,17 +165,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Get the current budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} - REF_NAME: ${{ github.ref_name}} + - uses: peter-evans/repository-dispatch@v2 with: repository: budibase/budibase-deploys event: budicloud-qa-deploy github_pat: ${{ secrets.GH_ACCESS_TOKEN }} + client-payload: |- + { + "PAYLOAD_VERSION": "${{ github.ref_name }}", + "REF_NAME": "${{ github.ref_name}}" + } diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml index a3444d5e7a..16b1da186a 100644 --- a/.github/workflows/release-singleimage.yml +++ b/.github/workflows/release-singleimage.yml @@ -66,7 +66,7 @@ jobs: context: . push: true platforms: linux/amd64,linux/arm64 - build-args: BUDIBASE_VERSION=$BUDIBASE_VERSION + build-args: BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }} file: ./hosting/single/Dockerfile.v2 env: @@ -79,7 +79,7 @@ jobs: platforms: linux/amd64 build-args: | TARGETBUILD=aas - BUDIBASE_VERSION=$BUDIBASE_VERSION + BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }} file: ./hosting/single/Dockerfile.v2 env: diff --git a/lerna.json b/lerna.json index f0f51242d1..ff0cbafedd 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.12.4", + "version": "2.12.6", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 417fb31e0e..8a27cde104 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "scripts": { "preinstall": "node scripts/syncProPackage.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", - "build": "lerna run build --stream", + "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "check:types": "lerna run check:types", "build:sdk": "lerna run --stream build:sdk", diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index ffffd8240a..c7cf9f56cc 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -30,6 +30,7 @@ export * as timers from "./timers" export { default as env } from "./environment" export * as blacklist from "./blacklist" export * as docUpdates from "./docUpdates" +export * from "./utils/Duration" export { SearchParams } from "./db" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index af2ec6dbaa..a8add7ecb6 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -36,7 +36,7 @@ class InMemoryQueue { * @param opts This is not used by the in memory queue as there is no real use * case when in memory, but is the same API as Bull */ - constructor(name: string, opts = null) { + constructor(name: string, opts?: any) { this._name = name this._opts = opts this._messages = [] diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 0658147709..c0d1861de3 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -2,11 +2,18 @@ import env from "../environment" import { getRedisOptions } from "../redis/utils" import { JobQueue } from "./constants" import InMemoryQueue from "./inMemoryQueue" -import BullQueue from "bull" +import BullQueue, { QueueOptions } from "bull" import { addListeners, StalledFn } from "./listeners" +import { Duration } from "../utils" import * as timers from "../timers" +import * as Redis from "ioredis" -const CLEANUP_PERIOD_MS = 60 * 1000 +// the queue lock is held for 5 minutes +const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs() +// queue lock is refreshed every 30 seconds +const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs() +// cleanup the queue every 60 seconds +const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs() let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = [] let cleanupInterval: NodeJS.Timeout @@ -21,7 +28,14 @@ export function createQueue( opts: { removeStalledCb?: StalledFn } = {} ): BullQueue.Queue { const { opts: redisOpts, redisProtocolUrl } = getRedisOptions() - const queueConfig: any = redisProtocolUrl || { redis: redisOpts } + const queueConfig: QueueOptions = { + redis: redisProtocolUrl! || (redisOpts as Redis.RedisOptions), + settings: { + maxStalledCount: 0, + lockDuration: QUEUE_LOCK_MS, + lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS, + }, + } let queue: any if (!env.isTest()) { queue = new BullQueue(jobQueue, queueConfig) diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts new file mode 100644 index 0000000000..f376c2f7c7 --- /dev/null +++ b/packages/backend-core/src/utils/Duration.ts @@ -0,0 +1,49 @@ +export enum DurationType { + MILLISECONDS = "milliseconds", + SECONDS = "seconds", + MINUTES = "minutes", + HOURS = "hours", + DAYS = "days", +} + +const conversion: Record = { + milliseconds: 1, + seconds: 1000, + minutes: 60 * 1000, + hours: 60 * 60 * 1000, + days: 24 * 60 * 60 * 1000, +} + +export class Duration { + static convert(from: DurationType, to: DurationType, duration: number) { + const milliseconds = duration * conversion[from] + return milliseconds / conversion[to] + } + + static from(from: DurationType, duration: number) { + return { + to: (to: DurationType) => { + return Duration.convert(from, to, duration) + }, + toMs: () => { + return Duration.convert(from, DurationType.MILLISECONDS, duration) + }, + } + } + + static fromSeconds(duration: number) { + return Duration.from(DurationType.SECONDS, duration) + } + + static fromMinutes(duration: number) { + return Duration.from(DurationType.MINUTES, duration) + } + + static fromHours(duration: number) { + return Duration.from(DurationType.HOURS, duration) + } + + static fromDays(duration: number) { + return Duration.from(DurationType.DAYS, duration) + } +} diff --git a/packages/backend-core/src/utils/index.ts b/packages/backend-core/src/utils/index.ts index 318a7f13ba..ac17227459 100644 --- a/packages/backend-core/src/utils/index.ts +++ b/packages/backend-core/src/utils/index.ts @@ -1,3 +1,4 @@ export * from "./hashing" export * from "./utils" export * from "./stringUtils" +export * from "./Duration" diff --git a/packages/backend-core/src/utils/tests/Duration.spec.ts b/packages/backend-core/src/utils/tests/Duration.spec.ts new file mode 100644 index 0000000000..46b996f788 --- /dev/null +++ b/packages/backend-core/src/utils/tests/Duration.spec.ts @@ -0,0 +1,19 @@ +import { Duration, DurationType } from "../Duration" + +describe("duration", () => { + it("should convert minutes to milliseconds", () => { + expect(Duration.fromMinutes(5).toMs()).toBe(300000) + }) + + it("should convert seconds to milliseconds", () => { + expect(Duration.fromSeconds(30).toMs()).toBe(30000) + }) + + it("should convert days to milliseconds", () => { + expect(Duration.fromDays(1).toMs()).toBe(86400000) + }) + + it("should convert minutes to days", () => { + expect(Duration.fromMinutes(1440).to(DurationType.DAYS)).toBe(1) + }) +}) diff --git a/packages/builder/src/components/design/settings/componentSettings.js b/packages/builder/src/components/design/settings/componentSettings.js index 232b4bef31..c2bd08760a 100644 --- a/packages/builder/src/components/design/settings/componentSettings.js +++ b/packages/builder/src/components/design/settings/componentSettings.js @@ -1,5 +1,5 @@ import { Checkbox, Select, RadioGroup, Stepper, Input } from "@budibase/bbui" -import DataSourceSelect from "./controls/DataSourceSelect.svelte" +import DataSourceSelect from "./controls/DataSourceSelect/DataSourceSelect.svelte" import S3DataSourceSelect from "./controls/S3DataSourceSelect.svelte" import DataProviderSelect from "./controls/DataProviderSelect.svelte" import ButtonActionEditor from "./controls/ButtonActionEditor/ButtonActionEditor.svelte" diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte new file mode 100644 index 0000000000..f7c79b3862 --- /dev/null +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte @@ -0,0 +1,55 @@ + + +{#if dividerState} + +{/if} +{#if heading} +
+ {heading} +
+{/if} +
    + {#each dataSet as data} +
  • onSelect(data)} + > + + {data.label} + + +
    • + {/each} +
+ + diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte similarity index 75% rename from packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte rename to packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index 29f32aa345..5cda0ebcca 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -7,10 +7,8 @@ import { Button, Popover, - Divider, Select, Layout, - Heading, Drawer, DrawerContent, Icon, @@ -32,6 +30,7 @@ import IntegrationQueryEditor from "components/integration/index.svelte" import { makePropSafe as safe } from "@budibase/string-templates" import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" + import DataSourceCategory from "components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte" import { API } from "api" export let value = {} @@ -279,102 +278,81 @@
-
- Tables -
-
    - {#each tables as table} -
  • handleSelected(table)}>{table.label}
    • - {/each} -
+ {#if views?.length} - -
- Views -
-
    - {#each views as view} -
  • handleSelected(view)}>{view.label}
    • - {/each} -
+ {/if} {#if queries?.length} - -
- Queries -
-
    - {#each queries as query} -
  • handleSelected(query)} - > - {query.label} -
    • - {/each} -
+ {/if} {#if links?.length} - -
- Relationships -
-
    - {#each links as link} -
  • handleSelected(link)}>{link.label}
    • - {/each} -
+ {/if} {#if fields?.length} - -
- Fields -
-
    - {#each fields as field} -
  • handleSelected(field)}>{field.label}
    • - {/each} -
+ {/if} {#if jsonArrays?.length} - -
- JSON Arrays -
-
    - {#each jsonArrays as field} -
  • handleSelected(field)}>{field.label}
    • - {/each} -
+ {/if} {#if showDataProviders && dataProviders?.length} - -
- Data Providers -
-
    - {#each dataProviders as provider} -
  • handleSelected(provider)} - > - {provider.label} -
    • - {/each} -
+ + {/if} + + {#if otherSources?.length} + {/if} - -
- Other -
-
    -
  • handleSelected(custom)}>{custom.label}
    • - {#if otherSources?.length} - {#each otherSources as source} -
  • handleSelected(source)}>{source.label}
    • - {/each} - {/if} -
@@ -398,31 +376,6 @@ .dropdown { padding: var(--spacing-m) 0; z-index: 99999999; - overflow-y: scroll; - } - .title { - padding: 0 var(--spacing-m) var(--spacing-s) var(--spacing-m); - } - - ul { - list-style: none; - padding-left: 0px; - margin: 0px; - } - - li { - cursor: pointer; - margin: 0px; - padding: var(--spacing-s) var(--spacing-m); - font-size: var(--font-size-m); - } - - .selected { - color: var(--spectrum-global-color-blue-600); - } - - li:hover { - background-color: var(--spectrum-global-color-gray-200); } .icon { diff --git a/packages/builder/src/components/design/settings/controls/SchemaSelect.svelte b/packages/builder/src/components/design/settings/controls/SchemaSelect.svelte index 80e36328f1..dbeeec53ef 100644 --- a/packages/builder/src/components/design/settings/controls/SchemaSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/SchemaSelect.svelte @@ -1,5 +1,5 @@ diff --git a/pull_request_template.md b/pull_request_template.md index 36e2f425d5..fa0f7f2143 100644 --- a/pull_request_template.md +++ b/pull_request_template.md @@ -1,18 +1,16 @@ ## Description + _Describe the problem or feature in addition to a link to the relevant github issues._ -Addresses: +Addresses: + - `` - ...more if required ## App Export + - If possible, attach an app export file along with your request template to make QA testing easier, with minimal setup. ## Screenshots + _If a UI facing feature, a short video of the happy path, and some screenshots of the new functionality._ - -## Documentation -- [ ] I have reviewed the budibase documentatation to verify if this feature requires any changes. If changes or new docs are required I have written them. - - -