From 0d67f000f03e1c27692e92dcf866cd36b55435cd Mon Sep 17 00:00:00 2001 From: Dean Date: Wed, 18 Oct 2023 12:19:24 +0100 Subject: [PATCH 01/25] Initial reflow of the block settings --- packages/client/manifest.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 8d0a4e456f..d5fca15fd7 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -5288,17 +5288,17 @@ }, "settings": [ { - "type": "select", + "type": "table", + "label": "Data", + "key": "dataSource" + }, + { + "type": "radio", "label": "Type", "key": "actionType", "options": ["Create", "Update", "View"], "defaultValue": "Create" }, - { - "type": "table", - "label": "Data", - "key": "dataSource" - }, { "type": "text", "label": "Title", @@ -5323,7 +5323,7 @@ }, { "type": "text", - "label": "Empty text", + "label": "No rows found", "key": "noRowsMessage", "defaultValue": "We couldn't find a row to display", "nested": true From f59d6222912443484cb98fb666dd92442199abc7 Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 19 Oct 2023 12:28:28 +0100 Subject: [PATCH 02/25] Moved buttons section below fields --- packages/client/manifest.json | 102 +++++++++++++++++----------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index d5fca15fd7..1919f4ccd2 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -5330,57 +5330,6 @@ } ] }, - { - "section": true, - "name": "Buttons", - "dependsOn": { - "setting": "actionType", - "value": "View", - "invert": true - }, - "settings": [ - { - "type": "text", - "key": "saveButtonLabel", - "label": "Save button", - "nested": true, - "defaultValue": "Save" - }, - { - "type": "text", - "key": "deleteButtonLabel", - "label": "Delete button", - "nested": true, - "defaultValue": "Delete", - "dependsOn": { - "setting": "actionType", - "value": "Update" - } - }, - { - "type": "url", - "label": "Navigate after button press", - "key": "actionUrl", - "placeholder": "Choose a screen", - "dependsOn": { - "setting": "actionType", - "value": "View", - "invert": true - } - }, - { - "type": "boolean", - "label": "Hide notifications", - "key": "notificationOverride", - "defaultValue": false, - "dependsOn": { - "setting": "actionType", - "value": "View", - "invert": true - } - } - ] - }, { "section": true, "name": "Fields", @@ -5436,6 +5385,57 @@ } } ] + }, + { + "section": true, + "name": "Buttons", + "dependsOn": { + "setting": "actionType", + "value": "View", + "invert": true + }, + "settings": [ + { + "type": "text", + "key": "saveButtonLabel", + "label": "Save button", + "nested": true, + "defaultValue": "Save" + }, + { + "type": "text", + "key": "deleteButtonLabel", + "label": "Delete button", + "nested": true, + "defaultValue": "Delete", + "dependsOn": { + "setting": "actionType", + "value": "Update" + } + }, + { + "type": "url", + "label": "Navigate after button press", + "key": "actionUrl", + "placeholder": "Choose a screen", + "dependsOn": { + "setting": "actionType", + "value": "View", + "invert": true + } + }, + { + "type": "boolean", + "label": "Hide notifications", + "key": "notificationOverride", + "defaultValue": false, + "dependsOn": { + "setting": "actionType", + "value": "View", + "invert": true + } + } + ] } ], "context": [ From d9f52295ee88dced451edabc7d32bdd07df63815 Mon Sep 17 00:00:00 2001 From: Dean Date: Fri, 27 Oct 2023 11:36:45 +0100 Subject: [PATCH 03/25] Moved the description block out of the header to ensure it's visible for the view mode --- .../components/app/blocks/form/InnerFormBlock.svelte | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte index e65d2cf90b..f7e9a0d2ed 100644 --- a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte @@ -220,15 +220,11 @@ {/if} - {#if description} - - {/if} {/if} + {#if description} + + {/if} {#key fields} {#each fields as field, idx} From 41d99f6a582500a500b2246968d624411e697929 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 30 Oct 2023 17:41:08 +0000 Subject: [PATCH 04/25] Updating app backup exports to not include automation logs as these bloat the backups. --- packages/backend-core/__mocks__/aws-sdk.ts | 1 + .../tests/core/utilities/mocks/date.ts | 5 ++ packages/pro | 2 +- packages/server/__mocks__/aws-sdk.ts | 7 +++ .../src/api/routes/tests/backup.spec.ts | 37 ++++++++++---- .../server/src/sdk/app/backups/exports.ts | 6 ++- .../src/tests/utilities/TestConfiguration.ts | 5 +- .../server/src/tests/utilities/api/backup.ts | 49 +++++++++++++++++++ .../server/src/tests/utilities/api/index.ts | 3 ++ packages/types/src/api/web/app/backup.ts | 5 ++ 10 files changed, 106 insertions(+), 14 deletions(-) create mode 100644 packages/server/src/tests/utilities/api/backup.ts diff --git a/packages/backend-core/__mocks__/aws-sdk.ts b/packages/backend-core/__mocks__/aws-sdk.ts index b8d91dbaa9..e3be511d08 100644 --- a/packages/backend-core/__mocks__/aws-sdk.ts +++ b/packages/backend-core/__mocks__/aws-sdk.ts @@ -3,6 +3,7 @@ const mockS3 = { deleteObject: jest.fn().mockReturnThis(), deleteObjects: jest.fn().mockReturnThis(), createBucket: jest.fn().mockReturnThis(), + getObject: jest.fn().mockReturnThis(), listObject: jest.fn().mockReturnThis(), getSignedUrl: jest.fn((operation: string, params: any) => { return `http://s3.example.com/${params.Bucket}/${params.Key}` diff --git a/packages/backend-core/tests/core/utilities/mocks/date.ts b/packages/backend-core/tests/core/utilities/mocks/date.ts index f580b68349..9f33058d29 100644 --- a/packages/backend-core/tests/core/utilities/mocks/date.ts +++ b/packages/backend-core/tests/core/utilities/mocks/date.ts @@ -1,2 +1,7 @@ export const MOCK_DATE = new Date("2020-01-01T00:00:00.000Z") + +export const MOCK_DATE_PLUS = (ms: number) => { + return new Date(Date.now() + ms) +} + export const MOCK_DATE_TIMESTAMP = 1577836800000 diff --git a/packages/pro b/packages/pro index 3820c0c93a..132b080643 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8 +Subproject commit 132b080643f0b1ef1488bb8362d1c41ea0bd263f diff --git a/packages/server/__mocks__/aws-sdk.ts b/packages/server/__mocks__/aws-sdk.ts index 8a66f0e213..fa6d099f56 100644 --- a/packages/server/__mocks__/aws-sdk.ts +++ b/packages/server/__mocks__/aws-sdk.ts @@ -70,6 +70,13 @@ module AwsMock { Contents: {}, }) ) + + // @ts-ignore + this.getObject = jest.fn( + response({ + Body: "", + }) + ) } aws.DynamoDB = { DocumentClient } diff --git a/packages/server/src/api/routes/tests/backup.spec.ts b/packages/server/src/api/routes/tests/backup.spec.ts index 92e0176060..0c62b32155 100644 --- a/packages/server/src/api/routes/tests/backup.spec.ts +++ b/packages/server/src/api/routes/tests/backup.spec.ts @@ -5,6 +5,8 @@ import sdk from "../../../sdk" import { checkBuilderEndpoint } from "./utilities/TestFunctions" import { mocks } from "@budibase/backend-core/tests" +mocks.licenses.useBackups() + describe("/backups", () => { let request = setup.getRequest() let config = setup.getConfig() @@ -15,13 +17,13 @@ describe("/backups", () => { await config.init() }) - describe("exportAppDump", () => { + describe("/api/backups/export", () => { it("should be able to export app", async () => { - const res = await request - .post(`/api/backups/export?appId=${config.getAppId()}`) - .set(config.defaultHeaders()) - .expect(200) - expect(res.headers["content-type"]).toEqual("application/gzip") + const { body, headers } = await config.api.backup.exportBasicBackup( + config.getAppId()! + ) + expect(body instanceof Buffer).toBe(true) + expect(headers["content-type"]).toEqual("application/gzip") expect(events.app.exported).toBeCalledTimes(1) }) @@ -36,11 +38,11 @@ describe("/backups", () => { it("should infer the app name from the app", async () => { tk.freeze(mocks.date.MOCK_DATE) - const res = await request - .post(`/api/backups/export?appId=${config.getAppId()}`) - .set(config.defaultHeaders()) + const { headers } = await config.api.backup.exportBasicBackup( + config.getAppId()! + ) - expect(res.headers["content-disposition"]).toEqual( + expect(headers["content-disposition"]).toEqual( `attachment; filename="${ config.getApp()!.name }-export-${mocks.date.MOCK_DATE.getTime()}.tar.gz"` @@ -48,6 +50,21 @@ describe("/backups", () => { }) }) + describe("/api/backups/import", () => { + it("should be able to import an app", async () => { + const appId = config.getAppId()! + const automation = await config.createAutomation() + await config.createAutomationLog(automation, appId) + await config.createScreen() + const exportRes = await config.api.backup.createBackup(appId) + expect(exportRes.backupId).toBeDefined() + const importRes = await config.api.backup.importBackup( + appId, + exportRes.backupId + ) + }) + }) + describe("calculateBackupStats", () => { it("should be able to calculate the backup statistics", async () => { await config.createAutomation() diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index d5ea31cdf5..6432bd2e0d 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -84,7 +84,11 @@ export async function exportDB( } function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) { - const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX] + const ids = [ + USER_METDATA_PREFIX, + LINK_USER_METADATA_PREFIX, + AUTOMATION_LOG_PREFIX, + ] if (excludeRows) { ids.push(TABLE_ROW_PREFIX) } diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index cec8c8aa12..869a42012a 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -774,8 +774,9 @@ class TestConfiguration { // AUTOMATION LOG - async createAutomationLog(automation: Automation) { - return await context.doInAppContext(this.getProdAppId(), async () => { + async createAutomationLog(automation: Automation, appId?: string) { + appId = appId || this.getProdAppId() + return await context.doInAppContext(appId!, async () => { return await pro.sdk.automations.logs.storeLog( automation, basicAutomationResults(automation._id!) diff --git a/packages/server/src/tests/utilities/api/backup.ts b/packages/server/src/tests/utilities/api/backup.ts new file mode 100644 index 0000000000..4cee19fdc6 --- /dev/null +++ b/packages/server/src/tests/utilities/api/backup.ts @@ -0,0 +1,49 @@ +import { + CreateAppBackupResponse, + ImportAppBackupResponse, +} from "@budibase/types" +import TestConfiguration from "../TestConfiguration" +import { TestAPI } from "./base" +import tk from "timekeeper" +import { mocks } from "@budibase/backend-core/tests" + +export class BackupAPI extends TestAPI { + constructor(config: TestConfiguration) { + super(config) + } + + exportBasicBackup = async (appId: string) => { + const result = await this.request + .post(`/api/backups/export?appId=${appId}`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /application\/gzip/) + .expect(200) + return { + body: result.body as Buffer, + headers: result.headers, + } + } + + createBackup = async (appId: string) => { + tk.freeze(mocks.date.MOCK_DATE_PLUS(1)) + const result = await this.request + .post(`/api/apps/${appId}/backups`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + return result.body as CreateAppBackupResponse + } + + importBackup = async ( + appId: string, + backupId: string + ): Promise => { + tk.freeze(mocks.date.MOCK_DATE_PLUS(1)) + const result = await this.request + .post(`/api/apps/${appId}/backups/${backupId}/import`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + return result.body as ImportAppBackupResponse + } +} diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts index fce8237760..c344068295 100644 --- a/packages/server/src/tests/utilities/api/index.ts +++ b/packages/server/src/tests/utilities/api/index.ts @@ -7,6 +7,7 @@ import { DatasourceAPI } from "./datasource" import { LegacyViewAPI } from "./legacyView" import { ScreenAPI } from "./screen" import { ApplicationAPI } from "./application" +import { BackupAPI } from "./backup" export default class API { table: TableAPI @@ -17,6 +18,7 @@ export default class API { datasource: DatasourceAPI screen: ScreenAPI application: ApplicationAPI + backup: BackupAPI constructor(config: TestConfiguration) { this.table = new TableAPI(config) @@ -27,5 +29,6 @@ export default class API { this.datasource = new DatasourceAPI(config) this.screen = new ScreenAPI(config) this.application = new ApplicationAPI(config) + this.backup = new BackupAPI(config) } } diff --git a/packages/types/src/api/web/app/backup.ts b/packages/types/src/api/web/app/backup.ts index c9a8d07f5e..f77707e9c6 100644 --- a/packages/types/src/api/web/app/backup.ts +++ b/packages/types/src/api/web/app/backup.ts @@ -20,3 +20,8 @@ export interface CreateAppBackupResponse { export interface UpdateAppBackupRequest { name: string } + +export interface ImportAppBackupResponse { + restoreId: string + message: string +} From 15b1f3efe63fb8d4eceb3dc66d4c3ed3f90fd6ee Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 31 Oct 2023 10:51:46 +0000 Subject: [PATCH 05/25] Removing duplicate filtering of automation logs. --- packages/server/src/sdk/app/backups/exports.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index 6432bd2e0d..c349dcb927 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -26,7 +26,6 @@ export interface DBDumpOpts { export interface ExportOpts extends DBDumpOpts { tar?: boolean excludeRows?: boolean - excludeLogs?: boolean encryptPassword?: string } @@ -83,7 +82,7 @@ export async function exportDB( }) } -function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) { +function defineFilter(excludeRows?: boolean) { const ids = [ USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, @@ -92,9 +91,6 @@ function defineFilter(excludeRows?: boolean, excludeLogs?: boolean) { if (excludeRows) { ids.push(TABLE_ROW_PREFIX) } - if (excludeLogs) { - ids.push(AUTOMATION_LOG_PREFIX) - } return (doc: any) => !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr) } @@ -122,7 +118,7 @@ export async function exportApp(appId: string, config?: ExportOpts) { fs.writeFileSync(join(tmpPath, path), contents) } } - // get all of the files + // get all the files else { tmpPath = await objectStore.retrieveDirectory( ObjectStoreBuckets.APPS, @@ -145,7 +141,7 @@ export async function exportApp(appId: string, config?: ExportOpts) { // enforce an export of app DB to the tmp path const dbPath = join(tmpPath, DB_EXPORT_FILE) await exportDB(appId, { - filter: defineFilter(config?.excludeRows, config?.excludeLogs), + filter: defineFilter(config?.excludeRows), exportPath: dbPath, }) @@ -195,7 +191,6 @@ export async function streamExportApp({ }) { const tmpPath = await exportApp(appId, { excludeRows, - excludeLogs: true, tar: true, encryptPassword, }) From 88cc8a19eef23af0ace19b1fab3014d70cc44ba5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 31 Oct 2023 11:19:53 +0000 Subject: [PATCH 06/25] Resetting timekeeper and resetting pro back to normal. --- packages/backend-core/tests/core/utilities/mocks/date.ts | 4 ---- packages/pro | 2 +- packages/server/src/api/routes/tests/backup.spec.ts | 1 + packages/server/src/tests/utilities/api/backup.ts | 4 ---- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/tests/core/utilities/mocks/date.ts b/packages/backend-core/tests/core/utilities/mocks/date.ts index 9f33058d29..1e6d105d93 100644 --- a/packages/backend-core/tests/core/utilities/mocks/date.ts +++ b/packages/backend-core/tests/core/utilities/mocks/date.ts @@ -1,7 +1,3 @@ export const MOCK_DATE = new Date("2020-01-01T00:00:00.000Z") -export const MOCK_DATE_PLUS = (ms: number) => { - return new Date(Date.now() + ms) -} - export const MOCK_DATE_TIMESTAMP = 1577836800000 diff --git a/packages/pro b/packages/pro index 132b080643..3820c0c93a 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 132b080643f0b1ef1488bb8362d1c41ea0bd263f +Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8 diff --git a/packages/server/src/api/routes/tests/backup.spec.ts b/packages/server/src/api/routes/tests/backup.spec.ts index 0c62b32155..d12b5e1507 100644 --- a/packages/server/src/api/routes/tests/backup.spec.ts +++ b/packages/server/src/api/routes/tests/backup.spec.ts @@ -14,6 +14,7 @@ describe("/backups", () => { afterAll(setup.afterAll) beforeEach(async () => { + tk.reset() await config.init() }) diff --git a/packages/server/src/tests/utilities/api/backup.ts b/packages/server/src/tests/utilities/api/backup.ts index 4cee19fdc6..f9cbc7086e 100644 --- a/packages/server/src/tests/utilities/api/backup.ts +++ b/packages/server/src/tests/utilities/api/backup.ts @@ -4,8 +4,6 @@ import { } from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI } from "./base" -import tk from "timekeeper" -import { mocks } from "@budibase/backend-core/tests" export class BackupAPI extends TestAPI { constructor(config: TestConfiguration) { @@ -25,7 +23,6 @@ export class BackupAPI extends TestAPI { } createBackup = async (appId: string) => { - tk.freeze(mocks.date.MOCK_DATE_PLUS(1)) const result = await this.request .post(`/api/apps/${appId}/backups`) .set(this.config.defaultHeaders()) @@ -38,7 +35,6 @@ export class BackupAPI extends TestAPI { appId: string, backupId: string ): Promise => { - tk.freeze(mocks.date.MOCK_DATE_PLUS(1)) const result = await this.request .post(`/api/apps/${appId}/backups/${backupId}/import`) .set(this.config.defaultHeaders()) From 7e33aacbb1a77b0fa989d8647d9bf680c9b9ddfe Mon Sep 17 00:00:00 2001 From: Dean Date: Tue, 31 Oct 2023 14:48:23 +0000 Subject: [PATCH 07/25] Stop the sample data being identified as an external source. --- packages/server/src/integrations/utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index b4fff0737a..33895e4fe1 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -11,6 +11,7 @@ import { InvalidColumns, NoEmptyFilterStrings } from "../constants" import { helpers } from "@budibase/shared-core" import * as external from "../api/controllers/table/external" import * as internal from "../api/controllers/table/internal" +import { DEFAULT_BB_DATASOURCE_ID } from "../db/defaultData/datasource_bb_default" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const ROW_ID_REGEX = /^\[.*]$/g @@ -96,7 +97,8 @@ export function isInternalTableID(tableId: string) { export function isExternalTable(table: Table) { if ( table?.sourceId && - table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) + table.sourceId.includes(DocumentType.DATASOURCE + SEPARATOR) && + table?.sourceId !== DEFAULT_BB_DATASOURCE_ID ) { return true } else if (table?.sourceType === TableSourceType.EXTERNAL) { From df6f8dad7e725e5c208bd65682e7db61a3a72d03 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 31 Oct 2023 16:56:19 +0000 Subject: [PATCH 08/25] Updating bull parameters to see if this helps with queue stalling. --- .../backend-core/src/queue/inMemoryQueue.ts | 2 +- packages/backend-core/src/queue/queue.ts | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index af2ec6dbaa..a8add7ecb6 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -36,7 +36,7 @@ class InMemoryQueue { * @param opts This is not used by the in memory queue as there is no real use * case when in memory, but is the same API as Bull */ - constructor(name: string, opts = null) { + constructor(name: string, opts?: any) { this._name = name this._opts = opts this._messages = [] diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 0658147709..c6b02921bf 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -2,10 +2,16 @@ import env from "../environment" import { getRedisOptions } from "../redis/utils" import { JobQueue } from "./constants" import InMemoryQueue from "./inMemoryQueue" -import BullQueue from "bull" +import BullQueue, { QueueOptions } from "bull" import { addListeners, StalledFn } from "./listeners" import * as timers from "../timers" +import * as Redis from "ioredis" +// the queue lock is held for 5 minutes +const QUEUE_LOCK_MS = 300000 +// queue lock is refreshed every 30 seconds +const QUEUE_LOCK_RENEW_INTERNAL_MS = 30000 +// cleanup the queue every 60 seconds const CLEANUP_PERIOD_MS = 60 * 1000 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = [] let cleanupInterval: NodeJS.Timeout @@ -21,7 +27,14 @@ export function createQueue( opts: { removeStalledCb?: StalledFn } = {} ): BullQueue.Queue { const { opts: redisOpts, redisProtocolUrl } = getRedisOptions() - const queueConfig: any = redisProtocolUrl || { redis: redisOpts } + const queueConfig: QueueOptions = { + redis: redisProtocolUrl! || (redisOpts as Redis.RedisOptions), + settings: { + maxStalledCount: 0, + lockDuration: QUEUE_LOCK_MS, + lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS, + }, + } let queue: any if (!env.isTest()) { queue = new BullQueue(jobQueue, queueConfig) From 17319a6981a79ef41abe8fe4f7a26a6b02d4f9cd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 31 Oct 2023 17:52:39 +0000 Subject: [PATCH 09/25] Moving things around so that DEFAULT_BB_DATASOURCE_ID can be imported without cyclics occurring. --- .../server/src/api/controllers/application.ts | 7 ++----- packages/server/src/constants/index.ts | 5 +++++ .../src/db/defaultData/datasource_bb_default.ts | 16 +++++++++------- packages/server/src/integrations/utils.ts | 9 +++++---- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 4afd7b23f9..4e4c66858e 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -32,11 +32,8 @@ import { tenancy, users, } from "@budibase/backend-core" -import { USERS_TABLE_SCHEMA } from "../../constants" -import { - buildDefaultDocs, - DEFAULT_BB_DATASOURCE_ID, -} from "../../db/defaultData/datasource_bb_default" +import { USERS_TABLE_SCHEMA, DEFAULT_BB_DATASOURCE_ID } from "../../constants" +import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default" import { removeAppFromUserRoles } from "../../utilities/workerRequests" import { stringToReadStream } from "../../utilities" import { doesUserHaveLock } from "../../utilities/redis" diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index 1104fb9b29..fb5c42e7b8 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -172,3 +172,8 @@ export enum AutomationErrors { export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets export const MAX_AUTOMATION_RECURRING_ERRORS = 5 export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber" +export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" +export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory" +export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses" +export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee" +export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts index 584b76f879..b430f9ffb6 100644 --- a/packages/server/src/db/defaultData/datasource_bb_default.ts +++ b/packages/server/src/db/defaultData/datasource_bb_default.ts @@ -1,4 +1,12 @@ -import { AutoFieldSubTypes, FieldTypes } from "../../constants" +import { + AutoFieldSubTypes, + FieldTypes, + DEFAULT_BB_DATASOURCE_ID, + DEFAULT_INVENTORY_TABLE_ID, + DEFAULT_EMPLOYEE_TABLE_ID, + DEFAULT_EXPENSES_TABLE_ID, + DEFAULT_JOBS_TABLE_ID, +} from "../../constants" import { importToRows } from "../../api/controllers/table/utils" import { cloneDeep } from "lodash/fp" import LinkDocument from "../linkedRows/LinkDocument" @@ -16,12 +24,6 @@ import { TableSourceType, } from "@budibase/types" -export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" -export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory" -export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses" -export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee" -export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" - const defaultDatasource = { _id: DEFAULT_BB_DATASOURCE_ID, type: dbCore.BUDIBASE_DATASOURCE_TYPE, diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 33895e4fe1..fe8a9055b0 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -7,11 +7,12 @@ import { TableSourceType, } from "@budibase/types" import { DocumentType, SEPARATOR } from "../db/utils" -import { InvalidColumns, NoEmptyFilterStrings } from "../constants" +import { + InvalidColumns, + NoEmptyFilterStrings, + DEFAULT_BB_DATASOURCE_ID, +} from "../constants" import { helpers } from "@budibase/shared-core" -import * as external from "../api/controllers/table/external" -import * as internal from "../api/controllers/table/internal" -import { DEFAULT_BB_DATASOURCE_ID } from "../db/defaultData/datasource_bb_default" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const ROW_ID_REGEX = /^\[.*]$/g From e05821d6d7c78320fe427abd8237738c61c88e31 Mon Sep 17 00:00:00 2001 From: Mitch-Budibase Date: Tue, 31 Oct 2023 18:16:58 +0000 Subject: [PATCH 10/25] License Management Test This test retrieves plans, creates checkout session, and updates license Essential changes have been made to linkStripeCustomer & updatePlan functions to support the test Modified "test:self:ci" to include 'licensing' instead of 'license' Modified environment.ts to include STRIPE_SECRET_KEY --- qa-core/package.json | 2 +- .../src/account-api/api/apis/LicenseAPI.ts | 10 +- qa-core/src/account-api/api/apis/StripeAPI.ts | 13 +- .../tests/licensing/license.manage.spec.ts | 113 ++++++++++++++++++ qa-core/src/environment.ts | 1 + 5 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 qa-core/src/account-api/tests/licensing/license.manage.spec.ts diff --git a/qa-core/package.json b/qa-core/package.json index d266ca9def..cfccd5e650 100644 --- a/qa-core/package.json +++ b/qa-core/package.json @@ -17,7 +17,7 @@ "test:notify": "node scripts/testResultsWebhook", "test:cloud:prod": "yarn run test --testPathIgnorePatterns=\\.integration\\.", "test:cloud:qa": "yarn run test", - "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.license\\.", + "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.licensing\\.", "serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci", "serve": "start-server-and-test dev:built http://localhost:4001/health", "dev:built": "cd ../ && yarn dev:built" diff --git a/qa-core/src/account-api/api/apis/LicenseAPI.ts b/qa-core/src/account-api/api/apis/LicenseAPI.ts index b371f00f05..b3b54c5102 100644 --- a/qa-core/src/account-api/api/apis/LicenseAPI.ts +++ b/qa-core/src/account-api/api/apis/LicenseAPI.ts @@ -99,9 +99,15 @@ export default class LicenseAPI extends BaseAPI { }, opts) } - async updatePlan(opts: APIRequestOpts = { status: 200 }) { + async updatePlan( + priceId: string, + opts: APIRequestOpts = { status: 200 } + ) { return this.doRequest(() => { - return this.client.put(`/api/license/plan`) + return this.client.put(`/api/license/plan`, + { + body: { priceId }, + }) }, opts) } diff --git a/qa-core/src/account-api/api/apis/StripeAPI.ts b/qa-core/src/account-api/api/apis/StripeAPI.ts index c9c776e89b..6fef944206 100644 --- a/qa-core/src/account-api/api/apis/StripeAPI.ts +++ b/qa-core/src/account-api/api/apis/StripeAPI.ts @@ -38,9 +38,18 @@ export default class StripeAPI extends BaseAPI { }, opts) } - async linkStripeCustomer(opts: APIRequestOpts = { status: 200 }) { + async linkStripeCustomer( + accountId: string, + stripeCustomerId: string, + opts: APIRequestOpts = { status: 200 }) { return this.doRequest(() => { - return this.client.post(`/api/stripe/link`) + return this.client.post(`/api/stripe/link`, { + body: { + accountId, + stripeCustomerId + }, + internal: true, + }) }, opts) } diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts new file mode 100644 index 0000000000..a62b733c24 --- /dev/null +++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts @@ -0,0 +1,113 @@ +import TestConfiguration from "../../config/TestConfiguration" +import * as fixtures from "../../fixtures" +import {Hosting, PlanType} from "@budibase/types" + +describe("license management", () => { + const config = new TestConfiguration() + + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + + it("retrieves plans, creates checkout session, and updates license", async () => { + // Create cloud account + const createAccountRequest = fixtures.accounts.generateAccount({ + hosting: Hosting.CLOUD, + }) + const [createAccountRes, account] = + await config.accountsApi.accounts.create(createAccountRequest, { + autoVerify: true, + }) + + // Self response has free license + await config.doInNewState(async () => { + await config.loginAsAccount(createAccountRequest) + const [selfRes, selfBody] = await config.api.accounts.self() + expect(selfBody.license.plan.type).toBe(PlanType.FREE) + }) + + // Retrieve plans + const [plansRes, planBody] = await config.api.licenses.getPlans() + + // Select priceId from premium plan + let premiumPriceId = null + let businessPriceId = '' + for (const plan of planBody) { + if (plan.type === PlanType.PREMIUM) { + premiumPriceId = plan.prices[0].priceId + } + if (plan.type === PlanType.BUSINESS) { + businessPriceId = plan.prices[0].priceId + } + } + + // Create checkout session for price + const checkoutSessionRes = await config.api.stripe.createCheckoutSession( + premiumPriceId + ) + const checkoutSessionUrl = checkoutSessionRes[1].url + expect(checkoutSessionUrl).toContain("checkout.stripe.com") + + // Create stripe customer + const stripe = require('stripe')(process.env.STRIPE_SECRET_KEY) + const customer = await stripe.customers.create({ + email: createAccountRequest.email + }) + + // Create payment method + const paymentMethod = await stripe.paymentMethods.create({ + type: 'card', + card: { + token: 'tok_visa' // Test Visa Card + }, + }) + + // Attach payment method to customer + await stripe.paymentMethods.attach(paymentMethod.id, { + customer: customer.id, + }) + + // Update customer + await stripe.customers.update(customer.id, { + invoice_settings: { + default_payment_method: paymentMethod.id, + }, + }) + + // Create subscription for premium plan + const subscription = await stripe.subscriptions.create({ + customer: customer.id, + items: [ + { + price: premiumPriceId, + quantity: 1, + }, + ], + default_payment_method: paymentMethod.id, + collection_method: 'charge_automatically', + }) + + await config.doInNewState(async () => { + // License updated from Free to Premium + await config.loginAsAccount(createAccountRequest) + await config.api.stripe.linkStripeCustomer(account.accountId, customer.id) + const [_, selfBodyPremium] = await config.api.accounts.self() + expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM) + + // Create portal session - Check URL + const [portalRes, portalSessionBody] = await config.api.stripe.createPortalSession(customer.id) + expect(portalSessionBody.url).toContain("billing.stripe.com") + + // Update subscription from premium to business license + await config.api.licenses.updatePlan(businessPriceId) + + // License updated to Business + const [selfRes, selfBodyBusiness] = await config.api.accounts.self() + expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS) + }) + }) +}) diff --git a/qa-core/src/environment.ts b/qa-core/src/environment.ts index 0257b10831..a805503474 100644 --- a/qa-core/src/environment.ts +++ b/qa-core/src/environment.ts @@ -28,6 +28,7 @@ const env = { MARIADB_DB: process.env.MARIADB_DB, MARIADB_USER: process.env.MARIADB_USER, MARIADB_PASSWORD: process.env.MARIADB_PASSWORD, + STRIPE_SECRET_KEY: process.env.STRIPE_SECRET_KEY, } export = env From 84079450f973ad957d8bd031329832d155603b1b Mon Sep 17 00:00:00 2001 From: Mitch-Budibase Date: Tue, 31 Oct 2023 18:20:27 +0000 Subject: [PATCH 11/25] lint --- .../src/account-api/api/apis/LicenseAPI.ts | 12 +- qa-core/src/account-api/api/apis/StripeAPI.ts | 9 +- .../tests/licensing/license.manage.spec.ts | 205 +++++++++--------- 3 files changed, 112 insertions(+), 114 deletions(-) diff --git a/qa-core/src/account-api/api/apis/LicenseAPI.ts b/qa-core/src/account-api/api/apis/LicenseAPI.ts index b3b54c5102..a9b0a35269 100644 --- a/qa-core/src/account-api/api/apis/LicenseAPI.ts +++ b/qa-core/src/account-api/api/apis/LicenseAPI.ts @@ -99,15 +99,11 @@ export default class LicenseAPI extends BaseAPI { }, opts) } - async updatePlan( - priceId: string, - opts: APIRequestOpts = { status: 200 } - ) { + async updatePlan(priceId: string, opts: APIRequestOpts = { status: 200 }) { return this.doRequest(() => { - return this.client.put(`/api/license/plan`, - { - body: { priceId }, - }) + return this.client.put(`/api/license/plan`, { + body: { priceId }, + }) }, opts) } diff --git a/qa-core/src/account-api/api/apis/StripeAPI.ts b/qa-core/src/account-api/api/apis/StripeAPI.ts index 6fef944206..5a4e810655 100644 --- a/qa-core/src/account-api/api/apis/StripeAPI.ts +++ b/qa-core/src/account-api/api/apis/StripeAPI.ts @@ -39,14 +39,15 @@ export default class StripeAPI extends BaseAPI { } async linkStripeCustomer( - accountId: string, - stripeCustomerId: string, - opts: APIRequestOpts = { status: 200 }) { + accountId: string, + stripeCustomerId: string, + opts: APIRequestOpts = { status: 200 } + ) { return this.doRequest(() => { return this.client.post(`/api/stripe/link`, { body: { accountId, - stripeCustomerId + stripeCustomerId, }, internal: true, }) diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts index a62b733c24..a9109dad3a 100644 --- a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts +++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts @@ -1,113 +1,114 @@ import TestConfiguration from "../../config/TestConfiguration" import * as fixtures from "../../fixtures" -import {Hosting, PlanType} from "@budibase/types" +import { Hosting, PlanType } from "@budibase/types" describe("license management", () => { - const config = new TestConfiguration() + const config = new TestConfiguration() - beforeAll(async () => { - await config.beforeAll() + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + + it("retrieves plans, creates checkout session, and updates license", async () => { + // Create cloud account + const createAccountRequest = fixtures.accounts.generateAccount({ + hosting: Hosting.CLOUD, + }) + const [createAccountRes, account] = + await config.accountsApi.accounts.create(createAccountRequest, { + autoVerify: true, + }) + + // Self response has free license + await config.doInNewState(async () => { + await config.loginAsAccount(createAccountRequest) + const [selfRes, selfBody] = await config.api.accounts.self() + expect(selfBody.license.plan.type).toBe(PlanType.FREE) }) - afterAll(async () => { - await config.afterAll() + // Retrieve plans + const [plansRes, planBody] = await config.api.licenses.getPlans() + + // Select priceId from premium plan + let premiumPriceId = null + let businessPriceId = "" + for (const plan of planBody) { + if (plan.type === PlanType.PREMIUM) { + premiumPriceId = plan.prices[0].priceId + } + if (plan.type === PlanType.BUSINESS) { + businessPriceId = plan.prices[0].priceId + } + } + + // Create checkout session for price + const checkoutSessionRes = await config.api.stripe.createCheckoutSession( + premiumPriceId + ) + const checkoutSessionUrl = checkoutSessionRes[1].url + expect(checkoutSessionUrl).toContain("checkout.stripe.com") + + // Create stripe customer + const stripe = require("stripe")(process.env.STRIPE_SECRET_KEY) + const customer = await stripe.customers.create({ + email: createAccountRequest.email, }) - it("retrieves plans, creates checkout session, and updates license", async () => { - // Create cloud account - const createAccountRequest = fixtures.accounts.generateAccount({ - hosting: Hosting.CLOUD, - }) - const [createAccountRes, account] = - await config.accountsApi.accounts.create(createAccountRequest, { - autoVerify: true, - }) - - // Self response has free license - await config.doInNewState(async () => { - await config.loginAsAccount(createAccountRequest) - const [selfRes, selfBody] = await config.api.accounts.self() - expect(selfBody.license.plan.type).toBe(PlanType.FREE) - }) - - // Retrieve plans - const [plansRes, planBody] = await config.api.licenses.getPlans() - - // Select priceId from premium plan - let premiumPriceId = null - let businessPriceId = '' - for (const plan of planBody) { - if (plan.type === PlanType.PREMIUM) { - premiumPriceId = plan.prices[0].priceId - } - if (plan.type === PlanType.BUSINESS) { - businessPriceId = plan.prices[0].priceId - } - } - - // Create checkout session for price - const checkoutSessionRes = await config.api.stripe.createCheckoutSession( - premiumPriceId - ) - const checkoutSessionUrl = checkoutSessionRes[1].url - expect(checkoutSessionUrl).toContain("checkout.stripe.com") - - // Create stripe customer - const stripe = require('stripe')(process.env.STRIPE_SECRET_KEY) - const customer = await stripe.customers.create({ - email: createAccountRequest.email - }) - - // Create payment method - const paymentMethod = await stripe.paymentMethods.create({ - type: 'card', - card: { - token: 'tok_visa' // Test Visa Card - }, - }) - - // Attach payment method to customer - await stripe.paymentMethods.attach(paymentMethod.id, { - customer: customer.id, - }) - - // Update customer - await stripe.customers.update(customer.id, { - invoice_settings: { - default_payment_method: paymentMethod.id, - }, - }) - - // Create subscription for premium plan - const subscription = await stripe.subscriptions.create({ - customer: customer.id, - items: [ - { - price: premiumPriceId, - quantity: 1, - }, - ], - default_payment_method: paymentMethod.id, - collection_method: 'charge_automatically', - }) - - await config.doInNewState(async () => { - // License updated from Free to Premium - await config.loginAsAccount(createAccountRequest) - await config.api.stripe.linkStripeCustomer(account.accountId, customer.id) - const [_, selfBodyPremium] = await config.api.accounts.self() - expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM) - - // Create portal session - Check URL - const [portalRes, portalSessionBody] = await config.api.stripe.createPortalSession(customer.id) - expect(portalSessionBody.url).toContain("billing.stripe.com") - - // Update subscription from premium to business license - await config.api.licenses.updatePlan(businessPriceId) - - // License updated to Business - const [selfRes, selfBodyBusiness] = await config.api.accounts.self() - expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS) - }) + // Create payment method + const paymentMethod = await stripe.paymentMethods.create({ + type: "card", + card: { + token: "tok_visa", // Test Visa Card + }, }) + + // Attach payment method to customer + await stripe.paymentMethods.attach(paymentMethod.id, { + customer: customer.id, + }) + + // Update customer + await stripe.customers.update(customer.id, { + invoice_settings: { + default_payment_method: paymentMethod.id, + }, + }) + + // Create subscription for premium plan + const subscription = await stripe.subscriptions.create({ + customer: customer.id, + items: [ + { + price: premiumPriceId, + quantity: 1, + }, + ], + default_payment_method: paymentMethod.id, + collection_method: "charge_automatically", + }) + + await config.doInNewState(async () => { + // License updated from Free to Premium + await config.loginAsAccount(createAccountRequest) + await config.api.stripe.linkStripeCustomer(account.accountId, customer.id) + const [_, selfBodyPremium] = await config.api.accounts.self() + expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM) + + // Create portal session - Check URL + const [portalRes, portalSessionBody] = + await config.api.stripe.createPortalSession(customer.id) + expect(portalSessionBody.url).toContain("billing.stripe.com") + + // Update subscription from premium to business license + await config.api.licenses.updatePlan(businessPriceId) + + // License updated to Business + const [selfRes, selfBodyBusiness] = await config.api.accounts.self() + expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS) + }) + }) }) From 234cdbbf2441c4f3921a7226e14b920ba4506bab Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 1 Nov 2023 10:51:34 +0000 Subject: [PATCH 12/25] Bump version to 2.12.2 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 6df4a4c4cd..cb92b3ba0d 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.12.1", + "version": "2.12.2", "npmClient": "yarn", "packages": [ "packages/*" From 984d9497811a6daa049c743f7e4095c06eca8b5a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 1 Nov 2023 15:02:55 +0000 Subject: [PATCH 13/25] Updating single image build - the old method of installing Node 18 was deprecated, switching to the new mechanism. --- hosting/scripts/install-minio.sh | 10 ---------- hosting/single/Dockerfile.v2 | 11 +++++------ scripts/install-minio.sh | 6 +++--- scripts/install-node.sh | 8 ++++++++ 4 files changed, 16 insertions(+), 19 deletions(-) delete mode 100755 hosting/scripts/install-minio.sh create mode 100644 scripts/install-node.sh diff --git a/hosting/scripts/install-minio.sh b/hosting/scripts/install-minio.sh deleted file mode 100755 index 8297593599..0000000000 --- a/hosting/scripts/install-minio.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -if [[ $TARGETARCH == arm* ]] ; -then - echo "INSTALLING ARM64 MINIO" - wget https://dl.min.io/server/minio/release/linux-arm64/minio -else - echo "INSTALLING AMD64 MINIO" - wget https://dl.min.io/server/minio/release/linux-amd64/minio -fi -chmod +x minio diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2 index 5b07a51b27..ec03a1b5a2 100644 --- a/hosting/single/Dockerfile.v2 +++ b/hosting/single/Dockerfile.v2 @@ -42,6 +42,7 @@ COPY packages/string-templates packages/string-templates FROM budibase/couchdb as runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH +ENV NODE_MAJOR 18 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) # e.g. docker build --build-arg TARGETBUILD=aas .... ARG TARGETBUILD=single @@ -49,10 +50,10 @@ ENV TARGETBUILD $TARGETBUILD # install base dependencies RUN apt-get update && \ - apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server + apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server libaio1 # Install postgres client for pg_dump utils -RUN apt install software-properties-common apt-transport-https gpg -y \ +RUN apt install -y software-properties-common apt-transport-https ca-certificates gnupg \ && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ && apt update -y \ @@ -61,10 +62,8 @@ RUN apt install software-properties-common apt-transport-https gpg -y \ # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx WORKDIR /nodejs -RUN curl -sL https://deb.nodesource.com/setup_18.x -o /tmp/nodesource_setup.sh && \ - bash /tmp/nodesource_setup.sh && \ - apt-get install -y --no-install-recommends libaio1 nodejs && \ - npm install --global yarn pm2 +COPY scripts/install-node.sh ./install.sh +RUN chmod +x install.sh && ./install.sh # setup nginx COPY hosting/single/nginx/nginx.conf /etc/nginx diff --git a/scripts/install-minio.sh b/scripts/install-minio.sh index 8297593599..b1e0d9ee80 100755 --- a/scripts/install-minio.sh +++ b/scripts/install-minio.sh @@ -2,9 +2,9 @@ if [[ $TARGETARCH == arm* ]] ; then echo "INSTALLING ARM64 MINIO" - wget https://dl.min.io/server/minio/release/linux-arm64/minio + wget wget https://dl.min.io/server/minio/release/linux-arm64/archive/minio.deb -O minio.deb else echo "INSTALLING AMD64 MINIO" - wget https://dl.min.io/server/minio/release/linux-amd64/minio + wget wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio.deb -O minio.deb fi -chmod +x minio +dpkg -i minio.deb diff --git a/scripts/install-node.sh b/scripts/install-node.sh new file mode 100644 index 0000000000..562bdf2cd3 --- /dev/null +++ b/scripts/install-node.sh @@ -0,0 +1,8 @@ +#!/bin/bash +apt-get install -y gnupg +curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | tee /usr/share/keyrings/nodesource.gpg > /dev/null +echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list +apt-get update +echo "INSTALLING NODE $NODE_MAJOR" +apt-get install -y --no-install-recommends nodejs +npm install --global yarn pm2 \ No newline at end of file From cf40b87d81f7161f2294c6f00c1768ec8e7aa49f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 1 Nov 2023 15:10:32 +0000 Subject: [PATCH 14/25] Use installed MinIO rather than binary. --- hosting/single/runner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 9dc7aa25d8..770b23eec1 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -77,7 +77,7 @@ mkdir -p ${DATA_DIR}/minio chown -R couchdb:couchdb ${DATA_DIR}/couch redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & /bbcouch-runner.sh & -/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & +minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & /etc/init.d/nginx restart if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then # Add monthly cron job to renew certbot certificate From 85325814d06ab47f3fb8fba6bba3717f66c779de Mon Sep 17 00:00:00 2001 From: Mitch-Budibase Date: Wed, 1 Nov 2023 15:18:20 +0000 Subject: [PATCH 15/25] Moving stripe import to top of test file --- qa-core/src/account-api/tests/licensing/license.manage.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts index a9109dad3a..9cad980038 100644 --- a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts +++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts @@ -1,6 +1,7 @@ import TestConfiguration from "../../config/TestConfiguration" import * as fixtures from "../../fixtures" import { Hosting, PlanType } from "@budibase/types" +const stripe = require("stripe")(process.env.STRIPE_SECRET_KEY) describe("license management", () => { const config = new TestConfiguration() @@ -53,7 +54,6 @@ describe("license management", () => { expect(checkoutSessionUrl).toContain("checkout.stripe.com") // Create stripe customer - const stripe = require("stripe")(process.env.STRIPE_SECRET_KEY) const customer = await stripe.customers.create({ email: createAccountRequest.email, }) From d6a4b71655991f27ee45986f0339bd1e33d8dc2b Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 1 Nov 2023 15:37:08 +0000 Subject: [PATCH 16/25] Bump version to 2.12.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index cb92b3ba0d..dc6ffb2da2 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.12.2", + "version": "2.12.3", "npmClient": "yarn", "packages": [ "packages/*" From 586d539109f38d9a48b8fe23b0d4ea9084d2320d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 1 Nov 2023 16:23:51 +0000 Subject: [PATCH 17/25] Adding release information to single image. --- .github/workflows/release-singleimage.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml index 4d35916f4d..a3444d5e7a 100644 --- a/.github/workflows/release-singleimage.yml +++ b/.github/workflows/release-singleimage.yml @@ -66,14 +66,21 @@ jobs: context: . push: true platforms: linux/amd64,linux/arm64 + build-args: BUDIBASE_VERSION=$BUDIBASE_VERSION tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }} file: ./hosting/single/Dockerfile.v2 + env: + BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} - name: Tag and release Budibase Azure App Service docker image uses: docker/build-push-action@v2 with: context: . push: true platforms: linux/amd64 - build-args: TARGETBUILD=aas + build-args: | + TARGETBUILD=aas + BUDIBASE_VERSION=$BUDIBASE_VERSION tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }} file: ./hosting/single/Dockerfile.v2 + env: + BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} From 0477ed64f6edd6b2a2e02a25fb1baaff591584fb Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 1 Nov 2023 16:40:29 +0000 Subject: [PATCH 18/25] Bump version to 2.12.4 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index dc6ffb2da2..f0f51242d1 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.12.3", + "version": "2.12.4", "npmClient": "yarn", "packages": [ "packages/*" From 17b6985f06aa5faf256c3f5f45b76a1a55278414 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 1 Nov 2023 17:48:13 +0000 Subject: [PATCH 19/25] Adding a duration utility for changing between time types. --- packages/backend-core/src/index.ts | 1 + packages/backend-core/src/queue/queue.ts | 7 +-- packages/backend-core/src/utils/Duration.ts | 49 +++++++++++++++++++ packages/backend-core/src/utils/index.ts | 1 + .../src/utils/tests/Duration.spec.ts | 19 +++++++ 5 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 packages/backend-core/src/utils/Duration.ts create mode 100644 packages/backend-core/src/utils/tests/Duration.spec.ts diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index ffffd8240a..c7cf9f56cc 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -30,6 +30,7 @@ export * as timers from "./timers" export { default as env } from "./environment" export * as blacklist from "./blacklist" export * as docUpdates from "./docUpdates" +export * from "./utils/Duration" export { SearchParams } from "./db" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index c6b02921bf..c0d1861de3 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -4,15 +4,16 @@ import { JobQueue } from "./constants" import InMemoryQueue from "./inMemoryQueue" import BullQueue, { QueueOptions } from "bull" import { addListeners, StalledFn } from "./listeners" +import { Duration } from "../utils" import * as timers from "../timers" import * as Redis from "ioredis" // the queue lock is held for 5 minutes -const QUEUE_LOCK_MS = 300000 +const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs() // queue lock is refreshed every 30 seconds -const QUEUE_LOCK_RENEW_INTERNAL_MS = 30000 +const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs() // cleanup the queue every 60 seconds -const CLEANUP_PERIOD_MS = 60 * 1000 +const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs() let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = [] let cleanupInterval: NodeJS.Timeout diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts new file mode 100644 index 0000000000..f376c2f7c7 --- /dev/null +++ b/packages/backend-core/src/utils/Duration.ts @@ -0,0 +1,49 @@ +export enum DurationType { + MILLISECONDS = "milliseconds", + SECONDS = "seconds", + MINUTES = "minutes", + HOURS = "hours", + DAYS = "days", +} + +const conversion: Record = { + milliseconds: 1, + seconds: 1000, + minutes: 60 * 1000, + hours: 60 * 60 * 1000, + days: 24 * 60 * 60 * 1000, +} + +export class Duration { + static convert(from: DurationType, to: DurationType, duration: number) { + const milliseconds = duration * conversion[from] + return milliseconds / conversion[to] + } + + static from(from: DurationType, duration: number) { + return { + to: (to: DurationType) => { + return Duration.convert(from, to, duration) + }, + toMs: () => { + return Duration.convert(from, DurationType.MILLISECONDS, duration) + }, + } + } + + static fromSeconds(duration: number) { + return Duration.from(DurationType.SECONDS, duration) + } + + static fromMinutes(duration: number) { + return Duration.from(DurationType.MINUTES, duration) + } + + static fromHours(duration: number) { + return Duration.from(DurationType.HOURS, duration) + } + + static fromDays(duration: number) { + return Duration.from(DurationType.DAYS, duration) + } +} diff --git a/packages/backend-core/src/utils/index.ts b/packages/backend-core/src/utils/index.ts index 318a7f13ba..ac17227459 100644 --- a/packages/backend-core/src/utils/index.ts +++ b/packages/backend-core/src/utils/index.ts @@ -1,3 +1,4 @@ export * from "./hashing" export * from "./utils" export * from "./stringUtils" +export * from "./Duration" diff --git a/packages/backend-core/src/utils/tests/Duration.spec.ts b/packages/backend-core/src/utils/tests/Duration.spec.ts new file mode 100644 index 0000000000..46b996f788 --- /dev/null +++ b/packages/backend-core/src/utils/tests/Duration.spec.ts @@ -0,0 +1,19 @@ +import { Duration, DurationType } from "../Duration" + +describe("duration", () => { + it("should convert minutes to milliseconds", () => { + expect(Duration.fromMinutes(5).toMs()).toBe(300000) + }) + + it("should convert seconds to milliseconds", () => { + expect(Duration.fromSeconds(30).toMs()).toBe(30000) + }) + + it("should convert days to milliseconds", () => { + expect(Duration.fromDays(1).toMs()).toBe(86400000) + }) + + it("should convert minutes to days", () => { + expect(Duration.fromMinutes(1440).to(DurationType.DAYS)).toBe(1) + }) +}) From e0bf0f6891e60132615742b01338fb1a2b2a3c0a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Nov 2023 11:52:18 +0100 Subject: [PATCH 20/25] Fix payloads --- .github/workflows/deploy-qa.yml | 8 +++++--- .github/workflows/release-master.yml | 10 ++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml index d850d289ff..0580ac1c3a 100644 --- a/.github/workflows/deploy-qa.yml +++ b/.github/workflows/deploy-qa.yml @@ -11,10 +11,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: peter-evans/repository-dispatch@v2 - env: - PAYLOAD_VERSION: ${{ github.sha }} - REF_NAME: ${{ github.ref_name}} with: repository: budibase/budibase-deploys event-type: budicloud-qa-deploy token: ${{ secrets.GH_ACCESS_TOKEN }} + client-payload: |- + { + "PAYLOAD_VERSION": "${{ github.sha }}", + "REF_NAME": "${{ github.ref_name}}" + } diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index df25182cd6..d579670cf6 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -171,11 +171,13 @@ jobs: release_version=$(cat lerna.json | jq -r '.version') echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} - REF_NAME: ${{ github.ref_name}} + - uses: peter-evans/repository-dispatch@v2 with: repository: budibase/budibase-deploys event: budicloud-qa-deploy github_pat: ${{ secrets.GH_ACCESS_TOKEN }} + client-payload: |- + { + "PAYLOAD_VERSION": "${{ github.sha }}", + "REF_NAME": "${{ github.ref_name}}" + } From f74556cb7aacaaa0cc9447481b830a370c49c41e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 2 Nov 2023 11:19:26 +0000 Subject: [PATCH 21/25] Hardcoding the memory used by the build process to make sure it always tries to claim as much as it needs to run the build fully (avoid heap errors as seen on feature branch builds). --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 417fb31e0e..8a27cde104 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "scripts": { "preinstall": "node scripts/syncProPackage.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", - "build": "lerna run build --stream", + "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "check:types": "lerna run check:types", "build:sdk": "lerna run --stream build:sdk", From f08c42f338f381951599fac6c6a17b6fe524773a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Nov 2023 13:15:39 +0100 Subject: [PATCH 22/25] Use ref name as payload version --- .github/workflows/release-master.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index d579670cf6..7648ae32e7 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -165,11 +165,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Get the current budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - uses: peter-evans/repository-dispatch@v2 with: @@ -178,6 +173,6 @@ jobs: github_pat: ${{ secrets.GH_ACCESS_TOKEN }} client-payload: |- { - "PAYLOAD_VERSION": "${{ github.sha }}", + "PAYLOAD_VERSION": "${{ github.ref_name }}", "REF_NAME": "${{ github.ref_name}}" } From 2ac06465bfdcb6463ba5c3c484ef70da7b2682bd Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Thu, 2 Nov 2023 13:11:16 +0000 Subject: [PATCH 23/25] Remove docs section (#12248) --- pull_request_template.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pull_request_template.md b/pull_request_template.md index 36e2f425d5..fa0f7f2143 100644 --- a/pull_request_template.md +++ b/pull_request_template.md @@ -1,18 +1,16 @@ ## Description + _Describe the problem or feature in addition to a link to the relevant github issues._ -Addresses: +Addresses: + - `` - ...more if required ## App Export + - If possible, attach an app export file along with your request template to make QA testing easier, with minimal setup. ## Screenshots + _If a UI facing feature, a short video of the happy path, and some screenshots of the new functionality._ - -## Documentation -- [ ] I have reviewed the budibase documentatation to verify if this feature requires any changes. If changes or new docs are required I have written them. - - - From 4751f9b240407abee4e60bc212ecca1994c51710 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 2 Nov 2023 13:23:31 +0000 Subject: [PATCH 24/25] Correcting BUDIBASE_VERSION in single image. --- hosting/single/Dockerfile.v2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2 index ec03a1b5a2..272860cca2 100644 --- a/hosting/single/Dockerfile.v2 +++ b/hosting/single/Dockerfile.v2 @@ -120,7 +120,7 @@ VOLUME /data ARG BUDIBASE_VERSION # Ensuring the version argument is sent RUN test -n "$BUDIBASE_VERSION" -ENV BUDIBASE_VERSION=$BUDIBASE_VERSION +ENV BUDIBASE_VERSION $BUDIBASE_VERSION HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh" From 52510c5e48506dbd1e0cb1f9cdf03949b101fd6c Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 2 Nov 2023 13:24:05 +0000 Subject: [PATCH 25/25] Bump version to 2.12.5 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index f0f51242d1..5dd081ec14 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.12.4", + "version": "2.12.5", "npmClient": "yarn", "packages": [ "packages/*"