From 0d67f000f03e1c27692e92dcf866cd36b55435cd Mon Sep 17 00:00:00 2001 From: Dean Date: Wed, 18 Oct 2023 12:19:24 +0100 Subject: [PATCH 01/12] Initial reflow of the block settings --- packages/client/manifest.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 8d0a4e456f..d5fca15fd7 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -5288,17 +5288,17 @@ }, "settings": [ { - "type": "select", + "type": "table", + "label": "Data", + "key": "dataSource" + }, + { + "type": "radio", "label": "Type", "key": "actionType", "options": ["Create", "Update", "View"], "defaultValue": "Create" }, - { - "type": "table", - "label": "Data", - "key": "dataSource" - }, { "type": "text", "label": "Title", @@ -5323,7 +5323,7 @@ }, { "type": "text", - "label": "Empty text", + "label": "No rows found", "key": "noRowsMessage", "defaultValue": "We couldn't find a row to display", "nested": true From f59d6222912443484cb98fb666dd92442199abc7 Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 19 Oct 2023 12:28:28 +0100 Subject: [PATCH 02/12] Moved buttons section below fields --- packages/client/manifest.json | 102 +++++++++++++++++----------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index d5fca15fd7..1919f4ccd2 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -5330,57 +5330,6 @@ } ] }, - { - "section": true, - "name": "Buttons", - "dependsOn": { - "setting": "actionType", - "value": "View", - "invert": true - }, - "settings": [ - { - "type": "text", - "key": "saveButtonLabel", - "label": "Save button", - "nested": true, - "defaultValue": "Save" - }, - { - "type": "text", - "key": "deleteButtonLabel", - "label": "Delete button", - "nested": true, - "defaultValue": "Delete", - "dependsOn": { - "setting": "actionType", - "value": "Update" - } - }, - { - "type": "url", - "label": "Navigate after button press", - "key": "actionUrl", - "placeholder": "Choose a screen", - "dependsOn": { - "setting": "actionType", - "value": "View", - "invert": true - } - }, - { - "type": "boolean", - "label": "Hide notifications", - "key": "notificationOverride", - "defaultValue": false, - "dependsOn": { - "setting": "actionType", - "value": "View", - "invert": true - } - } - ] - }, { "section": true, "name": "Fields", @@ -5436,6 +5385,57 @@ } } ] + }, + { + "section": true, + "name": "Buttons", + "dependsOn": { + "setting": "actionType", + "value": "View", + "invert": true + }, + "settings": [ + { + "type": "text", + "key": "saveButtonLabel", + "label": "Save button", + "nested": true, + "defaultValue": "Save" + }, + { + "type": "text", + "key": "deleteButtonLabel", + "label": "Delete button", + "nested": true, + "defaultValue": "Delete", + "dependsOn": { + "setting": "actionType", + "value": "Update" + } + }, + { + "type": "url", + "label": "Navigate after button press", + "key": "actionUrl", + "placeholder": "Choose a screen", + "dependsOn": { + "setting": "actionType", + "value": "View", + "invert": true + } + }, + { + "type": "boolean", + "label": "Hide notifications", + "key": "notificationOverride", + "defaultValue": false, + "dependsOn": { + "setting": "actionType", + "value": "View", + "invert": true + } + } + ] } ], "context": [ From df6f8dad7e725e5c208bd65682e7db61a3a72d03 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 31 Oct 2023 16:56:19 +0000 Subject: [PATCH 03/12] Updating bull parameters to see if this helps with queue stalling. --- .../backend-core/src/queue/inMemoryQueue.ts | 2 +- packages/backend-core/src/queue/queue.ts | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index af2ec6dbaa..a8add7ecb6 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -36,7 +36,7 @@ class InMemoryQueue { * @param opts This is not used by the in memory queue as there is no real use * case when in memory, but is the same API as Bull */ - constructor(name: string, opts = null) { + constructor(name: string, opts?: any) { this._name = name this._opts = opts this._messages = [] diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 0658147709..c6b02921bf 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -2,10 +2,16 @@ import env from "../environment" import { getRedisOptions } from "../redis/utils" import { JobQueue } from "./constants" import InMemoryQueue from "./inMemoryQueue" -import BullQueue from "bull" +import BullQueue, { QueueOptions } from "bull" import { addListeners, StalledFn } from "./listeners" import * as timers from "../timers" +import * as Redis from "ioredis" +// the queue lock is held for 5 minutes +const QUEUE_LOCK_MS = 300000 +// queue lock is refreshed every 30 seconds +const QUEUE_LOCK_RENEW_INTERNAL_MS = 30000 +// cleanup the queue every 60 seconds const CLEANUP_PERIOD_MS = 60 * 1000 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = [] let cleanupInterval: NodeJS.Timeout @@ -21,7 +27,14 @@ export function createQueue( opts: { removeStalledCb?: StalledFn } = {} ): BullQueue.Queue { const { opts: redisOpts, redisProtocolUrl } = getRedisOptions() - const queueConfig: any = redisProtocolUrl || { redis: redisOpts } + const queueConfig: QueueOptions = { + redis: redisProtocolUrl! || (redisOpts as Redis.RedisOptions), + settings: { + maxStalledCount: 0, + lockDuration: QUEUE_LOCK_MS, + lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS, + }, + } let queue: any if (!env.isTest()) { queue = new BullQueue(jobQueue, queueConfig) From e05821d6d7c78320fe427abd8237738c61c88e31 Mon Sep 17 00:00:00 2001 From: Mitch-Budibase Date: Tue, 31 Oct 2023 18:16:58 +0000 Subject: [PATCH 04/12] License Management Test This test retrieves plans, creates checkout session, and updates license Essential changes have been made to linkStripeCustomer & updatePlan functions to support the test Modified "test:self:ci" to include 'licensing' instead of 'license' Modified environment.ts to include STRIPE_SECRET_KEY --- qa-core/package.json | 2 +- .../src/account-api/api/apis/LicenseAPI.ts | 10 +- qa-core/src/account-api/api/apis/StripeAPI.ts | 13 +- .../tests/licensing/license.manage.spec.ts | 113 ++++++++++++++++++ qa-core/src/environment.ts | 1 + 5 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 qa-core/src/account-api/tests/licensing/license.manage.spec.ts diff --git a/qa-core/package.json b/qa-core/package.json index d266ca9def..cfccd5e650 100644 --- a/qa-core/package.json +++ b/qa-core/package.json @@ -17,7 +17,7 @@ "test:notify": "node scripts/testResultsWebhook", "test:cloud:prod": "yarn run test --testPathIgnorePatterns=\\.integration\\.", "test:cloud:qa": "yarn run test", - "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.license\\.", + "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.licensing\\.", "serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci", "serve": "start-server-and-test dev:built http://localhost:4001/health", "dev:built": "cd ../ && yarn dev:built" diff --git a/qa-core/src/account-api/api/apis/LicenseAPI.ts b/qa-core/src/account-api/api/apis/LicenseAPI.ts index b371f00f05..b3b54c5102 100644 --- a/qa-core/src/account-api/api/apis/LicenseAPI.ts +++ b/qa-core/src/account-api/api/apis/LicenseAPI.ts @@ -99,9 +99,15 @@ export default class LicenseAPI extends BaseAPI { }, opts) } - async updatePlan(opts: APIRequestOpts = { status: 200 }) { + async updatePlan( + priceId: string, + opts: APIRequestOpts = { status: 200 } + ) { return this.doRequest(() => { - return this.client.put(`/api/license/plan`) + return this.client.put(`/api/license/plan`, + { + body: { priceId }, + }) }, opts) } diff --git a/qa-core/src/account-api/api/apis/StripeAPI.ts b/qa-core/src/account-api/api/apis/StripeAPI.ts index c9c776e89b..6fef944206 100644 --- a/qa-core/src/account-api/api/apis/StripeAPI.ts +++ b/qa-core/src/account-api/api/apis/StripeAPI.ts @@ -38,9 +38,18 @@ export default class StripeAPI extends BaseAPI { }, opts) } - async linkStripeCustomer(opts: APIRequestOpts = { status: 200 }) { + async linkStripeCustomer( + accountId: string, + stripeCustomerId: string, + opts: APIRequestOpts = { status: 200 }) { return this.doRequest(() => { - return this.client.post(`/api/stripe/link`) + return this.client.post(`/api/stripe/link`, { + body: { + accountId, + stripeCustomerId + }, + internal: true, + }) }, opts) } diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts new file mode 100644 index 0000000000..a62b733c24 --- /dev/null +++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts @@ -0,0 +1,113 @@ +import TestConfiguration from "../../config/TestConfiguration" +import * as fixtures from "../../fixtures" +import {Hosting, PlanType} from "@budibase/types" + +describe("license management", () => { + const config = new TestConfiguration() + + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + + it("retrieves plans, creates checkout session, and updates license", async () => { + // Create cloud account + const createAccountRequest = fixtures.accounts.generateAccount({ + hosting: Hosting.CLOUD, + }) + const [createAccountRes, account] = + await config.accountsApi.accounts.create(createAccountRequest, { + autoVerify: true, + }) + + // Self response has free license + await config.doInNewState(async () => { + await config.loginAsAccount(createAccountRequest) + const [selfRes, selfBody] = await config.api.accounts.self() + expect(selfBody.license.plan.type).toBe(PlanType.FREE) + }) + + // Retrieve plans + const [plansRes, planBody] = await config.api.licenses.getPlans() + + // Select priceId from premium plan + let premiumPriceId = null + let businessPriceId = '' + for (const plan of planBody) { + if (plan.type === PlanType.PREMIUM) { + premiumPriceId = plan.prices[0].priceId + } + if (plan.type === PlanType.BUSINESS) { + businessPriceId = plan.prices[0].priceId + } + } + + // Create checkout session for price + const checkoutSessionRes = await config.api.stripe.createCheckoutSession( + premiumPriceId + ) + const checkoutSessionUrl = checkoutSessionRes[1].url + expect(checkoutSessionUrl).toContain("checkout.stripe.com") + + // Create stripe customer + const stripe = require('stripe')(process.env.STRIPE_SECRET_KEY) + const customer = await stripe.customers.create({ + email: createAccountRequest.email + }) + + // Create payment method + const paymentMethod = await stripe.paymentMethods.create({ + type: 'card', + card: { + token: 'tok_visa' // Test Visa Card + }, + }) + + // Attach payment method to customer + await stripe.paymentMethods.attach(paymentMethod.id, { + customer: customer.id, + }) + + // Update customer + await stripe.customers.update(customer.id, { + invoice_settings: { + default_payment_method: paymentMethod.id, + }, + }) + + // Create subscription for premium plan + const subscription = await stripe.subscriptions.create({ + customer: customer.id, + items: [ + { + price: premiumPriceId, + quantity: 1, + }, + ], + default_payment_method: paymentMethod.id, + collection_method: 'charge_automatically', + }) + + await config.doInNewState(async () => { + // License updated from Free to Premium + await config.loginAsAccount(createAccountRequest) + await config.api.stripe.linkStripeCustomer(account.accountId, customer.id) + const [_, selfBodyPremium] = await config.api.accounts.self() + expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM) + + // Create portal session - Check URL + const [portalRes, portalSessionBody] = await config.api.stripe.createPortalSession(customer.id) + expect(portalSessionBody.url).toContain("billing.stripe.com") + + // Update subscription from premium to business license + await config.api.licenses.updatePlan(businessPriceId) + + // License updated to Business + const [selfRes, selfBodyBusiness] = await config.api.accounts.self() + expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS) + }) + }) +}) diff --git a/qa-core/src/environment.ts b/qa-core/src/environment.ts index 0257b10831..a805503474 100644 --- a/qa-core/src/environment.ts +++ b/qa-core/src/environment.ts @@ -28,6 +28,7 @@ const env = { MARIADB_DB: process.env.MARIADB_DB, MARIADB_USER: process.env.MARIADB_USER, MARIADB_PASSWORD: process.env.MARIADB_PASSWORD, + STRIPE_SECRET_KEY: process.env.STRIPE_SECRET_KEY, } export = env From 84079450f973ad957d8bd031329832d155603b1b Mon Sep 17 00:00:00 2001 From: Mitch-Budibase Date: Tue, 31 Oct 2023 18:20:27 +0000 Subject: [PATCH 05/12] lint --- .../src/account-api/api/apis/LicenseAPI.ts | 12 +- qa-core/src/account-api/api/apis/StripeAPI.ts | 9 +- .../tests/licensing/license.manage.spec.ts | 205 +++++++++--------- 3 files changed, 112 insertions(+), 114 deletions(-) diff --git a/qa-core/src/account-api/api/apis/LicenseAPI.ts b/qa-core/src/account-api/api/apis/LicenseAPI.ts index b3b54c5102..a9b0a35269 100644 --- a/qa-core/src/account-api/api/apis/LicenseAPI.ts +++ b/qa-core/src/account-api/api/apis/LicenseAPI.ts @@ -99,15 +99,11 @@ export default class LicenseAPI extends BaseAPI { }, opts) } - async updatePlan( - priceId: string, - opts: APIRequestOpts = { status: 200 } - ) { + async updatePlan(priceId: string, opts: APIRequestOpts = { status: 200 }) { return this.doRequest(() => { - return this.client.put(`/api/license/plan`, - { - body: { priceId }, - }) + return this.client.put(`/api/license/plan`, { + body: { priceId }, + }) }, opts) } diff --git a/qa-core/src/account-api/api/apis/StripeAPI.ts b/qa-core/src/account-api/api/apis/StripeAPI.ts index 6fef944206..5a4e810655 100644 --- a/qa-core/src/account-api/api/apis/StripeAPI.ts +++ b/qa-core/src/account-api/api/apis/StripeAPI.ts @@ -39,14 +39,15 @@ export default class StripeAPI extends BaseAPI { } async linkStripeCustomer( - accountId: string, - stripeCustomerId: string, - opts: APIRequestOpts = { status: 200 }) { + accountId: string, + stripeCustomerId: string, + opts: APIRequestOpts = { status: 200 } + ) { return this.doRequest(() => { return this.client.post(`/api/stripe/link`, { body: { accountId, - stripeCustomerId + stripeCustomerId, }, internal: true, }) diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts index a62b733c24..a9109dad3a 100644 --- a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts +++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts @@ -1,113 +1,114 @@ import TestConfiguration from "../../config/TestConfiguration" import * as fixtures from "../../fixtures" -import {Hosting, PlanType} from "@budibase/types" +import { Hosting, PlanType } from "@budibase/types" describe("license management", () => { - const config = new TestConfiguration() + const config = new TestConfiguration() - beforeAll(async () => { - await config.beforeAll() + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + + it("retrieves plans, creates checkout session, and updates license", async () => { + // Create cloud account + const createAccountRequest = fixtures.accounts.generateAccount({ + hosting: Hosting.CLOUD, + }) + const [createAccountRes, account] = + await config.accountsApi.accounts.create(createAccountRequest, { + autoVerify: true, + }) + + // Self response has free license + await config.doInNewState(async () => { + await config.loginAsAccount(createAccountRequest) + const [selfRes, selfBody] = await config.api.accounts.self() + expect(selfBody.license.plan.type).toBe(PlanType.FREE) }) - afterAll(async () => { - await config.afterAll() + // Retrieve plans + const [plansRes, planBody] = await config.api.licenses.getPlans() + + // Select priceId from premium plan + let premiumPriceId = null + let businessPriceId = "" + for (const plan of planBody) { + if (plan.type === PlanType.PREMIUM) { + premiumPriceId = plan.prices[0].priceId + } + if (plan.type === PlanType.BUSINESS) { + businessPriceId = plan.prices[0].priceId + } + } + + // Create checkout session for price + const checkoutSessionRes = await config.api.stripe.createCheckoutSession( + premiumPriceId + ) + const checkoutSessionUrl = checkoutSessionRes[1].url + expect(checkoutSessionUrl).toContain("checkout.stripe.com") + + // Create stripe customer + const stripe = require("stripe")(process.env.STRIPE_SECRET_KEY) + const customer = await stripe.customers.create({ + email: createAccountRequest.email, }) - it("retrieves plans, creates checkout session, and updates license", async () => { - // Create cloud account - const createAccountRequest = fixtures.accounts.generateAccount({ - hosting: Hosting.CLOUD, - }) - const [createAccountRes, account] = - await config.accountsApi.accounts.create(createAccountRequest, { - autoVerify: true, - }) - - // Self response has free license - await config.doInNewState(async () => { - await config.loginAsAccount(createAccountRequest) - const [selfRes, selfBody] = await config.api.accounts.self() - expect(selfBody.license.plan.type).toBe(PlanType.FREE) - }) - - // Retrieve plans - const [plansRes, planBody] = await config.api.licenses.getPlans() - - // Select priceId from premium plan - let premiumPriceId = null - let businessPriceId = '' - for (const plan of planBody) { - if (plan.type === PlanType.PREMIUM) { - premiumPriceId = plan.prices[0].priceId - } - if (plan.type === PlanType.BUSINESS) { - businessPriceId = plan.prices[0].priceId - } - } - - // Create checkout session for price - const checkoutSessionRes = await config.api.stripe.createCheckoutSession( - premiumPriceId - ) - const checkoutSessionUrl = checkoutSessionRes[1].url - expect(checkoutSessionUrl).toContain("checkout.stripe.com") - - // Create stripe customer - const stripe = require('stripe')(process.env.STRIPE_SECRET_KEY) - const customer = await stripe.customers.create({ - email: createAccountRequest.email - }) - - // Create payment method - const paymentMethod = await stripe.paymentMethods.create({ - type: 'card', - card: { - token: 'tok_visa' // Test Visa Card - }, - }) - - // Attach payment method to customer - await stripe.paymentMethods.attach(paymentMethod.id, { - customer: customer.id, - }) - - // Update customer - await stripe.customers.update(customer.id, { - invoice_settings: { - default_payment_method: paymentMethod.id, - }, - }) - - // Create subscription for premium plan - const subscription = await stripe.subscriptions.create({ - customer: customer.id, - items: [ - { - price: premiumPriceId, - quantity: 1, - }, - ], - default_payment_method: paymentMethod.id, - collection_method: 'charge_automatically', - }) - - await config.doInNewState(async () => { - // License updated from Free to Premium - await config.loginAsAccount(createAccountRequest) - await config.api.stripe.linkStripeCustomer(account.accountId, customer.id) - const [_, selfBodyPremium] = await config.api.accounts.self() - expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM) - - // Create portal session - Check URL - const [portalRes, portalSessionBody] = await config.api.stripe.createPortalSession(customer.id) - expect(portalSessionBody.url).toContain("billing.stripe.com") - - // Update subscription from premium to business license - await config.api.licenses.updatePlan(businessPriceId) - - // License updated to Business - const [selfRes, selfBodyBusiness] = await config.api.accounts.self() - expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS) - }) + // Create payment method + const paymentMethod = await stripe.paymentMethods.create({ + type: "card", + card: { + token: "tok_visa", // Test Visa Card + }, }) + + // Attach payment method to customer + await stripe.paymentMethods.attach(paymentMethod.id, { + customer: customer.id, + }) + + // Update customer + await stripe.customers.update(customer.id, { + invoice_settings: { + default_payment_method: paymentMethod.id, + }, + }) + + // Create subscription for premium plan + const subscription = await stripe.subscriptions.create({ + customer: customer.id, + items: [ + { + price: premiumPriceId, + quantity: 1, + }, + ], + default_payment_method: paymentMethod.id, + collection_method: "charge_automatically", + }) + + await config.doInNewState(async () => { + // License updated from Free to Premium + await config.loginAsAccount(createAccountRequest) + await config.api.stripe.linkStripeCustomer(account.accountId, customer.id) + const [_, selfBodyPremium] = await config.api.accounts.self() + expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM) + + // Create portal session - Check URL + const [portalRes, portalSessionBody] = + await config.api.stripe.createPortalSession(customer.id) + expect(portalSessionBody.url).toContain("billing.stripe.com") + + // Update subscription from premium to business license + await config.api.licenses.updatePlan(businessPriceId) + + // License updated to Business + const [selfRes, selfBodyBusiness] = await config.api.accounts.self() + expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS) + }) + }) }) From 984d9497811a6daa049c743f7e4095c06eca8b5a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 1 Nov 2023 15:02:55 +0000 Subject: [PATCH 06/12] Updating single image build - the old method of installing Node 18 was deprecated, switching to the new mechanism. --- hosting/scripts/install-minio.sh | 10 ---------- hosting/single/Dockerfile.v2 | 11 +++++------ scripts/install-minio.sh | 6 +++--- scripts/install-node.sh | 8 ++++++++ 4 files changed, 16 insertions(+), 19 deletions(-) delete mode 100755 hosting/scripts/install-minio.sh create mode 100644 scripts/install-node.sh diff --git a/hosting/scripts/install-minio.sh b/hosting/scripts/install-minio.sh deleted file mode 100755 index 8297593599..0000000000 --- a/hosting/scripts/install-minio.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -if [[ $TARGETARCH == arm* ]] ; -then - echo "INSTALLING ARM64 MINIO" - wget https://dl.min.io/server/minio/release/linux-arm64/minio -else - echo "INSTALLING AMD64 MINIO" - wget https://dl.min.io/server/minio/release/linux-amd64/minio -fi -chmod +x minio diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2 index 5b07a51b27..ec03a1b5a2 100644 --- a/hosting/single/Dockerfile.v2 +++ b/hosting/single/Dockerfile.v2 @@ -42,6 +42,7 @@ COPY packages/string-templates packages/string-templates FROM budibase/couchdb as runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH +ENV NODE_MAJOR 18 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) # e.g. docker build --build-arg TARGETBUILD=aas .... ARG TARGETBUILD=single @@ -49,10 +50,10 @@ ENV TARGETBUILD $TARGETBUILD # install base dependencies RUN apt-get update && \ - apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server + apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server libaio1 # Install postgres client for pg_dump utils -RUN apt install software-properties-common apt-transport-https gpg -y \ +RUN apt install -y software-properties-common apt-transport-https ca-certificates gnupg \ && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ && apt update -y \ @@ -61,10 +62,8 @@ RUN apt install software-properties-common apt-transport-https gpg -y \ # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx WORKDIR /nodejs -RUN curl -sL https://deb.nodesource.com/setup_18.x -o /tmp/nodesource_setup.sh && \ - bash /tmp/nodesource_setup.sh && \ - apt-get install -y --no-install-recommends libaio1 nodejs && \ - npm install --global yarn pm2 +COPY scripts/install-node.sh ./install.sh +RUN chmod +x install.sh && ./install.sh # setup nginx COPY hosting/single/nginx/nginx.conf /etc/nginx diff --git a/scripts/install-minio.sh b/scripts/install-minio.sh index 8297593599..b1e0d9ee80 100755 --- a/scripts/install-minio.sh +++ b/scripts/install-minio.sh @@ -2,9 +2,9 @@ if [[ $TARGETARCH == arm* ]] ; then echo "INSTALLING ARM64 MINIO" - wget https://dl.min.io/server/minio/release/linux-arm64/minio + wget wget https://dl.min.io/server/minio/release/linux-arm64/archive/minio.deb -O minio.deb else echo "INSTALLING AMD64 MINIO" - wget https://dl.min.io/server/minio/release/linux-amd64/minio + wget wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio.deb -O minio.deb fi -chmod +x minio +dpkg -i minio.deb diff --git a/scripts/install-node.sh b/scripts/install-node.sh new file mode 100644 index 0000000000..562bdf2cd3 --- /dev/null +++ b/scripts/install-node.sh @@ -0,0 +1,8 @@ +#!/bin/bash +apt-get install -y gnupg +curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | tee /usr/share/keyrings/nodesource.gpg > /dev/null +echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list +apt-get update +echo "INSTALLING NODE $NODE_MAJOR" +apt-get install -y --no-install-recommends nodejs +npm install --global yarn pm2 \ No newline at end of file From cf40b87d81f7161f2294c6f00c1768ec8e7aa49f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 1 Nov 2023 15:10:32 +0000 Subject: [PATCH 07/12] Use installed MinIO rather than binary. --- hosting/single/runner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 9dc7aa25d8..770b23eec1 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -77,7 +77,7 @@ mkdir -p ${DATA_DIR}/minio chown -R couchdb:couchdb ${DATA_DIR}/couch redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & /bbcouch-runner.sh & -/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & +minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & /etc/init.d/nginx restart if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then # Add monthly cron job to renew certbot certificate From 85325814d06ab47f3fb8fba6bba3717f66c779de Mon Sep 17 00:00:00 2001 From: Mitch-Budibase Date: Wed, 1 Nov 2023 15:18:20 +0000 Subject: [PATCH 08/12] Moving stripe import to top of test file --- qa-core/src/account-api/tests/licensing/license.manage.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts index a9109dad3a..9cad980038 100644 --- a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts +++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts @@ -1,6 +1,7 @@ import TestConfiguration from "../../config/TestConfiguration" import * as fixtures from "../../fixtures" import { Hosting, PlanType } from "@budibase/types" +const stripe = require("stripe")(process.env.STRIPE_SECRET_KEY) describe("license management", () => { const config = new TestConfiguration() @@ -53,7 +54,6 @@ describe("license management", () => { expect(checkoutSessionUrl).toContain("checkout.stripe.com") // Create stripe customer - const stripe = require("stripe")(process.env.STRIPE_SECRET_KEY) const customer = await stripe.customers.create({ email: createAccountRequest.email, }) From d6a4b71655991f27ee45986f0339bd1e33d8dc2b Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 1 Nov 2023 15:37:08 +0000 Subject: [PATCH 09/12] Bump version to 2.12.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index cb92b3ba0d..dc6ffb2da2 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.12.2", + "version": "2.12.3", "npmClient": "yarn", "packages": [ "packages/*" From 586d539109f38d9a48b8fe23b0d4ea9084d2320d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 1 Nov 2023 16:23:51 +0000 Subject: [PATCH 10/12] Adding release information to single image. --- .github/workflows/release-singleimage.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml index 4d35916f4d..a3444d5e7a 100644 --- a/.github/workflows/release-singleimage.yml +++ b/.github/workflows/release-singleimage.yml @@ -66,14 +66,21 @@ jobs: context: . push: true platforms: linux/amd64,linux/arm64 + build-args: BUDIBASE_VERSION=$BUDIBASE_VERSION tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }} file: ./hosting/single/Dockerfile.v2 + env: + BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} - name: Tag and release Budibase Azure App Service docker image uses: docker/build-push-action@v2 with: context: . push: true platforms: linux/amd64 - build-args: TARGETBUILD=aas + build-args: | + TARGETBUILD=aas + BUDIBASE_VERSION=$BUDIBASE_VERSION tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }} file: ./hosting/single/Dockerfile.v2 + env: + BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} From 0477ed64f6edd6b2a2e02a25fb1baaff591584fb Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 1 Nov 2023 16:40:29 +0000 Subject: [PATCH 11/12] Bump version to 2.12.4 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index dc6ffb2da2..f0f51242d1 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.12.3", + "version": "2.12.4", "npmClient": "yarn", "packages": [ "packages/*" From 17b6985f06aa5faf256c3f5f45b76a1a55278414 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 1 Nov 2023 17:48:13 +0000 Subject: [PATCH 12/12] Adding a duration utility for changing between time types. --- packages/backend-core/src/index.ts | 1 + packages/backend-core/src/queue/queue.ts | 7 +-- packages/backend-core/src/utils/Duration.ts | 49 +++++++++++++++++++ packages/backend-core/src/utils/index.ts | 1 + .../src/utils/tests/Duration.spec.ts | 19 +++++++ 5 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 packages/backend-core/src/utils/Duration.ts create mode 100644 packages/backend-core/src/utils/tests/Duration.spec.ts diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index ffffd8240a..c7cf9f56cc 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -30,6 +30,7 @@ export * as timers from "./timers" export { default as env } from "./environment" export * as blacklist from "./blacklist" export * as docUpdates from "./docUpdates" +export * from "./utils/Duration" export { SearchParams } from "./db" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index c6b02921bf..c0d1861de3 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -4,15 +4,16 @@ import { JobQueue } from "./constants" import InMemoryQueue from "./inMemoryQueue" import BullQueue, { QueueOptions } from "bull" import { addListeners, StalledFn } from "./listeners" +import { Duration } from "../utils" import * as timers from "../timers" import * as Redis from "ioredis" // the queue lock is held for 5 minutes -const QUEUE_LOCK_MS = 300000 +const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs() // queue lock is refreshed every 30 seconds -const QUEUE_LOCK_RENEW_INTERNAL_MS = 30000 +const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs() // cleanup the queue every 60 seconds -const CLEANUP_PERIOD_MS = 60 * 1000 +const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs() let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = [] let cleanupInterval: NodeJS.Timeout diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts new file mode 100644 index 0000000000..f376c2f7c7 --- /dev/null +++ b/packages/backend-core/src/utils/Duration.ts @@ -0,0 +1,49 @@ +export enum DurationType { + MILLISECONDS = "milliseconds", + SECONDS = "seconds", + MINUTES = "minutes", + HOURS = "hours", + DAYS = "days", +} + +const conversion: Record = { + milliseconds: 1, + seconds: 1000, + minutes: 60 * 1000, + hours: 60 * 60 * 1000, + days: 24 * 60 * 60 * 1000, +} + +export class Duration { + static convert(from: DurationType, to: DurationType, duration: number) { + const milliseconds = duration * conversion[from] + return milliseconds / conversion[to] + } + + static from(from: DurationType, duration: number) { + return { + to: (to: DurationType) => { + return Duration.convert(from, to, duration) + }, + toMs: () => { + return Duration.convert(from, DurationType.MILLISECONDS, duration) + }, + } + } + + static fromSeconds(duration: number) { + return Duration.from(DurationType.SECONDS, duration) + } + + static fromMinutes(duration: number) { + return Duration.from(DurationType.MINUTES, duration) + } + + static fromHours(duration: number) { + return Duration.from(DurationType.HOURS, duration) + } + + static fromDays(duration: number) { + return Duration.from(DurationType.DAYS, duration) + } +} diff --git a/packages/backend-core/src/utils/index.ts b/packages/backend-core/src/utils/index.ts index 318a7f13ba..ac17227459 100644 --- a/packages/backend-core/src/utils/index.ts +++ b/packages/backend-core/src/utils/index.ts @@ -1,3 +1,4 @@ export * from "./hashing" export * from "./utils" export * from "./stringUtils" +export * from "./Duration" diff --git a/packages/backend-core/src/utils/tests/Duration.spec.ts b/packages/backend-core/src/utils/tests/Duration.spec.ts new file mode 100644 index 0000000000..46b996f788 --- /dev/null +++ b/packages/backend-core/src/utils/tests/Duration.spec.ts @@ -0,0 +1,19 @@ +import { Duration, DurationType } from "../Duration" + +describe("duration", () => { + it("should convert minutes to milliseconds", () => { + expect(Duration.fromMinutes(5).toMs()).toBe(300000) + }) + + it("should convert seconds to milliseconds", () => { + expect(Duration.fromSeconds(30).toMs()).toBe(30000) + }) + + it("should convert days to milliseconds", () => { + expect(Duration.fromDays(1).toMs()).toBe(86400000) + }) + + it("should convert minutes to days", () => { + expect(Duration.fromMinutes(1440).to(DurationType.DAYS)).toBe(1) + }) +})