diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml index 4d35916f4d..a3444d5e7a 100644 --- a/.github/workflows/release-singleimage.yml +++ b/.github/workflows/release-singleimage.yml @@ -66,14 +66,21 @@ jobs: context: . push: true platforms: linux/amd64,linux/arm64 + build-args: BUDIBASE_VERSION=$BUDIBASE_VERSION tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }} file: ./hosting/single/Dockerfile.v2 + env: + BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} - name: Tag and release Budibase Azure App Service docker image uses: docker/build-push-action@v2 with: context: . push: true platforms: linux/amd64 - build-args: TARGETBUILD=aas + build-args: | + TARGETBUILD=aas + BUDIBASE_VERSION=$BUDIBASE_VERSION tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }} file: ./hosting/single/Dockerfile.v2 + env: + BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} diff --git a/hosting/scripts/install-minio.sh b/hosting/scripts/install-minio.sh deleted file mode 100755 index 8297593599..0000000000 --- a/hosting/scripts/install-minio.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -if [[ $TARGETARCH == arm* ]] ; -then - echo "INSTALLING ARM64 MINIO" - wget https://dl.min.io/server/minio/release/linux-arm64/minio -else - echo "INSTALLING AMD64 MINIO" - wget https://dl.min.io/server/minio/release/linux-amd64/minio -fi -chmod +x minio diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2 index 5b07a51b27..ec03a1b5a2 100644 --- a/hosting/single/Dockerfile.v2 +++ b/hosting/single/Dockerfile.v2 @@ -42,6 +42,7 @@ COPY packages/string-templates packages/string-templates FROM budibase/couchdb as runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH +ENV NODE_MAJOR 18 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) # e.g. docker build --build-arg TARGETBUILD=aas .... ARG TARGETBUILD=single @@ -49,10 +50,10 @@ ENV TARGETBUILD $TARGETBUILD # install base dependencies RUN apt-get update && \ - apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server + apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server libaio1 # Install postgres client for pg_dump utils -RUN apt install software-properties-common apt-transport-https gpg -y \ +RUN apt install -y software-properties-common apt-transport-https ca-certificates gnupg \ && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ && apt update -y \ @@ -61,10 +62,8 @@ RUN apt install software-properties-common apt-transport-https gpg -y \ # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx WORKDIR /nodejs -RUN curl -sL https://deb.nodesource.com/setup_18.x -o /tmp/nodesource_setup.sh && \ - bash /tmp/nodesource_setup.sh && \ - apt-get install -y --no-install-recommends libaio1 nodejs && \ - npm install --global yarn pm2 +COPY scripts/install-node.sh ./install.sh +RUN chmod +x install.sh && ./install.sh # setup nginx COPY hosting/single/nginx/nginx.conf /etc/nginx diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 9dc7aa25d8..770b23eec1 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -77,7 +77,7 @@ mkdir -p ${DATA_DIR}/minio chown -R couchdb:couchdb ${DATA_DIR}/couch redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & /bbcouch-runner.sh & -/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & +minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & /etc/init.d/nginx restart if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then # Add monthly cron job to renew certbot certificate diff --git a/lerna.json b/lerna.json index cb92b3ba0d..f0f51242d1 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.12.2", + "version": "2.12.4", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index ffffd8240a..c7cf9f56cc 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -30,6 +30,7 @@ export * as timers from "./timers" export { default as env } from "./environment" export * as blacklist from "./blacklist" export * as docUpdates from "./docUpdates" +export * from "./utils/Duration" export { SearchParams } from "./db" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index af2ec6dbaa..a8add7ecb6 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -36,7 +36,7 @@ class InMemoryQueue { * @param opts This is not used by the in memory queue as there is no real use * case when in memory, but is the same API as Bull */ - constructor(name: string, opts = null) { + constructor(name: string, opts?: any) { this._name = name this._opts = opts this._messages = [] diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 0658147709..c0d1861de3 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -2,11 +2,18 @@ import env from "../environment" import { getRedisOptions } from "../redis/utils" import { JobQueue } from "./constants" import InMemoryQueue from "./inMemoryQueue" -import BullQueue from "bull" +import BullQueue, { QueueOptions } from "bull" import { addListeners, StalledFn } from "./listeners" +import { Duration } from "../utils" import * as timers from "../timers" +import * as Redis from "ioredis" -const CLEANUP_PERIOD_MS = 60 * 1000 +// the queue lock is held for 5 minutes +const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs() +// queue lock is refreshed every 30 seconds +const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs() +// cleanup the queue every 60 seconds +const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs() let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = [] let cleanupInterval: NodeJS.Timeout @@ -21,7 +28,14 @@ export function createQueue( opts: { removeStalledCb?: StalledFn } = {} ): BullQueue.Queue { const { opts: redisOpts, redisProtocolUrl } = getRedisOptions() - const queueConfig: any = redisProtocolUrl || { redis: redisOpts } + const queueConfig: QueueOptions = { + redis: redisProtocolUrl! || (redisOpts as Redis.RedisOptions), + settings: { + maxStalledCount: 0, + lockDuration: QUEUE_LOCK_MS, + lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS, + }, + } let queue: any if (!env.isTest()) { queue = new BullQueue(jobQueue, queueConfig) diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts new file mode 100644 index 0000000000..f376c2f7c7 --- /dev/null +++ b/packages/backend-core/src/utils/Duration.ts @@ -0,0 +1,49 @@ +export enum DurationType { + MILLISECONDS = "milliseconds", + SECONDS = "seconds", + MINUTES = "minutes", + HOURS = "hours", + DAYS = "days", +} + +const conversion: Record = { + milliseconds: 1, + seconds: 1000, + minutes: 60 * 1000, + hours: 60 * 60 * 1000, + days: 24 * 60 * 60 * 1000, +} + +export class Duration { + static convert(from: DurationType, to: DurationType, duration: number) { + const milliseconds = duration * conversion[from] + return milliseconds / conversion[to] + } + + static from(from: DurationType, duration: number) { + return { + to: (to: DurationType) => { + return Duration.convert(from, to, duration) + }, + toMs: () => { + return Duration.convert(from, DurationType.MILLISECONDS, duration) + }, + } + } + + static fromSeconds(duration: number) { + return Duration.from(DurationType.SECONDS, duration) + } + + static fromMinutes(duration: number) { + return Duration.from(DurationType.MINUTES, duration) + } + + static fromHours(duration: number) { + return Duration.from(DurationType.HOURS, duration) + } + + static fromDays(duration: number) { + return Duration.from(DurationType.DAYS, duration) + } +} diff --git a/packages/backend-core/src/utils/index.ts b/packages/backend-core/src/utils/index.ts index 318a7f13ba..ac17227459 100644 --- a/packages/backend-core/src/utils/index.ts +++ b/packages/backend-core/src/utils/index.ts @@ -1,3 +1,4 @@ export * from "./hashing" export * from "./utils" export * from "./stringUtils" +export * from "./Duration" diff --git a/packages/backend-core/src/utils/tests/Duration.spec.ts b/packages/backend-core/src/utils/tests/Duration.spec.ts new file mode 100644 index 0000000000..46b996f788 --- /dev/null +++ b/packages/backend-core/src/utils/tests/Duration.spec.ts @@ -0,0 +1,19 @@ +import { Duration, DurationType } from "../Duration" + +describe("duration", () => { + it("should convert minutes to milliseconds", () => { + expect(Duration.fromMinutes(5).toMs()).toBe(300000) + }) + + it("should convert seconds to milliseconds", () => { + expect(Duration.fromSeconds(30).toMs()).toBe(30000) + }) + + it("should convert days to milliseconds", () => { + expect(Duration.fromDays(1).toMs()).toBe(86400000) + }) + + it("should convert minutes to days", () => { + expect(Duration.fromMinutes(1440).to(DurationType.DAYS)).toBe(1) + }) +}) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index c8ef406472..eef1e50b7c 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -5467,17 +5467,17 @@ }, "settings": [ { - "type": "select", + "type": "table", + "label": "Data", + "key": "dataSource" + }, + { + "type": "radio", "label": "Type", "key": "actionType", "options": ["Create", "Update", "View"], "defaultValue": "Create" }, - { - "type": "table", - "label": "Data", - "key": "dataSource" - }, { "type": "text", "label": "Title", @@ -5508,13 +5508,37 @@ }, { "type": "text", - "label": "Empty text", + "label": "No rows found", "key": "noRowsMessage", "defaultValue": "We couldn't find a row to display", "nested": true } ] }, + { + "section": true, + "name": "Fields", + "settings": [ + { + "type": "fieldConfiguration", + "key": "fields", + "nested": true, + "resetOn": "dataSource", + "selectAllFields": true + }, + { + "type": "boolean", + "label": "Disabled", + "key": "disabled", + "defaultValue": false, + "dependsOn": { + "setting": "actionType", + "value": "View", + "invert": true + } + } + ] + }, { "section": true, "name": "Buttons", @@ -5566,30 +5590,6 @@ } ] }, - { - "section": true, - "name": "Fields", - "settings": [ - { - "type": "fieldConfiguration", - "key": "fields", - "nested": true, - "resetOn": "dataSource", - "selectAllFields": true - }, - { - "type": "boolean", - "label": "Disabled", - "key": "disabled", - "defaultValue": false, - "dependsOn": { - "setting": "actionType", - "value": "View", - "invert": true - } - } - ] - }, { "tag": "style", "type": "select", @@ -5924,4 +5924,4 @@ } ] } -} \ No newline at end of file +} diff --git a/qa-core/package.json b/qa-core/package.json index d266ca9def..cfccd5e650 100644 --- a/qa-core/package.json +++ b/qa-core/package.json @@ -17,7 +17,7 @@ "test:notify": "node scripts/testResultsWebhook", "test:cloud:prod": "yarn run test --testPathIgnorePatterns=\\.integration\\.", "test:cloud:qa": "yarn run test", - "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.license\\.", + "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.licensing\\.", "serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci", "serve": "start-server-and-test dev:built http://localhost:4001/health", "dev:built": "cd ../ && yarn dev:built" diff --git a/qa-core/src/account-api/api/apis/LicenseAPI.ts b/qa-core/src/account-api/api/apis/LicenseAPI.ts index b371f00f05..a9b0a35269 100644 --- a/qa-core/src/account-api/api/apis/LicenseAPI.ts +++ b/qa-core/src/account-api/api/apis/LicenseAPI.ts @@ -99,9 +99,11 @@ export default class LicenseAPI extends BaseAPI { }, opts) } - async updatePlan(opts: APIRequestOpts = { status: 200 }) { + async updatePlan(priceId: string, opts: APIRequestOpts = { status: 200 }) { return this.doRequest(() => { - return this.client.put(`/api/license/plan`) + return this.client.put(`/api/license/plan`, { + body: { priceId }, + }) }, opts) } diff --git a/qa-core/src/account-api/api/apis/StripeAPI.ts b/qa-core/src/account-api/api/apis/StripeAPI.ts index c9c776e89b..5a4e810655 100644 --- a/qa-core/src/account-api/api/apis/StripeAPI.ts +++ b/qa-core/src/account-api/api/apis/StripeAPI.ts @@ -38,9 +38,19 @@ export default class StripeAPI extends BaseAPI { }, opts) } - async linkStripeCustomer(opts: APIRequestOpts = { status: 200 }) { + async linkStripeCustomer( + accountId: string, + stripeCustomerId: string, + opts: APIRequestOpts = { status: 200 } + ) { return this.doRequest(() => { - return this.client.post(`/api/stripe/link`) + return this.client.post(`/api/stripe/link`, { + body: { + accountId, + stripeCustomerId, + }, + internal: true, + }) }, opts) } diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts new file mode 100644 index 0000000000..9cad980038 --- /dev/null +++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts @@ -0,0 +1,114 @@ +import TestConfiguration from "../../config/TestConfiguration" +import * as fixtures from "../../fixtures" +import { Hosting, PlanType } from "@budibase/types" +const stripe = require("stripe")(process.env.STRIPE_SECRET_KEY) + +describe("license management", () => { + const config = new TestConfiguration() + + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + + it("retrieves plans, creates checkout session, and updates license", async () => { + // Create cloud account + const createAccountRequest = fixtures.accounts.generateAccount({ + hosting: Hosting.CLOUD, + }) + const [createAccountRes, account] = + await config.accountsApi.accounts.create(createAccountRequest, { + autoVerify: true, + }) + + // Self response has free license + await config.doInNewState(async () => { + await config.loginAsAccount(createAccountRequest) + const [selfRes, selfBody] = await config.api.accounts.self() + expect(selfBody.license.plan.type).toBe(PlanType.FREE) + }) + + // Retrieve plans + const [plansRes, planBody] = await config.api.licenses.getPlans() + + // Select priceId from premium plan + let premiumPriceId = null + let businessPriceId = "" + for (const plan of planBody) { + if (plan.type === PlanType.PREMIUM) { + premiumPriceId = plan.prices[0].priceId + } + if (plan.type === PlanType.BUSINESS) { + businessPriceId = plan.prices[0].priceId + } + } + + // Create checkout session for price + const checkoutSessionRes = await config.api.stripe.createCheckoutSession( + premiumPriceId + ) + const checkoutSessionUrl = checkoutSessionRes[1].url + expect(checkoutSessionUrl).toContain("checkout.stripe.com") + + // Create stripe customer + const customer = await stripe.customers.create({ + email: createAccountRequest.email, + }) + + // Create payment method + const paymentMethod = await stripe.paymentMethods.create({ + type: "card", + card: { + token: "tok_visa", // Test Visa Card + }, + }) + + // Attach payment method to customer + await stripe.paymentMethods.attach(paymentMethod.id, { + customer: customer.id, + }) + + // Update customer + await stripe.customers.update(customer.id, { + invoice_settings: { + default_payment_method: paymentMethod.id, + }, + }) + + // Create subscription for premium plan + const subscription = await stripe.subscriptions.create({ + customer: customer.id, + items: [ + { + price: premiumPriceId, + quantity: 1, + }, + ], + default_payment_method: paymentMethod.id, + collection_method: "charge_automatically", + }) + + await config.doInNewState(async () => { + // License updated from Free to Premium + await config.loginAsAccount(createAccountRequest) + await config.api.stripe.linkStripeCustomer(account.accountId, customer.id) + const [_, selfBodyPremium] = await config.api.accounts.self() + expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM) + + // Create portal session - Check URL + const [portalRes, portalSessionBody] = + await config.api.stripe.createPortalSession(customer.id) + expect(portalSessionBody.url).toContain("billing.stripe.com") + + // Update subscription from premium to business license + await config.api.licenses.updatePlan(businessPriceId) + + // License updated to Business + const [selfRes, selfBodyBusiness] = await config.api.accounts.self() + expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS) + }) + }) +}) diff --git a/qa-core/src/environment.ts b/qa-core/src/environment.ts index 0257b10831..a805503474 100644 --- a/qa-core/src/environment.ts +++ b/qa-core/src/environment.ts @@ -28,6 +28,7 @@ const env = { MARIADB_DB: process.env.MARIADB_DB, MARIADB_USER: process.env.MARIADB_USER, MARIADB_PASSWORD: process.env.MARIADB_PASSWORD, + STRIPE_SECRET_KEY: process.env.STRIPE_SECRET_KEY, } export = env diff --git a/scripts/install-minio.sh b/scripts/install-minio.sh index 8297593599..b1e0d9ee80 100755 --- a/scripts/install-minio.sh +++ b/scripts/install-minio.sh @@ -2,9 +2,9 @@ if [[ $TARGETARCH == arm* ]] ; then echo "INSTALLING ARM64 MINIO" - wget https://dl.min.io/server/minio/release/linux-arm64/minio + wget wget https://dl.min.io/server/minio/release/linux-arm64/archive/minio.deb -O minio.deb else echo "INSTALLING AMD64 MINIO" - wget https://dl.min.io/server/minio/release/linux-amd64/minio + wget wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio.deb -O minio.deb fi -chmod +x minio +dpkg -i minio.deb diff --git a/scripts/install-node.sh b/scripts/install-node.sh new file mode 100644 index 0000000000..562bdf2cd3 --- /dev/null +++ b/scripts/install-node.sh @@ -0,0 +1,8 @@ +#!/bin/bash +apt-get install -y gnupg +curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | tee /usr/share/keyrings/nodesource.gpg > /dev/null +echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list +apt-get update +echo "INSTALLING NODE $NODE_MAJOR" +apt-get install -y --no-install-recommends nodejs +npm install --global yarn pm2 \ No newline at end of file