Merge branch 'master' into fix/relationship-issue-one-to-many

.github/workflows/release-singleimage.yml

@@ -66,14 +66,21 @@ jobs:
           context: .
           push: true
           platforms: linux/amd64,linux/arm64
+          build-args: BUDIBASE_VERSION=$BUDIBASE_VERSION
           tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }}
           file: ./hosting/single/Dockerfile.v2
+        env:
+          BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }}
       - name: Tag and release Budibase Azure App Service docker image
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
           platforms: linux/amd64
-          build-args: TARGETBUILD=aas
+          build-args: |
+            TARGETBUILD=aas
+            BUDIBASE_VERSION=$BUDIBASE_VERSION
           tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }}
           file: ./hosting/single/Dockerfile.v2
+        env:
+          BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }}

hosting/scripts/install-minio.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-if [[ $TARGETARCH == arm* ]] ;
-then
-  echo "INSTALLING ARM64 MINIO"
-  wget https://dl.min.io/server/minio/release/linux-arm64/minio
-else
-  echo "INSTALLING AMD64 MINIO"
-  wget https://dl.min.io/server/minio/release/linux-amd64/minio
-fi
-chmod +x minio

hosting/single/Dockerfile.v2

@@ -42,6 +42,7 @@ COPY packages/string-templates packages/string-templates
 FROM budibase/couchdb as runner
 ARG TARGETARCH
 ENV TARGETARCH $TARGETARCH
+ENV NODE_MAJOR 18
 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
 # e.g. docker build --build-arg TARGETBUILD=aas ....
 ARG TARGETBUILD=single
@@ -49,10 +50,10 @@ ENV TARGETBUILD $TARGETBUILD
 
 # install base dependencies
 RUN apt-get update && \
-  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server
+  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server libaio1
 
 # Install postgres client for pg_dump utils
-RUN apt install software-properties-common apt-transport-https gpg -y \
+RUN apt install -y software-properties-common apt-transport-https ca-certificates gnupg \
     && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \
     && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \
     && apt update -y \
@@ -61,10 +62,8 @@ RUN apt install software-properties-common apt-transport-https gpg -y \
 
 # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx
 WORKDIR /nodejs
-RUN curl -sL https://deb.nodesource.com/setup_18.x -o /tmp/nodesource_setup.sh && \
-  bash /tmp/nodesource_setup.sh && \
-  apt-get install -y --no-install-recommends libaio1 nodejs && \
-  npm install --global yarn pm2
+COPY scripts/install-node.sh ./install.sh
+RUN chmod +x install.sh && ./install.sh
 
 # setup nginx
 COPY hosting/single/nginx/nginx.conf /etc/nginx

hosting/single/runner.sh

@@ -77,7 +77,7 @@ mkdir -p ${DATA_DIR}/minio
 chown -R couchdb:couchdb ${DATA_DIR}/couch
 redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
 /bbcouch-runner.sh &
-/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
+minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
 /etc/init.d/nginx restart
 if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
     # Add monthly cron job to renew certbot certificate

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.12.2",
+  "version": "2.12.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/src/index.ts

@@ -30,6 +30,7 @@ export * as timers from "./timers"
 export { default as env } from "./environment"
 export * as blacklist from "./blacklist"
 export * as docUpdates from "./docUpdates"
+export * from "./utils/Duration"
 export { SearchParams } from "./db"
 // Add context to tenancy for backwards compatibility
 // only do this for external usages to prevent internal

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -36,7 +36,7 @@ class InMemoryQueue {
    * @param opts This is not used by the in memory queue as there is no real use
    * case when in memory, but is the same API as Bull
    */
-  constructor(name: string, opts = null) {
+  constructor(name: string, opts?: any) {
     this._name = name
     this._opts = opts
     this._messages = []

packages/backend-core/src/queue/queue.ts

@@ -2,11 +2,18 @@ import env from "../environment"
 import { getRedisOptions } from "../redis/utils"
 import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
-import BullQueue from "bull"
+import BullQueue, { QueueOptions } from "bull"
 import { addListeners, StalledFn } from "./listeners"
+import { Duration } from "../utils"
 import * as timers from "../timers"
+import * as Redis from "ioredis"
 
-const CLEANUP_PERIOD_MS = 60 * 1000
+// the queue lock is held for 5 minutes
+const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
+// queue lock is refreshed every 30 seconds
+const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs()
+// cleanup the queue every 60 seconds
+const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs()
 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
 let cleanupInterval: NodeJS.Timeout
 
@@ -21,7 +28,14 @@ export function createQueue<T>(
   opts: { removeStalledCb?: StalledFn } = {}
 ): BullQueue.Queue<T> {
   const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
-  const queueConfig: any = redisProtocolUrl || { redis: redisOpts }
+  const queueConfig: QueueOptions = {
+    redis: redisProtocolUrl! || (redisOpts as Redis.RedisOptions),
+    settings: {
+      maxStalledCount: 0,
+      lockDuration: QUEUE_LOCK_MS,
+      lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
+    },
+  }
   let queue: any
   if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)

packages/backend-core/src/utils/Duration.ts

@@ -0,0 +1,49 @@
+export enum DurationType {
+  MILLISECONDS = "milliseconds",
+  SECONDS = "seconds",
+  MINUTES = "minutes",
+  HOURS = "hours",
+  DAYS = "days",
+}
+
+const conversion: Record<DurationType, number> = {
+  milliseconds: 1,
+  seconds: 1000,
+  minutes: 60 * 1000,
+  hours: 60 * 60 * 1000,
+  days: 24 * 60 * 60 * 1000,
+}
+
+export class Duration {
+  static convert(from: DurationType, to: DurationType, duration: number) {
+    const milliseconds = duration * conversion[from]
+    return milliseconds / conversion[to]
+  }
+
+  static from(from: DurationType, duration: number) {
+    return {
+      to: (to: DurationType) => {
+        return Duration.convert(from, to, duration)
+      },
+      toMs: () => {
+        return Duration.convert(from, DurationType.MILLISECONDS, duration)
+      },
+    }
+  }
+
+  static fromSeconds(duration: number) {
+    return Duration.from(DurationType.SECONDS, duration)
+  }
+
+  static fromMinutes(duration: number) {
+    return Duration.from(DurationType.MINUTES, duration)
+  }
+
+  static fromHours(duration: number) {
+    return Duration.from(DurationType.HOURS, duration)
+  }
+
+  static fromDays(duration: number) {
+    return Duration.from(DurationType.DAYS, duration)
+  }
+}

packages/backend-core/src/utils/index.ts

@@ -1,3 +1,4 @@
 export * from "./hashing"
 export * from "./utils"
 export * from "./stringUtils"
+export * from "./Duration"

packages/backend-core/src/utils/tests/Duration.spec.ts

@@ -0,0 +1,19 @@
+import { Duration, DurationType } from "../Duration"
+
+describe("duration", () => {
+  it("should convert minutes to milliseconds", () => {
+    expect(Duration.fromMinutes(5).toMs()).toBe(300000)
+  })
+
+  it("should convert seconds to milliseconds", () => {
+    expect(Duration.fromSeconds(30).toMs()).toBe(30000)
+  })
+
+  it("should convert days to milliseconds", () => {
+    expect(Duration.fromDays(1).toMs()).toBe(86400000)
+  })
+
+  it("should convert minutes to days", () => {
+    expect(Duration.fromMinutes(1440).to(DurationType.DAYS)).toBe(1)
+  })
+})

packages/client/manifest.json

@@ -5467,17 +5467,17 @@
     },
     "settings": [
       {
-        "type": "select",
+        "type": "table",
+        "label": "Data",
+        "key": "dataSource"
+      },
+      {
+        "type": "radio",
         "label": "Type",
         "key": "actionType",
         "options": ["Create", "Update", "View"],
         "defaultValue": "Create"
       },
-      {
-        "type": "table",
-        "label": "Data",
-        "key": "dataSource"
-      },
       {
         "type": "text",
         "label": "Title",
@@ -5508,13 +5508,37 @@
           },
           {
             "type": "text",
-            "label": "Empty text",
+            "label": "No rows found",
             "key": "noRowsMessage",
             "defaultValue": "We couldn't find a row to display",
             "nested": true
           }
         ]
       },
+      {
+        "section": true,
+        "name": "Fields",
+        "settings": [
+          {
+            "type": "fieldConfiguration",
+            "key": "fields",
+            "nested": true,
+            "resetOn": "dataSource",
+            "selectAllFields": true
+          },
+          {
+            "type": "boolean",
+            "label": "Disabled",
+            "key": "disabled",
+            "defaultValue": false,
+            "dependsOn": {
+              "setting": "actionType",
+              "value": "View",
+              "invert": true
+            }
+          }
+        ]
+      },
       {
         "section": true,
         "name": "Buttons",
@@ -5566,30 +5590,6 @@
           }
         ]
       },
-      {
-        "section": true,
-        "name": "Fields",
-        "settings": [
-          {
-            "type": "fieldConfiguration",
-            "key": "fields",
-            "nested": true,
-            "resetOn": "dataSource",
-            "selectAllFields": true
-          },
-          {
-            "type": "boolean",
-            "label": "Disabled",
-            "key": "disabled",
-            "defaultValue": false,
-            "dependsOn": {
-              "setting": "actionType",
-              "value": "View",
-              "invert": true
-            }
-          }
-        ]
-      },
       {
         "tag": "style",
         "type": "select",
@@ -5924,4 +5924,4 @@
       }
     ]
   }
-}
+}

qa-core/package.json

@@ -17,7 +17,7 @@
     "test:notify": "node scripts/testResultsWebhook",
     "test:cloud:prod": "yarn run test --testPathIgnorePatterns=\\.integration\\.",
     "test:cloud:qa": "yarn run test",
-    "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.license\\.",
+    "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.licensing\\.",
     "serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci",
     "serve": "start-server-and-test dev:built http://localhost:4001/health",
     "dev:built": "cd ../ && yarn dev:built"

qa-core/src/account-api/api/apis/LicenseAPI.ts

@@ -99,9 +99,11 @@ export default class LicenseAPI extends BaseAPI {
     }, opts)
   }
 
-  async updatePlan(opts: APIRequestOpts = { status: 200 }) {
+  async updatePlan(priceId: string, opts: APIRequestOpts = { status: 200 }) {
     return this.doRequest(() => {
-      return this.client.put(`/api/license/plan`)
+      return this.client.put(`/api/license/plan`, {
+        body: { priceId },
+      })
     }, opts)
   }
 

qa-core/src/account-api/api/apis/StripeAPI.ts

@@ -38,9 +38,19 @@ export default class StripeAPI extends BaseAPI {
     }, opts)
   }
 
-  async linkStripeCustomer(opts: APIRequestOpts = { status: 200 }) {
+  async linkStripeCustomer(
+    accountId: string,
+    stripeCustomerId: string,
+    opts: APIRequestOpts = { status: 200 }
+  ) {
     return this.doRequest(() => {
-      return this.client.post(`/api/stripe/link`)
+      return this.client.post(`/api/stripe/link`, {
+        body: {
+          accountId,
+          stripeCustomerId,
+        },
+        internal: true,
+      })
     }, opts)
   }
 

qa-core/src/account-api/tests/licensing/license.manage.spec.ts

@@ -0,0 +1,114 @@
+import TestConfiguration from "../../config/TestConfiguration"
+import * as fixtures from "../../fixtures"
+import { Hosting, PlanType } from "@budibase/types"
+const stripe = require("stripe")(process.env.STRIPE_SECRET_KEY)
+
+describe("license management", () => {
+  const config = new TestConfiguration()
+
+  beforeAll(async () => {
+    await config.beforeAll()
+  })
+
+  afterAll(async () => {
+    await config.afterAll()
+  })
+
+  it("retrieves plans, creates checkout session, and updates license", async () => {
+    // Create cloud account
+    const createAccountRequest = fixtures.accounts.generateAccount({
+      hosting: Hosting.CLOUD,
+    })
+    const [createAccountRes, account] =
+      await config.accountsApi.accounts.create(createAccountRequest, {
+        autoVerify: true,
+      })
+
+    // Self response has free license
+    await config.doInNewState(async () => {
+      await config.loginAsAccount(createAccountRequest)
+      const [selfRes, selfBody] = await config.api.accounts.self()
+      expect(selfBody.license.plan.type).toBe(PlanType.FREE)
+    })
+
+    // Retrieve plans
+    const [plansRes, planBody] = await config.api.licenses.getPlans()
+
+    // Select priceId from premium plan
+    let premiumPriceId = null
+    let businessPriceId = ""
+    for (const plan of planBody) {
+      if (plan.type === PlanType.PREMIUM) {
+        premiumPriceId = plan.prices[0].priceId
+      }
+      if (plan.type === PlanType.BUSINESS) {
+        businessPriceId = plan.prices[0].priceId
+      }
+    }
+
+    // Create checkout session for price
+    const checkoutSessionRes = await config.api.stripe.createCheckoutSession(
+      premiumPriceId
+    )
+    const checkoutSessionUrl = checkoutSessionRes[1].url
+    expect(checkoutSessionUrl).toContain("checkout.stripe.com")
+
+    // Create stripe customer
+    const customer = await stripe.customers.create({
+      email: createAccountRequest.email,
+    })
+
+    // Create payment method
+    const paymentMethod = await stripe.paymentMethods.create({
+      type: "card",
+      card: {
+        token: "tok_visa", // Test Visa Card
+      },
+    })
+
+    // Attach payment method to customer
+    await stripe.paymentMethods.attach(paymentMethod.id, {
+      customer: customer.id,
+    })
+
+    // Update customer
+    await stripe.customers.update(customer.id, {
+      invoice_settings: {
+        default_payment_method: paymentMethod.id,
+      },
+    })
+
+    // Create subscription for premium plan
+    const subscription = await stripe.subscriptions.create({
+      customer: customer.id,
+      items: [
+        {
+          price: premiumPriceId,
+          quantity: 1,
+        },
+      ],
+      default_payment_method: paymentMethod.id,
+      collection_method: "charge_automatically",
+    })
+
+    await config.doInNewState(async () => {
+      // License updated from Free to Premium
+      await config.loginAsAccount(createAccountRequest)
+      await config.api.stripe.linkStripeCustomer(account.accountId, customer.id)
+      const [_, selfBodyPremium] = await config.api.accounts.self()
+      expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM)
+
+      // Create portal session - Check URL
+      const [portalRes, portalSessionBody] =
+        await config.api.stripe.createPortalSession(customer.id)
+      expect(portalSessionBody.url).toContain("billing.stripe.com")
+
+      // Update subscription from premium to business license
+      await config.api.licenses.updatePlan(businessPriceId)
+
+      // License updated to Business
+      const [selfRes, selfBodyBusiness] = await config.api.accounts.self()
+      expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS)
+    })
+  })
+})

qa-core/src/environment.ts

@@ -28,6 +28,7 @@ const env = {
   MARIADB_DB: process.env.MARIADB_DB,
   MARIADB_USER: process.env.MARIADB_USER,
   MARIADB_PASSWORD: process.env.MARIADB_PASSWORD,
+  STRIPE_SECRET_KEY: process.env.STRIPE_SECRET_KEY,
 }
 
 export = env

scripts/install-minio.sh

@@ -2,9 +2,9 @@
 if [[ $TARGETARCH == arm* ]] ;
 then
   echo "INSTALLING ARM64 MINIO"
-  wget https://dl.min.io/server/minio/release/linux-arm64/minio
+  wget wget https://dl.min.io/server/minio/release/linux-arm64/archive/minio.deb -O minio.deb
 else
   echo "INSTALLING AMD64 MINIO"
-  wget https://dl.min.io/server/minio/release/linux-amd64/minio
+  wget wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio.deb -O minio.deb
 fi
-chmod +x minio
+dpkg -i minio.deb

scripts/install-node.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+apt-get install -y gnupg
+curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | tee /usr/share/keyrings/nodesource.gpg > /dev/null
+echo "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+apt-get update
+echo "INSTALLING NODE $NODE_MAJOR"
+apt-get install -y --no-install-recommends nodejs
+npm install --global yarn pm2