Merge branch 'release' into feature/event-backfill

charts/budibase/Chart.yaml

@@ -11,7 +11,7 @@ sources:
   - https://github.com/Budibase/budibase
   - https://budibase.com
 type: application
-version: 0.2.9
+version: 0.2.10
 appVersion: 1.0.48
 dependencies:
   - name: couchdb

charts/budibase/templates/app-service-deployment.yaml

@@ -80,6 +80,10 @@ spec:
           value: {{ .Values.services.objectStore.url }}
         - name: PORT
           value: {{ .Values.services.apps.port | quote }}
+        {{ if .Values.services.worker.publicApiRateLimitPerSecond }}
+        - name: API_REQ_LIMIT_PER_SEC
+          value: {{ .Values.globals.apps.publicApiRateLimitPerSecond | quote }}
+        {{ end }}
         - name: MULTI_TENANCY
           value: {{ .Values.globals.multiTenancy | quote }}
         - name: LOG_LEVEL
@@ -121,6 +125,12 @@ spec:
 
         image: budibase/apps:{{ .Values.globals.appVersion }}
         imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: {{ .Values.services.apps.port }}
+          initialDelaySeconds: 5
+          periodSeconds: 5
         name: bbapps
         ports:
         - containerPort: {{ .Values.services.apps.port }}

charts/budibase/templates/worker-service-deployment.yaml

@@ -121,6 +121,12 @@ spec:
           value: {{ .Values.globals.google.secret | quote }}
         image: budibase/worker:{{ .Values.globals.appVersion }}
         imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: {{ .Values.services.worker.port }}
+          initialDelaySeconds: 5
+          periodSeconds: 5
         name: bbworker
         ports:
         - containerPort: {{ .Values.services.worker.port }}

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.189",
+  "version": "1.0.195",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/logging.js

@@ -0,0 +1 @@
+module.exports = require("./src/logging")

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.189",
+  "version": "1.0.195",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.ts",
   "types": "dist/src/index.d.ts",

packages/backend-core/src/logging.js

@@ -0,0 +1,16 @@
+const NonErrors = ["AccountError"]
+
+function isSuppressed(e) {
+  return e && e["suppressAlert"]
+}
+
+module.exports.logAlert = (message, e = null) => {
+  if (e && NonErrors.includes(e.name) && isSuppressed(e)) {
+    return
+  }
+  let errorJson = ""
+  if (e) {
+    errorJson = ": " + JSON.stringify(e, Object.getOwnPropertyNames(e))
+  }
+  console.error(`bb-alert: ${message} ${errorJson}`)
+}

packages/backend-core/src/middleware/passport/datasource/google.js

@@ -1,4 +1,5 @@
 const google = require("../google")
+const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 const { Cookies, Configs } = require("../../../constants")
 const { clearCookie, getCookie } = require("../../../utils")
 const { getScopedConfig, getPlatformUrl } = require("../../../db/utils")
@@ -46,19 +47,20 @@ async function postAuth(passport, ctx, next) {
   const platformUrl = await getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
-  const strategy = await google.strategyFactory(
-    config,
-    callbackUrl,
-    (accessToken, refreshToken, profile, done) => {
-      clearCookie(ctx, Cookies.DatasourceAuth)
-      done(null, { refreshToken })
-    }
-  )
-
   const authStateCookie = getCookie(ctx, Cookies.DatasourceAuth)
 
   return passport.authenticate(
-    strategy,
+    new GoogleStrategy(
+      {
+        clientID: config.clientID,
+        clientSecret: config.clientSecret,
+        callbackURL: callbackUrl,
+      },
+      (accessToken, refreshToken, profile, done) => {
+        clearCookie(ctx, Cookies.DatasourceAuth)
+        done(null, { accessToken, refreshToken })
+      }
+    ),
     { successRedirect: "/", failureRedirect: "/error" },
     async (err, tokens) => {
       // update the DB for the datasource with all the user info

packages/backend-core/src/middleware/passport/google.js

@@ -11,8 +11,8 @@ const buildVerifyFn = saveUserFn => {
       profile: profile,
       email: profile._json.email,
       oauth2: {
-        accessToken: accessToken,
+        accessToken,
-        refreshToken: refreshToken,
+        refreshToken,
       },
     }
 

packages/backend-core/src/redis/index.js

@@ -23,7 +23,7 @@ function connectionError(timeout, err) {
   if (CLOSED) {
     return
   }
-  CLIENT.end()
+  CLIENT.disconnect()
   CLOSED = true
   // always clear this on error
   clearTimeout(timeout)

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.189",
+  "version": "1.0.195",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "^1.0.189",
+    "@budibase/string-templates": "^1.0.195",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/builder/cypress/integration/addMultiOptionDatatype.spec.js

@@ -41,7 +41,7 @@ filterTests(['all'], () => {
       }
       // Check items have been selected
       cy.getComponent(componentId)
-        .find(interact.SPECTRUM_Picker_LABEL)
+        .find(interact.SPECTRUM_PICKER_LABEL)
         .contains("(5)")
       })
     })

packages/builder/cypress/integration/appPublishWorkflow.spec.js

@@ -1,4 +1,5 @@
 import filterTests from "../support/filterTests"
+const interact = require('../support/interact')
 
 filterTests(['all'], () => {
   context("Publish Application Workflow", () => {
@@ -11,49 +12,63 @@ filterTests(['all'], () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.wait(1000)
 
-      cy.get(".appTable .app-status").eq(0)
+      cy.get(interact.APP_TABLE_STATUS).eq(0)
       .within(() => {
         cy.contains("Unpublished")
-        cy.get("svg[aria-label='GlobeStrike']").should("exist")
+        cy.get(interact.GLOBESTRIKE).should("exist")
       })
 
-      cy.get(".appTable .app-row-actions").eq(0)
+      cy.get(interact.APP_TABLE_ROW_ACTION).eq(0)
       .within(() => {
-        cy.get(".spectrum-Button").contains("View")
+        cy.get(interact.SPECTRUM_BUTTON_TEMPLATE).contains("Preview")
-        cy.get(".spectrum-Button").contains("Edit").click({ force: true })
+        cy.get(interact.SPECTRUM_BUTTON_TEMPLATE).contains("Edit").click({ force: true })
       })
       
-      cy.get(".deployment-top-nav svg[aria-label='GlobeStrike']").should("exist")
+      cy.get(interact.DEPLOYMENT_TOP_NAV_GLOBESTRIKE).should("exist")
-      cy.get(".deployment-top-nav svg[aria-label='Globe']").should("not.exist")
+      cy.get(interact.DEPLOYMENT_TOP_GLOBE).should("not.exist")
     })
 
     it("Should publish an application and correctly reflect that", () => {
       //Assuming the previous test was run and the unpublished app is open in edit mode.
+      cy.get(interact.TOPRIGHTNAV_BUTTON_SPECTRUM).contains("Publish").click({ force : true })
 
-      cy.publishApp("cypress-tests")
+      cy.get(interact.DEPLOY_APP_MODAL).should("be.visible")
+      .within(() => {
+        cy.get(interact.SPECTRUM_BUTTON).contains("Publish").click({ force : true })
+        cy.wait(1000)
+      });
+
+      //Verify that the app url is presented correctly to the user
+      cy.get(interact.DEPLOY_APP_MODAL)
+      .should("be.visible")
+      .within(() => {
+        let appUrl = Cypress.config().baseUrl + '/app/cypress-tests'
+        cy.get(interact.DEPLOY_APP_URL_INPUT).should('have.value', appUrl)
+        cy.get(interact.SPECTRUM_BUTTON).contains("Done").click({ force: true })
+      })
 
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.wait(1000)
 
-      cy.get(".appTable .app-status").eq(0)
+      cy.get(interact.APP_TABLE_STATUS).eq(0)
       .within(() => {
         cy.contains("Published")
-        cy.get("svg[aria-label='Globe']").should("exist")
+        cy.get(interact.GLOBE).should("exist")
       })
 
-      cy.get(".appTable .app-row-actions").eq(0)
+      cy.get(interact.APP_TABLE_ROW_ACTION).eq(0)
       .within(() => {
-        cy.get(".spectrum-Button").contains("View")
+        cy.get(interact.SPECTRUM_BUTTON).contains("View")
-        cy.get(".spectrum-Button").contains("Edit").click({ force: true })
+        cy.get(interact.SPECTRUM_BUTTON).contains("Edit").click({ force: true })
       })
 
-      cy.get(".deployment-top-nav svg[aria-label='Globe']").should("exist").click({ force: true })
+      cy.get(interact.DEPLOYMENT_TOP_GLOBE).should("exist").click({ force: true })
       
-      cy.get("[data-cy='publish-popover-menu']").should("be.visible")
+      cy.get(interact.PUBLISH_POPOVER_MENU).should("be.visible")
       .within(() => {
-        cy.get("[data-cy='publish-popover-action']").should("exist")
+        cy.get(interact.PUBLISH_POPOVER_ACTION).should("exist")
-        cy.get("button").contains("View").should("exist")
+        cy.get("button").contains("View app").should("exist")
-        cy.get(".publish-popover-message").should("have.text", "Last published a few seconds ago")
+        cy.get(interact.PUBLISH_POPOVER_MESSAGE).should("have.text", "Last published a few seconds ago")
       })
     })
 
@@ -62,36 +77,36 @@ filterTests(['all'], () => {
       
       cy.visit(`${Cypress.config().baseUrl}/builder`)
 
-      cy.get(".appTable .app-status").eq(0)
+      cy.get(interact.APP_TABLE_STATUS).eq(0)
       .within(() => {
         cy.contains("Published")
         cy.get("svg[aria-label='Globe']").should("exist")
       })
 
-      cy.get(".appTable .app-row-actions").eq(0)
+      cy.get(interact.APP_TABLE_ROW_ACTION).eq(0)
       .within(() => {
-        cy.get(".spectrum-Button").contains("View")
+        cy.get(interact.SPECTRUM_BUTTON).contains("View app")
-        cy.get(".spectrum-Button").contains("Edit").click({ force: true })
+        cy.get(interact.SPECTRUM_BUTTON).contains("Edit").click({ force: true })
       })
 
       //The published status 
-      cy.get(".deployment-top-nav svg[aria-label='Globe']").should("exist")
+      cy.get(interact.DEPLOYMENT_TOP_GLOBE).should("exist")
       .click({ force: true })
 
-      cy.get("[data-cy='publish-popover-menu']").should("be.visible")
+      cy.get(interact.PUBLISH_POPOVER_MENU).should("be.visible")
       cy.get("[data-cy='publish-popover-menu'] [data-cy='publish-popover-action']")
       .click({ force : true })
 
-      cy.get("[data-cy='unpublish-modal']").should("be.visible")
+      cy.get(interact.UNPUBLISH_MODAL).should("be.visible")
       .within(() => {
-        cy.get(".confirm-wrap button").click({ force: true }
+        cy.get(interact.CONFIRM_WRAP_BUTTON).click({ force: true }
       )})
 
-      cy.get(".deployment-top-nav svg[aria-label='GlobeStrike']").should("exist")
+      cy.get(interact.DEPLOYMENT_TOP_NAV_GLOBESTRIKE).should("exist")
 
       cy.visit(`${Cypress.config().baseUrl}/builder`)
 
-      cy.get(".appTable .app-status").eq(0).contains("Unpublished")
+      cy.get(interact.APP_TABLE_STATUS).eq(0).contains("Unpublished")
 
     })
   })

packages/builder/cypress/integration/autoScreensUI.spec.js

@@ -1,4 +1,5 @@
 import filterTests from "../support/filterTests"
+const interact = require('../support/interact')
 
 filterTests(['smoke', 'all'], () => {
   context("Auto Screens UI", () => {
@@ -12,10 +13,10 @@ filterTests(['smoke', 'all'], () => {
       cy.closeModal();
 
       cy.contains("Design").click()
-      cy.get("[aria-label=AddCircle]").click()
+      cy.get(interact.LABEL_ADD_CIRCLE).click()
-      cy.get(".spectrum-Modal").within(() => {
+      cy.get(interact.SPECTRUM_MODAL).within(() => {
-        cy.get(".item.disabled").contains("Autogenerated screens")
+        cy.get(interact.ITEM_DISABLED).contains("Autogenerated screens")
-        cy.get(".confirm-wrap .spectrum-Button").should('be.disabled')
+        cy.get(interact.CONFIRM_WRAP_SPE_BUTTON).should('be.disabled')
       })
 
       cy.deleteAllApps()
@@ -26,14 +27,14 @@ filterTests(['smoke', 'all'], () => {
 
       cy.selectExternalDatasource("REST")
       cy.selectExternalDatasource("S3")
-      cy.get(".spectrum-Modal").within(() => {
+      cy.get(interact.SPECTRUM_MODAL).within(() => {
-        cy.get(".spectrum-Button").contains("Save and continue to query").click({ force : true })
+        cy.get(interact.SPECTRUM_BUTTON).contains("Save and continue to query").click({ force : true })
       })
       
       cy.navigateToAutogeneratedModal()
 
-      cy.get('.data-source-entry').should('have.length', 1)
+      cy.get(interact.DATA_SOURCE_ENTRY).should('have.length', 1)
-      cy.get('.data-source-entry')
+      cy.get(interact.DATA_SOURCE_ENTRY)
 
       cy.deleteAllApps()
     });
@@ -43,8 +44,8 @@ filterTests(['smoke', 'all'], () => {
       // Create Autogenerated screens from the internal table
       cy.createDatasourceScreen(["Cypress Tests"])
       // Confirm screens have been auto generated
-      cy.get(".nav-items-container").contains("cypress-tests").click({ force: true })
+      cy.get(interact.NAV_ITEMS_CONTAINER).contains("cypress-tests").click({ force: true })
-      cy.get(".nav-items-container").should('contain', 'cypress-tests/:id')
+      cy.get(interact.NAV_ITEMS_CONTAINER).should('contain', 'cypress-tests/:id')
         .and('contain', 'cypress-tests/new/row')
     })
 
@@ -56,12 +57,12 @@ filterTests(['smoke', 'all'], () => {
       // Create Autogenerated screens from the internal tables
       cy.createDatasourceScreen([initialTable, secondTable])
       // Confirm screens have been auto generated
-      cy.get(".nav-items-container").contains("cypress-tests").click({ force: true })
+      cy.get(interact.NAV_ITEMS_CONTAINER).contains("cypress-tests").click({ force: true })
       // Previously generated tables are suffixed with numbers - as expected
-      cy.get(".nav-items-container").should('contain', 'cypress-tests-2/:id')
+      cy.get(interact.NAV_ITEMS_CONTAINER).should('contain', 'cypress-tests-2/:id')
         .and('contain', 'cypress-tests-2/new/row')
-      cy.get(".nav-items-container").contains("table-two").click()
+      cy.get(interact.NAV_ITEMS_CONTAINER).contains("table-two").click()
-      cy.get(".nav-items-container").should('contain', 'table-two/:id')
+      cy.get(interact.NAV_ITEMS_CONTAINER).should('contain', 'table-two/:id')
         .and('contain', 'table-two/new/row')
     })
 
@@ -71,17 +72,17 @@ filterTests(['smoke', 'all'], () => {
       cy.createTable("Table Four")
       cy.createDatasourceScreen(["Table Three", "Table Four"], "Admin")
 
-      cy.get(".nav-items-container").contains("table-three").click()
+      cy.get(interact.NAV_ITEMS_CONTAINER).contains("table-three").click()
-      cy.get(".nav-items-container").should('contain', 'table-three/:id')
+      cy.get(interact.NAV_ITEMS_CONTAINER).should('contain', 'table-three/:id')
       .and('contain', 'table-three/new/row')
 
-      cy.get(".nav-items-container").contains("table-four").click()
+      cy.get(interact.NAV_ITEMS_CONTAINER).contains("table-four").click()
-      cy.get(".nav-items-container").should('contain', 'table-four/:id')
+      cy.get(interact.NAV_ITEMS_CONTAINER).should('contain', 'table-four/:id')
       .and('contain', 'table-four/new/row')
 
       //The access level should now be set to admin. Previous screens should be filtered.
-      cy.get(".nav-items-container").contains("table-two").should('not.exist')
+      cy.get(interact.NAV_ITEMS_CONTAINER).contains("table-two").should('not.exist')
-      cy.get(".nav-items-container").contains("cypress-tests").should('not.exist')
+      cy.get(interact.NAV_ITEMS_CONTAINER).contains("cypress-tests").should('not.exist')
     })
 
     if (Cypress.env("TEST_ENV")) {
@@ -94,8 +95,8 @@ filterTests(['smoke', 'all'], () => {
         // Create Autogenerated screens from a MySQL table - MySQL contains books table
         cy.createDatasourceScreen(["books"])
         
-        cy.get(".nav-items-container").contains("books").click()
+        cy.get(interact.NAV_ITEMS_CONTAINER).contains("books").click()
-        cy.get(".nav-items-container").should('contain', 'books/:id')
+        cy.get(interact.NAV_ITEMS_CONTAINER).should('contain', 'books/:id')
           .and('contain', 'books/new/row')
       })
     }

packages/builder/cypress/integration/createApp.spec.js

@@ -30,7 +30,7 @@ filterTests(['smoke', 'all'], () => {
         .its("body")
         .then(val => {
           if (val.length > 0) {
-            cy.get(interact.SPECTRUM_BUTTON_TEMPLATE).contains("Templates").click({force: true})
+            cy.get(interact.SPECTRUM_BUTTON).contains("Templates").click({force: true})
           }
         })
 

packages/builder/cypress/integration/createComponents.spec.js

@@ -1,6 +1,7 @@
 // TODO for now components are skipped, might not be good to keep doing this
 
 import filterTests from "../support/filterTests"
+const interact = require('../support/interact')
 
 filterTests(['all'], () => {
   xcontext("Create Components", () => {
@@ -31,32 +32,32 @@ filterTests(['all'], () => {
 
     it("should change the text of the headline", () => {
       const text = "Lorem ipsum dolor sit amet."
-      cy.get("[data-cy=Settings]").click()
+      cy.get(interact.SETTINGS).click()
-      cy.get("[data-cy=setting-text] input")
+      cy.get(interact.SETTINGS_INPUT)
         .type(text)
         .blur()
       cy.getComponent(headlineId).should("have.text", text)
     })
 
     it("should change the size of the headline", () => {
-      cy.get("[data-cy=Design]").click()
+      cy.get(interact.DESIGN).click()
       cy.contains("Typography").click()
-      cy.get("[data-cy=font-size-prop-control]").click()
+      cy.get(interact.FONT_SIZE_PROP_CONTROL).click()
       cy.contains("60px").click()
       cy.getComponent(headlineId).should("have.css", "font-size", "60px")
     })
 
     it("should create a form and reset to match schema", () => {
       cy.addComponent("Form", "Form").then(() => {
-        cy.get("[data-cy=Settings]").click()
+        cy.get(interact.SETTINGS).click()
-        cy.get("[data-cy=setting-dataSource]")
+        cy.get(interact.DATA_CY_DATASOURCE)
           .contains("Choose option")
           .click()
-        cy.get(".dropdown")
+        cy.get(interact.DROPDOWN)
           .contains("dog")
           .click()
         cy.addComponent("Form", "Field Group").then(fieldGroupId => {
-          cy.get("[data-cy=Settings]").click()
+          cy.get(interact.SETTINGS).click()
           cy.contains("Update Form Fields").click()
           cy.get(".modal")
             .get("button.primary")
@@ -70,7 +71,7 @@ filterTests(['all'], () => {
             .find("input")
             .should("have.length", 2)
           cy.getComponent(fieldGroupId)
-            .find(".spectrum-Picker")
+            .find(interact.SPECTRUM_PICKER)
             .should("have.length", 1)
         })
       })
@@ -84,7 +85,7 @@ filterTests(['all'], () => {
         cy.get(".ui-nav ul .nav-item.selected .ri-more-line").click({
           force: true,
         })
-        cy.get(".dropdown-container")
+        cy.get(interact.DROPDOWN_CONTAINER)
           .contains("Delete")
           .click()
         cy.get(".modal")

packages/builder/cypress/support/interact.js

@@ -17,10 +17,44 @@ export const CATEGORY_DATA = '[data-cy="category-Data"]'
 export const COMPONENT_DATA_PROVIDER = '[data-cy="component-Data Provider"]'
 export const DATASOURCE_PROP_CONTROL = '[data-cy="dataSource-prop-control"]'
 export const DROPDOWN = ".dropdown"
-export const SPECTRUM_Picker_LABEL = ".spectrum-Picker-label"
+export const SPECTRUM_PICKER_LABEL = ".spectrum-Picker-label"
 export const DATASOURCE_FIELD_CONTROL = '[data-cy="field-prop-control"]'
 export const OPTION_TYPE_PROP_CONTROL = '[data-cy="optionsType-prop-control'
 
 //AddRadioButtons
 export const SPECTRUM_POPOVER = ".spectrum-Popover"
 export const OPTION_SOURCE_PROP_CONROL = '[data-cy="optionsSource-prop-control'
+export const APP_TABLE_STATUS = ".appTable .app-status"
+export const APP_TABLE_ROW_ACTION = ".appTable .app-row-actions"
+export const DEPLOYMENT_TOP_NAV_GLOBESTRIKE =
+  ".deployment-top-nav svg[aria-label=GlobeStrike]"
+export const DEPLOYMENT_TOP_GLOBE = ".deployment-top-nav svg[aria-label=Globe]"
+export const PUBLISH_POPOVER_MENU = '[data-cy="publish-popover-menu"]'
+export const PUBLISH_POPOVER_ACTION = '[data-cy="publish-popover-action"]'
+export const PUBLISH_POPOVER_MESSAGE = ".publish-popover-message"
+export const SPECTRUM_BUTTON = ".spectrum-Button"
+export const TOPRIGHTNAV_BUTTON_SPECTRUM = ".toprightnav button.spectrum-Button"
+
+//createComponents
+export const SETTINGS = "[data-cy=Settings]"
+export const SETTINGS_INPUT = "[data-cy=setting-text] input"
+export const DESIGN = "[data-cy=Design]"
+export const FONT_SIZE_PROP_CONTRO = "[data-cy=font-size-prop-control]"
+export const DATA_CY_DATASOURCE = "[data-cy=setting-dataSource]"
+export const DROPDOWN_CONTAINER = ".dropdown-container"
+export const SPECTRUM_PICKER = ".spectrum-Picker"
+
+//autoScreens
+export const LABEL_ADD_CIRCLE = "[aria-label=AddCircle]"
+export const ITEM_DISABLED = ".item.disabled"
+export const CONFIRM_WRAP_SPE_BUTTON = ".confirm-wrap .spectrum-Button"
+export const DATA_SOURCE_ENTRY = ".data-source-entry"
+export const NAV_ITEMS_CONTAINER = ".nav-items-container"
+
+//publishWorkFlow
+export const DEPLOY_APP_MODAL = ".spectrum-Modal [data-cy=deploy-app-modal]"
+export const DEPLOY_APP_URL_INPUT = "[data-cy=deployed-app-url] input"
+export const GLOBESTRIKE = "svg[aria-label=GlobeStrike]"
+export const GLOBE = "svg[aria-label=Globe]"
+export const UNPUBLISH_MODAL = "[data-cy=unpublish-modal]"
+export const CONFIRM_WRAP_BUTTON = ".confirm-wrap button"

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.189",
+  "version": "1.0.195",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -69,10 +69,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.189",
+    "@budibase/bbui": "^1.0.195",
-    "@budibase/client": "^1.0.189",
+    "@budibase/client": "^1.0.195",
-    "@budibase/frontend-core": "^1.0.189",
+    "@budibase/frontend-core": "^1.0.195",
-    "@budibase/string-templates": "^1.0.189",
+    "@budibase/string-templates": "^1.0.195",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte

@@ -91,7 +91,11 @@
       notifications.success(`Datasource ${name} tables updated successfully.`)
       await tables.fetch()
     } catch (error) {
-      notifications.error("Error updating datasource schema")
+      notifications.error(
+        `Error updating datasource schema ${
+          error?.message ? `: ${error.message}` : ""
+        }`
+      )
     }
   }
 

packages/builder/src/components/common/LinkedRowSelector.svelte

@@ -8,7 +8,9 @@
   export let linkedRows = []
 
   let rows = []
-  let linkedIds = (linkedRows || [])?.map(row => row?._id || row)
+  let linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
+    row => row?._id || row
+  )
 
   $: linkedRows = linkedIds
   $: label = capitalise(schema.name)

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.189",
+  "version": "1.0.195",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.189",
+  "version": "1.0.195",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.189",
+    "@budibase/bbui": "^1.0.195",
-    "@budibase/frontend-core": "^1.0.189",
+    "@budibase/frontend-core": "^1.0.195",
-    "@budibase/string-templates": "^1.0.189",
+    "@budibase/string-templates": "^1.0.195",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "1.0.189",
+  "version": "1.0.195",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "^1.0.189",
+    "@budibase/bbui": "^1.0.195",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.189",
+  "version": "1.0.195",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -71,10 +71,10 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.189",
+    "@budibase/backend-core": "^1.0.195",
-    "@budibase/client": "^1.0.189",
+    "@budibase/client": "^1.0.195",
-    "@budibase/pro": "1.0.189",
+    "@budibase/pro": "1.0.195",
-    "@budibase/string-templates": "^1.0.189",
+    "@budibase/string-templates": "^1.0.195",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/index.js

@@ -12,6 +12,7 @@ const { mainRoutes, staticRoutes, publicRoutes } = require("./routes")
 const pkg = require("../../package.json")
 const env = require("../environment")
 const { middleware: pro } = require("@budibase/pro")
+const { shutdown } = require("./routes/public")
 
 const router = new Router()
 
@@ -88,4 +89,5 @@ router.use(publicRoutes.allowedMethods())
 router.use(staticRoutes.routes())
 router.use(staticRoutes.allowedMethods())
 
-module.exports = router
+module.exports.router = router
+module.exports.shutdown = shutdown

packages/server/src/api/routes/public/index.ts

@@ -29,6 +29,7 @@ function getApiLimitPerSecond(): number {
   return parseInt(env.API_REQ_LIMIT_PER_SEC)
 }
 
+let rateLimitStore: any = null
 if (!env.isTest()) {
   const REDIS_OPTS = getRedisOptions()
   let options
@@ -47,8 +48,9 @@ if (!env.isTest()) {
       database: 1,
     }
   }
+  rateLimitStore = new Stores.Redis(options)
   RateLimit.defaultOptions({
-    store: new Stores.Redis(options),
+    store: rateLimitStore,
   })
 }
 // rate limiting, allows for 2 requests per second
@@ -128,3 +130,10 @@ applyRoutes(queryEndpoints, PermissionTypes.QUERY, "queryId")
 applyRoutes(rowEndpoints, PermissionTypes.TABLE, "tableId", "rowId")
 
 export default publicRouter
+
+export const shutdown = () => {
+  if (rateLimitStore) {
+    rateLimitStore.client.disconnect()
+    rateLimitStore = null
+  }
+}

packages/server/src/app.ts

@@ -14,6 +14,8 @@ const automations = require("./automations/index")
 const Sentry = require("@sentry/node")
 const fileSystem = require("./utilities/fileSystem")
 const bullboard = require("./automations/bullboard")
+const { logAlert } = require("@budibase/backend-core/logging")
+const { Thread } = require("./threads")
 import redis from "./utilities/redis"
 import * as migrations from "./migrations"
 import { events, installation } from "@budibase/backend-core"
@@ -50,7 +52,7 @@ app.context.eventEmitter = eventEmitter
 app.context.auth = {}
 
 // api routes
-app.use(api.routes())
+app.use(api.router.routes())
 
 if (env.isProd()) {
   env._set("NODE_ENV", "production")
@@ -69,12 +71,25 @@ if (env.isProd()) {
 const server = http.createServer(app.callback())
 destroyable(server)
 
+let shuttingDown = false,
+  errCode = 0
 server.on("close", async () => {
-  if (env.NODE_ENV !== "jest") {
+  // already in process
+  if (shuttingDown) {
+    return
+  }
+  shuttingDown = true
+  if (!env.isTest()) {
     console.log("Server Closed")
   }
+  await automations.shutdown()
   await redis.shutdown()
   await events.shutdown()
+  await Thread.shutdown()
+  api.shutdown()
+  if (!env.isTest()) {
+    process.exit(errCode)
+  }
 })
 
 module.exports = server.listen(env.PORT || 0, async () => {
@@ -90,7 +105,7 @@ module.exports = server.listen(env.PORT || 0, async () => {
     try {
       await migrations.migrate()
     } catch (e) {
-      console.error("Error performing migrations. Exiting.\n", e)
+      logAlert("Error performing migrations. Exiting.", e)
       shutdown()
     }
   }
@@ -108,7 +123,13 @@ const shutdown = () => {
 }
 
 process.on("uncaughtException", err => {
-  console.error(err)
+  // @ts-ignore
+  // don't worry about this error, comes from zlib isn't important
+  if (err && err["code"] === "ERR_INVALID_CHAR") {
+    return
+  }
+  errCode = -1
+  logAlert("Uncaught exception.", err)
   shutdown()
 })
 

packages/server/src/automations/bullboard.js

@@ -45,4 +45,12 @@ exports.init = () => {
   return serverAdapter.registerPlugin()
 }
 
+exports.shutdown = async () => {
+  if (automationQueue) {
+    clearInterval(cleanupInternal)
+    await automationQueue.close()
+    automationQueue = null
+  }
+}
+
 exports.queue = automationQueue

packages/server/src/automations/index.js

@@ -1,5 +1,5 @@
 const { processEvent } = require("./utils")
-const { queue } = require("./bullboard")
+const { queue, shutdown } = require("./bullboard")
 const { TRIGGER_DEFINITIONS } = require("./triggers")
 const { ACTION_DEFINITIONS } = require("./actions")
 
@@ -17,6 +17,10 @@ exports.getQueues = () => {
   return [queue]
 }
 
+exports.shutdown = () => {
+  return shutdown()
+}
+
 exports.queue = queue
 exports.TRIGGER_DEFINITIONS = TRIGGER_DEFINITIONS
 exports.ACTION_DEFINITIONS = ACTION_DEFINITIONS

packages/server/src/integrations/googlesheets.ts

@@ -164,11 +164,15 @@ module GoogleSheetsModule {
         }
       )
 
+      const json = await response.json()
+
       if (response.status !== 200) {
-        throw new Error("Error authenticating with google sheets.")
+        throw new Error(
+          `Error authenticating with google sheets. ${json.error_description}`
+        )
       }
 
-      return response.json()
+      return json
     }
 
     async connect() {

packages/server/src/threads/index.ts

@@ -28,6 +28,8 @@ export class Thread {
   workers: any
   timeoutMs: any
 
+  static workerRefs: any[] = []
+
   constructor(type: any, opts: any = { timeoutMs: null, count: 1 }) {
     this.type = type
     this.count = opts.count ? opts.count : 1
@@ -46,6 +48,7 @@ export class Thread {
         workerOpts.maxCallTime = opts.timeoutMs
       }
       this.workers = workerFarm(workerOpts, typeToFile(type))
+      Thread.workerRefs.push(this.workers)
     }
   }
 
@@ -73,4 +76,23 @@ export class Thread {
       })
     })
   }
+
+  static shutdown() {
+    return new Promise<void>(resolve => {
+      if (Thread.workerRefs.length === 0) {
+        resolve()
+      }
+      let count = 0
+      function complete() {
+        count++
+        if (count >= Thread.workerRefs.length) {
+          resolve()
+        }
+      }
+      for (let worker of Thread.workerRefs) {
+        workerFarm.end(worker, complete)
+      }
+      Thread.workerRefs = []
+    })
+  }
 }

packages/server/src/utilities/queue/inMemoryQueue.js

@@ -75,6 +75,13 @@ class InMemoryQueue {
     this._emitter.emit("message")
   }
 
+  /**
+   * replicating the close function from bull, which waits for jobs to finish.
+   */
+  async close() {
+    return []
+  }
+
   /**
    * This removes a cron which has been implemented, this is part of Bull API.
    * @param {string} cronJobId The cron which is to be removed.

packages/server/yarn.lock

@@ -1021,14 +1021,15 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.189":
+"@budibase/backend-core@1.0.194":
-  version "1.0.189"
+  version "1.0.194"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.189.tgz#689edf9ccb7bfaaa397eaf6cf76c5a75043d60b9"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.194.tgz#08b2b1aec3c88efbc7cfb14145ce6f199475c538"
-  integrity sha512-j+/IZM6p7N0DZQNqNOkghFqjPrKyMRLZPnhHhtpCVwVPv379Bv2YxHZ7wDSoSLXVN3N1Yn72jGZKvlLw/Z1Lkw==
+  integrity sha512-BbnJFtAioJeD9tQfSwc2uftFK8SagREgSfUYv06dfnf/NsmkrONzZiTon1oA57S7ifcSiu+WZf87lNX0k8pwfQ==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
     bcrypt "^5.0.1"
+    dotenv "^16.0.1"
     emitter-listener "^1.1.2"
     ioredis "^4.27.1"
     jsonwebtoken "^8.5.1"
@@ -1098,12 +1099,12 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/pro@1.0.189":
+"@budibase/pro@1.0.194":
-  version "1.0.189"
+  version "1.0.194"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.189.tgz#11fb6ce151818f246dbbcafc329f65ff0bf0c9bb"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.194.tgz#fbf977b292b9a6dbf7b072b2e0379dcf4379943a"
-  integrity sha512-0L95Ce8Ll3mhv21QWXgtbdvIekaUXoblBVebsoAzFxsA/w4xuxiTIX8RtcGYhu1BqMEfeaetorXSM5cAIfOk8g==
+  integrity sha512-LSqVwlhKWwFNnC3acvLnCzbeBoze1Ma6GELE/b5ZxS65owsigu0KC6T/UuujEbU9xqbFJ3R6uV+4Fz4NUibLIw==
   dependencies:
-    "@budibase/backend-core" "1.0.189"
+    "@budibase/backend-core" "1.0.194"
     node-fetch "^2.6.1"
 
 "@budibase/standard-components@^0.9.139":
@@ -5053,6 +5054,11 @@ dotenv@8.2.0:
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a"
   integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==
 
+dotenv@^16.0.1:
+  version "16.0.1"
+  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.1.tgz#8f8f9d94876c35dac989876a5d3a82a267fdce1d"
+  integrity sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==
+
 dotenv@^8.2.0:
   version "8.6.0"
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.6.0.tgz#061af664d19f7f4d8fc6e4ff9b584ce237adcb8b"

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.0.189",
+  "version": "1.0.195",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.189",
+  "version": "1.0.195",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -33,9 +33,9 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.189",
+    "@budibase/backend-core": "^1.0.195",
-    "@budibase/pro": "1.0.189",
+    "@budibase/pro": "1.0.195",
-    "@budibase/string-templates": "^1.0.189",
+    "@budibase/string-templates": "^1.0.195",
     "@koa/router": "^8.0.0",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "^0.3.0",

packages/worker/src/api/routes/global/users.js

@@ -6,6 +6,7 @@ const Joi = require("joi")
 const cloudRestricted = require("../../../middleware/cloudRestricted")
 const { users } = require("../validation")
 const selfController = require("../../controllers/global/self")
+const builderOrAdmin = require("../../../middleware/builderOrAdmin")
 
 const router = Router()
 
@@ -44,7 +45,7 @@ router
     users.buildUserSaveValidation(),
     controller.save
   )
-  .get("/api/global/users", adminOnly, controller.fetch)
+  .get("/api/global/users", builderOrAdmin, controller.fetch)
   .delete("/api/global/users/:id", adminOnly, controller.destroy)
   .get("/api/global/roles/:appId")
   .post(

packages/worker/src/index.ts

@@ -12,6 +12,7 @@ const destroyable = require("server-destroy")
 const koaBody = require("koa-body")
 const koaSession = require("koa-session")
 const { passport } = require("@budibase/backend-core/auth")
+const { logAlert } = require("@budibase/backend-core/logging")
 const logger = require("koa-pino-logger")
 const http = require("http")
 const api = require("./api")
@@ -29,7 +30,6 @@ app.keys = ["secret", "key"]
 // set up top level koa middleware
 app.use(koaBody({ multipart: true }))
 app.use(koaSession(app))
-
 app.use(
   logger({
     prettyPrint: {
@@ -63,26 +63,39 @@ if (env.isProd()) {
 const server = http.createServer(app.callback())
 destroyable(server)
 
+let shuttingDown = false,
+  errCode = 0
 server.on("close", async () => {
-  if (env.isProd()) {
+  if (shuttingDown) {
+    return
+  }
+  shuttingDown = true
+  if (!env.isTest()) {
     console.log("Server Closed")
   }
   await redis.shutdown()
   await events.shutdown()
+  if (!env.isTest()) {
+    process.exit(errCode)
+  }
 })
 
+const shutdown = () => {
+  server.close()
+  server.destroy()
+}
+
 module.exports = server.listen(parseInt(env.PORT || 4002), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
   await redis.init()
 })
 
 process.on("uncaughtException", err => {
-  console.error(err)
+  errCode = -1
-  server.close()
+  logAlert("Uncaught exception.", err)
-  server.destroy()
+  shutdown()
 })
 
 process.on("SIGTERM", () => {
-  server.close()
+  shutdown()
-  server.destroy()
 })

packages/worker/src/middleware/builderOrAdmin.js

@@ -0,0 +1,10 @@
+module.exports = async (ctx, next) => {
+  if (
+    !ctx.internal &&
+    (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) &&
+    (!ctx.user || !ctx.user.admin || !ctx.user.admin.global)
+  ) {
+    ctx.throw(403, "Builder user only endpoint.")
+  }
+  return next()
+}

packages/worker/yarn.lock

@@ -293,14 +293,15 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.189":
+"@budibase/backend-core@1.0.194":
-  version "1.0.189"
+  version "1.0.194"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.189.tgz#689edf9ccb7bfaaa397eaf6cf76c5a75043d60b9"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.194.tgz#08b2b1aec3c88efbc7cfb14145ce6f199475c538"
-  integrity sha512-j+/IZM6p7N0DZQNqNOkghFqjPrKyMRLZPnhHhtpCVwVPv379Bv2YxHZ7wDSoSLXVN3N1Yn72jGZKvlLw/Z1Lkw==
+  integrity sha512-BbnJFtAioJeD9tQfSwc2uftFK8SagREgSfUYv06dfnf/NsmkrONzZiTon1oA57S7ifcSiu+WZf87lNX0k8pwfQ==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
     bcrypt "^5.0.1"
+    dotenv "^16.0.1"
     emitter-listener "^1.1.2"
     ioredis "^4.27.1"
     jsonwebtoken "^8.5.1"
@@ -321,12 +322,12 @@
     uuid "^8.3.2"
     zlib "^1.0.5"
 
-"@budibase/pro@1.0.189":
+"@budibase/pro@1.0.194":
-  version "1.0.189"
+  version "1.0.194"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.189.tgz#11fb6ce151818f246dbbcafc329f65ff0bf0c9bb"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.194.tgz#fbf977b292b9a6dbf7b072b2e0379dcf4379943a"
-  integrity sha512-0L95Ce8Ll3mhv21QWXgtbdvIekaUXoblBVebsoAzFxsA/w4xuxiTIX8RtcGYhu1BqMEfeaetorXSM5cAIfOk8g==
+  integrity sha512-LSqVwlhKWwFNnC3acvLnCzbeBoze1Ma6GELE/b5ZxS65owsigu0KC6T/UuujEbU9xqbFJ3R6uV+4Fz4NUibLIw==
   dependencies:
-    "@budibase/backend-core" "1.0.189"
+    "@budibase/backend-core" "1.0.194"
     node-fetch "^2.6.1"
 
 "@cspotcode/source-map-consumer@0.8.0":
@@ -2258,6 +2259,11 @@ dot-prop@^5.2.0:
   dependencies:
     is-obj "^2.0.0"
 
+dotenv@^16.0.1:
+  version "16.0.1"
+  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.1.tgz#8f8f9d94876c35dac989876a5d3a82a267fdce1d"
+  integrity sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==
+
 dotenv@^8.2.0:
   version "8.6.0"
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.6.0.tgz#061af664d19f7f4d8fc6e4ff9b584ce237adcb8b"