Merge branch 'develop' into cypress-testing

.github/workflows/smoke_test.yaml

@@ -31,6 +31,7 @@ jobs:
         continue-on-error: true
         uses: cypress-io/github-action@v2
         with:
+          record: true
           install: false
           command: yarn test:e2e:ci:record
         env:
@@ -48,7 +49,7 @@ jobs:
         uses: tsickert/discord-webhook@v4.0.0
         with:
           webhook-url: ${{ secrets.BUDI_QA_WEBHOOK }}
-          content: "Smoke test run completed with ${{ steps.cypress.outcome }}. See results at ${{ steps.cypress.dashboardUrl }}"
+          content: "Smoke test run completed with ${{ steps.cypress.outcome }}. See results at ${{ steps.cypress.outputs.dashboardUrl }}"
           embed-title: ${{ steps.cypress.outcome }}
           embed-color: ${{ steps.cypress.outcome == 'success' && '3066993' || '15548997' }}
 

charts/budibase/templates/app-service-deployment.yaml

@@ -114,8 +114,8 @@ spec:
           value: {{ .Values.globals.google.secret | quote }}
         - name: AUTOMATION_MAX_ITERATIONS
           value: {{ .Values.globals.automationMaxIterations | quote }}
-        - name: EXCLUDE_QUOTAS_TENANTS
+        - name: TENANT_FEATURE_FLAGS
-          value: {{ .Values.globals.excludeQuotasTenants | quote }}
+          value: {{ .Values.globals.tenantFeatureFlags | quote }}
 
         image: budibase/apps:{{ .Values.globals.appVersion }}
         imagePullPolicy: Always

hosting/nginx.prod.conf.hbs

@@ -30,7 +30,7 @@ http {
   log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                     '$status $body_bytes_sent "$http_referer" '
                     '"$http_user_agent" "$http_x_forwarded_for"';
-  
+
   map $http_upgrade $connection_upgrade {
     default     "upgrade";
   }
@@ -42,13 +42,13 @@ http {
     client_max_body_size 1000m;
     ignore_invalid_headers off;
     proxy_buffering off;
-    
+
     set $csp_default "default-src 'self'";
     set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io";
     set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
     set $csp_object "object-src 'none'";
     set $csp_base_uri "base-uri 'self'";
-    set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.s3.*.amazonaws.com";
+    set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com";
     set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
     set $csp_frame "frame-src 'self' https:";
     set $csp_img "img-src http: https: data: blob:";
@@ -58,7 +58,7 @@ http {
 
     error_page 502 503 504 /error.html;
     location = /error.html {
-      root /usr/share/nginx/html; 
+      root /usr/share/nginx/html;
       internal;
     }
 
@@ -154,4 +154,4 @@ http {
     gzip_comp_level 6;
     gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
   }
-}
+}

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.143-alpha.2",
+  "version": "1.0.151-alpha.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.143-alpha.2",
+  "version": "1.0.151-alpha.2",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",

packages/backend-core/src/db/conversions.js

@@ -23,24 +23,30 @@ exports.isDevApp = app => {
 }
 
 /**
- * Convert a development app ID to a deployed app ID.
+ * Generates a development app ID from a real app ID.
+ * @returns {string} the dev app ID which can be used for dev database.
  */
-exports.getProdAppID = appId => {
+exports.getDevelopmentAppID = appId => {
-  // if dev, convert it
+  if (!appId || appId.startsWith(APP_DEV_PREFIX)) {
-  if (appId.startsWith(APP_DEV_PREFIX)) {
+    return appId
-    const id = appId.split(APP_DEV_PREFIX)[1]
-    return `${APP_PREFIX}${id}`
   }
-  return appId
+  // split to take off the app_ element, then join it together incase any other app_ exist
+  const split = appId.split(APP_PREFIX)
+  split.shift()
+  const rest = split.join(APP_PREFIX)
+  return `${APP_DEV_PREFIX}${rest}`
 }
 
 /**
- * Convert a deployed app ID to a development app ID.
+ * Convert a development app ID to a deployed app ID.
  */
-exports.getDevelopmentAppID = appId => {
+exports.getProdAppID = appId => {
-  if (!appId.startsWith(APP_DEV_PREFIX)) {
+  if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
-    const id = appId.split(APP_PREFIX)[1]
+    return appId
-    return `${APP_DEV_PREFIX}${id}`
   }
-  return appId
+  // split to take off the app_dev element, then join it together incase any other app_ exist
+  const split = appId.split(APP_DEV_PREFIX)
+  split.shift()
+  const rest = split.join(APP_DEV_PREFIX)
+  return `${APP_PREFIX}${rest}`
 }

packages/backend-core/src/db/tests/utils.spec.js

@@ -0,0 +1,61 @@
+const {
+  generateAppID,
+  getDevelopmentAppID,
+  getProdAppID,
+  isDevAppID,
+  isProdAppID,
+} = require("../utils")
+
+function getID() {
+  const appId = generateAppID()
+  const split = appId.split("_")
+  const uuid = split[split.length - 1]
+  const devAppId = `app_dev_${uuid}`
+  return { appId, devAppId, split, uuid }
+}
+
+describe("app ID manipulation", () => {
+  it("should be able to generate a new app ID", () => {
+    expect(generateAppID().startsWith("app_")).toEqual(true)
+  })
+
+  it("should be able to convert a production app ID to development", () => {
+    const { appId, uuid } = getID()
+    expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
+  })
+
+  it("should be able to convert a development app ID to development", () => {
+    const { devAppId, uuid } = getID()
+    expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
+  })
+
+  it("should be able to convert a development ID to a production", () => {
+    const { devAppId, uuid } = getID()
+    expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
+  })
+
+  it("should be able to convert a production ID to production", () => {
+    const { appId, uuid } = getID()
+    expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
+  })
+
+  it("should be able to confirm dev app ID is development", () => {
+    const { devAppId } = getID()
+    expect(isDevAppID(devAppId)).toEqual(true)
+  })
+
+  it("should be able to confirm prod app ID is not development", () => {
+    const { appId } = getID()
+    expect(isDevAppID(appId)).toEqual(false)
+  })
+
+  it("should be able to confirm prod app ID is prod", () => {
+    const { appId } = getID()
+    expect(isProdAppID(appId)).toEqual(true)
+  })
+
+  it("should be able to confirm dev app ID is not prod", () => {
+    const { devAppId } = getID()
+    expect(isProdAppID(devAppId)).toEqual(false)
+  })
+})

packages/backend-core/src/db/utils.js

@@ -43,6 +43,18 @@ exports.isDevAppID = isDevAppID
 exports.getDevelopmentAppID = getDevelopmentAppID
 exports.getProdAppID = getProdAppID
 
+/**
+ * Generates a new app ID.
+ * @returns {string} The new app ID which the app doc can be stored under.
+ */
+exports.generateAppID = (tenantId = null) => {
+  let id = APP_PREFIX
+  if (tenantId) {
+    id += `${tenantId}${SEPARATOR}`
+  }
+  return `${id}${newid()}`
+}
+
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this
  * is usually the case as most of our docs are top level e.g. tables, automations, users and so on.

packages/backend-core/src/environment.js

@@ -34,6 +34,12 @@ module.exports = {
   COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
   PLATFORM_URL: process.env.PLATFORM_URL,
   TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
+  BACKUPS_BUCKET_NAME: process.env.BACKUPS_BUCKET_NAME || "backups",
+  APPS_BUCKET_NAME: process.env.APPS_BUCKET_NAME || "prod-budi-app-assets",
+  TEMPLATES_BUCKET_NAME: process.env.TEMPLATES_BUCKET_NAME || "templates",
+  GLOBAL_BUCKET_NAME: process.env.GLOBAL_BUCKET_NAME || "global",
+  GLOBAL_CLOUD_BUCKET_NAME:
+    process.env.GLOBAL_CLOUD_BUCKET_NAME || "prod-budi-tenant-uploads",
   USE_COUCH: process.env.USE_COUCH || true,
   isTest,
   isDev,

packages/backend-core/src/featureFlags/index.js

@@ -49,4 +49,5 @@ exports.getTenantFeatureFlags = tenantId => {
 
 exports.FeatureFlag = {
   LICENSING: "LICENSING",
+  GOOGLE_SHEETS: "GOOGLE_SHEETS",
 }

packages/backend-core/src/objectStore/utils.js

@@ -1,12 +1,13 @@
 const { join } = require("path")
 const { tmpdir } = require("os")
+const env = require("../environment")
 
 exports.ObjectStoreBuckets = {
-  BACKUPS: "backups",
+  BACKUPS: env.BACKUPS_BUCKET_NAME,
-  APPS: "prod-budi-app-assets",
+  APPS: env.APPS_BUCKET_NAME,
-  TEMPLATES: "templates",
+  TEMPLATES: env.TEMPLATES_BUCKET_NAME,
-  GLOBAL: "global",
+  GLOBAL: env.GLOBAL_BUCKET_NAME,
-  GLOBAL_CLOUD: "prod-budi-tenant-uploads",
+  GLOBAL_CLOUD: env.GLOBAL_CLOUD_BUCKET_NAME,
 }
 
 exports.budibaseTempDir = function () {

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.143-alpha.2",
+  "version": "1.0.151-alpha.2",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "^1.0.143-alpha.2",
+    "@budibase/string-templates": "^1.0.151-alpha.2",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/builder/cypress/integration/createScreen.js

@@ -9,7 +9,7 @@ filterTests(["smoke", "all"], () => {
     })
 
     it("Should successfully create a screen", () => {
-      cy.createScreen("/test")
+      cy.createScreen("test")
       cy.get(".nav-items-container").within(() => {
         cy.contains("/test").should("exist")
       })

packages/builder/cypress/integration/revertApp.spec.js

@@ -26,6 +26,8 @@ filterTests(['smoke', 'all'], () => {
         })
         
         it("should revert a published app", () => {
+            cy.navigateToFrontend()
+
             // Add initial component - Paragraph
             cy.addComponent("Elements", "Paragraph")
             // Publish app
@@ -37,6 +39,7 @@ filterTests(['smoke', 'all'], () => {
             cy.get(".spectrum-ButtonGroup").within(() => {
                 cy.get(".spectrum-Button").contains("Done").click({ force: true })
             })
+
             // Add second component - Button
             cy.addComponent("Elements", "Button")
             // Click Revert

packages/builder/cypress/support/commands.js

@@ -142,7 +142,6 @@ Cypress.Commands.add("createTestApp", () => {
   const appName = "Cypress Tests"
   cy.deleteApp(appName)
   cy.createApp(appName, "This app is used for Cypress testing.")
-  cy.createScreen("home")
 })
 
 Cypress.Commands.add("createTestTableWithData", () => {

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.143-alpha.2",
+  "version": "1.0.151-alpha.2",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -67,10 +67,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.143-alpha.2",
+    "@budibase/bbui": "^1.0.151-alpha.2",
-    "@budibase/client": "^1.0.143-alpha.2",
+    "@budibase/client": "^1.0.151-alpha.2",
-    "@budibase/frontend-core": "^1.0.143-alpha.2",
+    "@budibase/frontend-core": "^1.0.151-alpha.2",
-    "@budibase/string-templates": "^1.0.143-alpha.2",
+    "@budibase/string-templates": "^1.0.151-alpha.2",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -53,6 +53,7 @@
     : { schema: {} }
   $: schema = getSchemaForTable(tableId, { searchableSchema: true }).schema
   $: schemaFields = Object.values(schema || {})
+  $: queryLimit = tableId?.includes("datasource") ? "∞" : "1000"
 
   const onChange = Utils.sequential(async (e, key) => {
     try {
@@ -330,6 +331,7 @@
               on:change={e => onChange(e, key)}
               {bindings}
               updateOnChange={false}
+              placeholder={value.customType === "queryLimit" ? queryLimit : ""}
             />
           </div>
         {/if}

packages/builder/src/components/start/AppRow.svelte

@@ -19,6 +19,7 @@
   export let unpublishApp
   export let releaseLock
   export let editIcon
+  export let copyAppId
 </script>
 
 <div class="title">
@@ -102,6 +103,9 @@
       <MenuItem on:click={() => unpublishApp(app)} icon="GlobeRemove">
         Unpublish
       </MenuItem>
+      <MenuItem on:click={() => copyAppId(app)} icon="Copy">
+        Copy App ID
+      </MenuItem>
     {/if}
     {#if !app.deployed}
       <MenuItem on:click={() => updateApp(app)} icon="Edit">Edit</MenuItem>

packages/builder/src/components/start/CreateAppModal.svelte

@@ -10,6 +10,8 @@
   import { createValidationStore } from "helpers/validation/yup"
   import * as appValidation from "helpers/validation/yup/app"
   import TemplateCard from "components/common/TemplateCard.svelte"
+  import createFromScratchScreen from "builderStore/store/screenTemplates/createFromScratchScreen"
+  import { Roles } from "constants/backend"
 
   export let template
 
@@ -81,7 +83,7 @@
       }
       data.append("useTemplate", template != null)
       if (template) {
-        data.append("templateName", template.name)
+        data.append("templateName", template.name) //or here?
         data.append("templateKey", template.key)
         data.append("templateFile", $values.file)
       }
@@ -104,6 +106,22 @@
       // Create user
       await API.updateOwnMetadata({ roleId: $values.roleId })
       await auth.setInitInfo({})
+
+      // Create a default home screen if no template was selected
+      if (template == null) {
+        let defaultScreenTemplate = createFromScratchScreen.create()
+        defaultScreenTemplate.routing.route = "/home"
+        defaultScreenTemplate.routing.roldId = Roles.BASIC
+        try {
+          await store.actions.screens.save(defaultScreenTemplate)
+        } catch (err) {
+          console.error("Could not create a default application screen", err)
+          notifications.warning(
+            "Encountered an issue creating the default screen."
+          )
+        }
+      }
+
       $goto(`/builder/app/${createdApp.instance._id}`)
     } catch (error) {
       creating = false

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -12,6 +12,7 @@
     Body,
     Search,
     Divider,
+    Helpers,
   } from "@budibase/bbui"
   import TemplateDisplay from "components/common/TemplateDisplay.svelte"
   import Spinner from "components/common/Spinner.svelte"
@@ -261,6 +262,11 @@
     }
   }
 
+  const copyAppId = async app => {
+    await Helpers.copyToClipboard(app.prodId)
+    notifications.success("App ID copied to clipboard.")
+  }
+
   function createAppFromTemplateUrl(templateKey) {
     // validate the template key just to make sure
     const templateParts = templateKey.split("/")
@@ -394,6 +400,7 @@
           <div class="appTable">
             {#each filteredApps as app (app.appId)}
               <AppRow
+                {copyAppId}
                 {releaseLock}
                 {editIcon}
                 {app}

packages/builder/src/pages/builder/portal/manage/users/_components/BasicOnboardingModal.svelte

@@ -47,7 +47,7 @@
   </Body>
   <Input
     type="email"
-    label="Username"
+    label="Email"
     bind:value={$email}
     error={$touched && $error}
   />

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.143-alpha.2",
+  "version": "1.0.151-alpha.2",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.143-alpha.2",
+  "version": "1.0.151-alpha.2",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.143-alpha.2",
+    "@budibase/bbui": "^1.0.151-alpha.2",
-    "@budibase/frontend-core": "^1.0.143-alpha.2",
+    "@budibase/frontend-core": "^1.0.151-alpha.2",
-    "@budibase/string-templates": "^1.0.143-alpha.2",
+    "@budibase/string-templates": "^1.0.151-alpha.2",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/client/src/components/app/forms/FormStep.svelte

@@ -22,7 +22,7 @@
     if (
       formContext &&
       $builderStore.inBuilder &&
-      $componentStore?.selectedComponentPath?.includes($component.id)
+      $componentStore.selectedComponentPath?.includes($component.id)
     ) {
       formContext.formApi.setStep(step)
     }

packages/client/src/components/preview/SettingsBar.svelte

@@ -155,7 +155,7 @@
       icon="Duplicate"
       on:click={() => {
         builderStore.actions.duplicateComponent(
-          $builderStore.selectedComponent._id
+          $builderStore.selectedComponentId
         )
       }}
       title="Duplicate component"

packages/client/src/sdk.js

@@ -7,6 +7,7 @@ import {
   builderStore,
   uploadStore,
   rowSelectionStore,
+  componentStore,
 } from "stores"
 import { styleable } from "utils/styleable"
 import { linkable } from "utils/linkable"
@@ -24,6 +25,7 @@ export default {
   screenStore,
   builderStore,
   uploadStore,
+  componentStore,
   styleable,
   linkable,
   getAction,

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "1.0.143-alpha.2",
+  "version": "1.0.151-alpha.2",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "^1.0.143-alpha.2",
+    "@budibase/bbui": "^1.0.151-alpha.2",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/server/__mocks__/@google-cloud/firestore.ts

@@ -0,0 +1,36 @@
+module FirebaseMock {
+  const firebase: any = {}
+
+  firebase.Firestore = function () {
+    this.get = jest.fn(() => [
+      {
+        data: jest.fn(() => ({ result: "test" })),
+      },
+    ])
+
+    this.update = jest.fn()
+    this.set = jest.fn()
+    this.delete = jest.fn()
+
+    this.doc = jest.fn(() => ({
+      update: this.update,
+      set: this.set,
+      delete: this.delete,
+      get: jest.fn(() => ({
+        data: jest.fn(() => ({ result: "test" })),
+      })),
+      id: "test_id",
+    }))
+
+    this.where = jest.fn(() => ({
+      get: this.get,
+    }))
+
+    this.collection = jest.fn(() => ({
+      doc: this.doc,
+      where: this.where,
+    }))
+  }
+
+  module.exports = firebase
+}

packages/server/__mocks__/pg.ts

@@ -14,21 +14,13 @@ module PgMock {
   function Client() {}
 
   Client.prototype.query = query
+  Client.prototype.end = jest.fn()
   Client.prototype.connect = jest.fn()
   Client.prototype.release = jest.fn()
 
-  function Pool() {}
-
   const on = jest.fn()
-  Pool.prototype.query = query
-  Pool.prototype.connect = jest.fn(() => {
-    // @ts-ignore
-    return new Client()
-  })
-  Pool.prototype.on = on
 
   pg.Client = Client
-  pg.Pool = Pool
   pg.queryMock = query
   pg.on = on
 

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.143-alpha.2",
+  "version": "1.0.151-alpha.2",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -68,10 +68,10 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.143-alpha.2",
+    "@budibase/backend-core": "^1.0.151-alpha.2",
-    "@budibase/client": "^1.0.143-alpha.2",
+    "@budibase/client": "^1.0.151-alpha.2",
-    "@budibase/pro": "1.0.143-alpha.2",
+    "@budibase/pro": "1.0.151-alpha.2",
-    "@budibase/string-templates": "^1.0.143-alpha.2",
+    "@budibase/string-templates": "^1.0.151-alpha.2",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/controllers/integration.js

@@ -1,21 +1,16 @@
 const { cloneDeep } = require("lodash")
 const { definitions } = require("../../integrations")
-const { getTenantId } = require("@budibase/backend-core/tenancy")
 const { SourceNames } = require("../../definitions/datasource")
 const googlesheets = require("../../integrations/googlesheets")
-const env = require("../../environment")
+const { featureFlags } = require("@budibase/backend-core")
 
 exports.fetch = async function (ctx) {
   ctx.status = 200
   const defs = cloneDeep(definitions)
 
   // for google sheets integration google verification
-  if (env.EXCLUDE_QUOTAS_TENANTS) {
+  if (featureFlags.isEnabled(featureFlags.FeatureFlag.GOOGLE_SHEETS)) {
-    const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.split(",")
+    defs[SourceNames.GOOGLE_SHEETS] = googlesheets.schema
-    const tenantId = getTenantId()
-    if (excludedTenants.includes(tenantId)) {
-      defs[SourceNames.GOOGLE_SHEETS] = googlesheets.schema
-    }
   }
 
   ctx.body = defs

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -175,9 +175,10 @@ module External {
     const thisRow: Row = {}
     // filter the row down to what is actually the row (not joined)
     for (let fieldName of Object.keys(table.schema)) {
-      const value = row[`${table.name}.${fieldName}`] || row[fieldName]
+      const pathValue = row[`${table.name}.${fieldName}`]
+      const value = pathValue != null ? pathValue : row[fieldName]
       // all responses include "select col as table.col" so that overlaps are handled
-      if (value) {
+      if (value != null) {
         thisRow[fieldName] = value
       }
     }

packages/server/src/api/controllers/view/exporters.js

@@ -6,7 +6,7 @@ exports.csv = function (headers, rows) {
       .map(header => {
         let val = row[header]
         val =
-          typeof val === "object"
+          typeof val === "object" && !(val instanceof Date)
             ? `"${JSON.stringify(val).replace(/"/g, "'")}"`
             : `"${val}"`
         return val.trim()

packages/server/src/automations/steps/queryRows.js

@@ -50,6 +50,7 @@ exports.definition = {
         limit: {
           type: "number",
           title: "Limit",
+          customType: "queryLimit",
         },
       },
       required: ["tableId"],

packages/server/src/db/utils.js

@@ -9,6 +9,8 @@ const {
   StaticDatabases,
   isDevAppID,
   isProdAppID,
+  getDevelopmentAppID,
+  generateAppID,
 } = require("@budibase/backend-core/db")
 
 const UNICODE_MAX = "\ufff0"
@@ -80,6 +82,8 @@ exports.UNICODE_MAX = UNICODE_MAX
 exports.SearchIndexes = SearchIndexes
 exports.AppStatus = AppStatus
 exports.BudibaseInternalDB = BudibaseInternalDB
+exports.generateAppID = generateAppID
+exports.generateDevAppID = getDevelopmentAppID
 
 exports.generateRoleID = generateRoleID
 exports.getRoleParams = getRoleParams
@@ -243,28 +247,6 @@ exports.getLinkParams = (otherProps = {}) => {
   return getDocParams(DocumentTypes.LINK, null, otherProps)
 }
 
-/**
- * Generates a new app ID.
- * @returns {string} The new app ID which the app doc can be stored under.
- */
-exports.generateAppID = (tenantId = null) => {
-  let id = `${DocumentTypes.APP}${SEPARATOR}`
-  if (tenantId) {
-    id += `${tenantId}${SEPARATOR}`
-  }
-  return `${id}${newid()}`
-}
-
-/**
- * Generates a development app ID from a real app ID.
- * @returns {string} the dev app ID which can be used for dev database.
- */
-exports.generateDevAppID = appId => {
-  const prefix = `${DocumentTypes.APP}${SEPARATOR}`
-  const rest = appId.split(prefix)[1]
-  return `${DocumentTypes.APP_DEV}${SEPARATOR}${rest}`
-}
-
 /**
  * Generates a new layout ID.
  * @returns {string} The new layout ID which the layout doc can be stored under.

packages/server/src/integrations/firebase.ts

@@ -92,13 +92,13 @@ module Firebase {
 
   class FirebaseIntegration implements IntegrationBase {
     private config: FirebaseConfig
-    private db: Firestore
+    private client: Firestore
 
     constructor(config: FirebaseConfig) {
       this.config = config
       if (config.serviceAccount) {
         const serviceAccount = JSON.parse(config.serviceAccount)
-        this.db = new Firestore({
+        this.client = new Firestore({
           projectId: serviceAccount.project_id,
           credentials: {
             client_email: serviceAccount.client_email,
@@ -106,7 +106,7 @@ module Firebase {
           },
         })
       } else {
-        this.db = new Firestore({
+        this.client = new Firestore({
           projectId: config.projectId,
           credentials: {
             client_email: config.email,
@@ -118,7 +118,7 @@ module Firebase {
 
     async create(query: { json: object; extra: { [key: string]: string } }) {
       try {
-        const documentReference = this.db
+        const documentReference = this.client
           .collection(query.extra.collection)
           .doc()
         await documentReference.set({ ...query.json, id: documentReference.id })
@@ -133,7 +133,7 @@ module Firebase {
     async read(query: { json: object; extra: { [key: string]: string } }) {
       try {
         let snapshot
-        const collectionRef = this.db.collection(query.extra.collection)
+        const collectionRef = this.client.collection(query.extra.collection)
         if (
           query.extra.filterField &&
           query.extra.filter &&
@@ -164,19 +164,19 @@ module Firebase {
       extra: { [key: string]: string }
     }) {
       try {
-        await this.db
+        await this.client
           .collection(query.extra.collection)
           .doc(query.json.id)
           .update(query.json)
 
         return (
-          await this.db
+          await this.client
             .collection(query.extra.collection)
             .doc(query.json.id)
             .get()
         ).data()
       } catch (err) {
-        console.error("Error writing to firebase", err)
+        console.error("Error writing to Firestore", err)
         throw err
       }
     }
@@ -186,13 +186,13 @@ module Firebase {
       extra: { [key: string]: string }
     }) {
       try {
-        await this.db
+        await this.client
           .collection(query.extra.collection)
           .doc(query.json.id)
           .delete()
         return true
       } catch (err) {
-        console.error("Error writing to mongodb", err)
+        console.error("Error deleting from Firestore", err)
         throw err
       }
     }

packages/server/src/integrations/index.ts

@@ -46,6 +46,7 @@ const INTEGRATIONS = {
   [SourceNames.FIREBASE]: firebase.integration,
   [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
   [SourceNames.REDIS]: redis.integration,
+  [SourceNames.FIREBASE]: firebase.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

packages/server/src/integrations/postgres.ts

@@ -16,7 +16,7 @@ import {
 import { DatasourcePlus } from "./base/datasourcePlus"
 
 module PostgresModule {
-  const { Pool } = require("pg")
+  const { Client } = require("pg")
   const Sql = require("./base/sql")
   const { escapeDangerousCharacters } = require("../utilities")
 
@@ -104,7 +104,6 @@ module PostgresModule {
   }
 
   class PostgresIntegration extends Sql implements DatasourcePlus {
-    static pool: any
     private readonly client: any
     private readonly config: PostgresConfig
     private index: number = 1
@@ -136,11 +135,7 @@ module PostgresModule {
             }
           : undefined,
       }
-      if (!this.pool) {
+      this.client = new Client(newConfig)
-        this.pool = new Pool(newConfig)
-      }
-
-      this.client = this.pool
       this.setSchema()
     }
 
@@ -152,7 +147,7 @@ module PostgresModule {
       return parts.join(" || ")
     }
 
-    async internalQuery(query: SqlQuery) {
+    async internalQuery(query: SqlQuery, close: boolean = true) {
       const client = this.client
       this.index = 1
       // need to handle a specific issue with json data types in postgres,
@@ -169,18 +164,20 @@ module PostgresModule {
       try {
         return await client.query(query.sql, query.bindings || [])
       } catch (err) {
+        await this.client.end()
         // @ts-ignore
         throw new Error(err)
+      } finally {
+        if (close) await this.client.end()
       }
     }
 
-    setSchema() {
+    async setSchema() {
+      await this.client.connect()
       if (!this.config.schema) {
         this.config.schema = "public"
       }
-      this.client.on("connect", (client: any) => {
+      this.client.query(`SET search_path TO ${this.config.schema}`)
-        client.query(`SET search_path TO ${this.config.schema}`)
-      })
       this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
     }
 
@@ -210,44 +207,52 @@ module PostgresModule {
         tableKeys = {}
       }
 
-      const columnsResponse = await this.client.query(this.COLUMNS_SQL)
+      try {
-      const tables: { [key: string]: Table } = {}
+        const columnsResponse = await this.client.query(this.COLUMNS_SQL)
 
-      for (let column of columnsResponse.rows) {
+        const tables: { [key: string]: Table } = {}
-        const tableName: string = column.table_name
-        const columnName: string = column.column_name
 
-        // table key doesn't exist yet
+        for (let column of columnsResponse.rows) {
-        if (!tables[tableName] || !tables[tableName].schema) {
+          const tableName: string = column.table_name
-          tables[tableName] = {
+          const columnName: string = column.column_name
-            _id: buildExternalTableId(datasourceId, tableName),
+
-            primary: tableKeys[tableName] || [],
+          // table key doesn't exist yet
-            name: tableName,
+          if (!tables[tableName] || !tables[tableName].schema) {
-            schema: {},
+            tables[tableName] = {
+              _id: buildExternalTableId(datasourceId, tableName),
+              primary: tableKeys[tableName] || [],
+              name: tableName,
+              schema: {},
+            }
+          }
+
+          const identity = !!(
+            column.identity_generation ||
+            column.identity_start ||
+            column.identity_increment
+          )
+          const hasDefault =
+            typeof column.column_default === "string" &&
+            column.column_default.startsWith("nextval")
+          const isGenerated =
+            column.is_generated && column.is_generated !== "NEVER"
+          const isAuto: boolean = hasDefault || identity || isGenerated
+          tables[tableName].schema[columnName] = {
+            autocolumn: isAuto,
+            name: columnName,
+            ...convertSqlType(column.data_type),
           }
         }
 
-        const identity = !!(
+        const final = finaliseExternalTables(tables, entities)
-          column.identity_generation ||
+        this.tables = final.tables
-          column.identity_start ||
+        this.schemaErrors = final.errors
-          column.identity_increment
+      } catch (err) {
-        )
+        // @ts-ignore
-        const hasDefault =
+        throw new Error(err)
-          typeof column.column_default === "string" &&
+      } finally {
-          column.column_default.startsWith("nextval")
+        await this.client.end()
-        const isGenerated =
-          column.is_generated && column.is_generated !== "NEVER"
-        const isAuto: boolean = hasDefault || identity || isGenerated
-        tables[tableName].schema[columnName] = {
-          autocolumn: isAuto,
-          name: columnName,
-          ...convertSqlType(column.data_type),
-        }
       }
-
-      const final = finaliseExternalTables(tables, entities)
-      this.tables = final.tables
-      this.schemaErrors = final.errors
     }
 
     async create(query: SqlQuery | string) {
@@ -276,8 +281,9 @@ module PostgresModule {
       if (Array.isArray(input)) {
         const responses = []
         for (let query of input) {
-          responses.push(await this.internalQuery(query))
+          responses.push(await this.internalQuery(query, false))
         }
+        await this.client.end()
         return responses
       } else {
         const response = await this.internalQuery(input)

packages/server/src/integrations/tests/firebase.spec.js

@@ -0,0 +1,92 @@
+const firebase = require("@google-cloud/firestore")
+const FirebaseIntegration = require("../firebase")
+jest.mock("@google-cloud/firestore")
+
+class TestConfiguration {
+  constructor(config = {}) {
+    this.integration = new FirebaseIntegration.integration(config) 
+  }
+}
+
+describe("Firebase Integration", () => {
+  let config 
+  let tableName = "Users"
+
+  beforeEach(() => {
+    config = new TestConfiguration({
+      serviceAccount: "{}"
+    })
+  })
+
+  it("calls the create method with the correct params", async () => {
+    await config.integration.create({ 
+      table: tableName,
+      json: {
+        Name: "Test Name"
+      },
+      extra: {
+        collection: "test"
+      }
+    })
+    expect(config.integration.client.collection).toHaveBeenCalledWith("test")
+    expect(config.integration.client.set).toHaveBeenCalledWith({ 
+      Name: "Test Name",
+      id: "test_id"
+    })
+  })
+
+  it("calls the read method with the correct params", async () => {
+    const response = await config.integration.read({ 
+      table: tableName,
+      json: {
+        Name: "Test"
+      },
+      extra: {
+        collection: "test",
+        filterField: "field",
+        filter: "==",
+        filterValue: "value",
+      }
+    })
+    expect(config.integration.client.collection).toHaveBeenCalledWith("test")
+    expect(config.integration.client.where).toHaveBeenCalledWith("field", "==", "value")
+    expect(response).toEqual([{ result: "test"}])
+  })
+
+  it("calls the update method with the correct params", async () => {
+    const response = await config.integration.update({ 
+      table: tableName,
+      json: {
+        id: "test",
+        Name: "Test"
+      },
+      extra: {
+        collection: "test"
+      }
+    })
+    expect(config.integration.client.collection).toHaveBeenCalledWith("test")
+    expect(config.integration.client.update).toHaveBeenCalledWith({
+      Name: "Test",
+      id: "test"
+    })
+    expect(response).toEqual({
+      result: "test"
+    })
+  })
+
+  it("calls the delete method with the correct params", async () => {
+    const response = await config.integration.delete({ 
+      table: tableName,
+      json: {
+        id: "test",
+        Name: "Test"
+      },
+      extra: {
+        collection: "test"
+      }
+    })
+    expect(config.integration.client.collection).toHaveBeenCalledWith("test")
+    expect(config.integration.client.doc).toHaveBeenCalledWith("test")
+    expect(config.integration.client.delete).toHaveBeenCalled()
+  })
+})

packages/server/src/integrations/tests/postgres.spec.js

@@ -15,10 +15,6 @@ describe("Postgres Integration", () => {
     config = new TestConfiguration()
   })
 
-  it("calls the connection callback", async () => {
-    expect(pg.on).toHaveBeenCalledWith('connect', expect.anything())
-  })
-
   it("calls the create method with the correct params", async () => {
     const sql = "insert into users (name, age) values ('Joe', 123);"
     await config.integration.create({

packages/server/yarn.lock

@@ -1014,10 +1014,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.138":
+"@budibase/backend-core@1.0.147":
-  version "1.0.138"
+  version "1.0.147"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.138.tgz#5297d6cf5b9ec8c15f0a6df4c7d8273b8ac900f0"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.147.tgz#fc93a803e97e304170b33bd28b4f5deda32bec18"
-  integrity sha512-1qN/5urKX8bBXwEz266Z94rco8dTI7VqIh75m8ZcqrAfoUpjvZJS76gZxfc5U/QWPwrgVFnLtYvnEjaLbGEflg==
+  integrity sha512-0GcF9G/tTAsko9g352MB8K3EtMTVb7v7Av6RdsYyKBdVtOD42HKFzSOD0n/RQUL4v70YByiu99zJAB6z1m1Pcg==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
@@ -1091,12 +1091,12 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/pro@1.0.138":
+"@budibase/pro@1.0.147":
-  version "1.0.138"
+  version "1.0.147"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.138.tgz#cacbebe5ce93eb533af62a794a638944c2c61544"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.147.tgz#c1ec9d92ffaa40285a83c57ddbec1f32802cd9a1"
-  integrity sha512-4ABlUZvl2h8sd8awJATf3KJeoFWV/8SoqdbKiH1ICdUcM/6dad7nhbJ15QqJL+Uuh/+mN2yEbr8V6Un2+yF+CA==
+  integrity sha512-e4im6Byqeeit/QFHkVhVdRmgIlJJY/W06cQrOuiG/1ngO4PGnXeJqtjQjHVfvchD5Xi3y1MgQ4AtH2Bzb5LEhw==
   dependencies:
-    "@budibase/backend-core" "1.0.138"
+    "@budibase/backend-core" "1.0.147"
     node-fetch "^2.6.1"
 
 "@budibase/standard-components@^0.9.139":

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.0.143-alpha.2",
+  "version": "1.0.151-alpha.2",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.143-alpha.2",
+  "version": "1.0.151-alpha.2",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -31,9 +31,9 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.143-alpha.2",
+    "@budibase/backend-core": "^1.0.151-alpha.2",
-    "@budibase/pro": "1.0.143-alpha.2",
+    "@budibase/pro": "1.0.151-alpha.2",
-    "@budibase/string-templates": "^1.0.143-alpha.2",
+    "@budibase/string-templates": "^1.0.151-alpha.2",
     "@koa/router": "^8.0.0",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "^0.3.0",

packages/worker/yarn.lock

@@ -293,10 +293,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.138":
+"@budibase/backend-core@1.0.147":
-  version "1.0.138"
+  version "1.0.147"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.138.tgz#5297d6cf5b9ec8c15f0a6df4c7d8273b8ac900f0"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.147.tgz#fc93a803e97e304170b33bd28b4f5deda32bec18"
-  integrity sha512-1qN/5urKX8bBXwEz266Z94rco8dTI7VqIh75m8ZcqrAfoUpjvZJS76gZxfc5U/QWPwrgVFnLtYvnEjaLbGEflg==
+  integrity sha512-0GcF9G/tTAsko9g352MB8K3EtMTVb7v7Av6RdsYyKBdVtOD42HKFzSOD0n/RQUL4v70YByiu99zJAB6z1m1Pcg==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
@@ -321,12 +321,12 @@
     uuid "^8.3.2"
     zlib "^1.0.5"
 
-"@budibase/pro@1.0.138":
+"@budibase/pro@1.0.147":
-  version "1.0.138"
+  version "1.0.147"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.138.tgz#cacbebe5ce93eb533af62a794a638944c2c61544"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.147.tgz#c1ec9d92ffaa40285a83c57ddbec1f32802cd9a1"
-  integrity sha512-4ABlUZvl2h8sd8awJATf3KJeoFWV/8SoqdbKiH1ICdUcM/6dad7nhbJ15QqJL+Uuh/+mN2yEbr8V6Un2+yF+CA==
+  integrity sha512-e4im6Byqeeit/QFHkVhVdRmgIlJJY/W06cQrOuiG/1ngO4PGnXeJqtjQjHVfvchD5Xi3y1MgQ4AtH2Bzb5LEhw==
   dependencies:
-    "@budibase/backend-core" "1.0.138"
+    "@budibase/backend-core" "1.0.147"
     node-fetch "^2.6.1"
 
 "@cspotcode/source-map-consumer@0.8.0":

pull_request_template.md

@@ -1,8 +1,12 @@
 ## Description
 _Describe the problem or feature in addition to a link to the relevant github issues._
 
+Addresses: 
+- `<Enter the Link to the issue(s) this PR addresses>`
+- ...more if required
+
 ## Screenshots
-_If a UI facing feature, some screenshots of the new functionality._
+_If a UI facing feature, a short video of the happy path, and some screenshots of the new functionality._