Merge branch 'develop' of github.com:Budibase/budibase into new-design-ui-dirty

.github/stale.yml

@@ -14,7 +14,6 @@ staleLabel: stale
 # Comment to post when marking an issue as stale. Set to `false` to disable
 markComment: >
   This issue has been automatically marked as stale because it has not had
-  recent activity. It will be closed if no further activity occurs. Thank you
+  recent activity.
-  for your contributions.
 # Comment to post when closing a stale issue. Set to `false` to disable
 closeComment: false

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.105-alpha.35",
+  "version": "1.0.105-alpha.38",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.105-alpha.35",
+  "version": "1.0.105-alpha.38",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",

packages/backend-core/src/environment.js

@@ -22,7 +22,8 @@ module.exports = {
   MINIO_URL: process.env.MINIO_URL,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
-  ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
+  ACCOUNT_PORTAL_URL:
+    process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app",
   ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY,
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
   SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.105-alpha.35",
+  "version": "1.0.105-alpha.38",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "^1.0.105-alpha.35",
+    "@budibase/string-templates": "^1.0.105-alpha.38",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/builder/cypress/integration/autoScreensUI.spec.js

@@ -4,12 +4,44 @@ filterTests(['smoke', 'all'], () => {
   context("Auto Screens UI", () => {
     before(() => {
       cy.login()
-        cy.createTestApp()
     })
 
+    it("should disable the autogenerated screen options if no sources are available", () => {
+      cy.createApp("First Test App", false)
+      
+      cy.closeModal();
+
+      cy.contains("Design").click()
+      cy.get("[aria-label=AddCircle]").click()
+      cy.get(".spectrum-Modal").within(() => {
+        cy.get(".item.disabled").contains("Autogenerated screens")
+        cy.get(".confirm-wrap .spectrum-Button").should('be.disabled')
+      })
+
+      cy.deleteAllApps()
+    });
+
+    it("should not display incompatible sources", () => {
+      cy.createApp("Test App")
+
+      cy.selectExternalDatasource("REST")
+      cy.selectExternalDatasource("S3")
+      cy.get(".spectrum-Modal").within(() => {
+        cy.get(".spectrum-Button").contains("Save and continue to query").click({ force : true })
+      })
+      
+      cy.navigateToAutogeneratedModal()
+
+      cy.get('.data-source-entry').should('have.length', 1)
+      cy.get('.data-source-entry')
+
+      cy.deleteAllApps()
+    });
+      
     it("should generate internal table screens", () => {
-            // Create autogenerated screens from the internal table
+      cy.createTestApp()
-            cy.createAutogeneratedScreens(["Cypress Tests"])
+      // Create Autogenerated screens from the internal table
+      cy.createDatasourceScreen(["Cypress Tests"])
       // Confirm screens have been auto generated
       cy.get(".nav-items-container").contains("cypress-tests").click({ force: true })
       cy.get(".nav-items-container").should('contain', 'cypress-tests/:id')
@@ -21,8 +53,8 @@ filterTests(['smoke', 'all'], () => {
       const initialTable = "Cypress Tests"
       const secondTable = "Table Two"
       cy.createTable(secondTable)
-            // Create autogenerated screens from the internal tables
+      // Create Autogenerated screens from the internal tables
-            cy.createAutogeneratedScreens([initialTable, secondTable])
+      cy.createDatasourceScreen([initialTable, secondTable])
       // Confirm screens have been auto generated
       cy.get(".nav-items-container").contains("cypress-tests").click({ force: true })
       // Previously generated tables are suffixed with numbers - as expected
@@ -33,6 +65,25 @@ filterTests(['smoke', 'all'], () => {
         .and('contain', 'table-two/new/row')
     })
 
+    it("should generate multiple internal table screens with the same screen access level", () => {
+      //The tables created in the previous step still exist
+      cy.createTable("Table Three")
+      cy.createTable("Table Four")
+      cy.createDatasourceScreen(["Table Three", "Table Four"], "Admin")
+
+      cy.get(".nav-items-container").contains("table-three").click()
+      cy.get(".nav-items-container").should('contain', 'table-three/:id')
+      .and('contain', 'table-three/new/row')
+
+      cy.get(".nav-items-container").contains("table-four").click()
+      cy.get(".nav-items-container").should('contain', 'table-four/:id')
+      .and('contain', 'table-four/new/row')
+
+      //The access level should now be set to admin. Previous screens should be filtered.
+      cy.get(".nav-items-container").contains("table-two").should('not.exist')
+      cy.get(".nav-items-container").contains("cypress-tests").should('not.exist')
+    })
+
     if (Cypress.env("TEST_ENV")) {
       it("should generate data source screens", () => {
         // Using MySQL data source for testing this
@@ -40,8 +91,9 @@ filterTests(['smoke', 'all'], () => {
         // Select & configure MySQL data source
         cy.selectExternalDatasource(datasource)
         cy.addDatasourceConfig(datasource)
-                // Create autogenerated screens from a MySQL table - MySQL contains books table
+        // Create Autogenerated screens from a MySQL table - MySQL contains books table
-                cy.createAutogeneratedScreens(["books"])
+        cy.createDatasourceScreen(["books"])
+        
         cy.get(".nav-items-container").contains("books").click()
         cy.get(".nav-items-container").should('contain', 'books/:id')
           .and('contain', 'books/new/row')

packages/builder/cypress/integration/createBinding.spec.js

@@ -26,7 +26,7 @@ filterTests(['smoke', 'all'], () => {
 
     it("should add a URL param binding", () => {
       const paramName = "foo"
-      cy.createScreen("Test Param", `/test/:${paramName}`)
+      cy.createScreen(`/test/:${paramName}`)
       cy.addComponent("Elements", "Paragraph").then(componentId => {
         addSettingBinding("text", `URL.${paramName}`)
         // The builder preview pages don't have a real URL, so all we can do

packages/builder/cypress/integration/createScreen.js

@@ -9,17 +9,33 @@ filterTests(["smoke", "all"], () => {
     })
 
     it("Should successfully create a screen", () => {
-      cy.createScreen("Test Screen", "/test")
+      cy.createScreen("/test")
       cy.get(".nav-items-container").within(() => {
         cy.contains("/test").should("exist")
       })
     })
 
     it("Should update the url", () => {
-      cy.createScreen("Test Screen", "test with spaces")
+      cy.createScreen("test with spaces")
       cy.get(".nav-items-container").within(() => {
         cy.contains("/test-with-spaces").should("exist")
       })
     })
+
+    it("Should create a blank screen with the selected access level", () => {
+      cy.createScreen("admin only", "Admin")
+
+      cy.get(".nav-items-container").within(() => {
+        cy.contains("/admin-only").should("exist")
+      })
+
+      cy.createScreen("open to all", "Public")
+
+      cy.get(".nav-items-container").within(() => {
+        cy.contains("/open-to-all").should("exist")
+        //The access level should now be set to admin. Previous screens should be filtered.
+        cy.get(".nav-item").contains("/test-screen").should("not.exist")
+      })
+    })
   })
 })

packages/builder/cypress/support/commands.js

@@ -32,7 +32,17 @@ Cypress.Commands.add("login", () => {
   })
 })
 
-Cypress.Commands.add("createApp", name => {
+Cypress.Commands.add("closeModal", () => {
+  cy.get(".spectrum-Modal").within(() => {
+    cy.get(".close-icon").click()
+    cy.wait(500)
+  })
+})
+
+Cypress.Commands.add("createApp", (name, addDefaultTable) => {
+  const shouldCreateDefaultTable =
+    typeof addDefaultTable != "boolean" ? true : addDefaultTable
+
   cy.visit(`${Cypress.config().baseUrl}/builder`)
   cy.wait(500)
   cy.get(`[data-cy="create-app-btn"]`).click({ force: true })
@@ -51,7 +61,9 @@ Cypress.Commands.add("createApp", name => {
     cy.get(".spectrum-ButtonGroup").contains("Create app").click()
     cy.wait(10000)
   })
+  if (shouldCreateDefaultTable) {
     cy.createTable("Cypress Tests", true)
+  }
 })
 
 Cypress.Commands.add("deleteApp", name => {
@@ -135,7 +147,7 @@ Cypress.Commands.add("createTestApp", () => {
   const appName = "Cypress Tests"
   cy.deleteApp(appName)
   cy.createApp(appName, "This app is used for Cypress testing.")
-  cy.createScreen("home", "home")
+  cy.createScreen("home")
 })
 
 Cypress.Commands.add("createTestTableWithData", () => {
@@ -275,32 +287,98 @@ Cypress.Commands.add("navigateToDataSection", () => {
   cy.contains("Data").click()
 })
 
-Cypress.Commands.add("createScreen", (screenName, route) => {
+//Blank
+Cypress.Commands.add("createScreen", (route, accessLevelLabel) => {
   cy.contains("Design").click()
   cy.get("[aria-label=AddCircle]").click()
   cy.get(".spectrum-Modal").within(() => {
-    cy.get(".item").contains("Blank").click()
+    cy.get(".item").contains("Blank screen").click()
-    cy.get(".spectrum-Button").contains("Add screens").click({ force: true })
+    cy.get(".spectrum-Button").contains("Continue").click({ force: true })
     cy.wait(500)
   })
   cy.get(".spectrum-Dialog-grid").within(() => {
-    cy.get(".spectrum-Form-itemField").eq(0).type(screenName)
+    cy.get(".spectrum-Form-itemField").eq(0).type(route)
-    cy.get(".spectrum-Form-itemField").eq(1).type(route)
     cy.get(".spectrum-Button").contains("Continue").click({ force: true })
     cy.wait(1000)
   })
+
+  cy.get(".spectrum-Modal").within(() => {
+    if (accessLevelLabel) {
+      cy.get(".spectrum-Picker-label").click()
+      cy.wait(500)
+      cy.contains(accessLevelLabel).click()
+    }
+    cy.get(".spectrum-Button").contains("Done").click({ force: true })
+  })
 })
 
-Cypress.Commands.add("createAutogeneratedScreens", screenNames => {
+Cypress.Commands.add(
+  "createDatasourceScreen",
+  (datasourceNames, accessLevelLabel) => {
+    cy.contains("Design").click()
+    cy.get("[aria-label=AddCircle]").click()
+    cy.get(".spectrum-Modal").within(() => {
+      cy.get(".item").contains("Autogenerated screens").click()
+      cy.get(".spectrum-Button").contains("Continue").click({ force: true })
+      cy.wait(500)
+    })
+    cy.get(".spectrum-Modal [data-cy='data-source-modal']").within(() => {
+      for (let i = 0; i < datasourceNames.length; i++) {
+        cy.get(".data-source-entry").contains(datasourceNames[i]).click()
+        //Ensure the check mark is visible
+        cy.get(".data-source-entry")
+          .contains(datasourceNames[i])
+          .get(".data-source-check")
+          .should("exist")
+      }
+
+      cy.get(".spectrum-Button").contains("Confirm").click({ force: true })
+    })
+
+    cy.get(".spectrum-Modal").within(() => {
+      if (accessLevelLabel) {
+        cy.get(".spectrum-Picker-label").click()
+        cy.wait(500)
+        cy.contains(accessLevelLabel).click()
+      }
+      cy.get(".spectrum-Button").contains("Done").click({ force: true })
+    })
+
+    cy.contains("Design").click()
+  }
+)
+
+Cypress.Commands.add("navigateToAutogeneratedModal", () => {
   // Screen name must already exist within data source
   cy.contains("Design").click()
   cy.get("[aria-label=AddCircle]").click()
+  cy.get(".spectrum-Modal").within(() => {
+    cy.get(".item").contains("Autogenerated screens").click()
+    cy.get(".spectrum-Button").contains("Continue").click({ force: true })
+    cy.wait(500)
+  })
+})
+
+Cypress.Commands.add(
+  "createAutogeneratedScreens",
+  (screenNames, accessLevelLabel) => {
+    cy.navigateToAutogeneratedModal()
+
     for (let i = 0; i < screenNames.length; i++) {
-    cy.get(".item").contains(screenNames[i]).click()
+      cy.get(".data-source-entry").contains(screenNames[i]).click()
     }
-  cy.get(".spectrum-Button").contains("Add screens").click({ force: true })
+
+    cy.get(".spectrum-Modal").within(() => {
+      if (accessLevelLabel) {
+        cy.get(".spectrum-Picker-label").click()
+        cy.wait(500)
+        cy.contains(accessLevelLabel).click()
+      }
+      cy.get(".spectrum-Button").contains("Confirm").click({ force: true })
       cy.wait(4000)
     })
+  }
+)
 
 Cypress.Commands.add("addRow", values => {
   cy.contains("Create row").click()

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.105-alpha.35",
+  "version": "1.0.105-alpha.38",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.105-alpha.35",
+    "@budibase/bbui": "^1.0.105-alpha.38",
-    "@budibase/client": "^1.0.105-alpha.35",
+    "@budibase/client": "^1.0.105-alpha.38",
-    "@budibase/frontend-core": "^1.0.105-alpha.35",
+    "@budibase/frontend-core": "^1.0.105-alpha.38",
-    "@budibase/string-templates": "^1.0.105-alpha.35",
+    "@budibase/string-templates": "^1.0.105-alpha.38",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/analytics/constants.js

@@ -22,6 +22,7 @@ export const Events = {
   },
   SCREEN: {
     CREATED: "Screen Created",
+    CREATE_ROLE_UPDATED: "Changed Role On Screen Creation",
   },
   AUTOMATION: {
     CREATED: "Automation Created",

packages/builder/src/components/backend/DatasourceNavigator/icons/Redis.svelte

@@ -0,0 +1,49 @@
+<script>
+  export let width = "100"
+  export let height = "100"
+</script>
+
+<svg
+  {width}
+  {height}
+  viewBox="0 0 256 220"
+  fill="none"
+  xmlns="http://www.w3.org/2000/svg"
+>
+  <path
+    d="M245.97 168.943C232.308 176.064 161.536 205.163 146.469 213.018C131.402 220.874 123.032 220.798 111.129 215.108C99.227 209.418 23.913 178.996 10.346 172.511C3.566 169.271 0 166.535 0 163.951V138.075C0 138.075 98.05 116.73 113.879 111.051C129.707 105.372 135.199 105.167 148.669 110.101C162.141 115.037 242.687 129.569 256 134.445L255.994 159.955C255.996 162.513 252.924 165.319 245.97 168.943"
+    fill="#912626"
+  />
+  <path
+    d="M245.965 143.22C232.304 150.338 161.534 179.438 146.467 187.292C131.401 195.149 123.031 195.072 111.129 189.382C99.226 183.696 23.915 153.269 10.349 146.788C-3.21698 140.303 -3.50098 135.84 9.82502 130.622C23.151 125.402 98.049 96.017 113.88 90.338C129.708 84.661 135.199 84.454 148.669 89.39C162.14 94.324 232.488 122.325 245.799 127.2C259.115 132.081 259.626 136.1 245.965 143.22"
+    fill="#C6302B"
+  />
+  <path
+    d="M245.97 127.074C232.308 134.196 161.536 163.294 146.469 171.152C131.402 179.005 123.032 178.929 111.129 173.239C99.226 167.552 23.913 137.127 10.346 130.642C3.566 127.402 0 124.67 0 122.085V96.206C0 96.206 98.05 74.862 113.879 69.183C129.707 63.504 135.199 63.298 148.669 68.233C162.142 73.168 242.688 87.697 256 92.574L255.994 118.087C255.996 120.644 252.924 123.45 245.97 127.074Z"
+    fill="#912626"
+  />
+  <path
+    d="M245.965 101.351C232.304 108.471 161.534 137.569 146.467 145.426C131.401 153.28 123.031 153.203 111.129 147.513C99.226 141.827 23.915 111.401 10.349 104.919C-3.21698 98.436 -3.50098 93.972 9.82502 88.752C23.151 83.535 98.05 54.148 113.88 48.47C129.708 42.792 135.199 42.586 148.669 47.521C162.14 52.455 232.488 80.454 245.799 85.331C259.115 90.211 259.626 94.231 245.965 101.351"
+    fill="#C6302B"
+  />
+  <path
+    d="M245.97 83.653C232.308 90.773 161.536 119.873 146.469 127.731C131.402 135.585 123.032 135.508 111.129 129.818C99.226 124.131 23.913 93.705 10.346 87.223C3.566 83.98 0 81.247 0 78.665V52.785C0 52.785 98.05 31.442 113.879 25.764C129.707 20.084 135.199 19.88 148.669 24.814C162.142 29.749 242.688 44.278 256 49.155L255.994 74.667C255.996 77.222 252.924 80.028 245.97 83.653Z"
+    fill="#912626"
+  />
+  <path
+    d="M245.965 57.93C232.304 65.05 161.534 94.15 146.467 102.004C131.401 109.858 123.031 109.781 111.129 104.094C99.227 98.404 23.915 67.98 10.35 61.497C-3.21699 55.015 -3.49999 50.55 9.82501 45.331C23.151 40.113 98.05 10.73 113.88 5.04999C129.708 -0.629006 135.199 -0.833006 148.669 4.10199C162.14 9.03699 232.488 37.036 245.799 41.913C259.115 46.789 259.626 50.81 245.965 57.93"
+    fill="#C6302B"
+  />
+  <path
+    d="M159.283 32.757L137.273 35.042L132.346 46.898L124.388 33.668L98.9729 31.384L117.937 24.545L112.247 14.047L130.002 20.991L146.74 15.511L142.216 26.366L159.283 32.757V32.757ZM131.032 90.275L89.9549 73.238L148.815 64.203L131.032 90.275V90.275ZM74.0819 39.347C91.4569 39.347 105.542 44.807 105.542 51.541C105.542 58.277 91.4569 63.736 74.0819 63.736C56.7069 63.736 42.6219 58.276 42.6219 51.541C42.6219 44.807 56.7069 39.347 74.0819 39.347"
+    fill="white"
+  />
+  <path
+    d="M185.295 35.998L220.131 49.764L185.325 63.517L185.295 35.997"
+    fill="#621B1C"
+  />
+  <path
+    d="M146.755 51.243L185.295 35.998L185.325 63.517L181.546 64.995L146.755 51.243Z"
+    fill="#9A2928"
+  />
+</svg>

packages/builder/src/components/backend/DatasourceNavigator/icons/index.js

@@ -13,6 +13,7 @@ import Budibase from "./Budibase.svelte"
 import Oracle from "./Oracle.svelte"
 import GoogleSheets from "./GoogleSheets.svelte"
 import Firebase from "./Firebase.svelte"
+import Redis from "./Redis.svelte"
 
 export default {
   BUDIBASE: Budibase,
@@ -30,4 +31,5 @@ export default {
   ORACLE: Oracle,
   GOOGLE_SHEETS: GoogleSheets,
   FIREBASE: Firebase,
+  REDIS: Redis,
 }

packages/builder/src/components/integration/QueryViewer.svelte

@@ -72,7 +72,7 @@
       fields = response.schema
       notifications.success("Query executed successfully")
     } catch (error) {
-      notifications.error("Error previewing query")
+      notifications.error(`Query Error: ${error.message}`)
     }
   }
 

packages/builder/src/constants/backend/index.js

@@ -179,6 +179,7 @@ export const IntegrationTypes = {
   INTERNAL: "INTERNAL",
   GOOGLE_SHEETS: "GOOGLE_SHEETS",
   FIREBASE: "FIREBASE",
+  REDIS: "REDIS",
 }
 
 export const IntegrationNames = {
@@ -197,6 +198,7 @@ export const IntegrationNames = {
   [IntegrationTypes.INTERNAL]: "Internal",
   [IntegrationTypes.GOOGLE_SHEETS]: "Google Sheets",
   [IntegrationTypes.FIREBASE]: "Firebase",
+  [IntegrationTypes.REDIS]: "Redis",
 }
 
 export const SchemaTypeOptions = [

packages/builder/src/pages/builder/app/[application]/design/[screenId]/screens/_components/DatasourceModal.svelte

@@ -0,0 +1,199 @@
+<script>
+  import { store } from "builderStore"
+  import { ModalContent, Layout, notifications, Icon } from "@budibase/bbui"
+  import { tables, datasources } from "stores/backend"
+  import getTemplates from "builderStore/store/screenTemplates"
+  import ICONS from "components/backend/DatasourceNavigator/icons"
+  import { IntegrationNames } from "constants"
+  import { onMount } from "svelte"
+
+  export let onCancel
+  export let onConfirm
+  export let initalScreens = []
+
+  let selectedScreens = [...initalScreens]
+
+  const toggleScreenSelection = (table, datasource) => {
+    if (selectedScreens.find(s => s.table === table.name)) {
+      selectedScreens = selectedScreens.filter(
+        screen => screen.table !== table.name
+      )
+    } else {
+      let partialTemplates = getTemplates($store, $tables.list).reduce(
+        (acc, template) => {
+          if (template.table === table.name) {
+            template.datasource = datasource.name
+            acc.push(template)
+          }
+          return acc
+        },
+        []
+      )
+      selectedScreens = [...partialTemplates, ...selectedScreens]
+    }
+  }
+
+  const confirmDatasourceSelection = async () => {
+    await onConfirm({
+      templates: selectedScreens,
+    })
+  }
+
+  $: filteredSources = Array.isArray($datasources.list)
+    ? $datasources.list.reduce((acc, datasource) => {
+        if (
+          datasource.source !== IntegrationNames.REST &&
+          datasource["entities"]
+        ) {
+          acc.push(datasource)
+        }
+        return acc
+      }, [])
+    : []
+
+  onMount(async () => {
+    try {
+      await datasources.fetch()
+    } catch (error) {
+      notifications.error("Error fetching datasources")
+    }
+  })
+</script>
+
+<span data-cy="data-source-modal">
+  <ModalContent
+    title="Create CRUD Screens"
+    confirmText="Confirm"
+    cancelText="Back"
+    onConfirm={confirmDatasourceSelection}
+    {onCancel}
+    disabled={!selectedScreens.length}
+    size="L"
+  >
+    <Layout noPadding gap="S">
+      {#each filteredSources as datasource}
+        <div class="data-source-wrap">
+          <div class="data-source-header">
+            <div class="datasource-icon">
+              <svelte:component
+                this={ICONS[datasource.source]}
+                height="24"
+                width="24"
+              />
+            </div>
+            <div class="data-source-name">{datasource.name}</div>
+          </div>
+          {#if Array.isArray(datasource.entities)}
+            {#each datasource.entities.filter(table => table._id !== "ta_users") as table}
+              <div
+                class="data-source-entry"
+                class:selected={selectedScreens.find(
+                  x => x.table === table.name
+                )}
+                on:click={() => toggleScreenSelection(table, datasource)}
+              >
+                <svg
+                  width="16px"
+                  height="16px"
+                  class="spectrum-Icon"
+                  style="color: white"
+                  focusable="false"
+                >
+                  <use xlink:href="#spectrum-icon-18-Table" />
+                </svg>
+                {table.name}
+
+                {#if selectedScreens.find(x => x.table === table.name)}
+                  <span class="data-source-check">
+                    <Icon size="S" name="CheckmarkCircle" />
+                  </span>
+                {/if}
+              </div>
+            {/each}
+          {/if}
+          {#if datasource["entities"] && !Array.isArray(datasource.entities)}
+            {#each Object.keys(datasource.entities).filter(table => table._id !== "ta_users") as table_key}
+              <div
+                class="data-source-entry"
+                class:selected={selectedScreens.find(
+                  x => x.table === datasource.entities[table_key].name
+                )}
+                on:click={() =>
+                  toggleScreenSelection(
+                    datasource.entities[table_key],
+                    datasource
+                  )}
+              >
+                <svg
+                  width="16px"
+                  height="16px"
+                  class="spectrum-Icon"
+                  style="color: white"
+                  focusable="false"
+                >
+                  <use xlink:href="#spectrum-icon-18-Table" />
+                </svg>
+                {datasource.entities[table_key].name}
+
+                {#if selectedScreens.find(x => x.table === datasource.entities[table_key].name)}
+                  <span class="data-source-check">
+                    <Icon size="S" name="CheckmarkCircle" />
+                  </span>
+                {/if}
+              </div>
+            {/each}
+          {/if}
+        </div>
+      {/each}
+    </Layout>
+  </ModalContent>
+</span>
+
+<style>
+  .data-source-wrap {
+    padding-bottom: var(--spectrum-alias-item-padding-s);
+    display: grid;
+    grid-gap: var(--spacing-xs);
+  }
+
+  .data-source-header {
+    display: flex;
+    align-items: center;
+  }
+
+  .data-source-entry {
+    cursor: pointer;
+    grid-gap: var(--spectrum-alias-grid-margin-xsmall);
+    padding: var(--spectrum-alias-item-padding-s);
+    background: var(--spectrum-alias-background-color-primary);
+    transition: 0.3s all;
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    border-radius: 4px;
+    border-width: 1px;
+    display: flex;
+    align-items: center;
+  }
+
+  .data-source-entry:hover,
+  .selected {
+    background: var(--spectrum-alias-background-color-tertiary);
+  }
+
+  .data-source-name {
+    padding: var(--spectrum-alias-item-padding-s);
+    min-height: var(--spectrum-icon-size-s);
+  }
+
+  .data-source-entry .data-source-check {
+    margin-left: auto;
+  }
+
+  .data-source-entry :global(.spectrum-Icon) {
+    min-width: 16px;
+  }
+
+  .data-source-entry .data-source-check :global(.spectrum-Icon) {
+    color: var(--spectrum-global-color-green-600);
+    display: block;
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/screens/_components/NewScreenModal.svelte

@@ -1,117 +1,98 @@
 <script>
-  import { store } from "builderStore"
   import { tables } from "stores/backend"
-  import {
+  import { ModalContent, Body, Layout, Icon, Heading } from "@budibase/bbui"
-    ModalContent,
-    Body,
-    Detail,
-    Layout,
-    Icon,
-    ProgressCircle,
-  } from "@budibase/bbui"
-  import getTemplates from "builderStore/store/screenTemplates"
 
   export let onConfirm
   export let onCancel
-  export let showProgressCircle = false
 
-  const blankScreen = "createFromScratch"
+  let autoCreateModeKey = "autoCreate"
+  let blankScreenModeKey = "blankScreen"
 
-  let selectedScreens = []
+  let selectedScreenMode
-  let templates = getTemplates($store, $tables.list)
-
-  $: blankSelected = selectedScreens?.length === 1
-  $: autoSelected = selectedScreens?.length > 0 && !blankSelected
-
-  const toggleScreenSelection = table => {
-    if (selectedScreens.find(s => s.table === table.name)) {
-      selectedScreens = selectedScreens.filter(
-        screen => screen.table !== table.name
-      )
-    } else {
-      let partialTemplates = getTemplates($store, $tables.list).filter(
-        template => template.table === table.name
-      )
-      selectedScreens = [...partialTemplates, ...selectedScreens]
-    }
-  }
 
   const confirmScreenSelection = async () => {
-    await onConfirm(selectedScreens)
+    await onConfirm(selectedScreenMode)
   }
 </script>
 
 <div>
   <ModalContent
     title="Add screens"
-    confirmText="Add screens"
+    confirmText="Continue"
     cancelText="Cancel"
     onConfirm={confirmScreenSelection}
     {onCancel}
-    disabled={!selectedScreens.length}
+    disabled={!selectedScreenMode}
     size="L"
   >
-    <Body size="S">
-      Please select the screens you would like to add to your application.
-      Autogenerated screens come with CRUD functionality.
-    </Body>
     <Layout noPadding gap="S">
-      <Detail size="S">Blank screen</Detail>
       <div
-        class="item"
+        class="screen-type item"
-        class:selected={selectedScreens.find(x => x.id.includes(blankScreen))}
+        class:selected={selectedScreenMode == blankScreenModeKey}
-        on:click={() =>
+        on:click={() => {
-          toggleScreenSelection(templates.find(t => t.id === blankScreen))}
+          selectedScreenMode = blankScreenModeKey
-        class:disabled={autoSelected}
+        }}
       >
-        <div data-cy="blank-screen" class="content">
+        <div data-cy="blank-screen" class="content screen-type-wrap">
-          <div class="text">Blank</div>
+          <Icon name="WebPage" />
+          <div class="screen-type-text">
+            <Heading size="XS">Blank screen</Heading>
+            <Body size="S">Add a blank screen</Body>
+          </div>
         </div>
         <div
           style="color: var(--spectrum-global-color-green-600); float: right"
         >
-          {#if selectedScreens.find(x => x.id === blankScreen)}
+          {#if selectedScreenMode == blankScreenModeKey}
             <div class="checkmark-spacing">
-              <Icon size="S" name="CheckmarkCircleOutline" />
+              <Icon size="S" name="CheckmarkCircle" />
             </div>
           {/if}
         </div>
       </div>
-      {#if $tables.list.filter(table => table._id !== "ta_users").length > 0}
-        <Detail size="S">Autogenerated Screens</Detail>
 
-        {#each $tables.list.filter(table => table._id !== "ta_users") as table}
       <div
-            class:disabled={blankSelected}
+        class="screen-type item"
-            class:selected={selectedScreens.find(x => x.table === table.name)}
+        class:selected={selectedScreenMode == autoCreateModeKey}
-            on:click={() => toggleScreenSelection(table)}
+        on:click={() => {
-            class="item"
+          selectedScreenMode = autoCreateModeKey
+        }}
+        class:disabled={!$tables.list.filter(table => table._id !== "ta_users")
+          .length}
       >
-            <div class="content">
+        <div data-cy="autogenerated-screens" class="content screen-type-wrap">
-              <div class="text">{table.name}</div>
+          <Icon name="WebPages" />
+          <div class="screen-type-text">
+            <Heading size="XS">Autogenerated screens</Heading>
+            <Body size="S">
+              Add autogenerated screens with CRUD functionality to get a working
+              app quickly! (Requires a data source)
+            </Body>
+          </div>
         </div>
         <div
           style="color: var(--spectrum-global-color-green-600); float: right"
         >
-              {#if selectedScreens.find(x => x.table === table.name)}
+          {#if selectedScreenMode == autoCreateModeKey}
             <div class="checkmark-spacing">
-                  <Icon size="S" name="CheckmarkCircleOutline" />
+              <Icon size="S" name="CheckmarkCircle" />
             </div>
           {/if}
         </div>
       </div>
-        {/each}
-      {/if}
     </Layout>
-    <div slot="footer">
-      {#if showProgressCircle}
-        <div class="footer-progress"><ProgressCircle size="S" /></div>
-      {/if}
-    </div>
   </ModalContent>
 </div>
 
 <style>
+  .screen-type.item {
+    padding: var(--spectrum-alias-item-padding-xl);
+  }
+  .screen-type-wrap {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+  }
   .disabled {
     opacity: 0.3;
     pointer-events: none;
@@ -119,22 +100,9 @@
   .checkmark-spacing {
     margin-right: var(--spacing-m);
   }
-
   .content {
     letter-spacing: 0px;
   }
-
-  .footer-progress {
-    margin-top: var(--spacing-s);
-  }
-
-  .text {
-    font-weight: 600;
-    margin-left: var(--spacing-m);
-    font-size: 14px;
-    text-transform: capitalize;
-  }
-
   .item {
     cursor: pointer;
     grid-gap: var(--spectrum-alias-grid-margin-xsmall);
@@ -143,16 +111,22 @@
     transition: 0.3s all;
     border: 1px solid var(--spectrum-global-color-gray-300);
     border-radius: 4px;
-    box-sizing: border-box;
     border-width: 1px;
     display: flex;
     justify-content: space-between;
     align-items: center;
-    height: 60px;
   }
-
   .item:hover,
   .selected {
     background: var(--spectrum-alias-background-color-tertiary);
   }
+  .screen-type-wrap .screen-type-text {
+    padding-left: var(--spectrum-alias-item-padding-xl);
+  }
+  .screen-type-wrap :global(.spectrum-Icon) {
+    min-width: var(--spectrum-icon-size-m);
+  }
+  .screen-type-wrap :global(.spectrum-Heading) {
+    padding-bottom: var(--spectrum-alias-item-padding-s);
+  }
 </style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/screens/_components/ScreenDetailsModal.svelte

@@ -1,18 +1,23 @@
 <script>
-  import { ModalContent, Input, ProgressCircle } from "@budibase/bbui"
+  import { ModalContent, Input } from "@budibase/bbui"
   import sanitizeUrl from "builderStore/store/screenTemplates/utils/sanitizeUrl"
   import { selectedAccessRole, allScreens } from "builderStore"
   import { get } from "svelte/store"
 
   export let onConfirm
   export let onCancel
-  export let showProgressCircle = false
-  export let screenName
   export let screenUrl
   export let confirmText = "Continue"
 
   let routeError
   let touched = false
+  let screenAccessRole = $selectedAccessRole + ""
+
+  const appPrefix = "/app"
+
+  $: appUrl = screenUrl
+    ? `${window.location.origin}${appPrefix}${screenUrl}`
+    : `${window.location.origin}${appPrefix}`
 
   const routeChanged = event => {
     if (!event.detail.startsWith("/")) {
@@ -38,7 +43,6 @@
 
   const confirmScreenDetails = async () => {
     await onConfirm({
-      screenName,
       screenUrl,
     })
   }
@@ -51,24 +55,25 @@
   onConfirm={confirmScreenDetails}
   {onCancel}
   cancelText={"Back"}
-  disabled={!screenName || !screenUrl || routeError || !touched}
+  disabled={!screenAccessRole || !screenUrl || routeError || !touched}
 >
-  <Input label="Name" bind:value={screenName} />
   <Input
-    label="URL"
+    label="Enter a URL for the new screen"
     error={routeError}
     bind:value={screenUrl}
     on:change={routeChanged}
   />
-  <div slot="footer">
+  <div class="app-server" title={appUrl}>
-    {#if showProgressCircle}
+    {appUrl}
-      <div class="footer-progress"><ProgressCircle size="S" /></div>
-    {/if}
   </div>
 </ModalContent>
 
 <style>
-  .footer-progress {
+  .app-server {
-    margin-top: var(--spacing-s);
+    color: var(--spectrum-global-color-gray-600);
+    width: 320px;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
   }
 </style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/screens/_components/ScreenWizard.svelte

@@ -1,34 +1,46 @@
 <script>
   import ScreenDetailsModal from "./ScreenDetailsModal.svelte"
   import NewScreenModal from "./NewScreenModal.svelte"
+  import DatasourceModal from "./DatasourceModal.svelte"
   import sanitizeUrl from "builderStore/store/screenTemplates/utils/sanitizeUrl"
-  import { Modal, notifications } from "@budibase/bbui"
+  import { Modal, ModalContent, Select, notifications } from "@budibase/bbui"
   import { store, selectedAccessRole } from "builderStore"
   import analytics, { Events } from "analytics"
   import { get } from "svelte/store"
+  import getTemplates from "builderStore/store/screenTemplates"
+  import { tables, roles } from "stores/backend"
 
   let pendingScreen
-  let showProgressCircle = false
 
   // Modal refs
   let newScreenModal
   let screenDetailsModal
+  let datasourceModal
+  let screenAccessRoleModal
+
+  // Cache variables for workflow
+  let screenAccessRole = $selectedAccessRole + ""
+  let selectedTemplates = null
+  let blankScreenUrl = null
+
+  let screenMode = null
 
   // External handler to show the screen wizard
   export const showModal = () => {
-    newScreenModal.show()
+    selectedTemplates = null
+    blankScreenUrl = null
+    screenMode = null
 
+    newScreenModal.show()
     // Reset state when showing modal again
     pendingScreen = null
-    showProgressCircle = false
   }
 
   // Creates an array of screens, checking and sanitising their URLs
-  const createScreens = async screens => {
+  const createScreens = async ({ screens, screenAccessRole }) => {
     if (!screens?.length) {
       return
     }
-    showProgressCircle = true
 
     try {
       for (let screen of screens) {
@@ -46,7 +58,9 @@
         screen.routing.route = sanitizeUrl(screen.routing.route)
 
         // Use the currently selected role
-        screen.routing.roleId = get(selectedAccessRole) || "BASIC"
+        screen.routing.roleId = screenAccessRole
+          ? screenAccessRole
+          : get(selectedAccessRole) || "BASIC"
 
         // Create the screen
         await store.actions.screens.save(screen)
@@ -55,6 +69,8 @@
         if (screen.template) {
           analytics.captureEvent(Events.SCREEN.CREATED, {
             template: screen.template,
+            datasource: screen.datasource,
+            screenAccessRole,
           })
         }
 
@@ -69,8 +85,6 @@
     } catch (error) {
       notifications.error("Error creating screens")
     }
-
-    showProgressCircle = false
   }
 
   // Checks if any screens exist in the store with the given route and
@@ -98,38 +112,120 @@
   }
 
   // Handler for NewScreenModal
-  const confirmScreenSelection = async templates => {
+  const confirmScreenSelection = async mode => {
-    // Handle template selection
+    screenMode = mode
-    if (templates?.length > 1) {
+
-      // Autoscreens, so create immediately
+    if (mode == "autoCreate") {
-      const screens = templates.map(template => template.create())
+      datasourceModal.show()
-      await createScreens(screens)
     } else {
-      // Empty screen, so proceed to the next modal
+      let templates = getTemplates($store, $tables.list)
-      pendingScreen = templates[0].create()
+      const blankScreenTemplate = templates.find(
+        t => t.id === "createFromScratch"
+      )
+      pendingScreen = blankScreenTemplate.create()
       screenDetailsModal.show()
     }
   }
 
-  // Handler for ScreenDetailsModal
+  // Handler for DatasourceModal confirmation, move to screen access select
-  const confirmScreenDetails = async ({ screenName, screenUrl }) => {
+  const confirmScreenDatasources = async ({ templates }) => {
+    selectedTemplates = templates
+    screenAccessRoleModal.show()
+  }
+
+  // Handler for Datasource Screen Creation
+  const completeDatasourceScreenCreation = async () => {
+    // // Handle template selection
+    if (selectedTemplates?.length > 1) {
+      // Autoscreens, so create immediately
+      const screens = selectedTemplates.map(template => {
+        let screenTemplate = template.create()
+        screenTemplate.datasource = template.datasource
+        return screenTemplate
+      })
+      await createScreens({ screens, screenAccessRole })
+    }
+  }
+
+  const confirmScreenBlank = async ({ screenUrl }) => {
+    blankScreenUrl = screenUrl
+    screenAccessRoleModal.show()
+  }
+
+  // Submit request for a blank screen
+  const confirmBlankScreenCreation = async ({
+    screenUrl,
+    screenAccessRole,
+  }) => {
     if (!pendingScreen) {
       return
     }
-    pendingScreen.props._instanceName = screenName
     pendingScreen.routing.route = screenUrl
-    await createScreens([pendingScreen])
+    await createScreens({ screens: [pendingScreen], screenAccessRole })
+  }
+
+  // Submit screen config for creation.
+  const confirmScreenCreation = async () => {
+    if (screenMode === "blankScreen") {
+      confirmBlankScreenCreation({
+        screenUrl: blankScreenUrl,
+        screenAccessRole,
+      })
+    } else {
+      completeDatasourceScreenCreation()
+    }
+  }
+
+  const roleSelectBack = () => {
+    if (screenMode === "blankScreen") {
+      screenDetailsModal.show()
+    } else {
+      datasourceModal.show()
+    }
   }
 </script>
 
 <Modal bind:this={newScreenModal}>
-  <NewScreenModal onConfirm={confirmScreenSelection} {showProgressCircle} />
+  <NewScreenModal onConfirm={confirmScreenSelection} />
+</Modal>
+
+<Modal bind:this={datasourceModal}>
+  <DatasourceModal
+    onConfirm={confirmScreenDatasources}
+    onCancel={() => newScreenModal.show()}
+    initalScreens={!selectedTemplates ? [] : [...selectedTemplates]}
+  />
+</Modal>
+
+<Modal bind:this={screenAccessRoleModal}>
+  <ModalContent
+    title={"Create CRUD Screens"}
+    confirmText={"Done"}
+    cancelText={"Back"}
+    onConfirm={confirmScreenCreation}
+    onCancel={roleSelectBack}
+  >
+    Select which level of access you want your screens to have
+    <Select
+      bind:value={screenAccessRole}
+      on:change={() => {
+        analytics.captureEvent(Events.SCREEN.CREATE_ROLE_UPDATED, {
+          screenAccessRole,
+        })
+      }}
+      label="Access"
+      getOptionLabel={role => role.name}
+      getOptionValue={role => role._id}
+      getOptionColor={role => role.color}
+      options={$roles}
+    />
+  </ModalContent>
 </Modal>
 
 <Modal bind:this={screenDetailsModal}>
   <ScreenDetailsModal
-    {showProgressCircle}
+    onConfirm={confirmScreenBlank}
-    onConfirm={confirmScreenDetails}
     onCancel={() => newScreenModal.show()}
+    initialUrl={blankScreenUrl}
   />
 </Modal>

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.105-alpha.35",
+  "version": "1.0.105-alpha.38",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.105-alpha.35",
+  "version": "1.0.105-alpha.38",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.105-alpha.35",
+    "@budibase/bbui": "^1.0.105-alpha.38",
-    "@budibase/frontend-core": "^1.0.105-alpha.35",
+    "@budibase/frontend-core": "^1.0.105-alpha.38",
-    "@budibase/string-templates": "^1.0.105-alpha.35",
+    "@budibase/string-templates": "^1.0.105-alpha.38",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "1.0.105-alpha.35",
+  "version": "1.0.105-alpha.38",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "^1.0.105-alpha.35",
+    "@budibase/bbui": "^1.0.105-alpha.38",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.105-alpha.35",
+  "version": "1.0.105-alpha.38",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -68,10 +68,10 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.105-alpha.35",
+    "@budibase/backend-core": "^1.0.105-alpha.38",
-    "@budibase/client": "^1.0.105-alpha.35",
+    "@budibase/client": "^1.0.105-alpha.38",
-    "@budibase/pro": "1.0.105-alpha.34",
+    "@budibase/pro": "1.0.105-alpha.38",
-    "@budibase/string-templates": "^1.0.105-alpha.35",
+    "@budibase/string-templates": "^1.0.105-alpha.38",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -160,6 +160,7 @@
     "copyfiles": "^2.4.1",
     "docker-compose": "^0.23.6",
     "eslint": "^6.8.0",
+    "ioredis-mock": "^7.2.0",
     "is-wsl": "^2.2.0",
     "jest": "^27.0.5",
     "jest-openapi": "^0.14.2",

packages/server/src/definitions/datasource.ts

@@ -49,6 +49,7 @@ export enum SourceNames {
   ORACLE = "ORACLE",
   GOOGLE_SHEETS = "GOOGLE_SHEETS",
   FIREBASE = "FIREBASE",
+  REDIS = "REDIS",
 }
 
 export enum IncludeRelationships {

packages/server/src/integrations/index.ts

@@ -11,6 +11,7 @@ const arangodb = require("./arangodb")
 const rest = require("./rest")
 const googlesheets = require("./googlesheets")
 const firebase = require("./firebase")
+const redis = require("./redis")
 const { SourceNames } = require("../definitions/datasource")
 const environment = require("../environment")
 
@@ -26,6 +27,8 @@ const DEFINITIONS = {
   [SourceNames.MYSQL]: mysql.schema,
   [SourceNames.ARANGODB]: arangodb.schema,
   [SourceNames.REST]: rest.schema,
+  [SourceNames.FIREBASE]: firebase.schema,
+  [SourceNames.REDIS]: redis.schema,
 }
 
 const INTEGRATIONS = {
@@ -42,6 +45,7 @@ const INTEGRATIONS = {
   [SourceNames.REST]: rest.integration,
   [SourceNames.FIREBASE]: firebase.integration,
   [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
+  [SourceNames.REDIS]: redis.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

packages/server/src/integrations/redis.ts

@@ -0,0 +1,152 @@
+import {
+  DatasourceFieldTypes,
+  Integration,
+  QueryTypes,
+} from "../definitions/datasource"
+import Redis from "ioredis"
+
+module RedisModule {
+  interface RedisConfig {
+    host: string
+    port: number
+    username: string
+    password?: string
+  }
+
+  const SCHEMA: Integration = {
+    docs: "https://redis.io/docs/",
+    description: "",
+    friendlyName: "Redis",
+    datasource: {
+      host: {
+        type: "string",
+        required: true,
+        default: "localhost",
+      },
+      port: {
+        type: "number",
+        required: true,
+        default: 6379,
+      },
+      username: {
+        type: "string",
+        required: false,
+      },
+      password: {
+        type: "password",
+        required: false,
+      },
+    },
+    query: {
+      create: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          key: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          value: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          ttl: {
+            type: DatasourceFieldTypes.NUMBER,
+          },
+        },
+      },
+      read: {
+        readable: true,
+        type: QueryTypes.FIELDS,
+        fields: {
+          key: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+        },
+      },
+      delete: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          key: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+        },
+      },
+      command: {
+        readable: true,
+        displayName: "Redis Command",
+        type: QueryTypes.JSON,
+      },
+    },
+  }
+
+  class RedisIntegration {
+    private readonly config: RedisConfig
+    private client: any
+
+    constructor(config: RedisConfig) {
+      this.config = config
+      this.client = new Redis({
+        host: this.config.host,
+        port: this.config.port,
+        username: this.config.username,
+        password: this.config.password,
+      })
+    }
+
+    async disconnect() {
+      this.client.disconnect()
+    }
+
+    async redisContext(query: Function) {
+      try {
+        return await query()
+      } catch (err) {
+        throw new Error(`Redis error: ${err}`)
+      } finally {
+        this.disconnect()
+      }
+    }
+
+    async create(query: { key: string; value: string; ttl: number }) {
+      return this.redisContext(async () => {
+        const response = await this.client.set(query.key, query.value)
+        if (query.ttl) {
+          await this.client.expire(query.key, query.ttl)
+        }
+        return response
+      })
+    }
+
+    async read(query: { key: string }) {
+      return this.redisContext(async () => {
+        const response = await this.client.get(query.key)
+        return response
+      })
+    }
+
+    async delete(query: { key: string }) {
+      return this.redisContext(async () => {
+        const response = await this.client.del(query.key)
+        return response
+      })
+    }
+
+    async command(query: { json: string }) {
+      return this.redisContext(async () => {
+        const commands = query.json.trim().split(" ")
+        const pipeline = this.client.pipeline([commands])
+        const result = await pipeline.exec()
+        return {
+          response: result[0][1],
+        }
+      })
+    }
+  }
+
+  module.exports = {
+    schema: SCHEMA,
+    integration: RedisIntegration,
+  }
+}

packages/server/src/integrations/tests/redis.spec.js

@@ -0,0 +1,60 @@
+const Redis = require("ioredis-mock")
+const RedisIntegration = require("../redis")
+
+class TestConfiguration {
+  constructor(config = {}) {
+    this.integration = new RedisIntegration.integration(config) 
+    this.redis = new Redis({
+      data: {
+        test: 'test',
+        result: "1"
+      },
+    })
+    this.integration.client = this.redis
+  }
+}
+
+describe("Redis Integration", () => {
+  let config 
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  it("calls the create method with the correct params", async () => {
+    const body = {
+      key: "key",
+      value: "value"
+    }
+    const response = await config.integration.create(body)
+    expect(await config.redis.get("key")).toEqual("value")
+  })
+
+  it("calls the read method with the correct params", async () => {
+    const body = {
+      key: "test"
+    }
+    const response = await config.integration.read(body)
+    expect(response).toEqual("test")
+  })
+
+  it("calls the delete method with the correct params", async () => {
+    const body = {
+      key: "test"
+    }
+    await config.integration.delete(body)
+    expect(await config.redis.get(body.key)).toEqual(null)
+  })
+
+  it("calls the command method with the correct params", async () => {
+    const body = {
+      json: "KEYS *"
+    }
+
+    // ioredis-mock doesn't support pipelines
+    config.integration.client.pipeline = jest.fn(() => ({ exec: jest.fn(() => [[]]) }))
+
+    await config.integration.command(body)
+    expect(config.integration.client.pipeline).toHaveBeenCalledWith([["KEYS", "*"]])
+  })
+})

packages/server/src/threads/utils.js

@@ -72,6 +72,7 @@ exports.hasExtraData = response => {
   return (
     typeof response === "object" &&
     !Array.isArray(response) &&
+    response &&
     response.data != null &&
     response.info != null
   )

packages/server/yarn.lock

@@ -1014,10 +1014,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@^1.0.0":
+"@budibase/backend-core@1.0.105-alpha.38":
-  version "1.0.115"
+  version "1.0.105-alpha.38"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.115.tgz#c188dc9d4abe8f7d8088c54aeaa9f9c620bbbdba"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.105-alpha.38.tgz#399bc37877392f04c0072936d1c74d35d2997d6c"
-  integrity sha512-QGTaYyXWIInlKFzL514vDUh2gNob3Tckt1Lvtjk3Z5hhx2K7JZ5T2JwxSJ3qOBRKG6h2jI6HxlKEXPUefETAVA==
+  integrity sha512-IKVw3+a42Yea49Qc8vbVd+KZzOIAfgAFEohApJZSxqKA5UaFeWPJ343LQ7oC6jbYOkRVlGRnkrvXXa1jRggqbA==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
@@ -1087,12 +1087,12 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/pro@1.0.105-alpha.34":
+"@budibase/pro@1.0.105-alpha.38":
-  version "1.0.105-alpha.34"
+  version "1.0.105-alpha.38"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.105-alpha.34.tgz#75cdb7e29a22b6dc7e2d2846d8fce95be22bedb4"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.105-alpha.38.tgz#f025daf5667798083a7b0af301d6231dc3e58f00"
-  integrity sha512-OSPNjXYI2awQfKzGEDRjPVn/4R1Dd9xFhBwirrwq3BRrhBJbHg7uKRELCJfiLu7wsEqZSZqqbH5Ugyxcj8n+PQ==
+  integrity sha512-TXSov/c2KT7YuxZRspSd4iNPZPiJOdpl91ZTxGrqaPfK8PDmgk/XBWkHci+z1WLCvf6YY2kuQJGp1moldoLO1g==
   dependencies:
-    "@budibase/backend-core" "^1.0.0"
+    "@budibase/backend-core" "1.0.105-alpha.38"
     node-fetch "^2.6.1"
 
 "@budibase/standard-components@^0.9.139":
@@ -5844,6 +5844,20 @@ fecha@^4.2.0:
   resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.1.tgz#0a83ad8f86ef62a091e22bb5a039cd03d23eecce"
   integrity sha512-MMMQ0ludy/nBs1/o0zVOiKTpG7qMbonKUzjJgQFEuvq6INZ1OraKPRAWkBq5vlKLOUMpmNYG1JoN3oDPUQ9m3Q==
 
+fengari-interop@^0.1.3:
+  version "0.1.3"
+  resolved "https://registry.yarnpkg.com/fengari-interop/-/fengari-interop-0.1.3.tgz#3ad37a90e7430b69b365441e9fc0ba168942a146"
+  integrity sha512-EtZ+oTu3kEwVJnoymFPBVLIbQcCoy9uWCVnMA6h3M/RqHkUBsLYp29+RRHf9rKr6GwjubWREU1O7RretFIXjHw==
+
+fengari@^0.1.4:
+  version "0.1.4"
+  resolved "https://registry.yarnpkg.com/fengari/-/fengari-0.1.4.tgz#72416693cd9e43bd7d809d7829ddc0578b78b0bb"
+  integrity sha512-6ujqUuiIYmcgkGz8MGAdERU57EIluGGPSUgGPTsco657EHa+srq0S3/YUl/r9kx1+D+d4rGfYObd+m8K22gB1g==
+  dependencies:
+    readline-sync "^1.4.9"
+    sprintf-js "^1.1.1"
+    tmp "^0.0.33"
+
 fetch-cookie@0.10.1:
   version "0.10.1"
   resolved "https://registry.yarnpkg.com/fetch-cookie/-/fetch-cookie-0.10.1.tgz#5ea88f3d36950543c87997c27ae2aeafb4b5c4d4"
@@ -6960,6 +6974,16 @@ invert-kv@^2.0.0:
   resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02"
   integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==
 
+ioredis-mock@^7.2.0:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/ioredis-mock/-/ioredis-mock-7.2.0.tgz#48f006c07ef7f1f93f75e60d8f9035fa46c4ef0a"
+  integrity sha512-xzABBG3NhfDBGxH1KX9n6vs7WGNn9lhcxMT3b+vjynVImxlUV+vOXU+tjGzSUnGmx4IYllA8RqbXN8z6ROMPVA==
+  dependencies:
+    fengari "^0.1.4"
+    fengari-interop "^0.1.3"
+    redis-commands "^1.7.0"
+    standard-as-callback "^2.1.0"
+
 ioredis@^4.27.0:
   version "4.28.0"
   resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.0.tgz#5a2be3f37ff2075e2332f280eaeb02ab4d9ff0d3"
@@ -11122,6 +11146,11 @@ readdirp@~3.6.0:
   dependencies:
     picomatch "^2.2.1"
 
+readline-sync@^1.4.9:
+  version "1.4.10"
+  resolved "https://registry.yarnpkg.com/readline-sync/-/readline-sync-1.4.10.tgz#41df7fbb4b6312d673011594145705bf56d8873b"
+  integrity sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==
+
 realpath-native@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/realpath-native/-/realpath-native-1.1.0.tgz#2003294fea23fb0672f2476ebe22fcf498a2d65c"
@@ -11163,7 +11192,7 @@ rechoir@^0.7.0:
   dependencies:
     resolve "^1.9.0"
 
-redis-commands@1.7.0:
+redis-commands@1.7.0, redis-commands@^1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89"
   integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==
@@ -11974,7 +12003,7 @@ split2@^4.1.0:
   resolved "https://registry.yarnpkg.com/split2/-/split2-4.1.0.tgz#101907a24370f85bb782f08adaabe4e281ecf809"
   integrity sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==
 
-sprintf-js@^1.1.2:
+sprintf-js@^1.1.1, sprintf-js@^1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.2.tgz#da1765262bf8c0f571749f2ad6c26300207ae673"
   integrity sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.0.105-alpha.35",
+  "version": "1.0.105-alpha.38",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.105-alpha.35",
+  "version": "1.0.105-alpha.38",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -31,9 +31,9 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.105-alpha.35",
+    "@budibase/backend-core": "^1.0.105-alpha.38",
-    "@budibase/pro": "1.0.105-alpha.34",
+    "@budibase/pro": "1.0.105-alpha.38",
-    "@budibase/string-templates": "^1.0.105-alpha.35",
+    "@budibase/string-templates": "^1.0.105-alpha.38",
     "@koa/router": "^8.0.0",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "^0.3.0",

packages/worker/yarn.lock

@@ -286,10 +286,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@^1.0.0":
+"@budibase/backend-core@1.0.105-alpha.38":
-  version "1.0.115"
+  version "1.0.105-alpha.38"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.115.tgz#c188dc9d4abe8f7d8088c54aeaa9f9c620bbbdba"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.105-alpha.38.tgz#399bc37877392f04c0072936d1c74d35d2997d6c"
-  integrity sha512-QGTaYyXWIInlKFzL514vDUh2gNob3Tckt1Lvtjk3Z5hhx2K7JZ5T2JwxSJ3qOBRKG6h2jI6HxlKEXPUefETAVA==
+  integrity sha512-IKVw3+a42Yea49Qc8vbVd+KZzOIAfgAFEohApJZSxqKA5UaFeWPJ343LQ7oC6jbYOkRVlGRnkrvXXa1jRggqbA==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
@@ -310,12 +310,12 @@
     uuid "^8.3.2"
     zlib "^1.0.5"
 
-"@budibase/pro@1.0.105-alpha.34":
+"@budibase/pro@1.0.105-alpha.38":
-  version "1.0.105-alpha.34"
+  version "1.0.105-alpha.38"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.105-alpha.34.tgz#75cdb7e29a22b6dc7e2d2846d8fce95be22bedb4"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.105-alpha.38.tgz#f025daf5667798083a7b0af301d6231dc3e58f00"
-  integrity sha512-OSPNjXYI2awQfKzGEDRjPVn/4R1Dd9xFhBwirrwq3BRrhBJbHg7uKRELCJfiLu7wsEqZSZqqbH5Ugyxcj8n+PQ==
+  integrity sha512-TXSov/c2KT7YuxZRspSd4iNPZPiJOdpl91ZTxGrqaPfK8PDmgk/XBWkHci+z1WLCvf6YY2kuQJGp1moldoLO1g==
   dependencies:
-    "@budibase/backend-core" "^1.0.0"
+    "@budibase/backend-core" "1.0.105-alpha.38"
     node-fetch "^2.6.1"
 
 "@cspotcode/source-map-consumer@0.8.0":

scripts/pro/release.sh

@@ -5,18 +5,16 @@ if [[ -z "${CI}" ]]; then
   exit 0
 fi
 
-# Release pro as same version as budibase
+#############################################
+#                   SETUP                   #
+#############################################
+
+# Release pro with same version as budibase
 VERSION=$(jq -r .version lerna.json)
 echo "Version: $VERSION"
 COMMAND=$1
 echo "Command: $COMMAND"
 
-# Go to pro package
-cd ../budibase-pro
-
-# Install NPM credentials
-echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc 
-
 # Determine tag to use
 TAG=""
 if [[ $COMMAND == "develop" ]]; then
@@ -27,24 +25,70 @@ fi
 
 echo "Releasing version $VERSION"
 echo "Releasing tag $TAG"
+
+#############################################
+#                PRE-PUBLISH                #
+#############################################
+
+# Go to pro repo root
+cd ../budibase-pro
+
+# Install NPM credentials
+echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc 
+
+# Sync backend-core version in packages/pro/package.json
+# Ensures pro does not use out of date dependency
+cd packages/pro
+jq '.dependencies."@budibase/backend-core"="'$VERSION'"' package.json > package.json.tmp && mv package.json.tmp package.json
+
+# Go back to pro repo root
+cd -
+
+#############################################
+#                  PUBLISH                  #
+#############################################
+
 lerna publish $VERSION --yes --force-publish --dist-tag $TAG
 
-# reset main and types to point to src for dev
+#############################################
+#              POST-PUBLISH - PRO           #
+#############################################
+
+# Revert build changes on packages/pro/package.json
 cd packages/pro
 jq '.main = "src/index.ts" | .types = "src/index.ts"' package.json > package.json.tmp && mv package.json.tmp package.json
+
+# Go back to pro repo root
 cd -
+
+# Commit and push changes
 git add packages/pro/package.json
-git commit -m 'Prep dev'
+git commit -m "Prep next development iteration"
 git push 
 
+#############################################
+#            POST-PUBLISH - BUDIBASE        #
+#############################################
+
+# Go to budibase repo root
 cd ../budibase
 
-if [[ $COMMAND == "develop" ]]; then
+# Update pro version in packages/server/package.json
-  # Pin pro version for develop container build
-  echo "Pinning pro version"
 cd packages/server
 jq '.dependencies."@budibase/pro"="'$VERSION'"' package.json > package.json.tmp && mv package.json.tmp package.json
+
+# Go back to budibase repo root
 cd -
+
+# Update pro version in packages/worker/package.json
 cd packages/worker
 jq '.dependencies."@budibase/pro"="'$VERSION'"' package.json > package.json.tmp && mv package.json.tmp package.json
-fi
+
+# Go back to budibase repo root
+cd -
+
+# Commit and push changes
+git add packages/server/package.json
+git add packages/worker/package.json
+git commit -m "Update pro version to $VERSION"
+git push