Merge branch 'develop' into feature/oidc-support

hosting/scripts/linux/install-docker.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 echo "**** WARNING - not for production environments ****"
-# warning this is a convience script, for production installations install docker 
+# warning this is a convenience script, for production installations install docker
 # properly for your environment!
 curl -fsSL https://get.docker.com -o get-docker.sh
 sudo sh get-docker.sh

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.71",
+  "version": "0.9.74",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.71",
+  "version": "0.9.74",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/auth/src/objectStore/index.js

@@ -22,11 +22,13 @@ const CONTENT_TYPE_MAP = {
   html: "text/html",
   css: "text/css",
   js: "application/javascript",
+  json: "application/json",
 }
 const STRING_CONTENT_TYPES = [
   CONTENT_TYPE_MAP.html,
   CONTENT_TYPE_MAP.css,
   CONTENT_TYPE_MAP.js,
+  CONTENT_TYPE_MAP.json,
 ]
 
 // does normal sanitization and then swaps dev apps to apps

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.71",
+  "version": "0.9.74",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Form/Core/Select.svelte

@@ -31,7 +31,7 @@
       return ""
     }
 
-    // Render the label if the selected option is found, otherwide raw value
+    // Render the label if the selected option is found, otherwise raw value
     const index = options.findIndex(
       (option, idx) => getOptionValue(option, idx) === value
     )

packages/builder/CONTRIBUTING.md

@@ -9,7 +9,7 @@ Please read this if you are unfamiliar with it.
 
 * Please maintain the existing code style. 
 
-* Please try to keep your commits small and focussed.
+* Please try to keep your commits small and focused.
 
 * If the project diverges from your branch, please rebase instead of merging. This makes the commit graph easier to read.
 

packages/builder/cypress/setup.js

@@ -34,7 +34,7 @@ async function run() {
     process.exit(-1)
   }
 
-  // dont make this a variable or top level require
+  // don't make this a variable or top level require
   // it will cause environment module to be loaded prematurely
   const server = require("../../server/dist/app")
   process.env.PORT = WORKER_PORT

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.71",
+  "version": "0.9.74",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.71",
+    "@budibase/bbui": "^0.9.74",
-    "@budibase/client": "^0.9.71",
+    "@budibase/client": "^0.9.74",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.71",
+    "@budibase/string-templates": "^0.9.74",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/store/frontend.js

@@ -32,6 +32,10 @@ const INITIAL_FRONTEND_STATE = {
   layouts: [],
   screens: [],
   components: [],
+  clientFeatures: {
+    spectrumThemes: false,
+    intelligentLoading: false,
+  },
   currentFrontEndType: "none",
   selectedScreenId: "",
   selectedLayoutId: "",
@@ -56,6 +60,10 @@ export const getFrontendStore = () => {
         ...state,
         libraries: application.componentLibraries,
         components,
+        clientFeatures: {
+          ...state.clientFeatures,
+          ...components.features,
+        },
         name: application.name,
         description: application.description,
         appId: application.appId,
@@ -67,6 +75,8 @@ export const getFrontendStore = () => {
         appInstance: application.instance,
         clientLibPath,
         previousTopNavPath: {},
+        version: application.version,
+        revertableVersion: application.revertableVersion,
       }))
       await hostingStore.actions.fetch()
 

packages/builder/src/components/deploy/VersionModal.svelte

@@ -0,0 +1,120 @@
+<script>
+  import {
+    Icon,
+    Modal,
+    notifications,
+    ModalContent,
+    Body,
+    Button,
+  } from "@budibase/bbui"
+  import { store } from "builderStore"
+  import api from "builderStore/api"
+  import clientPackage from "@budibase/client/package.json"
+
+  let updateModal
+
+  $: appId = $store.appId
+  $: updateAvailable = clientPackage.version !== $store.version
+  $: revertAvailable = $store.revertableVersion != null
+
+  const refreshAppPackage = async () => {
+    const applicationPkg = await api.get(
+      `/api/applications/${appId}/appPackage`
+    )
+    const pkg = await applicationPkg.json()
+    if (applicationPkg.ok) {
+      await store.actions.initialise(pkg)
+    } else {
+      throw new Error(pkg)
+    }
+  }
+
+  const update = async () => {
+    try {
+      const response = await api.post(
+        `/api/applications/${appId}/client/update`
+      )
+      const json = await response.json()
+      if (response.status !== 200) {
+        throw json.message
+      }
+
+      // Don't wait for the async refresh, since this causes modal flashing
+      refreshAppPackage()
+      notifications.success(
+        `App updated successfully to version ${clientPackage.version}`
+      )
+    } catch (err) {
+      notifications.error(`Error updating app: ${err}`)
+    }
+  }
+
+  const revert = async () => {
+    try {
+      const revertableVersion = $store.revertableVersion
+      const response = await api.post(
+        `/api/applications/${appId}/client/revert`
+      )
+      const json = await response.json()
+      if (response.status !== 200) {
+        throw json.message
+      }
+
+      // Don't wait for the async refresh, since this causes modal flashing
+      refreshAppPackage()
+      notifications.success(
+        `App reverted successfully to version ${revertableVersion}`
+      )
+    } catch (err) {
+      notifications.error(`Error reverting app: ${err}`)
+    }
+    updateModal.hide()
+  }
+</script>
+
+<div class="icon-wrapper" class:highlight={updateAvailable}>
+  <Icon name="Refresh" hoverable on:click={updateModal.show} />
+</div>
+<Modal bind:this={updateModal}>
+  <ModalContent
+    title="App version"
+    confirmText="Update"
+    cancelText={updateAvailable ? "Cancel" : "Close"}
+    onConfirm={update}
+    showConfirmButton={updateAvailable}
+  >
+    <div slot="footer">
+      {#if revertAvailable}
+        <Button quiet secondary on:click={revert}>Revert</Button>
+      {/if}
+    </div>
+    {#if updateAvailable}
+      <Body size="S">
+        This app is currently using version <b>{$store.version}</b>, but version
+        <b>{clientPackage.version}</b> is available. Updates can contain new features,
+        performance improvements and bug fixes.
+      </Body>
+    {:else}
+      <Body size="S">
+        This app is currently using version <b>{$store.version}</b> which is the
+        latest version available.
+      </Body>
+    {/if}
+    {#if revertAvailable}
+      <Body size="S">
+        You can revert this app to version
+        <b>{$store.revertableVersion}</b>
+        if you're experiencing issues with the current version.
+      </Body>
+    {/if}
+  </ModalContent>
+</Modal>
+
+<style>
+  .icon-wrapper {
+    display: contents;
+  }
+  .icon-wrapper.highlight :global(svg) {
+    color: var(--spectrum-global-color-blue-600);
+  }
+</style>

packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte

@@ -74,14 +74,14 @@
     // Initialise the app when mounted
     iframe.contentWindow.addEventListener(
       "ready",
-      () => refreshContent(strippedJson),
+      () => {
-      { once: true }
+        // Display preview immediately if the intelligent loading feature
-    )
+        // is not supported
-
+        if (!$store.clientFeatures.intelligentLoading) {
-    // Display the client app once the iframe is initialised
+          loading = false
-    iframe.contentWindow.addEventListener(
+        }
-      "iframe-loaded",
+        refreshContent(strippedJson)
-      () => (loading = false),
+      },
       { once: true }
     )
 
@@ -106,11 +106,9 @@
         idToDelete = data.id
         confirmDeleteDialog.show()
       } else if (type === "preview-loaded") {
-        // We can use this in future to delay displaying the preview
+        // Wait for this event to show the client library if intelligent
-        // until the client app has actually initialised.
+        // loading is supported
-        // This makes a smoother loading experience, but is not backwards
+        loading = false
-        // compatible with old client library versions.
-        // So do nothing with this for now.
       } else {
         console.warning(`Client sent unknown event type: ${type}`)
       }

packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte

@@ -65,7 +65,7 @@
   }
 </script>
 
-{#if definition.editable !== false}
+{#if definition?.editable !== false}
   <ActionMenu>
     <div slot="control" class="icon">
       <Icon size="S" hoverable name="MoreSmallList" />

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte

@@ -48,7 +48,7 @@
           Add your first filter column.
         {:else}
           Results are filtered to only those which match all of the following
-          constaints.
+          constraints.
         {/if}
       </Body>
       <LuceneFilterBuilder bind:value={tempValue} {schemaFields} />

packages/builder/src/components/design/PropertiesPanel/PropertyControls/IconSelect/icons.js

@@ -1153,10 +1153,10 @@ export default {
     space: ["空格"],
     "page-separator": ["insert", "分页符", "插入"],
     "code-view": ["代码视图"],
-    "double-quotes-l": ["left", "quotaion marks", "双引号"],
+    "double-quotes-l": ["left", "quotation marks", "双引号"],
-    "double-quotes-r": ["right", "quotaion marks", "双引号"],
+    "double-quotes-r": ["right", "quotation marks", "双引号"],
-    "single-quotes-l": ["left", "quotaion marks", "单引号"],
+    "single-quotes-l": ["left", "quotation marks", "单引号"],
-    "single-quotes-r": ["right", "quotaion marks", "单引号"],
+    "single-quotes-r": ["right", "quotation marks", "单引号"],
     "table-2": ["表格"],
     subscript: ["角标", "下标", "脚注"],
     "subscript-2": ["角标", "下标", "脚注"],
@@ -1463,7 +1463,7 @@ export default {
     alipay: ["zhifubao", "支付宝"],
     amazon: ["亚马逊"],
     android: ["applications", "安卓", "应用"],
-    angularjs: ["angular", "programing framework"],
+    angularjs: ["angular", "programming framework"],
     "app-store": ["applications", "苹果应用商店"],
     apple: ["苹果"],
     baidu: ["du", "claw", "百度", "爪"],
@@ -1519,7 +1519,7 @@ export default {
     playstation: ["ps"],
     "product-hunt": ["product hunt"],
     qq: ["penguin", "tencent", "腾讯", "企鹅"],
-    reactjs: ["react", "programing framework", "facebook"],
+    reactjs: ["react", "programming framework", "facebook"],
     reddit: ["reddit"],
     remixicon: ["remix icon", "图标"],
     safari: ["safari浏览器"],
@@ -1543,7 +1543,7 @@ export default {
     unsplash: ["photos"],
     vimeo: ["视频"],
     visa: ["bank card", "银行卡"],
-    vuejs: ["vue", "programing framework"],
+    vuejs: ["vue", "programming framework"],
     wechat: ["微信"],
     "wechat-2": ["微信"],
     "wechat-pay": ["微信支付"],

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -4,6 +4,7 @@
   import { Icon, ActionGroup, Tabs, Tab } from "@budibase/bbui"
   import DeployModal from "components/deploy/DeployModal.svelte"
   import RevertModal from "components/deploy/RevertModal.svelte"
+  import VersionModal from "components/deploy/VersionModal.svelte"
   import { get } from "builderStore/api"
   import { isActive, goto, layout } from "@roxi/routify"
   import Logo from "assets/bb-emblem.svg"
@@ -80,6 +81,7 @@
         <ActionGroup />
       </div>
       <div class="toprightnav">
+        <VersionModal />
         <RevertModal />
         <Icon
           name="Play"

packages/builder/src/pages/builder/app/[application]/design/[assetType]/_layout.svelte

@@ -150,10 +150,14 @@
     {#if $currentAsset}
       <div class="preview-header">
         <ComponentSelectionList />
+        {#if $store.clientFeatures.spectrumThemes}
           <AppThemeSelect />
+        {/if}
       </div>
       <div class="preview-content">
+        {#key $store.version}
           <CurrentItemPreview />
+        {/key}
       </div>
     {/if}
   </div>

packages/builder/src/pages/builder/portal/manage/users/_components/BasicOnboardingModal.svelte

@@ -20,7 +20,7 @@
     if (res.status) {
       notifications.error(res.message)
     } else {
-      notifications.success("Succesfully created user")
+      notifications.success("Successfully created user")
     }
   }
 </script>

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.71",
+  "version": "0.9.74",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.71",
+  "version": "0.9.74",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -18,9 +18,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.71",
+    "@budibase/bbui": "^0.9.74",
-    "@budibase/standard-components": "^0.9.71",
+    "@budibase/standard-components": "^0.9.74",
-    "@budibase/string-templates": "^0.9.71",
+    "@budibase/string-templates": "^0.9.74",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/server/.gitignore

@@ -2,6 +2,7 @@ node_modules/
 myapps/
 .env
 builder/*
+client/*
 public/
 db/dev.db/
 dist

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.71",
+  "version": "0.9.74",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -13,7 +13,8 @@
     "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
     "test": "jest --coverage --maxWorkers=2",
     "test:watch": "jest --watch",
-    "build:docker": "docker build . -t app-service",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../standard-components/manifest.json client",
+    "build:docker": "yarn run predocker && docker build . -t app-service",
     "run:docker": "node dist/index.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
     "dev:stack:down": "node scripts/dev/manage.js down",
@@ -59,9 +60,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.71",
+    "@budibase/auth": "^0.9.74",
-    "@budibase/client": "^0.9.71",
+    "@budibase/client": "^0.9.74",
-    "@budibase/string-templates": "^0.9.71",
+    "@budibase/string-templates": "^0.9.74",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -114,7 +115,7 @@
   "devDependencies": {
     "@babel/core": "^7.14.3",
     "@babel/preset-env": "^7.14.4",
-    "@budibase/standard-components": "^0.9.71",
+    "@budibase/standard-components": "^0.9.74",
     "@jest/test-sequencer": "^24.8.0",
     "@types/bull": "^3.15.1",
     "@types/jest": "^26.0.23",

packages/server/scripts/integrations/mysql/init.sql

@@ -2,6 +2,8 @@ CREATE DATABASE IF NOT EXISTS main;
 USE main;
 CREATE TABLE Persons (
     PersonID int NOT NULL AUTO_INCREMENT,
+    CreatedAt datetime,
+    Age float,
     LastName varchar(255),
     FirstName varchar(255),
     Address varchar(255),
@@ -17,6 +19,6 @@ CREATE TABLE Tasks (
         FOREIGN KEY(PersonID)
 	    REFERENCES Persons(PersonID)
 );
-INSERT INTO Persons (FirstName, LastName, Address, City) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast');
+INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Mike', 'Hughes', 28.2, '123 Fake Street', 'Belfast', '2021-01-19 03:14:07');
 INSERT INTO Tasks (PersonID, TaskName) VALUES (1, 'assembling');
 INSERT INTO Tasks (PersonID, TaskName) VALUES (1, 'processing');

packages/server/src/api/controllers/application.js

@@ -33,6 +33,11 @@ const {
 } = require("../../utilities/workerRequests")
 const { clientLibraryPath } = require("../../utilities")
 const { getAllLocks } = require("../../utilities/redis")
+const {
+  updateClientLibrary,
+  backupClientLibrary,
+  revertClientLibrary,
+} = require("../../utilities/fileSystem/clientLibrary")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -231,27 +236,54 @@ exports.create = async function (ctx) {
 }
 
 exports.update = async function (ctx) {
-  const url = await getAppUrlIfNotInUse(ctx)
+  const data = await updateAppPackage(ctx, ctx.request.body, ctx.params.appId)
+  ctx.status = 200
+  ctx.body = data
+}
+
+exports.updateClient = async function (ctx) {
+  // Get current app version
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
+  const currentVersion = application.version
 
-  const data = ctx.request.body
+  // Update client library and manifest
-  const newData = { ...application, ...data, url }
+  if (!env.isTest()) {
-  if (ctx.request.body._rev !== application._rev) {
+    await backupClientLibrary(ctx.params.appId)
-    newData._rev = application._rev
+    await updateClientLibrary(ctx.params.appId)
   }
 
-  // the locked by property is attached by server but generated from
+  // Update versions in app package
-  // Redis, shouldn't ever store it
+  const appPackageUpdates = {
-  if (newData.lockedBy) {
+    version: packageJson.version,
-    delete newData.lockedBy
+    revertableVersion: currentVersion,
   }
-
+  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
-  const response = await db.put(newData)
-  data._rev = response.rev
-
   ctx.status = 200
-  ctx.body = response
+  ctx.body = data
+}
+
+exports.revertClient = async function (ctx) {
+  // Check app can be reverted
+  const db = new CouchDB(ctx.params.appId)
+  const application = await db.get(DocumentTypes.APP_METADATA)
+  if (!application.revertableVersion) {
+    ctx.throw(400, "There is no version to revert to")
+  }
+
+  // Update client library and manifest
+  if (!env.isTest()) {
+    await revertClientLibrary(ctx.params.appId)
+  }
+
+  // Update versions in app package
+  const appPackageUpdates = {
+    version: application.revertableVersion,
+    revertableVersion: null,
+  }
+  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
+  ctx.status = 200
+  ctx.body = data
 }
 
 exports.delete = async function (ctx) {
@@ -269,6 +301,23 @@ exports.delete = async function (ctx) {
   ctx.body = result
 }
 
+const updateAppPackage = async (ctx, appPackage, appId) => {
+  const url = await getAppUrlIfNotInUse(ctx)
+  const db = new CouchDB(appId)
+  const application = await db.get(DocumentTypes.APP_METADATA)
+
+  const newAppPackage = { ...application, ...appPackage, url }
+  if (appPackage._rev !== application._rev) {
+    newAppPackage._rev = application._rev
+  }
+
+  // the locked by property is attached by server but generated from
+  // Redis, shouldn't ever store it
+  delete newAppPackage.lockedBy
+
+  return await db.put(newAppPackage)
+}
+
 const createEmptyAppPackage = async (ctx, app) => {
   const db = new CouchDB(app.appId)
 

packages/server/src/api/controllers/component.js

@@ -20,6 +20,9 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
   const definitions = {}
   for (let { manifest, library } of componentManifests) {
     for (let key of Object.keys(manifest)) {
+      if (key === "features") {
+        definitions[key] = manifest[key]
+      } else {
         const fullComponentName = `${library}/${key}`.toLowerCase()
         definitions[fullComponentName] = {
           component: fullComponentName,
@@ -27,5 +30,6 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
         }
       }
     }
+  }
   ctx.body = definitions
 }

packages/server/src/api/controllers/view/viewBuilder.js

@@ -108,7 +108,7 @@ function parseEmitExpression(field, groupBy) {
  * calculation: an optional calculation to be performed over the view data.
  */
 function viewTemplate({ field, tableId, groupBy, filters = [], calculation }) {
-  // first filter can't have a conjuction
+  // first filter can't have a conjunction
   if (filters && filters.length > 0 && filters[0].conjunction) {
     delete filters[0].conjunction
   }

packages/server/src/api/routes/application.js

@@ -11,6 +11,16 @@ router
   .get("/api/applications/:appId/appPackage", controller.fetchAppPackage)
   .put("/api/applications/:appId", authorized(BUILDER), controller.update)
   .post("/api/applications", authorized(BUILDER), controller.create)
+  .post(
+    "/api/applications/:appId/client/update",
+    authorized(BUILDER),
+    controller.updateClient
+  )
+  .post(
+    "/api/applications/:appId/client/revert",
+    authorized(BUILDER),
+    controller.revertClient
+  )
   .delete("/api/applications/:appId", authorized(BUILDER), controller.delete)
 
 module.exports = router

packages/server/src/api/routes/tests/application.spec.js

@@ -94,7 +94,7 @@ describe("/applications", () => {
   })
 
   describe("update", () => {
-    it("should be able to fetch the app package", async () => {
+    it("should be able to update the app package", async () => {
       const res = await request
         .put(`/api/applications/${config.getAppId()}`)
         .send({
@@ -107,6 +107,30 @@ describe("/applications", () => {
     })
   })
 
+  describe("manage client library version", () => {
+    it("should be able to update the app client library version", async () => {
+      console.log(config.getAppId())
+      await request
+        .post(`/api/applications/${config.getAppId()}/client/update`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+    })
+    it("should be able to revert the app client library version", async () => {
+      // We need to first update the version so that we can then revert
+      await request
+        .post(`/api/applications/${config.getAppId()}/client/update`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      await request
+        .post(`/api/applications/${config.getAppId()}/client/revert`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+    })
+  })
+
   describe("edited at", () => {
     it("middleware should set edited at", async () => {
       const headers = config.defaultHeaders()

packages/server/src/definitions/datasource.ts

@@ -129,7 +129,9 @@ export interface QueryJson {
 
 export interface SqlQuery {
   sql: string
-  bindings?: {
+  bindings?:
+    | string[]
+    | {
         [key: string]: any
       }
 }

packages/server/src/integrations/base/sql.ts

@@ -8,9 +8,24 @@ import {
   Operation,
   RelationshipsJson,
 } from "../../definitions/datasource"
+import { isIsoDateString } from "../utils"
 
 type KnexQuery = Knex.QueryBuilder | Knex
 
+function parseBody(body: any) {
+  for (let [key, value] of Object.entries(body)) {
+    if (typeof value !== "string") {
+      continue
+    }
+    if (isIsoDateString(value)) {
+      body[key] = new Date(value)
+    } else if (!isNaN(parseFloat(value))) {
+      body[key] = parseFloat(value)
+    }
+  }
+  return body
+}
+
 // right now we only do filters on the specific table being queried
 function addFilters(
   tableName: string,
@@ -119,11 +134,12 @@ function buildCreate(
 ): KnexQuery {
   const { endpoint, body } = json
   let query: KnexQuery = knex(endpoint.entityId)
+  const parsedBody = parseBody(body)
   // mysql can't use returning
   if (opts.disableReturning) {
-    return query.insert(body)
+    return query.insert(parsedBody)
   } else {
-    return query.insert(body).returning("*")
+    return query.insert(parsedBody).returning("*")
   }
 }
 
@@ -173,12 +189,13 @@ function buildUpdate(
 ): KnexQuery {
   const { endpoint, body, filters } = json
   let query: KnexQuery = knex(endpoint.entityId)
+  const parsedBody = parseBody(body)
   query = addFilters(endpoint.entityId, query, filters)
   // mysql can't use returning
   if (opts.disableReturning) {
-    return query.update(body)
+    return query.update(parsedBody)
   } else {
-    return query.update(body).returning("*")
+    return query.update(parsedBody).returning("*")
   }
 }
 

packages/server/src/integrations/mysql.ts

@@ -29,6 +29,7 @@ module MySQLModule {
     blob: FieldTypes.LONGFORM,
     enum: FieldTypes.STRING,
     varchar: FieldTypes.STRING,
+    float: FieldTypes.NUMBER,
     int: FieldTypes.NUMBER,
     numeric: FieldTypes.NUMBER,
     bigint: FieldTypes.NUMBER,

packages/server/src/integrations/postgres.ts

@@ -92,7 +92,7 @@ module PostgresModule {
 
   async function internalQuery(client: any, query: SqlQuery) {
     try {
-      return await client.query(query.sql, query.bindings || {})
+      return await client.query(query.sql, query.bindings || [])
     } catch (err) {
       throw new Error(err)
     }

packages/server/src/integrations/tests/postgres.spec.js

@@ -20,7 +20,7 @@ describe("Postgres Integration", () => {
     await config.integration.create({
       sql
     })
-    expect(pg.queryMock).toHaveBeenCalledWith(sql, {})
+    expect(pg.queryMock).toHaveBeenCalledWith(sql, [])
   })
 
   it("calls the read method with the correct params", async () => {
@@ -28,7 +28,7 @@ describe("Postgres Integration", () => {
     await config.integration.read({
       sql
     })
-    expect(pg.queryMock).toHaveBeenCalledWith(sql, {})
+    expect(pg.queryMock).toHaveBeenCalledWith(sql, [])
   })
 
   it("calls the update method with the correct params", async () => {
@@ -36,7 +36,7 @@ describe("Postgres Integration", () => {
     const response = await config.integration.update({ 
       sql
     })
-    expect(pg.queryMock).toHaveBeenCalledWith(sql, {})
+    expect(pg.queryMock).toHaveBeenCalledWith(sql, [])
   })
 
   it("calls the delete method with the correct params", async () => {
@@ -44,7 +44,7 @@ describe("Postgres Integration", () => {
     await config.integration.delete({
       sql
     })
-    expect(pg.queryMock).toHaveBeenCalledWith(sql, {})
+    expect(pg.queryMock).toHaveBeenCalledWith(sql, [])
   })
 
   describe("no rows returned", () => {

packages/server/src/integrations/utils.ts

@@ -68,3 +68,11 @@ export function isSQL(datasource: Datasource): boolean {
   const SQL = [SourceNames.POSTGRES, SourceNames.SQL_SERVER, SourceNames.MYSQL]
   return SQL.indexOf(datasource.source) !== -1
 }
+
+export function isIsoDateString(str: string) {
+  if (!/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/.test(str)) {
+    return false
+  }
+  let d = new Date(str)
+  return d.toISOString() === str
+}

packages/server/src/utilities/fileSystem/clientLibrary.js

@@ -0,0 +1,154 @@
+const { join } = require("path")
+const { ObjectStoreBuckets } = require("../../constants")
+const fs = require("fs")
+const { upload, retrieveToTmp, streamUpload } = require("./utilities")
+const { resolve } = require("../centralPath")
+const env = require("../../environment")
+const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+
+/**
+ * Client library paths in the object store:
+ * Previously, the entire standard-components package was downloaded from NPM
+ * as a tarball and extracted to the object store, even though only the manifest
+ * was ever needed. Therefore we need to support old apps which may still have
+ * the manifest at this location for the first update.
+ *
+ * The new paths for the in-use version are:
+ * {appId}/manifest.json
+ * {appId}/budibase-client.js
+ *
+ * The paths for the backups are:
+ * {appId}/manifest.json.bak
+ * {appId}/budibase-client.js.bak
+ *
+ * We don't rely on NPM at all any more, as when updating to the latest version
+ * we pull both the manifest and client bundle from the server's dependencies
+ * in the local file system.
+ */
+
+/**
+ * Backs up the current client library version by copying both the manifest
+ * and client bundle to .bak extensions in the object store. Only the one
+ * previous version is stored as a backup, which can be reverted to.
+ * @param appId The app ID to backup
+ * @returns {Promise<void>}
+ */
+exports.backupClientLibrary = async appId => {
+  // Copy existing manifest to tmp
+  let tmpManifestPath
+  try {
+    // Try to load the manifest from the new file location
+    tmpManifestPath = await retrieveToTmp(
+      ObjectStoreBuckets.APPS,
+      join(appId, "manifest.json")
+    )
+  } catch (error) {
+    // Fallback to loading it from the old location for old apps
+    tmpManifestPath = await retrieveToTmp(
+      ObjectStoreBuckets.APPS,
+      join(
+        appId,
+        "node_modules",
+        "budibase",
+        "standard-components",
+        "package",
+        "manifest.json"
+      )
+    )
+  }
+
+  // Copy existing client lib to tmp
+  const tmpClientPath = await retrieveToTmp(
+    ObjectStoreBuckets.APPS,
+    join(appId, "budibase-client.js")
+  )
+
+  // Upload manifest and client library as backups
+  const manifestUpload = upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: join(appId, "manifest.json.bak"),
+    path: tmpManifestPath,
+    type: "application/json",
+  })
+  const clientUpload = upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: join(appId, "budibase-client.js.bak"),
+    path: tmpClientPath,
+    type: "application/javascript",
+  })
+  await Promise.all([manifestUpload, clientUpload])
+}
+
+/**
+ * Uploads the latest version of the component manifest and the client library
+ * to the object store, overwriting the existing version.
+ * @param appId The app ID to update
+ * @returns {Promise<void>}
+ */
+exports.updateClientLibrary = async appId => {
+  let manifest, client
+
+  if (env.isDev()) {
+    // Load the symlinked version in dev which is always the newest
+    manifest = require.resolve("@budibase/standard-components/manifest.json")
+    client = require.resolve("@budibase/client")
+  } else {
+    // Load the bundled version in prod
+    manifest = resolve(TOP_LEVEL_PATH, "client", "manifest.json")
+    client = resolve(TOP_LEVEL_PATH, "client", "budibase-client.js")
+  }
+
+  // Upload latest manifest and client library
+  const manifestUpload = streamUpload(
+    ObjectStoreBuckets.APPS,
+    join(appId, "manifest.json"),
+    fs.createReadStream(manifest),
+    {
+      ContentType: "application/json",
+    }
+  )
+  const clientUpload = streamUpload(
+    ObjectStoreBuckets.APPS,
+    join(appId, "budibase-client.js"),
+    fs.createReadStream(client),
+    {
+      ContentType: "application/javascript",
+    }
+  )
+  await Promise.all([manifestUpload, clientUpload])
+}
+
+/**
+ * Reverts the version of the client library and manifest to the previously
+ * used version for an app.
+ * @param appId The app ID to revert
+ * @returns {Promise<void>}
+ */
+exports.revertClientLibrary = async appId => {
+  // Copy backups manifest to tmp directory
+  const tmpManifestPath = await retrieveToTmp(
+    ObjectStoreBuckets.APPS,
+    join(appId, "manifest.json.bak")
+  )
+
+  // Copy backup client lib to tmp
+  const tmpClientPath = await retrieveToTmp(
+    ObjectStoreBuckets.APPS,
+    join(appId, "budibase-client.js.bak")
+  )
+
+  // Upload backups as new versions
+  const manifestUpload = upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: join(appId, "manifest.json"),
+    path: tmpManifestPath,
+    type: "application/json",
+  })
+  const clientUpload = upload({
+    bucket: ObjectStoreBuckets.APPS,
+    filename: join(appId, "budibase-client.js"),
+    path: tmpClientPath,
+    type: "application/javascript",
+  })
+  await Promise.all([manifestUpload, clientUpload])
+}

packages/server/src/utilities/fileSystem/index.js

@@ -13,7 +13,7 @@ const {
   deleteFolder,
   downloadTarball,
 } = require("./utilities")
-const { downloadLibraries, uploadClientLibrary } = require("./newApp")
+const { updateClientLibrary } = require("./clientLibrary")
 const download = require("download")
 const env = require("../../environment")
 const { homedir } = require("os")
@@ -139,13 +139,12 @@ exports.performBackup = async (appId, backupName) => {
 }
 
 /**
- * Downloads required libraries and creates a new path in the object store.
+ * Uploads the latest client library to the object store.
  * @param {string} appId The ID of the app which is being created.
  * @return {Promise<void>} once promise completes app resources should be ready in object store.
  */
 exports.createApp = async appId => {
-  await downloadLibraries(appId)
+  await updateClientLibrary(appId)
-  await uploadClientLibrary(appId)
 }
 
 /**
@@ -193,8 +192,17 @@ exports.getComponentLibraryManifest = async (appId, library) => {
     delete require.cache[require.resolve(path)]
     return require(path)
   }
+
+  let resp
+  try {
+    // Try to load the manifest from the new file location
+    const path = join(appId, filename)
+    resp = await retrieve(ObjectStoreBuckets.APPS, path)
+  } catch (error) {
+    // Fallback to loading it from the old location for old apps
     const path = join(appId, "node_modules", library, "package", filename)
-  let resp = await retrieve(ObjectStoreBuckets.APPS, path)
+    resp = await retrieve(ObjectStoreBuckets.APPS, path)
+  }
   if (typeof resp !== "string") {
     resp = resp.toString("utf8")
   }

packages/server/src/utilities/fileSystem/newApp.js

@@ -1,36 +0,0 @@
-const packageJson = require("../../../package.json")
-const { join } = require("path")
-const { ObjectStoreBuckets } = require("../../constants")
-const { streamUpload, downloadTarball } = require("./utilities")
-const fs = require("fs")
-
-const BUCKET_NAME = ObjectStoreBuckets.APPS
-
-// can't really test this due to the downloading nature of it, wouldn't be a great test case
-/* istanbul ignore next */
-exports.downloadLibraries = async appId => {
-  const LIBRARIES = ["standard-components"]
-
-  const paths = {}
-  // Need to download tarballs directly from NPM as our users may not have node on their machine
-  for (let lib of LIBRARIES) {
-    // download tarball
-    const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-${packageJson.version}.tgz`
-    const path = join(appId, "node_modules", "@budibase", lib)
-    paths[`@budibase/${lib}`] = await downloadTarball(
-      registryUrl,
-      BUCKET_NAME,
-      path
-    )
-  }
-  return paths
-}
-
-exports.uploadClientLibrary = async appId => {
-  const sourcepath = require.resolve("@budibase/client")
-  const destPath = join(appId, "budibase-client.js")
-
-  await streamUpload(BUCKET_NAME, destPath, fs.createReadStream(sourcepath), {
-    ContentType: "application/javascript",
-  })
-}

packages/server/src/utilities/usageQuota.js

@@ -85,7 +85,7 @@ exports.update = async (apiKey, property, usage) => {
         await apiKeyTable.put({ item: keyObj })
         return
       }
-      // we have infact breached the reset period
+      // we have in fact breached the reset period
       else if (keyObj && keyObj.quotaReset <= Date.now()) {
         // update the quota reset period and reset the values for all properties
         keyObj.quotaReset = getNewQuotaReset()

packages/standard-components/manifest.json

@@ -1,4 +1,8 @@
 {
+  "features": {
+    "spectrumThemes": true,
+    "intelligentLoading": true
+  },
   "layout": {
     "name": "Layout",
     "description": "This component is specific only to layouts",
@@ -27,7 +31,7 @@
         "type": "select",
         "label": "Width",
         "key": "width",
-        "options": ["Small", "Medium", "Large"],
+        "options": ["Small", "Medium", "Large", "Max"],
         "defaultValue": "Large"
       },
       {

packages/standard-components/package.json

@@ -29,12 +29,12 @@
   "keywords": [
     "svelte"
   ],
-  "version": "0.9.71",
+  "version": "0.9.74",
   "license": "MIT",
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc",
   "dependencies": {
-    "@budibase/bbui": "^0.9.71",
     "@spectrum-css/card": "^3.0.3",
+    "@budibase/bbui": "^0.9.74",
     "@spectrum-css/link": "^3.1.3",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/typography": "^3.0.2",

packages/standard-components/src/Layout.svelte

@@ -20,6 +20,7 @@
     None: "none",
   }
   const widthClasses = {
+    Max: "max",
     Large: "l",
     Medium: "m",
     Small: "s",
@@ -178,6 +179,9 @@
     position: relative;
     padding: 32px;
   }
+  .layout--none .main {
+    padding: 0;
+  }
   .size--s {
     width: 800px;
   }
@@ -187,6 +191,9 @@
   .size--l {
     width: 1400px;
   }
+  .size--max {
+    width: 100%;
+  }
 
   /*  Nav components */
   .burger {

packages/standard-components/src/helpers.js

@@ -22,7 +22,7 @@ export const generateID = (size = 21) => {
     // It is incorrect to use bytes exceeding the alphabet size.
     // The following mask reduces the random byte in the 0-255 value
     // range to the 0-63 value range. Therefore, adding hacks, such
-    // as empty string fallback or magic numbers, is unneccessary because
+    // as empty string fallback or magic numbers, is unnecessary because
     // the bitmask trims bytes down to the alphabet size.
     let byte = bytes[size] & 63
     if (byte < 36) {

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "0.9.71",
+  "version": "0.9.74",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/string-templates/src/processors/preprocessor.js

@@ -28,7 +28,7 @@ module.exports.processors = [
     let startBraceIdx = statement.indexOf("[")
     let lastIdx = 0
     while (startBraceIdx !== -1) {
-      // if the character previous to the literal specifier is alpha-numeric this should happen
+      // if the character previous to the literal specifier is alphanumeric this should happen
       if (isAlphaNumeric(statement.charAt(startBraceIdx - 1))) {
         statement = swapStrings(statement, startBraceIdx + lastIdx, 1, ".[")
       }

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "0.9.71",
+  "version": "0.9.74",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -21,8 +21,8 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.71",
+    "@budibase/auth": "^0.9.74",
-    "@budibase/string-templates": "^0.9.71",
+    "@budibase/string-templates": "^0.9.74",
     "@koa/router": "^8.0.0",
     "aws-sdk": "^2.811.0",
     "bcryptjs": "^2.4.3",

packages/worker/src/utilities/email.js

@@ -173,7 +173,7 @@ exports.sendEmail = async (
 }
 
 /**
- * Given an SMTP configuration this runs it through nodemailer to see if it is infact functional.
+ * Given an SMTP configuration this runs it through nodemailer to see if it is in fact functional.
  * @param {object} config an SMTP configuration - this is based on the nodemailer API.
  * @return {Promise<boolean>} returns true if the configuration is valid.
  */