Merge branch 'master' into feature/automation-sidebar

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.7.2",
+  "version": "3.7.3",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/sql/sql.ts

@@ -6,6 +6,7 @@ import {
   isInvalidISODateString,
   isValidFilter,
   isValidISODateString,
+  isValidTime,
   sqlLog,
   validateManyToMany,
 } from "./utils"
@@ -417,6 +418,11 @@ class InternalBuilder {
     }
 
     if (typeof input === "string" && schema.type === FieldType.DATETIME) {
+      if (schema.timeOnly) {
+        if (!isValidTime(input)) {
+          return null
+        }
+      } else {
         if (isInvalidISODateString(input)) {
           return null
         }
@@ -424,6 +430,7 @@ class InternalBuilder {
           return new Date(input.trim())
         }
       }
+    }
     return input
   }
 

packages/backend-core/src/sql/tests/utils.spec.ts

@@ -0,0 +1,35 @@
+import { isValidISODateString, isInvalidISODateString } from "../utils"
+
+describe("ISO date string validity checks", () => {
+  it("accepts a valid ISO date string without a time", () => {
+    const str = "2013-02-01"
+    const valid = isValidISODateString(str)
+    const invalid = isInvalidISODateString(str)
+    expect(valid).toEqual(true)
+    expect(invalid).toEqual(false)
+  })
+
+  it("accepts a valid ISO date string with a time", () => {
+    const str = "2013-02-01T01:23:45Z"
+    const valid = isValidISODateString(str)
+    const invalid = isInvalidISODateString(str)
+    expect(valid).toEqual(true)
+    expect(invalid).toEqual(false)
+  })
+
+  it("accepts a valid ISO date string with a time and millis", () => {
+    const str = "2013-02-01T01:23:45.678Z"
+    const valid = isValidISODateString(str)
+    const invalid = isInvalidISODateString(str)
+    expect(valid).toEqual(true)
+    expect(invalid).toEqual(false)
+  })
+
+  it("rejects an invalid ISO date string", () => {
+    const str = "2013-523-814T444:22:11Z"
+    const valid = isValidISODateString(str)
+    const invalid = isInvalidISODateString(str)
+    expect(valid).toEqual(false)
+    expect(invalid).toEqual(true)
+  })
+})

packages/backend-core/src/sql/utils.ts

@@ -14,7 +14,7 @@ import environment from "../environment"
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
 const ENCODED_SPACE = encodeURIComponent(" ")
-const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}.\d{3}Z)?$/
+const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:.\d{3})?Z)?$/
 const TIME_REGEX = /^(?:\d{2}:)?(?:\d{2}:)(?:\d{2})$/
 
 export function isExternalTableID(tableId: string) {
@@ -139,17 +139,17 @@ export function breakRowIdField(_id: string | { _id: string }): any[] {
   }
 }
 
-export function isInvalidISODateString(str: string) {
+export function isValidISODateString(str: string) {
   const trimmedValue = str.trim()
   if (!ISO_DATE_REGEX.test(trimmedValue)) {
     return false
   }
-  let d = new Date(trimmedValue)
-  return isNaN(d.getTime())
+  const d = new Date(trimmedValue)
+  return !isNaN(d.getTime())
 }
 
-export function isValidISODateString(str: string) {
-  return ISO_DATE_REGEX.test(str.trim())
+export function isInvalidISODateString(str: string) {
+  return !isValidISODateString(str)
 }
 
 export function isValidFilter(value: any) {

packages/bbui/src/Form/Core/TextArea.svelte

@@ -75,6 +75,7 @@
   class:is-disabled={disabled}
   class:is-focused={isFocused}
 >
+  <!-- We need to ignore prettier here as we want no whitespace -->
   <!-- prettier-ignore -->
   <textarea
     bind:this={textarea}
@@ -90,6 +91,7 @@
     on:blur
     on:keypress
   >{value || ""}</textarea>
+  <slot />
 </div>
 
 <style>

packages/bbui/src/Form/Core/TextField.svelte

@@ -114,6 +114,7 @@
     inputmode={getInputMode(type)}
     autocomplete={autocompleteValue}
   />
+  <slot />
 </div>
 
 <style>

packages/bbui/src/Form/Input.svelte

@@ -41,5 +41,7 @@
     on:blur
     on:focus
     on:keyup
-  />
+  >
+    <slot />
+  </TextField>
 </Field>

packages/bbui/src/Form/TextArea.svelte

@@ -7,11 +7,13 @@
   export let label: string | undefined = undefined
   export let labelPosition = "above"
   export let placeholder: string | undefined = undefined
-  export let disabled = false
+  export let readonly: boolean = false
+  export let disabled: boolean = false
   export let error: string | undefined = undefined
   export let height: number | undefined = undefined
   export let minHeight: number | undefined = undefined
   export let helpText: string | undefined = undefined
+  export let updateOnChange: boolean = false
 
   let textarea: TextArea
   export function focus() {
@@ -33,11 +35,16 @@
   <TextArea
     bind:this={textarea}
     {disabled}
+    {readonly}
     {value}
     {placeholder}
     {height}
     {minHeight}
+    {updateOnChange}
     on:change={onChange}
     on:keypress
-  />
+    on:scrollable
+  >
+    <slot />
+  </TextArea>
 </Field>

packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte

@@ -45,7 +45,7 @@
     },
     {
       label: "Multi-select",
-      value: FieldType.ARRAY.type,
+      value: FieldType.ARRAY,
     },
     {
       label: "Barcode/QR",

packages/builder/src/components/common/bindings/DrawerBindableInput.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-  import { Icon, Input, Drawer, Button, CoreTextArea } from "@budibase/bbui"
+  import { Icon, Input, Drawer, Button, TextArea } from "@budibase/bbui"
   import {
     readableToRuntimeBinding,
     runtimeToReadableBinding,
@@ -67,7 +67,7 @@
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <div class="control" class:multiline class:disabled class:scrollable>
   <svelte:component
-    this={multiline ? CoreTextArea : Input}
+    this={multiline ? TextArea : Input}
     {label}
     {disabled}
     readonly={isJS}
@@ -78,7 +78,7 @@
     {placeholder}
     {updateOnChange}
     {autocomplete}
-  />
+  >
     {#if !disabled && !disableBindings}
       <div
         class="icon"
@@ -90,6 +90,7 @@
         <Icon size="S" name="FlashOn" />
       </div>
     {/if}
+  </svelte:component>
 </div>
 <Drawer
   on:drawerHide={onDrawerHide}

packages/client/src/components/app/Text.svelte

@@ -2,7 +2,7 @@
   import { getContext } from "svelte"
   import { MarkdownViewer } from "@budibase/bbui"
 
-  export let text: string = ""
+  export let text: any = ""
   export let color: string | undefined = undefined
   export let align: "left" | "center" | "right" | "justify" = "left"
 
@@ -12,6 +12,9 @@
   // Add in certain settings to styles
   $: styles = enrichStyles($component.styles, color, align)
 
+  // Ensure we're always passing in a string value to the markdown editor
+  $: safeText = stringify(text)
+
   const enrichStyles = (
     styles: any,
     colorStyle: typeof color,
@@ -31,10 +34,24 @@
       },
     }
   }
+
+  const stringify = (text: any): string => {
+    if (text == null) {
+      return ""
+    }
+    if (typeof text !== "string") {
+      try {
+        return JSON.stringify(text)
+      } catch (e) {
+        return ""
+      }
+    }
+    return text
+  }
 </script>
 
 <div use:styleable={styles}>
-  <MarkdownViewer value={text} />
+  <MarkdownViewer value={safeText} />
 </div>
 
 <style>

packages/frontend-core/src/components/grid/stores/config.ts

@@ -82,8 +82,9 @@ export const deriveStores = (context: StoreContext): ConfigDerivedStore => {
         config.canEditColumns = false
       }
 
-      // Determine if we can select rows
-      config.canSelectRows = !!config.canDeleteRows || !!config.canAddRows
+      // Determine if we can select rows. Always true in the meantime as you can
+      // use the selected rows binding regardless of readonly state.
+      config.canSelectRows = true
 
       return config
     }

packages/server/src/api/controllers/assets.ts

@@ -0,0 +1,12 @@
+import { join } from "../../utilities/centralPath"
+import { TOP_LEVEL_PATH, DEV_ASSET_PATH } from "../../utilities/fileSystem"
+import { Ctx } from "@budibase/types"
+import env from "../../environment"
+import send from "koa-send"
+
+// this is a public endpoint with no middlewares
+export const serveBuilderAssets = async function (ctx: Ctx<void, void>) {
+  let topLevelPath = env.isDev() ? DEV_ASSET_PATH : TOP_LEVEL_PATH
+  const builderPath = join(topLevelPath, "builder")
+  await send(ctx, ctx.file, { root: builderPath })
+}

packages/server/src/api/controllers/static/index.ts

@@ -76,11 +76,6 @@ export const toggleBetaUiFeature = async function (
   }
 }
 
-export const serveBuilder = async function (ctx: Ctx<void, void>) {
-  const builderPath = join(TOP_LEVEL_PATH, "builder")
-  await send(ctx, ctx.file, { root: builderPath })
-}
-
 export const uploadFile = async function (
   ctx: Ctx<void, ProcessAttachmentResponse>
 ) {

packages/server/src/api/controllers/table/tests/utils.spec.ts

@@ -1,4 +1,9 @@
-import { AIOperationEnum, AutoFieldSubType, FieldType } from "@budibase/types"
+import {
+  AIOperationEnum,
+  AutoFieldSubType,
+  FieldType,
+  JsonFieldSubType,
+} from "@budibase/types"
 import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
 import { importToRows } from "../utils"
 
@@ -152,5 +157,33 @@ describe("utils", () => {
         ])
       })
     })
+
+    it("coerces strings into arrays for array fields", async () => {
+      await config.doInContext(config.appId, async () => {
+        const table = await config.createTable({
+          name: "table",
+          type: "table",
+          schema: {
+            colours: {
+              name: "colours",
+              type: FieldType.ARRAY,
+              constraints: {
+                type: JsonFieldSubType.ARRAY,
+                inclusion: ["red"],
+              },
+            },
+          },
+        })
+
+        const data = [{ colours: "red" }]
+
+        const result = await importToRows(data, table, config.user?._id)
+        expect(result).toEqual([
+          expect.objectContaining({
+            colours: ["red"],
+          }),
+        ])
+      })
+    })
   })
 })

packages/server/src/api/controllers/table/utils.ts

@@ -155,13 +155,19 @@ export async function importToRows(
           schema.type === FieldType.ARRAY) &&
         row[fieldName]
       ) {
-        const rowVal = Array.isArray(row[fieldName])
-          ? row[fieldName]
-          : [row[fieldName]]
+        const isArray = Array.isArray(row[fieldName])
+
+        // Add option to inclusion constraints
+        const rowVal = isArray ? row[fieldName] : [row[fieldName]]
         let merged = [...schema.constraints!.inclusion!, ...rowVal]
         let superSet = new Set(merged)
         schema.constraints!.inclusion = Array.from(superSet)
         schema.constraints!.inclusion.sort()
+
+        // If array type, ensure we import the value as an array
+        if (!isArray && schema.type === FieldType.ARRAY) {
+          row[fieldName] = rowVal
+        }
       }
     }
 

packages/server/src/api/index.ts

@@ -8,6 +8,7 @@ import { middleware as pro } from "@budibase/pro"
 import { apiEnabled, automationsEnabled } from "../features"
 import migrations from "../middleware/appMigrations"
 import { automationQueue } from "../automations"
+import assetRouter from "./routes/assets"
 
 export { shutdown } from "./routes/public"
 const compress = require("koa-compress")
@@ -44,6 +45,12 @@ if (apiEnabled()) {
     )
     // re-direct before any middlewares occur
     .redirect("/", "/builder")
+
+  // send assets before middleware
+  router.use(assetRouter.routes())
+  router.use(assetRouter.allowedMethods())
+
+  router
     .use(
       auth.buildAuthMiddleware([], {
         publicAllowed: true,

packages/server/src/api/routes/assets.ts

@@ -0,0 +1,11 @@
+import { addFileManagement } from "../utils"
+import { serveBuilderAssets } from "../controllers/assets"
+import Router from "@koa/router"
+
+const router: Router = new Router()
+
+addFileManagement(router)
+
+router.get("/builder/:file*", serveBuilderAssets)
+
+export default router

packages/server/src/api/routes/static.ts

@@ -1,35 +1,17 @@
 import Router from "@koa/router"
 import * as controller from "../controllers/static"
-import { budibaseTempDir } from "../../utilities/budibaseDir"
 import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
-import env from "../../environment"
+import { addFileManagement } from "../utils"
 import { paramResource } from "../../middleware/resourceId"
-import { devClientLibPath } from "../../utilities/fileSystem"
 
 const { BUILDER, PermissionType, PermissionLevel } = permissions
 
 const router: Router = new Router()
 
-/* istanbul ignore next */
-router.param("file", async (file: any, ctx: any, next: any) => {
-  ctx.file = file && file.includes(".") ? file : "index.html"
-  if (!ctx.file.startsWith("budibase-client")) {
-    return next()
-  }
-  // test serves from require
-  if (env.isTest()) {
-    const path = devClientLibPath()
-    ctx.devPath = path.split(ctx.file)[0]
-  } else if (env.isDev()) {
-    // Serving the client library from your local dir in dev
-    ctx.devPath = budibaseTempDir()
-  }
-  return next()
-})
+addFileManagement(router)
 
 router
-  .get("/builder/:file*", controller.serveBuilder)
   .get("/api/assets/client", controller.serveClientLibrary)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
   .post("/api/beta/:feature", controller.toggleBetaUiFeature)

packages/server/src/api/routes/tests/assets.spec.ts

@@ -0,0 +1,34 @@
+import fs from "fs"
+import { join } from "path"
+import { DEV_ASSET_PATH } from "../../../utilities/fileSystem"
+import * as setup from "./utilities"
+
+const path = join(DEV_ASSET_PATH, "builder")
+let addedPath = false
+const config = setup.getConfig()
+
+beforeAll(async () => {
+  if (!fs.existsSync(path)) {
+    addedPath = true
+    fs.mkdirSync(path)
+  }
+  const indexPath = join(path, "index.html")
+  if (!fs.existsSync(indexPath)) {
+    fs.writeFileSync(indexPath, "<html></html>", "utf8")
+    addedPath = true
+  }
+  await config.init()
+})
+
+afterAll(() => {
+  if (addedPath) {
+    fs.rmSync(path, { recursive: true })
+  }
+})
+
+describe("/builder/:file*", () => {
+  it("should be able to retrieve the builder file", async () => {
+    const res = await config.api.assets.get("index.html")
+    expect(res.text).toContain("<html")
+  })
+})

packages/server/src/api/utils.ts

@@ -0,0 +1,23 @@
+import env from "../environment"
+import { devClientLibPath } from "../utilities/fileSystem"
+import { budibaseTempDir } from "../utilities/budibaseDir"
+import Router from "@koa/router"
+
+export function addFileManagement(router: Router) {
+  /* istanbul ignore next */
+  router.param("file", async (file: any, ctx: any, next: any) => {
+    ctx.file = file && file.includes(".") ? file : "index.html"
+    if (!ctx.file.startsWith("budibase-client")) {
+      return next()
+    }
+    // test serves from require
+    if (env.isTest()) {
+      const path = devClientLibPath()
+      ctx.devPath = path.split(ctx.file)[0]
+    } else if (env.isDev()) {
+      // Serving the client library from your local dir in dev
+      ctx.devPath = budibaseTempDir()
+    }
+    return next()
+  })
+}

packages/server/src/tests/utilities/api/assets.ts

@@ -0,0 +1,8 @@
+import { TestAPI } from "./base"
+
+export class AssetsAPI extends TestAPI {
+  get = async (path: string) => {
+    // has to be raw, body isn't JSON
+    return await this._requestRaw("get", `/builder/${path}`)
+  }
+}

packages/server/src/tests/utilities/api/index.ts

@@ -21,6 +21,7 @@ import { EnvironmentAPI } from "./environment"
 import { UserPublicAPI } from "./public/user"
 import { MiscAPI } from "./misc"
 import { OAuth2API } from "./oauth2"
+import { AssetsAPI } from "./assets"
 
 export default class API {
   application: ApplicationAPI
@@ -44,6 +45,7 @@ export default class API {
   user: UserAPI
   viewV2: ViewV2API
   webhook: WebhookAPI
+  assets: AssetsAPI
 
   public: {
     user: UserPublicAPI
@@ -71,6 +73,7 @@ export default class API {
     this.user = new UserAPI(config)
     this.viewV2 = new ViewV2API(config)
     this.webhook = new WebhookAPI(config)
+    this.assets = new AssetsAPI(config)
     this.public = {
       user: new UserPublicAPI(config),
     }

packages/server/src/utilities/fileSystem/filesystem.ts

@@ -8,6 +8,7 @@ import { v4 as uuid } from "uuid"
 
 export const TOP_LEVEL_PATH =
   env.TOP_LEVEL_PATH || resolve(join(__dirname, "..", "..", ".."))
+export const DEV_ASSET_PATH = join(TOP_LEVEL_PATH, "packages", "server")
 
 /**
  * Upon first startup of instance there may not be everything we need in tmp directory, set it up.

packages/server/src/utilities/schema.ts

@@ -106,7 +106,7 @@ export function validate(
       } else if (
         // If provided must be a valid date
         columnType === FieldType.DATETIME &&
-        isNaN(new Date(columnData).getTime())
+        sql.utils.isInvalidISODateString(columnData)
       ) {
         results.schemaValidation[columnName] = false
       } else if (