Merge master.

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.7.0",
+  "version": "3.7.4",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/package.json

@@ -56,6 +56,7 @@
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",
     "node-fetch": "2.6.7",
+    "object-sizeof": "2.6.1",
     "passport-google-oauth": "2.0.0",
     "passport-local": "1.0.0",
     "passport-oauth2-refresh": "^2.1.0",

packages/backend-core/src/cache/docWritethrough.ts

@@ -1,6 +1,6 @@
-import { AnyDocument, Database, Document } from "@budibase/types"
+import { AnyDocument, Database, Document, DocumentType } from "@budibase/types"
 
-import { JobQueue, Queue, createQueue } from "../queue"
+import { BudibaseQueue, JobQueue } from "../queue"
 import * as dbUtils from "../db"
 
 interface ProcessDocMessage {
@@ -13,11 +13,11 @@ const PERSIST_MAX_ATTEMPTS = 100
 let processor: DocWritethroughProcessor | undefined
 
 export class DocWritethroughProcessor {
-  private static _queue: Queue
+  private static _queue: BudibaseQueue<ProcessDocMessage>
 
   public static get queue() {
     if (!DocWritethroughProcessor._queue) {
-      DocWritethroughProcessor._queue = createQueue<ProcessDocMessage>(
+      DocWritethroughProcessor._queue = new BudibaseQueue<ProcessDocMessage>(
         JobQueue.DOC_WRITETHROUGH_QUEUE,
         {
           jobOptions: {
@@ -57,6 +57,10 @@ export class DocWritethroughProcessor {
     docId: string
     data: Record<string, any>
   }) {
+    // HACK - for now drop SCIM events
+    if (docId.startsWith(DocumentType.SCIM_LOG)) {
+      return
+    }
     const db = dbUtils.getDB(dbName)
     let doc: AnyDocument | undefined
     try {

packages/backend-core/src/cache/tests/docWritethrough.spec.ts

@@ -18,7 +18,9 @@ import {
 const initialTime = Date.now()
 
 async function waitForQueueCompletion() {
-  await utils.queue.processMessages(DocWritethroughProcessor.queue)
+  await utils.queue.processMessages(
+    DocWritethroughProcessor.queue.getBullQueue()
+  )
 }
 
 beforeAll(() => utils.queue.useRealQueues())

packages/backend-core/src/events/asyncEvents/queue.ts

@@ -1,5 +1,4 @@
-import BullQueue from "bull"
-import { createQueue, JobQueue } from "../../queue"
+import { BudibaseQueue, JobQueue } from "../../queue"
 import { Event, Identity } from "@budibase/types"
 
 export interface EventPayload {
@@ -9,10 +8,19 @@ export interface EventPayload {
   timestamp?: string | number
 }
 
-export let asyncEventQueue: BullQueue.Queue
+export let asyncEventQueue: BudibaseQueue<EventPayload>
 
 export function init() {
-  asyncEventQueue = createQueue<EventPayload>(JobQueue.SYSTEM_EVENT_QUEUE)
+  asyncEventQueue = new BudibaseQueue<EventPayload>(
+    JobQueue.SYSTEM_EVENT_QUEUE,
+    {
+      jobTags: (event: EventPayload) => {
+        return {
+          "event.name": event.event,
+        }
+      },
+    }
+  )
 }
 
 export async function shutdown() {

packages/backend-core/src/events/processors/AuditLogsProcessor.ts

@@ -8,24 +8,30 @@ import {
 } from "@budibase/types"
 import { EventProcessor } from "./types"
 import { getAppId, doInTenant, getTenantId } from "../../context"
-import BullQueue from "bull"
-import { createQueue, JobQueue } from "../../queue"
+import { BudibaseQueue, JobQueue } from "../../queue"
 import { isAudited } from "../../utils"
 import env from "../../environment"
 
 export default class AuditLogsProcessor implements EventProcessor {
   static auditLogsEnabled = false
-  static auditLogQueue: BullQueue.Queue<AuditLogQueueEvent>
+  static auditLogQueue: BudibaseQueue<AuditLogQueueEvent>
 
   // can't use constructor as need to return promise
   static init(fn: AuditLogFn) {
     AuditLogsProcessor.auditLogsEnabled = true
     const writeAuditLogs = fn
-    AuditLogsProcessor.auditLogQueue = createQueue<AuditLogQueueEvent>(
-      JobQueue.AUDIT_LOG
+    AuditLogsProcessor.auditLogQueue = new BudibaseQueue<AuditLogQueueEvent>(
+      JobQueue.AUDIT_LOG,
+      {
+        jobTags: (event: AuditLogQueueEvent) => {
+          return {
+            "event.name": event.event,
+          }
+        },
+      }
     )
     return AuditLogsProcessor.auditLogQueue.process(async job => {
-      return doInTenant(job.data.tenantId, async () => {
+      await doInTenant(job.data.tenantId, async () => {
         let properties = job.data.properties
         if (properties.audited) {
           properties = {

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -96,12 +96,17 @@ export class InMemoryQueue<T = any> implements Partial<Queue<T>> {
 
       let resp = func(message)
 
-      async function retryFunc(fnc: any) {
+      async function retryFunc(fnc: any, attempt = 0) {
         try {
           await fnc
         } catch (e: any) {
-          await helpers.wait(50)
-          await retryFunc(func(message))
+          attempt++
+          if (attempt < 3) {
+            await helpers.wait(100 * attempt)
+            await retryFunc(func(message), attempt)
+          } else {
+            throw e
+          }
         }
       }
 

packages/backend-core/src/queue/queue.ts

@@ -2,10 +2,12 @@ import env from "../environment"
 import { getRedisOptions } from "../redis/utils"
 import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
-import BullQueue, { QueueOptions, JobOptions } from "bull"
+import BullQueue, { Queue, QueueOptions, JobOptions, Job } from "bull"
 import { addListeners, StalledFn } from "./listeners"
 import { Duration } from "../utils"
 import * as timers from "../timers"
+import tracer from "dd-trace"
+import sizeof from "object-sizeof"
 
 export type { QueueOptions, Queue, JobOptions } from "bull"
 
@@ -15,7 +17,7 @@ const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
 const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs()
 // cleanup the queue every 60 seconds
 const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs()
-let QUEUES: BullQueue.Queue[] = []
+let QUEUES: Queue[] = []
 let cleanupInterval: NodeJS.Timeout
 
 async function cleanup() {
@@ -25,51 +27,207 @@ async function cleanup() {
   }
 }
 
-export function createQueue<T>(
-  jobQueue: JobQueue,
-  opts: {
+async function withMetrics<T>(
+  name: string,
+  cb: () => Promise<T>,
+  tags?: Record<string, string | number>
+): Promise<T> {
+  const start = performance.now()
+  try {
+    const result = await cb()
+    tracer.dogstatsd.increment(`${name}.success`, 1, tags)
+    return result
+  } catch (err) {
+    tracer.dogstatsd.increment(`${name}.error`, 1, tags)
+    throw err
+  } finally {
+    const durationMs = performance.now() - start
+    tracer.dogstatsd.distribution(`${name}.duration.ms`, durationMs, tags)
+    tracer.dogstatsd.increment(name, 1, tags)
+  }
+}
+
+function jobOptsTags(opts: JobOptions) {
+  return {
+    "job.opts.attempts": opts.attempts,
+    "job.opts.backoff": opts.backoff,
+    "job.opts.delay": opts.delay,
+    "job.opts.jobId": opts.jobId,
+    "job.opts.lifo": opts.lifo,
+    "job.opts.preventParsingData": opts.preventParsingData,
+    "job.opts.priority": opts.priority,
+    "job.opts.removeOnComplete": opts.removeOnComplete,
+    "job.opts.removeOnFail": opts.removeOnFail,
+    "job.opts.repeat": opts.repeat,
+    "job.opts.stackTraceLimit": opts.stackTraceLimit,
+    "job.opts.timeout": opts.timeout,
+  }
+}
+
+function jobTags(job: Job) {
+  return {
+    "job.id": job.id,
+    "job.attemptsMade": job.attemptsMade,
+    "job.timestamp": job.timestamp,
+    "job.data.sizeBytes": sizeof(job.data),
+    ...jobOptsTags(job.opts || {}),
+  }
+}
+
+export interface BudibaseQueueOpts<T> {
   removeStalledCb?: StalledFn
   maxStalledCount?: number
   jobOptions?: JobOptions
-  } = {}
-): BullQueue.Queue<T> {
+  jobTags?: (job: T) => Record<string, any>
+}
+
+export class BudibaseQueue<T> {
+  private queue: Queue<T>
+  private opts: BudibaseQueueOpts<T>
+  private jobQueue: JobQueue
+
+  constructor(jobQueue: JobQueue, opts: BudibaseQueueOpts<T> = {}) {
+    this.opts = opts
+    this.jobQueue = jobQueue
+    this.queue = this.initQueue()
+  }
+
+  private initQueue() {
     const redisOpts = getRedisOptions()
     const queueConfig: QueueOptions = {
       redis: redisOpts,
       settings: {
-      maxStalledCount: opts.maxStalledCount ? opts.maxStalledCount : 0,
+        maxStalledCount: this.opts.maxStalledCount
+          ? this.opts.maxStalledCount
+          : 0,
         lockDuration: QUEUE_LOCK_MS,
         lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
       },
     }
-  if (opts.jobOptions) {
-    queueConfig.defaultJobOptions = opts.jobOptions
+    if (this.opts.jobOptions) {
+      queueConfig.defaultJobOptions = this.opts.jobOptions
     }
-  let queue: BullQueue.Queue<T>
+    let queue: Queue<T>
     if (!env.isTest()) {
-    queue = new BullQueue(jobQueue, queueConfig)
+      queue = new BullQueue(this.jobQueue, queueConfig)
     } else if (
       process.env.BULL_TEST_REDIS_PORT &&
       !isNaN(+process.env.BULL_TEST_REDIS_PORT)
     ) {
-    queue = new BullQueue(jobQueue, {
+      queue = new BullQueue(this.jobQueue, {
         redis: { host: "localhost", port: +process.env.BULL_TEST_REDIS_PORT },
       })
     } else {
-    queue = new InMemoryQueue(jobQueue, queueConfig) as any
+      queue = new InMemoryQueue(this.jobQueue, queueConfig) as any
     }
-  addListeners(queue, jobQueue, opts?.removeStalledCb)
+
+    addListeners(queue, this.jobQueue, this.opts.removeStalledCb)
     QUEUES.push(queue)
     if (!cleanupInterval && !env.isTest()) {
       cleanupInterval = timers.set(cleanup, CLEANUP_PERIOD_MS)
       // fire off an initial cleanup
       cleanup().catch(err => {
-      console.error(`Unable to cleanup ${jobQueue} initially - ${err}`)
+        console.error(`Unable to cleanup ${this.jobQueue} initially - ${err}`)
       })
     }
     return queue
   }
 
+  getBullQueue() {
+    return this.queue
+  }
+
+  process(
+    concurrency: number,
+    cb: (job: Job<T>) => Promise<void>
+  ): Promise<void>
+  process(cb: (job: Job<T>) => Promise<void>): Promise<void>
+  process(...args: any[]) {
+    let concurrency: number | undefined = undefined
+    let cb: (job: Job<T>) => Promise<void>
+    if (args.length === 2) {
+      concurrency = args[0]
+      cb = args[1]
+    } else {
+      cb = args[0]
+    }
+
+    const wrappedCb = async (job: Job<T>) => {
+      await tracer.trace("queue.process", async span => {
+        // @ts-expect-error monkey patching the parent span id
+        if (job.data._parentSpanContext) {
+          // @ts-expect-error monkey patching the parent span id
+          const parentContext = job.data._parentSpanContext
+          const parent = {
+            traceId: parentContext.traceId,
+            spanId: parentContext.spanId,
+            toTraceId: () => parentContext.traceId,
+            toSpanId: () => parentContext.spanId,
+            toTraceparent: () => "",
+          }
+          span.addLink(parent)
+        }
+        span.addTags({ "queue.name": this.jobQueue, ...jobTags(job) })
+        if (this.opts.jobTags) {
+          span.addTags(this.opts.jobTags(job.data))
+        }
+
+        tracer.dogstatsd.distribution(
+          "queue.process.sizeBytes",
+          sizeof(job.data),
+          this.metricTags()
+        )
+        await this.withMetrics("queue.process", () => cb(job))
+      })
+    }
+
+    if (concurrency) {
+      return this.queue.process(concurrency, wrappedCb)
+    } else {
+      return this.queue.process(wrappedCb)
+    }
+  }
+
+  async add(data: T, opts?: JobOptions): Promise<Job<T>> {
+    return await tracer.trace("queue.add", async span => {
+      span.addTags({
+        "queue.name": this.jobQueue,
+        "job.data.sizeBytes": sizeof(data),
+        ...jobOptsTags(opts || {}),
+      })
+      if (this.opts.jobTags) {
+        span.addTags(this.opts.jobTags(data))
+      }
+      // @ts-expect-error monkey patching the parent span id
+      data._parentSpanContext = {
+        traceId: span.context().toTraceId(),
+        spanId: span.context().toSpanId(),
+      }
+
+      tracer.dogstatsd.distribution(
+        "queue.add.sizeBytes",
+        sizeof(data),
+        this.metricTags()
+      )
+      return await this.withMetrics("queue.add", () =>
+        this.queue.add(data, opts)
+      )
+    })
+  }
+
+  private withMetrics<T>(name: string, cb: () => Promise<T>) {
+    return withMetrics(name, cb, this.metricTags())
+  }
+
+  private metricTags() {
+    return { queueName: this.jobQueue }
+  }
+
+  close() {
+    return this.queue.close()
+  }
+}
+
 export async function shutdown() {
   if (cleanupInterval) {
     timers.clear(cleanupInterval)

packages/backend-core/src/sql/sql.ts

@@ -6,6 +6,7 @@ import {
   isInvalidISODateString,
   isValidFilter,
   isValidISODateString,
+  isValidTime,
   sqlLog,
   validateManyToMany,
 } from "./utils"
@@ -417,6 +418,11 @@ class InternalBuilder {
     }
 
     if (typeof input === "string" && schema.type === FieldType.DATETIME) {
+      if (schema.timeOnly) {
+        if (!isValidTime(input)) {
+          return null
+        }
+      } else {
         if (isInvalidISODateString(input)) {
           return null
         }
@@ -424,6 +430,7 @@ class InternalBuilder {
           return new Date(input.trim())
         }
       }
+    }
     return input
   }
 

packages/backend-core/src/sql/tests/utils.spec.ts

@@ -0,0 +1,35 @@
+import { isValidISODateString, isInvalidISODateString } from "../utils"
+
+describe("ISO date string validity checks", () => {
+  it("accepts a valid ISO date string without a time", () => {
+    const str = "2013-02-01"
+    const valid = isValidISODateString(str)
+    const invalid = isInvalidISODateString(str)
+    expect(valid).toEqual(true)
+    expect(invalid).toEqual(false)
+  })
+
+  it("accepts a valid ISO date string with a time", () => {
+    const str = "2013-02-01T01:23:45Z"
+    const valid = isValidISODateString(str)
+    const invalid = isInvalidISODateString(str)
+    expect(valid).toEqual(true)
+    expect(invalid).toEqual(false)
+  })
+
+  it("accepts a valid ISO date string with a time and millis", () => {
+    const str = "2013-02-01T01:23:45.678Z"
+    const valid = isValidISODateString(str)
+    const invalid = isInvalidISODateString(str)
+    expect(valid).toEqual(true)
+    expect(invalid).toEqual(false)
+  })
+
+  it("rejects an invalid ISO date string", () => {
+    const str = "2013-523-814T444:22:11Z"
+    const valid = isValidISODateString(str)
+    const invalid = isInvalidISODateString(str)
+    expect(valid).toEqual(false)
+    expect(invalid).toEqual(true)
+  })
+})

packages/backend-core/src/sql/utils.ts

@@ -14,7 +14,7 @@ import environment from "../environment"
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
 const ENCODED_SPACE = encodeURIComponent(" ")
-const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}.\d{3}Z)?$/
+const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:.\d{3})?Z)?$/
 const TIME_REGEX = /^(?:\d{2}:)?(?:\d{2}:)(?:\d{2})$/
 
 export function isExternalTableID(tableId: string) {
@@ -139,17 +139,17 @@ export function breakRowIdField(_id: string | { _id: string }): any[] {
   }
 }
 
-export function isInvalidISODateString(str: string) {
+export function isValidISODateString(str: string) {
   const trimmedValue = str.trim()
   if (!ISO_DATE_REGEX.test(trimmedValue)) {
     return false
   }
-  let d = new Date(trimmedValue)
-  return isNaN(d.getTime())
+  const d = new Date(trimmedValue)
+  return !isNaN(d.getTime())
 }
 
-export function isValidISODateString(str: string) {
-  return ISO_DATE_REGEX.test(str.trim())
+export function isInvalidISODateString(str: string) {
+  return !isValidISODateString(str)
 }
 
 export function isValidFilter(value: any) {

packages/bbui/src/Form/Core/TextArea.svelte

@@ -75,6 +75,7 @@
   class:is-disabled={disabled}
   class:is-focused={isFocused}
 >
+  <!-- We need to ignore prettier here as we want no whitespace -->
   <!-- prettier-ignore -->
   <textarea
     bind:this={textarea}
@@ -90,6 +91,7 @@
     on:blur
     on:keypress
   >{value || ""}</textarea>
+  <slot />
 </div>
 
 <style>

packages/bbui/src/Form/Core/TextField.svelte

@@ -114,6 +114,7 @@
     inputmode={getInputMode(type)}
     autocomplete={autocompleteValue}
   />
+  <slot />
 </div>
 
 <style>

packages/bbui/src/Form/Input.svelte

@@ -41,5 +41,7 @@
     on:blur
     on:focus
     on:keyup
-  />
+  >
+    <slot />
+  </TextField>
 </Field>

packages/bbui/src/Form/TextArea.svelte

@@ -7,11 +7,13 @@
   export let label: string | undefined = undefined
   export let labelPosition = "above"
   export let placeholder: string | undefined = undefined
-  export let disabled = false
+  export let readonly: boolean = false
+  export let disabled: boolean = false
   export let error: string | undefined = undefined
   export let height: number | undefined = undefined
   export let minHeight: number | undefined = undefined
   export let helpText: string | undefined = undefined
+  export let updateOnChange: boolean = false
 
   let textarea: TextArea
   export function focus() {
@@ -33,11 +35,16 @@
   <TextArea
     bind:this={textarea}
     {disabled}
+    {readonly}
     {value}
     {placeholder}
     {height}
     {minHeight}
+    {updateOnChange}
     on:change={onChange}
     on:keypress
-  />
+    on:scrollable
+  >
+    <slot />
+  </TextArea>
 </Field>

packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte

@@ -45,7 +45,7 @@
     },
     {
       label: "Multi-select",
-      value: FieldType.ARRAY.type,
+      value: FieldType.ARRAY,
     },
     {
       label: "Barcode/QR",

packages/builder/src/components/common/bindings/DrawerBindableInput.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-  import { Icon, Input, Drawer, Button, CoreTextArea } from "@budibase/bbui"
+  import { Icon, Input, Drawer, Button, TextArea } from "@budibase/bbui"
   import {
     readableToRuntimeBinding,
     runtimeToReadableBinding,
@@ -67,7 +67,7 @@
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <div class="control" class:multiline class:disabled class:scrollable>
   <svelte:component
-    this={multiline ? CoreTextArea : Input}
+    this={multiline ? TextArea : Input}
     {label}
     {disabled}
     readonly={isJS}
@@ -78,7 +78,7 @@
     {placeholder}
     {updateOnChange}
     {autocomplete}
-  />
+  >
     {#if !disabled && !disableBindings}
       <div
         class="icon"
@@ -90,6 +90,7 @@
         <Icon size="S" name="FlashOn" />
       </div>
     {/if}
+  </svelte:component>
 </div>
 <Drawer
   on:drawerHide={onDrawerHide}

packages/client/src/components/app/Text.svelte

@@ -2,7 +2,7 @@
   import { getContext } from "svelte"
   import { MarkdownViewer } from "@budibase/bbui"
 
-  export let text: string = ""
+  export let text: any = ""
   export let color: string | undefined = undefined
   export let align: "left" | "center" | "right" | "justify" = "left"
 
@@ -12,6 +12,9 @@
   // Add in certain settings to styles
   $: styles = enrichStyles($component.styles, color, align)
 
+  // Ensure we're always passing in a string value to the markdown editor
+  $: safeText = stringify(text)
+
   const enrichStyles = (
     styles: any,
     colorStyle: typeof color,
@@ -31,10 +34,24 @@
       },
     }
   }
+
+  const stringify = (text: any): string => {
+    if (text == null) {
+      return ""
+    }
+    if (typeof text !== "string") {
+      try {
+        return JSON.stringify(text)
+      } catch (e) {
+        return ""
+      }
+    }
+    return text
+  }
 </script>
 
 <div use:styleable={styles}>
-  <MarkdownViewer value={text} />
+  <MarkdownViewer value={safeText} />
 </div>
 
 <style>

packages/frontend-core/src/components/grid/stores/config.ts

@@ -82,8 +82,9 @@ export const deriveStores = (context: StoreContext): ConfigDerivedStore => {
         config.canEditColumns = false
       }
 
-      // Determine if we can select rows
-      config.canSelectRows = !!config.canDeleteRows || !!config.canAddRows
+      // Determine if we can select rows. Always true in the meantime as you can
+      // use the selected rows binding regardless of readonly state.
+      config.canSelectRows = true
 
       return config
     }

packages/server/src/api/controllers/assets.ts

@@ -0,0 +1,12 @@
+import { join } from "../../utilities/centralPath"
+import { TOP_LEVEL_PATH, DEV_ASSET_PATH } from "../../utilities/fileSystem"
+import { Ctx } from "@budibase/types"
+import env from "../../environment"
+import send from "koa-send"
+
+// this is a public endpoint with no middlewares
+export const serveBuilderAssets = async function (ctx: Ctx<void, void>) {
+  let topLevelPath = env.isDev() ? DEV_ASSET_PATH : TOP_LEVEL_PATH
+  const builderPath = join(topLevelPath, "builder")
+  await send(ctx, ctx.file, { root: builderPath })
+}

packages/server/src/api/controllers/static/index.ts

@@ -76,11 +76,6 @@ export const toggleBetaUiFeature = async function (
   }
 }
 
-export const serveBuilder = async function (ctx: Ctx<void, void>) {
-  const builderPath = join(TOP_LEVEL_PATH, "builder")
-  await send(ctx, ctx.file, { root: builderPath })
-}
-
 export const uploadFile = async function (
   ctx: Ctx<void, ProcessAttachmentResponse>
 ) {

packages/server/src/api/controllers/table/tests/utils.spec.ts

@@ -1,4 +1,9 @@
-import { AIOperationEnum, AutoFieldSubType, FieldType } from "@budibase/types"
+import {
+  AIOperationEnum,
+  AutoFieldSubType,
+  FieldType,
+  JsonFieldSubType,
+} from "@budibase/types"
 import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
 import { importToRows } from "../utils"
 
@@ -152,5 +157,33 @@ describe("utils", () => {
         ])
       })
     })
+
+    it("coerces strings into arrays for array fields", async () => {
+      await config.doInContext(config.appId, async () => {
+        const table = await config.createTable({
+          name: "table",
+          type: "table",
+          schema: {
+            colours: {
+              name: "colours",
+              type: FieldType.ARRAY,
+              constraints: {
+                type: JsonFieldSubType.ARRAY,
+                inclusion: ["red"],
+              },
+            },
+          },
+        })
+
+        const data = [{ colours: "red" }]
+
+        const result = await importToRows(data, table, config.user?._id)
+        expect(result).toEqual([
+          expect.objectContaining({
+            colours: ["red"],
+          }),
+        ])
+      })
+    })
   })
 })

packages/server/src/api/controllers/table/utils.ts

@@ -155,13 +155,19 @@ export async function importToRows(
           schema.type === FieldType.ARRAY) &&
         row[fieldName]
       ) {
-        const rowVal = Array.isArray(row[fieldName])
-          ? row[fieldName]
-          : [row[fieldName]]
+        const isArray = Array.isArray(row[fieldName])
+
+        // Add option to inclusion constraints
+        const rowVal = isArray ? row[fieldName] : [row[fieldName]]
         let merged = [...schema.constraints!.inclusion!, ...rowVal]
         let superSet = new Set(merged)
         schema.constraints!.inclusion = Array.from(superSet)
         schema.constraints!.inclusion.sort()
+
+        // If array type, ensure we import the value as an array
+        if (!isArray && schema.type === FieldType.ARRAY) {
+          row[fieldName] = rowVal
+        }
       }
     }
 

packages/server/src/api/index.ts

@@ -8,6 +8,7 @@ import { middleware as pro } from "@budibase/pro"
 import { apiEnabled, automationsEnabled } from "../features"
 import migrations from "../middleware/appMigrations"
 import { automationQueue } from "../automations"
+import assetRouter from "./routes/assets"
 
 export { shutdown } from "./routes/public"
 const compress = require("koa-compress")
@@ -16,7 +17,7 @@ export const router: Router = new Router()
 
 router.get("/health", async ctx => {
   if (automationsEnabled()) {
-    if (!(await automationQueue.isReady())) {
+    if (!(await automationQueue.getBullQueue().isReady())) {
       ctx.status = 503
       return
     }
@@ -44,6 +45,12 @@ if (apiEnabled()) {
     )
     // re-direct before any middlewares occur
     .redirect("/", "/builder")
+
+  // send assets before middleware
+  router.use(assetRouter.routes())
+  router.use(assetRouter.allowedMethods())
+
+  router
     .use(
       auth.buildAuthMiddleware([], {
         publicAllowed: true,

packages/server/src/api/routes/assets.ts

@@ -0,0 +1,11 @@
+import { addFileManagement } from "../utils"
+import { serveBuilderAssets } from "../controllers/assets"
+import Router from "@koa/router"
+
+const router: Router = new Router()
+
+addFileManagement(router)
+
+router.get("/builder/:file*", serveBuilderAssets)
+
+export default router

packages/server/src/api/routes/static.ts

@@ -1,35 +1,17 @@
 import Router from "@koa/router"
 import * as controller from "../controllers/static"
-import { budibaseTempDir } from "../../utilities/budibaseDir"
 import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
-import env from "../../environment"
+import { addFileManagement } from "../utils"
 import { paramResource } from "../../middleware/resourceId"
-import { devClientLibPath } from "../../utilities/fileSystem"
 
 const { BUILDER, PermissionType, PermissionLevel } = permissions
 
 const router: Router = new Router()
 
-/* istanbul ignore next */
-router.param("file", async (file: any, ctx: any, next: any) => {
-  ctx.file = file && file.includes(".") ? file : "index.html"
-  if (!ctx.file.startsWith("budibase-client")) {
-    return next()
-  }
-  // test serves from require
-  if (env.isTest()) {
-    const path = devClientLibPath()
-    ctx.devPath = path.split(ctx.file)[0]
-  } else if (env.isDev()) {
-    // Serving the client library from your local dir in dev
-    ctx.devPath = budibaseTempDir()
-  }
-  return next()
-})
+addFileManagement(router)
 
 router
-  .get("/builder/:file*", controller.serveBuilder)
   .get("/api/assets/client", controller.serveClientLibrary)
   .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
   .post("/api/beta/:feature", controller.toggleBetaUiFeature)

packages/server/src/api/routes/tests/assets.spec.ts

@@ -0,0 +1,34 @@
+import fs from "fs"
+import { join } from "path"
+import { DEV_ASSET_PATH } from "../../../utilities/fileSystem"
+import * as setup from "./utilities"
+
+const path = join(DEV_ASSET_PATH, "builder")
+let addedPath = false
+const config = setup.getConfig()
+
+beforeAll(async () => {
+  if (!fs.existsSync(path)) {
+    addedPath = true
+    fs.mkdirSync(path)
+  }
+  const indexPath = join(path, "index.html")
+  if (!fs.existsSync(indexPath)) {
+    fs.writeFileSync(indexPath, "<html></html>", "utf8")
+    addedPath = true
+  }
+  await config.init()
+})
+
+afterAll(() => {
+  if (addedPath) {
+    fs.rmSync(path, { recursive: true })
+  }
+})
+
+describe("/builder/:file*", () => {
+  it("should be able to retrieve the builder file", async () => {
+    const res = await config.api.assets.get("index.html")
+    expect(res.text).toContain("<html")
+  })
+})

packages/server/src/api/routes/tests/row.spec.ts

@@ -2276,6 +2276,25 @@ if (descriptions.length) {
           expect(updated.attachment.key).toBe(newAttachment.key)
         })
 
+        it("should allow updating signature row", async () => {
+          const { table, row } = await coreAttachmentEnrichment(
+            {
+              signature: {
+                type: FieldType.SIGNATURE_SINGLE,
+                name: "signature",
+                constraints: { presence: false },
+              },
+            },
+            "signature",
+            `${uuid.v4()}.png`
+          )
+
+          const newSignature = generateAttachment(`${uuid.v4()}.png`)
+          row["signature"] = newSignature
+          const updated = await config.api.row.save(table._id!, row)
+          expect(updated.signature.key).toBe(newSignature.key)
+        })
+
         it("should allow enriching attachment list rows", async () => {
           await coreAttachmentEnrichment(
             {

packages/server/src/api/utils.ts

@@ -0,0 +1,23 @@
+import env from "../environment"
+import { devClientLibPath } from "../utilities/fileSystem"
+import { budibaseTempDir } from "../utilities/budibaseDir"
+import Router from "@koa/router"
+
+export function addFileManagement(router: Router) {
+  /* istanbul ignore next */
+  router.param("file", async (file: any, ctx: any, next: any) => {
+    ctx.file = file && file.includes(".") ? file : "index.html"
+    if (!ctx.file.startsWith("budibase-client")) {
+      return next()
+    }
+    // test serves from require
+    if (env.isTest()) {
+      const path = devClientLibPath()
+      ctx.devPath = path.split(ctx.file)[0]
+    } else if (env.isDev()) {
+      // Serving the client library from your local dir in dev
+      ctx.devPath = budibaseTempDir()
+    }
+    return next()
+  })
+}

packages/server/src/appMigrations/queue.ts

@@ -13,7 +13,7 @@ export type AppMigrationJob = {
 
 // always create app migration queue - so that events can be pushed and read from it
 // across the different api and automation services
-const appMigrationQueue = queue.createQueue<AppMigrationJob>(
+const appMigrationQueue = new queue.BudibaseQueue<AppMigrationJob>(
   queue.JobQueue.APP_MIGRATION,
   {
     jobOptions: {

packages/server/src/automations/bullboard.ts

@@ -7,25 +7,36 @@ import { getAppMigrationQueue } from "../appMigrations/queue"
 import { createBullBoard } from "@bull-board/api"
 import { AutomationData } from "@budibase/types"
 
-export const automationQueue = queue.createQueue<AutomationData>(
+export const automationQueue = new queue.BudibaseQueue<AutomationData>(
   queue.JobQueue.AUTOMATION,
-  { removeStalledCb: automation.removeStalled }
+  {
+    removeStalledCb: automation.removeStalled,
+    jobTags: (job: AutomationData) => {
+      return {
+        "automation.id": job.automation._id,
+        "automation.name": job.automation.name,
+        "automation.appId": job.automation.appId,
+        "automation.createdAt": job.automation.createdAt,
+        "automation.trigger": job.automation.definition.trigger.stepId,
+      }
+    },
+  }
 )
 
 const PATH_PREFIX = "/bulladmin"
 
 export async function init() {
   // Set up queues for bull board admin
-  const queues = [new BullAdapter(automationQueue)]
+  const queues = [new BullAdapter(automationQueue.getBullQueue())]
 
   const backupQueue = backups.getBackupQueue()
   if (backupQueue) {
-    queues.push(new BullAdapter(backupQueue))
+    queues.push(new BullAdapter(backupQueue.getBullQueue()))
   }
 
   const appMigrationQueue = getAppMigrationQueue()
   if (appMigrationQueue) {
-    queues.push(new BullAdapter(appMigrationQueue))
+    queues.push(new BullAdapter(appMigrationQueue.getBullQueue()))
   }
 
   const serverAdapter = new KoaAdapter()

packages/server/src/automations/tests/utilities/index.ts

@@ -22,7 +22,7 @@ export function afterAll() {
 }
 
 export function getTestQueue(): queue.InMemoryQueue<AutomationData> {
-  return getQueue() as unknown as queue.InMemoryQueue<AutomationData>
+  return getQueue().getBullQueue() as unknown as queue.InMemoryQueue<AutomationData>
 }
 
 export function triggerCron(message: Job<AutomationData>) {
@@ -48,7 +48,7 @@ export async function runInProd(fn: any) {
 
 export async function captureAllAutomationRemovals(f: () => Promise<unknown>) {
   const messages: Job<AutomationData>[] = []
-  const queue = getQueue()
+  const queue = getQueue().getBullQueue()
 
   const messageListener = async (message: Job<AutomationData>) => {
     messages.push(message)
@@ -82,7 +82,7 @@ export async function captureAutomationRemovals(
 
 export async function captureAllAutomationMessages(f: () => Promise<unknown>) {
   const messages: Job<AutomationData>[] = []
-  const queue = getQueue()
+  const queue = getQueue().getBullQueue()
 
   const messageListener = async (message: Job<AutomationData>) => {
     messages.push(message)
@@ -122,7 +122,7 @@ export async function captureAllAutomationResults(
   f: () => Promise<unknown>
 ): Promise<queue.TestQueueMessage<AutomationData>[]> {
   const runs: queue.TestQueueMessage<AutomationData>[] = []
-  const queue = getQueue()
+  const queue = getQueue().getBullQueue()
   let messagesOutstanding = 0
 
   const completedListener = async (

packages/server/src/automations/utils.ts

@@ -107,12 +107,14 @@ export async function updateTestHistory(
 // end the repetition and the job itself
 export async function disableAllCrons(appId: any) {
   const promises = []
-  const jobs = await automationQueue.getRepeatableJobs()
+  const jobs = await automationQueue.getBullQueue().getRepeatableJobs()
   for (let job of jobs) {
     if (job.key.includes(`${appId}_cron`)) {
-      promises.push(automationQueue.removeRepeatableByKey(job.key))
+      promises.push(
+        automationQueue.getBullQueue().removeRepeatableByKey(job.key)
+      )
       if (job.id) {
-        promises.push(automationQueue.removeJobs(job.id))
+        promises.push(automationQueue.getBullQueue().removeJobs(job.id))
       }
     }
   }
@@ -121,10 +123,10 @@ export async function disableAllCrons(appId: any) {
 }
 
 export async function disableCronById(jobId: JobId) {
-  const jobs = await automationQueue.getRepeatableJobs()
+  const jobs = await automationQueue.getBullQueue().getRepeatableJobs()
   for (const job of jobs) {
     if (job.id === jobId) {
-      await automationQueue.removeRepeatableByKey(job.key)
+      await automationQueue.getBullQueue().removeRepeatableByKey(job.key)
     }
   }
   console.log(`jobId=${jobId} disabled`)

packages/server/src/integrations/microsoftSqlServer.ts

@@ -371,8 +371,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
           ? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;`
           : query.sql
       this.log(sql, query.bindings)
-      const resp = await request.query(sql)
-      return resp
+      return await request.query(sql)
     } catch (err: any) {
       let readableMessage = getReadableErrorMessage(
         SourceName.SQL_SERVER,

packages/server/src/sdk/app/oauth2/tests/utils.spec.ts

@@ -190,7 +190,7 @@ describe("oauth2 utils", () => {
 
         await config.doInContext(config.appId, () => getToken(oauthConfig._id))
         await testUtils.queue.processMessages(
-          cache.docWritethrough.DocWritethroughProcessor.queue
+          cache.docWritethrough.DocWritethroughProcessor.queue.getBullQueue()
         )
 
         const usageLog = await config.doInContext(config.appId, () =>
@@ -216,7 +216,7 @@ describe("oauth2 utils", () => {
           config.doInContext(config.appId, () => getToken(oauthConfig._id))
         ).rejects.toThrow()
         await testUtils.queue.processMessages(
-          cache.docWritethrough.DocWritethroughProcessor.queue
+          cache.docWritethrough.DocWritethroughProcessor.queue.getBullQueue()
         )
 
         const usageLog = await config.doInContext(config.appId, () =>
@@ -247,7 +247,7 @@ describe("oauth2 utils", () => {
           getToken(oauthConfig._id)
         )
         await testUtils.queue.processMessages(
-          cache.docWritethrough.DocWritethroughProcessor.queue
+          cache.docWritethrough.DocWritethroughProcessor.queue.getBullQueue()
         )
 
         for (const appId of [config.appId, config.prodAppId]) {

packages/server/src/tests/utilities/api/assets.ts

@@ -0,0 +1,8 @@
+import { TestAPI } from "./base"
+
+export class AssetsAPI extends TestAPI {
+  get = async (path: string) => {
+    // has to be raw, body isn't JSON
+    return await this._requestRaw("get", `/builder/${path}`)
+  }
+}

packages/server/src/tests/utilities/api/index.ts

@@ -21,6 +21,7 @@ import { EnvironmentAPI } from "./environment"
 import { UserPublicAPI } from "./public/user"
 import { MiscAPI } from "./misc"
 import { OAuth2API } from "./oauth2"
+import { AssetsAPI } from "./assets"
 
 export default class API {
   application: ApplicationAPI
@@ -44,6 +45,7 @@ export default class API {
   user: UserAPI
   viewV2: ViewV2API
   webhook: WebhookAPI
+  assets: AssetsAPI
 
   public: {
     user: UserPublicAPI
@@ -71,6 +73,7 @@ export default class API {
     this.user = new UserAPI(config)
     this.viewV2 = new ViewV2API(config)
     this.webhook = new WebhookAPI(config)
+    this.assets = new AssetsAPI(config)
     this.public = {
       user: new UserPublicAPI(config),
     }

packages/server/src/utilities/fileSystem/filesystem.ts

@@ -8,6 +8,7 @@ import { v4 as uuid } from "uuid"
 
 export const TOP_LEVEL_PATH =
   env.TOP_LEVEL_PATH || resolve(join(__dirname, "..", "..", ".."))
+export const DEV_ASSET_PATH = join(TOP_LEVEL_PATH, "packages", "server")
 
 /**
  * Upon first startup of instance there may not be everything we need in tmp directory, set it up.

packages/server/src/utilities/schema.ts

@@ -106,7 +106,7 @@ export function validate(
       } else if (
         // If provided must be a valid date
         columnType === FieldType.DATETIME &&
-        isNaN(new Date(columnData).getTime())
+        sql.utils.isInvalidISODateString(columnData)
       ) {
         results.schemaValidation[columnName] = false
       } else if (

packages/shared-core/src/constants/fields.ts

@@ -30,4 +30,12 @@ export const SWITCHABLE_TYPES: SwitchableTypes = {
     FieldType.LONGFORM,
   ],
   [FieldType.NUMBER]: [FieldType.NUMBER, FieldType.BOOLEAN],
+  [FieldType.JSON]: [
+    FieldType.JSON,
+    FieldType.ARRAY,
+    FieldType.ATTACHMENTS,
+    FieldType.ATTACHMENT_SINGLE,
+    FieldType.BB_REFERENCE,
+    FieldType.SIGNATURE_SINGLE,
+  ],
 }

packages/types/src/documents/app/row.ts

@@ -128,6 +128,7 @@ export enum FieldType {
 export const JsonTypes = [
   FieldType.ATTACHMENT_SINGLE,
   FieldType.ATTACHMENTS,
+  FieldType.SIGNATURE_SINGLE,
   // only BB_REFERENCE is JSON, it's an array, BB_REFERENCE_SINGLE is a string type
   FieldType.BB_REFERENCE,
   FieldType.JSON,