Merge branch 'develop' of github.com:Budibase/budibase into more-grid-tweaks

hosting/proxy/nginx.prod.conf

@@ -222,9 +222,9 @@ http {
       rewrite ^/files/signed/(.*)$ /$1 break;
     }
 
-    client_header_timeout 60;
-    client_body_timeout   60;
-    keepalive_timeout     60;
+    client_header_timeout 120;
+    client_body_timeout   120;
+    keepalive_timeout     120;
 
     # gzip
     gzip on;

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.5.6-alpha.39",
+  "version": "2.5.10-alpha.1",
   "npmClient": "yarn",
   "useWorkspaces": true,
   "packages": ["packages/*"],

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.5.6-alpha.39",
+  "version": "2.5.10-alpha.1",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -24,7 +24,7 @@
   "dependencies": {
     "@budibase/nano": "10.1.2",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.5.6-alpha.39",
+    "@budibase/types": "2.5.10-alpha.1",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",

packages/backend-core/src/cache/writethrough.ts

@@ -47,7 +47,7 @@ async function put(
         type: LockType.TRY_ONCE,
         name: LockName.PERSIST_WRITETHROUGH,
         resource: key,
-        ttl: 1000,
+        ttl: 15000,
       },
       async () => {
         const writeDb = async (toWrite: any) => {
@@ -71,6 +71,7 @@ async function put(
         }
       }
     )
+
     if (!lockResponse.executed) {
       logWarn(`Ignoring redlock conflict in write-through cache`)
     }

packages/backend-core/src/db/lucene.ts

@@ -434,7 +434,7 @@ export class QueryBuilder<T> {
       })
     }
     if (this.#query.empty) {
-      build(this.#query.empty, (key: string) => `!${key}:["" TO *]`)
+      build(this.#query.empty, (key: string) => `(*:* -${key}:["" TO *])`)
     }
     if (this.#query.notEmpty) {
       build(this.#query.notEmpty, (key: string) => `${key}:["" TO *]`)

packages/backend-core/src/environment.ts

@@ -154,6 +154,7 @@ const environment = {
     ? process.env.ENABLE_SSO_MAINTENANCE_MODE
     : false,
   VERSION: findVersion(),
+  DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore

packages/backend-core/src/events/publishers/license.ts

@@ -3,7 +3,6 @@ import {
   Event,
   LicenseActivatedEvent,
   LicensePlanChangedEvent,
-  LicenseTierChangedEvent,
   PlanType,
   Account,
   LicensePortalOpenedEvent,
@@ -11,22 +10,23 @@ import {
   LicenseCheckoutOpenedEvent,
   LicensePaymentFailedEvent,
   LicensePaymentRecoveredEvent,
+  PriceDuration,
 } from "@budibase/types"
 
-async function tierChanged(account: Account, from: number, to: number) {
-  const properties: LicenseTierChangedEvent = {
-    accountId: account.accountId,
-    to,
-    from,
+async function planChanged(
+  account: Account,
+  opts: {
+    from: PlanType
+    to: PlanType
+    fromQuantity: number | undefined
+    toQuantity: number | undefined
+    fromDuration: PriceDuration | undefined
+    toDuration: PriceDuration | undefined
   }
-  await publishEvent(Event.LICENSE_TIER_CHANGED, properties)
-}
-
-async function planChanged(account: Account, from: PlanType, to: PlanType) {
+) {
   const properties: LicensePlanChangedEvent = {
     accountId: account.accountId,
-    to,
-    from,
+    ...opts,
   }
   await publishEvent(Event.LICENSE_PLAN_CHANGED, properties)
 }
@@ -74,7 +74,6 @@ async function paymentRecovered(account: Account) {
 }
 
 export default {
-  tierChanged,
   planChanged,
   activated,
   checkoutOpened,

packages/backend-core/src/logging/index.ts

@@ -1,5 +1,5 @@
 export * as correlation from "./correlation/correlation"
-export { logger, disableLogger } from "./pino/logger"
+export { logger } from "./pino/logger"
 export * from "./alerts"
 
 // turn off or on context logging i.e. tenantId, appId etc

packages/backend-core/src/logging/pino/logger.ts

@@ -5,184 +5,169 @@ import * as correlation from "../correlation"
 import { IdentityType } from "@budibase/types"
 import { LOG_CONTEXT } from "../index"
 
-// CORE LOGGERS - for disabling
-
-const BUILT_INS = {
-  log: console.log,
-  error: console.error,
-  info: console.info,
-  warn: console.warn,
-  trace: console.trace,
-  debug: console.debug,
-}
-
 // LOGGER
 
-const pinoOptions: LoggerOptions = {
-  level: env.LOG_LEVEL,
-  formatters: {
-    level: label => {
-      return { level: label.toUpperCase() }
-    },
-    bindings: () => {
-      return {}
-    },
-  },
-  timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
-}
-
-if (env.isDev()) {
-  pinoOptions.transport = {
-    target: "pino-pretty",
-    options: {
-      singleLine: true,
+let pinoInstance: pino.Logger | undefined
+if (!env.DISABLE_PINO_LOGGER) {
+  const pinoOptions: LoggerOptions = {
+    level: env.LOG_LEVEL,
+    formatters: {
+      level: label => {
+        return { level: label.toUpperCase() }
+      },
+      bindings: () => {
+        return {}
+      },
     },
+    timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
   }
-}
 
-export const logger = pino(pinoOptions)
-
-export function disableLogger() {
-  console.log = BUILT_INS.log
-  console.error = BUILT_INS.error
-  console.info = BUILT_INS.info
-  console.warn = BUILT_INS.warn
-  console.trace = BUILT_INS.trace
-  console.debug = BUILT_INS.debug
-}
-
-// CONSOLE OVERRIDES
-
-interface MergingObject {
-  objects?: any[]
-  tenantId?: string
-  appId?: string
-  identityId?: string
-  identityType?: IdentityType
-  correlationId?: string
-  err?: Error
-}
-
-function isPlainObject(obj: any) {
-  return typeof obj === "object" && obj !== null && !(obj instanceof Error)
-}
-
-function isError(obj: any) {
-  return obj instanceof Error
-}
-
-function isMessage(obj: any) {
-  return typeof obj === "string"
-}
-
-/**
- * Backwards compatibility between console logging statements
- * and pino logging requirements.
- */
-function getLogParams(args: any[]): [MergingObject, string] {
-  let error = undefined
-  let objects: any[] = []
-  let message = ""
-
-  args.forEach(arg => {
-    if (isMessage(arg)) {
-      message = `${message} ${arg}`.trimStart()
-    }
-    if (isPlainObject(arg)) {
-      objects.push(arg)
-    }
-    if (isError(arg)) {
-      error = arg
-    }
-  })
-
-  const identity = getIdentity()
-
-  let contextObject = {}
-
-  if (LOG_CONTEXT) {
-    contextObject = {
-      tenantId: getTenantId(),
-      appId: getAppId(),
-      identityId: identity?._id,
-      identityType: identity?.type,
-      correlationId: correlation.getId(),
+  if (env.isDev()) {
+    pinoOptions.transport = {
+      target: "pino-pretty",
+      options: {
+        singleLine: true,
+      },
     }
   }
 
-  const mergingObject = {
-    objects: objects.length ? objects : undefined,
-    err: error,
-    ...contextObject,
+  pinoInstance = pino(pinoOptions)
+
+  // CONSOLE OVERRIDES
+
+  interface MergingObject {
+    objects?: any[]
+    tenantId?: string
+    appId?: string
+    identityId?: string
+    identityType?: IdentityType
+    correlationId?: string
+    err?: Error
   }
 
-  return [mergingObject, message]
-}
-
-console.log = (...arg: any[]) => {
-  const [obj, msg] = getLogParams(arg)
-  logger.info(obj, msg)
-}
-console.info = (...arg: any[]) => {
-  const [obj, msg] = getLogParams(arg)
-  logger.info(obj, msg)
-}
-console.warn = (...arg: any[]) => {
-  const [obj, msg] = getLogParams(arg)
-  logger.warn(obj, msg)
-}
-console.error = (...arg: any[]) => {
-  const [obj, msg] = getLogParams(arg)
-  logger.error(obj, msg)
-}
-
-/**
- * custom trace impl - this resembles the node trace behaviour rather
- * than traditional trace logging
- * @param arg
- */
-console.trace = (...arg: any[]) => {
-  const [obj, msg] = getLogParams(arg)
-  if (!obj.err) {
-    // to get stack trace
-    obj.err = new Error()
+  function isPlainObject(obj: any) {
+    return typeof obj === "object" && obj !== null && !(obj instanceof Error)
   }
-  logger.trace(obj, msg)
-}
 
-console.debug = (...arg: any) => {
-  const [obj, msg] = getLogParams(arg)
-  logger.debug(obj, msg)
-}
-
-// CONTEXT
-
-const getTenantId = () => {
-  let tenantId
-  try {
-    tenantId = context.getTenantId()
-  } catch (e: any) {
-    // do nothing
+  function isError(obj: any) {
+    return obj instanceof Error
+  }
+
+  function isMessage(obj: any) {
+    return typeof obj === "string"
+  }
+
+  /**
+   * Backwards compatibility between console logging statements
+   * and pino logging requirements.
+   */
+  function getLogParams(args: any[]): [MergingObject, string] {
+    let error = undefined
+    let objects: any[] = []
+    let message = ""
+
+    args.forEach(arg => {
+      if (isMessage(arg)) {
+        message = `${message} ${arg}`.trimStart()
+      }
+      if (isPlainObject(arg)) {
+        objects.push(arg)
+      }
+      if (isError(arg)) {
+        error = arg
+      }
+    })
+
+    const identity = getIdentity()
+
+    let contextObject = {}
+
+    if (LOG_CONTEXT) {
+      contextObject = {
+        tenantId: getTenantId(),
+        appId: getAppId(),
+        identityId: identity?._id,
+        identityType: identity?.type,
+        correlationId: correlation.getId(),
+      }
+    }
+
+    const mergingObject = {
+      objects: objects.length ? objects : undefined,
+      err: error,
+      ...contextObject,
+    }
+
+    return [mergingObject, message]
+  }
+
+  console.log = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.info(obj, msg)
+  }
+  console.info = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.info(obj, msg)
+  }
+  console.warn = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.warn(obj, msg)
+  }
+  console.error = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.error(obj, msg)
+  }
+
+  /**
+   * custom trace impl - this resembles the node trace behaviour rather
+   * than traditional trace logging
+   * @param arg
+   */
+  console.trace = (...arg: any[]) => {
+    const [obj, msg] = getLogParams(arg)
+    if (!obj.err) {
+      // to get stack trace
+      obj.err = new Error()
+    }
+    pinoInstance?.trace(obj, msg)
+  }
+
+  console.debug = (...arg: any) => {
+    const [obj, msg] = getLogParams(arg)
+    pinoInstance?.debug(obj, msg)
+  }
+
+  // CONTEXT
+
+  const getTenantId = () => {
+    let tenantId
+    try {
+      tenantId = context.getTenantId()
+    } catch (e: any) {
+      // do nothing
+    }
+    return tenantId
+  }
+
+  const getAppId = () => {
+    let appId
+    try {
+      appId = context.getAppId()
+    } catch (e) {
+      // do nothing
+    }
+    return appId
+  }
+
+  const getIdentity = () => {
+    let identity
+    try {
+      identity = context.getIdentity()
+    } catch (e) {
+      // do nothing
+    }
+    return identity
   }
-  return tenantId
 }
 
-const getAppId = () => {
-  let appId
-  try {
-    appId = context.getAppId()
-  } catch (e) {
-    // do nothing
-  }
-  return appId
-}
-
-const getIdentity = () => {
-  let identity
-  try {
-    identity = context.getIdentity()
-  } catch (e) {
-    // do nothing
-  }
-  return identity
-}
+export const logger = pinoInstance

packages/backend-core/tests/core/utilities/mocks/events.ts

@@ -123,7 +123,6 @@ beforeAll(async () => {
   jest.spyOn(events.plugin, "imported")
   jest.spyOn(events.plugin, "deleted")
 
-  jest.spyOn(events.license, "tierChanged")
   jest.spyOn(events.license, "planChanged")
   jest.spyOn(events.license, "activated")
   jest.spyOn(events.license, "checkoutOpened")

packages/backend-core/tests/core/utilities/structures/licenses.ts

@@ -7,16 +7,29 @@ import {
   PlanType,
   PriceDuration,
   PurchasedPlan,
+  PurchasedPrice,
   Quotas,
   Subscription,
 } from "@budibase/types"
 
+export function price(): PurchasedPrice {
+  return {
+    amount: 10000,
+    amountMonthly: 10000,
+    currency: "usd",
+    duration: PriceDuration.MONTHLY,
+    priceId: "price_123",
+    dayPasses: undefined,
+    isPerUser: true,
+  }
+}
+
 export const plan = (type: PlanType = PlanType.FREE): PurchasedPlan => {
   return {
     type,
     usesInvoicing: false,
-    minUsers: 1,
     model: PlanModel.PER_USER,
+    price: type !== PlanType.FREE ? price() : undefined,
   }
 }
 

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.5.6-alpha.39",
+  "version": "2.5.10-alpha.1",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,8 +38,8 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/shared-core": "2.5.6-alpha.39",
-    "@budibase/string-templates": "2.5.6-alpha.39",
+    "@budibase/shared-core": "2.5.10-alpha.1",
+    "@budibase/string-templates": "2.5.10-alpha.1",
     "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.5.6-alpha.39",
+  "version": "2.5.10-alpha.1",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -58,10 +58,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.5.6-alpha.39",
-    "@budibase/frontend-core": "2.5.6-alpha.39",
-    "@budibase/shared-core": "2.5.6-alpha.39",
-    "@budibase/string-templates": "2.5.6-alpha.39",
+    "@budibase/bbui": "2.5.10-alpha.1",
+    "@budibase/frontend-core": "2.5.10-alpha.1",
+    "@budibase/shared-core": "2.5.10-alpha.1",
+    "@budibase/string-templates": "2.5.10-alpha.1",
     "@fortawesome/fontawesome-svg-core": "^6.2.1",
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",

packages/builder/src/components/backend/TableNavigator/utils.js

@@ -42,16 +42,7 @@ export const parseFile = e => {
 
     reader.addEventListener("load", function (e) {
       const fileData = e.target.result
-
-      if (file.type === "text/csv") {
-        API.csvToJson(fileData)
-          .then(rows => {
-            resolveRows(rows)
-          })
-          .catch(() => {
-            reject("can't convert csv to json")
-          })
-      } else if (file.type === "application/json") {
+      if (file.type?.includes("json")) {
         const parsedFileData = JSON.parse(fileData)
 
         if (Array.isArray(parsedFileData)) {
@@ -62,7 +53,13 @@ export const parseFile = e => {
           reject("invalid json format")
         }
       } else {
-        reject("invalid file type")
+        API.csvToJson(fileData)
+          .then(rows => {
+            resolveRows(rows)
+          })
+          .catch(() => {
+            reject("cannot parse csv")
+          })
       }
     })
 

packages/builder/src/components/usage/Usage.svelte

@@ -27,7 +27,7 @@
   onMount(() => {
     unlimited = isUnlimited()
     percentage = getPercentage()
-    if (warnWhenFull && percentage === 100) {
+    if (warnWhenFull && percentage >= 100) {
       showWarning = true
     }
   })

packages/builder/src/stores/portal/users.js

@@ -41,7 +41,7 @@ export function createUsersStore() {
       inviteCode,
       password,
       firstName,
-      lastName,
+      lastName: !lastName?.trim() ? undefined : lastName,
     })
   }
 

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.5.6-alpha.39",
+  "version": "2.5.10-alpha.1",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "dist/index.js",
   "bin": {
@@ -29,14 +29,14 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.5.6-alpha.39",
-    "@budibase/string-templates": "2.5.6-alpha.39",
-    "@budibase/types": "2.5.6-alpha.39",
+    "@budibase/backend-core": "2.5.10-alpha.1",
+    "@budibase/string-templates": "2.5.10-alpha.1",
+    "@budibase/types": "2.5.10-alpha.1",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",
     "commander": "7.1.0",
-    "docker-compose": "0.23.12",
+    "docker-compose": "0.24.0",
     "dotenv": "16.0.1",
     "download": "8.0.0",
     "find-free-port": "^2.0.0",

packages/cli/src/index.ts

@@ -1,6 +1,5 @@
 #!/usr/bin/env node
-import { logging } from "@budibase/backend-core"
-logging.disableLogger()
+process.env.DISABLE_PINO_LOGGER = "1"
 import "./prebuilds"
 import "./environment"
 import { env } from "@budibase/backend-core"

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.5.6-alpha.39",
+  "version": "2.5.10-alpha.1",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,11 +19,11 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.5.6-alpha.39",
-    "@budibase/frontend-core": "2.5.6-alpha.39",
-    "@budibase/shared-core": "2.5.6-alpha.39",
-    "@budibase/string-templates": "2.5.6-alpha.39",
-    "@budibase/types": "2.5.6-alpha.39",
+    "@budibase/bbui": "2.5.10-alpha.1",
+    "@budibase/frontend-core": "2.5.10-alpha.1",
+    "@budibase/shared-core": "2.5.10-alpha.1",
+    "@budibase/string-templates": "2.5.10-alpha.1",
+    "@budibase/types": "2.5.10-alpha.1",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.5.6-alpha.39",
+  "version": "2.5.10-alpha.1",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.5.6-alpha.39",
-    "@budibase/shared-core": "2.5.6-alpha.39",
+    "@budibase/bbui": "2.5.10-alpha.1",
+    "@budibase/shared-core": "2.5.10-alpha.1",
     "dayjs": "^1.11.7",
     "lodash": "^4.17.21",
     "socket.io-client": "^4.6.1",

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.5.6-alpha.39",
+  "version": "2.5.10-alpha.1",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.5.6-alpha.39",
+  "version": "2.5.10-alpha.1",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -45,12 +45,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.5.6-alpha.39",
-    "@budibase/client": "2.5.6-alpha.39",
-    "@budibase/pro": "2.5.6-alpha.39",
-    "@budibase/shared-core": "2.5.6-alpha.39",
-    "@budibase/string-templates": "2.5.6-alpha.39",
-    "@budibase/types": "2.5.6-alpha.39",
+    "@budibase/backend-core": "2.5.10-alpha.1",
+    "@budibase/client": "2.5.10-alpha.1",
+    "@budibase/pro": "2.5.10-alpha.1",
+    "@budibase/shared-core": "2.5.10-alpha.1",
+    "@budibase/string-templates": "2.5.10-alpha.1",
+    "@budibase/types": "2.5.10-alpha.1",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/controllers/row/internal.ts

@@ -37,7 +37,7 @@ import {
   Table,
 } from "@budibase/types"
 
-const { cleanExportRows } = require("./utils")
+import { cleanExportRows } from "./utils"
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -391,6 +391,9 @@ export async function exportRows(ctx: UserCtx) {
   const table = await db.get(ctx.params.tableId)
   const rowIds = ctx.request.body.rows
   let format = ctx.query.format
+  if (typeof format !== "string") {
+    ctx.throw(400, "Format parameter is not valid")
+  }
   const { columns, query } = ctx.request.body
 
   let result

packages/server/src/api/controllers/row/utils.ts

@@ -137,8 +137,8 @@ export function cleanExportRows(
     delete schema[column]
   })
 
-  // Intended to avoid 'undefined' in export
   if (format === Format.CSV) {
+    // Intended to append empty values in export
     const schemaKeys = Object.keys(schema)
     for (let key of schemaKeys) {
       if (columns?.length && columns.indexOf(key) > 0) {
@@ -146,7 +146,7 @@ export function cleanExportRows(
       }
       for (let row of cleanRows) {
         if (row[key] == null) {
-          row[key] = ""
+          row[key] = undefined
         }
       }
     }

packages/server/src/api/controllers/table/index.ts

@@ -10,7 +10,7 @@ import { getDatasourceParams } from "../../../db/utils"
 import { context, events } from "@budibase/backend-core"
 import { Table, UserCtx } from "@budibase/types"
 import sdk from "../../../sdk"
-import csv from "csvtojson"
+import { jsonFromCsvString } from "../../../utilities/csv"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
@@ -104,7 +104,7 @@ export async function bulkImport(ctx: UserCtx) {
 export async function csvToJson(ctx: UserCtx) {
   const { csvString } = ctx.request.body
 
-  const result = await csv().fromString(csvString)
+  const result = await jsonFromCsvString(csvString)
 
   ctx.status = 200
   ctx.body = result

packages/server/src/api/controllers/view/exporters.ts

@@ -10,7 +10,9 @@ export function csv(headers: string[], rows: Row[]) {
         val =
           typeof val === "object" && !(val instanceof Date)
             ? `"${JSON.stringify(val).replace(/"/g, "'")}"`
-            : `"${val}"`
+            : val !== undefined
+            ? `"${val}"`
+            : ""
         return val.trim()
       })
       .join(",")}`

packages/server/src/api/routes/tests/internalSearch.spec.js

@@ -105,7 +105,7 @@ describe("internal search", () => {
         "column": "",
       },
     }, PARAMS)
-    checkLucene(response, `*:* AND !column:["" TO *]`, PARAMS)
+    checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
   })
 
   it("test notEmpty query", async () => {

packages/server/src/utilities/csv.ts

@@ -0,0 +1,22 @@
+import csv from "csvtojson"
+
+export async function jsonFromCsvString(csvString: string) {
+  const castedWithEmptyValues = await csv({ ignoreEmpty: true }).fromString(
+    csvString
+  )
+
+  // By default the csvtojson library casts empty values as empty strings. This is causing issues on conversion.
+  // ignoreEmpty will remove the key completly if empty, so creating this empty object will ensure we return the values with the keys but empty values
+  const result = await csv({ ignoreEmpty: false }).fromString(csvString)
+  result.forEach((r, i) => {
+    for (const [key] of Object.entries(r).filter(
+      ([key, value]) => value === ""
+    )) {
+      if (castedWithEmptyValues[i][key] === undefined) {
+        r[key] = null
+      }
+    }
+  })
+
+  return result
+}

packages/server/src/utilities/schema.ts

@@ -4,6 +4,9 @@ interface SchemaColumn {
   readonly name: string
   readonly type: FieldTypes
   readonly autocolumn?: boolean
+  readonly constraints?: {
+    presence: boolean
+  }
 }
 
 interface Schema {
@@ -76,6 +79,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
       // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
       if (typeof columnType !== "string") {
         results.invalidColumns.push(columnName)
+      } else if (
+        columnData == null &&
+        !schema[columnName].constraints?.presence
+      ) {
+        results.schemaValidation[columnName] = true
       } else if (
         // If there's no data for this field don't bother with further checks
         // If the field is already marked as invalid there's no need for further checks

packages/server/src/utilities/tests/csv.spec.ts

@@ -0,0 +1,33 @@
+import { jsonFromCsvString } from "../csv"
+
+describe("csv", () => {
+  describe("jsonFromCsvString", () => {
+    test("multiple lines csv can be casted", async () => {
+      const csvString = '"id","title"\n"1","aaa"\n"2","bbb"'
+
+      const result = await jsonFromCsvString(csvString)
+
+      expect(result).toEqual([
+        { id: "1", title: "aaa" },
+        { id: "2", title: "bbb" },
+      ])
+      result.forEach(r => expect(Object.keys(r)).toEqual(["id", "title"]))
+    })
+
+    test("empty values are casted as undefined", async () => {
+      const csvString =
+        '"id","optional","title"\n1,,"aaa"\n2,"value","bbb"\n3,,"ccc"'
+
+      const result = await jsonFromCsvString(csvString)
+
+      expect(result).toEqual([
+        { id: "1", optional: null, title: "aaa" },
+        { id: "2", optional: "value", title: "bbb" },
+        { id: "3", optional: null, title: "ccc" },
+      ])
+      result.forEach(r =>
+        expect(Object.keys(r)).toEqual(["id", "optional", "title"])
+      )
+    })
+  })
+})

packages/shared-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/shared-core",
-  "version": "2.5.6-alpha.39",
+  "version": "2.5.10-alpha.1",
   "description": "Shared data utils",
   "main": "dist/cjs/src/index.js",
   "types": "dist/mjs/src/index.d.ts",
@@ -20,7 +20,7 @@
     "dev:builder": "yarn prebuild && concurrently \"tsc -p tsconfig.build.json --watch\" \"tsc -p tsconfig-cjs.build.json --watch\""
   },
   "dependencies": {
-    "@budibase/types": "2.5.6-alpha.39"
+    "@budibase/types": "2.5.10-alpha.1"
   },
   "devDependencies": {
     "concurrently": "^7.6.0",

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "2.5.6-alpha.39",
+  "version": "2.5.10-alpha.1",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/types",
-  "version": "2.5.6-alpha.39",
+  "version": "2.5.10-alpha.1",
   "description": "Budibase types",
   "main": "dist/cjs/index.js",
   "types": "dist/mjs/index.d.ts",

packages/types/src/documents/account/account.ts

@@ -39,6 +39,7 @@ export interface Account extends CreateAccount {
   // licensing
   tier: string // deprecated
   planType?: PlanType
+  /** @deprecated */
   planTier?: number
   license?: License
   installId?: string
@@ -47,6 +48,7 @@ export interface Account extends CreateAccount {
   stripeCustomerId?: string
   licenseKey?: string
   licenseKeyActivatedAt?: number
+  licenseRequestedAt?: number
   licenseOverrides?: LicenseOverrides
   quotaUsage?: QuotaUsage
 }

packages/types/src/sdk/events/event.ts

@@ -138,7 +138,6 @@ export enum Event {
 
   // LICENSE
   LICENSE_PLAN_CHANGED = "license:plan:changed",
-  LICENSE_TIER_CHANGED = "license:tier:changed",
   LICENSE_ACTIVATED = "license:activated",
   LICENSE_PAYMENT_FAILED = "license:payment:failed",
   LICENSE_PAYMENT_RECOVERED = "license:payment:recovered",
@@ -328,7 +327,6 @@ export const AuditedEventFriendlyName: Record<Event, string | undefined> = {
 
   // LICENSE - NOT AUDITED
   [Event.LICENSE_PLAN_CHANGED]: undefined,
-  [Event.LICENSE_TIER_CHANGED]: undefined,
   [Event.LICENSE_ACTIVATED]: undefined,
   [Event.LICENSE_PAYMENT_FAILED]: undefined,
   [Event.LICENSE_PAYMENT_RECOVERED]: undefined,

packages/types/src/sdk/events/license.ts

@@ -1,15 +1,14 @@
-import { PlanType } from "../licensing"
-
-export interface LicenseTierChangedEvent {
-  accountId: string
-  from: number
-  to: number
-}
+import { PlanType, PriceDuration } from "../licensing"
 
 export interface LicensePlanChangedEvent {
   accountId: string
   from: PlanType
   to: PlanType
+  // may not be on historical events
+  fromDuration: PriceDuration | undefined
+  toDuration: PriceDuration | undefined
+  fromQuantity: number | undefined
+  toQuantity: number | undefined
 }
 
 export interface LicenseActivatedEvent {

packages/types/src/sdk/licensing/plan.ts

@@ -17,7 +17,6 @@ export enum PriceDuration {
 export interface AvailablePlan {
   type: PlanType
   maxUsers: number
-  minUsers: number
   prices: AvailablePrice[]
 }
 
@@ -38,7 +37,6 @@ export interface PurchasedPlan {
   type: PlanType
   model: PlanModel
   usesInvoicing: boolean
-  minUsers: number
   price?: PurchasedPrice
 }
 

packages/types/src/sdk/licensing/quota.ts

@@ -55,12 +55,6 @@ export const isConstantQuota = (
   return quotaType === QuotaType.CONSTANT
 }
 
-export interface Minimums {
-  users: number
-}
-
-export type PlanMinimums = { [key in PlanType]: Minimums }
-
 export type PlanQuotas = { [key in PlanType]: Quotas | undefined }
 
 export type MonthlyQuotas = {

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.5.6-alpha.39",
+  "version": "2.5.10-alpha.1",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -37,10 +37,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "2.5.6-alpha.39",
-    "@budibase/pro": "2.5.6-alpha.39",
-    "@budibase/string-templates": "2.5.6-alpha.39",
-    "@budibase/types": "2.5.6-alpha.39",
+    "@budibase/backend-core": "2.5.10-alpha.1",
+    "@budibase/pro": "2.5.10-alpha.1",
+    "@budibase/string-templates": "2.5.10-alpha.1",
+    "@budibase/types": "2.5.10-alpha.1",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",

yarn.lock

@@ -1486,15 +1486,15 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
-"@budibase/pro@2.5.6-alpha.38":
-  version "2.5.6-alpha.38"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.6-alpha.38.tgz#09e8dfbb6cefe856b5c01845a5a5c02a406faedf"
-  integrity sha512-CBZv6V+163USHPN0SuEIrXeGA+9gB1QNmHrMEPSmwlOCZbNW2dhniz00EVSuVh3ypHSjDECgozasDJTNRhiufQ==
+"@budibase/pro@2.5.10-alpha.0":
+  version "2.5.10-alpha.0"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.10-alpha.0.tgz#f3688cc61d2130fbf8b191e515b8ac9789f2a9df"
+  integrity sha512-vYa1F4NhMh7veSZyhgrE54iziwZLHiuRwXbU+unEo3xqyFZIvERAAQZ88nrnwssubWBWnPhpN/8JMHrsyM8mXg==
   dependencies:
-    "@budibase/backend-core" "2.5.6-alpha.38"
-    "@budibase/shared-core" "2.4.44-alpha.1"
-    "@budibase/string-templates" "2.4.44-alpha.1"
-    "@budibase/types" "2.5.6-alpha.38"
+    "@budibase/backend-core" "2.5.10-alpha.0"
+    "@budibase/shared-core" "2.5.9"
+    "@budibase/string-templates" "2.5.9"
+    "@budibase/types" "2.5.10-alpha.0"
     "@koa/router" "8.0.8"
     bull "4.10.1"
     joi "17.6.0"
@@ -1505,12 +1505,12 @@
     scim-patch "^0.7.0"
     scim2-parse-filter "^0.2.8"
 
-"@budibase/shared-core@2.4.44-alpha.1":
-  version "2.4.44-alpha.1"
-  resolved "https://registry.yarnpkg.com/@budibase/shared-core/-/shared-core-2.4.44-alpha.1.tgz#3d499e40e7e6c646e13a87cd08e01ba116c2ff1d"
-  integrity sha512-cN8LaDczijtsfWUYiXC4sg9Z+US4020i3Mb8TwCbf8TQyA1b06U5PwPCp+GHVA/wDFqfwcpcE1GXf8GwVuYs7A==
+"@budibase/shared-core@2.5.9":
+  version "2.5.9"
+  resolved "https://registry.yarnpkg.com/@budibase/shared-core/-/shared-core-2.5.9.tgz#f22f22637fb7618ded1c7292b10793d7969827ce"
+  integrity sha512-l417Rb2+1tuXbjNL42wJHmqIQQyla2pPglOnapxfOdRuvzng+5GqlrTV1caLNf/53TS9U6ueGJdPOtxZTTFGUA==
   dependencies:
-    "@budibase/types" "2.4.44-alpha.1"
+    "@budibase/types" "^2.5.9"
 
 "@budibase/standard-components@^0.9.139":
   version "0.9.139"
@@ -1530,22 +1530,22 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/string-templates@2.4.44-alpha.1":
-  version "2.4.44-alpha.1"
-  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-2.4.44-alpha.1.tgz#6c2aee594d16eac1f173c509e087a817dd3172f0"
-  integrity sha512-4gC2+0kccK0ilLnd0i/dmJzC0Ur7UgSAmV6zbzDDYNL4spU0qSy5VhBh7E3qKieg5RKMMzbpXLMWERpoPLlnqA==
+"@budibase/string-templates@2.5.9":
+  version "2.5.9"
+  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-2.5.9.tgz#64582730421801e1e829b430136b449b1adc0314"
+  integrity sha512-Szu06M0JFHuUVIil2aHZWU8hvsROYpDx9raX9uIv4DcwBOAtyvVzD16wOCHzlmj8wWeV8fbKe4JF4q4GXnilJg==
   dependencies:
     "@budibase/handlebars-helpers" "^0.11.8"
     dayjs "^1.10.4"
     handlebars "^4.7.6"
     handlebars-utils "^1.0.6"
     lodash "^4.17.20"
-    vm2 "^3.9.4"
+    vm2 "^3.9.15"
 
-"@budibase/types@2.4.44-alpha.1":
-  version "2.4.44-alpha.1"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.4.44-alpha.1.tgz#1679657aa180d9c59afa1dffa611bff0638bd933"
-  integrity sha512-Sq+8HfM75EBMoOvKYFwELdlxmVN6wNZMofDjT/2G+9aF+Zfe5Tzw69C+unmdBgcGGjGCHEYWSz4mF0v8FPAGbg==
+"@budibase/types@^2.5.9":
+  version "2.5.9"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.5.9.tgz#4b1253e76b95cd8d6a42e86ebc3363a305c62653"
+  integrity sha512-7NELdWB3iV5y+pmXhm9g/v1AlN+wqHtoy67DZxKY5dlcK4DCcTdlqkToGiiw9d3sDPgDAVznjdZYnB5pGzd3TQ==
 
 "@bull-board/api@3.7.0":
   version "3.7.0"
@@ -9545,13 +9545,6 @@ dir-glob@^3.0.1:
   dependencies:
     path-type "^4.0.0"
 
-docker-compose@0.23.12:
-  version "0.23.12"
-  resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.12.tgz#fa883b98be08f6926143d06bf9e522ef7ed3210c"
-  integrity sha512-KFbSMqQBuHjTGZGmYDOCO0L4SaML3BsWTId5oSUyaBa22vALuFHNv+UdDWs3HcMylHWKsxCbLB7hnM/nCosWZw==
-  dependencies:
-    yaml "^1.10.2"
-
 docker-compose@0.23.17:
   version "0.23.17"
   resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.17.tgz#8816bef82562d9417dc8c790aa4871350f93a2ba"
@@ -9559,6 +9552,13 @@ docker-compose@0.23.17:
   dependencies:
     yaml "^1.10.2"
 
+docker-compose@0.24.0:
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.24.0.tgz#474cd38b05b3887a56ffb2c39f16a7ae4d7d5077"
+  integrity sha512-RN/oSPLPa6ZG5e4dHg8tD8EMpd1WJqomNMBQT+d2M5MwcmfrPW/xHTent4TVqX0zZvHemv7qhhNlzXjxCnFaQw==
+  dependencies:
+    yaml "^1.10.2"
+
 docker-compose@^0.23.5:
   version "0.23.19"
   resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.19.tgz#9947726e2fe67bdfa9e8efe1ff15aa0de2e10eb8"
@@ -24502,7 +24502,7 @@ vm2@3.9.17:
     acorn "^8.7.0"
     acorn-walk "^8.2.0"
 
-vm2@^3.9.11, vm2@^3.9.15, vm2@^3.9.4:
+vm2@^3.9.11, vm2@^3.9.15:
   version "3.9.16"
   resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.16.tgz#0fbc2a265f7bf8b837cea6f4a908f88a3f93b8e6"
   integrity sha512-3T9LscojNTxdOyG+e8gFeyBXkMlOBYDoF6dqZbj+MPVHi9x10UfiTAJIobuchRCp3QvC+inybTbMJIUrLsig0w==