diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 8954106feb..dce1a71918 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -222,9 +222,9 @@ http { rewrite ^/files/signed/(.*)$ /$1 break; } - client_header_timeout 60; - client_body_timeout 60; - keepalive_timeout 60; + client_header_timeout 120; + client_body_timeout 120; + keepalive_timeout 120; # gzip gzip on; diff --git a/lerna.json b/lerna.json index 7356540633..c226dea49e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.5.6-alpha.39", + "version": "2.5.10-alpha.1", "npmClient": "yarn", "useWorkspaces": true, "packages": ["packages/*"], diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 4db1619306..b66112e8de 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.5.6-alpha.39", + "version": "2.5.10-alpha.1", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -24,7 +24,7 @@ "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "2.5.6-alpha.39", + "@budibase/types": "2.5.10-alpha.1", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index a3b1ecc08d..e64c116663 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -47,7 +47,7 @@ async function put( type: LockType.TRY_ONCE, name: LockName.PERSIST_WRITETHROUGH, resource: key, - ttl: 1000, + ttl: 15000, }, async () => { const writeDb = async (toWrite: any) => { @@ -71,6 +71,7 @@ async function put( } } ) + if (!lockResponse.executed) { logWarn(`Ignoring redlock conflict in write-through cache`) } diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index 5d21d4f4e4..4660be81aa 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -434,7 +434,7 @@ export class QueryBuilder { }) } if (this.#query.empty) { - build(this.#query.empty, (key: string) => `!${key}:["" TO *]`) + build(this.#query.empty, (key: string) => `(*:* -${key}:["" TO *])`) } if (this.#query.notEmpty) { build(this.#query.notEmpty, (key: string) => `${key}:["" TO *]`) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 0d413b8fa9..1bea1f3692 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -154,6 +154,7 @@ const environment = { ? process.env.ENABLE_SSO_MAINTENANCE_MODE : false, VERSION: findVersion(), + DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER, _set(key: any, value: any) { process.env[key] = value // @ts-ignore diff --git a/packages/backend-core/src/events/publishers/license.ts b/packages/backend-core/src/events/publishers/license.ts index aff3286c87..094cf7fac1 100644 --- a/packages/backend-core/src/events/publishers/license.ts +++ b/packages/backend-core/src/events/publishers/license.ts @@ -3,7 +3,6 @@ import { Event, LicenseActivatedEvent, LicensePlanChangedEvent, - LicenseTierChangedEvent, PlanType, Account, LicensePortalOpenedEvent, @@ -11,22 +10,23 @@ import { LicenseCheckoutOpenedEvent, LicensePaymentFailedEvent, LicensePaymentRecoveredEvent, + PriceDuration, } from "@budibase/types" -async function tierChanged(account: Account, from: number, to: number) { - const properties: LicenseTierChangedEvent = { - accountId: account.accountId, - to, - from, +async function planChanged( + account: Account, + opts: { + from: PlanType + to: PlanType + fromQuantity: number | undefined + toQuantity: number | undefined + fromDuration: PriceDuration | undefined + toDuration: PriceDuration | undefined } - await publishEvent(Event.LICENSE_TIER_CHANGED, properties) -} - -async function planChanged(account: Account, from: PlanType, to: PlanType) { +) { const properties: LicensePlanChangedEvent = { accountId: account.accountId, - to, - from, + ...opts, } await publishEvent(Event.LICENSE_PLAN_CHANGED, properties) } @@ -74,7 +74,6 @@ async function paymentRecovered(account: Account) { } export default { - tierChanged, planChanged, activated, checkoutOpened, diff --git a/packages/backend-core/src/logging/index.ts b/packages/backend-core/src/logging/index.ts index b229f47dea..b87062c478 100644 --- a/packages/backend-core/src/logging/index.ts +++ b/packages/backend-core/src/logging/index.ts @@ -1,5 +1,5 @@ export * as correlation from "./correlation/correlation" -export { logger, disableLogger } from "./pino/logger" +export { logger } from "./pino/logger" export * from "./alerts" // turn off or on context logging i.e. tenantId, appId etc diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index dd4b505d30..276377eb00 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -5,184 +5,169 @@ import * as correlation from "../correlation" import { IdentityType } from "@budibase/types" import { LOG_CONTEXT } from "../index" -// CORE LOGGERS - for disabling - -const BUILT_INS = { - log: console.log, - error: console.error, - info: console.info, - warn: console.warn, - trace: console.trace, - debug: console.debug, -} - // LOGGER -const pinoOptions: LoggerOptions = { - level: env.LOG_LEVEL, - formatters: { - level: label => { - return { level: label.toUpperCase() } - }, - bindings: () => { - return {} - }, - }, - timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`, -} - -if (env.isDev()) { - pinoOptions.transport = { - target: "pino-pretty", - options: { - singleLine: true, +let pinoInstance: pino.Logger | undefined +if (!env.DISABLE_PINO_LOGGER) { + const pinoOptions: LoggerOptions = { + level: env.LOG_LEVEL, + formatters: { + level: label => { + return { level: label.toUpperCase() } + }, + bindings: () => { + return {} + }, }, + timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`, } -} -export const logger = pino(pinoOptions) - -export function disableLogger() { - console.log = BUILT_INS.log - console.error = BUILT_INS.error - console.info = BUILT_INS.info - console.warn = BUILT_INS.warn - console.trace = BUILT_INS.trace - console.debug = BUILT_INS.debug -} - -// CONSOLE OVERRIDES - -interface MergingObject { - objects?: any[] - tenantId?: string - appId?: string - identityId?: string - identityType?: IdentityType - correlationId?: string - err?: Error -} - -function isPlainObject(obj: any) { - return typeof obj === "object" && obj !== null && !(obj instanceof Error) -} - -function isError(obj: any) { - return obj instanceof Error -} - -function isMessage(obj: any) { - return typeof obj === "string" -} - -/** - * Backwards compatibility between console logging statements - * and pino logging requirements. - */ -function getLogParams(args: any[]): [MergingObject, string] { - let error = undefined - let objects: any[] = [] - let message = "" - - args.forEach(arg => { - if (isMessage(arg)) { - message = `${message} ${arg}`.trimStart() - } - if (isPlainObject(arg)) { - objects.push(arg) - } - if (isError(arg)) { - error = arg - } - }) - - const identity = getIdentity() - - let contextObject = {} - - if (LOG_CONTEXT) { - contextObject = { - tenantId: getTenantId(), - appId: getAppId(), - identityId: identity?._id, - identityType: identity?.type, - correlationId: correlation.getId(), + if (env.isDev()) { + pinoOptions.transport = { + target: "pino-pretty", + options: { + singleLine: true, + }, } } - const mergingObject = { - objects: objects.length ? objects : undefined, - err: error, - ...contextObject, + pinoInstance = pino(pinoOptions) + + // CONSOLE OVERRIDES + + interface MergingObject { + objects?: any[] + tenantId?: string + appId?: string + identityId?: string + identityType?: IdentityType + correlationId?: string + err?: Error } - return [mergingObject, message] -} - -console.log = (...arg: any[]) => { - const [obj, msg] = getLogParams(arg) - logger.info(obj, msg) -} -console.info = (...arg: any[]) => { - const [obj, msg] = getLogParams(arg) - logger.info(obj, msg) -} -console.warn = (...arg: any[]) => { - const [obj, msg] = getLogParams(arg) - logger.warn(obj, msg) -} -console.error = (...arg: any[]) => { - const [obj, msg] = getLogParams(arg) - logger.error(obj, msg) -} - -/** - * custom trace impl - this resembles the node trace behaviour rather - * than traditional trace logging - * @param arg - */ -console.trace = (...arg: any[]) => { - const [obj, msg] = getLogParams(arg) - if (!obj.err) { - // to get stack trace - obj.err = new Error() + function isPlainObject(obj: any) { + return typeof obj === "object" && obj !== null && !(obj instanceof Error) } - logger.trace(obj, msg) -} -console.debug = (...arg: any) => { - const [obj, msg] = getLogParams(arg) - logger.debug(obj, msg) -} - -// CONTEXT - -const getTenantId = () => { - let tenantId - try { - tenantId = context.getTenantId() - } catch (e: any) { - // do nothing + function isError(obj: any) { + return obj instanceof Error + } + + function isMessage(obj: any) { + return typeof obj === "string" + } + + /** + * Backwards compatibility between console logging statements + * and pino logging requirements. + */ + function getLogParams(args: any[]): [MergingObject, string] { + let error = undefined + let objects: any[] = [] + let message = "" + + args.forEach(arg => { + if (isMessage(arg)) { + message = `${message} ${arg}`.trimStart() + } + if (isPlainObject(arg)) { + objects.push(arg) + } + if (isError(arg)) { + error = arg + } + }) + + const identity = getIdentity() + + let contextObject = {} + + if (LOG_CONTEXT) { + contextObject = { + tenantId: getTenantId(), + appId: getAppId(), + identityId: identity?._id, + identityType: identity?.type, + correlationId: correlation.getId(), + } + } + + const mergingObject = { + objects: objects.length ? objects : undefined, + err: error, + ...contextObject, + } + + return [mergingObject, message] + } + + console.log = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + pinoInstance?.info(obj, msg) + } + console.info = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + pinoInstance?.info(obj, msg) + } + console.warn = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + pinoInstance?.warn(obj, msg) + } + console.error = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + pinoInstance?.error(obj, msg) + } + + /** + * custom trace impl - this resembles the node trace behaviour rather + * than traditional trace logging + * @param arg + */ + console.trace = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + if (!obj.err) { + // to get stack trace + obj.err = new Error() + } + pinoInstance?.trace(obj, msg) + } + + console.debug = (...arg: any) => { + const [obj, msg] = getLogParams(arg) + pinoInstance?.debug(obj, msg) + } + + // CONTEXT + + const getTenantId = () => { + let tenantId + try { + tenantId = context.getTenantId() + } catch (e: any) { + // do nothing + } + return tenantId + } + + const getAppId = () => { + let appId + try { + appId = context.getAppId() + } catch (e) { + // do nothing + } + return appId + } + + const getIdentity = () => { + let identity + try { + identity = context.getIdentity() + } catch (e) { + // do nothing + } + return identity } - return tenantId } -const getAppId = () => { - let appId - try { - appId = context.getAppId() - } catch (e) { - // do nothing - } - return appId -} - -const getIdentity = () => { - let identity - try { - identity = context.getIdentity() - } catch (e) { - // do nothing - } - return identity -} +export const logger = pinoInstance diff --git a/packages/backend-core/tests/core/utilities/mocks/events.ts b/packages/backend-core/tests/core/utilities/mocks/events.ts index dacf7dcce8..81de1f8175 100644 --- a/packages/backend-core/tests/core/utilities/mocks/events.ts +++ b/packages/backend-core/tests/core/utilities/mocks/events.ts @@ -123,7 +123,6 @@ beforeAll(async () => { jest.spyOn(events.plugin, "imported") jest.spyOn(events.plugin, "deleted") - jest.spyOn(events.license, "tierChanged") jest.spyOn(events.license, "planChanged") jest.spyOn(events.license, "activated") jest.spyOn(events.license, "checkoutOpened") diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts index 24b120451e..22e73f2871 100644 --- a/packages/backend-core/tests/core/utilities/structures/licenses.ts +++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts @@ -7,16 +7,29 @@ import { PlanType, PriceDuration, PurchasedPlan, + PurchasedPrice, Quotas, Subscription, } from "@budibase/types" +export function price(): PurchasedPrice { + return { + amount: 10000, + amountMonthly: 10000, + currency: "usd", + duration: PriceDuration.MONTHLY, + priceId: "price_123", + dayPasses: undefined, + isPerUser: true, + } +} + export const plan = (type: PlanType = PlanType.FREE): PurchasedPlan => { return { type, usesInvoicing: false, - minUsers: 1, model: PlanModel.PER_USER, + price: type !== PlanType.FREE ? price() : undefined, } } diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 5d4b67752e..0e99b34384 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.5.6-alpha.39", + "version": "2.5.10-alpha.1", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,8 +38,8 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/shared-core": "2.5.6-alpha.39", - "@budibase/string-templates": "2.5.6-alpha.39", + "@budibase/shared-core": "2.5.10-alpha.1", + "@budibase/string-templates": "2.5.10-alpha.1", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", diff --git a/packages/builder/package.json b/packages/builder/package.json index 5245b5dc88..9607e51344 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.5.6-alpha.39", + "version": "2.5.10-alpha.1", "license": "GPL-3.0", "private": true, "scripts": { @@ -58,10 +58,10 @@ } }, "dependencies": { - "@budibase/bbui": "2.5.6-alpha.39", - "@budibase/frontend-core": "2.5.6-alpha.39", - "@budibase/shared-core": "2.5.6-alpha.39", - "@budibase/string-templates": "2.5.6-alpha.39", + "@budibase/bbui": "2.5.10-alpha.1", + "@budibase/frontend-core": "2.5.10-alpha.1", + "@budibase/shared-core": "2.5.10-alpha.1", + "@budibase/string-templates": "2.5.10-alpha.1", "@fortawesome/fontawesome-svg-core": "^6.2.1", "@fortawesome/free-brands-svg-icons": "^6.2.1", "@fortawesome/free-solid-svg-icons": "^6.2.1", diff --git a/packages/builder/src/components/backend/TableNavigator/utils.js b/packages/builder/src/components/backend/TableNavigator/utils.js index 658f037912..b7e46042be 100644 --- a/packages/builder/src/components/backend/TableNavigator/utils.js +++ b/packages/builder/src/components/backend/TableNavigator/utils.js @@ -42,16 +42,7 @@ export const parseFile = e => { reader.addEventListener("load", function (e) { const fileData = e.target.result - - if (file.type === "text/csv") { - API.csvToJson(fileData) - .then(rows => { - resolveRows(rows) - }) - .catch(() => { - reject("can't convert csv to json") - }) - } else if (file.type === "application/json") { + if (file.type?.includes("json")) { const parsedFileData = JSON.parse(fileData) if (Array.isArray(parsedFileData)) { @@ -62,7 +53,13 @@ export const parseFile = e => { reject("invalid json format") } } else { - reject("invalid file type") + API.csvToJson(fileData) + .then(rows => { + resolveRows(rows) + }) + .catch(() => { + reject("cannot parse csv") + }) } }) diff --git a/packages/builder/src/components/usage/Usage.svelte b/packages/builder/src/components/usage/Usage.svelte index 6e81abfe63..23d8ddc2f3 100644 --- a/packages/builder/src/components/usage/Usage.svelte +++ b/packages/builder/src/components/usage/Usage.svelte @@ -27,7 +27,7 @@ onMount(() => { unlimited = isUnlimited() percentage = getPercentage() - if (warnWhenFull && percentage === 100) { + if (warnWhenFull && percentage >= 100) { showWarning = true } }) diff --git a/packages/builder/src/stores/portal/users.js b/packages/builder/src/stores/portal/users.js index 898d47c0e2..e522cd7958 100644 --- a/packages/builder/src/stores/portal/users.js +++ b/packages/builder/src/stores/portal/users.js @@ -41,7 +41,7 @@ export function createUsersStore() { inviteCode, password, firstName, - lastName, + lastName: !lastName?.trim() ? undefined : lastName, }) } diff --git a/packages/cli/package.json b/packages/cli/package.json index 9efe246270..3d663babd5 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.5.6-alpha.39", + "version": "2.5.10-alpha.1", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "dist/index.js", "bin": { @@ -29,14 +29,14 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "2.5.6-alpha.39", - "@budibase/string-templates": "2.5.6-alpha.39", - "@budibase/types": "2.5.6-alpha.39", + "@budibase/backend-core": "2.5.10-alpha.1", + "@budibase/string-templates": "2.5.10-alpha.1", + "@budibase/types": "2.5.10-alpha.1", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", "commander": "7.1.0", - "docker-compose": "0.23.12", + "docker-compose": "0.24.0", "dotenv": "16.0.1", "download": "8.0.0", "find-free-port": "^2.0.0", diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index cca2c91862..8082d3f0a0 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,6 +1,5 @@ #!/usr/bin/env node -import { logging } from "@budibase/backend-core" -logging.disableLogger() +process.env.DISABLE_PINO_LOGGER = "1" import "./prebuilds" import "./environment" import { env } from "@budibase/backend-core" diff --git a/packages/client/package.json b/packages/client/package.json index ce1bfa0e68..4016899f8c 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.5.6-alpha.39", + "version": "2.5.10-alpha.1", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,11 +19,11 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "2.5.6-alpha.39", - "@budibase/frontend-core": "2.5.6-alpha.39", - "@budibase/shared-core": "2.5.6-alpha.39", - "@budibase/string-templates": "2.5.6-alpha.39", - "@budibase/types": "2.5.6-alpha.39", + "@budibase/bbui": "2.5.10-alpha.1", + "@budibase/frontend-core": "2.5.10-alpha.1", + "@budibase/shared-core": "2.5.10-alpha.1", + "@budibase/string-templates": "2.5.10-alpha.1", + "@budibase/types": "2.5.10-alpha.1", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 2f14d2eb36..937574814e 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,13 +1,13 @@ { "name": "@budibase/frontend-core", - "version": "2.5.6-alpha.39", + "version": "2.5.10-alpha.1", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "2.5.6-alpha.39", - "@budibase/shared-core": "2.5.6-alpha.39", + "@budibase/bbui": "2.5.10-alpha.1", + "@budibase/shared-core": "2.5.10-alpha.1", "dayjs": "^1.11.7", "lodash": "^4.17.21", "socket.io-client": "^4.6.1", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 0120f5df39..0843bd509f 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.5.6-alpha.39", + "version": "2.5.10-alpha.1", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index 9d50771c8f..d0ff56531b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.5.6-alpha.39", + "version": "2.5.10-alpha.1", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -45,12 +45,12 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "2.5.6-alpha.39", - "@budibase/client": "2.5.6-alpha.39", - "@budibase/pro": "2.5.6-alpha.39", - "@budibase/shared-core": "2.5.6-alpha.39", - "@budibase/string-templates": "2.5.6-alpha.39", - "@budibase/types": "2.5.6-alpha.39", + "@budibase/backend-core": "2.5.10-alpha.1", + "@budibase/client": "2.5.10-alpha.1", + "@budibase/pro": "2.5.10-alpha.1", + "@budibase/shared-core": "2.5.10-alpha.1", + "@budibase/string-templates": "2.5.10-alpha.1", + "@budibase/types": "2.5.10-alpha.1", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index a0bebc2490..cc1450060c 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -37,7 +37,7 @@ import { Table, } from "@budibase/types" -const { cleanExportRows } = require("./utils") +import { cleanExportRows } from "./utils" const CALCULATION_TYPES = { SUM: "sum", @@ -391,6 +391,9 @@ export async function exportRows(ctx: UserCtx) { const table = await db.get(ctx.params.tableId) const rowIds = ctx.request.body.rows let format = ctx.query.format + if (typeof format !== "string") { + ctx.throw(400, "Format parameter is not valid") + } const { columns, query } = ctx.request.body let result diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index e96a4fe6ee..a7c467ea61 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -137,8 +137,8 @@ export function cleanExportRows( delete schema[column] }) - // Intended to avoid 'undefined' in export if (format === Format.CSV) { + // Intended to append empty values in export const schemaKeys = Object.keys(schema) for (let key of schemaKeys) { if (columns?.length && columns.indexOf(key) > 0) { @@ -146,7 +146,7 @@ export function cleanExportRows( } for (let row of cleanRows) { if (row[key] == null) { - row[key] = "" + row[key] = undefined } } } diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 2ab7ad7b38..bc967a90f4 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -10,7 +10,7 @@ import { getDatasourceParams } from "../../../db/utils" import { context, events } from "@budibase/backend-core" import { Table, UserCtx } from "@budibase/types" import sdk from "../../../sdk" -import csv from "csvtojson" +import { jsonFromCsvString } from "../../../utilities/csv" function pickApi({ tableId, table }: { tableId?: string; table?: Table }) { if (table && !tableId) { @@ -104,7 +104,7 @@ export async function bulkImport(ctx: UserCtx) { export async function csvToJson(ctx: UserCtx) { const { csvString } = ctx.request.body - const result = await csv().fromString(csvString) + const result = await jsonFromCsvString(csvString) ctx.status = 200 ctx.body = result diff --git a/packages/server/src/api/controllers/view/exporters.ts b/packages/server/src/api/controllers/view/exporters.ts index 4d927bca27..ec0aca95a9 100644 --- a/packages/server/src/api/controllers/view/exporters.ts +++ b/packages/server/src/api/controllers/view/exporters.ts @@ -10,7 +10,9 @@ export function csv(headers: string[], rows: Row[]) { val = typeof val === "object" && !(val instanceof Date) ? `"${JSON.stringify(val).replace(/"/g, "'")}"` - : `"${val}"` + : val !== undefined + ? `"${val}"` + : "" return val.trim() }) .join(",")}` diff --git a/packages/server/src/api/routes/tests/internalSearch.spec.js b/packages/server/src/api/routes/tests/internalSearch.spec.js index 6ad333b87a..8d57d2ee1c 100644 --- a/packages/server/src/api/routes/tests/internalSearch.spec.js +++ b/packages/server/src/api/routes/tests/internalSearch.spec.js @@ -105,7 +105,7 @@ describe("internal search", () => { "column": "", }, }, PARAMS) - checkLucene(response, `*:* AND !column:["" TO *]`, PARAMS) + checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS) }) it("test notEmpty query", async () => { diff --git a/packages/server/src/utilities/csv.ts b/packages/server/src/utilities/csv.ts new file mode 100644 index 0000000000..477ebb896d --- /dev/null +++ b/packages/server/src/utilities/csv.ts @@ -0,0 +1,22 @@ +import csv from "csvtojson" + +export async function jsonFromCsvString(csvString: string) { + const castedWithEmptyValues = await csv({ ignoreEmpty: true }).fromString( + csvString + ) + + // By default the csvtojson library casts empty values as empty strings. This is causing issues on conversion. + // ignoreEmpty will remove the key completly if empty, so creating this empty object will ensure we return the values with the keys but empty values + const result = await csv({ ignoreEmpty: false }).fromString(csvString) + result.forEach((r, i) => { + for (const [key] of Object.entries(r).filter( + ([key, value]) => value === "" + )) { + if (castedWithEmptyValues[i][key] === undefined) { + r[key] = null + } + } + }) + + return result +} diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts index f3d98d35c9..7b5de7898a 100644 --- a/packages/server/src/utilities/schema.ts +++ b/packages/server/src/utilities/schema.ts @@ -4,6 +4,9 @@ interface SchemaColumn { readonly name: string readonly type: FieldTypes readonly autocolumn?: boolean + readonly constraints?: { + presence: boolean + } } interface Schema { @@ -76,6 +79,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults { // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array if (typeof columnType !== "string") { results.invalidColumns.push(columnName) + } else if ( + columnData == null && + !schema[columnName].constraints?.presence + ) { + results.schemaValidation[columnName] = true } else if ( // If there's no data for this field don't bother with further checks // If the field is already marked as invalid there's no need for further checks diff --git a/packages/server/src/utilities/tests/csv.spec.ts b/packages/server/src/utilities/tests/csv.spec.ts new file mode 100644 index 0000000000..14063d0e8e --- /dev/null +++ b/packages/server/src/utilities/tests/csv.spec.ts @@ -0,0 +1,33 @@ +import { jsonFromCsvString } from "../csv" + +describe("csv", () => { + describe("jsonFromCsvString", () => { + test("multiple lines csv can be casted", async () => { + const csvString = '"id","title"\n"1","aaa"\n"2","bbb"' + + const result = await jsonFromCsvString(csvString) + + expect(result).toEqual([ + { id: "1", title: "aaa" }, + { id: "2", title: "bbb" }, + ]) + result.forEach(r => expect(Object.keys(r)).toEqual(["id", "title"])) + }) + + test("empty values are casted as undefined", async () => { + const csvString = + '"id","optional","title"\n1,,"aaa"\n2,"value","bbb"\n3,,"ccc"' + + const result = await jsonFromCsvString(csvString) + + expect(result).toEqual([ + { id: "1", optional: null, title: "aaa" }, + { id: "2", optional: "value", title: "bbb" }, + { id: "3", optional: null, title: "ccc" }, + ]) + result.forEach(r => + expect(Object.keys(r)).toEqual(["id", "optional", "title"]) + ) + }) + }) +}) diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json index c342939aa9..2b8f75f738 100644 --- a/packages/shared-core/package.json +++ b/packages/shared-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/shared-core", - "version": "2.5.6-alpha.39", + "version": "2.5.10-alpha.1", "description": "Shared data utils", "main": "dist/cjs/src/index.js", "types": "dist/mjs/src/index.d.ts", @@ -20,7 +20,7 @@ "dev:builder": "yarn prebuild && concurrently \"tsc -p tsconfig.build.json --watch\" \"tsc -p tsconfig-cjs.build.json --watch\"" }, "dependencies": { - "@budibase/types": "2.5.6-alpha.39" + "@budibase/types": "2.5.10-alpha.1" }, "devDependencies": { "concurrently": "^7.6.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 023d9e9c2b..3349aad7ba 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.5.6-alpha.39", + "version": "2.5.10-alpha.1", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index 7bcec178a1..4367270c85 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.5.6-alpha.39", + "version": "2.5.10-alpha.1", "description": "Budibase types", "main": "dist/cjs/index.js", "types": "dist/mjs/index.d.ts", diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts index 8678085df0..dad8abed30 100644 --- a/packages/types/src/documents/account/account.ts +++ b/packages/types/src/documents/account/account.ts @@ -39,6 +39,7 @@ export interface Account extends CreateAccount { // licensing tier: string // deprecated planType?: PlanType + /** @deprecated */ planTier?: number license?: License installId?: string @@ -47,6 +48,7 @@ export interface Account extends CreateAccount { stripeCustomerId?: string licenseKey?: string licenseKeyActivatedAt?: number + licenseRequestedAt?: number licenseOverrides?: LicenseOverrides quotaUsage?: QuotaUsage } diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index c4990f869b..0d0b166253 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -138,7 +138,6 @@ export enum Event { // LICENSE LICENSE_PLAN_CHANGED = "license:plan:changed", - LICENSE_TIER_CHANGED = "license:tier:changed", LICENSE_ACTIVATED = "license:activated", LICENSE_PAYMENT_FAILED = "license:payment:failed", LICENSE_PAYMENT_RECOVERED = "license:payment:recovered", @@ -328,7 +327,6 @@ export const AuditedEventFriendlyName: Record = { // LICENSE - NOT AUDITED [Event.LICENSE_PLAN_CHANGED]: undefined, - [Event.LICENSE_TIER_CHANGED]: undefined, [Event.LICENSE_ACTIVATED]: undefined, [Event.LICENSE_PAYMENT_FAILED]: undefined, [Event.LICENSE_PAYMENT_RECOVERED]: undefined, diff --git a/packages/types/src/sdk/events/license.ts b/packages/types/src/sdk/events/license.ts index a12fc6bbb5..12bdd92184 100644 --- a/packages/types/src/sdk/events/license.ts +++ b/packages/types/src/sdk/events/license.ts @@ -1,15 +1,14 @@ -import { PlanType } from "../licensing" - -export interface LicenseTierChangedEvent { - accountId: string - from: number - to: number -} +import { PlanType, PriceDuration } from "../licensing" export interface LicensePlanChangedEvent { accountId: string from: PlanType to: PlanType + // may not be on historical events + fromDuration: PriceDuration | undefined + toDuration: PriceDuration | undefined + fromQuantity: number | undefined + toQuantity: number | undefined } export interface LicenseActivatedEvent { diff --git a/packages/types/src/sdk/licensing/plan.ts b/packages/types/src/sdk/licensing/plan.ts index 360d7d08e5..3e214a01ff 100644 --- a/packages/types/src/sdk/licensing/plan.ts +++ b/packages/types/src/sdk/licensing/plan.ts @@ -17,7 +17,6 @@ export enum PriceDuration { export interface AvailablePlan { type: PlanType maxUsers: number - minUsers: number prices: AvailablePrice[] } @@ -38,7 +37,6 @@ export interface PurchasedPlan { type: PlanType model: PlanModel usesInvoicing: boolean - minUsers: number price?: PurchasedPrice } diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index ea51f7a490..73afa1ed05 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -55,12 +55,6 @@ export const isConstantQuota = ( return quotaType === QuotaType.CONSTANT } -export interface Minimums { - users: number -} - -export type PlanMinimums = { [key in PlanType]: Minimums } - export type PlanQuotas = { [key in PlanType]: Quotas | undefined } export type MonthlyQuotas = { diff --git a/packages/worker/package.json b/packages/worker/package.json index 7c8a65caa8..165689cf7c 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.5.6-alpha.39", + "version": "2.5.10-alpha.1", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -37,10 +37,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "2.5.6-alpha.39", - "@budibase/pro": "2.5.6-alpha.39", - "@budibase/string-templates": "2.5.6-alpha.39", - "@budibase/types": "2.5.6-alpha.39", + "@budibase/backend-core": "2.5.10-alpha.1", + "@budibase/pro": "2.5.10-alpha.1", + "@budibase/string-templates": "2.5.10-alpha.1", + "@budibase/types": "2.5.10-alpha.1", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", diff --git a/yarn.lock b/yarn.lock index ecdb35847a..5f17870a6f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1486,15 +1486,15 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@2.5.6-alpha.38": - version "2.5.6-alpha.38" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.6-alpha.38.tgz#09e8dfbb6cefe856b5c01845a5a5c02a406faedf" - integrity sha512-CBZv6V+163USHPN0SuEIrXeGA+9gB1QNmHrMEPSmwlOCZbNW2dhniz00EVSuVh3ypHSjDECgozasDJTNRhiufQ== +"@budibase/pro@2.5.10-alpha.0": + version "2.5.10-alpha.0" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.10-alpha.0.tgz#f3688cc61d2130fbf8b191e515b8ac9789f2a9df" + integrity sha512-vYa1F4NhMh7veSZyhgrE54iziwZLHiuRwXbU+unEo3xqyFZIvERAAQZ88nrnwssubWBWnPhpN/8JMHrsyM8mXg== dependencies: - "@budibase/backend-core" "2.5.6-alpha.38" - "@budibase/shared-core" "2.4.44-alpha.1" - "@budibase/string-templates" "2.4.44-alpha.1" - "@budibase/types" "2.5.6-alpha.38" + "@budibase/backend-core" "2.5.10-alpha.0" + "@budibase/shared-core" "2.5.9" + "@budibase/string-templates" "2.5.9" + "@budibase/types" "2.5.10-alpha.0" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" @@ -1505,12 +1505,12 @@ scim-patch "^0.7.0" scim2-parse-filter "^0.2.8" -"@budibase/shared-core@2.4.44-alpha.1": - version "2.4.44-alpha.1" - resolved "https://registry.yarnpkg.com/@budibase/shared-core/-/shared-core-2.4.44-alpha.1.tgz#3d499e40e7e6c646e13a87cd08e01ba116c2ff1d" - integrity sha512-cN8LaDczijtsfWUYiXC4sg9Z+US4020i3Mb8TwCbf8TQyA1b06U5PwPCp+GHVA/wDFqfwcpcE1GXf8GwVuYs7A== +"@budibase/shared-core@2.5.9": + version "2.5.9" + resolved "https://registry.yarnpkg.com/@budibase/shared-core/-/shared-core-2.5.9.tgz#f22f22637fb7618ded1c7292b10793d7969827ce" + integrity sha512-l417Rb2+1tuXbjNL42wJHmqIQQyla2pPglOnapxfOdRuvzng+5GqlrTV1caLNf/53TS9U6ueGJdPOtxZTTFGUA== dependencies: - "@budibase/types" "2.4.44-alpha.1" + "@budibase/types" "^2.5.9" "@budibase/standard-components@^0.9.139": version "0.9.139" @@ -1530,22 +1530,22 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/string-templates@2.4.44-alpha.1": - version "2.4.44-alpha.1" - resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-2.4.44-alpha.1.tgz#6c2aee594d16eac1f173c509e087a817dd3172f0" - integrity sha512-4gC2+0kccK0ilLnd0i/dmJzC0Ur7UgSAmV6zbzDDYNL4spU0qSy5VhBh7E3qKieg5RKMMzbpXLMWERpoPLlnqA== +"@budibase/string-templates@2.5.9": + version "2.5.9" + resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-2.5.9.tgz#64582730421801e1e829b430136b449b1adc0314" + integrity sha512-Szu06M0JFHuUVIil2aHZWU8hvsROYpDx9raX9uIv4DcwBOAtyvVzD16wOCHzlmj8wWeV8fbKe4JF4q4GXnilJg== dependencies: "@budibase/handlebars-helpers" "^0.11.8" dayjs "^1.10.4" handlebars "^4.7.6" handlebars-utils "^1.0.6" lodash "^4.17.20" - vm2 "^3.9.4" + vm2 "^3.9.15" -"@budibase/types@2.4.44-alpha.1": - version "2.4.44-alpha.1" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.4.44-alpha.1.tgz#1679657aa180d9c59afa1dffa611bff0638bd933" - integrity sha512-Sq+8HfM75EBMoOvKYFwELdlxmVN6wNZMofDjT/2G+9aF+Zfe5Tzw69C+unmdBgcGGjGCHEYWSz4mF0v8FPAGbg== +"@budibase/types@^2.5.9": + version "2.5.9" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.5.9.tgz#4b1253e76b95cd8d6a42e86ebc3363a305c62653" + integrity sha512-7NELdWB3iV5y+pmXhm9g/v1AlN+wqHtoy67DZxKY5dlcK4DCcTdlqkToGiiw9d3sDPgDAVznjdZYnB5pGzd3TQ== "@bull-board/api@3.7.0": version "3.7.0" @@ -9545,13 +9545,6 @@ dir-glob@^3.0.1: dependencies: path-type "^4.0.0" -docker-compose@0.23.12: - version "0.23.12" - resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.12.tgz#fa883b98be08f6926143d06bf9e522ef7ed3210c" - integrity sha512-KFbSMqQBuHjTGZGmYDOCO0L4SaML3BsWTId5oSUyaBa22vALuFHNv+UdDWs3HcMylHWKsxCbLB7hnM/nCosWZw== - dependencies: - yaml "^1.10.2" - docker-compose@0.23.17: version "0.23.17" resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.17.tgz#8816bef82562d9417dc8c790aa4871350f93a2ba" @@ -9559,6 +9552,13 @@ docker-compose@0.23.17: dependencies: yaml "^1.10.2" +docker-compose@0.24.0: + version "0.24.0" + resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.24.0.tgz#474cd38b05b3887a56ffb2c39f16a7ae4d7d5077" + integrity sha512-RN/oSPLPa6ZG5e4dHg8tD8EMpd1WJqomNMBQT+d2M5MwcmfrPW/xHTent4TVqX0zZvHemv7qhhNlzXjxCnFaQw== + dependencies: + yaml "^1.10.2" + docker-compose@^0.23.5: version "0.23.19" resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.19.tgz#9947726e2fe67bdfa9e8efe1ff15aa0de2e10eb8" @@ -24502,7 +24502,7 @@ vm2@3.9.17: acorn "^8.7.0" acorn-walk "^8.2.0" -vm2@^3.9.11, vm2@^3.9.15, vm2@^3.9.4: +vm2@^3.9.11, vm2@^3.9.15: version "3.9.16" resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.16.tgz#0fbc2a265f7bf8b837cea6f4a908f88a3f93b8e6" integrity sha512-3T9LscojNTxdOyG+e8gFeyBXkMlOBYDoF6dqZbj+MPVHi9x10UfiTAJIobuchRCp3QvC+inybTbMJIUrLsig0w==