Merge remote-tracking branch 'origin/develop' into feature/db-query-save-prompt

This commit is contained in:
Dean 2023-05-04 16:13:48 +01:00
commit 12ce3291eb
27 changed files with 154 additions and 88 deletions

View File

@ -222,9 +222,9 @@ http {
rewrite ^/files/signed/(.*)$ /$1 break;
}
client_header_timeout 60;
client_body_timeout 60;
keepalive_timeout 60;
client_header_timeout 120;
client_body_timeout 120;
keepalive_timeout 120;
# gzip
gzip on;

View File

@ -1,5 +1,5 @@
{
"version": "2.5.6-alpha.45",
"version": "2.5.10-alpha.1",
"npmClient": "yarn",
"useWorkspaces": true,
"packages": ["packages/*"],

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
"version": "2.5.6-alpha.45",
"version": "2.5.10-alpha.1",
"description": "Budibase backend core libraries used in server and worker",
"main": "dist/src/index.js",
"types": "dist/src/index.d.ts",
@ -24,7 +24,7 @@
"dependencies": {
"@budibase/nano": "10.1.2",
"@budibase/pouchdb-replication-stream": "1.2.10",
"@budibase/types": "2.5.6-alpha.45",
"@budibase/types": "2.5.10-alpha.1",
"@shopify/jest-koa-mocks": "5.0.1",
"@techpass/passport-openidconnect": "0.3.2",
"aws-cloudfront-sign": "2.2.0",

View File

@ -47,7 +47,7 @@ async function put(
type: LockType.TRY_ONCE,
name: LockName.PERSIST_WRITETHROUGH,
resource: key,
ttl: 1000,
ttl: 15000,
},
async () => {
const writeDb = async (toWrite: any) => {
@ -71,6 +71,7 @@ async function put(
}
}
)
if (!lockResponse.executed) {
logWarn(`Ignoring redlock conflict in write-through cache`)
}

View File

@ -434,7 +434,7 @@ export class QueryBuilder<T> {
})
}
if (this.#query.empty) {
build(this.#query.empty, (key: string) => `!${key}:["" TO *]`)
build(this.#query.empty, (key: string) => `(*:* -${key}:["" TO *])`)
}
if (this.#query.notEmpty) {
build(this.#query.notEmpty, (key: string) => `${key}:["" TO *]`)

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "2.5.6-alpha.45",
"version": "2.5.10-alpha.1",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",
@ -38,8 +38,8 @@
],
"dependencies": {
"@adobe/spectrum-css-workflow-icons": "1.2.1",
"@budibase/shared-core": "2.5.6-alpha.45",
"@budibase/string-templates": "2.5.6-alpha.45",
"@budibase/shared-core": "2.5.10-alpha.1",
"@budibase/string-templates": "2.5.10-alpha.1",
"@spectrum-css/accordion": "3.0.24",
"@spectrum-css/actionbutton": "1.0.1",
"@spectrum-css/actiongroup": "1.0.1",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "2.5.6-alpha.45",
"version": "2.5.10-alpha.1",
"license": "GPL-3.0",
"private": true,
"scripts": {
@ -58,10 +58,10 @@
}
},
"dependencies": {
"@budibase/bbui": "2.5.6-alpha.45",
"@budibase/frontend-core": "2.5.6-alpha.45",
"@budibase/shared-core": "2.5.6-alpha.45",
"@budibase/string-templates": "2.5.6-alpha.45",
"@budibase/bbui": "2.5.10-alpha.1",
"@budibase/frontend-core": "2.5.10-alpha.1",
"@budibase/shared-core": "2.5.10-alpha.1",
"@budibase/string-templates": "2.5.10-alpha.1",
"@fortawesome/fontawesome-svg-core": "^6.2.1",
"@fortawesome/free-brands-svg-icons": "^6.2.1",
"@fortawesome/free-solid-svg-icons": "^6.2.1",

View File

@ -42,16 +42,7 @@ export const parseFile = e => {
reader.addEventListener("load", function (e) {
const fileData = e.target.result
if (file.type === "text/csv") {
API.csvToJson(fileData)
.then(rows => {
resolveRows(rows)
})
.catch(() => {
reject("can't convert csv to json")
})
} else if (file.type === "application/json") {
if (file.type?.includes("json")) {
const parsedFileData = JSON.parse(fileData)
if (Array.isArray(parsedFileData)) {
@ -62,7 +53,13 @@ export const parseFile = e => {
reject("invalid json format")
}
} else {
reject("invalid file type")
API.csvToJson(fileData)
.then(rows => {
resolveRows(rows)
})
.catch(() => {
reject("cannot parse csv")
})
}
})

View File

@ -41,7 +41,7 @@ export function createUsersStore() {
inviteCode,
password,
firstName,
lastName,
lastName: !lastName?.trim() ? undefined : lastName,
})
}

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "2.5.6-alpha.45",
"version": "2.5.10-alpha.1",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "dist/index.js",
"bin": {
@ -29,9 +29,9 @@
"outputPath": "build"
},
"dependencies": {
"@budibase/backend-core": "2.5.6-alpha.45",
"@budibase/string-templates": "2.5.6-alpha.45",
"@budibase/types": "2.5.6-alpha.45",
"@budibase/backend-core": "2.5.10-alpha.1",
"@budibase/string-templates": "2.5.10-alpha.1",
"@budibase/types": "2.5.10-alpha.1",
"axios": "0.21.2",
"chalk": "4.1.0",
"cli-progress": "3.11.2",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "2.5.6-alpha.45",
"version": "2.5.10-alpha.1",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,11 +19,11 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "2.5.6-alpha.45",
"@budibase/frontend-core": "2.5.6-alpha.45",
"@budibase/shared-core": "2.5.6-alpha.45",
"@budibase/string-templates": "2.5.6-alpha.45",
"@budibase/types": "2.5.6-alpha.45",
"@budibase/bbui": "2.5.10-alpha.1",
"@budibase/frontend-core": "2.5.10-alpha.1",
"@budibase/shared-core": "2.5.10-alpha.1",
"@budibase/string-templates": "2.5.10-alpha.1",
"@budibase/types": "2.5.10-alpha.1",
"@spectrum-css/button": "^3.0.3",
"@spectrum-css/card": "^3.0.3",
"@spectrum-css/divider": "^1.0.3",

View File

@ -1,13 +1,13 @@
{
"name": "@budibase/frontend-core",
"version": "2.5.6-alpha.45",
"version": "2.5.10-alpha.1",
"description": "Budibase frontend core libraries used in builder and client",
"author": "Budibase",
"license": "MPL-2.0",
"svelte": "src/index.js",
"dependencies": {
"@budibase/bbui": "2.5.6-alpha.45",
"@budibase/shared-core": "2.5.6-alpha.45",
"@budibase/bbui": "2.5.10-alpha.1",
"@budibase/shared-core": "2.5.10-alpha.1",
"dayjs": "^1.11.7",
"lodash": "^4.17.21",
"socket.io-client": "^4.6.1",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/sdk",
"version": "2.5.6-alpha.45",
"version": "2.5.10-alpha.1",
"description": "Budibase Public API SDK",
"author": "Budibase",
"license": "MPL-2.0",

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "2.5.6-alpha.45",
"version": "2.5.10-alpha.1",
"description": "Budibase Web Server",
"main": "src/index.ts",
"repository": {
@ -45,12 +45,12 @@
"license": "GPL-3.0",
"dependencies": {
"@apidevtools/swagger-parser": "10.0.3",
"@budibase/backend-core": "2.5.6-alpha.45",
"@budibase/client": "2.5.6-alpha.45",
"@budibase/pro": "2.5.6-alpha.45",
"@budibase/shared-core": "2.5.6-alpha.45",
"@budibase/string-templates": "2.5.6-alpha.45",
"@budibase/types": "2.5.6-alpha.45",
"@budibase/backend-core": "2.5.10-alpha.1",
"@budibase/client": "2.5.10-alpha.1",
"@budibase/pro": "2.5.10-alpha.1",
"@budibase/shared-core": "2.5.10-alpha.1",
"@budibase/string-templates": "2.5.10-alpha.1",
"@budibase/types": "2.5.10-alpha.1",
"@bull-board/api": "3.7.0",
"@bull-board/koa": "3.9.4",
"@elastic/elasticsearch": "7.10.0",

View File

@ -37,7 +37,7 @@ import {
Table,
} from "@budibase/types"
const { cleanExportRows } = require("./utils")
import { cleanExportRows } from "./utils"
const CALCULATION_TYPES = {
SUM: "sum",
@ -391,6 +391,9 @@ export async function exportRows(ctx: UserCtx) {
const table = await db.get(ctx.params.tableId)
const rowIds = ctx.request.body.rows
let format = ctx.query.format
if (typeof format !== "string") {
ctx.throw(400, "Format parameter is not valid")
}
const { columns, query } = ctx.request.body
let result

View File

@ -137,8 +137,8 @@ export function cleanExportRows(
delete schema[column]
})
// Intended to avoid 'undefined' in export
if (format === Format.CSV) {
// Intended to append empty values in export
const schemaKeys = Object.keys(schema)
for (let key of schemaKeys) {
if (columns?.length && columns.indexOf(key) > 0) {
@ -146,7 +146,7 @@ export function cleanExportRows(
}
for (let row of cleanRows) {
if (row[key] == null) {
row[key] = ""
row[key] = undefined
}
}
}

View File

@ -10,7 +10,7 @@ import { getDatasourceParams } from "../../../db/utils"
import { context, events } from "@budibase/backend-core"
import { Table, UserCtx } from "@budibase/types"
import sdk from "../../../sdk"
import csv from "csvtojson"
import { jsonFromCsvString } from "../../../utilities/csv"
function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
if (table && !tableId) {
@ -104,7 +104,7 @@ export async function bulkImport(ctx: UserCtx) {
export async function csvToJson(ctx: UserCtx) {
const { csvString } = ctx.request.body
const result = await csv().fromString(csvString)
const result = await jsonFromCsvString(csvString)
ctx.status = 200
ctx.body = result

View File

@ -10,7 +10,9 @@ export function csv(headers: string[], rows: Row[]) {
val =
typeof val === "object" && !(val instanceof Date)
? `"${JSON.stringify(val).replace(/"/g, "'")}"`
: `"${val}"`
: val !== undefined
? `"${val}"`
: ""
return val.trim()
})
.join(",")}`

View File

@ -105,7 +105,7 @@ describe("internal search", () => {
"column": "",
},
}, PARAMS)
checkLucene(response, `*:* AND !column:["" TO *]`, PARAMS)
checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
})
it("test notEmpty query", async () => {

View File

@ -0,0 +1,22 @@
import csv from "csvtojson"
export async function jsonFromCsvString(csvString: string) {
const castedWithEmptyValues = await csv({ ignoreEmpty: true }).fromString(
csvString
)
// By default the csvtojson library casts empty values as empty strings. This is causing issues on conversion.
// ignoreEmpty will remove the key completly if empty, so creating this empty object will ensure we return the values with the keys but empty values
const result = await csv({ ignoreEmpty: false }).fromString(csvString)
result.forEach((r, i) => {
for (const [key] of Object.entries(r).filter(
([key, value]) => value === ""
)) {
if (castedWithEmptyValues[i][key] === undefined) {
r[key] = null
}
}
})
return result
}

View File

@ -4,6 +4,9 @@ interface SchemaColumn {
readonly name: string
readonly type: FieldTypes
readonly autocolumn?: boolean
readonly constraints?: {
presence: boolean
}
}
interface Schema {
@ -76,6 +79,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
// If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
if (typeof columnType !== "string") {
results.invalidColumns.push(columnName)
} else if (
columnData == null &&
!schema[columnName].constraints?.presence
) {
results.schemaValidation[columnName] = true
} else if (
// If there's no data for this field don't bother with further checks
// If the field is already marked as invalid there's no need for further checks

View File

@ -0,0 +1,33 @@
import { jsonFromCsvString } from "../csv"
describe("csv", () => {
describe("jsonFromCsvString", () => {
test("multiple lines csv can be casted", async () => {
const csvString = '"id","title"\n"1","aaa"\n"2","bbb"'
const result = await jsonFromCsvString(csvString)
expect(result).toEqual([
{ id: "1", title: "aaa" },
{ id: "2", title: "bbb" },
])
result.forEach(r => expect(Object.keys(r)).toEqual(["id", "title"]))
})
test("empty values are casted as undefined", async () => {
const csvString =
'"id","optional","title"\n1,,"aaa"\n2,"value","bbb"\n3,,"ccc"'
const result = await jsonFromCsvString(csvString)
expect(result).toEqual([
{ id: "1", optional: null, title: "aaa" },
{ id: "2", optional: "value", title: "bbb" },
{ id: "3", optional: null, title: "ccc" },
])
result.forEach(r =>
expect(Object.keys(r)).toEqual(["id", "optional", "title"])
)
})
})
})

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/shared-core",
"version": "2.5.6-alpha.45",
"version": "2.5.10-alpha.1",
"description": "Shared data utils",
"main": "dist/cjs/src/index.js",
"types": "dist/mjs/src/index.d.ts",
@ -20,7 +20,7 @@
"dev:builder": "yarn prebuild && concurrently \"tsc -p tsconfig.build.json --watch\" \"tsc -p tsconfig-cjs.build.json --watch\""
},
"dependencies": {
"@budibase/types": "2.5.6-alpha.45"
"@budibase/types": "2.5.10-alpha.1"
},
"devDependencies": {
"concurrently": "^7.6.0",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "2.5.6-alpha.45",
"version": "2.5.10-alpha.1",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/types",
"version": "2.5.6-alpha.45",
"version": "2.5.10-alpha.1",
"description": "Budibase types",
"main": "dist/cjs/index.js",
"types": "dist/mjs/index.d.ts",

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "2.5.6-alpha.45",
"version": "2.5.10-alpha.1",
"description": "Budibase background service",
"main": "src/index.ts",
"repository": {
@ -37,10 +37,10 @@
"author": "Budibase",
"license": "GPL-3.0",
"dependencies": {
"@budibase/backend-core": "2.5.6-alpha.45",
"@budibase/pro": "2.5.6-alpha.45",
"@budibase/string-templates": "2.5.6-alpha.45",
"@budibase/types": "2.5.6-alpha.45",
"@budibase/backend-core": "2.5.10-alpha.1",
"@budibase/pro": "2.5.10-alpha.1",
"@budibase/string-templates": "2.5.10-alpha.1",
"@budibase/types": "2.5.10-alpha.1",
"@koa/router": "8.0.8",
"@sentry/node": "6.17.7",
"@techpass/passport-openidconnect": "0.3.2",

View File

@ -1486,15 +1486,15 @@
pouchdb-promise "^6.0.4"
through2 "^2.0.0"
"@budibase/pro@2.5.6-alpha.44":
version "2.5.6-alpha.44"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.6-alpha.44.tgz#f849e7bceb06144083cc4b1d406ffa195348c4f1"
integrity sha512-duU1w9OhnycNvdfJx941OwrI3siTL3bOF793/yp8GisqSH/76vEwpu1VsAV/pqRywkwpr1LkilPkPPufiNURLg==
"@budibase/pro@2.5.10-alpha.0":
version "2.5.10-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.10-alpha.0.tgz#f3688cc61d2130fbf8b191e515b8ac9789f2a9df"
integrity sha512-vYa1F4NhMh7veSZyhgrE54iziwZLHiuRwXbU+unEo3xqyFZIvERAAQZ88nrnwssubWBWnPhpN/8JMHrsyM8mXg==
dependencies:
"@budibase/backend-core" "2.5.6-alpha.44"
"@budibase/shared-core" "2.4.44-alpha.1"
"@budibase/string-templates" "2.4.44-alpha.1"
"@budibase/types" "2.5.6-alpha.44"
"@budibase/backend-core" "2.5.10-alpha.0"
"@budibase/shared-core" "2.5.9"
"@budibase/string-templates" "2.5.9"
"@budibase/types" "2.5.10-alpha.0"
"@koa/router" "8.0.8"
bull "4.10.1"
joi "17.6.0"
@ -1505,12 +1505,12 @@
scim-patch "^0.7.0"
scim2-parse-filter "^0.2.8"
"@budibase/shared-core@2.4.44-alpha.1":
version "2.4.44-alpha.1"
resolved "https://registry.yarnpkg.com/@budibase/shared-core/-/shared-core-2.4.44-alpha.1.tgz#3d499e40e7e6c646e13a87cd08e01ba116c2ff1d"
integrity sha512-cN8LaDczijtsfWUYiXC4sg9Z+US4020i3Mb8TwCbf8TQyA1b06U5PwPCp+GHVA/wDFqfwcpcE1GXf8GwVuYs7A==
"@budibase/shared-core@2.5.9":
version "2.5.9"
resolved "https://registry.yarnpkg.com/@budibase/shared-core/-/shared-core-2.5.9.tgz#f22f22637fb7618ded1c7292b10793d7969827ce"
integrity sha512-l417Rb2+1tuXbjNL42wJHmqIQQyla2pPglOnapxfOdRuvzng+5GqlrTV1caLNf/53TS9U6ueGJdPOtxZTTFGUA==
dependencies:
"@budibase/types" "2.4.44-alpha.1"
"@budibase/types" "^2.5.9"
"@budibase/standard-components@^0.9.139":
version "0.9.139"
@ -1530,22 +1530,22 @@
svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0"
"@budibase/string-templates@2.4.44-alpha.1":
version "2.4.44-alpha.1"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-2.4.44-alpha.1.tgz#6c2aee594d16eac1f173c509e087a817dd3172f0"
integrity sha512-4gC2+0kccK0ilLnd0i/dmJzC0Ur7UgSAmV6zbzDDYNL4spU0qSy5VhBh7E3qKieg5RKMMzbpXLMWERpoPLlnqA==
"@budibase/string-templates@2.5.9":
version "2.5.9"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-2.5.9.tgz#64582730421801e1e829b430136b449b1adc0314"
integrity sha512-Szu06M0JFHuUVIil2aHZWU8hvsROYpDx9raX9uIv4DcwBOAtyvVzD16wOCHzlmj8wWeV8fbKe4JF4q4GXnilJg==
dependencies:
"@budibase/handlebars-helpers" "^0.11.8"
dayjs "^1.10.4"
handlebars "^4.7.6"
handlebars-utils "^1.0.6"
lodash "^4.17.20"
vm2 "^3.9.4"
vm2 "^3.9.15"
"@budibase/types@2.4.44-alpha.1":
version "2.4.44-alpha.1"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.4.44-alpha.1.tgz#1679657aa180d9c59afa1dffa611bff0638bd933"
integrity sha512-Sq+8HfM75EBMoOvKYFwELdlxmVN6wNZMofDjT/2G+9aF+Zfe5Tzw69C+unmdBgcGGjGCHEYWSz4mF0v8FPAGbg==
"@budibase/types@^2.5.9":
version "2.5.9"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.5.9.tgz#4b1253e76b95cd8d6a42e86ebc3363a305c62653"
integrity sha512-7NELdWB3iV5y+pmXhm9g/v1AlN+wqHtoy67DZxKY5dlcK4DCcTdlqkToGiiw9d3sDPgDAVznjdZYnB5pGzd3TQ==
"@bull-board/api@3.7.0":
version "3.7.0"
@ -24502,7 +24502,7 @@ vm2@3.9.17:
acorn "^8.7.0"
acorn-walk "^8.2.0"
vm2@^3.9.11, vm2@^3.9.15, vm2@^3.9.4:
vm2@^3.9.11, vm2@^3.9.15:
version "3.9.16"
resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.16.tgz#0fbc2a265f7bf8b837cea6f4a908f88a3f93b8e6"
integrity sha512-3T9LscojNTxdOyG+e8gFeyBXkMlOBYDoF6dqZbj+MPVHi9x10UfiTAJIobuchRCp3QvC+inybTbMJIUrLsig0w==