Merge branch 'plugins-dev-experience' of github.com:Budibase/budibase into plugins-dev-experience

This commit is contained in:
Andrew Kingston 2022-08-15 22:24:41 +01:00
commit 30406f7f35
38 changed files with 710 additions and 772 deletions

View File

@ -1,5 +1,5 @@
{
"version": "1.2.39-alpha.1",
"version": "1.2.39-alpha.6",
"npmClient": "yarn",
"packages": [
"packages/*"

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
"version": "1.2.39-alpha.1",
"version": "1.2.39-alpha.6",
"description": "Budibase backend core libraries used in server and worker",
"main": "dist/src/index.js",
"types": "dist/src/index.d.ts",
@ -20,7 +20,7 @@
"test:watch": "jest --watchAll"
},
"dependencies": {
"@budibase/types": "1.2.39-alpha.1",
"@budibase/types": "1.2.39-alpha.6",
"@techpass/passport-openidconnect": "0.3.2",
"aws-sdk": "2.1030.0",
"bcrypt": "5.0.1",

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "1.2.39-alpha.1",
"version": "1.2.39-alpha.6",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",
@ -38,7 +38,7 @@
],
"dependencies": {
"@adobe/spectrum-css-workflow-icons": "^1.2.1",
"@budibase/string-templates": "1.2.39-alpha.1",
"@budibase/string-templates": "1.2.39-alpha.6",
"@spectrum-css/actionbutton": "^1.0.1",
"@spectrum-css/actiongroup": "^1.0.1",
"@spectrum-css/avatar": "^3.0.2",

View File

@ -23,7 +23,7 @@
$: toggleOption = makeToggleOption(selectedLookupMap, value)
const getFieldText = (value, map, placeholder) => {
if (value?.length) {
if (Array.isArray(value) && value.length > 0) {
if (!map) {
return ""
}
@ -36,7 +36,7 @@
const getSelectedLookupMap = value => {
let map = {}
if (value?.length) {
if (Array.isArray(value) && value.length > 0) {
value.forEach(option => {
if (option) {
map[option] = true

View File

@ -63,7 +63,7 @@
<style>
.spectrum-Popover {
min-width: var(--spectrum-global-dimension-size-2000) !important;
min-width: var(--spectrum-global-dimension-size-2000);
}
.spectrum-Popover.is-open.spectrum-Popover--withTip {
margin-top: var(--spacing-xs);

View File

@ -266,7 +266,7 @@ filterTests(["all"], () => {
cy.reload()
cy.log("Current deployment version: " + clientPackage.version)
cy.get(".version-status a", { timeout: 1000 }).contains("Update").click()
cy.get(".version-status a", { timeout: 5000 }).contains("Update").click()
cy.get(".spectrum-Tabs-item.is-selected").contains("Settings")
cy.get(".version-section .page-action button")

View File

@ -128,7 +128,9 @@ Cypress.Commands.add("updateUserInformation", (firstName, lastName) => {
.should("have.value", lastName)
.blur()
}
cy.get("button").contains("Update information").click({ force: true })
cy.get(".confirm-wrap").within(() => {
cy.get("button").contains("Update information").click({ force: true })
})
cy.get(".spectrum-Dialog-grid").should("not.exist")
})
})
@ -162,7 +164,7 @@ Cypress.Commands.add("createApp", (name, addDefaultTable) => {
typeof addDefaultTable != "boolean" ? true : addDefaultTable
cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 10000 })
cy.wait(1000)
cy.url({ timeout: 30000 }).should("include", "/apps")
cy.get(`[data-cy="create-app-btn"]`, { timeout: 5000 }).click({ force: true })
// If apps already exist
@ -432,6 +434,7 @@ Cypress.Commands.add("createAppFromScratch", appName => {
// TABLES
Cypress.Commands.add("createTable", (tableName, initialTable) => {
// Creates an internal Budibase DB table
if (!initialTable) {
cy.navigateToDataSection()
cy.get(`[data-cy="new-table"]`, { timeout: 2000 }).click()
@ -445,6 +448,10 @@ Cypress.Commands.add("createTable", (tableName, initialTable) => {
.contains("Continue")
.click({ force: true })
})
cy.get(".spectrum-Modal", { timeout: 10000 }).should(
"not.contain",
"Add data source"
)
cy.get(".spectrum-Modal", { timeout: 2000 }).within(() => {
cy.get("input", { timeout: 2000 }).first().type(tableName).blur()
cy.get(".spectrum-ButtonGroup").contains("Create").click()

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "1.2.39-alpha.1",
"version": "1.2.39-alpha.6",
"license": "GPL-3.0",
"private": true,
"scripts": {
@ -69,10 +69,10 @@
}
},
"dependencies": {
"@budibase/bbui": "1.2.39-alpha.1",
"@budibase/client": "1.2.39-alpha.1",
"@budibase/frontend-core": "1.2.39-alpha.1",
"@budibase/string-templates": "1.2.39-alpha.1",
"@budibase/bbui": "1.2.39-alpha.6",
"@budibase/client": "1.2.39-alpha.6",
"@budibase/frontend-core": "1.2.39-alpha.6",
"@budibase/string-templates": "1.2.39-alpha.6",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",

View File

@ -0,0 +1,39 @@
<script>
export let width = "100"
export let height = "100"
let color =
"var(--spectrum-heading-xxs-text-color, var(--spectrum-alias-heading-text-color))"
</script>
<svg
{width}
{height}
xmlns="http://www.w3.org/2000/svg"
x="0px"
y="0px"
viewBox="0 0 230.795 230.795"
style="enable-background:new 0 0 230.795 230.795;"
xml:space="preserve"
>
<g>
<path
d="M60.357,63.289c-2.929-2.929-7.678-2.93-10.606-0.001L2.197,110.836C0.79,112.243,0,114.151,0,116.14
c0,1.989,0.79,3.896,2.196,5.303l47.348,47.35c1.465,1.465,3.384,2.197,5.304,2.197c1.919,0,3.839-0.732,5.303-2.196
c2.93-2.929,2.93-7.678,0.001-10.606L18.107,116.14l42.25-42.245C63.286,70.966,63.286,66.217,60.357,63.289z"
fill={color}
/>
<path
d="M228.598,110.639l-47.355-47.352c-2.928-2.928-7.677-2.929-10.606,0.001c-2.929,2.929-2.929,7.678,0.001,10.607
l42.051,42.048l-42.249,42.243c-2.93,2.929-2.93,7.678-0.001,10.606c1.465,1.465,3.384,2.197,5.304,2.197
c1.919,0,3.839-0.732,5.303-2.196l47.554-47.547c1.407-1.406,2.197-3.314,2.197-5.304
C230.795,113.954,230.005,112.046,228.598,110.639z"
fill={color}
/>
<path
d="M155.889,61.302c-3.314-2.484-8.017-1.806-10.498,1.51l-71.994,96.184c-2.482,3.316-1.807,8.017,1.51,10.498
c1.348,1.01,2.925,1.496,4.488,1.496c2.282,0,4.537-1.038,6.01-3.006L157.398,71.8C159.881,68.484,159.205,63.784,155.889,61.302z"
fill={color}
/>
</g>
<g />
</svg>

View File

@ -15,6 +15,7 @@ import GoogleSheets from "./GoogleSheets.svelte"
import Firebase from "./Firebase.svelte"
import Redis from "./Redis.svelte"
import Snowflake from "./Snowflake.svelte"
import Custom from "./Custom.svelte"
export default {
BUDIBASE: Budibase,
@ -34,4 +35,5 @@ export default {
FIRESTORE: Firebase,
REDIS: Redis,
SNOWFLAKE: Snowflake,
CUSTOM: Custom,
}

View File

@ -92,6 +92,14 @@
}
integrations = newIntegrations
}
function getIcon(integrationType, schema) {
if (schema.custom) {
return ICONS.CUSTOM
} else {
return ICONS[integrationType]
}
}
</script>
<Modal bind:this={internalTableModal}>
@ -158,7 +166,7 @@
>
<div class="item-body" class:with-type={!!schema.type}>
<svelte:component
this={ICONS[integrationType]}
this={getIcon(integrationType, schema)}
height="20"
width="20"
/>

View File

@ -3,6 +3,7 @@
Body,
Button,
Combobox,
Multiselect,
DatePicker,
DrawerContent,
Icon,
@ -97,6 +98,16 @@
if (expression.noValue) {
expression.value = null
}
if (
operator === Constants.OperatorOptions.In.value &&
!Array.isArray(expression.value)
) {
if (expression.value) {
expression.value = [expression.value]
} else {
expression.value = []
}
}
}
const getFieldOptions = field => {
@ -169,7 +180,13 @@
/>
{:else if ["string", "longform", "number", "formula"].includes(filter.type)}
<Input disabled={filter.noValue} bind:value={filter.value} />
{:else if ["options", "array"].includes(filter.type)}
{:else if filter.type === "array" || (filter.type === "options" && filter.operator === "oneOf")}
<Multiselect
disabled={filter.noValue}
options={getFieldOptions(filter.field)}
bind:value={filter.value}
/>
{:else if filter.type === "options"}
<Combobox
disabled={filter.noValue}
options={getFieldOptions(filter.field)}

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "1.2.39-alpha.1",
"version": "1.2.39-alpha.6",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {

View File

@ -1466,10 +1466,11 @@
},
{
"type": "select",
"label": "Colours",
"label": "Colors",
"key": "palette",
"defaultValue": "Palette 1",
"options": [
"Custom",
"Palette 1",
"Palette 2",
"Palette 3",
@ -1482,6 +1483,51 @@
"Palette 10"
]
},
{
"type": "color",
"label": "C1",
"key": "c1",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C2",
"key": "c2",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C3",
"key": "c3",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C4",
"key": "c4",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C5",
"key": "c5",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "boolean",
"label": "Stacked",
@ -1581,10 +1627,11 @@
},
{
"type": "select",
"label": "Colours",
"label": "Colors",
"key": "palette",
"defaultValue": "Palette 1",
"options": [
"Custom",
"Palette 1",
"Palette 2",
"Palette 3",
@ -1597,6 +1644,51 @@
"Palette 10"
]
},
{
"type": "color",
"label": "C1",
"key": "c1",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C2",
"key": "c2",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C3",
"key": "c3",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C4",
"key": "c4",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C5",
"key": "c5",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "select",
"label": "Curve",
@ -1695,10 +1787,11 @@
},
{
"type": "select",
"label": "Colours",
"label": "Colors",
"key": "palette",
"defaultValue": "Palette 1",
"options": [
"Custom",
"Palette 1",
"Palette 2",
"Palette 3",
@ -1711,6 +1804,51 @@
"Palette 10"
]
},
{
"type": "color",
"label": "C1",
"key": "c1",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C2",
"key": "c2",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C3",
"key": "c3",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C4",
"key": "c4",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C5",
"key": "c5",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "select",
"label": "Curve",
@ -1800,10 +1938,11 @@
},
{
"type": "select",
"label": "Colours",
"label": "Colors",
"key": "palette",
"defaultValue": "Palette 1",
"options": [
"Custom",
"Palette 1",
"Palette 2",
"Palette 3",
@ -1816,6 +1955,51 @@
"Palette 10"
]
},
{
"type": "color",
"label": "C1",
"key": "c1",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C2",
"key": "c2",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C3",
"key": "c3",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C4",
"key": "c4",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C5",
"key": "c5",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "boolean",
"label": "Data Labels",
@ -1882,10 +2066,11 @@
},
{
"type": "select",
"label": "Colours",
"label": "Colors",
"key": "palette",
"defaultValue": "Palette 1",
"options": [
"Custom",
"Palette 1",
"Palette 2",
"Palette 3",
@ -1898,6 +2083,51 @@
"Palette 10"
]
},
{
"type": "color",
"label": "C1",
"key": "c1",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C2",
"key": "c2",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C3",
"key": "c3",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C4",
"key": "c4",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "color",
"label": "C5",
"key": "c5",
"dependsOn": {
"setting": "palette",
"value": "Custom"
}
},
{
"type": "boolean",
"label": "Data Labels",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "1.2.39-alpha.1",
"version": "1.2.39-alpha.6",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "1.2.39-alpha.1",
"@budibase/frontend-core": "1.2.39-alpha.1",
"@budibase/string-templates": "1.2.39-alpha.1",
"@budibase/bbui": "1.2.39-alpha.6",
"@budibase/frontend-core": "1.2.39-alpha.6",
"@budibase/string-templates": "1.2.39-alpha.6",
"@spectrum-css/button": "^3.0.3",
"@spectrum-css/card": "^3.0.3",
"@spectrum-css/divider": "^1.0.3",

View File

@ -10,7 +10,9 @@
</script>
{#if options}
<div use:chart={options} use:styleable={$component.styles} />
{#key options.customColor}
<div use:chart={options} use:styleable={$component.styles} />
{/key}
{:else if $builderStore.inBuilder}
<div use:styleable={$component.styles}>
<Placeholder />

View File

@ -62,8 +62,14 @@ export class ApexOptionsBuilder {
return this.setOption(["title", "text"], title)
}
color(color) {
return this.setOption(["colors"], [color])
colors(colors) {
if (!colors) {
delete this.options.colors
this.options["customColor"] = false
return this
}
this.options["customColor"] = true
return this.setOption(["colors"], colors)
}
width(width) {

View File

@ -16,6 +16,7 @@
export let stacked
export let yAxisUnits
export let palette
export let c1, c2, c3, c4, c5
export let horizontal
$: options = setUpChart(
@ -33,9 +34,13 @@
stacked,
yAxisUnits,
palette,
horizontal
horizontal,
c1 && c2 && c3 && c4 && c5 ? [c1, c2, c3, c4, c5] : null,
customColor
)
$: customColor = palette === "Custom"
const setUpChart = (
title,
dataProvider,
@ -51,7 +56,9 @@
stacked,
yAxisUnits,
palette,
horizontal
horizontal,
colors,
customColor
) => {
const allCols = [labelColumn, ...(valueColumns || [null])]
if (
@ -85,6 +92,7 @@
.stacked(stacked)
.palette(palette)
.horizontal(horizontal)
.colors(customColor ? colors : null)
// Add data
let useDates = false

View File

@ -17,6 +17,7 @@
export let legend
export let yAxisUnits
export let palette
export let c1, c2, c3, c4, c5
// Area specific props
export let area
@ -40,9 +41,13 @@
palette,
area,
stacked,
gradient
gradient,
c1 && c2 && c3 && c4 && c5 ? [c1, c2, c3, c4, c5] : null,
customColor
)
$: customColor = palette === "Custom"
const setUpChart = (
title,
dataProvider,
@ -60,7 +65,9 @@
palette,
area,
stacked,
gradient
gradient,
colors,
customColor
) => {
const allCols = [labelColumn, ...(valueColumns || [null])]
if (
@ -96,6 +103,7 @@
.legend(legend)
.yUnits(yAxisUnits)
.palette(palette)
.colors(customColor ? colors : null)
// Add data
let useDates = false

View File

@ -13,6 +13,7 @@
export let legend
export let donut
export let palette
export let c1, c2, c3, c4, c5
$: options = setUpChart(
title,
@ -25,9 +26,13 @@
animate,
legend,
donut,
palette
palette,
c1 && c2 && c3 && c4 && c5 ? [c1, c2, c3, c4, c5] : null,
customColor
)
$: customColor = palette === "Custom"
const setUpChart = (
title,
dataProvider,
@ -39,7 +44,9 @@
animate,
legend,
donut,
palette
palette,
colors,
customColor
) => {
if (
!dataProvider ||
@ -70,6 +77,7 @@
.legend(legend)
.legendPosition("right")
.palette(palette)
.colors(customColor ? colors : null)
// Add data if valid datasource
const series = data.map(row => parseFloat(row[valueColumn]))

View File

@ -1,12 +1,12 @@
{
"name": "@budibase/frontend-core",
"version": "1.2.39-alpha.1",
"version": "1.2.39-alpha.6",
"description": "Budibase frontend core libraries used in builder and client",
"author": "Budibase",
"license": "MPL-2.0",
"svelte": "src/index.js",
"dependencies": {
"@budibase/bbui": "1.2.39-alpha.1",
"@budibase/bbui": "1.2.39-alpha.6",
"lodash": "^4.17.21",
"svelte": "^3.46.2"
}

View File

@ -39,13 +39,17 @@ export const OperatorOptions = {
label: "Contains",
},
NotContains: {
value: "notEqual",
value: "notContains",
label: "Does Not Contain",
},
In: {
value: "oneOf",
label: "Is in",
},
ContainsAny: {
value: "containsAny",
label: "Has any",
},
}
// Cookie names

View File

@ -32,9 +32,9 @@ export const getValidOperatorsForType = type => {
} else if (type === "number") {
return numOps
} else if (type === "options") {
return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
} else if (type === "array") {
return [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty]
return [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny]
} else if (type === "boolean") {
return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
} else if (type === "longform") {
@ -96,6 +96,7 @@ export const buildLuceneQuery = filter => {
contains: {},
notContains: {},
oneOf: {},
containsAny: {},
}
if (Array.isArray(filter)) {
filter.forEach(expression => {
@ -128,6 +129,13 @@ export const buildLuceneQuery = filter => {
if (type === "boolean") {
value = `${value}`?.toLowerCase() === "true"
}
if (
["contains", "notContains", "containsAny"].includes(operator) &&
type === "array" &&
typeof value === "string"
) {
value = value.split(",")
}
if (operator.startsWith("range")) {
const minint =
SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER
@ -244,6 +252,18 @@ export const runLuceneQuery = (docs, query) => {
return !testValue?.includes(docValue)
})
const containsAny = match("containsAny", (docValue, testValue) => {
return !docValue?.includes(...testValue)
})
const contains = match("contains", (docValue, testValue) => {
return !testValue?.every(item => docValue?.includes(item))
})
const notContains = match("notContains", (docValue, testValue) => {
return testValue?.every(item => docValue?.includes(item))
})
// Match a document against all criteria
const docMatch = doc => {
return (
@ -254,7 +274,10 @@ export const runLuceneQuery = (docs, query) => {
notEqualMatch(doc) &&
emptyMatch(doc) &&
notEmptyMatch(doc) &&
oneOf(doc)
oneOf(doc) &&
contains(doc) &&
containsAny(doc) &&
notContains(doc)
)
}

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "1.2.39-alpha.1",
"version": "1.2.39-alpha.6",
"description": "Budibase Web Server",
"main": "src/index.ts",
"repository": {
@ -77,11 +77,11 @@
"license": "GPL-3.0",
"dependencies": {
"@apidevtools/swagger-parser": "10.0.3",
"@budibase/backend-core": "1.2.39-alpha.1",
"@budibase/client": "1.2.39-alpha.1",
"@budibase/pro": "1.2.39-alpha.0",
"@budibase/string-templates": "1.2.39-alpha.1",
"@budibase/types": "1.2.39-alpha.1",
"@budibase/backend-core": "1.2.39-alpha.6",
"@budibase/client": "1.2.39-alpha.6",
"@budibase/pro": "1.2.39-alpha.6",
"@budibase/string-templates": "1.2.39-alpha.6",
"@budibase/types": "1.2.39-alpha.6",
"@bull-board/api": "3.7.0",
"@bull-board/koa": "3.9.4",
"@elastic/elasticsearch": "7.10.0",

View File

@ -7,7 +7,7 @@ const {
getTableParams,
} = require("../../db/utils")
const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
const { integrations } = require("../../integrations")
const { getIntegration } = require("../../integrations")
const { getDatasourceAndQuery } = require("./row/utils")
const { invalidateDynamicVariables } = require("../../threads/utils")
const { getAppDB } = require("@budibase/backend-core/context")
@ -114,7 +114,7 @@ exports.update = async function (ctx) {
// Drain connection pools when configuration is changed
if (datasource.source) {
const source = integrations[datasource.source]
const source = await getIntegration(datasource.source)
if (source && source.pool) {
await source.pool.end()
}
@ -149,7 +149,7 @@ exports.save = async function (ctx) {
// Drain connection pools when configuration is changed
if (datasource.source) {
const source = integrations[datasource.source]
const source = await getIntegration(datasource.source)
if (source && source.pool) {
await source.pool.end()
}
@ -218,7 +218,7 @@ function updateError(error, newError, tables) {
}
const buildSchemaHelper = async datasource => {
const Connector = integrations[datasource.source]
const Connector = await getIntegration(datasource.source)
// Connect to the DB and build the schema
const connector = new Connector(datasource.config)

View File

@ -21,6 +21,8 @@ class QueryBuilder {
notEmpty: {},
oneOf: {},
contains: {},
notContains: {},
containsAny: {},
...base,
}
this.limit = 50
@ -126,6 +128,16 @@ class QueryBuilder {
return this
}
addNotContains(key, value) {
this.query.notContains[key] = value
return this
}
addContainsAny(key, value) {
this.query.containsAny[key] = value
return this
}
/**
* Preprocesses a value before going into a lucene search.
* Transforms strings to lowercase and wraps strings and bools in quotes.
@ -171,11 +183,29 @@ class QueryBuilder {
return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`
}
const contains = (key, value) => {
if (!value && value !== 0) {
const contains = (key, value, mode = "AND") => {
if (Array.isArray(value) && value.length === 0) {
return null
}
return `${key}:${builder.preprocess(value, { escape: true })}`
if (!Array.isArray(value)) {
return `${key}:${value}`
}
let statement = `${builder.preprocess(value[0], { escape: true })}`
for (let i = 1; i < value.length; i++) {
statement += ` ${mode} ${builder.preprocess(value[i], {
escape: true,
})}`
}
return `${key}:(${statement})`
}
const notContains = (key, value) => {
const allPrefix = allOr === "" ? "*:* AND" : ""
return allPrefix + "NOT " + contains(key, value)
}
const containsAny = (key, value) => {
return contains(key, value, "OR")
}
const oneOf = (key, value) => {
@ -278,6 +308,12 @@ class QueryBuilder {
if (this.query.contains) {
build(this.query.contains, contains)
}
if (this.query.notContains) {
build(this.query.notContains, notContains)
}
if (this.query.containsAny) {
build(this.query.containsAny, containsAny)
}
// make sure table ID is always added as an AND
if (tableId) {
query = `(${query})`

View File

@ -129,9 +129,10 @@ describe("internal search", () => {
const response = await search.paginatedSearch({
contains: {
"column": "a",
"colArr": [1, 2, 3],
},
}, PARAMS)
checkLucene(response, `*:* AND column:a`, PARAMS)
checkLucene(response, `*:* AND column:a AND colArr:(1 AND 2 AND 3)`, PARAMS)
})
it("test multiple of same column", async () => {
@ -154,4 +155,22 @@ describe("internal search", () => {
}, PARAMS)
checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
})
it("test containsAny query", async () => {
const response = await search.paginatedSearch({
containsAny: {
"column": ["a", "b", "c"]
},
}, PARAMS)
checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
})
it("test notContains query", async () => {
const response = await search.paginatedSearch({
notContains: {
"column": ["a", "b", "c"]
},
}, PARAMS)
checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
})
})

View File

@ -1,11 +1,11 @@
import { QueryJson, Datasource } from "@budibase/types"
const { integrations } = require("../index")
const { getIntegration } = require("../index")
export async function makeExternalQuery(
datasource: Datasource,
json: QueryJson
) {
const Integration = integrations[datasource.source]
const Integration = await getIntegration(datasource.source)
// query is the opinionated function
if (Integration.prototype.query) {
const integration = new Integration(datasource.config)

View File

@ -159,6 +159,61 @@ class InternalBuilder {
}
}
const contains = (mode: object, any: boolean = false) => {
const fnc = allOr ? "orWhere" : "where"
const rawFnc = `${fnc}Raw`
const not = mode === filters?.notContains ? "NOT " : ""
function stringifyArray(value: Array<any>, quoteStyle = '"'): string {
for (let i in value) {
if (typeof value[i] === "string") {
value[i] = `${quoteStyle}${value[i]}${quoteStyle}`
}
}
return `[${value.join(",")}]`
}
if (this.client === SqlClient.POSTGRES) {
iterate(mode, (key: string, value: Array<any>) => {
const wrap = any ? "" : "'"
const containsOp = any ? "\\?| array" : "@>"
const fieldNames = key.split(/\./g)
const tableName = fieldNames[0]
const columnName = fieldNames[1]
// @ts-ignore
query = query[rawFnc](
`${not}"${tableName}"."${columnName}"::jsonb ${containsOp} ${wrap}${stringifyArray(
value,
any ? "'" : '"'
)}${wrap}`
)
})
} else if (this.client === SqlClient.MY_SQL) {
const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS"
iterate(mode, (key: string, value: Array<any>) => {
// @ts-ignore
query = query[rawFnc](
`${not}${jsonFnc}(${key}, '${stringifyArray(value)}')`
)
})
} else {
const andOr = mode === filters?.containsAny ? " OR " : " AND "
iterate(mode, (key: string, value: Array<any>) => {
let statement = ""
for (let i in value) {
if (typeof value[i] === "string") {
value[i] = `%"${value[i]}"%`
} else {
value[i] = `%${value[i]}%`
}
statement +=
(statement ? andOr : "") +
`LOWER(${likeKey(this.client, key)}) LIKE ?`
}
// @ts-ignore
query = query[rawFnc](`${not}(${statement})`, value)
})
}
}
if (!filters) {
return query
}
@ -229,32 +284,13 @@ class InternalBuilder {
})
}
if (filters.contains) {
const fnc = allOr ? "orWhere" : "where"
const rawFnc = `${fnc}Raw`
if (this.client === SqlClient.POSTGRES) {
iterate(filters.contains, (key: string, value: any) => {
const fieldNames = key.split(/\./g)
const tableName = fieldNames[0]
const columnName = fieldNames[1]
if (typeof value === "string") {
value = `"${value}"`
}
// @ts-ignore
query = query[rawFnc](
`"${tableName}"."${columnName}"::jsonb @> '[${value}]'`
)
})
} else if (this.client === SqlClient.MY_SQL) {
iterate(filters.contains, (key: string, value: any) => {
if (typeof value === "string") {
value = `"${value}"`
}
// @ts-ignore
query = query[rawFnc](`JSON_CONTAINS(${key}, '${value}')`)
})
} else {
iterate(filters.contains, like)
}
contains(filters.contains)
}
if (filters.notContains) {
contains(filters.notContains)
}
if (filters.containsAny) {
contains(filters.containsAny, true)
}
return query
}

View File

@ -67,8 +67,22 @@ if (environment.SELF_HOSTED) {
module.exports = {
getDefinitions: async () => {
const custom = await getPlugins(PluginType.DATASOURCE)
return cloneDeep(DEFINITIONS)
const plugins = await getPlugins(PluginType.DATASOURCE)
// extract the actual schema from each custom
const pluginSchemas: { [key: string]: Integration } = {}
for (let plugin of plugins) {
const sourceId = plugin.name
pluginSchemas[sourceId] = {
...plugin.schema["schema"],
custom: true,
}
}
return {
...cloneDeep(DEFINITIONS),
...pluginSchemas,
}
},
getIntegration: async () => {
return INTEGRATIONS
},
integrations: INTEGRATIONS,
}

View File

@ -240,18 +240,18 @@ describe("SQL query builder", () => {
})
})
it("should use like expression for MS-SQL when filter is contains", () => {
it("should use AND like expression for MS-SQL when filter is contains", () => {
const query = new Sql(SqlClient.MS_SQL, 10)._query(generateReadJson({
filters: {
contains: {
age: 20,
name: "John"
age: [20, 25],
name: ["John", "Mary"]
}
}
}))
expect(query).toEqual({
bindings: [10, "%20%", "%John%"],
sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where LOWER(${TABLE_NAME}.age) LIKE @p1 and LOWER(${TABLE_NAME}.name) LIKE @p2) as [${TABLE_NAME}]`
bindings: [10, "%20%", "%25%", `%"John"%`, `%"Mary"%`],
sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where (LOWER(${TABLE_NAME}.age) LIKE @p1 AND LOWER(${TABLE_NAME}.age) LIKE @p2) and (LOWER(${TABLE_NAME}.name) LIKE @p3 AND LOWER(${TABLE_NAME}.name) LIKE @p4)) as [${TABLE_NAME}]`
})
})
@ -259,14 +259,14 @@ describe("SQL query builder", () => {
const query = new Sql(SqlClient.MY_SQL, 10)._query(generateReadJson({
filters: {
contains: {
age: 20,
name: "John"
age: [20],
name: ["John"]
}
}
}))
expect(query).toEqual({
bindings: [10],
sql: `select * from (select * from \`${TABLE_NAME}\` where JSON_CONTAINS(${TABLE_NAME}.age, '20') and JSON_CONTAINS(${TABLE_NAME}.name, '"John"') limit ?) as \`${TABLE_NAME}\``
sql: `select * from (select * from \`${TABLE_NAME}\` where JSON_CONTAINS(${TABLE_NAME}.age, '[20]') and JSON_CONTAINS(${TABLE_NAME}.name, '["John"]') limit ?) as \`${TABLE_NAME}\``
})
})
@ -274,8 +274,8 @@ describe("SQL query builder", () => {
const query = new Sql(SqlClient.POSTGRES, 10)._query(generateReadJson({
filters: {
contains: {
age: 20,
name: "John"
age: [20],
name: ["John"]
}
}
}))
@ -284,4 +284,94 @@ describe("SQL query builder", () => {
sql: `select * from (select * from \"${TABLE_NAME}\" where \"${TABLE_NAME}\".\"age\"::jsonb @> '[20]' and \"${TABLE_NAME}\".\"name\"::jsonb @> '["John"]' limit $1) as \"${TABLE_NAME}\"`
})
})
it("should use NOT like expression for MS-SQL when filter is notContains", () => {
const query = new Sql(SqlClient.MS_SQL, 10)._query(generateReadJson({
filters: {
notContains: {
age: [20],
name: ["John"]
}
}
}))
expect(query).toEqual({
bindings: [10, "%20%", `%"John"%`],
sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where NOT (LOWER(${TABLE_NAME}.age) LIKE @p1) and NOT (LOWER(${TABLE_NAME}.name) LIKE @p2)) as [${TABLE_NAME}]`
})
})
it("should use NOT JSON_CONTAINS expression for MySQL when filter is notContains", () => {
const query = new Sql(SqlClient.MY_SQL, 10)._query(generateReadJson({
filters: {
notContains: {
age: [20],
name: ["John"]
}
}
}))
expect(query).toEqual({
bindings: [10],
sql: `select * from (select * from \`${TABLE_NAME}\` where NOT JSON_CONTAINS(${TABLE_NAME}.age, '[20]') and NOT JSON_CONTAINS(${TABLE_NAME}.name, '["John"]') limit ?) as \`${TABLE_NAME}\``
})
})
it("should use jsonb operator NOT expression for PostgreSQL when filter is notContains", () => {
const query = new Sql(SqlClient.POSTGRES, 10)._query(generateReadJson({
filters: {
notContains: {
age: [20],
name: ["John"]
}
}
}))
expect(query).toEqual({
bindings: [10],
sql: `select * from (select * from \"${TABLE_NAME}\" where NOT \"${TABLE_NAME}\".\"age\"::jsonb @> '[20]' and NOT \"${TABLE_NAME}\".\"name\"::jsonb @> '["John"]' limit $1) as \"${TABLE_NAME}\"`
})
})
it("should use OR like expression for MS-SQL when filter is containsAny", () => {
const query = new Sql(SqlClient.MS_SQL, 10)._query(generateReadJson({
filters: {
containsAny: {
age: [20, 25],
name: ["John", "Mary"]
}
}
}))
expect(query).toEqual({
bindings: [10, "%20%", "%25%", `%"John"%`, `%"Mary"%`],
sql: `select * from (select top (@p0) * from [${TABLE_NAME}] where (LOWER(${TABLE_NAME}.age) LIKE @p1 OR LOWER(${TABLE_NAME}.age) LIKE @p2) and (LOWER(${TABLE_NAME}.name) LIKE @p3 OR LOWER(${TABLE_NAME}.name) LIKE @p4)) as [${TABLE_NAME}]`
})
})
it("should use JSON_OVERLAPS expression for MySQL when filter is containsAny", () => {
const query = new Sql(SqlClient.MY_SQL, 10)._query(generateReadJson({
filters: {
containsAny: {
age: [20, 25],
name: ["John", "Mary"]
}
}
}))
expect(query).toEqual({
bindings: [10],
sql: `select * from (select * from \`${TABLE_NAME}\` where JSON_OVERLAPS(${TABLE_NAME}.age, '[20,25]') and JSON_OVERLAPS(${TABLE_NAME}.name, '["John","Mary"]') limit ?) as \`${TABLE_NAME}\``
})
})
it("should use ?| operator expression for PostgreSQL when filter is containsAny", () => {
const query = new Sql(SqlClient.POSTGRES, 10)._query(generateReadJson({
filters: {
containsAny: {
age: [20, 25],
name: ["John", "Mary"]
}
}
}))
expect(query).toEqual({
bindings: [10],
sql: `select * from (select * from \"${TABLE_NAME}\" where \"${TABLE_NAME}\".\"age\"::jsonb ?| array [20,25] and \"${TABLE_NAME}\".\"name\"::jsonb ?| array ['John','Mary'] limit $1) as \"${TABLE_NAME}\"`
})
})
})

View File

@ -2,7 +2,7 @@ import { default as threadUtils } from "./utils"
threadUtils.threadSetup()
import { WorkerCallback, QueryEvent, QueryVariable } from "./definitions"
const ScriptRunner = require("../utilities/scriptRunner")
const { integrations } = require("../integrations")
const { getIntegration } = require("../integrations")
const { processStringSync } = require("@budibase/string-templates")
const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
const {
@ -62,7 +62,7 @@ class QueryRunner {
let datasourceClone = cloneDeep(datasource)
let fieldsClone = cloneDeep(fields)
const Integration = integrations[datasourceClone.source]
const Integration = await getIntegration(datasourceClone.source)
if (!Integration) {
throw "Integration type does not exist."
}

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "1.2.39-alpha.1",
"version": "1.2.39-alpha.6",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/types",
"version": "1.2.39-alpha.1",
"version": "1.2.39-alpha.6",
"description": "Budibase types",
"main": "dist/index.js",
"types": "dist/index.d.ts",

View File

@ -31,7 +31,13 @@ export interface SearchFilters {
[key: string]: any[]
}
contains?: {
[key: string]: any
[key: string]: any[]
}
notContains?: {
[key: string]: any[]
}
containsAny?: {
[key: string]: any[]
}
}

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "1.2.39-alpha.1",
"version": "1.2.39-alpha.6",
"description": "Budibase background service",
"main": "src/index.ts",
"repository": {
@ -35,10 +35,10 @@
"author": "Budibase",
"license": "GPL-3.0",
"dependencies": {
"@budibase/backend-core": "1.2.39-alpha.1",
"@budibase/pro": "1.2.39-alpha.0",
"@budibase/string-templates": "1.2.39-alpha.1",
"@budibase/types": "1.2.39-alpha.1",
"@budibase/backend-core": "1.2.39-alpha.6",
"@budibase/pro": "1.2.39-alpha.6",
"@budibase/string-templates": "1.2.39-alpha.6",
"@budibase/types": "1.2.39-alpha.6",
"@koa/router": "8.0.8",
"@sentry/node": "6.17.7",
"@techpass/passport-openidconnect": "0.3.2",

View File

@ -291,12 +291,12 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@1.2.39-alpha.0":
version "1.2.39-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.2.39-alpha.0.tgz#52a1d340d28fa714822827aeb843e1f097ad8f60"
integrity sha512-yHvWAUH1j8+UJx8hYl11iAGOpxmkxSbzf7qVCJ8a84Ms46Clmwp+qxtnDzKxMit2KJW72dJM9+PvI+OUG0Bixg==
"@budibase/backend-core@1.2.39-alpha.6":
version "1.2.39-alpha.6"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.2.39-alpha.6.tgz#ebfddd4fa74fff043a60df7cba335b5fcdb84d51"
integrity sha512-1kz9K3MSe5E4d/dSm6jD4BMIX/SSW58d8U787N+1BfzvSkIEd32j7HFs5ij0w9sromvK9FEjpFBfZKqSDRp+5g==
dependencies:
"@budibase/types" "1.2.39-alpha.0"
"@budibase/types" "1.2.39-alpha.6"
"@techpass/passport-openidconnect" "0.3.2"
aws-sdk "2.1030.0"
bcrypt "5.0.1"
@ -325,21 +325,21 @@
uuid "8.3.2"
zlib "1.0.5"
"@budibase/pro@1.2.39-alpha.0":
version "1.2.39-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.2.39-alpha.0.tgz#272e3ffd5e2189c787df6b6fbe22b7084832ade7"
integrity sha512-B0XakaUHW2Y1RtkEcrPEBcp8CDFJ8/fC9agLI3WK0ARnTGx9ROd8Liokq3/pW3vmlOgsfijzNHL3LUYcPTMEXw==
"@budibase/pro@1.2.39-alpha.6":
version "1.2.39-alpha.6"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.2.39-alpha.6.tgz#94a19338c6f59755f34b4bf181ba15ac5eba5404"
integrity sha512-l5IcibGGpd9VCIzY5vZU6rYeJa+1AYNCsWWKMnm7ZeZQgiD62Hyus53pv9GvT4pwpUF60pxybAoTWhbbityCPQ==
dependencies:
"@budibase/backend-core" "1.2.39-alpha.0"
"@budibase/types" "1.2.39-alpha.0"
"@budibase/backend-core" "1.2.39-alpha.6"
"@budibase/types" "1.2.39-alpha.6"
"@koa/router" "8.0.8"
joi "17.6.0"
node-fetch "^2.6.1"
"@budibase/types@1.2.39-alpha.0":
version "1.2.39-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.2.39-alpha.0.tgz#d0d48e4da36a93a4a9354cc49663f38d0181b6c0"
integrity sha512-i+lQeqlKmFORLuKdwmKnlmCCCEXZH6NdXHrOqNcebJzcNlx3Gx342+cHe4ZGYu0L+wCkWWelD5XHFuNMVMcuHg==
"@budibase/types@1.2.39-alpha.6":
version "1.2.39-alpha.6"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.2.39-alpha.6.tgz#5c6dc1a130913fb1c7ac6999fc6e03d970d0142f"
integrity sha512-0FHmwsfvAL80PGQM0OSJ6n9CDCtQQrbX8PRdcWOSIhRNlJXNcTsPcBsPbQpZPbgT57GHX/86Fo1rljiMWDwNpw==
"@cspotcode/source-map-consumer@0.8.0":
version "0.8.0"