Merge branch 'master' into fix/collect-data

This commit is contained in:
José Vte. Calderón 2024-01-10 15:51:21 +01:00 committed by GitHub
commit 58994a6435
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 1368 additions and 418 deletions

View File

@ -1,5 +1,5 @@
{
"version": "2.14.4",
"version": "2.14.5",
"npmClient": "yarn",
"packages": [
"packages/*",

@ -1 +1 @@
Subproject commit bcd86d9034ba954f013da4c10171bf495ab88189
Subproject commit b23fb3b17961fb04badd9487913a683fcf26dbe6

View File

@ -85,7 +85,6 @@ const INITIAL_FRONTEND_STATE = {
selectedScreenId: null,
selectedComponentId: null,
selectedLayoutId: null,
hoverComponentId: null,
// Client state
selectedComponentInstance: null,
@ -93,6 +92,9 @@ const INITIAL_FRONTEND_STATE = {
// Onboarding
onboarding: false,
tourNodes: null,
// UI state
hoveredComponentId: null,
}
export const getFrontendStore = () => {
@ -1413,6 +1415,18 @@ export const getFrontendStore = () => {
return state
})
},
hover: (componentId, notifyClient = true) => {
if (componentId === get(store).hoveredComponentId) {
return
}
store.update(state => {
state.hoveredComponentId = componentId
return state
})
if (notifyClient) {
store.actions.preview.sendEvent("hover-component", componentId)
}
},
},
links: {
save: async (url, title) => {

View File

@ -152,7 +152,7 @@
{#if isDisabled && !syncAutomationsEnabled && action.stepId === ActionStepID.COLLECT}
<div class="tag-color">
<Tags>
<Tag icon="LockClosed">Business</Tag>
<Tag icon="LockClosed">Premium</Tag>
</Tags>
</div>
{:else if isDisabled}

View File

@ -41,7 +41,7 @@
{ label: "False", value: "false" },
]}
/>
{:else if schema.type === "array"}
{:else if schemaHasOptions(schema) && schema.type === "array"}
<Multiselect
bind:value={value[field]}
options={schema.constraints.inclusion}
@ -77,7 +77,7 @@
on:change={e => onChange(e, field)}
useLabel={false}
/>
{:else if ["string", "number", "bigint", "barcodeqr"].includes(schema.type)}
{:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)}
<svelte:component
this={isTestModal ? ModalBindableInput : DrawerBindableInput}
panel={AutomationBindingPanel}

View File

@ -1,6 +1,9 @@
<script>
import { currentAsset } from "builderStore"
import { findClosestMatchingComponent } from "builderStore/componentUtils"
import { currentAsset, store } from "builderStore"
import {
findClosestMatchingComponent,
findComponent,
} from "builderStore/componentUtils"
import {
getDatasourceForProvider,
getSchemaForDatasource,
@ -20,8 +23,23 @@
component => component._component.endsWith("/form")
)
const resolveDatasource = (currentAsset, componentInstance, form) => {
if (!form && componentInstance._id != $store.selectedComponentId) {
const block = findComponent(
currentAsset.props,
$store.selectedComponentId
)
const def = store.actions.components.getDefinition(block._component)
return def?.block === true
? getDatasourceForProvider(currentAsset, block)
: {}
} else {
return getDatasourceForProvider(currentAsset, form)
}
}
// Get that form's schema
$: datasource = getDatasourceForProvider($currentAsset, form)
$: datasource = resolveDatasource($currentAsset, componentInstance, form)
$: formSchema = getSchemaForDatasource($currentAsset, datasource)?.schema
// Get the schema for the relationship field that this picker is using

View File

@ -108,6 +108,8 @@
{componentInstance}
{componentDefinition}
{bindings}
iconTooltip={componentName}
componentTitle={title}
/>
{/if}
{#if section == "conditions"}

View File

@ -5,6 +5,9 @@
Drawer,
Button,
notifications,
AbsTooltip,
Icon,
Body,
} from "@budibase/bbui"
import { selectedScreen, store } from "builderStore"
import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
@ -15,6 +18,9 @@
} from "builderStore/dataBinding"
export let componentInstance
export let componentDefinition
export let iconTooltip
export let componentTitle
let tempValue
let drawer
@ -24,6 +30,8 @@
$store.selectedComponentId
)
$: icon = componentDefinition?.icon
const openDrawer = () => {
tempValue = runtimeToReadableBinding(
bindings,
@ -54,7 +62,19 @@
{#key componentInstance?._id}
<Drawer bind:this={drawer} title="Custom CSS">
<svelte:fragment slot="description">
Custom CSS overrides all other component styles.
<div class="header">
Your CSS will overwrite styles for:
{#if icon}
<AbsTooltip type="info" text={iconTooltip}>
<Icon
color={`var(--spectrum-global-color-gray-600)`}
size="S"
name={icon}
/>
</AbsTooltip>
<Body size="S"><b>{componentTitle || ""}</b></Body>
{/if}
</div>
</svelte:fragment>
<Button cta slot="buttons" on:click={save}>Save</Button>
<svelte:component
@ -68,3 +88,13 @@
/>
</Drawer>
{/key}
<style>
.header {
display: flex;
flex-direction: row;
justify-content: space-between;
align-items: center;
gap: var(--spacing-m);
}
</style>

View File

@ -36,14 +36,12 @@
// Determine selected component ID
$: selectedComponentId = $store.selectedComponentId
$: hoverComponentId = $store.hoverComponentId
$: previewData = {
appId: $store.appId,
layout,
screen,
selectedComponentId,
hoverComponentId,
theme: $store.theme,
customTheme: $store.customTheme,
previewDevice: $store.previewDevice,
@ -119,8 +117,8 @@
error = event.error || "An unknown error occurred"
} else if (type === "select-component" && data.id) {
$store.selectedComponentId = data.id
} else if (type === "hover-component" && data.id) {
$store.hoverComponentId = data.id
} else if (type === "hover-component") {
store.actions.components.hover(data.id, false)
} else if (type === "update-prop") {
await store.actions.components.updateSetting(data.prop, data.value)
} else if (type === "update-styles") {

View File

@ -90,16 +90,7 @@
return findComponentPath($selectedComponent, component._id)?.length > 0
}
const handleMouseover = componentId => {
if ($store.hoverComponentId !== componentId) {
$store.hoverComponentId = componentId
}
}
const handleMouseout = componentId => {
if ($store.hoverComponentId === componentId) {
$store.hoverComponentId = null
}
}
const hover = store.actions.components.hover
</script>
<ul>
@ -120,9 +111,9 @@
on:dragover={dragover(component, index)}
on:iconClick={() => toggleNodeOpen(component._id)}
on:drop={onDrop}
hovering={$store.hoverComponentId === component._id}
on:mouseenter={() => handleMouseover(component._id)}
on:mouseleave={() => handleMouseout(component._id)}
hovering={$store.hoveredComponentId === component._id}
on:mouseenter={() => hover(component._id)}
on:mouseleave={() => hover(null)}
text={getComponentText(component)}
icon={getComponentIcon(component)}
iconTooltip={getComponentName(component)}

View File

@ -12,6 +12,9 @@
let scrolling = false
$: screenComponentId = `${$store.selectedScreenId}-screen`
$: navComponentId = `${$store.selectedScreenId}-navigation`
const toNewComponentRoute = () => {
if ($isActive(`./:componentId/new`)) {
$goto(`./:componentId`)
@ -33,16 +36,7 @@
scrolling = e.target.scrollTop !== 0
}
const handleMouseover = componentId => {
if ($store.hoverComponentId !== componentId) {
$store.hoverComponentId = componentId
}
}
const handleMouseout = componentId => {
if ($store.hoverComponentId === componentId) {
$store.hoverComponentId = null
}
}
const hover = store.actions.components.hover
</script>
<div class="components">
@ -65,46 +59,31 @@
scrollable
icon="WebPage"
on:drop={onDrop}
on:click={() => {
$store.selectedComponentId = `${$store.selectedScreenId}-screen`
}}
hovering={$store.hoverComponentId ===
`${$store.selectedScreenId}-screen`}
on:mouseenter={() =>
handleMouseover(`${$store.selectedScreenId}-screen`)}
on:mouseleave={() =>
handleMouseout(`${$store.selectedScreenId}-screen`)}
id={`component-screen`}
selectedBy={$userSelectedResourceMap[
`${$store.selectedScreenId}-screen`
]}
on:click={() => ($store.selectedComponentId = screenComponentId)}
hovering={$store.hoveredComponentId === screenComponentId}
on:mouseenter={() => hover(screenComponentId)}
on:mouseleave={() => hover(null)}
id="component-screen"
selectedBy={$userSelectedResourceMap[screenComponentId]}
>
<ScreenslotDropdownMenu component={$selectedScreen?.props} />
</NavItem>
<NavItem
text="Navigation"
indentLevel={0}
selected={$store.selectedComponentId ===
`${$store.selectedScreenId}-navigation`}
selected={$store.selectedComponentId === navComponentId}
opened
scrollable
icon={$selectedScreen.showNavigation
? "Visibility"
: "VisibilityOff"}
on:drop={onDrop}
on:click={() => {
$store.selectedComponentId = `${$store.selectedScreenId}-navigation`
}}
hovering={$store.hoverComponentId ===
`${$store.selectedScreenId}-navigation`}
on:mouseenter={() =>
handleMouseover(`${$store.selectedScreenId}-navigation`)}
on:mouseleave={() =>
handleMouseout(`${$store.selectedScreenId}-navigation`)}
id={`component-nav`}
selectedBy={$userSelectedResourceMap[
`${$store.selectedScreenId}-navigation`
]}
on:click={() => ($store.selectedComponentId = navComponentId)}
hovering={$store.hoveredComponentId === navComponentId}
on:mouseenter={() => hover(navComponentId)}
on:mouseleave={() => hover(null)}
id="component-nav"
selectedBy={$userSelectedResourceMap[navComponentId]}
/>
<ComponentTree
level={0}

View File

@ -214,7 +214,7 @@
<Heading size="M">Branding</Heading>
{#if !isCloud && !brandingEnabled}
<Tags>
<Tag icon="LockClosed">Business</Tag>
<Tag icon="LockClosed">Premium</Tag>
</Tags>
{/if}
{#if isCloud && !brandingEnabled}

View File

@ -97,7 +97,7 @@
<Heading size="M">Groups</Heading>
{#if !$licensing.groupsEnabled}
<Tags>
<Tag icon="LockClosed">Business</Tag>
<Tag icon="LockClosed">Enterpise</Tag>
</Tags>
{/if}
</div>

View File

@ -1,9 +1,9 @@
<script>
import { onMount, onDestroy } from "svelte"
import IndicatorSet from "./IndicatorSet.svelte"
import { builderStore, dndIsDragging } from "stores"
import { builderStore, dndIsDragging, hoverStore } from "stores"
$: componentId = $builderStore.hoverComponentId
$: componentId = $hoverStore.hoveredComponentId
$: zIndex = componentId === $builderStore.selectedComponentId ? 900 : 920
const onMouseOver = e => {
@ -23,12 +23,12 @@
}
if (newId !== componentId) {
builderStore.actions.hoverComponent(newId)
hoverStore.actions.hoverComponent(newId)
}
}
const onMouseLeave = () => {
builderStore.actions.hoverComponent(null)
hoverStore.actions.hoverComponent(null)
}
onMount(() => {

View File

@ -8,6 +8,7 @@ import {
environmentStore,
dndStore,
eventStore,
hoverStore,
} from "./stores"
import loadSpectrumIcons from "@budibase/bbui/spectrum-icons-rollup.js"
import { get } from "svelte/store"
@ -33,7 +34,6 @@ const loadBudibase = async () => {
layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],
hoverComponentId: window["##BUDIBASE_HOVER_COMPONENT_ID##"],
previewId: window["##BUDIBASE_PREVIEW_ID##"],
theme: window["##BUDIBASE_PREVIEW_THEME##"],
customTheme: window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"],
@ -84,6 +84,8 @@ const loadBudibase = async () => {
} else {
dndStore.actions.reset()
}
} else if (type === "hover-component") {
hoverStore.actions.hoverComponent(data)
} else if (type === "builder-meta") {
builderStore.actions.setMetadata(data)
}

View File

@ -8,7 +8,6 @@ const createBuilderStore = () => {
inBuilder: false,
screen: null,
selectedComponentId: null,
hoverComponentId: null,
editMode: false,
previewId: null,
theme: null,
@ -25,16 +24,6 @@ const createBuilderStore = () => {
}
const store = writable(initialState)
const actions = {
hoverComponent: id => {
if (id === get(store).hoverComponentId) {
return
}
store.update(state => ({
...state,
hoverComponentId: id,
}))
eventStore.actions.dispatchEvent("hover-component", { id })
},
selectComponent: id => {
if (id === get(store).selectedComponentId) {
return

View File

@ -0,0 +1,25 @@
import { get, writable } from "svelte/store"
import { eventStore } from "./events.js"
const createHoverStore = () => {
const store = writable({
hoveredComponentId: null,
})
const hoverComponent = id => {
if (id === get(store).hoveredComponentId) {
return
}
store.set({ hoveredComponentId: id })
eventStore.actions.dispatchEvent("hover-component", { id })
}
return {
...store,
actions: {
hoverComponent,
},
}
}
export const hoverStore = createHoverStore()

View File

@ -27,6 +27,7 @@ export {
dndIsDragging,
} from "./dnd"
export { sidePanelStore } from "./sidePanel"
export { hoverStore } from "./hover"
// Context stores are layered and duplicated, so it is not a singleton
export { createContextStore } from "./context"

View File

@ -80,7 +80,7 @@
"koa": "2.13.4",
"koa-body": "4.2.0",
"koa-compress": "4.0.1",
"koa-send": "5.0.0",
"koa-send": "5.0.1",
"koa-useragent": "^4.1.0",
"koa2-ratelimit": "1.1.1",
"lodash": "4.17.21",
@ -120,6 +120,7 @@
"@types/jest": "29.5.5",
"@types/koa": "2.13.4",
"@types/koa__router": "8.0.8",
"@types/koa-send": "^4.1.6",
"@types/lodash": "4.14.200",
"@types/mssql": "9.1.4",
"@types/node-fetch": "2.6.4",

View File

@ -1,13 +1,10 @@
import {
DocumentType,
generateDatasourceID,
getQueryParams,
getTableParams,
} from "../../db/utils"
import { getQueryParams, getTableParams } from "../../db/utils"
import { getIntegration } from "../../integrations"
import { invalidateDynamicVariables } from "../../threads/utils"
import { context, db as dbCore, events } from "@budibase/backend-core"
import {
BuildSchemaFromSourceRequest,
BuildSchemaFromSourceResponse,
CreateDatasourceRequest,
CreateDatasourceResponse,
Datasource,
@ -22,7 +19,6 @@ import {
} from "@budibase/types"
import sdk from "../../sdk"
import { builderSocket } from "../../websockets"
import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
import { isEqual } from "lodash"
export async function fetch(ctx: UserCtx) {
@ -67,22 +63,16 @@ export async function information(
}
}
export async function buildSchemaFromDb(ctx: UserCtx) {
const db = context.getAppDB()
export async function buildSchemaFromSource(
ctx: UserCtx<BuildSchemaFromSourceRequest, BuildSchemaFromSourceResponse>
) {
const datasourceId = ctx.params.datasourceId
const tablesFilter = ctx.request.body.tablesFilter
const datasource = await sdk.datasources.get(ctx.params.datasourceId)
const { tables, errors } = await sdk.datasources.buildFilteredSchema(
datasource,
const { datasource, errors } = await sdk.datasources.buildSchemaFromSource(
datasourceId,
tablesFilter
)
datasource.entities = tables
setDefaultDisplayColumns(datasource)
const dbResp = await db.put(
sdk.tables.populateExternalTableSchemas(datasource)
)
datasource._rev = dbResp.rev
ctx.body = {
datasource: await sdk.datasources.removeSecretSingle(datasource),
@ -90,24 +80,6 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
}
}
/**
* Make sure all datasource entities have a display name selected
*/
function setDefaultDisplayColumns(datasource: Datasource) {
//
for (let entity of Object.values(datasource.entities || {})) {
if (entity.primaryDisplay) {
continue
}
const notAutoColumn = Object.values(entity.schema).find(
schema => !schema.autocolumn
)
if (notAutoColumn) {
entity.primaryDisplay = notAutoColumn.name
}
}
}
/**
* Check for variables that have been updated or removed and invalidate them.
*/
@ -205,54 +177,18 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
}
}
const preSaveAction: Partial<Record<SourceName, any>> = {
[SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => {
await googleSetupCreationAuth(datasource.config as any)
},
}
export async function save(
ctx: UserCtx<CreateDatasourceRequest, CreateDatasourceResponse>
) {
const db = context.getAppDB()
const plus = ctx.request.body.datasource.plus
const fetchSchema = ctx.request.body.fetchSchema
const tablesFilter = ctx.request.body.tablesFilter
const datasource = {
_id: generateDatasourceID({ plus }),
...ctx.request.body.datasource,
type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
}
let errors: Record<string, string> = {}
if (fetchSchema) {
const schema = await sdk.datasources.buildFilteredSchema(
datasource,
tablesFilter
)
datasource.entities = schema.tables
setDefaultDisplayColumns(datasource)
errors = schema.errors
}
if (preSaveAction[datasource.source]) {
await preSaveAction[datasource.source](datasource)
}
const dbResp = await db.put(
sdk.tables.populateExternalTableSchemas(datasource)
)
await events.datasource.created(datasource)
datasource._rev = dbResp.rev
// Drain connection pools when configuration is changed
if (datasource.source) {
const source = await getIntegration(datasource.source)
if (source && source.pool) {
await source.pool.end()
}
}
const {
datasource: datasourceData,
fetchSchema,
tablesFilter,
} = ctx.request.body
const { datasource, errors } = await sdk.datasources.save(datasourceData, {
fetchSchema,
tablesFilter,
})
ctx.body = {
datasource: await sdk.datasources.removeSecretSingle(datasource),

View File

@ -63,7 +63,6 @@
// Extract data from message
const {
selectedComponentId,
hoverComponentId,
layout,
screen,
appId,
@ -82,7 +81,6 @@
window["##BUDIBASE_PREVIEW_LAYOUT##"] = layout
window["##BUDIBASE_PREVIEW_SCREEN##"] = screen
window["##BUDIBASE_SELECTED_COMPONENT_ID##"] = selectedComponentId
window["##BUDIBASE_HOVER_COMPONENT_ID##"] = hoverComponentId
window["##BUDIBASE_PREVIEW_ID##"] = Math.random()
window["##BUDIBASE_PREVIEW_THEME##"] = theme
window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"] = customTheme

View File

@ -53,7 +53,7 @@ router
.post(
"/api/datasources/:datasourceId/schema",
authorized(permissions.BUILDER),
datasourceController.buildSchemaFromDb
datasourceController.buildSchemaFromSource
)
.post(
"/api/datasources",

View File

@ -103,8 +103,7 @@ function typeCoercion(filters: SearchFilters, table: Table) {
return filters
}
for (let key of Object.keys(filters)) {
// @ts-ignore
const searchParam = filters[key]
const searchParam = filters[key as keyof SearchFilters]
if (typeof searchParam === "object") {
for (let [property, value] of Object.entries(searchParam)) {
// We need to strip numerical prefixes here, so that we can look up
@ -117,7 +116,13 @@ function typeCoercion(filters: SearchFilters, table: Table) {
continue
}
if (column.type === FieldTypes.NUMBER) {
searchParam[property] = parseFloat(value)
if (key === "oneOf") {
searchParam[property] = value
.split(",")
.map(item => parseFloat(item))
} else {
searchParam[property] = parseFloat(value)
}
}
}
}

View File

@ -149,8 +149,6 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
private index: number = 1
private open: boolean
COLUMNS_SQL!: string
PRIMARY_KEYS_SQL = () => `
SELECT pg_namespace.nspname table_schema
, pg_class.relname table_name
@ -171,6 +169,11 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace;
`
COLUMNS_SQL = () => `
select * from information_schema.columns where table_schema = ANY(current_schemas(false))
AND pg_table_is_visible(to_regclass(format('%I.%I', table_schema, table_name)));
`
constructor(config: PostgresConfig) {
super(SqlClient.POSTGRES)
this.config = config
@ -224,8 +227,6 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
.split(",")
.map(item => `"${item.trim()}"`)
await this.client.query(`SET search_path TO ${search_path.join(",")};`)
this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = ANY(current_schemas(false))
AND pg_table_is_visible(to_regclass(table_schema || '.' || table_name));`
this.open = true
}
@ -312,7 +313,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
try {
const columnsResponse: { rows: PostgresColumn[] } =
await this.client.query(this.COLUMNS_SQL)
await this.client.query(this.COLUMNS_SQL())
const tables: { [key: string]: Table } = {}
@ -382,7 +383,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
try {
await this.openConnection()
const columnsResponse: { rows: PostgresColumn[] } =
await this.client.query(this.COLUMNS_SQL)
await this.client.query(this.COLUMNS_SQL())
const names = columnsResponse.rows.map(row => row.table_name)
return [...new Set(names)]
} finally {

View File

@ -12,51 +12,49 @@ export function init() {
const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
let track: TrackerFn = f => f()
if (perRequestLimit) {
tracer.trace<any>("runJS.setupTracker", {}, span => {
const bbCtx = context.getCurrentContext()
if (bbCtx) {
if (!bbCtx.jsExecutionTracker) {
span?.addTags({
createdExecutionTracker: true,
})
bbCtx.jsExecutionTracker =
timers.ExecutionTimeTracker.withLimit(perRequestLimit)
}
const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span =>
context.getCurrentContext()
)
if (bbCtx) {
if (!bbCtx.jsExecutionTracker) {
span?.addTags({
js: {
limitMS: bbCtx.jsExecutionTracker.limitMs,
elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
},
createdExecutionTracker: true,
})
// We call checkLimit() here to prevent paying the cost of creating
// a new VM context below when we don't need to.
bbCtx.jsExecutionTracker.checkLimit()
track = bbCtx.jsExecutionTracker.track.bind(
bbCtx.jsExecutionTracker
bbCtx.jsExecutionTracker = tracer.trace(
"runJS.createExecutionTimeTracker",
{},
span => timers.ExecutionTimeTracker.withLimit(perRequestLimit)
)
}
})
span?.addTags({
js: {
limitMS: bbCtx.jsExecutionTracker.limitMs,
elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
},
})
// We call checkLimit() here to prevent paying the cost of creating
// a new VM context below when we don't need to.
tracer.trace("runJS.checkLimitAndBind", {}, span => {
bbCtx.jsExecutionTracker!.checkLimit()
track = bbCtx.jsExecutionTracker!.track.bind(
bbCtx.jsExecutionTracker
)
})
}
}
ctx = tracer.trace("runJS.ctxClone", {}, span => {
return {
...ctx,
alert: undefined,
setInterval: undefined,
setTimeout: undefined,
}
})
tracer.trace("runJS.vm.createContext", {}, span => {
vm.createContext(ctx)
})
ctx = {
...ctx,
alert: undefined,
setInterval: undefined,
setTimeout: undefined,
}
vm.createContext(ctx)
return track(() =>
tracer.trace("runJS.vm.runInNewContext", {}, span =>
vm.runInNewContext(js, ctx, {
timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
})
)
vm.runInNewContext(js, ctx, {
timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
})
)
})
})

View File

@ -1,4 +1,4 @@
import { context, db as dbCore } from "@budibase/backend-core"
import { context, db as dbCore, events } from "@budibase/backend-core"
import { findHBSBlocks, processObjectSync } from "@budibase/string-templates"
import {
Datasource,
@ -14,16 +14,22 @@ import {
} from "@budibase/types"
import { cloneDeep } from "lodash/fp"
import { getEnvironmentVariables } from "../../utils"
import { getDefinitions, getDefinition } from "../../../integrations"
import {
getDefinitions,
getDefinition,
getIntegration,
} from "../../../integrations"
import merge from "lodash/merge"
import {
BudibaseInternalDB,
generateDatasourceID,
getDatasourceParams,
getDatasourcePlusParams,
getTableParams,
DocumentType,
} from "../../../db/utils"
import sdk from "../../index"
import datasource from "../../../api/routes/datasource"
import { setupCreationAuth as googleSetupCreationAuth } from "../../../integrations/googlesheets"
const ENV_VAR_PREFIX = "env."
@ -273,3 +279,75 @@ export async function getExternalDatasources(): Promise<Datasource[]> {
return externalDatasources.rows.map(r => r.doc!)
}
export async function save(
datasource: Datasource,
opts?: { fetchSchema?: boolean; tablesFilter?: string[] }
): Promise<{ datasource: Datasource; errors: Record<string, string> }> {
const db = context.getAppDB()
const plus = datasource.plus
const fetchSchema = opts?.fetchSchema || false
const tablesFilter = opts?.tablesFilter || []
datasource = {
_id: generateDatasourceID({ plus }),
...datasource,
type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
}
let errors: Record<string, string> = {}
if (fetchSchema) {
const schema = await sdk.datasources.buildFilteredSchema(
datasource,
tablesFilter
)
datasource.entities = schema.tables
setDefaultDisplayColumns(datasource)
errors = schema.errors
}
if (preSaveAction[datasource.source]) {
await preSaveAction[datasource.source](datasource)
}
const dbResp = await db.put(
sdk.tables.populateExternalTableSchemas(datasource)
)
await events.datasource.created(datasource)
datasource._rev = dbResp.rev
// Drain connection pools when configuration is changed
if (datasource.source) {
const source = await getIntegration(datasource.source)
if (source && source.pool) {
await source.pool.end()
}
}
return { datasource, errors }
}
const preSaveAction: Partial<Record<SourceName, any>> = {
[SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => {
await googleSetupCreationAuth(datasource.config as any)
},
}
/**
* Make sure all datasource entities have a display name selected
*/
export function setDefaultDisplayColumns(datasource: Datasource) {
//
for (let entity of Object.values(datasource.entities || {})) {
if (entity.primaryDisplay) {
continue
}
const notAutoColumn = Object.values(entity.schema).find(
schema => !schema.autocolumn
)
if (notAutoColumn) {
entity.primaryDisplay = notAutoColumn.name
}
}
}

View File

@ -5,7 +5,9 @@ import {
Schema,
} from "@budibase/types"
import * as datasources from "./datasources"
import tableSdk from "../tables"
import { getIntegration } from "../../../integrations"
import { context } from "@budibase/backend-core"
export async function buildFilteredSchema(
datasource: Datasource,
@ -60,3 +62,24 @@ export async function getAndMergeDatasource(datasource: Datasource) {
}
return await datasources.enrich(datasource)
}
export async function buildSchemaFromSource(
datasourceId: string,
tablesFilter?: string[]
) {
const db = context.getAppDB()
const datasource = await datasources.get(datasourceId)
const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter)
datasource.entities = tables
datasources.setDefaultDisplayColumns(datasource)
const dbResp = await db.put(tableSdk.populateExternalTableSchemas(datasource))
datasource._rev = dbResp.rev
return {
datasource,
errors,
}
}

View File

@ -143,100 +143,104 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => {
oneOf: {},
containsAny: {},
}
if (Array.isArray(filter)) {
filter.forEach(expression => {
let { operator, field, type, value, externalType, onEmptyFilter } =
expression
const isHbs =
typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
// Parse all values into correct types
if (operator === "allOr") {
query.allOr = true
if (!Array.isArray(filter)) {
return query
}
filter.forEach(expression => {
let { operator, field, type, value, externalType, onEmptyFilter } =
expression
const isHbs =
typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
// Parse all values into correct types
if (operator === "allOr") {
query.allOr = true
return
}
if (onEmptyFilter) {
query.onEmptyFilter = onEmptyFilter
return
}
if (
type === "datetime" &&
!isHbs &&
operator !== "empty" &&
operator !== "notEmpty"
) {
// Ensure date value is a valid date and parse into correct format
if (!value) {
return
}
if (onEmptyFilter) {
query.onEmptyFilter = onEmptyFilter
try {
value = new Date(value).toISOString()
} catch (error) {
return
}
if (
type === "datetime" &&
!isHbs &&
operator !== "empty" &&
operator !== "notEmpty"
}
if (type === "number" && typeof value === "string" && !isHbs) {
if (operator === "oneOf") {
value = value.split(",").map(item => parseFloat(item))
} else {
value = parseFloat(value)
}
}
if (type === "boolean") {
value = `${value}`?.toLowerCase() === "true"
}
if (
["contains", "notContains", "containsAny"].includes(operator) &&
type === "array" &&
typeof value === "string"
) {
value = value.split(",")
}
if (operator.startsWith("range") && query.range) {
const minint =
SqlNumberTypeRangeMap[
externalType as keyof typeof SqlNumberTypeRangeMap
]?.min || Number.MIN_SAFE_INTEGER
const maxint =
SqlNumberTypeRangeMap[
externalType as keyof typeof SqlNumberTypeRangeMap
]?.max || Number.MAX_SAFE_INTEGER
if (!query.range[field]) {
query.range[field] = {
low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
}
}
if ((operator as any) === "rangeLow" && value != null && value !== "") {
query.range[field].low = value
} else if (
(operator as any) === "rangeHigh" &&
value != null &&
value !== ""
) {
// Ensure date value is a valid date and parse into correct format
if (!value) {
return
}
try {
value = new Date(value).toISOString()
} catch (error) {
return
}
}
if (type === "number" && typeof value === "string") {
if (operator === "oneOf") {
value = value.split(",").map(item => parseFloat(item))
} else if (!isHbs) {
value = parseFloat(value)
}
query.range[field].high = value
}
} else if (query[operator] && operator !== "onEmptyFilter") {
if (type === "boolean") {
value = `${value}`?.toLowerCase() === "true"
}
if (
["contains", "notContains", "containsAny"].includes(operator) &&
type === "array" &&
typeof value === "string"
) {
value = value.split(",")
}
if (operator.startsWith("range") && query.range) {
const minint =
SqlNumberTypeRangeMap[
externalType as keyof typeof SqlNumberTypeRangeMap
]?.min || Number.MIN_SAFE_INTEGER
const maxint =
SqlNumberTypeRangeMap[
externalType as keyof typeof SqlNumberTypeRangeMap
]?.max || Number.MAX_SAFE_INTEGER
if (!query.range[field]) {
query.range[field] = {
low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
}
}
if ((operator as any) === "rangeLow" && value != null && value !== "") {
query.range[field].low = value
} else if (
(operator as any) === "rangeHigh" &&
value != null &&
value !== ""
) {
query.range[field].high = value
}
} else if (query[operator] && operator !== "onEmptyFilter") {
if (type === "boolean") {
// Transform boolean filters to cope with null.
// "equals false" needs to be "not equals true"
// "not equals false" needs to be "equals true"
if (operator === "equal" && value === false) {
query.notEqual = query.notEqual || {}
query.notEqual[field] = true
} else if (operator === "notEqual" && value === false) {
query.equal = query.equal || {}
query.equal[field] = true
} else {
query[operator] = query[operator] || {}
query[operator]![field] = value
}
// Transform boolean filters to cope with null.
// "equals false" needs to be "not equals true"
// "not equals false" needs to be "equals true"
if (operator === "equal" && value === false) {
query.notEqual = query.notEqual || {}
query.notEqual[field] = true
} else if (operator === "notEqual" && value === false) {
query.equal = query.equal || {}
query.equal[field] = true
} else {
query[operator] = query[operator] || {}
query[operator]![field] = value
}
} else {
query[operator] = query[operator] || {}
query[operator]![field] = value
}
})
}
}
})
return query
}

View File

@ -1,6 +1,11 @@
import { SearchQuery, SearchQueryOperators } from "@budibase/types"
import { runLuceneQuery } from "../filters"
import { expect, describe, it } from "vitest"
import {
SearchQuery,
SearchQueryOperators,
FieldType,
SearchFilter,
} from "@budibase/types"
import { buildLuceneQuery, runLuceneQuery } from "../filters"
import { expect, describe, it, test } from "vitest"
describe("runLuceneQuery", () => {
const docs = [
@ -167,4 +172,186 @@ describe("runLuceneQuery", () => {
})
expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2, 3])
})
test.each([[523, 259], "523,259"])(
"should return rows with matches on numeric oneOf filter",
input => {
let query = buildQuery("oneOf", {
customer_id: input,
})
expect(runLuceneQuery(docs, query).map(row => row.customer_id)).toEqual([
259, 523,
])
}
)
})
describe("buildLuceneQuery", () => {
it("should return a basic search query template if the input is not an array", () => {
const filter: any = "NOT_AN_ARRAY"
expect(buildLuceneQuery(filter)).toEqual({
string: {},
fuzzy: {},
range: {},
equal: {},
notEqual: {},
empty: {},
notEmpty: {},
contains: {},
notContains: {},
oneOf: {},
containsAny: {},
})
})
it("should parseFloat if the type is a number, but the value is a numeric string", () => {
const filter: SearchFilter[] = [
{
operator: SearchQueryOperators.EQUAL,
field: "customer_id",
type: FieldType.NUMBER,
value: "1212",
},
{
operator: SearchQueryOperators.ONE_OF,
field: "customer_id",
type: FieldType.NUMBER,
value: "1000,1212,3400",
},
]
expect(buildLuceneQuery(filter)).toEqual({
string: {},
fuzzy: {},
range: {},
equal: {
customer_id: 1212,
},
notEqual: {},
empty: {},
notEmpty: {},
contains: {},
notContains: {},
oneOf: {
customer_id: [1000, 1212, 3400],
},
containsAny: {},
})
})
it("should not parseFloat if the type is a number, but the value is a handlebars binding string", () => {
const filter: SearchFilter[] = [
{
operator: SearchQueryOperators.EQUAL,
field: "customer_id",
type: FieldType.NUMBER,
value: "{{ customer_id }}",
},
{
operator: SearchQueryOperators.ONE_OF,
field: "customer_id",
type: FieldType.NUMBER,
value: "{{ list_of_customer_ids }}",
},
]
expect(buildLuceneQuery(filter)).toEqual({
string: {},
fuzzy: {},
range: {},
equal: {
customer_id: "{{ customer_id }}",
},
notEqual: {},
empty: {},
notEmpty: {},
contains: {},
notContains: {},
oneOf: {
customer_id: "{{ list_of_customer_ids }}",
},
containsAny: {},
})
})
it("should cast string to boolean if the type is boolean", () => {
const filter: SearchFilter[] = [
{
operator: SearchQueryOperators.EQUAL,
field: "a",
type: FieldType.BOOLEAN,
value: "not_true",
},
{
operator: SearchQueryOperators.NOT_EQUAL,
field: "b",
type: FieldType.BOOLEAN,
value: "not_true",
},
{
operator: SearchQueryOperators.EQUAL,
field: "c",
type: FieldType.BOOLEAN,
value: "true",
},
]
expect(buildLuceneQuery(filter)).toEqual({
string: {},
fuzzy: {},
range: {},
equal: {
b: true,
c: true,
},
notEqual: {
a: true,
},
empty: {},
notEmpty: {},
contains: {},
notContains: {},
oneOf: {},
containsAny: {},
})
})
it("should split the string for contains operators", () => {
const filter: SearchFilter[] = [
{
operator: SearchQueryOperators.CONTAINS,
field: "description",
type: FieldType.ARRAY,
value: "Large box,Heavy box,Small box",
},
{
operator: SearchQueryOperators.NOT_CONTAINS,
field: "description",
type: FieldType.ARRAY,
value: "Large box,Heavy box,Small box",
},
{
operator: SearchQueryOperators.CONTAINS_ANY,
field: "description",
type: FieldType.ARRAY,
value: "Large box,Heavy box,Small box",
},
]
expect(buildLuceneQuery(filter)).toEqual({
string: {},
fuzzy: {},
range: {},
equal: {},
notEqual: {},
empty: {},
notEmpty: {},
contains: {
description: ["Large box", "Heavy box", "Small box"],
},
notContains: {
description: ["Large box", "Heavy box", "Small box"],
},
oneOf: {},
containsAny: {
description: ["Large box", "Heavy box", "Small box"],
},
})
})
})

View File

@ -35,3 +35,12 @@ export interface FetchDatasourceInfoResponse {
export interface UpdateDatasourceRequest extends Datasource {
datasource: Datasource
}
export interface BuildSchemaFromSourceRequest {
tablesFilter?: string[]
}
export interface BuildSchemaFromSourceResponse {
datasource: Datasource
errors: Record<string, string>
}

View File

@ -39,10 +39,10 @@ describe("license management", () => {
let premiumPriceId = null
let businessPriceId = ""
for (const plan of planBody) {
if (plan.type === PlanType.PREMIUM) {
if (plan.type === PlanType.PREMIUM_PLUS) {
premiumPriceId = plan.prices[0].priceId
}
if (plan.type === PlanType.BUSINESS) {
if (plan.type === PlanType.ENTERPRISE_BASIC) {
businessPriceId = plan.prices[0].priceId
}
}
@ -97,7 +97,7 @@ describe("license management", () => {
await config.loginAsAccount(createAccountRequest)
await config.api.stripe.linkStripeCustomer(account.accountId, customer.id)
const [_, selfBodyPremium] = await config.api.accounts.self()
expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM)
expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM_PLUS)
// Create portal session - Check URL
const [portalRes, portalSessionBody] =
@ -109,7 +109,7 @@ describe("license management", () => {
// License updated to Business
const [selfRes, selfBodyBusiness] = await config.api.accounts.self()
expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS)
expect(selfBodyBusiness.license.plan.type).toBe(PlanType.ENTERPRISE_BASIC)
})
})
})

883
yarn.lock

File diff suppressed because it is too large Load Diff