Merge remote-tracking branch 'origin/develop' into feat/relationship-configuration

This commit is contained in:
Peter Clement 2023-09-27 15:25:39 +01:00
commit b32f1b5510
57 changed files with 1606 additions and 484 deletions

View File

@ -30,8 +30,8 @@ jobs:
with:
root-reserve-mb: 35000
swap-size-mb: 1024
remove-android: 'true'
remove-dotnet: 'true'
remove-android: "true"
remove-dotnet: "true"
- name: Checkout repo and submodules
uses: actions/checkout@v3
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
@ -121,7 +121,7 @@ jobs:
name: codecov-umbrella
verbose: true
test-services:
test-worker:
runs-on: ubuntu-latest
steps:
- name: Checkout repo and submodules
@ -143,12 +143,48 @@ jobs:
node-version: 18.x
cache: "yarn"
- run: yarn --frozen-lockfile
- name: Test worker and server
- name: Test worker
run: |
if ${{ env.USE_NX_AFFECTED }}; then
yarn test --scope=@budibase/worker --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
yarn test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }}
else
yarn test --scope=@budibase/worker --scope=@budibase/server
yarn test --scope=@budibase/worker
fi
- uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
name: codecov-umbrella
verbose: true
test-server:
runs-on: ubuntu-latest
steps:
- name: Checkout repo and submodules
uses: actions/checkout@v3
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
with:
submodules: true
token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
fetch-depth: 0
- name: Checkout repo only
uses: actions/checkout@v3
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
with:
fetch-depth: 0
- name: Use Node.js 18.x
uses: actions/setup-node@v3
with:
node-version: 18.x
cache: "yarn"
- run: yarn --frozen-lockfile
- name: Test server
run: |
if ${{ env.USE_NX_AFFECTED }}; then
yarn test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
else
yarn test --scope=@budibase/server
fi
- uses: codecov/codecov-action@v3
@ -259,4 +295,4 @@ jobs:
process.exit(1);
} else {
console.log('All good, the submodule had been merged and setup correctly!')
}
}

View File

@ -1,5 +1,5 @@
{
"version": "2.10.12-alpha.17",
"version": "2.10.12-alpha.23",
"npmClient": "yarn",
"packages": [
"packages/*"

View File

@ -120,7 +120,7 @@ export async function getUsers(
): Promise<{ users: User[]; notFoundIds?: string[] }> {
const client = await redis.getUserClient()
// try cache
let usersFromCache = await client.bulkGet(userIds)
let usersFromCache = await client.bulkGet<User>(userIds)
const missingUsersFromCache = userIds.filter(uid => !usersFromCache[uid])
const users = Object.values(usersFromCache)
let notFoundIds

View File

@ -242,7 +242,7 @@ class RedisWrapper {
}
}
async bulkGet(keys: string[]) {
async bulkGet<T>(keys: string[]) {
const db = this._db
if (keys.length === 0) {
return {}
@ -250,7 +250,7 @@ class RedisWrapper {
const prefixedKeys = keys.map(key => addDbPrefix(db, key))
let response = await this.getClient().mget(prefixedKeys)
if (Array.isArray(response)) {
let final: Record<string, any> = {}
let final: Record<string, T> = {}
let count = 0
for (let result of response) {
if (result) {

View File

@ -18,7 +18,7 @@ class DBTestConfiguration {
// TENANCY
doInTenant(task: any) {
doInTenant<T>(task: () => Promise<T>) {
return context.doInTenant(this.tenantId, () => {
return task()
})

View File

@ -1 +1,2 @@
export * from "./core/utilities"
export * from "./extra"

View File

@ -96,8 +96,8 @@
{disabled}
{readonly}
{id}
value={value || ""}
placeholder={placeholder || ""}
value={value ?? ""}
placeholder={placeholder ?? ""}
on:click
on:blur
on:focus

View File

@ -25,6 +25,7 @@
longform: StringRenderer,
array: ArrayRenderer,
internal: InternalRenderer,
bb_reference: RelationshipRenderer,
}
$: type = getType(schema)
$: customRenderer = customRenderers?.find(x => x.column === schema?.name)

View File

@ -1,33 +0,0 @@
<script>
import Table from "./Table.svelte"
export let query = {}
export let data = []
export let editRows = false
let loading = false
let error = false
let type = "external"
</script>
{#if error}
<div class="errors">{error}</div>
{/if}
<Table
schema={query.schema}
{data}
{loading}
{type}
rowCount={5}
allowEditing={editRows}
/>
<style>
.errors {
color: var(--red);
background: var(--red-light);
padding: var(--spacing-m);
border-radius: var(--border-radius-m);
margin-bottom: var(--spacing-m);
}
</style>

View File

@ -33,6 +33,7 @@
import { getBindings } from "components/backend/DataTable/formula"
import JSONSchemaModal from "./JSONSchemaModal.svelte"
import { ValidColumnNameRegex } from "@budibase/shared-core"
import { FieldSubtype, FieldType } from "@budibase/types"
import RelationshipSelector from "components/common/RelationshipSelector.svelte"
const AUTO_TYPE = "auto"
@ -42,6 +43,11 @@
const NUMBER_TYPE = FIELDS.NUMBER.type
const JSON_TYPE = FIELDS.JSON.type
const DATE_TYPE = FIELDS.DATETIME.type
const BB_REFERENCE_TYPE = FieldType.BB_REFERENCE
const BB_USER_REFERENCE_TYPE = composeType(
BB_REFERENCE_TYPE,
FieldSubtype.USER
)
const dispatch = createEventDispatcher()
const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@ -70,7 +76,7 @@
let jsonSchemaModal
let allowedTypes = []
let editableColumn = {
type: "string",
type: fieldDefinitions.STRING.type,
constraints: fieldDefinitions.STRING.constraints,
// Initial value for column name in other table for linked records
fieldName: $tables.selected.name,
@ -78,6 +84,33 @@
let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
const bbRefTypeMapping = {}
function composeType(fieldType, subtype) {
return `${fieldType}_${subtype}`
}
// Handling fields with subtypes
fieldDefinitions = Object.entries(fieldDefinitions).reduce(
(p, [key, field]) => {
if (field.type === BB_REFERENCE_TYPE) {
const composedType = composeType(field.type, field.subtype)
p[key] = {
...field,
type: composedType,
}
bbRefTypeMapping[composedType] = {
type: field.type,
subtype: field.subtype,
}
} else {
p[key] = field
}
return p
},
{}
)
$: if (primaryDisplay) {
editableColumn.constraints.presence = { allowEmpty: false }
}
@ -128,6 +161,7 @@
}
const initialiseField = (field, savingColumn) => {
isCreating = !field
if (field && !savingColumn) {
editableColumn = cloneDeep(field)
originalName = editableColumn.name ? editableColumn.name + "" : null
@ -148,6 +182,13 @@
relationshipPart1 = part1
relationshipPart2 = part2
}
const mapped = Object.entries(bbRefTypeMapping).find(
([_, v]) => v.type === field.type && v.subtype === field.subtype
)
if (mapped) {
editableColumn.type = mapped[0]
delete editableColumn.subtype
}
} else if (!savingColumn) {
let highestNumber = 0
@ -165,11 +206,14 @@
editableColumn.name = "Column 01"
}
}
allowedTypes = getAllowedTypes()
}
$: initialiseField(field, savingColumn)
$: isBBReference = !!bbRefTypeMapping[editableColumn.type]
$: checkConstraints(editableColumn)
$: required = !!editableColumn?.constraints?.presence || primaryDisplay
$: uneditable =
@ -242,6 +286,13 @@
let saveColumn = cloneDeep(editableColumn)
if (bbRefTypeMapping[saveColumn.type]) {
saveColumn = {
...saveColumn,
...bbRefTypeMapping[saveColumn.type],
}
}
if (saveColumn.type === AUTO_TYPE) {
saveColumn = buildAutoColumn(
$tables.selected.name,
@ -320,9 +371,10 @@
// Default relationships many to many
if (editableColumn.type === LINK_TYPE) {
editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
}
if (editableColumn.type === FORMULA_TYPE) {
} else if (editableColumn.type === FORMULA_TYPE) {
editableColumn.formulaType = "dynamic"
} else if (editableColumn.type === BB_USER_REFERENCE_TYPE) {
editableColumn.relationshipType = RelationshipType.ONE_TO_MANY
}
}
@ -361,7 +413,9 @@
ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1
) {
return ALLOWABLE_NUMBER_OPTIONS
} else if (!external) {
}
if (!external) {
return [
...Object.values(fieldDefinitions),
{ name: "Auto Column", type: AUTO_TYPE },
@ -382,6 +436,9 @@
if (!external || table.sql) {
fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
}
if (fieldDefinitions.USER) {
fields.push(fieldDefinitions.USER)
}
return fields
}
}
@ -637,6 +694,17 @@
<Button primary text on:click={openJsonSchemaEditor}
>Open schema editor</Button
>
{:else if isBBReference}
<Toggle
value={editableColumn.relationshipType === RelationshipType.MANY_TO_MANY}
on:change={e =>
(editableColumn.relationshipType = e.detail
? RelationshipType.MANY_TO_MANY
: RelationshipType.ONE_TO_MANY)}
disabled={!isCreating}
thin
text="Allow multiple users"
/>
{/if}
{#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
<Select

View File

@ -12,11 +12,19 @@
export let borderLeft = false
export let borderRight = false
export let wide = false
export let extraWide = false
export let closeButtonIcon = "Close"
$: customHeaderContent = $$slots["panel-header-content"]
</script>
<div class="panel" class:wide class:borderLeft class:borderRight>
<div
class="panel"
class:wide
class:extraWide
class:borderLeft
class:borderRight
>
<div class="header" class:custom={customHeaderContent}>
{#if showBackButton}
<Icon name="ArrowLeft" hoverable on:click={onClickBackButton} />
@ -33,7 +41,7 @@
</div>
{/if}
{#if showCloseButton}
<Icon name="Close" hoverable on:click={onClickCloseButton} />
<Icon name={closeButtonIcon} hoverable on:click={onClickCloseButton} />
{/if}
</div>
@ -70,6 +78,10 @@
width: 310px;
flex: 0 0 310px;
}
.panel.extraWide {
width: 450px;
flex: 0 0 450px;
}
.header {
flex: 0 0 48px;
display: flex;

View File

@ -65,6 +65,7 @@ const componentMap = {
"field/array": FormFieldSelect,
"field/json": FormFieldSelect,
"field/barcodeqr": FormFieldSelect,
"field/bb_reference": FormFieldSelect,
// Some validation types are the same as others, so not all types are
// explicitly listed here. e.g. options uses string validation
"validation/string": ValidationEditor,
@ -74,6 +75,7 @@ const componentMap = {
"validation/datetime": ValidationEditor,
"validation/attachment": ValidationEditor,
"validation/link": ValidationEditor,
"validation/bb_reference": ValidationEditor,
}
export const getComponentForSetting = setting => {

View File

@ -43,4 +43,5 @@ export const FieldTypeToComponentMap = {
link: "relationshipfield",
json: "jsonfield",
barcodeqr: "codescanner",
bb_reference: "bbreferencefield",
}

View File

@ -18,31 +18,20 @@
</script>
{#each extraFields as { key, displayName, type }}
<div class="config-field">
<Label>{displayName}</Label>
{#if type === "string"}
<Input
on:change={() => populateExtraQuery(extraQueryFields)}
bind:value={extraQueryFields[key]}
/>
{/if}
<Label>{displayName}</Label>
{#if type === "string"}
<Input
on:change={() => populateExtraQuery(extraQueryFields)}
bind:value={extraQueryFields[key]}
/>
{/if}
{#if type === "list"}
<Select
on:change={() => populateExtraQuery(extraQueryFields)}
bind:value={extraQueryFields[key]}
options={config[key].data[query.queryVerb]}
getOptionLabel={current => current}
/>
{/if}
</div>
{#if type === "list"}
<Select
on:change={() => populateExtraQuery(extraQueryFields)}
bind:value={extraQueryFields[key]}
options={config[key].data[query.queryVerb]}
getOptionLabel={current => current}
/>
{/if}
{/each}
<style>
.config-field {
display: grid;
grid-template-columns: 20% 1fr;
grid-gap: var(--spacing-l);
align-items: center;
}
</style>

View File

@ -34,6 +34,7 @@
export let bindings = []
export let bindingDrawerLeft
export let allowHelpers = true
export let customButtonText = null
let fields = Object.entries(object || {}).map(([name, value]) => ({
name,
@ -158,9 +159,13 @@
{/if}
{#if !readOnly && !noAddButton}
<div>
<ActionButton icon="Add" secondary thin outline on:click={addEntry}
>Add{name ? ` ${lowercase(name)}` : ""}</ActionButton
>
<ActionButton icon="Add" secondary thin outline on:click={addEntry}>
{#if customButtonText}
{customButtonText}
{:else}
{`Add${name ? ` ${lowercase(name)}` : ""}`}
{/if}
</ActionButton>
</div>
{/if}

View File

@ -1,364 +1,443 @@
<script>
import { goto, beforeUrlChange } from "@roxi/routify"
import { goto } from "@roxi/routify"
import { datasources, integrations, queries } from "stores/backend"
import {
Icon,
Select,
Button,
ButtonGroup,
Body,
Label,
Layout,
Input,
Heading,
Tabs,
Tab,
Modal,
ModalContent,
Label,
notifications,
Heading,
Body,
Divider,
Button,
} from "@budibase/bbui"
import ExtraQueryConfig from "./ExtraQueryConfig.svelte"
import IntegrationQueryEditor from "components/integration/index.svelte"
import ExternalDataSourceTable from "components/backend/DataTable/ExternalDataSourceTable.svelte"
import BindingBuilder from "components/integration/QueryViewerBindingBuilder.svelte"
import { datasources, integrations, queries } from "stores/backend"
import { capitalise } from "../../helpers"
import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
import JSONPreview from "./JSONPreview.svelte"
import { SchemaTypeOptions } from "constants/backend"
import KeyValueBuilder from "./KeyValueBuilder.svelte"
import { fieldsToSchema, schemaToFields } from "helpers/data/utils"
import { capitalise } from "helpers"
import AccessLevelSelect from "./AccessLevelSelect.svelte"
import IntegrationQueryEditor from "components/integration/index.svelte"
import QueryViewerSidePanel from "./QueryViewerSidePanel/index.svelte"
import { cloneDeep } from "lodash/fp"
import BindingBuilder from "components/integration/QueryViewerBindingBuilder.svelte"
import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
import { ValidQueryNameRegex } from "@budibase/shared-core"
import ExtraQueryConfig from "./ExtraQueryConfig.svelte"
import QueryViewerSavePromptModal from "./QueryViewerSavePromptModal.svelte"
import { Utils } from "@budibase/frontend-core"
export let query
let queryHash
const resumeNavigation = () => {
if (typeof navigateTo == "string") {
$goto(typeof navigateTo == "string" ? `${navigateTo}` : navigateTo)
}
let loading = false
let modified = false
let scrolling = false
let showSidePanel = false
let nameError
let newQuery
let datasource
let integration
let schemaType
let autoSchema = {}
let rows = []
const parseQuery = query => {
modified = false
datasource = $datasources.list.find(ds => ds._id === query.datasourceId)
integration = $integrations[datasource.source]
schemaType = integration.query[query.queryVerb].type
newQuery = cloneDeep(query)
// Set the location where the query code will be written to an empty string so that it doesn't
// get changed from undefined -> "" by the input, breaking our unsaved changes checks
newQuery.fields[schemaType] ??= ""
queryHash = JSON.stringify(newQuery)
}
const transformerDocs = "https://docs.budibase.com/docs/transformers"
$: parseQuery(query)
let fields = query?.schema ? schemaToFields(query.schema) : []
let parameters
let data = []
let saveId
let currentTab = "JSON"
let saveModal
let override = false
let navigateTo = null
let nameError = null
const checkIsModified = newQuery => {
const newQueryHash = JSON.stringify(newQuery)
modified = newQueryHash !== queryHash
// seed the transformer
if (query && !query.transformer) {
query.transformer = "return data"
return modified
}
// initialise a new empty schema
if (query && !query.schema) {
query.schema = {}
}
const debouncedCheckIsModified = Utils.debounce(checkIsModified, 1000)
let queryStr = JSON.stringify(query)
$: debouncedCheckIsModified(newQuery)
$beforeUrlChange(event => {
const updated = JSON.stringify(query)
if (updated !== queryStr && !override) {
navigateTo = event.type == "pushstate" ? event.url : null
saveModal.show()
return false
} else return true
})
$: datasource = $datasources.list.find(ds => ds._id === query.datasourceId)
$: query.schema = fieldsToSchema(fields)
$: datasourceType = datasource?.source
$: integrationInfo = datasourceType ? $integrations[datasourceType] : null
$: queryConfig = integrationInfo?.query
$: shouldShowQueryConfig = queryConfig && query.queryVerb
$: readQuery = query.queryVerb === "read" || query.readable
$: queryInvalid = !query.name || nameError || (readQuery && data.length === 0)
//Cast field in query preview response to number if specified by schema
$: {
for (let i = 0; i < data.length; i++) {
let row = data[i]
for (let fieldName of Object.keys(fields)) {
if (fields[fieldName] === "number" && !isNaN(Number(row[fieldName]))) {
row[fieldName] = Number(row[fieldName])
} else {
row[fieldName] = row[fieldName]?.toString()
}
}
}
}
function resetDependentFields() {
if (query.fields.extra) {
query.fields.extra = {}
}
}
function populateExtraQuery(extraQueryFields) {
query.fields.extra = extraQueryFields
}
async function previewQuery() {
async function runQuery({ suppressErrors = true }) {
try {
const response = await queries.preview(query)
showSidePanel = true
loading = true
const response = await queries.preview(newQuery)
if (response.rows.length === 0) {
notifications.info(
"Query results empty. Please execute a query with results to create your schema."
)
return
}
data = response.rows
// need to merge fields that already exist/might have changed
if (fields) {
for (let key of Object.keys(response.schema)) {
if (fields[key]) {
response.schema[key] = fields[key]
}
}
if (Object.keys(newQuery.schema).length === 0) {
// Assign this to a variable instead of directly to the newQuery.schema so that a user
// can change the table they're querying and have the schema update until they first
// edit it
autoSchema = response.schema
}
fields = response.schema
currentTab = "JSON"
rows = response.rows
notifications.success("Query executed successfully")
} catch (error) {
notifications.error(`Query Error: ${error.message}`)
if (!suppressErrors) {
throw error
}
} finally {
loading = false
}
}
// return the query.
async function saveQuery() {
try {
const response = await queries.save(query.datasourceId, query)
saveId = response._id
if (response?._rev) {
queryStr = JSON.stringify(query)
}
showSidePanel = true
loading = true
const response = await queries.save(newQuery.datasourceId, {
...newQuery,
schema:
Object.keys(newQuery.schema).length === 0
? autoSchema
: newQuery.schema,
})
notifications.success("Query saved successfully")
return response
} catch (error) {
notifications.error(error.message || "Error saving query")
} finally {
loading = false
}
}
function resetDependentFields() {
if (newQuery.fields.extra) {
newQuery.fields.extra = {}
}
}
function populateExtraQuery(extraQueryFields) {
newQuery.fields.extra = extraQueryFields
}
const handleScroll = e => {
scrolling = e.target.scrollTop !== 0
}
</script>
<Modal
bind:this={saveModal}
on:hide={() => {
navigateTo = null
}}
>
<ModalContent
title="You have unsaved changes"
confirmText="Save and Continue"
cancelText="Discard Changes"
size="L"
onConfirm={async () => {
await saveQuery()
override = true
resumeNavigation()
}}
onCancel={async () => {
override = true
resumeNavigation()
}}
>
<Body>Leaving this section will mean losing and changes to your query</Body>
</ModalContent>
</Modal>
<div class="wrapper">
<Layout gap="S" noPadding>
<Heading size="M">Query {integrationInfo?.friendlyName}</Heading>
<Divider />
<Heading size="S">Config</Heading>
<div class="config">
<div class="config-field">
<Label>Query Name</Label>
<Input
value={query.name}
on:input={e => {
let newValue = e.target.value || ""
if (newValue.match(ValidQueryNameRegex)) {
query.name = newValue.trim()
nameError = null
} else {
nameError = "Invalid query name"
}
}}
error={nameError}
/>
</div>
{#if queryConfig}
<div class="config-field">
<Label>Function</Label>
<Select
bind:value={query.queryVerb}
on:change={resetDependentFields}
options={Object.keys(queryConfig)}
getOptionLabel={verb =>
queryConfig[verb]?.displayName || capitalise(verb)}
/>
</div>
<div class="config-field">
<AccessLevelSelect {saveId} {query} label="Access Level" />
</div>
{#if integrationInfo?.extra && query.queryVerb}
<ExtraQueryConfig
{query}
{populateExtraQuery}
config={integrationInfo.extra}
/>
{/if}
{#key query.parameters}
<div class="binding-wrap">
<BindingBuilder
queryBindings={query.parameters}
bindable={false}
on:change={e => {
query.parameters = e.detail.map(binding => {
return {
name: binding.name,
default: binding.value,
}
})
}}
/>
</div>
{/key}
{/if}
</div>
{#if shouldShowQueryConfig}
<Divider />
<div class="config">
<Heading size="S">Fields</Heading>
<Body size="S">Fill in the fields specific to this query.</Body>
<IntegrationQueryEditor
{datasource}
{query}
height={200}
schema={queryConfig[query.queryVerb]}
bind:parameters
/>
<Divider />
</div>
<div class="config">
<div class="help-heading">
<Heading size="S">Transformer</Heading>
<Icon
on:click={() => window.open(transformerDocs)}
hoverable
name="Help"
size="L"
/>
</div>
<QueryViewerSavePromptModal
checkIsModified={() => checkIsModified(newQuery)}
attemptSave={() => runQuery({ suppressErrors: false }).then(saveQuery)}
/>
<div class="queryViewer">
<div class="main">
<div class="header" class:scrolling>
<div class="title">
<Body size="S">
Add a JavaScript function to transform the query result.
{newQuery.name || "Untitled query"}<span class="unsaved"
>{modified ? "*" : ""}</span
>
</Body>
<CodeMirrorEditor
height={200}
label="Transformer"
value={query.transformer}
resize="vertical"
on:change={e => (query.transformer = e.detail)}
/>
<Divider />
</div>
<div class="viewer-controls">
<Heading size="S">Results</Heading>
<ButtonGroup gap="S">
<div class="controls">
<Button disabled={loading} on:click={runQuery} overBackground>
<Icon size="S" name="Play" />
Run query</Button
>
<div class="tooltip" title="Run your query to enable saving">
<Button
cta
disabled={queryInvalid}
on:click={async () => {
await saveQuery()
// Go to the correct URL if we just created a new query
if (!query._rev) {
$goto(`../../${query._id}`)
const response = await saveQuery()
// When creating a new query the initally passed in query object will have no id.
if (response._id && !newQuery._id) {
// Set the comparison query hash to match the new query so that the user doesn't
// get nagged when navigating to the edit view
queryHash = JSON.stringify(newQuery)
$goto(`../../${response._id}`)
}
}}
disabled={loading ||
!newQuery.name ||
nameError ||
rows.length === 0}
overBackground
>
Save Query
<Icon size="S" name="SaveFloppy" />
Save
</Button>
<Button secondary on:click={previewQuery}>Run Query</Button>
</ButtonGroup>
</div>
</div>
<Body size="S">
Below, you can preview the results from your query and change the
schema.
</Body>
<section class="viewer">
{#if data}
<Tabs bind:selected={currentTab}>
<Tab title="JSON">
<JSONPreview data={data[0]} minHeight="120" />
</Tab>
<Tab title="Schema">
<KeyValueBuilder
bind:object={fields}
name="field"
headings
options={SchemaTypeOptions}
</div>
<div class="body" on:scroll={handleScroll}>
<div class="bodyInner">
<div class="configField">
<Label>Name</Label>
<Input
value={newQuery.name}
on:input={e => {
let newValue = e.target.value || ""
if (newValue.match(ValidQueryNameRegex)) {
newQuery.name = newValue.trim()
nameError = null
} else {
nameError = "Invalid query name"
}
}}
error={nameError}
/>
{#if integration.query}
<Label>Function</Label>
<Select
bind:value={newQuery.queryVerb}
on:change={resetDependentFields}
options={Object.keys(integration.query)}
getOptionLabel={verb =>
integration.query[verb]?.displayName || capitalise(verb)}
/>
<Label>Access</Label>
<AccessLevelSelect query={newQuery} />
{#if integration?.extra && newQuery.queryVerb}
<ExtraQueryConfig
query={newQuery}
{populateExtraQuery}
config={integration.extra}
/>
</Tab>
<Tab title="Preview">
<ExternalDataSourceTable {query} {data} />
</Tab>
</Tabs>
{/if}
</section>
{/if}
</Layout>
{/if}
{/if}
</div>
<Divider />
<div class="heading">
<Heading weight="L" size="XS">Query</Heading>
</div>
<div class="copy">
<Body size="S">
{#if schemaType === "sql"}
Add some SQL to query your data
{:else if schemaType === "json"}
Add some JSON to query your data
{:else if schemaType === "fields"}
Add some fields to query your data
{:else}
Enter your query below
{/if}
</Body>
</div>
<IntegrationQueryEditor
noLabel
{datasource}
bind:query={newQuery}
height={200}
schema={integration.query[newQuery.queryVerb]}
/>
<Divider />
<div class="heading">
<Heading weight="L" size="XS">Bindings</Heading>
</div>
<div class="copy">
<Body size="S">
Bindings come in two parts: the binding name, and a default/fallback
value. These bindings can be used as Handlebars expressions
throughout the query.
</Body>
</div>
{#key newQuery.parameters}
<BindingBuilder
hideHeading
queryBindings={newQuery.parameters}
on:change={e => {
newQuery.parameters = e.detail.map(binding => {
return {
name: binding.name,
default: binding.value,
}
})
}}
/>
{/key}
<Divider />
<div class="heading">
<Heading weight="L" size="XS">Transformer</Heading>
</div>
<div class="copy">
<Body size="S">
Add a JavaScript function to transform the query result.
</Body>
</div>
<CodeMirrorEditor
height={200}
value={newQuery.transformer}
resize="vertical"
on:change={e => (newQuery.transformer = e.detail)}
/>
</div>
</div>
</div>
<div class:showSidePanel class="sidePanel">
<QueryViewerSidePanel
onClose={() => (showSidePanel = false)}
onSchemaChange={newSchema => {
newQuery.schema = newSchema
}}
{rows}
schema={Object.keys(newQuery.schema).length === 0
? autoSchema
: newQuery.schema}
/>
</div>
</div>
<style>
.wrapper {
width: 640px;
.unsaved {
color: var(--grey-5);
font-style: italic;
}
.queryViewer {
height: 100%;
margin: -28px -40px -40px -40px;
display: flex;
flex: 1;
}
.queryViewer :global(.spectrum-Divider) {
margin: 35px 0;
}
.main {
flex-grow: 1;
height: 100%;
display: flex;
flex-direction: column;
}
.header {
align-items: center;
padding: 8px 10px 8px 16px;
display: flex;
border-bottom: 2px solid transparent;
transition: border-bottom 130ms ease-out;
}
.header.scrolling {
border-bottom: var(--border-light);
}
.body {
flex-grow: 1;
overflow-y: scroll;
padding: 23px 23px 80px;
box-sizing: border-box;
}
.bodyInner {
max-width: 520px;
margin: auto;
}
.config {
display: grid;
grid-gap: var(--spacing-s);
z-index: 1;
.title {
/* width 0 paired with flex-grow necessary here for the truncation to work properly*/
width: 0;
flex-grow: 1;
}
.config-field {
.title :global(p) {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.controls {
flex-shrink: 0;
}
.tooltip {
display: inline-block;
}
.controls :global(button) {
border: none;
color: var(--grey-7);
font-weight: 300;
}
.controls :global(button):hover {
background-color: transparent;
color: var(--ink);
}
.controls :global(.is-disabled) {
pointer-events: none;
background-color: transparent;
color: var(--grey-3);
}
.controls :global(span) {
display: flex;
align-items: center;
}
.controls :global(.icon) {
margin-right: 8px;
}
.configField {
display: grid;
grid-template-columns: 20% 1fr;
grid-gap: var(--spacing-l);
align-items: center;
}
.help-heading {
display: flex;
justify-content: space-between;
.configField :global(label) {
color: var(--grey-6);
}
.viewer {
min-height: 200px;
width: 640px;
.heading {
margin-bottom: 8px;
}
.viewer-controls {
display: flex;
flex-direction: row;
justify-content: space-between;
gap: var(--spacing-m);
min-width: 150px;
align-items: center;
.copy {
margin-bottom: 14px;
}
.binding-wrap :global(div.container) {
padding-left: 0px;
padding-right: 0px;
.copy :global(p) {
color: var(--grey-7);
}
.sidePanel {
flex-shrink: 0;
height: 100%;
width: 0;
overflow: hidden;
transition: width 150ms;
}
.sidePanel :global(.panel) {
height: 100%;
}
.showSidePanel {
width: 450px;
}
</style>

View File

@ -1,9 +1,9 @@
<script>
import { Body, Heading, Layout } from "@budibase/bbui"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import { getUserBindings } from "builderStore/dataBinding"
export let bindable = true
export let queryBindings = []
export let hideHeading = false
const userBindings = getUserBindings()
@ -13,44 +13,16 @@
}, {})
</script>
<Layout noPadding={bindable} gap="S">
<div class="controls" class:height={!bindable}>
<Heading size="XS">Bindings</Heading>
</div>
<Body size="S">
{#if !bindable}
Bindings come in two parts: the binding name, and a default/fallback
value. These bindings can be used as Handlebars expressions throughout the
query.
{:else}
Enter a value for each binding. The default values will be used for any
values left blank.
{/if}
</Body>
<div class="bindings" class:bindable>
<KeyValueBuilder
bind:object={internalBindings}
tooltip="Set the name of the binding which can be used in Handlebars statements throughout your query"
name="binding"
headings
keyPlaceholder="Binding name"
valuePlaceholder="Default"
bindings={[...userBindings]}
bindingDrawerLeft="260px"
allowHelpers={false}
on:change
/>
</div>
</Layout>
<style>
.controls {
display: flex;
align-items: center;
justify-content: space-between;
}
.height {
height: 40px;
}
</style>
<KeyValueBuilder
bind:object={internalBindings}
tooltip="Set the name of the binding which can be used in Handlebars statements throughout your query"
name="binding"
customButtonText="Bindings"
headings
keyPlaceholder="Binding name"
valuePlaceholder="Default"
bindings={[...userBindings]}
bindingDrawerLeft="260px"
allowHelpers={false}
on:change
/>

View File

@ -0,0 +1,53 @@
<script>
import { goto, beforeUrlChange } from "@roxi/routify"
import { Body, Modal, ModalContent } from "@budibase/bbui"
export let checkIsModified = () => {}
export let attemptSave = () => {}
let modal
let navigateTo
let override = false
$beforeUrlChange(event => {
if (checkIsModified() && !override) {
navigateTo = event.type == "pushstate" ? event.url : null
modal.show()
return false
} else return true
})
const resumeNavigation = () => {
if (typeof navigateTo == "string") {
$goto(typeof navigateTo == "string" ? `${navigateTo}` : navigateTo)
}
}
</script>
<Modal
bind:this={modal}
on:hide={() => {
navigateTo = null
}}
>
<ModalContent
title="You have unsaved changes"
confirmText="Save and Continue"
cancelText="Discard Changes"
size="L"
onConfirm={async () => {
try {
await attemptSave()
override = true
resumeNavigation()
} catch (e) {
navigateTo = false
}
}}
onCancel={async () => {
override = true
resumeNavigation()
}}
>
<Body>Leaving this section will mean losing any changes to your query</Body>
</ModalContent>
</Modal>

View File

@ -0,0 +1,22 @@
<script>
export let data
$: string = JSON.stringify(data || {}, null, 2)
</script>
<textarea class="json" disabled value={string} />
<style>
.json {
width: 100%;
height: 100%;
color: var(--ink);
padding: 12px;
box-sizing: border-box;
resize: none;
background-color: var(
--spectrum-textfield-m-background-color,
var(--spectrum-global-color-gray-50)
);
}
</style>

View File

@ -0,0 +1,33 @@
<script>
import Table from "components/backend/DataTable/Table.svelte"
import { cloneDeep } from "lodash/fp"
export let schema = {}
export let rows = []
$: rowsCopy = cloneDeep(rows)
// Cast field in query preview response to number if specified by schema
$: {
for (let i = 0; i < rowsCopy.length; i++) {
let row = rowsCopy[i]
for (let fieldName of Object.keys(schema)) {
if (schema[fieldName] === "number" && !isNaN(Number(row[fieldName]))) {
row[fieldName] = Number(row[fieldName])
} else {
row[fieldName] = row[fieldName]?.toString()
}
}
}
}
</script>
<div class="table">
<Table {schema} data={rowsCopy} type="external" allowEditing={false} />
</div>
<style>
.table :global(.spectrum-Table-cell) {
min-width: 100px;
}
</style>

View File

@ -0,0 +1,29 @@
<script>
import KeyValueBuilder from "../KeyValueBuilder.svelte"
import { SchemaTypeOptions } from "constants/backend"
export let schema
export let onSchemaChange = () => {}
const handleChange = e => {
let newSchema = {}
// KeyValueBuilder on change event returns an array of objects with each object
// containing a field
e.detail.forEach(({ name, value }) => {
newSchema[name] = value
})
onSchemaChange(newSchema)
}
</script>
{#key schema}
<KeyValueBuilder
on:change={handleChange}
object={schema}
name="field"
headings
options={SchemaTypeOptions}
/>
{/key}

View File

@ -0,0 +1,66 @@
<script>
import Panel from "components/design/Panel.svelte"
import { ActionButton } from "@budibase/bbui"
import JSONPanel from "./JSONPanel.svelte"
import SchemaPanel from "./SchemaPanel.svelte"
import PreviewPanel from "./PreviewPanel.svelte"
export let rows
export let schema
export let onSchemaChange = () => {}
export let onClose = () => {}
const tabs = ["JSON", "Schema", "Preview"]
let activeTab = "JSON"
</script>
<Panel
showCloseButton
closeButtonIcon="RailRightClose"
onClickCloseButton={onClose}
title="Query results"
icon={"SQLQuery"}
borderLeft
extraWide
>
<div slot="panel-header-content">
<div class="settings-tabs">
{#each tabs as tab}
<ActionButton
size="M"
quiet
selected={activeTab === tab}
on:click={() => {
activeTab = tab
}}
>
{tab}
</ActionButton>
{/each}
</div>
</div>
<div class="content">
{#if activeTab === "JSON"}
<JSONPanel data={rows[0] || {}} />
{:else if activeTab === "Schema"}
<SchemaPanel {onSchemaChange} {schema} />
{:else}
<PreviewPanel {schema} {rows} />
{/if}
</div>
</Panel>
<style>
.settings-tabs {
display: flex;
gap: var(--spacing-s);
padding: 0 var(--spacing-l);
padding-bottom: var(--spacing-l);
}
.content {
padding: 14px;
height: 100%;
overflow: scroll;
}
</style>

View File

@ -23,6 +23,7 @@
export let schema
export let editable = true
export let height = 500
export let noLabel = false
let stepEditors = []
@ -75,7 +76,7 @@
{#if schema.type === QueryTypes.SQL}
<Editor
editorHeight={height}
label="Query"
label={noLabel ? null : "Query"}
mode="sql"
on:change={updateQuery}
readOnly={!editable}
@ -85,7 +86,7 @@
{:else if shouldDisplayJsonBox}
<Editor
editorHeight={height}
label="Query"
label={noLabel ? null : "Query"}
mode="json"
on:change={updateQuery}
readOnly={!editable}

View File

@ -120,6 +120,12 @@ export const FIELDS = {
presence: false,
},
},
USER: {
name: "User",
type: "bb_reference",
subtype: "user",
icon: "User",
},
}
export const AUTO_COLUMN_SUB_TYPES = {

View File

@ -16,6 +16,9 @@
const buildNewQuery = isRestQuery => {
let query = {
name: "Untitled query",
transformer: "return data",
schema: {},
datasourceId: $params.datasourceId,
parameters: [],
fields: {},

View File

@ -69,7 +69,8 @@
"datetimefield",
"multifieldselect",
"s3upload",
"codescanner"
"codescanner",
"bbreferencefield"
]
},
{

View File

@ -39,7 +39,7 @@
<ActionButton
quiet
icon="Edit"
on:click={() => $goto(`../../app/${app.devId}`)}
on:click={() => $goto(`/builder/app/${app.devId}`)}
>
Edit
</ActionButton>

View File

@ -5623,5 +5623,73 @@
"defaultValue": false
}
]
},
"bbreferencefield": {
"devComment": "As bb reference is only used for user subtype for now, we are using user for icon and labels",
"name": "User Field",
"icon": "User",
"styles": ["size"],
"requiredAncestors": ["form"],
"editable": true,
"size": {
"width": 400,
"height": 50
},
"settings": [
{
"type": "field/bb_reference",
"label": "Field",
"key": "field",
"required": true
},
{
"type": "text",
"label": "Label",
"key": "label"
},
{
"type": "text",
"label": "Placeholder",
"key": "placeholder"
},
{
"type": "text",
"label": "Default value",
"key": "defaultValue"
},
{
"type": "event",
"label": "On change",
"key": "onChange",
"context": [
{
"label": "Field Value",
"key": "value"
}
]
},
{
"type": "validation/link",
"label": "Validation",
"key": "validation"
},
{
"type": "filter/relationship",
"label": "Filtering",
"key": "filter"
},
{
"type": "boolean",
"label": "Search",
"key": "autocomplete",
"defaultValue": true
},
{
"type": "boolean",
"label": "Disabled",
"key": "disabled",
"defaultValue": false
}
]
}
}

View File

@ -30,6 +30,7 @@
link: "relationshipfield",
json: "jsonfield",
barcodeqr: "codescanner",
bb_reference: "bbreferencefield",
}
let formId

View File

@ -0,0 +1,9 @@
<script>
import RelationshipField from "./RelationshipField.svelte"
</script>
<RelationshipField
{...$$props}
datasourceType={"user"}
primaryDisplay={"email"}
/>

View File

@ -16,6 +16,8 @@
export let defaultValue
export let onChange
export let filter
export let datasourceType = "table"
export let primaryDisplay
let fieldState
let fieldApi
@ -24,12 +26,15 @@
let searchTerm
let open
$: type =
datasourceType === "table" ? FieldTypes.LINK : FieldTypes.BB_REFERENCE
$: multiselect = fieldSchema?.relationshipType !== "one-to-many"
$: linkedTableId = fieldSchema?.tableId
$: fetch = fetchData({
API,
datasource: {
type: "table",
type: datasourceType,
tableId: linkedTableId,
},
options: {
@ -44,7 +49,7 @@
: flatten(fieldState?.value)?.[0]
$: component = multiselect ? CoreMultiselect : CoreSelect
$: expandedDefaultValue = expand(defaultValue)
$: primaryDisplay = tableDefinition?.primaryDisplay
$: primaryDisplay = primaryDisplay || tableDefinition?.primaryDisplay
let optionsObj = {}
let initialValuesProcessed
@ -54,7 +59,7 @@
// Persist the initial values as options, allowing them to be present in the dropdown,
// even if they are not in the inital fetch results
initialValuesProcessed = true
optionsObj = fieldState?.value?.reduce((accumulator, value) => {
optionsObj = (fieldState?.value || []).reduce((accumulator, value) => {
accumulator[value._id] = {
_id: value._id,
[primaryDisplay]: value.primaryDisplay,
@ -161,7 +166,7 @@
{disabled}
{validation}
defaultValue={expandedDefaultValue}
type={FieldTypes.LINK}
{type}
bind:fieldState
bind:fieldApi
bind:fieldSchema

View File

@ -15,3 +15,4 @@ export { default as formstep } from "./FormStep.svelte"
export { default as jsonfield } from "./JSONField.svelte"
export { default as s3upload } from "./S3Upload.svelte"
export { default as codescanner } from "./CodeScannerField.svelte"
export { default as bbreferencefield } from "./BBReferenceField.svelte"

View File

@ -1,17 +1,5 @@
export const FieldTypes = {
STRING: "string",
BARCODEQR: "barcodeqr",
LONGFORM: "longform",
OPTIONS: "options",
NUMBER: "number",
BOOLEAN: "boolean",
ARRAY: "array",
DATETIME: "datetime",
ATTACHMENT: "attachment",
LINK: "link",
FORMULA: "formula",
JSON: "json",
}
import { FieldType as FieldTypes } from "@budibase/types"
export { FieldType as FieldTypes } from "@budibase/types"
export const UnsortableTypes = [
FieldTypes.FORMULA,

View File

@ -0,0 +1,38 @@
<script>
import { getContext } from "svelte"
import RelationshipCell from "./RelationshipCell.svelte"
import { FieldSubtype } from "@budibase/types"
const { API } = getContext("grid")
const { subtype } = $$props.schema
const schema = {
...$$props.schema,
// This is not really used, just adding some content to be able to render the relationship cell
tableId: "external",
}
async function searchFunction(searchParams) {
if (subtype !== FieldSubtype.USER) {
throw `Search for '${subtype}' not implemented`
}
const results = await API.searchUsers({
...searchParams,
})
// Mapping to the expected data within RelationshipCell
return {
...results,
data: undefined,
rows: results.data,
}
}
</script>
<RelationshipCell
{...$$props}
{schema}
{searchFunction}
primaryDisplay={"email"}
/>

View File

@ -21,6 +21,8 @@
import { Icon, Input, ProgressCircle, clickOutside } from "@budibase/bbui"
import { debounce } from "../../../utils/utils"
const { API, dispatch } = getContext("grid")
export let value
export let api
export let readonly
@ -30,15 +32,15 @@
export let invertX = false
export let invertY = false
export let contentLines = 1
export let searchFunction = API.searchTable
export let primaryDisplay
const { API, dispatch } = getContext("grid")
const color = getColor(0)
let isOpen = false
let searchResults
let searchString
let lastSearchString
let primaryDisplay
let candidateIndex
let lastSearchId
let searching = false
@ -96,7 +98,7 @@
lastSearchId = Math.random()
searching = true
const thisSearchId = lastSearchId
const results = await API.searchTable({
const results = await searchFunction({
paginate: false,
tableId: schema.tableId,
limit: 20,
@ -259,14 +261,16 @@
on:wheel={e => (focused ? e.stopPropagation() : null)}
>
{#each value || [] as relationship}
{#if relationship.primaryDisplay}
{#if relationship[primaryDisplay] || relationship.primaryDisplay}
<div class="badge">
<span
on:click={editable
? () => showRelationship(relationship._id)
: null}
>
{readable(relationship.primaryDisplay)}
{readable(
relationship[primaryDisplay] || relationship.primaryDisplay
)}
</span>
{#if editable}
<Icon

View File

@ -45,7 +45,7 @@
on:focus={() => (active = true)}
on:blur={() => (active = false)}
{type}
value={value || ""}
value={value ?? ""}
on:change={handleChange}
spellcheck="false"
/>

View File

@ -9,6 +9,7 @@ import BooleanCell from "../cells/BooleanCell.svelte"
import FormulaCell from "../cells/FormulaCell.svelte"
import JSONCell from "../cells/JSONCell.svelte"
import AttachmentCell from "../cells/AttachmentCell.svelte"
import BBReferenceCell from "../cells/BBReferenceCell.svelte"
const TypeComponentMap = {
text: TextCell,
@ -23,6 +24,7 @@ const TypeComponentMap = {
link: RelationshipCell,
formula: FormulaCell,
json: JSONCell,
bb_reference: BBReferenceCell,
}
export const getCellRenderer = column => {
return TypeComponentMap[column?.schema?.type] || TextCell

View File

@ -19,12 +19,21 @@ const TypeIconMap = {
formula: "Calculator",
json: "Brackets",
bigint: "TagBold",
bb_reference: {
user: "User",
},
}
export const getColumnIcon = column => {
if (column.schema.autocolumn) {
return "MagicWand"
}
const type = column.schema.type
return TypeIconMap[type] || "Text"
const { type, subtype } = column.schema
const result =
typeof TypeIconMap[type] === "object" && subtype
? TypeIconMap[type][subtype]
: TypeIconMap[type]
return result || "Text"
}

View File

@ -3,9 +3,8 @@ set -e
if [[ -n $CI ]]
then
# --runInBand performs better in ci where resources are limited
# Running in ci, where resources are limited
export NODE_OPTIONS="--max-old-space-size=4096"
node ../../node_modules/jest/bin/jest.js --version
echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail"
jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail
else

View File

@ -166,7 +166,7 @@ export const addSampleData = async (ctx: UserCtx) => {
// Check if default datasource exists before creating it
await sdk.datasources.get(DEFAULT_BB_DATASOURCE_ID)
} catch (err: any) {
const defaultDbDocs = buildDefaultDocs()
const defaultDbDocs = await buildDefaultDocs()
// add in the default db data docs - tables, datasource, rows and links
await db.bulkDocs([...defaultDbDocs])

View File

@ -305,12 +305,7 @@ export class ExternalRequest {
manyRelationships: ManyRelationship[] = []
for (let [key, field] of Object.entries(table.schema)) {
// if set already, or not set just skip it
if (row[key] == null || newRow[key] || !isEditableColumn(field)) {
continue
}
// if its an empty string then it means return the column to null (if possible)
if (row[key] === "") {
newRow[key] = null
if (row[key] === undefined || newRow[key] || !isEditableColumn(field)) {
continue
}
// parse floats/numbers

View File

@ -80,8 +80,8 @@ export async function save(ctx: UserCtx) {
}
const table = await sdk.tables.getTable(tableId)
const { table: updatedTable, row } = inputProcessing(
ctx.user,
const { table: updatedTable, row } = await inputProcessing(
ctx.user?._id,
cloneDeep(table),
inputs
)

View File

@ -59,7 +59,11 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const tableClone = cloneDeep(dbTable)
// this returns the table and row incase they have been updated
let { table, row } = inputProcessing(ctx.user, tableClone, combinedRow)
let { table, row } = await inputProcessing(
ctx.user?._id,
tableClone,
combinedRow
)
const validateResult = await sdk.rows.utils.validate({
row,
table,
@ -106,7 +110,7 @@ export async function save(ctx: UserCtx) {
// need to copy the table so it can be differenced on way out
const tableClone = cloneDeep(dbTable)
let { table, row } = inputProcessing(ctx.user, tableClone, inputs)
let { table, row } = await inputProcessing(ctx.user?._id, tableClone, inputs)
const validateResult = await sdk.rows.utils.validate({
row,

View File

@ -42,7 +42,7 @@ describe("utils", () => {
const data = [{ name: "Alice" }, { name: "Bob" }, { name: "Claire" }]
const result = importToRows(data, table, config.user)
const result = await importToRows(data, table, config.user)
expect(result).toEqual([
expect.objectContaining({
autoId: 1,
@ -89,7 +89,7 @@ describe("utils", () => {
const data = [{ name: "Alice" }, { name: "Bob" }, { name: "Claire" }]
const result = importToRows(data, table)
const result = await importToRows(data, table)
expect(result).toHaveLength(3)
})
})

View File

@ -99,7 +99,7 @@ export function makeSureTableUpToDate(table: any, tableToSave: any) {
return tableToSave
}
export function importToRows(
export async function importToRows(
data: any[],
table: Table,
user: ContextUser | null = null
@ -113,7 +113,7 @@ export function importToRows(
// We use a reference to table here and update it after input processing,
// so that we can auto increment auto IDs in imported data properly
const processed = inputProcessing(user, table, row, {
const processed = await inputProcessing(user?._id, table, row, {
noAutoRelationships: true,
})
row = processed.row
@ -158,7 +158,7 @@ export async function handleDataImport(
const db = context.getAppDB()
const data = parse(rows, schema)
let finalData: any = importToRows(data, table, user)
let finalData: any = await importToRows(data, table, user)
//Set IDs of finalData to match existing row if an update is expected
if (identifierFields.length > 0) {
@ -422,13 +422,11 @@ export function hasTypeChanged(table: Table, oldTable: Table | undefined) {
if (!oldTable) {
return false
}
let key: any
let field: any
for ([key, field] of Object.entries(oldTable.schema)) {
const oldType = field.type
for (let [key, field] of Object.entries(oldTable.schema)) {
if (!table.schema[key]) {
continue
}
const oldType = field.type
const newType = table.schema[key].type
if (oldType !== newType && !areSwitchableTypes(oldType, newType)) {
return true

View File

@ -34,9 +34,9 @@ function syncLastIds(table: Table, rowCount: number) {
})
}
function tableImport(table: Table, data: Row[]) {
async function tableImport(table: Table, data: Row[]) {
const cloneTable = cloneDeep(table)
const rowDocs = importToRows(data, cloneTable)
const rowDocs = await importToRows(data, cloneTable)
syncLastIds(cloneTable, rowDocs.length)
return { rows: rowDocs, table: cloneTable }
}
@ -601,20 +601,20 @@ export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = {
},
}
export function buildDefaultDocs() {
const inventoryData = tableImport(
export async function buildDefaultDocs() {
const inventoryData = await tableImport(
DEFAULT_INVENTORY_TABLE_SCHEMA,
inventoryImport
)
const employeeData = tableImport(
const employeeData = await tableImport(
DEFAULT_EMPLOYEE_TABLE_SCHEMA,
employeeImport
)
const jobData = tableImport(DEFAULT_JOBS_TABLE_SCHEMA, jobsImport)
const jobData = await tableImport(DEFAULT_JOBS_TABLE_SCHEMA, jobsImport)
const expensesData = tableImport(
const expensesData = await tableImport(
DEFAULT_EXPENSES_TABLE_SCHEMA,
expensesImport
)

View File

@ -41,6 +41,7 @@ function generateSchema(
case FieldTypes.OPTIONS:
case FieldTypes.LONGFORM:
case FieldTypes.BARCODEQR:
case FieldTypes.BB_REFERENCE:
schema.text(key)
break
case FieldTypes.NUMBER:

View File

@ -41,7 +41,7 @@ export async function fetch(status: AppStatus, user: ContextUser) {
for (let app of apps) {
const lock = locks[app.appId]
if (lock) {
app.lockedBy = lock
app.lockedBy = lock as any
} else {
// make sure its definitely not present
delete app.lockedBy

View File

@ -0,0 +1,74 @@
import { cache } from "@budibase/backend-core"
import { utils } from "@budibase/shared-core"
import { FieldSubtype } from "@budibase/types"
import { InvalidBBRefError } from "./errors"
export async function processInputBBReferences(
value: string | string[] | { _id: string } | { _id: string }[],
subtype: FieldSubtype
): Promise<string | undefined> {
const referenceIds: string[] = []
if (Array.isArray(value)) {
referenceIds.push(
...value.map(idOrDoc =>
typeof idOrDoc === "string" ? idOrDoc : idOrDoc._id
)
)
} else if (typeof value !== "string") {
referenceIds.push(value._id)
} else {
referenceIds.push(
...value
.split(",")
.filter(x => x)
.map((id: string) => id.trim())
)
}
switch (subtype) {
case FieldSubtype.USER:
const { notFoundIds } = await cache.user.getUsers(referenceIds)
if (notFoundIds?.length) {
throw new InvalidBBRefError(notFoundIds[0], FieldSubtype.USER)
}
break
default:
throw utils.unreachable(subtype)
}
return referenceIds.join(",") || undefined
}
export async function processOutputBBReferences(
value: string,
subtype: FieldSubtype
) {
if (typeof value !== "string") {
// Already processed or nothing to process
return value
}
const ids = value.split(",").filter(id => !!id)
switch (subtype) {
case FieldSubtype.USER:
const { users } = await cache.user.getUsers(ids)
if (!users.length) {
return undefined
}
return users.map(u => ({
_id: u._id,
primaryDisplay: u.email,
email: u.email,
firstName: u.firstName,
lastName: u.lastName,
}))
default:
throw utils.unreachable(subtype)
}
}

View File

@ -0,0 +1,7 @@
import { FieldSubtype } from "@budibase/types"
export class InvalidBBRefError extends Error {
constructor(id: string, subtype: FieldSubtype) {
super(`Id "${id}" is not valid for the subtype "${subtype}"`)
}
}

View File

@ -5,8 +5,12 @@ import { ObjectStoreBuckets } from "../../constants"
import { context, db as dbCore, objectStore } from "@budibase/backend-core"
import { InternalTables } from "../../db/utils"
import { TYPE_TRANSFORM_MAP } from "./map"
import { Row, RowAttachment, Table, ContextUser } from "@budibase/types"
const { cloneDeep } = require("lodash/fp")
import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types"
import { cloneDeep } from "lodash/fp"
import {
processInputBBReferences,
processOutputBBReferences,
} from "./bbReferenceProcessor"
export * from "./utils"
type AutoColumnProcessingOpts = {
@ -48,12 +52,12 @@ function getRemovedAttachmentKeys(
* for automatic ID purposes.
*/
export function processAutoColumn(
user: ContextUser | null,
userId: string | null | undefined,
table: Table,
row: Row,
opts?: AutoColumnProcessingOpts
) {
let noUser = !user || !user.userId
let noUser = !userId
let isUserTable = table._id === InternalTables.USER_METADATA
let now = new Date().toISOString()
// if a row doesn't have a revision then it doesn't exist yet
@ -70,8 +74,8 @@ export function processAutoColumn(
}
switch (schema.subtype) {
case AutoFieldSubTypes.CREATED_BY:
if (creating && shouldUpdateUserFields && user) {
row[key] = [user.userId]
if (creating && shouldUpdateUserFields && userId) {
row[key] = [userId]
}
break
case AutoFieldSubTypes.CREATED_AT:
@ -80,8 +84,8 @@ export function processAutoColumn(
}
break
case AutoFieldSubTypes.UPDATED_BY:
if (shouldUpdateUserFields && user) {
row[key] = [user.userId]
if (shouldUpdateUserFields && userId) {
row[key] = [userId]
}
break
case AutoFieldSubTypes.UPDATED_AT:
@ -130,8 +134,8 @@ export function coerce(row: any, type: string) {
* @param {object} opts some input processing options (like disabling auto-column relationships).
* @returns {object} the row which has been prepared to be written to the DB.
*/
export function inputProcessing(
user: ContextUser | null,
export async function inputProcessing(
userId: string | null | undefined,
table: Table,
row: Row,
opts?: AutoColumnProcessingOpts
@ -166,6 +170,13 @@ export function inputProcessing(
})
}
}
if (field.type === FieldTypes.BB_REFERENCE && value) {
clonedRow[key] = await processInputBBReferences(
value,
field.subtype as FieldSubtype
)
}
}
if (!clonedRow._id || !clonedRow._rev) {
@ -174,7 +185,7 @@ export function inputProcessing(
}
// handle auto columns - this returns an object like {table, row}
return processAutoColumn(user, table, clonedRow, opts)
return processAutoColumn(userId, table, clonedRow, opts)
}
/**
@ -216,6 +227,16 @@ export async function outputProcessing<T extends Row[] | Row>(
attachment.url = objectStore.getAppFileUrl(attachment.key)
})
}
} else if (column.type == FieldTypes.BB_REFERENCE) {
for (let row of enriched) {
if (row[property] == null) {
continue
}
row[property] = await processOutputBBReferences(
row[property],
column.subtype as FieldSubtype
)
}
}
}
if (opts.squash) {

View File

@ -0,0 +1,212 @@
import _ from "lodash"
import * as backendCore from "@budibase/backend-core"
import { FieldSubtype, User } from "@budibase/types"
import {
processInputBBReferences,
processOutputBBReferences,
} from "../bbReferenceProcessor"
import {
DBTestConfiguration,
generator,
structures,
} from "@budibase/backend-core/tests"
import { InvalidBBRefError } from "../errors"
jest.mock("@budibase/backend-core", (): typeof backendCore => {
const actual: typeof backendCore = jest.requireActual(
"@budibase/backend-core"
)
return {
...actual,
cache: {
...actual.cache,
user: {
...actual.cache.user,
getUsers: jest.fn(actual.cache.user.getUsers),
},
},
}
})
const config = new DBTestConfiguration()
describe("bbReferenceProcessor", () => {
const cacheGetUsersSpy = backendCore.cache.user
.getUsers as jest.MockedFunction<typeof backendCore.cache.user.getUsers>
const users: User[] = []
beforeAll(async () => {
const userCount = 10
const userIds = generator.arrayOf(() => generator.guid(), {
min: userCount,
max: userCount,
})
await config.doInTenant(async () => {
const db = backendCore.context.getGlobalDB()
for (const userId of userIds) {
const user = structures.users.user({ _id: userId })
await db.put(user)
users.push(user)
}
})
})
beforeEach(() => {
jest.clearAllMocks()
})
describe("processInputBBReferences", () => {
describe("subtype user", () => {
it("validate valid string id", async () => {
const user = _.sample(users)
const userId = user!._id!
const result = await config.doInTenant(() =>
processInputBBReferences(userId, FieldSubtype.USER)
)
expect(result).toEqual(userId)
expect(cacheGetUsersSpy).toBeCalledTimes(1)
expect(cacheGetUsersSpy).toBeCalledWith([userId])
})
it("throws an error given an invalid id", async () => {
const userId = generator.guid()
await expect(
config.doInTenant(() =>
processInputBBReferences(userId, FieldSubtype.USER)
)
).rejects.toThrowError(new InvalidBBRefError(userId, FieldSubtype.USER))
expect(cacheGetUsersSpy).toBeCalledTimes(1)
expect(cacheGetUsersSpy).toBeCalledWith([userId])
})
it("validates valid user ids as csv", async () => {
const userIds = _.sampleSize(users, 5).map(x => x._id!)
const userIdCsv = userIds.join(" , ")
const result = await config.doInTenant(() =>
processInputBBReferences(userIdCsv, FieldSubtype.USER)
)
expect(result).toEqual(userIds.join(","))
expect(cacheGetUsersSpy).toBeCalledTimes(1)
expect(cacheGetUsersSpy).toBeCalledWith(userIds)
})
it("throws an error given an invalid id in a csv", async () => {
const expectedUserIds = _.sampleSize(users, 2).map(x => x._id!)
const wrongId = generator.guid()
const userIdCsv = [
expectedUserIds[0],
wrongId,
expectedUserIds[1],
].join(" , ")
await expect(
config.doInTenant(() =>
processInputBBReferences(userIdCsv, FieldSubtype.USER)
)
).rejects.toThrowError(
new InvalidBBRefError(wrongId, FieldSubtype.USER)
)
})
it("validate valid user object", async () => {
const userId = _.sample(users)!._id!
const result = await config.doInTenant(() =>
processInputBBReferences({ _id: userId }, FieldSubtype.USER)
)
expect(result).toEqual(userId)
expect(cacheGetUsersSpy).toBeCalledTimes(1)
expect(cacheGetUsersSpy).toBeCalledWith([userId])
})
it("validate valid user object array", async () => {
const userIds = _.sampleSize(users, 3).map(x => x._id!)
const result = await config.doInTenant(() =>
processInputBBReferences(userIds, FieldSubtype.USER)
)
expect(result).toEqual(userIds.join(","))
expect(cacheGetUsersSpy).toBeCalledTimes(1)
expect(cacheGetUsersSpy).toBeCalledWith(userIds)
})
it("empty strings will return undefined", async () => {
const result = await config.doInTenant(() =>
processInputBBReferences("", FieldSubtype.USER)
)
expect(result).toEqual(undefined)
})
it("empty arrays will return undefined", async () => {
const result = await config.doInTenant(() =>
processInputBBReferences([], FieldSubtype.USER)
)
expect(result).toEqual(undefined)
})
})
})
describe("processOutputBBReferences", () => {
describe("subtype user", () => {
it("fetches user given a valid string id", async () => {
const user = _.sample(users)!
const userId = user._id!
const result = await config.doInTenant(() =>
processOutputBBReferences(userId, FieldSubtype.USER)
)
expect(result).toEqual([
{
_id: user._id,
primaryDisplay: user.email,
email: user.email,
firstName: user.firstName,
lastName: user.lastName,
},
])
expect(cacheGetUsersSpy).toBeCalledTimes(1)
expect(cacheGetUsersSpy).toBeCalledWith([userId])
})
it("fetches user given a valid string id csv", async () => {
const [user1, user2] = _.sampleSize(users, 2)
const userId1 = user1._id!
const userId2 = user2._id!
const result = await config.doInTenant(() =>
processOutputBBReferences(
[userId1, userId2].join(","),
FieldSubtype.USER
)
)
expect(result).toHaveLength(2)
expect(result).toEqual(
expect.arrayContaining(
[user1, user2].map(u => ({
_id: u._id,
primaryDisplay: u.email,
email: u.email,
firstName: u.firstName,
lastName: u.lastName,
}))
)
)
expect(cacheGetUsersSpy).toBeCalledTimes(1)
expect(cacheGetUsersSpy).toBeCalledWith([userId1, userId2])
})
})
})
})

View File

@ -0,0 +1,186 @@
import { inputProcessing } from ".."
import { generator, structures } from "@budibase/backend-core/tests"
import { FieldType, FieldTypeSubtypes, Table } from "@budibase/types"
import * as bbReferenceProcessor from "../bbReferenceProcessor"
jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
processInputBBReferences: jest.fn(),
processOutputBBReferences: jest.fn(),
}))
describe("rowProcessor - inputProcessing", () => {
beforeEach(() => {
jest.resetAllMocks()
})
it("processes BB references if on the schema and it's populated", async () => {
const userId = generator.guid()
const table: Table = {
_id: generator.guid(),
name: "TestTable",
type: "table",
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: {
presence: true,
type: "string",
},
},
user: {
type: FieldType.BB_REFERENCE,
subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
name: "user",
constraints: {
presence: true,
type: "string",
},
},
},
}
const newRow = {
name: "Jack",
user: "123",
}
const user = structures.users.user()
;(
bbReferenceProcessor.processInputBBReferences as jest.Mock
).mockResolvedValue(user)
const { row } = await inputProcessing(userId, table, newRow)
expect(bbReferenceProcessor.processInputBBReferences).toBeCalledTimes(1)
expect(bbReferenceProcessor.processInputBBReferences).toBeCalledWith(
"123",
"user"
)
expect(row).toEqual({ ...newRow, user })
})
it("it does not process BB references if on the schema but it is not populated", async () => {
const userId = generator.guid()
const table: Table = {
_id: generator.guid(),
name: "TestTable",
type: "table",
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: {
presence: true,
type: "string",
},
},
user: {
type: FieldType.BB_REFERENCE,
subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
name: "user",
constraints: {
presence: false,
type: "string",
},
},
},
}
const newRow = {
name: "Jack",
}
const { row } = await inputProcessing(userId, table, newRow)
expect(bbReferenceProcessor.processInputBBReferences).not.toBeCalled()
expect(row).toEqual({ ...newRow, user: undefined })
})
it.each([undefined, null, ""])(
"it does not process BB references the field is $%",
async userValue => {
const userId = generator.guid()
const table: Table = {
_id: generator.guid(),
name: "TestTable",
type: "table",
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: {
presence: true,
type: "string",
},
},
user: {
type: FieldType.BB_REFERENCE,
subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
name: "user",
constraints: {
presence: false,
type: "string",
},
},
},
}
const newRow = {
name: "Jack",
user: userValue,
}
const { row } = await inputProcessing(userId, table, newRow)
expect(bbReferenceProcessor.processInputBBReferences).not.toBeCalled()
expect(row).toEqual(newRow)
}
)
it("it does not process BB references if not in the schema", async () => {
const userId = generator.guid()
const table: Table = {
_id: generator.guid(),
name: "TestTable",
type: "table",
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: {
presence: true,
type: "string",
},
},
user: {
type: FieldType.NUMBER,
name: "user",
constraints: {
presence: true,
type: "string",
},
},
},
}
const newRow = {
name: "Jack",
user: "123",
}
const { row } = await inputProcessing(userId, table, newRow)
expect(bbReferenceProcessor.processInputBBReferences).not.toBeCalled()
expect(row).toEqual({
name: "Jack",
user: 123,
})
})
})

View File

@ -0,0 +1,142 @@
import {
FieldSubtype,
FieldType,
FieldTypeSubtypes,
Table,
} from "@budibase/types"
import { outputProcessing } from ".."
import { generator, structures } from "@budibase/backend-core/tests"
import * as bbReferenceProcessor from "../bbReferenceProcessor"
jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
processInputBBReferences: jest.fn(),
processOutputBBReferences: jest.fn(),
}))
describe("rowProcessor - outputProcessing", () => {
beforeEach(() => {
jest.resetAllMocks()
})
const processOutputBBReferencesMock =
bbReferenceProcessor.processOutputBBReferences as jest.Mock
it("fetches bb user references given a populated field", async () => {
const table: Table = {
_id: generator.guid(),
name: "TestTable",
type: "table",
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: {
presence: true,
type: "string",
},
},
user: {
type: FieldType.BB_REFERENCE,
subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
name: "user",
constraints: {
presence: false,
type: "string",
},
},
},
}
const row = {
name: "Jack",
user: "123",
}
const user = structures.users.user()
processOutputBBReferencesMock.mockResolvedValue(user)
const result = await outputProcessing(table, row, { squash: false })
expect(result).toEqual({ name: "Jack", user })
expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledTimes(1)
expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledWith(
"123",
FieldSubtype.USER
)
})
it("does not fetch bb references when fields are empty", async () => {
const table: Table = {
_id: generator.guid(),
name: "TestTable",
type: "table",
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: {
presence: true,
type: "string",
},
},
user: {
type: FieldType.BB_REFERENCE,
subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
name: "user",
constraints: {
presence: false,
type: "string",
},
},
},
}
const row = {
name: "Jack",
}
const result = await outputProcessing(table, row, { squash: false })
expect(result).toEqual({ name: "Jack" })
expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled()
})
it("does not fetch bb references when not in the schema", async () => {
const table: Table = {
_id: generator.guid(),
name: "TestTable",
type: "table",
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: {
presence: true,
type: "string",
},
},
user: {
type: FieldType.NUMBER,
name: "user",
constraints: {
presence: false,
type: "string",
},
},
},
}
const row = {
name: "Jack",
user: "123",
}
const result = await outputProcessing(table, row, { squash: false })
expect(result).toEqual({ name: "Jack", user: "123" })
expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled()
})
})

View File

@ -150,7 +150,7 @@ export class BaseSocket {
if (room) {
const sessionIds = await this.getRoomSessionIds(room)
const keys = sessionIds.map(this.getSessionKey.bind(this))
const sessions = await this.redisClient?.bulkGet(keys)
const sessions = await this.redisClient?.bulkGet<SocketSession>(keys)
return Object.values(sessions || {})
} else {
return []

View File

@ -16,6 +16,7 @@ export enum FieldType {
INTERNAL = "internal",
BARCODEQR = "barcodeqr",
BIGINT = "bigint",
BB_REFERENCE = "bb_reference",
}
export interface RowAttachment {
@ -33,3 +34,13 @@ export interface Row extends Document {
_viewId?: string
[key: string]: any
}
export enum FieldSubtype {
USER = "user",
}
export const FieldTypeSubtypes = {
BB_REFERENCE: {
USER: FieldSubtype.USER,
},
}

View File

@ -3,9 +3,9 @@ set -e
if [[ -n $CI ]]
then
# --runInBand performs better in ci where resources are limited
echo "jest --coverage --runInBand --forceExit --bail"
jest --coverage --runInBand --forceExit --bail
# Running in ci, where resources are limited
echo "jest --coverage --maxWorkers=2 --forceExit --bail"
jest --coverage --maxWorkers=2 --forceExit --bail
else
# --maxWorkers performs better in development
echo "jest --coverage --maxWorkers=2 --forceExit"