MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:Budibase/budibase into table-improvements

This commit is contained in:
Andrew Kingston 2024-06-24 15:32:11 +01:00
parent df03a297c5 29a5ba5bb3
commit e7bc96fdee
No known key found for this signature in database
34 changed files with 1131 additions and 686 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.29.0",
"version": "2.29.1",
"npmClient": "yarn",
"packages": [
"packages/*",

2
packages/account-portal

@ -1 +1 @@
Subproject commit 247f56d455abbd64da17d865275ed978f577549f
Subproject commit b600cca314a5cc9971e44d46047d1a0019b46b08

2
packages/backend-core/src/constants/db.ts
View File

@ -72,4 +72,4 @@ export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
export { DEFAULT_BB_DATASOURCE_ID } from "@budibase/shared-core"

19
packages/backend-core/src/db/constants.ts
View File

@ -1,14 +1,5 @@
export const CONSTANT_INTERNAL_ROW_COLS = [
"_id",
"_rev",
"type",
"createdAt",
"updatedAt",
"tableId",
] as const
export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const
export function isInternalColumnName(name: string): boolean {
return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name)
}
export {
CONSTANT_INTERNAL_ROW_COLS,
CONSTANT_EXTERNAL_ROW_COLS,
isInternalColumnName,
} from "@budibase/shared-core"

6
packages/backend-core/src/sql/sqlTable.ts
View File

@ -109,8 +109,10 @@ function generateSchema(
const { tableName } = breakExternalTableId(column.tableId)
// @ts-ignore
const relatedTable = tables[tableName]
if (!relatedTable) {
throw new Error("Referenced table doesn't exist")
if (!relatedTable || !relatedTable.primary) {
throw new Error(
"Referenced table doesn't exist or has no primary keys"
)
}
const relatedPrimary = relatedTable.primary[0]
const externalType = relatedTable.schema[relatedPrimary].externalType

8
packages/backend-core/src/sql/utils.ts
View File

@ -55,10 +55,7 @@ export function buildExternalTableId(datasourceId: string, tableName: string) {
return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
}
export function breakExternalTableId(tableId: string | undefined) {
if (!tableId) {
return {}
}
export function breakExternalTableId(tableId: string) {
const parts = tableId.split(DOUBLE_SEPARATOR)
let datasourceId = parts.shift()
// if they need joined
@ -67,6 +64,9 @@ export function breakExternalTableId(tableId: string | undefined) {
if (tableName.includes(ENCODED_SPACE)) {
tableName = decodeURIComponent(tableName)
}
if (!datasourceId || !tableName) {
throw new Error("Unable to get datasource/table name from table ID")
}
return { datasourceId, tableName }
}

18
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -17,6 +17,8 @@
SWITCHABLE_TYPES,
ValidColumnNameRegex,
helpers,
CONSTANT_INTERNAL_ROW_COLS,
CONSTANT_EXTERNAL_ROW_COLS,
} from "@budibase/shared-core"
import { createEventDispatcher, getContext, onMount } from "svelte"
import { cloneDeep } from "lodash/fp"
@ -52,7 +54,6 @@
const DATE_TYPE = FieldType.DATETIME
const dispatch = createEventDispatcher()
const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
const { dispatch: gridDispatch, rows } = getContext("grid")
export let field
@ -487,20 +488,27 @@
})
}
const newError = {}
const prohibited = externalTable
? CONSTANT_EXTERNAL_ROW_COLS
: CONSTANT_INTERNAL_ROW_COLS
if (!externalTable && fieldInfo.name?.startsWith("_")) {
newError.name = `Column name cannot start with an underscore.`
} else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) {
newError.name = `Illegal character; must be alpha-numeric.`
} else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) {
newError.name = `${PROHIBITED_COLUMN_NAMES.join(
} else if (
prohibited.some(
name => fieldInfo?.name?.toLowerCase() === name.toLowerCase()
)
) {
newError.name = `${prohibited.join(
", "
)} are not allowed as column names`
)} are not allowed as column names - case insensitive.`
} else if (inUse($tables.selected, fieldInfo.name, originalName)) {
newError.name = `Column name already in use.`
}
if (fieldInfo.type === FieldType.AUTO && !fieldInfo.subtype) {
newError.subtype = `Auto Column requires a type`
newError.subtype = `Auto Column requires a type.`
}
if (fieldInfo.fieldName && fieldInfo.tableId) {

6
packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte
View File

@ -53,6 +53,12 @@
placeholder="Are you sure you want to delete?"
bind:value={parameters.confirmText}
/>
<Label small>Confirm Text</Label>
<Input placeholder="Confirm" bind:value={parameters.confirmButtonText} />
<Label small>Cancel Text</Label>
<Input placeholder="Cancel" bind:value={parameters.cancelButtonText} />
{/if}
</div>
</div>

6
packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte
View File

@ -83,6 +83,12 @@
placeholder="Are you sure you want to duplicate this row?"
bind:value={parameters.confirmText}
/>
<Label small>Confirm Text</Label>
<Input placeholder="Confirm" bind:value={parameters.confirmButtonText} />
<Label small>Cancel Text</Label>
<Input placeholder="Cancel" bind:value={parameters.cancelButtonText} />
{/if}
</div>

12
packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte
View File

@ -74,6 +74,18 @@
placeholder="Are you sure you want to execute this query?"
bind:value={parameters.confirmText}
/>
<Input
label="Confirm Text"
placeholder="Confirm"
bind:value={parameters.confirmButtonText}
/>
<Input
label="Cancel Text"
placeholder="Cancel"
bind:value={parameters.cancelButtonText}
/>
{/if}
{#if query?.parameters?.length > 0}

6
packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte
View File

@ -80,6 +80,12 @@
placeholder="Are you sure you want to save this row?"
bind:value={parameters.confirmText}
/>
<Label small>Confirm Text</Label>
<Input placeholder="Confirm" bind:value={parameters.confirmButtonText} />
<Label small>Cancel Text</Label>
<Input placeholder="Cancel" bind:value={parameters.cancelButtonText} />
{/if}
</div>

2
packages/client/src/components/overlay/ConfirmationDisplay.svelte
View File

@ -8,6 +8,8 @@
<ModalContent
title={$confirmationStore.title}
onConfirm={confirmationStore.actions.confirm}
confirmText={$confirmationStore.confirmButtonText}
cancelText={$confirmationStore.cancelButtonText}
>
{$confirmationStore.text}
</ModalContent>

13
packages/client/src/stores/confirmation.js
View File

@ -4,6 +4,8 @@ const initialState = {
showConfirmation: false,
title: null,
text: null,
confirmButtonText: null,
cancelButtonText: null,
onConfirm: null,
onCancel: null,
}
@ -11,11 +13,20 @@ const initialState = {
const createConfirmationStore = () => {
const store = writable(initialState)
const showConfirmation = (title, text, onConfirm, onCancel) => {
const showConfirmation = (
title,
text,
onConfirm,
onCancel,
confirmButtonText,
cancelButtonText
) => {
store.set({
showConfirmation: true,
title,
text,
confirmButtonText,
cancelButtonText,
onConfirm,
onCancel,
})

10
packages/client/src/utils/buttonActions.js
View File

@ -522,6 +522,7 @@ const confirmTextMap = {
["Execute Query"]: "Are you sure you want to execute this query?",
["Trigger Automation"]: "Are you sure you want to trigger this automation?",
["Prompt User"]: "Are you sure you want to continue?",
["Duplicate Row"]: "Are you sure you want to duplicate this row?",
}
/**
@ -582,6 +583,11 @@ export const enrichButtonActions = (actions, context) => {
const defaultTitleText = action["##eventHandlerType"]
const customTitleText =
action.parameters?.customTitleText || defaultTitleText
const cancelButtonText =
action.parameters?.cancelButtonText || "Cancel"
const confirmButtonText =
action.parameters?.confirmButtonText || "Confirm"
confirmationStore.actions.showConfirmation(
customTitleText,
confirmText,
@ -612,7 +618,9 @@ export const enrichButtonActions = (actions, context) => {
},
() => {
resolve(false)
}
},
confirmButtonText,
cancelButtonText
)
})
}

21
packages/frontend-core/src/components/FilterBuilder.svelte
View File

@ -18,7 +18,7 @@
import FilterUsers from "./FilterUsers.svelte"
import { getFields } from "../utils/searchFields"
const { OperatorOptions } = Constants
const { OperatorOptions, DEFAULT_BB_DATASOURCE_ID } = Constants
export let schemaFields
export let filters = []
@ -28,6 +28,23 @@
export let allowBindings = false
export let filtersLabel = "Filters"
$: {
if (
tables.find(
table =>
table._id === datasource.tableId &&
table.sourceId === DEFAULT_BB_DATASOURCE_ID
) &&
!schemaFields.some(field => field.name === "_id")
) {
schemaFields = [
...schemaFields,
{ name: "_id", type: "string" },
{ name: "_rev", type: "string" },
]
}
}
$: matchAny = filters?.find(filter => filter.operator === "allOr") != null
$: onEmptyFilter =
filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
@ -35,7 +52,6 @@
$: fieldFilters = filters.filter(
filter => filter.operator !== "allOr" && !filter.onEmptyFilter
)
const behaviourOptions = [
{ value: "and", label: "Match all filters" },
{ value: "or", label: "Match any filter" },
@ -44,7 +60,6 @@
{ value: "all", label: "Return all table rows" },
{ value: "none", label: "Return no rows" },
]
const context = getContext("context")
$: fieldOptions = getFields(tables, schemaFields || [], {

6
packages/frontend-core/src/constants.js
View File

@ -1,7 +1,11 @@
/**
* Operator options for lucene queries
*/
export { OperatorOptions, SqlNumberTypeRangeMap } from "@budibase/shared-core"
export {
OperatorOptions,
SqlNumberTypeRangeMap,
DEFAULT_BB_DATASOURCE_ID,
} from "@budibase/shared-core"
export { Feature as Features } from "@budibase/types"
import { BpmCorrelationKey } from "@budibase/shared-core"
import { FieldType, BBReferenceFieldSubType } from "@budibase/types"

258
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -72,92 +72,6 @@ export type ExternalRequestReturnType<T extends Operation> =
? number
: { row: Row; table: Table }
function buildFilters(
id: string | undefined | string[],
filters: SearchFilters,
table: Table
) {
const primary = table.primary
// if passed in array need to copy for shifting etc
let idCopy: undefined | string | any[] = cloneDeep(id)
if (filters) {
// need to map over the filters and make sure the _id field isn't present
let prefix = 1
for (let operator of Object.values(filters)) {
for (let field of Object.keys(operator || {})) {
if (dbCore.removeKeyNumbering(field) === "_id") {
if (primary) {
const parts = breakRowIdField(operator[field])
for (let field of primary) {
operator[`${prefix}:${field}`] = parts.shift()
}
prefix++
}
// make sure this field doesn't exist on any filter
delete operator[field]
}
}
}
}
// there is no id, just use the user provided filters
if (!idCopy || !table) {
return filters
}
// if used as URL parameter it will have been joined
if (!Array.isArray(idCopy)) {
idCopy = breakRowIdField(idCopy)
}
const equal: any = {}
if (primary && idCopy) {
for (let field of primary) {
// work through the ID and get the parts
equal[field] = idCopy.shift()
}
}
return {
equal,
}
}
async function removeManyToManyRelationships(
rowId: string,
table: Table,
colName: string
) {
const tableId = table._id!
const filters = buildFilters(rowId, {}, table)
// safety check, if there are no filters on deletion bad things happen
if (Object.keys(filters).length !== 0) {
return getDatasourceAndQuery({
endpoint: getEndpoint(tableId, Operation.DELETE),
body: { [colName]: null },
filters,
meta: {
table,
},
})
} else {
return []
}
}
async function removeOneToManyRelationships(rowId: string, table: Table) {
const tableId = table._id!
const filters = buildFilters(rowId, {}, table)
// safety check, if there are no filters on deletion bad things happen
if (Object.keys(filters).length !== 0) {
return getDatasourceAndQuery({
endpoint: getEndpoint(tableId, Operation.UPDATE),
filters,
meta: {
table,
},
})
} else {
return []
}
}
/**
* This function checks the incoming parameters to make sure all the inputs are
* valid based on on the table schema. The main thing this is looking for is when a
@ -212,8 +126,8 @@ function getEndpoint(tableId: string | undefined, operation: string) {
}
const { datasourceId, tableName } = breakExternalTableId(tableId)
return {
datasourceId: datasourceId!,
entityId: tableName!,
datasourceId: datasourceId,
entityId: tableName,
operation: operation as Operation,
}
}
@ -240,6 +154,7 @@ export class ExternalRequest<T extends Operation> {
private readonly tableId: string
private datasource?: Datasource
private tables: { [key: string]: Table } = {}
private tableList: Table[]
constructor(operation: T, tableId: string, datasource?: Datasource) {
this.operation = operation
@ -248,22 +163,134 @@ export class ExternalRequest<T extends Operation> {
if (datasource && datasource.entities) {
this.tables = datasource.entities
}
this.tableList = Object.values(this.tables)
}
private prepareFilters(
id: string | undefined | string[],
filters: SearchFilters,
table: Table
): SearchFilters {
// replace any relationship columns initially, table names and relationship column names are acceptable
const relationshipColumns = sdk.rows.filters.getRelationshipColumns(table)
filters = sdk.rows.filters.updateFilterKeys(
filters,
relationshipColumns.map(({ name, definition }) => {
const { tableName } = breakExternalTableId(definition.tableId)
return {
original: name,
updated: tableName,
}
})
)
const primary = table.primary
// if passed in array need to copy for shifting etc
let idCopy: undefined | string | any[] = cloneDeep(id)
if (filters) {
// need to map over the filters and make sure the _id field isn't present
let prefix = 1
for (let operator of Object.values(filters)) {
for (let field of Object.keys(operator || {})) {
if (dbCore.removeKeyNumbering(field) === "_id") {
if (primary) {
const parts = breakRowIdField(operator[field])
for (let field of primary) {
operator[`${prefix}:${field}`] = parts.shift()
}
prefix++
}
// make sure this field doesn't exist on any filter
delete operator[field]
}
}
}
}
// there is no id, just use the user provided filters
if (!idCopy || !table) {
return filters
}
// if used as URL parameter it will have been joined
if (!Array.isArray(idCopy)) {
idCopy = breakRowIdField(idCopy)
}
const equal: SearchFilters["equal"] = {}
if (primary && idCopy) {
for (let field of primary) {
// work through the ID and get the parts
equal[field] = idCopy.shift()
}
}
return {
equal,
}
}
private async removeManyToManyRelationships(
rowId: string,
table: Table,
colName: string
) {
const tableId = table._id!
const filters = this.prepareFilters(rowId, {}, table)
// safety check, if there are no filters on deletion bad things happen
if (Object.keys(filters).length !== 0) {
return getDatasourceAndQuery({
endpoint: getEndpoint(tableId, Operation.DELETE),
body: { [colName]: null },
filters,
meta: {
table,
},
})
} else {
return []
}
}
private async removeOneToManyRelationships(rowId: string, table: Table) {
const tableId = table._id!
const filters = this.prepareFilters(rowId, {}, table)
// safety check, if there are no filters on deletion bad things happen
if (Object.keys(filters).length !== 0) {
return getDatasourceAndQuery({
endpoint: getEndpoint(tableId, Operation.UPDATE),
filters,
meta: {
table,
},
})
} else {
return []
}
}
getTable(tableId: string | undefined): Table | undefined {
if (!tableId) {
throw "Table ID is unknown, cannot find table"
throw new Error("Table ID is unknown, cannot find table")
}
const { tableName } = breakExternalTableId(tableId)
if (tableName) {
return this.tables[tableName]
return this.tables[tableName]
}
// seeds the object with table and datasource information
async retrieveMetadata(
datasourceId: string
): Promise<{ tables: Record<string, Table>; datasource: Datasource }> {
if (!this.datasource) {
this.datasource = await sdk.datasources.get(datasourceId)
if (!this.datasource || !this.datasource.entities) {
throw "No tables found, fetch tables before query."
}
this.tables = this.datasource.entities
this.tableList = Object.values(this.tables)
}
return { tables: this.tables, datasource: this.datasource }
}
async getRow(table: Table, rowId: string): Promise<Row> {
const response = await getDatasourceAndQuery({
endpoint: getEndpoint(table._id!, Operation.READ),
filters: buildFilters(rowId, {}, table),
filters: this.prepareFilters(rowId, {}, table),
meta: {
table,
},
@ -296,9 +323,7 @@ export class ExternalRequest<T extends Operation> {
if (field.type === FieldType.NUMBER && !isNaN(parseFloat(row[key]))) {
newRow[key] = parseFloat(row[key])
} else if (field.type === FieldType.LINK) {
const { tableName: linkTableName } = breakExternalTableId(
field?.tableId
)
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
// table has to exist for many to many
if (!linkTableName || !this.tables[linkTableName]) {
continue
@ -379,9 +404,6 @@ export class ExternalRequest<T extends Operation> {
[key: string]: { rows: Row[]; isMany: boolean; tableId: string }
} = {}
const { tableName } = breakExternalTableId(tableId)
if (!tableName) {
return related
}
const table = this.tables[tableName]
// @ts-ignore
const primaryKey = table.primary[0]
@ -514,7 +536,7 @@ export class ExternalRequest<T extends Operation> {
endpoint: getEndpoint(tableId, operation),
// if we're doing many relationships then we're writing, only one response
body,
filters: buildFilters(id, {}, linkTable),
filters: this.prepareFilters(id, {}, linkTable),
meta: {
table: linkTable,
},
@ -538,8 +560,8 @@ export class ExternalRequest<T extends Operation> {
for (let row of rows) {
const rowId = generateIdForRow(row, table)
const promise: Promise<any> = isMany
? removeManyToManyRelationships(rowId, table, colName)
: removeOneToManyRelationships(rowId, table)
? this.removeManyToManyRelationships(rowId, table, colName)
: this.removeOneToManyRelationships(rowId, table)
if (promise) {
promises.push(promise)
}
@ -562,12 +584,12 @@ export class ExternalRequest<T extends Operation> {
rows.map(row => {
const rowId = generateIdForRow(row, table)
return isMany
? removeManyToManyRelationships(
? this.removeManyToManyRelationships(
rowId,
table,
relationshipColumn.fieldName
)
: removeOneToManyRelationships(rowId, table)
: this.removeOneToManyRelationships(rowId, table)
})
)
}
@ -575,21 +597,21 @@ export class ExternalRequest<T extends Operation> {
async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
const { operation, tableId } = this
let { datasourceId, tableName } = breakExternalTableId(tableId)
if (!tableName) {
throw "Unable to run without a table name"
if (!tableId) {
throw new Error("Unable to run without a table ID")
}
if (!this.datasource) {
this.datasource = await sdk.datasources.get(datasourceId!)
if (!this.datasource || !this.datasource.entities) {
throw "No tables found, fetch tables before query."
}
this.tables = this.datasource.entities
let { datasourceId, tableName } = breakExternalTableId(tableId)
let datasource = this.datasource
if (!datasource) {
const { datasource: ds } = await this.retrieveMetadata(datasourceId)
datasource = ds
}
const table = this.tables[tableName]
let isSql = isSQL(this.datasource)
let isSql = isSQL(datasource)
if (!table) {
throw `Unable to process query, table "${tableName}" not defined.`
throw new Error(
`Unable to process query, table "${tableName}" not defined.`
)
}
// look for specific components of config which may not be considered acceptable
let { id, row, filters, sort, paginate, rows } = cleanupConfig(
@ -612,7 +634,7 @@ export class ExternalRequest<T extends Operation> {
break
}
}
filters = buildFilters(id, filters || {}, table)
filters = this.prepareFilters(id, filters || {}, table)
const relationships = buildExternalRelationships(table, this.tables)
const incRelationships =
@ -660,7 +682,11 @@ export class ExternalRequest<T extends Operation> {
body: row || rows,
// pass an id filter into extra, purely for mysql/returning
extra: {
idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
idFilter: this.prepareFilters(
id || generateIdForRow(row, table),
{},
table
),
},
meta: {
table,

7
packages/server/src/api/controllers/row/external.ts
View File

@ -136,10 +136,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
const id = ctx.params.rowId
const tableId = utils.getTableId(ctx)
const { datasourceId, tableName } = breakExternalTableId(tableId)
const datasource: Datasource = await sdk.datasources.get(datasourceId!)
if (!tableName) {
ctx.throw(400, "Unable to find table.")
}
const datasource: Datasource = await sdk.datasources.get(datasourceId)
if (!datasource || !datasource.entities) {
ctx.throw(400, "Datasource has not been configured for plus API.")
}
@ -163,7 +160,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
}
const links = row[fieldName]
const linkedTableId = field.tableId
const linkedTableName = breakExternalTableId(linkedTableId).tableName!
const linkedTableName = breakExternalTableId(linkedTableId).tableName
const linkedTable = tables[linkedTableName]
// don't support composite keys right now
const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0])

2
packages/server/src/api/controllers/row/utils/basic.ts
View File

@ -99,7 +99,7 @@ export function basicProcessing({
row,
tableName: table._id!,
fieldName: internalColumn,
isLinked: false,
isLinked,
})
}
}

26
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -2,6 +2,8 @@ import {
DatasourcePlusQueryResponse,
DSPlusOperation,
FieldType,
isManyToOne,
isOneToMany,
ManyToManyRelationshipFieldMetadata,
RelationshipFieldMetadata,
RelationshipsJson,
@ -93,12 +95,12 @@ export function buildExternalRelationships(
): RelationshipsJson[] {
const relationships = []
for (let [fieldName, field] of Object.entries(table.schema)) {
if (field.type !== FieldType.LINK) {
if (field.type !== FieldType.LINK || !field.tableId) {
continue
}
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
// no table to link to, this is not a valid relationships
if (!linkTableName || !tables[linkTableName]) {
if (!tables[linkTableName]) {
continue
}
const linkTable = tables[linkTableName]
@ -110,7 +112,7 @@ export function buildExternalRelationships(
// need to specify where to put this back into
column: fieldName,
}
if (isManyToMany(field)) {
if (isManyToMany(field) && field.through) {
const { tableName: throughTableName } = breakExternalTableId(
field.through
)
@ -120,7 +122,7 @@ export function buildExternalRelationships(
definition.to = field.throughFrom || linkTable.primary[0]
definition.fromPrimary = table.primary[0]
definition.toPrimary = linkTable.primary[0]
} else {
} else if (isManyToOne(field) || isOneToMany(field)) {
// if no foreign key specified then use the name of the field in other table
definition.from = field.foreignKey || table.primary[0]
definition.to = field.fieldName
@ -180,16 +182,18 @@ export function buildSqlFieldList(
}
let fields = extractRealFields(table)
for (let field of Object.values(table.schema)) {
if (field.type !== FieldType.LINK || !opts?.relationships) {
if (
field.type !== FieldType.LINK ||
!opts?.relationships ||
!field.tableId
) {
continue
}
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
if (linkTableName) {
const linkTable = tables[linkTableName]
if (linkTable) {
const linkedFields = extractRealFields(linkTable, fields)
fields = fields.concat(linkedFields)
}
const linkTable = tables[linkTableName]
if (linkTable) {
const linkedFields = extractRealFields(linkTable, fields)
fields = fields.concat(linkedFields)
}
}
return fields

5
packages/server/src/api/controllers/table/external.ts
View File

@ -19,11 +19,14 @@ import { inputProcessing } from "../../../utilities/rowProcessor"
function getDatasourceId(table: Table) {
if (!table) {
throw "No table supplied"
throw new Error("No table supplied")
}
if (table.sourceId) {
return table.sourceId
}
if (!table._id) {
throw new Error("No table ID supplied")
}
return breakExternalTableId(table._id).datasourceId
}

1103
packages/server/src/api/routes/tests/search.spec.ts
View File

File diff suppressed because it is too large Load Diff

28
packages/server/src/api/routes/tests/table.spec.ts
View File

@ -276,6 +276,34 @@ describe.each([
})
})
isInternal &&
it("shouldn't allow duplicate column names", async () => {
const saveTableRequest: SaveTableRequest = {
...basicTable(),
}
saveTableRequest.schema["Type"] = {
type: FieldType.STRING,
name: "Type",
}
await config.api.table.save(saveTableRequest, {
status: 400,
body: {
message:
'Column(s) "type" are duplicated - check for other columns with these name (case in-sensitive)',
},
})
saveTableRequest.schema.foo = { type: FieldType.STRING, name: "foo" }
saveTableRequest.schema.FOO = { type: FieldType.STRING, name: "FOO" }
await config.api.table.save(saveTableRequest, {
status: 400,
body: {
message:
'Column(s) "type, foo" are duplicated - check for other columns with these name (case in-sensitive)',
},
})
})
it("should add a new column for an internal DB table", async () => {
const saveTableRequest: SaveTableRequest = {
...basicTable(),

2
packages/server/src/sdk/app/rows/index.ts
View File

@ -3,12 +3,14 @@ import * as rows from "./rows"
import * as search from "./search"
import * as utils from "./utils"
import * as external from "./external"
import * as filters from "./search/filters"
import AliasTables from "./sqlAlias"
export default {
...attachments,
...rows,
...search,
filters,
utils,
external,
AliasTables,

10
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -145,6 +145,10 @@ export async function exportRows(
delimiter,
customHeaders,
} = options
if (!tableId) {
throw new HTTPError("No table ID for search provided.", 400)
}
const { datasourceId, tableName } = breakExternalTableId(tableId)
let requestQuery: SearchFilters = {}
@ -167,7 +171,7 @@ export async function exportRows(
requestQuery = query || {}
}
const datasource = await sdk.datasources.get(datasourceId!)
const datasource = await sdk.datasources.get(datasourceId)
const table = await sdk.tables.getTable(tableId)
if (!datasource || !datasource.entities) {
throw new HTTPError("Datasource has not been configured for plus API.", 400)
@ -180,10 +184,6 @@ export async function exportRows(
let rows: Row[] = []
let headers
if (!tableName) {
throw new HTTPError("Could not find table name.", 400)
}
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.rows.length; i++) {

62
packages/server/src/sdk/app/rows/search/filters.ts Normal file
View File

@ -0,0 +1,62 @@
import {
FieldType,
RelationshipFieldMetadata,
SearchFilters,
Table,
} from "@budibase/types"
import { isPlainObject } from "lodash"
export function getRelationshipColumns(table: Table): {
name: string
definition: RelationshipFieldMetadata
}[] {
// performing this with a for loop rather than an array filter improves
// type guarding, as no casts are required
const linkEntries: [string, RelationshipFieldMetadata][] = []
for (let entry of Object.entries(table.schema)) {
if (entry[1].type === FieldType.LINK) {
const linkColumn: RelationshipFieldMetadata = entry[1]
linkEntries.push([entry[0], linkColumn])
}
}
return linkEntries.map(entry => ({
name: entry[0],
definition: entry[1],
}))
}
export function getTableIDList(
tables: Table[]
): { name: string; id: string }[] {
return tables
.filter(table => table.originalName && table._id)
.map(table => ({ id: table._id!, name: table.originalName! }))
}
export function updateFilterKeys(
filters: SearchFilters,
updates: { original: string; updated: string }[]
): SearchFilters {
const makeFilterKeyRegex = (str: string) =>
new RegExp(`^${str}\\.|:${str}\\.`)
for (let filter of Object.values(filters)) {
if (!isPlainObject(filter)) {
continue
}
for (let [key, keyFilter] of Object.entries(filter)) {
if (keyFilter === "") {
delete filter[key]
}
const possibleKey = updates.find(({ original }) =>
key.match(makeFilterKeyRegex(original))
)
if (possibleKey && possibleKey.original !== possibleKey.updated) {
// only replace the first, not replaceAll
filter[key.replace(possibleKey.original, possibleKey.updated)] =
filter[key]
delete filter[key]
}
}
}
return filters
}

60
packages/server/src/sdk/app/rows/search/sqs.ts
View File

@ -1,4 +1,5 @@
import {
Datasource,
DocumentType,
FieldType,
Operation,
@ -12,7 +13,6 @@ import {
SortType,
SqlClient,
Table,
Datasource,
} from "@budibase/types"
import {
buildInternalRelationships,
@ -30,6 +30,11 @@ import AliasTables from "../sqlAlias"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import pick from "lodash/pick"
import { processRowCountResponse } from "../utils"
import {
updateFilterKeys,
getRelationshipColumns,
getTableIDList,
} from "./filters"
const builder = new sql.Sql(SqlClient.SQL_LITE)
@ -60,34 +65,31 @@ function buildInternalFieldList(
return fieldList
}
function tableNameInFieldRegex(tableName: string) {
return new RegExp(`^${tableName}.|:${tableName}.`, "g")
}
function cleanupFilters(filters: SearchFilters, tables: Table[]) {
for (let filter of Object.values(filters)) {
if (typeof filter !== "object") {
continue
}
for (let [key, keyFilter] of Object.entries(filter)) {
if (keyFilter === "") {
delete filter[key]
}
// relationship, switch to table ID
const tableRelated = tables.find(
table =>
table.originalName &&
key.match(tableNameInFieldRegex(table.originalName))
function cleanupFilters(
filters: SearchFilters,
table: Table,
allTables: Table[]
) {
// get a list of all relationship columns in the table for updating
const relationshipColumns = getRelationshipColumns(table)
// get table names to ID map for relationships
const tableNameToID = getTableIDList(allTables)
// all should be applied at once
filters = updateFilterKeys(
filters,
relationshipColumns
.map(({ name, definition }) => ({
original: name,
updated: definition.tableId,
}))
.concat(
tableNameToID.map(({ name, id }) => ({
original: name,
updated: id,
}))
)
if (tableRelated && tableRelated.originalName) {
// only replace the first, not replaceAll
filter[key.replace(tableRelated.originalName, tableRelated._id!)] =
filter[key]
delete filter[key]
}
}
}
)
return filters
}
@ -176,7 +178,7 @@ export async function search(
operation: Operation.READ,
},
filters: {
...cleanupFilters(query, allTables),
...cleanupFilters(query, table, allTables),
documentType: DocumentType.ROW,
},
table,

6
packages/server/src/sdk/app/tables/getters.ts
View File

@ -90,10 +90,10 @@ export async function getExternalTable(
export async function getTable(tableId: string): Promise<Table> {
const db = context.getAppDB()
let output: Table
if (isExternalTableID(tableId)) {
if (tableId && isExternalTableID(tableId)) {
let { datasourceId, tableName } = breakExternalTableId(tableId)
const datasource = await datasources.get(datasourceId!)
const table = await getExternalTable(datasourceId!, tableName!)
const datasource = await datasources.get(datasourceId)
const table = await getExternalTable(datasourceId, tableName)
output = { ...table, sql: isSQL(datasource) }
} else {
output = await db.get<Table>(tableId)

12
packages/server/src/sdk/app/tables/internal/index.ts
View File

@ -17,6 +17,7 @@ import { cloneDeep } from "lodash/fp"
import isEqual from "lodash/isEqual"
import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula"
import { context } from "@budibase/backend-core"
import { findDuplicateInternalColumns } from "@budibase/shared-core"
import { getTable } from "../getters"
import { checkAutoColumns } from "./utils"
import * as viewsSdk from "../../views"
@ -44,6 +45,17 @@ export async function save(
if (hasTypeChanged(table, oldTable)) {
throw new Error("A column type has changed.")
}
// check for case sensitivity - we don't want to allow duplicated columns
const duplicateColumn = findDuplicateInternalColumns(table)
if (duplicateColumn.length) {
throw new Error(
`Column(s) "${duplicateColumn.join(
", "
)}" are duplicated - check for other columns with these name (case in-sensitive)`
)
}
// check that subtypes have been maintained
table = checkAutoColumns(table, oldTable)

24
packages/server/src/sdk/app/views/external.ts
View File

@ -10,9 +10,9 @@ export async function get(viewId: string): Promise<ViewV2> {
const { tableId } = utils.extractViewInfoFromID(viewId)
const { datasourceId, tableName } = breakExternalTableId(tableId)
const ds = await sdk.datasources.get(datasourceId!)
const ds = await sdk.datasources.get(datasourceId)
const table = ds.entities![tableName!]
const table = ds.entities![tableName]
const views = Object.values(table.views!).filter(isV2)
const found = views.find(v => v.id === viewId)
if (!found) {
@ -25,9 +25,9 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
const { tableId } = utils.extractViewInfoFromID(viewId)
const { datasourceId, tableName } = breakExternalTableId(tableId)
const ds = await sdk.datasources.get(datasourceId!)
const ds = await sdk.datasources.get(datasourceId)
const table = ds.entities![tableName!]
const table = ds.entities![tableName]
const views = Object.values(table.views!).filter(isV2)
const found = views.find(v => v.id === viewId)
if (!found) {
@ -49,9 +49,9 @@ export async function create(
const db = context.getAppDB()
const { datasourceId, tableName } = breakExternalTableId(tableId)
const ds = await sdk.datasources.get(datasourceId!)
ds.entities![tableName!].views ??= {}
ds.entities![tableName!].views![view.name] = view
const ds = await sdk.datasources.get(datasourceId)
ds.entities![tableName].views ??= {}
ds.entities![tableName].views![view.name] = view
await db.put(ds)
return view
}
@ -60,9 +60,9 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
const db = context.getAppDB()
const { datasourceId, tableName } = breakExternalTableId(tableId)
const ds = await sdk.datasources.get(datasourceId!)
ds.entities![tableName!].views ??= {}
const views = ds.entities![tableName!].views!
const ds = await sdk.datasources.get(datasourceId)
ds.entities![tableName].views ??= {}
const views = ds.entities![tableName].views!
const existingView = Object.values(views).find(
v => isV2(v) && v.id === view.id
@ -87,9 +87,9 @@ export async function remove(viewId: string): Promise<ViewV2> {
}
const { datasourceId, tableName } = breakExternalTableId(view.tableId)
const ds = await sdk.datasources.get(datasourceId!)
const ds = await sdk.datasources.get(datasourceId)
delete ds.entities![tableName!].views![view?.name]
delete ds.entities![tableName].views![view?.name]
await db.put(ds)
return view
}

3
packages/shared-core/src/constants/index.ts
View File

@ -1,5 +1,6 @@
export * from "./api"
export * from "./fields"
export * from "./rows"
export const OperatorOptions = {
Equals: {
@ -179,3 +180,5 @@ export enum BpmStatusValue {
VERIFYING_EMAIL = "verifying_email",
COMPLETED = "completed",
}
export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"

14
packages/shared-core/src/constants/rows.ts Normal file
View File

@ -0,0 +1,14 @@
export const CONSTANT_INTERNAL_ROW_COLS = [
"_id",
"_rev",
"type",
"createdAt",
"updatedAt",
"tableId",
] as const
export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const
export function isInternalColumnName(name: string): boolean {
return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name)
}

22
packages/shared-core/src/table.ts
View File

@ -1,4 +1,5 @@
import { FieldType } from "@budibase/types"
import { FieldType, Table } from "@budibase/types"
import { CONSTANT_INTERNAL_ROW_COLS } from "./constants"
const allowDisplayColumnByType: Record<FieldType, boolean> = {
[FieldType.STRING]: true,
@ -51,3 +52,22 @@ export function canBeDisplayColumn(type: FieldType): boolean {
export function canBeSortColumn(type: FieldType): boolean {
return !!allowSortColumnByType[type]
}
export function findDuplicateInternalColumns(table: Table): string[] {
// get the column names
const columnNames = Object.keys(table.schema)
.concat(CONSTANT_INTERNAL_ROW_COLS)
.map(colName => colName.toLowerCase())
// there are duplicates
const set = new Set(columnNames)
let duplicates: string[] = []
if (set.size !== columnNames.length) {
for (let key of set.keys()) {
const count = columnNames.filter(name => name === key).length
if (count > 1) {
duplicates.push(key)
}
}
}
return duplicates
}

34
yarn.lock
View File

@ -10296,7 +10296,7 @@ engine.io-parser@~5.0.3:
resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.0.6.tgz#7811244af173e157295dec9b2718dfe42a64ef45"
integrity sha512-tjuoZDMAdEhVnSFleYPCtdL2GXwVTGtNjoeJd9IhIG3C1xs9uwxqRNEu5WpnDZCaozwVlK/nuQhpodhXSIMaxw==
engine.io@~6.4.1:
engine.io@~6.4.2:
version "6.4.2"
resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-6.4.2.tgz#ffeaf68f69b1364b0286badddf15ff633476473f"
integrity sha512-FKn/3oMiJjrOEOeUub2WCox6JhxBXq/Zn3fZOMCBxKnNYtsdKjxhl7yR3fZhM9PV+rdE75SU5SYMc+2PGzo+Tg==
@ -20160,17 +20160,25 @@ socket.io-parser@~4.2.1:
"@socket.io/component-emitter" "~3.1.0"
debug "~4.3.1"
socket.io@4.6.1:
version "4.6.1"
resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.6.1.tgz#62ec117e5fce0692fa50498da9347cfb52c3bc70"
integrity sha512-KMcaAi4l/8+xEjkRICl6ak8ySoxsYG+gG6/XfRCPJPQ/haCRIJBTL4wIl8YCsmtaBovcAXGLOShyVWQ/FG8GZA==
socket.io-parser@~4.2.4:
version "4.2.4"
resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.4.tgz#c806966cf7270601e47469ddeec30fbdfda44c83"
integrity sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==
dependencies:
"@socket.io/component-emitter" "~3.1.0"
debug "~4.3.1"
socket.io@4.6.2:
version "4.6.2"
resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.6.2.tgz#d597db077d4df9cbbdfaa7a9ed8ccc3d49439786"
integrity sha512-Vp+lSks5k0dewYTfwgPT9UeGGd+ht7sCpB7p0e83VgO4X/AHYWhXITMrNk/pg8syY2bpx23ptClCQuHhqi2BgQ==
dependencies:
accepts "~1.3.4"
base64id "~2.0.0"
debug "~4.3.2"
engine.io "~6.4.1"
engine.io "~6.4.2"
socket.io-adapter "~2.5.2"
socket.io-parser "~4.2.1"
socket.io-parser "~4.2.4"
socks-proxy-agent@^7.0.0:
version "7.0.0"
@ -21102,18 +21110,6 @@ tar@6.1.11:
mkdirp "^1.0.3"
yallist "^4.0.0"
tar@6.1.15:
version "6.1.15"
resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.15.tgz#c9738b0b98845a3b344d334b8fa3041aaba53a69"
integrity sha512-/zKt9UyngnxIT/EAGYuxaMYgOIJiP81ab9ZfkILq4oNLPFX50qyYmu7jRj9qeXoxmJHjGlbH0+cm2uy1WCs10A==
dependencies:
chownr "^2.0.0"
fs-minipass "^2.0.0"
minipass "^5.0.0"
minizlib "^2.1.1"
mkdirp "^1.0.3"
yallist "^4.0.0"
tar@6.2.1, tar@^6.1.11, tar@^6.1.2:
version "6.2.1"
resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a"