MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix/automation-query-rows-table-with-spaces

This commit is contained in:
Michael Drury 2025-01-13 11:42:03 +00:00 committed by GitHub
parent 39f2ad021d 05b2c07b11
commit c39bda69c0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 1489 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
packages/backend-core/src/sql/sql.ts
View File

@ -272,17 +272,6 @@ class InternalBuilder {
return parts.join(".")
}
private isFullSelectStatementRequired(): boolean {
for (let column of Object.values(this.table.schema)) {
if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) {
return true
} else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) {
return true
}
}
return false
}
private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
const { table, resource } = this.query
@ -292,11 +281,9 @@ class InternalBuilder {
const alias = this.getTableName(table)
const schema = this.table.schema
if (!this.isFullSelectStatementRequired()) {
return [this.knex.raw("??", [`${alias}.*`])]
}
// get just the fields for this table
return resource.fields
const tableFields = resource.fields
.map(field => {
const parts = field.split(/\./g)
let table: string | undefined = undefined
@ -311,7 +298,8 @@ class InternalBuilder {
return { table, column, field }
})
.filter(({ table }) => !table || table === alias)
.map(({ table, column, field }) => {
return tableFields.map(({ table, column, field }) => {
const columnSchema = schema[column]
if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) {
@ -325,8 +313,6 @@ class InternalBuilder {
// Time gets returned as timestamp from mssql, not matching the expected
// HH:mm format
// TODO: figure out how to express this safely without string
// interpolation.
return this.knex.raw(`CONVERT(varchar, ??, 108) as ??`, [
this.rawQuotedIdentifier(field),
this.knex.raw(this.quote(field)),
@ -1291,6 +1277,7 @@ class InternalBuilder {
if (!toTable || !fromTable) {
continue
}
const relatedTable = tables[toTable]
if (!relatedTable) {
throw new Error(`related table "${toTable}" not found in datasource`)
@ -1319,6 +1306,10 @@ class InternalBuilder {
const fieldList = relationshipFields.map(field =>
this.buildJsonField(relatedTable, field)
)
if (!fieldList.length) {
continue
}
const fieldListFormatted = fieldList
.map(f => {
const separator = this.client === SqlClient.ORACLE ? " VALUE " : ","
@ -1359,7 +1350,9 @@ class InternalBuilder {
)
const standardWrap = (select: Knex.Raw): Knex.QueryBuilder => {
subQuery = subQuery.select(`${toAlias}.*`).limit(getRelationshipLimit())
subQuery = subQuery
.select(relationshipFields)
.limit(getRelationshipLimit())
// @ts-ignore - the from alias syntax isn't in Knex typing
return knex.select(select).from({
[toAlias]: subQuery,
@ -1589,11 +1582,12 @@ class InternalBuilder {
limits?: { base: number; query: number }
} = {}
): Knex.QueryBuilder {
let { operation, filters, paginate, relationships, table } = this.query
const { operation, filters, paginate, relationships, table } = this.query
const { limits } = opts
// start building the query
let query = this.qualifiedKnex()
// handle pagination
let foundOffset: number | null = null
let foundLimit = limits?.query || limits?.base
@ -1642,7 +1636,7 @@ class InternalBuilder {
const mainTable = this.query.tableAliases?.[table.name] || table.name
const cte = this.addSorting(
this.knex
.with("paginated", query)
.with("paginated", query.clone().clearSelect().select("*"))
.select(this.generateSelectStatement())
.from({
[mainTable]: "paginated",

5
packages/server/src/api/controllers/row/external.ts
View File

@ -45,6 +45,9 @@ export async function handleRequest<T extends Operation>(
export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const source = await utils.getSource(ctx)
const { viewId, tableId } = utils.getSourceId(ctx)
const sourceId = viewId || tableId
if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
ctx.throw(400, "Cannot update rows through a calculation view")
}
@ -86,7 +89,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
// The id might have been changed, so the refetching would fail. Recalculating the id just in case
const updatedId =
generateIdForRow({ ...beforeRow, ...dataToUpdate }, table) || _id
const row = await sdk.rows.external.getRow(table._id!, updatedId, {
const row = await sdk.rows.external.getRow(sourceId, updatedId, {
relationships: true,
})

121
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -14,7 +14,8 @@ import {
import { breakExternalTableId } from "../../../../integrations/utils"
import { generateJunctionTableID } from "../../../../db/utils"
import sdk from "../../../../sdk"
import { helpers } from "@budibase/shared-core"
import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
import { sql } from "@budibase/backend-core"
type TableMap = Record<string, Table>
@ -118,45 +119,131 @@ export async function buildSqlFieldList(
opts?: { relationships: boolean }
) {
const { relationships } = opts || {}
const nonMappedColumns = [FieldType.LINK, FieldType.FORMULA, FieldType.AI]
function extractRealFields(table: Table, existing: string[] = []) {
return Object.entries(table.schema)
.filter(
([columnName, column]) =>
column.type !== FieldType.LINK &&
column.type !== FieldType.FORMULA &&
column.type !== FieldType.AI &&
!existing.find(
(field: string) => field === `${table.name}.${columnName}`
!nonMappedColumns.includes(column.type) &&
!existing.find((field: string) => field === columnName)
)
.map(([columnName]) => columnName)
}
function getRequiredFields(table: Table, existing: string[] = []) {
const requiredFields: string[] = []
if (table.primary) {
requiredFields.push(...table.primary)
}
if (table.primaryDisplay) {
requiredFields.push(table.primaryDisplay)
}
if (!sql.utils.isExternalTable(table)) {
requiredFields.push(...PROTECTED_INTERNAL_COLUMNS)
}
return requiredFields.filter(
column =>
!existing.find((field: string) => field === column) &&
table.schema[column] &&
!nonMappedColumns.includes(table.schema[column].type)
)
.map(([columnName]) => `${table.name}.${columnName}`)
}
let fields: string[] = []
if (sdk.views.isView(source)) {
fields = Object.keys(helpers.views.basicFields(source))
} else {
fields = extractRealFields(source)
}
const isView = sdk.views.isView(source)
let table: Table
if (sdk.views.isView(source)) {
if (isView) {
table = await sdk.views.getTable(source.id)
fields = Object.keys(helpers.views.basicFields(source)).filter(
f => table.schema[f].type !== FieldType.LINK
)
} else {
table = source
fields = extractRealFields(source).filter(
f => table.schema[f].visible !== false
)
}
for (let field of Object.values(table.schema)) {
const containsFormula = (isView ? fields : Object.keys(table.schema)).some(
f => table.schema[f]?.type === FieldType.FORMULA
)
// If are requesting for a formula field, we need to retrieve all fields
if (containsFormula) {
fields = extractRealFields(table)
}
if (!isView || !helpers.views.isCalculationView(source)) {
fields.push(
...getRequiredFields(
{
...table,
primaryDisplay: source.primaryDisplay || table.primaryDisplay,
},
fields
)
)
}
fields = fields.map(c => `${table.name}.${c}`)
for (const field of Object.values(table.schema)) {
if (field.type !== FieldType.LINK || !relationships || !field.tableId) {
continue
}
if (
isView &&
(!source.schema?.[field.name] ||
!helpers.views.isVisible(source.schema[field.name])) &&
!containsFormula
) {
continue
}
const { tableName } = breakExternalTableId(field.tableId)
if (tables[tableName]) {
fields = fields.concat(extractRealFields(tables[tableName], fields))
const relatedTable = tables[tableName]
if (!relatedTable) {
continue
}
const viewFields = new Set<string>()
if (containsFormula) {
extractRealFields(relatedTable).forEach(f => viewFields.add(f))
} else {
relatedTable.primary?.forEach(f => viewFields.add(f))
if (relatedTable.primaryDisplay) {
viewFields.add(relatedTable.primaryDisplay)
}
if (isView) {
Object.entries(source.schema?.[field.name]?.columns || {})
.filter(
([columnName, columnConfig]) =>
relatedTable.schema[columnName] &&
helpers.views.isVisible(columnConfig) &&
![FieldType.LINK, FieldType.FORMULA].includes(
relatedTable.schema[columnName].type
)
)
.forEach(([field]) => viewFields.add(field))
}
}
return fields
const fieldsToAdd = Array.from(viewFields)
.filter(f => !nonMappedColumns.includes(relatedTable.schema[f].type))
.map(f => `${relatedTable.name}.${f}`)
.filter(f => !fields.includes(f))
fields.push(...fieldsToAdd)
}
return [...new Set(fields)]
}
export function isKnexEmptyReadResponse(resp: DatasourcePlusQueryResponse) {

511
packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts Normal file
View File

@ -0,0 +1,511 @@
import {
AIOperationEnum,
CalculationType,
FieldType,
RelationshipType,
SourceName,
Table,
ViewV2,
ViewV2Type,
} from "@budibase/types"
import { buildSqlFieldList } from "../sqlUtils"
import { structures } from "../../../../routes/tests/utilities"
import { sql } from "@budibase/backend-core"
import { generator } from "@budibase/backend-core/tests"
import { generateViewID } from "../../../../../db/utils"
import sdk from "../../../../../sdk"
import { cloneDeep } from "lodash"
import { utils } from "@budibase/shared-core"
jest.mock("../../../../../sdk/app/views", () => ({
...jest.requireActual("../../../../../sdk/app/views"),
getTable: jest.fn(),
}))
const getTableMock = sdk.views.getTable as jest.MockedFunction<
typeof sdk.views.getTable
>
describe("buildSqlFieldList", () => {
let allTables: Record<string, Table>
class TableConfig {
private _table: Table & { _id: string }
constructor(name: string) {
this._table = {
...structures.tableForDatasource({
type: "datasource",
source: SourceName.POSTGRES,
}),
name,
_id: sql.utils.buildExternalTableId("ds_id", name),
schema: {
name: {
name: "name",
type: FieldType.STRING,
},
description: {
name: "description",
type: FieldType.STRING,
},
amount: {
name: "amount",
type: FieldType.NUMBER,
},
},
}
allTables[name] = this._table
}
withHiddenField(field: string) {
this._table.schema[field].visible = false
return this
}
withField(
name: string,
type:
| FieldType.STRING
| FieldType.NUMBER
| FieldType.FORMULA
| FieldType.AI,
options?: { visible: boolean }
) {
switch (type) {
case FieldType.NUMBER:
case FieldType.STRING:
this._table.schema[name] = {
name,
type,
...options,
}
break
case FieldType.FORMULA:
this._table.schema[name] = {
name,
type,
formula: "any",
...options,
}
break
case FieldType.AI:
this._table.schema[name] = {
name,
type,
operation: AIOperationEnum.PROMPT,
...options,
}
break
default:
utils.unreachable(type)
}
return this
}
withRelation(name: string, toTableId: string) {
this._table.schema[name] = {
name,
type: FieldType.LINK,
relationshipType: RelationshipType.ONE_TO_MANY,
fieldName: "link",
tableId: toTableId,
}
return this
}
withPrimary(field: string) {
this._table.primary = [field]
return this
}
withDisplay(field: string) {
this._table.primaryDisplay = field
return this
}
create() {
return cloneDeep(this._table)
}
}
class ViewConfig {
private _table: Table
private _view: ViewV2
constructor(table: Table) {
this._table = table
this._view = {
version: 2,
id: generateViewID(table._id!),
name: generator.word(),
tableId: table._id!,
}
}
withVisible(field: string) {
this._view.schema ??= {}
this._view.schema[field] ??= {}
this._view.schema[field].visible = true
return this
}
withHidden(field: string) {
this._view.schema ??= {}
this._view.schema[field] ??= {}
this._view.schema[field].visible = false
return this
}
withRelationshipColumns(
field: string,
columns: Record<string, { visible: boolean }>
) {
this._view.schema ??= {}
this._view.schema[field] ??= {}
this._view.schema[field].columns = columns
return this
}
withCalculation(
name: string,
field: string,
calculationType: CalculationType
) {
this._view.type = ViewV2Type.CALCULATION
this._view.schema ??= {}
this._view.schema[name] = {
field,
calculationType,
visible: true,
}
return this
}
create() {
getTableMock.mockResolvedValueOnce(this._table)
return cloneDeep(this._view)
}
}
beforeEach(() => {
jest.clearAllMocks()
allTables = {}
})
describe("table", () => {
it("extracts fields from table schema", async () => {
const table = new TableConfig("table").create()
const result = await buildSqlFieldList(table, {})
expect(result).toEqual([
"table.name",
"table.description",
"table.amount",
])
})
it("excludes hidden fields", async () => {
const table = new TableConfig("table")
.withHiddenField("description")
.create()
const result = await buildSqlFieldList(table, {})
expect(result).toEqual(["table.name", "table.amount"])
})
it("excludes non-sql fields fields", async () => {
const table = new TableConfig("table")
.withField("formula", FieldType.FORMULA)
.withField("ai", FieldType.AI)
.withRelation("link", "otherTableId")
.create()
const result = await buildSqlFieldList(table, {})
expect(result).toEqual([
"table.name",
"table.description",
"table.amount",
])
})
it("includes hidden fields if there is a formula column", async () => {
const table = new TableConfig("table")
.withHiddenField("description")
.withField("formula", FieldType.FORMULA)
.create()
const result = await buildSqlFieldList(table, {})
expect(result).toEqual([
"table.name",
"table.description",
"table.amount",
])
})
it("includes relationships fields when flagged", async () => {
const otherTable = new TableConfig("linkedTable")
.withField("id", FieldType.NUMBER)
.withPrimary("id")
.withDisplay("name")
.create()
const table = new TableConfig("table")
.withRelation("link", otherTable._id)
.create()
const result = await buildSqlFieldList(table, allTables, {
relationships: true,
})
expect(result).toEqual([
"table.name",
"table.description",
"table.amount",
"linkedTable.id",
"linkedTable.name",
])
})
it("includes all relationship fields if there is a formula column", async () => {
const otherTable = new TableConfig("linkedTable")
.withField("hidden", FieldType.STRING, { visible: false })
.create()
const table = new TableConfig("table")
.withRelation("link", otherTable._id)
.withField("formula", FieldType.FORMULA)
.create()
const result = await buildSqlFieldList(table, allTables, {
relationships: true,
})
expect(result).toEqual([
"table.name",
"table.description",
"table.amount",
"linkedTable.name",
"linkedTable.description",
"linkedTable.amount",
"linkedTable.hidden",
])
})
it("never includes non-sql columns from relationships", async () => {
const otherTable = new TableConfig("linkedTable")
.withField("id", FieldType.NUMBER)
.withField("hidden", FieldType.STRING, { visible: false })
.withField("formula", FieldType.FORMULA)
.withField("ai", FieldType.AI)
.withRelation("link", "otherTableId")
.create()
const table = new TableConfig("table")
.withRelation("link", otherTable._id)
.withField("formula", FieldType.FORMULA)
.create()
const result = await buildSqlFieldList(table, allTables, {
relationships: true,
})
expect(result).toEqual([
"table.name",
"table.description",
"table.amount",
"linkedTable.name",
"linkedTable.description",
"linkedTable.amount",
"linkedTable.id",
"linkedTable.hidden",
])
})
})
describe("view", () => {
it("extracts fields from table schema", async () => {
const view = new ViewConfig(new TableConfig("table").create())
.withVisible("amount")
.withHidden("name")
.create()
const result = await buildSqlFieldList(view, {})
expect(result).toEqual(["table.amount"])
})
it("includes all fields if there is a formula column", async () => {
const table = new TableConfig("table")
.withField("formula", FieldType.FORMULA)
.create()
const view = new ViewConfig(table)
.withHidden("name")
.withVisible("amount")
.withVisible("formula")
.create()
const result = await buildSqlFieldList(view, {})
expect(result).toEqual([
"table.name",
"table.description",
"table.amount",
])
})
it("does not includes all fields if the formula column is not included", async () => {
const table = new TableConfig("table")
.withField("formula", FieldType.FORMULA)
.create()
const view = new ViewConfig(table)
.withHidden("name")
.withVisible("amount")
.withHidden("formula")
.create()
const result = await buildSqlFieldList(view, {})
expect(result).toEqual(["table.amount"])
})
it("includes relationships columns", async () => {
const otherTable = new TableConfig("linkedTable")
.withField("id", FieldType.NUMBER)
.withField("formula", FieldType.FORMULA)
.withPrimary("id")
.create()
const table = new TableConfig("table")
.withRelation("link", otherTable._id)
.create()
const view = new ViewConfig(table)
.withVisible("name")
.withVisible("link")
.withRelationshipColumns("link", {
name: { visible: false },
amount: { visible: true },
formula: { visible: false },
})
.create()
const result = await buildSqlFieldList(view, allTables, {
relationships: true,
})
expect(result).toEqual([
"table.name",
"linkedTable.id",
"linkedTable.amount",
])
})
it("excludes relationships fields when view is not included in the view", async () => {
const otherTable = new TableConfig("linkedTable")
.withField("id", FieldType.NUMBER)
.withPrimary("id")
.withDisplay("name")
.create()
const table = new TableConfig("table")
.withRelation("link", otherTable._id)
.withField("formula", FieldType.FORMULA)
.create()
const view = new ViewConfig(table)
.withVisible("name")
.withHidden("amount")
.create()
const result = await buildSqlFieldList(view, allTables, {
relationships: true,
})
expect(result).toEqual(["table.name"])
})
it("does not include relationships columns for hidden links", async () => {
const otherTable = new TableConfig("linkedTable")
.withField("id", FieldType.NUMBER)
.withField("formula", FieldType.FORMULA)
.withPrimary("id")
.create()
const table = new TableConfig("table")
.withRelation("link", otherTable._id)
.create()
const view = new ViewConfig(table)
.withVisible("name")
.withHidden("link")
.withRelationshipColumns("link", {
name: { visible: false },
amount: { visible: true },
formula: { visible: false },
})
.create()
const result = await buildSqlFieldList(view, allTables, {
relationships: true,
})
expect(result).toEqual(["table.name"])
})
it("includes all relationship fields if there is a formula column", async () => {
const otherTable = new TableConfig("linkedTable")
.withField("id", FieldType.NUMBER)
.withField("hidden", FieldType.STRING, { visible: false })
.withField("formula", FieldType.FORMULA)
.withField("ai", FieldType.AI)
.withRelation("link", "otherTableId")
.withPrimary("id")
.create()
const table = new TableConfig("table")
.withRelation("link", otherTable._id)
.withField("formula", FieldType.FORMULA)
.create()
const view = new ViewConfig(table)
.withVisible("name")
.withVisible("formula")
.withHidden("link")
.withRelationshipColumns("link", {
name: { visible: false },
amount: { visible: true },
formula: { visible: false },
})
.create()
const result = await buildSqlFieldList(view, allTables, {
relationships: true,
})
expect(result).toEqual([
"table.name",
"table.description",
"table.amount",
"linkedTable.name",
"linkedTable.description",
"linkedTable.amount",
"linkedTable.id",
"linkedTable.hidden",
])
})
})
describe("calculation view", () => {
it("does not include calculation fields", async () => {
const view = new ViewConfig(new TableConfig("table").create())
.withCalculation("average", "amount", CalculationType.AVG)
.create()
const result = await buildSqlFieldList(view, {})
expect(result).toEqual([])
})
it("includes visible fields calculation fields", async () => {
const view = new ViewConfig(new TableConfig("table").create())
.withCalculation("average", "amount", CalculationType.AVG)
.withHidden("name")
.withVisible("amount")
.create()
const result = await buildSqlFieldList(view, {})
expect(result).toEqual(["table.amount"])
})
})
})

2
packages/server/src/integration-test/postgres.spec.ts
View File

@ -13,7 +13,7 @@ const mainDescriptions = datasourceDescribe({
if (mainDescriptions.length) {
describe.each(mainDescriptions)(
"/postgres integrations",
"/postgres integrations ($dbName)",
({ config, dsProvider }) => {
let datasource: Datasource
let client: Knex

35
packages/server/src/integrations/tests/sqlAlias.spec.ts
View File

@ -73,6 +73,27 @@ describe("Captures of real examples", () => {
})
describe("read", () => {
it("should retrieve all fields if non are specified", () => {
const queryJson = getJson("basicFetch.json")
delete queryJson.resource
let query = new Sql(SqlClient.POSTGRES)._query(queryJson)
expect(query).toEqual({
bindings: [primaryLimit],
sql: `select * from "persons" as "a" order by "a"."firstname" asc nulls first, "a"."personid" asc limit $1`,
})
})
it("should retrieve only requested fields", () => {
const queryJson = getJson("basicFetch.json")
let query = new Sql(SqlClient.POSTGRES)._query(queryJson)
expect(query).toEqual({
bindings: [primaryLimit],
sql: `select "a"."year", "a"."firstname", "a"."personid", "a"."age", "a"."type", "a"."lastname" from "persons" as "a" order by "a"."firstname" asc nulls first, "a"."personid" asc limit $1`,
})
})
it("should handle basic retrieval with relationships", () => {
const queryJson = getJson("basicFetchWithRelationships.json")
let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
@ -112,9 +133,9 @@ describe("Captures of real examples", () => {
bindings: [primaryLimit, relationshipLimit],
sql: expect.stringContaining(
multiline(
`with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1)
select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname"))
from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks"
`with "paginated" as (select * from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1)
select "a"."productname", "a"."productid", (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname"))
from (select "b"."executorid", "b"."qaid", "b"."taskid", "b"."completed", "b"."taskname" from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks"
from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc`
)
),
@ -130,9 +151,9 @@ describe("Captures of real examples", () => {
expect(query).toEqual({
bindings: [...filters, relationshipLimit, relationshipLimit],
sql: multiline(
`with "paginated" as (select "a".* from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3)
select "a".*, (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname"))
from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid"
`with "paginated" as (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3)
select "a"."executorid", "a"."taskname", "a"."taskid", "a"."completed", "a"."qaid", (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname"))
from (select "b"."productid", "b"."productname" from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid"
where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc`
),
})
@ -209,7 +230,7 @@ describe("Captures of real examples", () => {
bindings: ["ddd", ""],
sql: multiline(`delete from "compositetable" as "a"
where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE)
returning "a".*`),
returning "a"."keyparttwo", "a"."keypartone", "a"."name"`),
})
})
})

137
packages/server/src/integrations/tests/sqlQueryJson/basicFetch.json Normal file
View File

@ -0,0 +1,137 @@
{
"operation": "READ",
"resource": {
"fields": [
"a.year",
"a.firstname",
"a.personid",
"a.age",
"a.type",
"a.lastname"
]
},
"filters": {},
"sort": {
"firstname": {
"direction": "ascending"
}
},
"paginate": {
"limit": 100,
"page": 1
},
"relationships": [],
"extra": {
"idFilter": {}
},
"table": {
"type": "table",
"_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
"primary": ["personid"],
"name": "persons",
"schema": {
"year": {
"type": "number",
"externalType": "integer",
"autocolumn": false,
"name": "year",
"constraints": {
"presence": false
}
},
"firstname": {
"type": "string",
"externalType": "character varying",
"autocolumn": false,
"name": "firstname",
"constraints": {
"presence": false
}
},
"personid": {
"type": "number",
"externalType": "integer",
"autocolumn": true,
"name": "personid",
"constraints": {
"presence": false
}
},
"address": {
"type": "string",
"externalType": "character varying",
"autocolumn": false,
"name": "address",
"constraints": {
"presence": false
}
},
"age": {
"type": "number",
"externalType": "integer",
"autocolumn": false,
"name": "age",
"constraints": {
"presence": false
}
},
"type": {
"type": "options",
"externalType": "USER-DEFINED",
"autocolumn": false,
"name": "type",
"constraints": {
"presence": false,
"inclusion": ["support", "designer", "programmer", "qa"]
}
},
"city": {
"type": "string",
"externalType": "character varying",
"autocolumn": false,
"name": "city",
"constraints": {
"presence": false
}
},
"lastname": {
"type": "string",
"externalType": "character varying",
"autocolumn": false,
"name": "lastname",
"constraints": {
"presence": false
}
},
"QA": {
"tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
"name": "QA",
"relationshipType": "many-to-one",
"fieldName": "qaid",
"type": "link",
"main": true,
"_id": "ccb68481c80c34217a4540a2c6c27fe46",
"foreignKey": "personid"
},
"executor": {
"tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
"name": "executor",
"relationshipType": "many-to-one",
"fieldName": "executorid",
"type": "link",
"main": true,
"_id": "c89530b9770d94bec851e062b5cff3001",
"foreignKey": "personid",
"tableName": "persons"
}
},
"sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
"sourceType": "external",
"primaryDisplay": "firstname",
"views": {}
},
"tableAliases": {
"persons": "a",
"tasks": "b"
}
}

65
packages/server/src/sdk/app/rows/search/internal/sqs.ts
View File

@ -55,32 +55,44 @@ const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`)
const MISSING_TABLE_REGX = new RegExp(`no such table: .+`)
const DUPLICATE_COLUMN_REGEX = new RegExp(`duplicate column name: .+`)
async function buildInternalFieldList(
export async function buildInternalFieldList(
source: Table | ViewV2,
tables: Table[],
opts?: { relationships?: RelationshipsJson[]; allowedFields?: string[] }
opts?: {
relationships?: RelationshipsJson[]
allowedFields?: string[]
includeHiddenFields?: boolean
}
) {
const { relationships, allowedFields } = opts || {}
const { relationships, allowedFields, includeHiddenFields } = opts || {}
let schemaFields: string[] = []
if (sdk.views.isView(source)) {
schemaFields = Object.keys(helpers.views.basicFields(source))
} else {
schemaFields = Object.keys(source.schema).filter(
key => source.schema[key].visible !== false
)
}
if (allowedFields) {
schemaFields = schemaFields.filter(field => allowedFields.includes(field))
}
const isView = sdk.views.isView(source)
let table: Table
if (sdk.views.isView(source)) {
if (isView) {
table = await sdk.views.getTable(source.id)
} else {
table = source
}
if (isView) {
schemaFields = Object.keys(helpers.views.basicFields(source))
} else {
schemaFields = Object.keys(source.schema).filter(
key => includeHiddenFields || source.schema[key].visible !== false
)
}
const containsFormula = schemaFields.some(
f => table.schema[f]?.type === FieldType.FORMULA
)
// If are requesting for a formula field, we need to retrieve all fields
if (containsFormula) {
schemaFields = Object.keys(table.schema)
} else if (allowedFields) {
schemaFields = schemaFields.filter(field => allowedFields.includes(field))
}
let fieldList: string[] = []
const getJunctionFields = (relatedTable: Table, fields: string[]) => {
const junctionFields: string[] = []
@ -101,10 +113,12 @@ async function buildInternalFieldList(
}
for (let key of schemaFields) {
const col = table.schema[key]
const isRelationship = col.type === FieldType.LINK
if (!relationships && isRelationship) {
continue
}
if (!isRelationship) {
fieldList.push(`${table._id}.${mapToUserColumn(key)}`)
} else {
@ -113,8 +127,17 @@ async function buildInternalFieldList(
if (!relatedTable) {
continue
}
// a quirk of how junction documents work in Budibase, refer to the "LinkDocument" type to see the full
// structure - essentially all relationships between two tables will be inserted into a single "table"
// we don't use an independent junction table ID for each separate relationship between two tables. For
// example if we have table A and B, with two relationships between them, all the junction documents will
// end up in the same junction table ID. We need to retrieve the field name property of the junction documents
// as part of the relationship to tell us which relationship column the junction is related to.
const relatedFields = (
await buildInternalFieldList(relatedTable, tables)
await buildInternalFieldList(relatedTable, tables, {
includeHiddenFields: containsFormula,
})
).concat(
getJunctionFields(relatedTable, ["doc1.fieldName", "doc2.fieldName"])
)
@ -125,6 +148,13 @@ async function buildInternalFieldList(
fieldList = fieldList.concat(relatedFields)
}
}
if (!isView || !helpers.views.isCalculationView(source)) {
for (const field of PROTECTED_INTERNAL_COLUMNS) {
fieldList.push(`${table._id}.${field}`)
}
}
return [...new Set(fieldList)]
}
@ -323,8 +353,9 @@ export async function search(
}
let aggregations: Aggregation[] = []
if (sdk.views.isView(source)) {
if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
const calculationFields = helpers.views.calculationFields(source)
for (const [key, field] of Object.entries(calculationFields)) {
if (options.fields && !options.fields.includes(key)) {
continue

618
packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts Normal file
View File

@ -0,0 +1,618 @@
import {
AIOperationEnum,
CalculationType,
FieldType,
RelationshipType,
SourceName,
Table,
ViewV2,
ViewV2Type,
} from "@budibase/types"
import { buildInternalFieldList } from "../sqs"
import { structures } from "../../../../../../api/routes/tests/utilities"
import { sql } from "@budibase/backend-core"
import { generator } from "@budibase/backend-core/tests"
import {
generateJunctionTableID,
generateViewID,
} from "../../../../../../db/utils"
import sdk from "../../../../../../sdk"
import { cloneDeep } from "lodash"
import { utils } from "@budibase/shared-core"
jest.mock("../../../../../../sdk/app/views", () => ({
...jest.requireActual("../../../../../../sdk/app/views"),
getTable: jest.fn(),
}))
const getTableMock = sdk.views.getTable as jest.MockedFunction<
typeof sdk.views.getTable
>
describe("buildInternalFieldList", () => {
let allTables: Table[]
class TableConfig {
private _table: Table & { _id: string }
constructor() {
const name = generator.word()
this._table = {
...structures.tableForDatasource({
type: "datasource",
source: SourceName.POSTGRES,
}),
name,
_id: sql.utils.buildExternalTableId("ds_id", name),
schema: {
name: {
name: "name",
type: FieldType.STRING,
},
description: {
name: "description",
type: FieldType.STRING,
},
amount: {
name: "amount",
type: FieldType.NUMBER,
},
},
}
allTables.push(this._table)
}
withHiddenField(field: string) {
this._table.schema[field].visible = false
return this
}
withField(
name: string,
type:
| FieldType.STRING
| FieldType.NUMBER
| FieldType.FORMULA
| FieldType.AI,
options?: { visible: boolean }
) {
switch (type) {
case FieldType.NUMBER:
case FieldType.STRING:
this._table.schema[name] = {
name,
type,
...options,
}
break
case FieldType.FORMULA:
this._table.schema[name] = {
name,
type,
formula: "any",
...options,
}
break
case FieldType.AI:
this._table.schema[name] = {
name,
type,
operation: AIOperationEnum.PROMPT,
...options,
}
break
default:
utils.unreachable(type)
}
return this
}
withRelation(name: string, toTableId: string) {
this._table.schema[name] = {
name,
type: FieldType.LINK,
relationshipType: RelationshipType.ONE_TO_MANY,
fieldName: "link",
tableId: toTableId,
}
return this
}
withEmptySchema() {
this._table.schema = {}
return this
}
create() {
return cloneDeep(this._table)
}
}
class ViewConfig {
private _table: Table
private _view: ViewV2
constructor(table: Table) {
this._table = table
this._view = {
version: 2,
id: generateViewID(table._id!),
name: generator.word(),
tableId: table._id!,
}
}
withVisible(field: string) {
this._view.schema ??= {}
this._view.schema[field] ??= {}
this._view.schema[field].visible = true
return this
}
withHidden(field: string) {
this._view.schema ??= {}
this._view.schema[field] ??= {}
this._view.schema[field].visible = false
return this
}
withRelationshipColumns(
field: string,
columns: Record<string, { visible: boolean }>
) {
this._view.schema ??= {}
this._view.schema[field] ??= {}
this._view.schema[field].columns = columns
return this
}
withCalculation(
name: string,
field: string,
calculationType: CalculationType
) {
this._view.type = ViewV2Type.CALCULATION
this._view.schema ??= {}
this._view.schema[name] = {
field,
calculationType,
visible: true,
}
return this
}
create() {
getTableMock.mockResolvedValueOnce(this._table)
return cloneDeep(this._view)
}
}
beforeEach(() => {
jest.clearAllMocks()
allTables = []
})
describe("table", () => {
it("includes internal columns by default", async () => {
const table = new TableConfig().withEmptySchema().create()
const result = await buildInternalFieldList(table, [])
expect(result).toEqual([
`${table._id}._id`,
`${table._id}._rev`,
`${table._id}.type`,
`${table._id}.createdAt`,
`${table._id}.updatedAt`,
`${table._id}.tableId`,
])
})
it("extracts fields from table schema", async () => {
const table = new TableConfig().create()
const result = await buildInternalFieldList(table, [])
expect(result).toEqual([
`${table._id}.data_name`,
`${table._id}.data_description`,
`${table._id}.data_amount`,
`${table._id}._id`,
`${table._id}._rev`,
`${table._id}.type`,
`${table._id}.createdAt`,
`${table._id}.updatedAt`,
`${table._id}.tableId`,
])
})
it("excludes hidden fields", async () => {
const table = new TableConfig().withHiddenField("description").create()
const result = await buildInternalFieldList(table, [])
expect(result).toEqual([
`${table._id}.data_name`,
`${table._id}.data_amount`,
`${table._id}._id`,
`${table._id}._rev`,
`${table._id}.type`,
`${table._id}.createdAt`,
`${table._id}.updatedAt`,
`${table._id}.tableId`,
])
})
it("includes hidden fields if there is a formula column", async () => {
const table = new TableConfig()
.withHiddenField("description")
.withField("formula", FieldType.FORMULA)
.create()
const result = await buildInternalFieldList(table, [])
expect(result).toEqual([
`${table._id}.data_name`,
`${table._id}.data_description`,
`${table._id}.data_amount`,
`${table._id}.data_formula`,
`${table._id}._id`,
`${table._id}._rev`,
`${table._id}.type`,
`${table._id}.createdAt`,
`${table._id}.updatedAt`,
`${table._id}.tableId`,
])
})
it("includes relationships fields when flagged", async () => {
const otherTable = new TableConfig()
.withHiddenField("description")
.create()
const table = new TableConfig()
.withHiddenField("amount")
.withRelation("link", otherTable._id)
.create()
const relationships = [{ tableName: otherTable.name, column: "link" }]
const result = await buildInternalFieldList(table, allTables, {
relationships,
})
expect(result).toEqual([
`${table._id}.data_name`,
`${table._id}.data_description`,
`${otherTable._id}.data_name`,
`${otherTable._id}.data_amount`,
`${otherTable._id}._id`,
`${otherTable._id}._rev`,
`${otherTable._id}.type`,
`${otherTable._id}.createdAt`,
`${otherTable._id}.updatedAt`,
`${otherTable._id}.tableId`,
`${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
`${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
`${table._id}._id`,
`${table._id}._rev`,
`${table._id}.type`,
`${table._id}.createdAt`,
`${table._id}.updatedAt`,
`${table._id}.tableId`,
])
})
it("includes all relationship fields if there is a formula column", async () => {
const otherTable = new TableConfig()
.withField("hidden", FieldType.STRING, { visible: false })
.create()
const table = new TableConfig()
.withRelation("link", otherTable._id)
.withHiddenField("description")
.withField("formula", FieldType.FORMULA)
.create()
const relationships = [{ tableName: otherTable.name, column: "link" }]
const result = await buildInternalFieldList(table, allTables, {
relationships,
})
expect(result).toEqual([
`${table._id}.data_name`,
`${table._id}.data_description`,
`${table._id}.data_amount`,
`${otherTable._id}.data_name`,
`${otherTable._id}.data_description`,
`${otherTable._id}.data_amount`,
`${otherTable._id}.data_hidden`,
`${otherTable._id}._id`,
`${otherTable._id}._rev`,
`${otherTable._id}.type`,
`${otherTable._id}.createdAt`,
`${otherTable._id}.updatedAt`,
`${otherTable._id}.tableId`,
`${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
`${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
`${table._id}.data_formula`,
`${table._id}._id`,
`${table._id}._rev`,
`${table._id}.type`,
`${table._id}.createdAt`,
`${table._id}.updatedAt`,
`${table._id}.tableId`,
])
})
})
describe("view", () => {
it("includes internal columns by default", async () => {
const view = new ViewConfig(new TableConfig().create()).create()
const result = await buildInternalFieldList(view, [])
expect(result).toEqual([
`${view.tableId}._id`,
`${view.tableId}._rev`,
`${view.tableId}.type`,
`${view.tableId}.createdAt`,
`${view.tableId}.updatedAt`,
`${view.tableId}.tableId`,
])
})
it("extracts fields from table schema", async () => {
const view = new ViewConfig(new TableConfig().create())
.withVisible("amount")
.withHidden("name")
.create()
const result = await buildInternalFieldList(view, [])
expect(result).toEqual([
`${view.tableId}.data_amount`,
`${view.tableId}._id`,
`${view.tableId}._rev`,
`${view.tableId}.type`,
`${view.tableId}.createdAt`,
`${view.tableId}.updatedAt`,
`${view.tableId}.tableId`,
])
})
it("includes all fields if there is a formula column", async () => {
const table = new TableConfig()
.withField("formula", FieldType.FORMULA)
.create()
const view = new ViewConfig(table)
.withHidden("name")
.withVisible("amount")
.withVisible("formula")
.create()
const result = await buildInternalFieldList(view, [])
expect(result).toEqual([
`${view.tableId}.data_name`,
`${view.tableId}.data_description`,
`${view.tableId}.data_amount`,
`${view.tableId}.data_formula`,
`${view.tableId}._id`,
`${view.tableId}._rev`,
`${view.tableId}.type`,
`${view.tableId}.createdAt`,
`${view.tableId}.updatedAt`,
`${view.tableId}.tableId`,
])
})
it("does not includes all fields if the formula column is not included", async () => {
const table = new TableConfig()
.withField("formula", FieldType.FORMULA)
.create()
const view = new ViewConfig(table)
.withHidden("name")
.withVisible("amount")
.withHidden("formula")
.create()
const result = await buildInternalFieldList(view, [])
expect(result).toEqual([
`${view.tableId}.data_amount`,
`${view.tableId}._id`,
`${view.tableId}._rev`,
`${view.tableId}.type`,
`${view.tableId}.createdAt`,
`${view.tableId}.updatedAt`,
`${view.tableId}.tableId`,
])
})
it("includes relationships fields", async () => {
const otherTable = new TableConfig().create()
const table = new TableConfig()
.withRelation("link", otherTable._id)
.withField("formula", FieldType.FORMULA)
.create()
const view = new ViewConfig(table)
.withVisible("name")
.withVisible("link")
.withHidden("amount")
.create()
const relationships = [{ tableName: otherTable.name, column: "link" }]
const result = await buildInternalFieldList(view, allTables, {
relationships,
})
expect(result).toEqual([
`${table._id}.data_name`,
`${otherTable._id}.data_name`,
`${otherTable._id}.data_description`,
`${otherTable._id}.data_amount`,
`${otherTable._id}._id`,
`${otherTable._id}._rev`,
`${otherTable._id}.type`,
`${otherTable._id}.createdAt`,
`${otherTable._id}.updatedAt`,
`${otherTable._id}.tableId`,
`${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
`${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
`${table._id}._id`,
`${table._id}._rev`,
`${table._id}.type`,
`${table._id}.createdAt`,
`${table._id}.updatedAt`,
`${table._id}.tableId`,
])
})
it("includes relationships columns", async () => {
const otherTable = new TableConfig()
.withField("formula", FieldType.FORMULA)
.create()
const table = new TableConfig()
.withRelation("link", otherTable._id)
.create()
const view = new ViewConfig(table)
.withVisible("name")
.withVisible("link")
.withRelationshipColumns("link", {
name: { visible: false },
amount: { visible: true },
formula: { visible: false },
})
.create()
const relationships = [{ tableName: otherTable.name, column: "link" }]
const result = await buildInternalFieldList(view, allTables, {
relationships,
})
expect(result).toEqual([
`${table._id}.data_name`,
`${otherTable._id}.data_name`,
`${otherTable._id}.data_description`,
`${otherTable._id}.data_amount`,
`${otherTable._id}.data_formula`,
`${otherTable._id}._id`,
`${otherTable._id}._rev`,
`${otherTable._id}.type`,
`${otherTable._id}.createdAt`,
`${otherTable._id}.updatedAt`,
`${otherTable._id}.tableId`,
`${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
`${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
`${table._id}._id`,
`${table._id}._rev`,
`${table._id}.type`,
`${table._id}.createdAt`,
`${table._id}.updatedAt`,
`${table._id}.tableId`,
])
})
it("does not include relationships columns for hidden links", async () => {
const otherTable = new TableConfig()
.withField("formula", FieldType.FORMULA)
.create()
const table = new TableConfig()
.withRelation("link", otherTable._id)
.create()
const view = new ViewConfig(table)
.withVisible("name")
.withHidden("link")
.withRelationshipColumns("link", {
name: { visible: false },
amount: { visible: true },
formula: { visible: false },
})
.create()
const relationships = [{ tableName: otherTable.name, column: "link" }]
const result = await buildInternalFieldList(view, allTables, {
relationships,
})
expect(result).toEqual([
`${table._id}.data_name`,
`${table._id}._id`,
`${table._id}._rev`,
`${table._id}.type`,
`${table._id}.createdAt`,
`${table._id}.updatedAt`,
`${table._id}.tableId`,
])
})
it("includes all relationship fields if there is a formula column", async () => {
const otherTable = new TableConfig()
.withField("hidden", FieldType.STRING, { visible: false })
.withField("formula", FieldType.FORMULA)
.withField("ai", FieldType.AI)
.withRelation("link", "otherTableId")
.create()
const table = new TableConfig()
.withRelation("link", otherTable._id)
.withField("formula", FieldType.FORMULA)
.create()
const view = new ViewConfig(table)
.withVisible("name")
.withVisible("formula")
.withHidden("link")
.withRelationshipColumns("link", {
name: { visible: false },
amount: { visible: true },
formula: { visible: false },
})
.create()
const relationships = [{ tableName: otherTable.name, column: "link" }]
const result = await buildInternalFieldList(view, allTables, {
relationships,
})
expect(result).toEqual([
`${table._id}.data_name`,
`${table._id}.data_description`,
`${table._id}.data_amount`,
`${otherTable._id}.data_name`,
`${otherTable._id}.data_description`,
`${otherTable._id}.data_amount`,
`${otherTable._id}.data_hidden`,
`${otherTable._id}.data_formula`,
`${otherTable._id}.data_ai`,
`${otherTable._id}._id`,
`${otherTable._id}._rev`,
`${otherTable._id}.type`,
`${otherTable._id}.createdAt`,
`${otherTable._id}.updatedAt`,
`${otherTable._id}.tableId`,
`${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
`${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
`${table._id}.data_formula`,
`${table._id}._id`,
`${table._id}._rev`,
`${table._id}.type`,
`${table._id}.createdAt`,
`${table._id}.updatedAt`,
`${table._id}.tableId`,
])
})
})
describe("calculation view", () => {
it("does not include calculation fields", async () => {
const view = new ViewConfig(new TableConfig().create())
.withCalculation("average", "amount", CalculationType.AVG)
.create()
const result = await buildInternalFieldList(view, [])
expect(result).toEqual([])
})
it("includes visible fields calculation fields", async () => {
const view = new ViewConfig(new TableConfig().create())
.withCalculation("average", "amount", CalculationType.AVG)
.withHidden("name")
.withVisible("amount")
.create()
const result = await buildInternalFieldList(view, [])
expect(result).toEqual([`${view.tableId}.data_amount`])
})
})
})