diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index 32124fa79d..7beadabd89 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -192,10 +192,10 @@ export function buildSqlFieldList( function extractRealFields(table: Table, existing: string[] = []) { return Object.entries(table.schema) .filter( - column => - column[1].type !== FieldType.LINK && - column[1].type !== FieldType.FORMULA && - !existing.find((field: string) => field === column[0]) + ([columnName, column]) => + column.type !== FieldType.LINK && + column.type !== FieldType.FORMULA && + !existing.find((field: string) => field === columnName) ) .map(column => `${table.name}.${column[0]}`) } diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 9351976d66..95e714fbd6 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1664,7 +1664,7 @@ describe.each([ isInternal && describe("attachments and signatures", () => { const coreAttachmentEnrichment = async ( - schema: any, + schema: TableSchema, field: string, attachmentCfg: string | string[] ) => { @@ -1691,7 +1691,7 @@ describe.each([ await withEnv({ SELF_HOSTED: "true" }, async () => { return context.doInAppContext(config.getAppId(), async () => { - const enriched: Row[] = await outputProcessing(table, [row]) + const enriched: Row[] = await outputProcessing(testTable, [row]) const [targetRow] = enriched const attachmentEntries = Array.isArray(targetRow[field]) ? targetRow[field] diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index e18cffeaa8..da38112e2b 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -30,6 +30,7 @@ import { withEnv as withCoreEnv, setEnv as setCoreEnv, } from "@budibase/backend-core" +import sdk from "../../../sdk" describe.each([ ["lucene", undefined], @@ -120,6 +121,7 @@ describe.each([ }) beforeEach(() => { + jest.clearAllMocks() mocks.licenses.useCloudFree() }) @@ -1602,6 +1604,28 @@ describe.each([ }) expect(response.rows).toHaveLength(0) }) + + it("queries the row api passing the view fields only", async () => { + const searchSpy = jest.spyOn(sdk.rows, "search") + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + id: { visible: true }, + one: { visible: false }, + }, + }) + + await config.api.viewV2.search(view.id, { query: {} }) + expect(searchSpy).toHaveBeenCalledTimes(1) + + expect(searchSpy).toHaveBeenCalledWith( + expect.objectContaining({ + fields: ["id"], + }) + ) + }) }) describe("permissions", () => { diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts new file mode 100644 index 0000000000..b033c6dd17 --- /dev/null +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -0,0 +1,102 @@ +import { db } from "@budibase/backend-core" +import { + FieldType, + isLogicalSearchOperator, + SearchFilters, + Table, +} from "@budibase/types" +import { cloneDeep } from "lodash/fp" +import sdk from "../../../sdk" + +export const removeInvalidFilters = ( + filters: SearchFilters, + validFields: string[] +) => { + const result = cloneDeep(filters) + + validFields = validFields.map(f => f.toLowerCase()) + for (const filterKey of Object.keys(result) as (keyof SearchFilters)[]) { + if (typeof result[filterKey] !== "object") { + continue + } + if (isLogicalSearchOperator(filterKey)) { + const resultingConditions: SearchFilters[] = [] + for (const condition of result[filterKey].conditions) { + const resultingCondition = removeInvalidFilters(condition, validFields) + if (Object.keys(resultingCondition).length) { + resultingConditions.push(resultingCondition) + } + } + if (resultingConditions.length) { + result[filterKey].conditions = resultingConditions + } else { + delete result[filterKey] + } + continue + } + + const filter = result[filterKey] + for (const columnKey of Object.keys(filter)) { + const possibleKeys = [columnKey, db.removeKeyNumbering(columnKey)].map( + c => c.toLowerCase() + ) + if (!validFields.some(f => possibleKeys.includes(f.toLowerCase()))) { + delete filter[columnKey] + } + } + if (!Object.keys(filter).length) { + delete result[filterKey] + } + } + + return result +} + +export const getQueryableFields = async ( + fields: string[], + table: Table +): Promise => { + const extractTableFields = async ( + table: Table, + allowedFields: string[], + fromTables: string[] + ): Promise => { + const result = [] + for (const field of Object.keys(table.schema).filter( + f => allowedFields.includes(f) && table.schema[f].visible !== false + )) { + const subSchema = table.schema[field] + if (subSchema.type === FieldType.LINK) { + if (fromTables.includes(subSchema.tableId)) { + // avoid circular loops + continue + } + const relatedTable = await sdk.tables.getTable(subSchema.tableId) + const relatedFields = await extractTableFields( + relatedTable, + Object.keys(relatedTable.schema), + [...fromTables, subSchema.tableId] + ) + + result.push( + ...relatedFields.flatMap(f => [ + `${subSchema.name}.${f}`, + // should be able to filter by relationship using table name + `${relatedTable.name}.${f}`, + ]) + ) + } else { + result.push(field) + } + } + return result + } + + const result = [ + "_id", // Querying by _id is always allowed, even if it's never part of the schema + ] + + result.push(...(await extractTableFields(table, fields, [table._id!]))) + + return result +} diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 1ccd89639b..6a4286814d 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -14,6 +14,7 @@ import sdk from "../../index" import { searchInputMapping } from "./search/utils" import { db as dbCore } from "@budibase/backend-core" import tracer from "dd-trace" +import { getQueryableFields, removeInvalidFilters } from "./queryUtils" export { isValidFilter } from "../../../integrations/utils" @@ -73,6 +74,18 @@ export async function search( const table = await sdk.tables.getTable(options.tableId) options = searchInputMapping(table, options) + if (options.query) { + const tableFields = Object.keys(table.schema).filter( + f => table.schema[f].visible !== false + ) + + const queriableFields = await getQueryableFields( + options.fields?.filter(f => tableFields.includes(f)) ?? tableFields, + table + ) + options.query = removeInvalidFilters(options.query, queriableFields) + } + let result: SearchResponse if (isExternalTable) { span?.addTags({ searchType: "external" }) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 252b9a6556..db1104bcf6 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -1,4 +1,11 @@ -import { Datasource, FieldType, Row, Table } from "@budibase/types" +import { + AutoColumnFieldMetadata, + AutoFieldSubType, + Datasource, + FieldType, + NumberFieldMetadata, + Table, +} from "@budibase/types" import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" import { search } from "../../../../../sdk/app/rows/search" @@ -32,7 +39,6 @@ describe.each([ let envCleanup: (() => void) | undefined let datasource: Datasource | undefined let table: Table - let rows: Row[] beforeAll(async () => { await withCoreEnv({ SQS_SEARCH_ENABLE: isSqs ? "true" : "false" }, () => @@ -51,16 +57,28 @@ describe.each([ datasource: await dsProvider, }) } + }) + + beforeEach(async () => { + const idFieldSchema: NumberFieldMetadata | AutoColumnFieldMetadata = + isInternal + ? { + name: "id", + type: FieldType.AUTO, + subtype: AutoFieldSubType.AUTO_ID, + autocolumn: true, + } + : { + name: "id", + type: FieldType.NUMBER, + autocolumn: true, + } table = await config.api.table.save( tableForDatasource(datasource, { primary: ["id"], schema: { - id: { - name: "id", - type: FieldType.NUMBER, - autocolumn: true, - }, + id: idFieldSchema, name: { name: "name", type: FieldType.STRING, @@ -81,16 +99,13 @@ describe.each([ }) ) - rows = [] for (let i = 0; i < 10; i++) { - rows.push( - await config.api.row.save(table._id!, { - name: generator.first(), - surname: generator.last(), - age: generator.age(), - address: generator.address(), - }) - ) + await config.api.row.save(table._id!, { + name: generator.first(), + surname: generator.last(), + age: generator.age(), + address: generator.address(), + }) } }) @@ -138,4 +153,100 @@ describe.each([ ) }) }) + + it("does not allow accessing hidden fields", async () => { + await config.doInContext(config.appId, async () => { + await config.api.table.save({ + ...table, + schema: { + ...table.schema, + name: { + ...table.schema.name, + visible: true, + }, + age: { + ...table.schema.age, + visible: false, + }, + }, + }) + const result = await search({ + tableId: table._id!, + query: {}, + }) + expect(result.rows).toHaveLength(10) + for (const row of result.rows) { + const keys = Object.keys(row) + expect(keys).toContain("name") + expect(keys).toContain("surname") + expect(keys).toContain("address") + expect(keys).not.toContain("age") + } + }) + }) + + it("does not allow accessing hidden fields even if requested", async () => { + await config.doInContext(config.appId, async () => { + await config.api.table.save({ + ...table, + schema: { + ...table.schema, + name: { + ...table.schema.name, + visible: true, + }, + age: { + ...table.schema.age, + visible: false, + }, + }, + }) + const result = await search({ + tableId: table._id!, + query: {}, + fields: ["name", "age"], + }) + expect(result.rows).toHaveLength(10) + for (const row of result.rows) { + const keys = Object.keys(row) + expect(keys).toContain("name") + expect(keys).not.toContain("age") + expect(keys).not.toContain("surname") + expect(keys).not.toContain("address") + } + }) + }) + + !isLucene && + it.each([ + [["id", "name", "age"], 3], + [["name", "age"], 10], + ])( + "cannot query by non search fields (fields: %s)", + async (queryFields, expectedRows) => { + await config.doInContext(config.appId, async () => { + const { rows } = await search({ + tableId: table._id!, + query: { + $or: { + conditions: [ + { + $and: { + conditions: [ + { range: { id: { low: 2, high: 4 } } }, + { range: { id: { low: 3, high: 5 } } }, + ], + }, + }, + { equal: { id: 7 } }, + ], + }, + }, + fields: queryFields, + }) + + expect(rows).toHaveLength(expectedRows) + }) + } + ) }) diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts new file mode 100644 index 0000000000..c156b2d5ba --- /dev/null +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -0,0 +1,563 @@ +import { + FieldType, + RelationshipType, + SearchFilters, + Table, +} from "@budibase/types" +import { getQueryableFields, removeInvalidFilters } from "../queryUtils" +import { structures } from "../../../../api/routes/tests/utilities" +import TestConfiguration from "../../../../tests/utilities/TestConfiguration" + +describe("query utils", () => { + describe("removeInvalidFilters", () => { + const fullFilters: SearchFilters = { + equal: { one: "foo" }, + $or: { + conditions: [ + { + equal: { one: "foo2", two: "bar" }, + notEmpty: { one: null }, + $and: { + conditions: [ + { + equal: { three: "baz" }, + notEmpty: { forth: null }, + }, + ], + }, + }, + ], + }, + $and: { + conditions: [{ equal: { one: "foo2" }, notEmpty: { one: null } }], + }, + } + + it("can filter empty queries", () => { + const filters: SearchFilters = {} + const result = removeInvalidFilters(filters, []) + expect(result).toEqual({}) + }) + + it("does not trim any valid field", () => { + const result = removeInvalidFilters(fullFilters, [ + "one", + "two", + "three", + "forth", + ]) + expect(result).toEqual(fullFilters) + }) + + it("trims invalid field", () => { + const result = removeInvalidFilters(fullFilters, [ + "one", + "three", + "forth", + ]) + expect(result).toEqual({ + equal: { one: "foo" }, + $or: { + conditions: [ + { + equal: { one: "foo2" }, + notEmpty: { one: null }, + $and: { + conditions: [ + { + equal: { three: "baz" }, + notEmpty: { forth: null }, + }, + ], + }, + }, + ], + }, + $and: { + conditions: [{ equal: { one: "foo2" }, notEmpty: { one: null } }], + }, + }) + }) + + it("trims invalid field keeping a valid fields", () => { + const result = removeInvalidFilters(fullFilters, ["three", "forth"]) + const expected: SearchFilters = { + $or: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { three: "baz" }, + notEmpty: { forth: null }, + }, + ], + }, + }, + ], + }, + } + expect(result).toEqual(expected) + }) + + it("keeps filter key numering", () => { + const prefixedFilters: SearchFilters = { + equal: { "1:one": "foo" }, + $or: { + conditions: [ + { + equal: { "2:one": "foo2", "3:two": "bar" }, + notEmpty: { "4:one": null }, + $and: { + conditions: [ + { + equal: { "5:three": "baz", two: "bar2" }, + notEmpty: { forth: null }, + }, + ], + }, + }, + ], + }, + $and: { + conditions: [{ equal: { "6:one": "foo2" }, notEmpty: { one: null } }], + }, + } + + const result = removeInvalidFilters(prefixedFilters, [ + "one", + "three", + "forth", + ]) + expect(result).toEqual({ + equal: { "1:one": "foo" }, + $or: { + conditions: [ + { + equal: { "2:one": "foo2" }, + notEmpty: { "4:one": null }, + $and: { + conditions: [ + { + equal: { "5:three": "baz" }, + notEmpty: { forth: null }, + }, + ], + }, + }, + ], + }, + $and: { + conditions: [{ equal: { "6:one": "foo2" }, notEmpty: { one: null } }], + }, + }) + }) + + it("handles relationship filters", () => { + const prefixedFilters: SearchFilters = { + $or: { + conditions: [ + { equal: { "1:other.one": "foo" } }, + { + equal: { + "2:other.one": "foo2", + "3:other.two": "bar", + "4:other.three": "baz", + }, + }, + { equal: { "another.three": "baz2" } }, + ], + }, + } + + const result = removeInvalidFilters(prefixedFilters, [ + "other.one", + "other.two", + "another.three", + ]) + expect(result).toEqual({ + $or: { + conditions: [ + { equal: { "1:other.one": "foo" } }, + { equal: { "2:other.one": "foo2", "3:other.two": "bar" } }, + { equal: { "another.three": "baz2" } }, + ], + }, + }) + }) + }) + + describe("getQueryableFields", () => { + const config = new TestConfiguration() + + beforeAll(async () => { + await config.init() + }) + + it("returns table schema fields and _id", async () => { + const table: Table = await config.api.table.save({ + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + age: { name: "age", type: FieldType.NUMBER }, + }, + }) + + const result = await getQueryableFields(Object.keys(table.schema), table) + expect(result).toEqual(["_id", "name", "age"]) + }) + + it("excludes hidden fields", async () => { + const table: Table = await config.api.table.save({ + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + age: { name: "age", type: FieldType.NUMBER, visible: false }, + }, + }) + + const result = await getQueryableFields(Object.keys(table.schema), table) + expect(result).toEqual(["_id", "name"]) + }) + + it("includes relationship fields", async () => { + const aux: Table = await config.api.table.save({ + ...structures.basicTable(), + name: "auxTable", + schema: { + title: { name: "title", type: FieldType.STRING }, + name: { name: "name", type: FieldType.STRING }, + }, + }) + + const table: Table = await config.api.table.save({ + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + aux: { + name: "aux", + type: FieldType.LINK, + tableId: aux._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "table", + }, + }, + }) + + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(table.schema), table) + }) + expect(result).toEqual([ + "_id", + "name", + "aux.title", + "auxTable.title", + "aux.name", + "auxTable.name", + ]) + }) + + it("excludes hidden relationship fields", async () => { + const aux: Table = await config.api.table.save({ + ...structures.basicTable(), + name: "auxTable", + schema: { + title: { name: "title", type: FieldType.STRING, visible: false }, + name: { name: "name", type: FieldType.STRING, visible: true }, + }, + }) + + const table: Table = await config.api.table.save({ + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + aux: { + name: "aux", + type: FieldType.LINK, + tableId: aux._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "table", + }, + }, + }) + + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(table.schema), table) + }) + expect(result).toEqual(["_id", "name", "aux.name", "auxTable.name"]) + }) + + it("excludes all relationship fields if hidden", async () => { + const aux: Table = await config.api.table.save({ + ...structures.basicTable(), + name: "auxTable", + schema: { + title: { name: "title", type: FieldType.STRING, visible: false }, + name: { name: "name", type: FieldType.STRING, visible: true }, + }, + }) + + const table: Table = await config.api.table.save({ + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + aux: { + name: "aux", + type: FieldType.LINK, + tableId: aux._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "table", + visible: false, + }, + }, + }) + + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(table.schema), table) + }) + expect(result).toEqual(["_id", "name"]) + }) + + describe("nested relationship", () => { + describe("one-to-many", () => { + let table: Table, aux1: Table, aux2: Table + + beforeAll(async () => { + const { _id: aux1Id } = await config.api.table.save({ + ...structures.basicTable(), + name: "aux1Table", + schema: { + name: { name: "name", type: FieldType.STRING }, + }, + }) + const { _id: aux2Id } = await config.api.table.save({ + ...structures.basicTable(), + name: "aux2Table", + schema: { + title: { name: "title", type: FieldType.STRING }, + aux1_1: { + name: "aux1_1", + type: FieldType.LINK, + tableId: aux1Id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "aux2_1", + }, + aux1_2: { + name: "aux1_2", + type: FieldType.LINK, + tableId: aux1Id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "aux2_2", + }, + }, + }) + + const { _id: tableId } = await config.api.table.save({ + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + aux1: { + name: "aux1", + type: FieldType.LINK, + tableId: aux1Id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "table", + }, + aux2: { + name: "aux2", + type: FieldType.LINK, + tableId: aux2Id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "table", + }, + }, + }) + + // We need to refech them to get the updated foreign keys + aux1 = await config.api.table.get(aux1Id!) + aux2 = await config.api.table.get(aux2Id!) + table = await config.api.table.get(tableId!) + }) + + it("includes nested relationship fields from main table", async () => { + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(table.schema), table) + }) + expect(result).toEqual([ + "_id", + "name", + // deep 1 aux1 primitive props + "aux1.name", + "aux1Table.name", + + // deep 2 aux1 primitive props + "aux1.aux2_1.title", + "aux1Table.aux2_1.title", + "aux1.aux2Table.title", + "aux1Table.aux2Table.title", + + // deep 2 aux2 primitive props + "aux1.aux2_2.title", + "aux1Table.aux2_2.title", + "aux1.aux2Table.title", + "aux1Table.aux2Table.title", + + // deep 1 aux2 primitive props + "aux2.title", + "aux2Table.title", + + // deep 2 aux2 primitive props + "aux2.aux1_1.name", + "aux2Table.aux1_1.name", + "aux2.aux1Table.name", + "aux2Table.aux1Table.name", + "aux2.aux1_2.name", + "aux2Table.aux1_2.name", + "aux2.aux1Table.name", + "aux2Table.aux1Table.name", + ]) + }) + + it("includes nested relationship fields from aux 1 table", async () => { + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(aux1.schema), aux1) + }) + expect(result).toEqual([ + "_id", + "name", + + // deep 1 aux2_1 primitive props + "aux2_1.title", + "aux2Table.title", + + // deep 2 aux2_1 primitive props + "aux2_1.table.name", + "aux2Table.table.name", + "aux2_1.TestTable.name", + "aux2Table.TestTable.name", + + // deep 1 aux2_2 primitive props + "aux2_2.title", + "aux2Table.title", + + // deep 2 aux2_2 primitive props + "aux2_2.table.name", + "aux2Table.table.name", + "aux2_2.TestTable.name", + "aux2Table.TestTable.name", + + // deep 1 table primitive props + "table.name", + "TestTable.name", + + // deep 2 table primitive props + "table.aux2.title", + "TestTable.aux2.title", + "table.aux2Table.title", + "TestTable.aux2Table.title", + ]) + }) + + it("includes nested relationship fields from aux 2 table", async () => { + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(aux2.schema), aux2) + }) + expect(result).toEqual([ + "_id", + "title", + + // deep 1 aux1_1 primitive props + "aux1_1.name", + "aux1Table.name", + + // deep 2 aux1_1 primitive props + "aux1_1.table.name", + "aux1Table.table.name", + "aux1_1.TestTable.name", + "aux1Table.TestTable.name", + + // deep 1 aux1_2 primitive props + "aux1_2.name", + "aux1Table.name", + + // deep 2 aux1_2 primitive props + "aux1_2.table.name", + "aux1Table.table.name", + "aux1_2.TestTable.name", + "aux1Table.TestTable.name", + + // deep 1 table primitive props + "table.name", + "TestTable.name", + + // deep 2 table primitive props + "table.aux1.name", + "TestTable.aux1.name", + "table.aux1Table.name", + "TestTable.aux1Table.name", + ]) + }) + }) + + describe("many-to-many", () => { + let table: Table, aux: Table + + beforeAll(async () => { + const { _id: auxId } = await config.api.table.save({ + ...structures.basicTable(), + name: "auxTable", + schema: { + title: { name: "title", type: FieldType.STRING }, + }, + }) + + const { _id: tableId } = await config.api.table.save({ + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + aux: { + name: "aux", + type: FieldType.LINK, + tableId: auxId!, + relationshipType: RelationshipType.MANY_TO_MANY, + fieldName: "table", + }, + }, + }) + + // We need to refech them to get the updated foreign keys + aux = await config.api.table.get(auxId!) + table = await config.api.table.get(tableId!) + }) + + it("includes nested relationship fields from main table", async () => { + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(table.schema), table) + }) + expect(result).toEqual([ + "_id", + "name", + + // deep 1 aux primitive props + "aux.title", + "auxTable.title", + ]) + }) + + it("includes nested relationship fields from aux table", async () => { + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(aux.schema), aux) + }) + expect(result).toEqual([ + "_id", + "title", + + // deep 1 dependency primitive props + "table.name", + "TestTable.name", + ]) + }) + }) + }) + }) +}) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 62a3b2dd74..4b2fd83882 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -26,8 +26,13 @@ import { processOutputBBReferences, } from "./bbReferenceProcessor" import { isExternalTableID } from "../../integrations/utils" -import { helpers } from "@budibase/shared-core" +import { + helpers, + PROTECTED_EXTERNAL_COLUMNS, + PROTECTED_INTERNAL_COLUMNS, +} from "@budibase/shared-core" import { processString } from "@budibase/string-templates" +import { isUserMetadataTable } from "../../api/controllers/row/utils" export * from "./utils" export * from "./attachments" @@ -53,9 +58,9 @@ export async function processAutoColumn( row: Row, opts?: AutoColumnProcessingOpts ) { - let noUser = !userId - let isUserTable = table._id === InternalTables.USER_METADATA - let now = new Date().toISOString() + const noUser = !userId + const isUserTable = table._id === InternalTables.USER_METADATA + const now = new Date().toISOString() // if a row doesn't have a revision then it doesn't exist yet const creating = !row._rev // check its not user table, or whether any of the processing options have been disabled @@ -111,7 +116,7 @@ async function processDefaultValues(table: Table, row: Row) { ctx.user = user } - for (let [key, schema] of Object.entries(table.schema)) { + for (const [key, schema] of Object.entries(table.schema)) { if ("default" in schema && schema.default != null && row[key] == null) { const processed = await processString(schema.default, ctx) @@ -165,10 +170,10 @@ export async function inputProcessing( row: Row, opts?: AutoColumnProcessingOpts ) { - let clonedRow = cloneDeep(row) + const clonedRow = cloneDeep(row) const dontCleanseKeys = ["type", "_id", "_rev", "tableId"] - for (let [key, value] of Object.entries(clonedRow)) { + for (const [key, value] of Object.entries(clonedRow)) { const field = table.schema[key] // cleanse fields that aren't in the schema if (!field) { @@ -268,13 +273,13 @@ export async function outputProcessing( } // process complex types: attachments, bb references... - for (let [property, column] of Object.entries(table.schema)) { + for (const [property, column] of Object.entries(table.schema)) { if ( column.type === FieldType.ATTACHMENTS || column.type === FieldType.ATTACHMENT_SINGLE || column.type === FieldType.SIGNATURE_SINGLE ) { - for (let row of enriched) { + for (const row of enriched) { if (row[property] == null) { continue } @@ -299,7 +304,7 @@ export async function outputProcessing( !opts.skipBBReferences && column.type == FieldType.BB_REFERENCE ) { - for (let row of enriched) { + for (const row of enriched) { row[property] = await processOutputBBReferences( row[property], column.subtype @@ -309,14 +314,14 @@ export async function outputProcessing( !opts.skipBBReferences && column.type == FieldType.BB_REFERENCE_SINGLE ) { - for (let row of enriched) { + for (const row of enriched) { row[property] = await processOutputBBReference( row[property], column.subtype ) } } else if (column.type === FieldType.DATETIME && column.timeOnly) { - for (let row of enriched) { + for (const row of enriched) { if (row[property] instanceof Date) { const hours = row[property].getUTCHours().toString().padStart(2, "0") const minutes = row[property] @@ -343,14 +348,36 @@ export async function outputProcessing( )) as Row[] } // remove null properties to match internal API - if (isExternalTableID(table._id!)) { - for (let row of enriched) { - for (let key of Object.keys(row)) { + const isExternal = isExternalTableID(table._id!) + if (isExternal) { + for (const row of enriched) { + for (const key of Object.keys(row)) { if (row[key] === null) { delete row[key] } } } } + + if (!isUserMetadataTable(table._id!)) { + const protectedColumns = isExternal + ? PROTECTED_EXTERNAL_COLUMNS + : PROTECTED_INTERNAL_COLUMNS + + const tableFields = Object.keys(table.schema).filter( + f => table.schema[f].visible !== false + ) + const fields = [...tableFields, ...protectedColumns].map(f => + f.toLowerCase() + ) + for (const row of enriched) { + for (const key of Object.keys(row)) { + if (!fields.includes(key.toLowerCase())) { + delete row[key] + } + } + } + } + return (wasArray ? enriched : enriched[0]) as T }