Merge pull request #14351 from Budibase/BUDI-8547/honor-table-and-view-schema-on-query

Honor table and view schema on query

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -192,10 +192,10 @@ export function buildSqlFieldList(
   function extractRealFields(table: Table, existing: string[] = []) {
     return Object.entries(table.schema)
       .filter(
-        column =>
+        ([columnName, column]) =>
-          column[1].type !== FieldType.LINK &&
+          column.type !== FieldType.LINK &&
-          column[1].type !== FieldType.FORMULA &&
+          column.type !== FieldType.FORMULA &&
-          !existing.find((field: string) => field === column[0])
+          !existing.find((field: string) => field === columnName)
       )
       .map(column => `${table.name}.${column[0]}`)
   }

packages/server/src/api/routes/tests/row.spec.ts

@@ -1664,7 +1664,7 @@ describe.each([
   isInternal &&
     describe("attachments and signatures", () => {
       const coreAttachmentEnrichment = async (
-        schema: any,
+        schema: TableSchema,
         field: string,
         attachmentCfg: string | string[]
       ) => {
@@ -1691,7 +1691,7 @@ describe.each([
 
         await withEnv({ SELF_HOSTED: "true" }, async () => {
           return context.doInAppContext(config.getAppId(), async () => {
-            const enriched: Row[] = await outputProcessing(table, [row])
+            const enriched: Row[] = await outputProcessing(testTable, [row])
             const [targetRow] = enriched
             const attachmentEntries = Array.isArray(targetRow[field])
               ? targetRow[field]

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -30,6 +30,7 @@ import {
   withEnv as withCoreEnv,
   setEnv as setCoreEnv,
 } from "@budibase/backend-core"
+import sdk from "../../../sdk"
 
 describe.each([
   ["lucene", undefined],
@@ -120,6 +121,7 @@ describe.each([
   })
 
   beforeEach(() => {
+    jest.clearAllMocks()
     mocks.licenses.useCloudFree()
   })
 
@@ -1602,6 +1604,28 @@ describe.each([
           })
           expect(response.rows).toHaveLength(0)
         })
+
+      it("queries the row api passing the view fields only", async () => {
+        const searchSpy = jest.spyOn(sdk.rows, "search")
+
+        const view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          schema: {
+            id: { visible: true },
+            one: { visible: false },
+          },
+        })
+
+        await config.api.viewV2.search(view.id, { query: {} })
+        expect(searchSpy).toHaveBeenCalledTimes(1)
+
+        expect(searchSpy).toHaveBeenCalledWith(
+          expect.objectContaining({
+            fields: ["id"],
+          })
+        )
+      })
     })
 
     describe("permissions", () => {

packages/server/src/sdk/app/rows/queryUtils.ts

@@ -0,0 +1,102 @@
+import { db } from "@budibase/backend-core"
+import {
+  FieldType,
+  isLogicalSearchOperator,
+  SearchFilters,
+  Table,
+} from "@budibase/types"
+import { cloneDeep } from "lodash/fp"
+import sdk from "../../../sdk"
+
+export const removeInvalidFilters = (
+  filters: SearchFilters,
+  validFields: string[]
+) => {
+  const result = cloneDeep(filters)
+
+  validFields = validFields.map(f => f.toLowerCase())
+  for (const filterKey of Object.keys(result) as (keyof SearchFilters)[]) {
+    if (typeof result[filterKey] !== "object") {
+      continue
+    }
+    if (isLogicalSearchOperator(filterKey)) {
+      const resultingConditions: SearchFilters[] = []
+      for (const condition of result[filterKey].conditions) {
+        const resultingCondition = removeInvalidFilters(condition, validFields)
+        if (Object.keys(resultingCondition).length) {
+          resultingConditions.push(resultingCondition)
+        }
+      }
+      if (resultingConditions.length) {
+        result[filterKey].conditions = resultingConditions
+      } else {
+        delete result[filterKey]
+      }
+      continue
+    }
+
+    const filter = result[filterKey]
+    for (const columnKey of Object.keys(filter)) {
+      const possibleKeys = [columnKey, db.removeKeyNumbering(columnKey)].map(
+        c => c.toLowerCase()
+      )
+      if (!validFields.some(f => possibleKeys.includes(f.toLowerCase()))) {
+        delete filter[columnKey]
+      }
+    }
+    if (!Object.keys(filter).length) {
+      delete result[filterKey]
+    }
+  }
+
+  return result
+}
+
+export const getQueryableFields = async (
+  fields: string[],
+  table: Table
+): Promise<string[]> => {
+  const extractTableFields = async (
+    table: Table,
+    allowedFields: string[],
+    fromTables: string[]
+  ): Promise<string[]> => {
+    const result = []
+    for (const field of Object.keys(table.schema).filter(
+      f => allowedFields.includes(f) && table.schema[f].visible !== false
+    )) {
+      const subSchema = table.schema[field]
+      if (subSchema.type === FieldType.LINK) {
+        if (fromTables.includes(subSchema.tableId)) {
+          // avoid circular loops
+          continue
+        }
+        const relatedTable = await sdk.tables.getTable(subSchema.tableId)
+        const relatedFields = await extractTableFields(
+          relatedTable,
+          Object.keys(relatedTable.schema),
+          [...fromTables, subSchema.tableId]
+        )
+
+        result.push(
+          ...relatedFields.flatMap(f => [
+            `${subSchema.name}.${f}`,
+            // should be able to filter by relationship using table name
+            `${relatedTable.name}.${f}`,
+          ])
+        )
+      } else {
+        result.push(field)
+      }
+    }
+    return result
+  }
+
+  const result = [
+    "_id", // Querying by _id is always allowed, even if it's never part of the schema
+  ]
+
+  result.push(...(await extractTableFields(table, fields, [table._id!])))
+
+  return result
+}

packages/server/src/sdk/app/rows/search.ts

@@ -14,6 +14,7 @@ import sdk from "../../index"
 import { searchInputMapping } from "./search/utils"
 import { db as dbCore } from "@budibase/backend-core"
 import tracer from "dd-trace"
+import { getQueryableFields, removeInvalidFilters } from "./queryUtils"
 
 export { isValidFilter } from "../../../integrations/utils"
 
@@ -73,6 +74,18 @@ export async function search(
     const table = await sdk.tables.getTable(options.tableId)
     options = searchInputMapping(table, options)
 
+    if (options.query) {
+      const tableFields = Object.keys(table.schema).filter(
+        f => table.schema[f].visible !== false
+      )
+
+      const queriableFields = await getQueryableFields(
+        options.fields?.filter(f => tableFields.includes(f)) ?? tableFields,
+        table
+      )
+      options.query = removeInvalidFilters(options.query, queriableFields)
+    }
+
     let result: SearchResponse<Row>
     if (isExternalTable) {
       span?.addTags({ searchType: "external" })

packages/server/src/sdk/app/rows/search/tests/search.spec.ts

@@ -1,4 +1,11 @@
-import { Datasource, FieldType, Row, Table } from "@budibase/types"
+import {
+  AutoColumnFieldMetadata,
+  AutoFieldSubType,
+  Datasource,
+  FieldType,
+  NumberFieldMetadata,
+  Table,
+} from "@budibase/types"
 
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
 import { search } from "../../../../../sdk/app/rows/search"
@@ -32,7 +39,6 @@ describe.each([
   let envCleanup: (() => void) | undefined
   let datasource: Datasource | undefined
   let table: Table
-  let rows: Row[]
 
   beforeAll(async () => {
     await withCoreEnv({ SQS_SEARCH_ENABLE: isSqs ? "true" : "false" }, () =>
@@ -51,16 +57,28 @@ describe.each([
         datasource: await dsProvider,
       })
     }
+  })
+
+  beforeEach(async () => {
+    const idFieldSchema: NumberFieldMetadata | AutoColumnFieldMetadata =
+      isInternal
+        ? {
+            name: "id",
+            type: FieldType.AUTO,
+            subtype: AutoFieldSubType.AUTO_ID,
+            autocolumn: true,
+          }
+        : {
+            name: "id",
+            type: FieldType.NUMBER,
+            autocolumn: true,
+          }
 
     table = await config.api.table.save(
       tableForDatasource(datasource, {
         primary: ["id"],
         schema: {
-          id: {
+          id: idFieldSchema,
-            name: "id",
-            type: FieldType.NUMBER,
-            autocolumn: true,
-          },
           name: {
             name: "name",
             type: FieldType.STRING,
@@ -81,16 +99,13 @@ describe.each([
       })
     )
 
-    rows = []
     for (let i = 0; i < 10; i++) {
-      rows.push(
+      await config.api.row.save(table._id!, {
-        await config.api.row.save(table._id!, {
+        name: generator.first(),
-          name: generator.first(),
+        surname: generator.last(),
-          surname: generator.last(),
+        age: generator.age(),
-          age: generator.age(),
+        address: generator.address(),
-          address: generator.address(),
+      })
-        })
-      )
     }
   })
 
@@ -138,4 +153,100 @@ describe.each([
         )
       })
     })
+
+  it("does not allow accessing hidden fields", async () => {
+    await config.doInContext(config.appId, async () => {
+      await config.api.table.save({
+        ...table,
+        schema: {
+          ...table.schema,
+          name: {
+            ...table.schema.name,
+            visible: true,
+          },
+          age: {
+            ...table.schema.age,
+            visible: false,
+          },
+        },
+      })
+      const result = await search({
+        tableId: table._id!,
+        query: {},
+      })
+      expect(result.rows).toHaveLength(10)
+      for (const row of result.rows) {
+        const keys = Object.keys(row)
+        expect(keys).toContain("name")
+        expect(keys).toContain("surname")
+        expect(keys).toContain("address")
+        expect(keys).not.toContain("age")
+      }
+    })
+  })
+
+  it("does not allow accessing hidden fields even if requested", async () => {
+    await config.doInContext(config.appId, async () => {
+      await config.api.table.save({
+        ...table,
+        schema: {
+          ...table.schema,
+          name: {
+            ...table.schema.name,
+            visible: true,
+          },
+          age: {
+            ...table.schema.age,
+            visible: false,
+          },
+        },
+      })
+      const result = await search({
+        tableId: table._id!,
+        query: {},
+        fields: ["name", "age"],
+      })
+      expect(result.rows).toHaveLength(10)
+      for (const row of result.rows) {
+        const keys = Object.keys(row)
+        expect(keys).toContain("name")
+        expect(keys).not.toContain("age")
+        expect(keys).not.toContain("surname")
+        expect(keys).not.toContain("address")
+      }
+    })
+  })
+
+  !isLucene &&
+    it.each([
+      [["id", "name", "age"], 3],
+      [["name", "age"], 10],
+    ])(
+      "cannot query by non search fields (fields: %s)",
+      async (queryFields, expectedRows) => {
+        await config.doInContext(config.appId, async () => {
+          const { rows } = await search({
+            tableId: table._id!,
+            query: {
+              $or: {
+                conditions: [
+                  {
+                    $and: {
+                      conditions: [
+                        { range: { id: { low: 2, high: 4 } } },
+                        { range: { id: { low: 3, high: 5 } } },
+                      ],
+                    },
+                  },
+                  { equal: { id: 7 } },
+                ],
+              },
+            },
+            fields: queryFields,
+          })
+
+          expect(rows).toHaveLength(expectedRows)
+        })
+      }
+    )
 })

packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts

@@ -0,0 +1,563 @@
+import {
+  FieldType,
+  RelationshipType,
+  SearchFilters,
+  Table,
+} from "@budibase/types"
+import { getQueryableFields, removeInvalidFilters } from "../queryUtils"
+import { structures } from "../../../../api/routes/tests/utilities"
+import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
+
+describe("query utils", () => {
+  describe("removeInvalidFilters", () => {
+    const fullFilters: SearchFilters = {
+      equal: { one: "foo" },
+      $or: {
+        conditions: [
+          {
+            equal: { one: "foo2", two: "bar" },
+            notEmpty: { one: null },
+            $and: {
+              conditions: [
+                {
+                  equal: { three: "baz" },
+                  notEmpty: { forth: null },
+                },
+              ],
+            },
+          },
+        ],
+      },
+      $and: {
+        conditions: [{ equal: { one: "foo2" }, notEmpty: { one: null } }],
+      },
+    }
+
+    it("can filter empty queries", () => {
+      const filters: SearchFilters = {}
+      const result = removeInvalidFilters(filters, [])
+      expect(result).toEqual({})
+    })
+
+    it("does not trim any valid field", () => {
+      const result = removeInvalidFilters(fullFilters, [
+        "one",
+        "two",
+        "three",
+        "forth",
+      ])
+      expect(result).toEqual(fullFilters)
+    })
+
+    it("trims invalid field", () => {
+      const result = removeInvalidFilters(fullFilters, [
+        "one",
+        "three",
+        "forth",
+      ])
+      expect(result).toEqual({
+        equal: { one: "foo" },
+        $or: {
+          conditions: [
+            {
+              equal: { one: "foo2" },
+              notEmpty: { one: null },
+              $and: {
+                conditions: [
+                  {
+                    equal: { three: "baz" },
+                    notEmpty: { forth: null },
+                  },
+                ],
+              },
+            },
+          ],
+        },
+        $and: {
+          conditions: [{ equal: { one: "foo2" }, notEmpty: { one: null } }],
+        },
+      })
+    })
+
+    it("trims invalid field keeping a valid fields", () => {
+      const result = removeInvalidFilters(fullFilters, ["three", "forth"])
+      const expected: SearchFilters = {
+        $or: {
+          conditions: [
+            {
+              $and: {
+                conditions: [
+                  {
+                    equal: { three: "baz" },
+                    notEmpty: { forth: null },
+                  },
+                ],
+              },
+            },
+          ],
+        },
+      }
+      expect(result).toEqual(expected)
+    })
+
+    it("keeps filter key numering", () => {
+      const prefixedFilters: SearchFilters = {
+        equal: { "1:one": "foo" },
+        $or: {
+          conditions: [
+            {
+              equal: { "2:one": "foo2", "3:two": "bar" },
+              notEmpty: { "4:one": null },
+              $and: {
+                conditions: [
+                  {
+                    equal: { "5:three": "baz", two: "bar2" },
+                    notEmpty: { forth: null },
+                  },
+                ],
+              },
+            },
+          ],
+        },
+        $and: {
+          conditions: [{ equal: { "6:one": "foo2" }, notEmpty: { one: null } }],
+        },
+      }
+
+      const result = removeInvalidFilters(prefixedFilters, [
+        "one",
+        "three",
+        "forth",
+      ])
+      expect(result).toEqual({
+        equal: { "1:one": "foo" },
+        $or: {
+          conditions: [
+            {
+              equal: { "2:one": "foo2" },
+              notEmpty: { "4:one": null },
+              $and: {
+                conditions: [
+                  {
+                    equal: { "5:three": "baz" },
+                    notEmpty: { forth: null },
+                  },
+                ],
+              },
+            },
+          ],
+        },
+        $and: {
+          conditions: [{ equal: { "6:one": "foo2" }, notEmpty: { one: null } }],
+        },
+      })
+    })
+
+    it("handles relationship filters", () => {
+      const prefixedFilters: SearchFilters = {
+        $or: {
+          conditions: [
+            { equal: { "1:other.one": "foo" } },
+            {
+              equal: {
+                "2:other.one": "foo2",
+                "3:other.two": "bar",
+                "4:other.three": "baz",
+              },
+            },
+            { equal: { "another.three": "baz2" } },
+          ],
+        },
+      }
+
+      const result = removeInvalidFilters(prefixedFilters, [
+        "other.one",
+        "other.two",
+        "another.three",
+      ])
+      expect(result).toEqual({
+        $or: {
+          conditions: [
+            { equal: { "1:other.one": "foo" } },
+            { equal: { "2:other.one": "foo2", "3:other.two": "bar" } },
+            { equal: { "another.three": "baz2" } },
+          ],
+        },
+      })
+    })
+  })
+
+  describe("getQueryableFields", () => {
+    const config = new TestConfiguration()
+
+    beforeAll(async () => {
+      await config.init()
+    })
+
+    it("returns table schema fields and _id", async () => {
+      const table: Table = await config.api.table.save({
+        ...structures.basicTable(),
+        schema: {
+          name: { name: "name", type: FieldType.STRING },
+          age: { name: "age", type: FieldType.NUMBER },
+        },
+      })
+
+      const result = await getQueryableFields(Object.keys(table.schema), table)
+      expect(result).toEqual(["_id", "name", "age"])
+    })
+
+    it("excludes hidden fields", async () => {
+      const table: Table = await config.api.table.save({
+        ...structures.basicTable(),
+        schema: {
+          name: { name: "name", type: FieldType.STRING },
+          age: { name: "age", type: FieldType.NUMBER, visible: false },
+        },
+      })
+
+      const result = await getQueryableFields(Object.keys(table.schema), table)
+      expect(result).toEqual(["_id", "name"])
+    })
+
+    it("includes relationship fields", async () => {
+      const aux: Table = await config.api.table.save({
+        ...structures.basicTable(),
+        name: "auxTable",
+        schema: {
+          title: { name: "title", type: FieldType.STRING },
+          name: { name: "name", type: FieldType.STRING },
+        },
+      })
+
+      const table: Table = await config.api.table.save({
+        ...structures.basicTable(),
+        schema: {
+          name: { name: "name", type: FieldType.STRING },
+          aux: {
+            name: "aux",
+            type: FieldType.LINK,
+            tableId: aux._id!,
+            relationshipType: RelationshipType.ONE_TO_MANY,
+            fieldName: "table",
+          },
+        },
+      })
+
+      const result = await config.doInContext(config.appId, () => {
+        return getQueryableFields(Object.keys(table.schema), table)
+      })
+      expect(result).toEqual([
+        "_id",
+        "name",
+        "aux.title",
+        "auxTable.title",
+        "aux.name",
+        "auxTable.name",
+      ])
+    })
+
+    it("excludes hidden relationship fields", async () => {
+      const aux: Table = await config.api.table.save({
+        ...structures.basicTable(),
+        name: "auxTable",
+        schema: {
+          title: { name: "title", type: FieldType.STRING, visible: false },
+          name: { name: "name", type: FieldType.STRING, visible: true },
+        },
+      })
+
+      const table: Table = await config.api.table.save({
+        ...structures.basicTable(),
+        schema: {
+          name: { name: "name", type: FieldType.STRING },
+          aux: {
+            name: "aux",
+            type: FieldType.LINK,
+            tableId: aux._id!,
+            relationshipType: RelationshipType.ONE_TO_MANY,
+            fieldName: "table",
+          },
+        },
+      })
+
+      const result = await config.doInContext(config.appId, () => {
+        return getQueryableFields(Object.keys(table.schema), table)
+      })
+      expect(result).toEqual(["_id", "name", "aux.name", "auxTable.name"])
+    })
+
+    it("excludes all relationship fields if hidden", async () => {
+      const aux: Table = await config.api.table.save({
+        ...structures.basicTable(),
+        name: "auxTable",
+        schema: {
+          title: { name: "title", type: FieldType.STRING, visible: false },
+          name: { name: "name", type: FieldType.STRING, visible: true },
+        },
+      })
+
+      const table: Table = await config.api.table.save({
+        ...structures.basicTable(),
+        schema: {
+          name: { name: "name", type: FieldType.STRING },
+          aux: {
+            name: "aux",
+            type: FieldType.LINK,
+            tableId: aux._id!,
+            relationshipType: RelationshipType.ONE_TO_MANY,
+            fieldName: "table",
+            visible: false,
+          },
+        },
+      })
+
+      const result = await config.doInContext(config.appId, () => {
+        return getQueryableFields(Object.keys(table.schema), table)
+      })
+      expect(result).toEqual(["_id", "name"])
+    })
+
+    describe("nested relationship", () => {
+      describe("one-to-many", () => {
+        let table: Table, aux1: Table, aux2: Table
+
+        beforeAll(async () => {
+          const { _id: aux1Id } = await config.api.table.save({
+            ...structures.basicTable(),
+            name: "aux1Table",
+            schema: {
+              name: { name: "name", type: FieldType.STRING },
+            },
+          })
+          const { _id: aux2Id } = await config.api.table.save({
+            ...structures.basicTable(),
+            name: "aux2Table",
+            schema: {
+              title: { name: "title", type: FieldType.STRING },
+              aux1_1: {
+                name: "aux1_1",
+                type: FieldType.LINK,
+                tableId: aux1Id!,
+                relationshipType: RelationshipType.ONE_TO_MANY,
+                fieldName: "aux2_1",
+              },
+              aux1_2: {
+                name: "aux1_2",
+                type: FieldType.LINK,
+                tableId: aux1Id!,
+                relationshipType: RelationshipType.ONE_TO_MANY,
+                fieldName: "aux2_2",
+              },
+            },
+          })
+
+          const { _id: tableId } = await config.api.table.save({
+            ...structures.basicTable(),
+            schema: {
+              name: { name: "name", type: FieldType.STRING },
+              aux1: {
+                name: "aux1",
+                type: FieldType.LINK,
+                tableId: aux1Id!,
+                relationshipType: RelationshipType.ONE_TO_MANY,
+                fieldName: "table",
+              },
+              aux2: {
+                name: "aux2",
+                type: FieldType.LINK,
+                tableId: aux2Id!,
+                relationshipType: RelationshipType.ONE_TO_MANY,
+                fieldName: "table",
+              },
+            },
+          })
+
+          // We need to refech them to get the updated foreign keys
+          aux1 = await config.api.table.get(aux1Id!)
+          aux2 = await config.api.table.get(aux2Id!)
+          table = await config.api.table.get(tableId!)
+        })
+
+        it("includes nested relationship fields from main table", async () => {
+          const result = await config.doInContext(config.appId, () => {
+            return getQueryableFields(Object.keys(table.schema), table)
+          })
+          expect(result).toEqual([
+            "_id",
+            "name",
+            // deep 1 aux1 primitive props
+            "aux1.name",
+            "aux1Table.name",
+
+            // deep 2 aux1 primitive props
+            "aux1.aux2_1.title",
+            "aux1Table.aux2_1.title",
+            "aux1.aux2Table.title",
+            "aux1Table.aux2Table.title",
+
+            // deep 2 aux2 primitive props
+            "aux1.aux2_2.title",
+            "aux1Table.aux2_2.title",
+            "aux1.aux2Table.title",
+            "aux1Table.aux2Table.title",
+
+            // deep 1 aux2 primitive props
+            "aux2.title",
+            "aux2Table.title",
+
+            // deep 2 aux2 primitive props
+            "aux2.aux1_1.name",
+            "aux2Table.aux1_1.name",
+            "aux2.aux1Table.name",
+            "aux2Table.aux1Table.name",
+            "aux2.aux1_2.name",
+            "aux2Table.aux1_2.name",
+            "aux2.aux1Table.name",
+            "aux2Table.aux1Table.name",
+          ])
+        })
+
+        it("includes nested relationship fields from aux 1 table", async () => {
+          const result = await config.doInContext(config.appId, () => {
+            return getQueryableFields(Object.keys(aux1.schema), aux1)
+          })
+          expect(result).toEqual([
+            "_id",
+            "name",
+
+            // deep 1 aux2_1 primitive props
+            "aux2_1.title",
+            "aux2Table.title",
+
+            // deep 2 aux2_1 primitive props
+            "aux2_1.table.name",
+            "aux2Table.table.name",
+            "aux2_1.TestTable.name",
+            "aux2Table.TestTable.name",
+
+            // deep 1 aux2_2 primitive props
+            "aux2_2.title",
+            "aux2Table.title",
+
+            // deep 2 aux2_2 primitive props
+            "aux2_2.table.name",
+            "aux2Table.table.name",
+            "aux2_2.TestTable.name",
+            "aux2Table.TestTable.name",
+
+            // deep 1 table primitive props
+            "table.name",
+            "TestTable.name",
+
+            // deep 2 table primitive props
+            "table.aux2.title",
+            "TestTable.aux2.title",
+            "table.aux2Table.title",
+            "TestTable.aux2Table.title",
+          ])
+        })
+
+        it("includes nested relationship fields from aux 2 table", async () => {
+          const result = await config.doInContext(config.appId, () => {
+            return getQueryableFields(Object.keys(aux2.schema), aux2)
+          })
+          expect(result).toEqual([
+            "_id",
+            "title",
+
+            // deep 1 aux1_1 primitive props
+            "aux1_1.name",
+            "aux1Table.name",
+
+            // deep 2 aux1_1 primitive props
+            "aux1_1.table.name",
+            "aux1Table.table.name",
+            "aux1_1.TestTable.name",
+            "aux1Table.TestTable.name",
+
+            // deep 1 aux1_2 primitive props
+            "aux1_2.name",
+            "aux1Table.name",
+
+            // deep 2 aux1_2 primitive props
+            "aux1_2.table.name",
+            "aux1Table.table.name",
+            "aux1_2.TestTable.name",
+            "aux1Table.TestTable.name",
+
+            // deep 1 table primitive props
+            "table.name",
+            "TestTable.name",
+
+            // deep 2 table primitive props
+            "table.aux1.name",
+            "TestTable.aux1.name",
+            "table.aux1Table.name",
+            "TestTable.aux1Table.name",
+          ])
+        })
+      })
+
+      describe("many-to-many", () => {
+        let table: Table, aux: Table
+
+        beforeAll(async () => {
+          const { _id: auxId } = await config.api.table.save({
+            ...structures.basicTable(),
+            name: "auxTable",
+            schema: {
+              title: { name: "title", type: FieldType.STRING },
+            },
+          })
+
+          const { _id: tableId } = await config.api.table.save({
+            ...structures.basicTable(),
+            schema: {
+              name: { name: "name", type: FieldType.STRING },
+              aux: {
+                name: "aux",
+                type: FieldType.LINK,
+                tableId: auxId!,
+                relationshipType: RelationshipType.MANY_TO_MANY,
+                fieldName: "table",
+              },
+            },
+          })
+
+          // We need to refech them to get the updated foreign keys
+          aux = await config.api.table.get(auxId!)
+          table = await config.api.table.get(tableId!)
+        })
+
+        it("includes nested relationship fields from main table", async () => {
+          const result = await config.doInContext(config.appId, () => {
+            return getQueryableFields(Object.keys(table.schema), table)
+          })
+          expect(result).toEqual([
+            "_id",
+            "name",
+
+            // deep 1 aux primitive props
+            "aux.title",
+            "auxTable.title",
+          ])
+        })
+
+        it("includes nested relationship fields from aux table", async () => {
+          const result = await config.doInContext(config.appId, () => {
+            return getQueryableFields(Object.keys(aux.schema), aux)
+          })
+          expect(result).toEqual([
+            "_id",
+            "title",
+
+            // deep 1 dependency primitive props
+            "table.name",
+            "TestTable.name",
+          ])
+        })
+      })
+    })
+  })
+})

packages/server/src/utilities/rowProcessor/index.ts

@@ -26,8 +26,13 @@ import {
   processOutputBBReferences,
 } from "./bbReferenceProcessor"
 import { isExternalTableID } from "../../integrations/utils"
-import { helpers } from "@budibase/shared-core"
+import {
+  helpers,
+  PROTECTED_EXTERNAL_COLUMNS,
+  PROTECTED_INTERNAL_COLUMNS,
+} from "@budibase/shared-core"
 import { processString } from "@budibase/string-templates"
+import { isUserMetadataTable } from "../../api/controllers/row/utils"
 
 export * from "./utils"
 export * from "./attachments"
@@ -53,9 +58,9 @@ export async function processAutoColumn(
   row: Row,
   opts?: AutoColumnProcessingOpts
 ) {
-  let noUser = !userId
+  const noUser = !userId
-  let isUserTable = table._id === InternalTables.USER_METADATA
+  const isUserTable = table._id === InternalTables.USER_METADATA
-  let now = new Date().toISOString()
+  const now = new Date().toISOString()
   // if a row doesn't have a revision then it doesn't exist yet
   const creating = !row._rev
   // check its not user table, or whether any of the processing options have been disabled
@@ -111,7 +116,7 @@ async function processDefaultValues(table: Table, row: Row) {
     ctx.user = user
   }
 
-  for (let [key, schema] of Object.entries(table.schema)) {
+  for (const [key, schema] of Object.entries(table.schema)) {
     if ("default" in schema && schema.default != null && row[key] == null) {
       const processed = await processString(schema.default, ctx)
 
@@ -165,10 +170,10 @@ export async function inputProcessing(
   row: Row,
   opts?: AutoColumnProcessingOpts
 ) {
-  let clonedRow = cloneDeep(row)
+  const clonedRow = cloneDeep(row)
 
   const dontCleanseKeys = ["type", "_id", "_rev", "tableId"]
-  for (let [key, value] of Object.entries(clonedRow)) {
+  for (const [key, value] of Object.entries(clonedRow)) {
     const field = table.schema[key]
     // cleanse fields that aren't in the schema
     if (!field) {
@@ -268,13 +273,13 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
 
   // process complex types: attachments, bb references...
-  for (let [property, column] of Object.entries(table.schema)) {
+  for (const [property, column] of Object.entries(table.schema)) {
     if (
       column.type === FieldType.ATTACHMENTS ||
       column.type === FieldType.ATTACHMENT_SINGLE ||
       column.type === FieldType.SIGNATURE_SINGLE
     ) {
-      for (let row of enriched) {
+      for (const row of enriched) {
         if (row[property] == null) {
           continue
         }
@@ -299,7 +304,7 @@ export async function outputProcessing<T extends Row[] | Row>(
       !opts.skipBBReferences &&
       column.type == FieldType.BB_REFERENCE
     ) {
-      for (let row of enriched) {
+      for (const row of enriched) {
         row[property] = await processOutputBBReferences(
           row[property],
           column.subtype
@@ -309,14 +314,14 @@ export async function outputProcessing<T extends Row[] | Row>(
       !opts.skipBBReferences &&
       column.type == FieldType.BB_REFERENCE_SINGLE
     ) {
-      for (let row of enriched) {
+      for (const row of enriched) {
         row[property] = await processOutputBBReference(
           row[property],
           column.subtype
         )
       }
     } else if (column.type === FieldType.DATETIME && column.timeOnly) {
-      for (let row of enriched) {
+      for (const row of enriched) {
         if (row[property] instanceof Date) {
           const hours = row[property].getUTCHours().toString().padStart(2, "0")
           const minutes = row[property]
@@ -343,14 +348,36 @@ export async function outputProcessing<T extends Row[] | Row>(
     )) as Row[]
   }
   // remove null properties to match internal API
-  if (isExternalTableID(table._id!)) {
+  const isExternal = isExternalTableID(table._id!)
-    for (let row of enriched) {
+  if (isExternal) {
-      for (let key of Object.keys(row)) {
+    for (const row of enriched) {
+      for (const key of Object.keys(row)) {
         if (row[key] === null) {
           delete row[key]
         }
       }
     }
   }
+
+  if (!isUserMetadataTable(table._id!)) {
+    const protectedColumns = isExternal
+      ? PROTECTED_EXTERNAL_COLUMNS
+      : PROTECTED_INTERNAL_COLUMNS
+
+    const tableFields = Object.keys(table.schema).filter(
+      f => table.schema[f].visible !== false
+    )
+    const fields = [...tableFields, ...protectedColumns].map(f =>
+      f.toLowerCase()
+    )
+    for (const row of enriched) {
+      for (const key of Object.keys(row)) {
+        if (!fields.includes(key.toLowerCase())) {
+          delete row[key]
+        }
+      }
+    }
+  }
+
   return (wasArray ? enriched : enriched[0]) as T
 }