Merge pull request #15169 from Budibase/BUDI-8885/only-select-required-columns-from-sql-databases

Only select required columns from sql databases

packages/backend-core/src/sql/sql.ts

@@ -272,17 +272,6 @@ class InternalBuilder {
     return parts.join(".")
   }
 
-  private isFullSelectStatementRequired(): boolean {
-    for (let column of Object.values(this.table.schema)) {
-      if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) {
-        return true
-      } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) {
-        return true
-      }
-    }
-    return false
-  }
-
   private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
     const { table, resource } = this.query
 
@@ -292,11 +281,9 @@ class InternalBuilder {
 
     const alias = this.getTableName(table)
     const schema = this.table.schema
-    if (!this.isFullSelectStatementRequired()) {
+
-      return [this.knex.raw("??", [`${alias}.*`])]
-    }
     // get just the fields for this table
-    return resource.fields
+    const tableFields = resource.fields
       .map(field => {
         const parts = field.split(/\./g)
         let table: string | undefined = undefined
@@ -311,7 +298,8 @@ class InternalBuilder {
         return { table, column, field }
       })
       .filter(({ table }) => !table || table === alias)
-      .map(({ table, column, field }) => {
+
+    return tableFields.map(({ table, column, field }) => {
       const columnSchema = schema[column]
 
       if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) {
@@ -325,8 +313,6 @@ class InternalBuilder {
         // Time gets returned as timestamp from mssql, not matching the expected
         // HH:mm format
 
-          // TODO: figure out how to express this safely without string
-          // interpolation.
         return this.knex.raw(`CONVERT(varchar, ??, 108) as ??`, [
           this.rawQuotedIdentifier(field),
           this.knex.raw(this.quote(field)),
@@ -1291,6 +1277,7 @@ class InternalBuilder {
       if (!toTable || !fromTable) {
         continue
       }
+
       const relatedTable = tables[toTable]
       if (!relatedTable) {
         throw new Error(`related table "${toTable}" not found in datasource`)
@@ -1319,6 +1306,10 @@ class InternalBuilder {
       const fieldList = relationshipFields.map(field =>
         this.buildJsonField(relatedTable, field)
       )
+      if (!fieldList.length) {
+        continue
+      }
+
       const fieldListFormatted = fieldList
         .map(f => {
           const separator = this.client === SqlClient.ORACLE ? " VALUE " : ","
@@ -1359,7 +1350,9 @@ class InternalBuilder {
       )
 
       const standardWrap = (select: Knex.Raw): Knex.QueryBuilder => {
-        subQuery = subQuery.select(`${toAlias}.*`).limit(getRelationshipLimit())
+        subQuery = subQuery
+          .select(relationshipFields)
+          .limit(getRelationshipLimit())
         // @ts-ignore - the from alias syntax isn't in Knex typing
         return knex.select(select).from({
           [toAlias]: subQuery,
@@ -1589,11 +1582,12 @@ class InternalBuilder {
       limits?: { base: number; query: number }
     } = {}
   ): Knex.QueryBuilder {
-    let { operation, filters, paginate, relationships, table } = this.query
+    const { operation, filters, paginate, relationships, table } = this.query
     const { limits } = opts
 
     // start building the query
     let query = this.qualifiedKnex()
+
     // handle pagination
     let foundOffset: number | null = null
     let foundLimit = limits?.query || limits?.base
@@ -1642,7 +1636,7 @@ class InternalBuilder {
       const mainTable = this.query.tableAliases?.[table.name] || table.name
       const cte = this.addSorting(
         this.knex
-          .with("paginated", query)
+          .with("paginated", query.clone().clearSelect().select("*"))
           .select(this.generateSelectStatement())
           .from({
             [mainTable]: "paginated",

packages/server/src/api/controllers/row/external.ts

@@ -45,6 +45,9 @@ export async function handleRequest<T extends Operation>(
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const source = await utils.getSource(ctx)
 
+  const { viewId, tableId } = utils.getSourceId(ctx)
+  const sourceId = viewId || tableId
+
   if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
     ctx.throw(400, "Cannot update rows through a calculation view")
   }
@@ -86,7 +89,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   // The id might have been changed, so the refetching would fail. Recalculating the id just in case
   const updatedId =
     generateIdForRow({ ...beforeRow, ...dataToUpdate }, table) || _id
-  const row = await sdk.rows.external.getRow(table._id!, updatedId, {
+  const row = await sdk.rows.external.getRow(sourceId, updatedId, {
     relationships: true,
   })
 

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -14,7 +14,8 @@ import {
 import { breakExternalTableId } from "../../../../integrations/utils"
 import { generateJunctionTableID } from "../../../../db/utils"
 import sdk from "../../../../sdk"
-import { helpers } from "@budibase/shared-core"
+import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
+import { sql } from "@budibase/backend-core"
 
 type TableMap = Record<string, Table>
 
@@ -118,45 +119,131 @@ export async function buildSqlFieldList(
   opts?: { relationships: boolean }
 ) {
   const { relationships } = opts || {}
+
+  const nonMappedColumns = [FieldType.LINK, FieldType.FORMULA, FieldType.AI]
+
   function extractRealFields(table: Table, existing: string[] = []) {
     return Object.entries(table.schema)
       .filter(
         ([columnName, column]) =>
-          column.type !== FieldType.LINK &&
+          !nonMappedColumns.includes(column.type) &&
-          column.type !== FieldType.FORMULA &&
+          !existing.find((field: string) => field === columnName)
-          column.type !== FieldType.AI &&
-          !existing.find(
-            (field: string) => field === `${table.name}.${columnName}`
       )
+      .map(([columnName]) => columnName)
+  }
+
+  function getRequiredFields(table: Table, existing: string[] = []) {
+    const requiredFields: string[] = []
+    if (table.primary) {
+      requiredFields.push(...table.primary)
+    }
+    if (table.primaryDisplay) {
+      requiredFields.push(table.primaryDisplay)
+    }
+
+    if (!sql.utils.isExternalTable(table)) {
+      requiredFields.push(...PROTECTED_INTERNAL_COLUMNS)
+    }
+
+    return requiredFields.filter(
+      column =>
+        !existing.find((field: string) => field === column) &&
+        table.schema[column] &&
+        !nonMappedColumns.includes(table.schema[column].type)
     )
-      .map(([columnName]) => `${table.name}.${columnName}`)
   }
 
   let fields: string[] = []
-  if (sdk.views.isView(source)) {
+
-    fields = Object.keys(helpers.views.basicFields(source))
+  const isView = sdk.views.isView(source)
-  } else {
-    fields = extractRealFields(source)
-  }
 
   let table: Table
-  if (sdk.views.isView(source)) {
+  if (isView) {
     table = await sdk.views.getTable(source.id)
+
+    fields = Object.keys(helpers.views.basicFields(source)).filter(
+      f => table.schema[f].type !== FieldType.LINK
+    )
   } else {
     table = source
+    fields = extractRealFields(source).filter(
+      f => table.schema[f].visible !== false
+    )
   }
 
-  for (let field of Object.values(table.schema)) {
+  const containsFormula = (isView ? fields : Object.keys(table.schema)).some(
+    f => table.schema[f]?.type === FieldType.FORMULA
+  )
+  // If are requesting for a formula field, we need to retrieve all fields
+  if (containsFormula) {
+    fields = extractRealFields(table)
+  }
+
+  if (!isView || !helpers.views.isCalculationView(source)) {
+    fields.push(
+      ...getRequiredFields(
+        {
+          ...table,
+          primaryDisplay: source.primaryDisplay || table.primaryDisplay,
+        },
+        fields
+      )
+    )
+  }
+
+  fields = fields.map(c => `${table.name}.${c}`)
+
+  for (const field of Object.values(table.schema)) {
     if (field.type !== FieldType.LINK || !relationships || !field.tableId) {
       continue
     }
+
+    if (
+      isView &&
+      (!source.schema?.[field.name] ||
+        !helpers.views.isVisible(source.schema[field.name])) &&
+      !containsFormula
+    ) {
+      continue
+    }
+
     const { tableName } = breakExternalTableId(field.tableId)
-    if (tables[tableName]) {
+    const relatedTable = tables[tableName]
-      fields = fields.concat(extractRealFields(tables[tableName], fields))
+    if (!relatedTable) {
+      continue
+    }
+
+    const viewFields = new Set<string>()
+    if (containsFormula) {
+      extractRealFields(relatedTable).forEach(f => viewFields.add(f))
+    } else {
+      relatedTable.primary?.forEach(f => viewFields.add(f))
+      if (relatedTable.primaryDisplay) {
+        viewFields.add(relatedTable.primaryDisplay)
+      }
+
+      if (isView) {
+        Object.entries(source.schema?.[field.name]?.columns || {})
+          .filter(
+            ([columnName, columnConfig]) =>
+              relatedTable.schema[columnName] &&
+              helpers.views.isVisible(columnConfig) &&
+              ![FieldType.LINK, FieldType.FORMULA].includes(
+                relatedTable.schema[columnName].type
+              )
+          )
+          .forEach(([field]) => viewFields.add(field))
       }
     }
 
-  return fields
+    const fieldsToAdd = Array.from(viewFields)
+      .filter(f => !nonMappedColumns.includes(relatedTable.schema[f].type))
+      .map(f => `${relatedTable.name}.${f}`)
+      .filter(f => !fields.includes(f))
+    fields.push(...fieldsToAdd)
+  }
+
+  return [...new Set(fields)]
 }
 
 export function isKnexEmptyReadResponse(resp: DatasourcePlusQueryResponse) {

packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts

@@ -0,0 +1,511 @@
+import {
+  AIOperationEnum,
+  CalculationType,
+  FieldType,
+  RelationshipType,
+  SourceName,
+  Table,
+  ViewV2,
+  ViewV2Type,
+} from "@budibase/types"
+import { buildSqlFieldList } from "../sqlUtils"
+import { structures } from "../../../../routes/tests/utilities"
+import { sql } from "@budibase/backend-core"
+import { generator } from "@budibase/backend-core/tests"
+import { generateViewID } from "../../../../../db/utils"
+
+import sdk from "../../../../../sdk"
+import { cloneDeep } from "lodash"
+import { utils } from "@budibase/shared-core"
+
+jest.mock("../../../../../sdk/app/views", () => ({
+  ...jest.requireActual("../../../../../sdk/app/views"),
+  getTable: jest.fn(),
+}))
+const getTableMock = sdk.views.getTable as jest.MockedFunction<
+  typeof sdk.views.getTable
+>
+
+describe("buildSqlFieldList", () => {
+  let allTables: Record<string, Table>
+
+  class TableConfig {
+    private _table: Table & { _id: string }
+
+    constructor(name: string) {
+      this._table = {
+        ...structures.tableForDatasource({
+          type: "datasource",
+          source: SourceName.POSTGRES,
+        }),
+        name,
+        _id: sql.utils.buildExternalTableId("ds_id", name),
+        schema: {
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+          },
+          description: {
+            name: "description",
+            type: FieldType.STRING,
+          },
+          amount: {
+            name: "amount",
+            type: FieldType.NUMBER,
+          },
+        },
+      }
+
+      allTables[name] = this._table
+    }
+
+    withHiddenField(field: string) {
+      this._table.schema[field].visible = false
+      return this
+    }
+
+    withField(
+      name: string,
+      type:
+        | FieldType.STRING
+        | FieldType.NUMBER
+        | FieldType.FORMULA
+        | FieldType.AI,
+      options?: { visible: boolean }
+    ) {
+      switch (type) {
+        case FieldType.NUMBER:
+        case FieldType.STRING:
+          this._table.schema[name] = {
+            name,
+            type,
+            ...options,
+          }
+          break
+        case FieldType.FORMULA:
+          this._table.schema[name] = {
+            name,
+            type,
+            formula: "any",
+            ...options,
+          }
+          break
+        case FieldType.AI:
+          this._table.schema[name] = {
+            name,
+            type,
+            operation: AIOperationEnum.PROMPT,
+            ...options,
+          }
+          break
+        default:
+          utils.unreachable(type)
+      }
+      return this
+    }
+
+    withRelation(name: string, toTableId: string) {
+      this._table.schema[name] = {
+        name,
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: toTableId,
+      }
+      return this
+    }
+
+    withPrimary(field: string) {
+      this._table.primary = [field]
+      return this
+    }
+
+    withDisplay(field: string) {
+      this._table.primaryDisplay = field
+      return this
+    }
+
+    create() {
+      return cloneDeep(this._table)
+    }
+  }
+
+  class ViewConfig {
+    private _table: Table
+    private _view: ViewV2
+
+    constructor(table: Table) {
+      this._table = table
+      this._view = {
+        version: 2,
+        id: generateViewID(table._id!),
+        name: generator.word(),
+        tableId: table._id!,
+      }
+    }
+
+    withVisible(field: string) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].visible = true
+      return this
+    }
+
+    withHidden(field: string) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].visible = false
+      return this
+    }
+
+    withRelationshipColumns(
+      field: string,
+      columns: Record<string, { visible: boolean }>
+    ) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].columns = columns
+      return this
+    }
+
+    withCalculation(
+      name: string,
+      field: string,
+      calculationType: CalculationType
+    ) {
+      this._view.type = ViewV2Type.CALCULATION
+      this._view.schema ??= {}
+      this._view.schema[name] = {
+        field,
+        calculationType,
+        visible: true,
+      }
+      return this
+    }
+
+    create() {
+      getTableMock.mockResolvedValueOnce(this._table)
+      return cloneDeep(this._view)
+    }
+  }
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+    allTables = {}
+  })
+
+  describe("table", () => {
+    it("extracts fields from table schema", async () => {
+      const table = new TableConfig("table").create()
+      const result = await buildSqlFieldList(table, {})
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+      ])
+    })
+
+    it("excludes hidden fields", async () => {
+      const table = new TableConfig("table")
+        .withHiddenField("description")
+        .create()
+      const result = await buildSqlFieldList(table, {})
+      expect(result).toEqual(["table.name", "table.amount"])
+    })
+
+    it("excludes non-sql fields fields", async () => {
+      const table = new TableConfig("table")
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .create()
+
+      const result = await buildSqlFieldList(table, {})
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+      ])
+    })
+
+    it("includes hidden fields if there is a formula column", async () => {
+      const table = new TableConfig("table")
+        .withHiddenField("description")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const result = await buildSqlFieldList(table, {})
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+      ])
+    })
+
+    it("includes relationships fields when flagged", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withPrimary("id")
+        .withDisplay("name")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const result = await buildSqlFieldList(table, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+        "linkedTable.id",
+        "linkedTable.name",
+      ])
+    })
+
+    it("includes all relationship fields if there is a formula column", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const result = await buildSqlFieldList(table, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+        "linkedTable.name",
+        "linkedTable.description",
+        "linkedTable.amount",
+        "linkedTable.hidden",
+      ])
+    })
+
+    it("never includes non-sql columns from relationships", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const result = await buildSqlFieldList(table, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+        "linkedTable.name",
+        "linkedTable.description",
+        "linkedTable.amount",
+        "linkedTable.id",
+        "linkedTable.hidden",
+      ])
+    })
+  })
+
+  describe("view", () => {
+    it("extracts fields from table schema", async () => {
+      const view = new ViewConfig(new TableConfig("table").create())
+        .withVisible("amount")
+        .withHidden("name")
+        .create()
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual(["table.amount"])
+    })
+
+    it("includes all fields if there is a formula column", async () => {
+      const table = new TableConfig("table")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+      const view = new ViewConfig(table)
+        .withHidden("name")
+        .withVisible("amount")
+        .withVisible("formula")
+        .create()
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+      ])
+    })
+
+    it("does not includes all fields if the formula column is not included", async () => {
+      const table = new TableConfig("table")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+      const view = new ViewConfig(table)
+        .withHidden("name")
+        .withVisible("amount")
+        .withHidden("formula")
+        .create()
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual(["table.amount"])
+    })
+
+    it("includes relationships columns", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withField("formula", FieldType.FORMULA)
+        .withPrimary("id")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const result = await buildSqlFieldList(view, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "linkedTable.id",
+        "linkedTable.amount",
+      ])
+    })
+
+    it("excludes relationships fields when view is not included in the view", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withPrimary("id")
+        .withDisplay("name")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withHidden("amount")
+        .create()
+
+      const result = await buildSqlFieldList(view, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual(["table.name"])
+    })
+
+    it("does not include relationships columns for hidden links", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withField("formula", FieldType.FORMULA)
+        .withPrimary("id")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withHidden("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const result = await buildSqlFieldList(view, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual(["table.name"])
+    })
+
+    it("includes all relationship fields if there is a formula column", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .withPrimary("id")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("formula")
+        .withHidden("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const result = await buildSqlFieldList(view, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+        "linkedTable.name",
+        "linkedTable.description",
+        "linkedTable.amount",
+        "linkedTable.id",
+        "linkedTable.hidden",
+      ])
+    })
+  })
+
+  describe("calculation view", () => {
+    it("does not include calculation fields", async () => {
+      const view = new ViewConfig(new TableConfig("table").create())
+        .withCalculation("average", "amount", CalculationType.AVG)
+
+        .create()
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual([])
+    })
+
+    it("includes visible fields calculation fields", async () => {
+      const view = new ViewConfig(new TableConfig("table").create())
+        .withCalculation("average", "amount", CalculationType.AVG)
+        .withHidden("name")
+        .withVisible("amount")
+
+        .create()
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual(["table.amount"])
+    })
+  })
+})

packages/server/src/integration-test/postgres.spec.ts

@@ -13,7 +13,7 @@ const mainDescriptions = datasourceDescribe({
 
 if (mainDescriptions.length) {
   describe.each(mainDescriptions)(
-    "/postgres integrations",
+    "/postgres integrations ($dbName)",
     ({ config, dsProvider }) => {
       let datasource: Datasource
       let client: Knex

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -73,6 +73,27 @@ describe("Captures of real examples", () => {
   })
 
   describe("read", () => {
+    it("should retrieve all fields if non are specified", () => {
+      const queryJson = getJson("basicFetch.json")
+      delete queryJson.resource
+
+      let query = new Sql(SqlClient.POSTGRES)._query(queryJson)
+      expect(query).toEqual({
+        bindings: [primaryLimit],
+        sql: `select * from "persons" as "a" order by "a"."firstname" asc nulls first, "a"."personid" asc limit $1`,
+      })
+    })
+
+    it("should retrieve only requested fields", () => {
+      const queryJson = getJson("basicFetch.json")
+
+      let query = new Sql(SqlClient.POSTGRES)._query(queryJson)
+      expect(query).toEqual({
+        bindings: [primaryLimit],
+        sql: `select "a"."year", "a"."firstname", "a"."personid", "a"."age", "a"."type", "a"."lastname" from "persons" as "a" order by "a"."firstname" asc nulls first, "a"."personid" asc limit $1`,
+      })
+    })
+
     it("should handle basic retrieval with relationships", () => {
       const queryJson = getJson("basicFetchWithRelationships.json")
       let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
@@ -112,9 +133,9 @@ describe("Captures of real examples", () => {
         bindings: [primaryLimit, relationshipLimit],
         sql: expect.stringContaining(
           multiline(
-            `with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) 
+            `with "paginated" as (select * from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) 
-                 select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) 
+                 select "a"."productname", "a"."productid", (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) 
-                 from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" 
+                 from (select "b"."executorid", "b"."qaid", "b"."taskid", "b"."completed", "b"."taskname" from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" 
                  from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc`
           )
         ),
@@ -130,9 +151,9 @@ describe("Captures of real examples", () => {
       expect(query).toEqual({
         bindings: [...filters, relationshipLimit, relationshipLimit],
         sql: multiline(
-          `with "paginated" as (select "a".* from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) 
+          `with "paginated" as (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) 
-               select "a".*, (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) 
+               select "a"."executorid", "a"."taskname", "a"."taskid", "a"."completed", "a"."qaid", (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) 
-               from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" 
+               from (select "b"."productid", "b"."productname" from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" 
                where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc`
         ),
       })
@@ -209,7 +230,7 @@ describe("Captures of real examples", () => {
         bindings: ["ddd", ""],
         sql: multiline(`delete from "compositetable" as "a" 
           where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE)
-          returning "a".*`),
+          returning "a"."keyparttwo", "a"."keypartone", "a"."name"`),
       })
     })
   })

packages/server/src/integrations/tests/sqlQueryJson/basicFetch.json

@@ -0,0 +1,137 @@
+{
+  "operation": "READ",
+  "resource": {
+    "fields": [
+      "a.year",
+      "a.firstname",
+      "a.personid",
+      "a.age",
+      "a.type",
+      "a.lastname"
+    ]
+  },
+  "filters": {},
+  "sort": {
+    "firstname": {
+      "direction": "ascending"
+    }
+  },
+  "paginate": {
+    "limit": 100,
+    "page": 1
+  },
+  "relationships": [],
+  "extra": {
+    "idFilter": {}
+  },
+  "table": {
+    "type": "table",
+    "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
+    "primary": ["personid"],
+    "name": "persons",
+    "schema": {
+      "year": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": false,
+        "name": "year",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "firstname": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "firstname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "personid": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": true,
+        "name": "personid",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "address": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "address",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "age": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": false,
+        "name": "age",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "type": {
+        "type": "options",
+        "externalType": "USER-DEFINED",
+        "autocolumn": false,
+        "name": "type",
+        "constraints": {
+          "presence": false,
+          "inclusion": ["support", "designer", "programmer", "qa"]
+        }
+      },
+      "city": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "city",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "lastname": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "lastname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "QA": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "QA",
+        "relationshipType": "many-to-one",
+        "fieldName": "qaid",
+        "type": "link",
+        "main": true,
+        "_id": "ccb68481c80c34217a4540a2c6c27fe46",
+        "foreignKey": "personid"
+      },
+      "executor": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "executor",
+        "relationshipType": "many-to-one",
+        "fieldName": "executorid",
+        "type": "link",
+        "main": true,
+        "_id": "c89530b9770d94bec851e062b5cff3001",
+        "foreignKey": "personid",
+        "tableName": "persons"
+      }
+    },
+    "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "sourceType": "external",
+    "primaryDisplay": "firstname",
+    "views": {}
+  },
+  "tableAliases": {
+    "persons": "a",
+    "tasks": "b"
+  }
+}

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -55,32 +55,44 @@ const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`)
 const MISSING_TABLE_REGX = new RegExp(`no such table: .+`)
 const DUPLICATE_COLUMN_REGEX = new RegExp(`duplicate column name: .+`)
 
-async function buildInternalFieldList(
+export async function buildInternalFieldList(
   source: Table | ViewV2,
   tables: Table[],
-  opts?: { relationships?: RelationshipsJson[]; allowedFields?: string[] }
+  opts?: {
+    relationships?: RelationshipsJson[]
+    allowedFields?: string[]
+    includeHiddenFields?: boolean
+  }
 ) {
-  const { relationships, allowedFields } = opts || {}
+  const { relationships, allowedFields, includeHiddenFields } = opts || {}
   let schemaFields: string[] = []
-  if (sdk.views.isView(source)) {
-    schemaFields = Object.keys(helpers.views.basicFields(source))
-  } else {
-    schemaFields = Object.keys(source.schema).filter(
-      key => source.schema[key].visible !== false
-    )
-  }
-
-  if (allowedFields) {
-    schemaFields = schemaFields.filter(field => allowedFields.includes(field))
-  }
 
+  const isView = sdk.views.isView(source)
   let table: Table
-  if (sdk.views.isView(source)) {
+  if (isView) {
     table = await sdk.views.getTable(source.id)
   } else {
     table = source
   }
 
+  if (isView) {
+    schemaFields = Object.keys(helpers.views.basicFields(source))
+  } else {
+    schemaFields = Object.keys(source.schema).filter(
+      key => includeHiddenFields || source.schema[key].visible !== false
+    )
+  }
+
+  const containsFormula = schemaFields.some(
+    f => table.schema[f]?.type === FieldType.FORMULA
+  )
+  // If are requesting for a formula field, we need to retrieve all fields
+  if (containsFormula) {
+    schemaFields = Object.keys(table.schema)
+  } else if (allowedFields) {
+    schemaFields = schemaFields.filter(field => allowedFields.includes(field))
+  }
+
   let fieldList: string[] = []
   const getJunctionFields = (relatedTable: Table, fields: string[]) => {
     const junctionFields: string[] = []
@@ -101,10 +113,12 @@ async function buildInternalFieldList(
   }
   for (let key of schemaFields) {
     const col = table.schema[key]
+
     const isRelationship = col.type === FieldType.LINK
     if (!relationships && isRelationship) {
       continue
     }
+
     if (!isRelationship) {
       fieldList.push(`${table._id}.${mapToUserColumn(key)}`)
     } else {
@@ -113,8 +127,17 @@ async function buildInternalFieldList(
       if (!relatedTable) {
         continue
       }
+
+      // a quirk of how junction documents work in Budibase, refer to the "LinkDocument" type to see the full
+      // structure - essentially all relationships between two tables will be inserted into a single "table"
+      // we don't use an independent junction table ID for each separate relationship between two tables. For
+      // example if we have table A and B, with two relationships between them, all the junction documents will
+      // end up in the same junction table ID. We need to retrieve the field name property of the junction documents
+      // as part of the relationship to tell us which relationship column the junction is related to.
       const relatedFields = (
-        await buildInternalFieldList(relatedTable, tables)
+        await buildInternalFieldList(relatedTable, tables, {
+          includeHiddenFields: containsFormula,
+        })
       ).concat(
         getJunctionFields(relatedTable, ["doc1.fieldName", "doc2.fieldName"])
       )
@@ -125,6 +148,13 @@ async function buildInternalFieldList(
       fieldList = fieldList.concat(relatedFields)
     }
   }
+
+  if (!isView || !helpers.views.isCalculationView(source)) {
+    for (const field of PROTECTED_INTERNAL_COLUMNS) {
+      fieldList.push(`${table._id}.${field}`)
+    }
+  }
+
   return [...new Set(fieldList)]
 }
 
@@ -323,8 +353,9 @@ export async function search(
   }
 
   let aggregations: Aggregation[] = []
-  if (sdk.views.isView(source)) {
+  if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
     const calculationFields = helpers.views.calculationFields(source)
+
     for (const [key, field] of Object.entries(calculationFields)) {
       if (options.fields && !options.fields.includes(key)) {
         continue

packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts

@@ -0,0 +1,618 @@
+import {
+  AIOperationEnum,
+  CalculationType,
+  FieldType,
+  RelationshipType,
+  SourceName,
+  Table,
+  ViewV2,
+  ViewV2Type,
+} from "@budibase/types"
+import { buildInternalFieldList } from "../sqs"
+import { structures } from "../../../../../../api/routes/tests/utilities"
+import { sql } from "@budibase/backend-core"
+import { generator } from "@budibase/backend-core/tests"
+import {
+  generateJunctionTableID,
+  generateViewID,
+} from "../../../../../../db/utils"
+
+import sdk from "../../../../../../sdk"
+import { cloneDeep } from "lodash"
+import { utils } from "@budibase/shared-core"
+
+jest.mock("../../../../../../sdk/app/views", () => ({
+  ...jest.requireActual("../../../../../../sdk/app/views"),
+  getTable: jest.fn(),
+}))
+const getTableMock = sdk.views.getTable as jest.MockedFunction<
+  typeof sdk.views.getTable
+>
+
+describe("buildInternalFieldList", () => {
+  let allTables: Table[]
+
+  class TableConfig {
+    private _table: Table & { _id: string }
+
+    constructor() {
+      const name = generator.word()
+      this._table = {
+        ...structures.tableForDatasource({
+          type: "datasource",
+          source: SourceName.POSTGRES,
+        }),
+        name,
+        _id: sql.utils.buildExternalTableId("ds_id", name),
+        schema: {
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+          },
+          description: {
+            name: "description",
+            type: FieldType.STRING,
+          },
+          amount: {
+            name: "amount",
+            type: FieldType.NUMBER,
+          },
+        },
+      }
+
+      allTables.push(this._table)
+    }
+
+    withHiddenField(field: string) {
+      this._table.schema[field].visible = false
+      return this
+    }
+
+    withField(
+      name: string,
+      type:
+        | FieldType.STRING
+        | FieldType.NUMBER
+        | FieldType.FORMULA
+        | FieldType.AI,
+      options?: { visible: boolean }
+    ) {
+      switch (type) {
+        case FieldType.NUMBER:
+        case FieldType.STRING:
+          this._table.schema[name] = {
+            name,
+            type,
+            ...options,
+          }
+          break
+        case FieldType.FORMULA:
+          this._table.schema[name] = {
+            name,
+            type,
+            formula: "any",
+            ...options,
+          }
+          break
+        case FieldType.AI:
+          this._table.schema[name] = {
+            name,
+            type,
+            operation: AIOperationEnum.PROMPT,
+            ...options,
+          }
+          break
+        default:
+          utils.unreachable(type)
+      }
+      return this
+    }
+
+    withRelation(name: string, toTableId: string) {
+      this._table.schema[name] = {
+        name,
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: toTableId,
+      }
+      return this
+    }
+
+    withEmptySchema() {
+      this._table.schema = {}
+      return this
+    }
+
+    create() {
+      return cloneDeep(this._table)
+    }
+  }
+
+  class ViewConfig {
+    private _table: Table
+    private _view: ViewV2
+
+    constructor(table: Table) {
+      this._table = table
+      this._view = {
+        version: 2,
+        id: generateViewID(table._id!),
+        name: generator.word(),
+        tableId: table._id!,
+      }
+    }
+
+    withVisible(field: string) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].visible = true
+      return this
+    }
+
+    withHidden(field: string) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].visible = false
+      return this
+    }
+
+    withRelationshipColumns(
+      field: string,
+      columns: Record<string, { visible: boolean }>
+    ) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].columns = columns
+      return this
+    }
+
+    withCalculation(
+      name: string,
+      field: string,
+      calculationType: CalculationType
+    ) {
+      this._view.type = ViewV2Type.CALCULATION
+      this._view.schema ??= {}
+      this._view.schema[name] = {
+        field,
+        calculationType,
+        visible: true,
+      }
+      return this
+    }
+
+    create() {
+      getTableMock.mockResolvedValueOnce(this._table)
+      return cloneDeep(this._view)
+    }
+  }
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+    allTables = []
+  })
+
+  describe("table", () => {
+    it("includes internal columns by default", async () => {
+      const table = new TableConfig().withEmptySchema().create()
+      const result = await buildInternalFieldList(table, [])
+      expect(result).toEqual([
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("extracts fields from table schema", async () => {
+      const table = new TableConfig().create()
+      const result = await buildInternalFieldList(table, [])
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("excludes hidden fields", async () => {
+      const table = new TableConfig().withHiddenField("description").create()
+      const result = await buildInternalFieldList(table, [])
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_amount`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes hidden fields if there is a formula column", async () => {
+      const table = new TableConfig()
+        .withHiddenField("description")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const result = await buildInternalFieldList(table, [])
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${table._id}.data_formula`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes relationships fields when flagged", async () => {
+      const otherTable = new TableConfig()
+        .withHiddenField("description")
+        .create()
+
+      const table = new TableConfig()
+        .withHiddenField("amount")
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+
+      const result = await buildInternalFieldList(table, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes all relationship fields if there is a formula column", async () => {
+      const otherTable = new TableConfig()
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .withHiddenField("description")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(table, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_description`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}.data_hidden`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}.data_formula`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+  })
+
+  describe("view", () => {
+    it("includes internal columns by default", async () => {
+      const view = new ViewConfig(new TableConfig().create()).create()
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([
+        `${view.tableId}._id`,
+        `${view.tableId}._rev`,
+        `${view.tableId}.type`,
+        `${view.tableId}.createdAt`,
+        `${view.tableId}.updatedAt`,
+        `${view.tableId}.tableId`,
+      ])
+    })
+
+    it("extracts fields from table schema", async () => {
+      const view = new ViewConfig(new TableConfig().create())
+        .withVisible("amount")
+        .withHidden("name")
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([
+        `${view.tableId}.data_amount`,
+        `${view.tableId}._id`,
+        `${view.tableId}._rev`,
+        `${view.tableId}.type`,
+        `${view.tableId}.createdAt`,
+        `${view.tableId}.updatedAt`,
+        `${view.tableId}.tableId`,
+      ])
+    })
+
+    it("includes all fields if there is a formula column", async () => {
+      const table = new TableConfig()
+        .withField("formula", FieldType.FORMULA)
+        .create()
+      const view = new ViewConfig(table)
+        .withHidden("name")
+        .withVisible("amount")
+        .withVisible("formula")
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([
+        `${view.tableId}.data_name`,
+        `${view.tableId}.data_description`,
+        `${view.tableId}.data_amount`,
+        `${view.tableId}.data_formula`,
+        `${view.tableId}._id`,
+        `${view.tableId}._rev`,
+        `${view.tableId}.type`,
+        `${view.tableId}.createdAt`,
+        `${view.tableId}.updatedAt`,
+        `${view.tableId}.tableId`,
+      ])
+    })
+
+    it("does not includes all fields if the formula column is not included", async () => {
+      const table = new TableConfig()
+        .withField("formula", FieldType.FORMULA)
+        .create()
+      const view = new ViewConfig(table)
+        .withHidden("name")
+        .withVisible("amount")
+        .withHidden("formula")
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([
+        `${view.tableId}.data_amount`,
+        `${view.tableId}._id`,
+        `${view.tableId}._rev`,
+        `${view.tableId}.type`,
+        `${view.tableId}.createdAt`,
+        `${view.tableId}.updatedAt`,
+        `${view.tableId}.tableId`,
+      ])
+    })
+
+    it("includes relationships fields", async () => {
+      const otherTable = new TableConfig().create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("link")
+        .withHidden("amount")
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(view, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_description`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes relationships columns", async () => {
+      const otherTable = new TableConfig()
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(view, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_description`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}.data_formula`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("does not include relationships columns for hidden links", async () => {
+      const otherTable = new TableConfig()
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withHidden("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(view, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes all relationship fields if there is a formula column", async () => {
+      const otherTable = new TableConfig()
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("formula")
+        .withHidden("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(view, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_description`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}.data_hidden`,
+        `${otherTable._id}.data_formula`,
+        `${otherTable._id}.data_ai`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}.data_formula`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+  })
+
+  describe("calculation view", () => {
+    it("does not include calculation fields", async () => {
+      const view = new ViewConfig(new TableConfig().create())
+        .withCalculation("average", "amount", CalculationType.AVG)
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([])
+    })
+
+    it("includes visible fields calculation fields", async () => {
+      const view = new ViewConfig(new TableConfig().create())
+        .withCalculation("average", "amount", CalculationType.AVG)
+        .withHidden("name")
+        .withVisible("amount")
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([`${view.tableId}.data_amount`])
+    })
+  })
+})