Trim unwanted columns in outputProcessing

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -192,10 +192,10 @@ export function buildSqlFieldList(
   function extractRealFields(table: Table, existing: string[] = []) {
     return Object.entries(table.schema)
       .filter(
-        column =>
+        ([columnName, column]) =>
-          column[1].type !== FieldType.LINK &&
+          column.type !== FieldType.LINK &&
-          column[1].type !== FieldType.FORMULA &&
+          column.type !== FieldType.FORMULA &&
-          !existing.find((field: string) => field === column[0])
+          !existing.find((field: string) => field === columnName)
       )
       .map(column => `${table.name}.${column[0]}`)
   }

packages/server/src/utilities/rowProcessor/index.ts

@@ -19,6 +19,7 @@ import {
   User,
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
+import { pick } from "lodash"
 import {
   processInputBBReference,
   processInputBBReferences,
@@ -26,7 +27,11 @@ import {
   processOutputBBReferences,
 } from "./bbReferenceProcessor"
 import { isExternalTableID } from "../../integrations/utils"
-import { helpers } from "@budibase/shared-core"
+import {
+  helpers,
+  PROTECTED_EXTERNAL_COLUMNS,
+  PROTECTED_INTERNAL_COLUMNS,
+} from "@budibase/shared-core"
 import { processString } from "@budibase/string-templates"
 
 export * from "./utils"
@@ -53,9 +58,9 @@ export async function processAutoColumn(
   row: Row,
   opts?: AutoColumnProcessingOpts
 ) {
-  let noUser = !userId
+  const noUser = !userId
-  let isUserTable = table._id === InternalTables.USER_METADATA
+  const isUserTable = table._id === InternalTables.USER_METADATA
-  let now = new Date().toISOString()
+  const now = new Date().toISOString()
   // if a row doesn't have a revision then it doesn't exist yet
   const creating = !row._rev
   // check its not user table, or whether any of the processing options have been disabled
@@ -111,7 +116,7 @@ async function processDefaultValues(table: Table, row: Row) {
     ctx.user = user
   }
 
-  for (let [key, schema] of Object.entries(table.schema)) {
+  for (const [key, schema] of Object.entries(table.schema)) {
     if ("default" in schema && schema.default != null && row[key] == null) {
       const processed = await processString(schema.default, ctx)
 
@@ -165,10 +170,10 @@ export async function inputProcessing(
   row: Row,
   opts?: AutoColumnProcessingOpts
 ) {
-  let clonedRow = cloneDeep(row)
+  const clonedRow = cloneDeep(row)
 
   const dontCleanseKeys = ["type", "_id", "_rev", "tableId"]
-  for (let [key, value] of Object.entries(clonedRow)) {
+  for (const [key, value] of Object.entries(clonedRow)) {
     const field = table.schema[key]
     // cleanse fields that aren't in the schema
     if (!field) {
@@ -268,13 +273,13 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
 
   // process complex types: attachments, bb references...
-  for (let [property, column] of Object.entries(table.schema)) {
+  for (const [property, column] of Object.entries(table.schema)) {
     if (
       column.type === FieldType.ATTACHMENTS ||
       column.type === FieldType.ATTACHMENT_SINGLE ||
       column.type === FieldType.SIGNATURE_SINGLE
     ) {
-      for (let row of enriched) {
+      for (const row of enriched) {
         if (row[property] == null) {
           continue
         }
@@ -299,7 +304,7 @@ export async function outputProcessing<T extends Row[] | Row>(
       !opts.skipBBReferences &&
       column.type == FieldType.BB_REFERENCE
     ) {
-      for (let row of enriched) {
+      for (const row of enriched) {
         row[property] = await processOutputBBReferences(
           row[property],
           column.subtype
@@ -309,14 +314,14 @@ export async function outputProcessing<T extends Row[] | Row>(
       !opts.skipBBReferences &&
       column.type == FieldType.BB_REFERENCE_SINGLE
     ) {
-      for (let row of enriched) {
+      for (const row of enriched) {
         row[property] = await processOutputBBReference(
           row[property],
           column.subtype
         )
       }
     } else if (column.type === FieldType.DATETIME && column.timeOnly) {
-      for (let row of enriched) {
+      for (const row of enriched) {
         if (row[property] instanceof Date) {
           const hours = row[property].getUTCHours().toString().padStart(2, "0")
           const minutes = row[property]
@@ -343,14 +348,27 @@ export async function outputProcessing<T extends Row[] | Row>(
     )) as Row[]
   }
   // remove null properties to match internal API
-  if (isExternalTableID(table._id!)) {
+  const isExternal = isExternalTableID(table._id!)
-    for (let row of enriched) {
+  if (isExternal) {
-      for (let key of Object.keys(row)) {
+    for (const row of enriched) {
+      for (const key of Object.keys(row)) {
         if (row[key] === null) {
           delete row[key]
         }
       }
     }
   }
+
+  const protectedColumns = isExternal
+    ? PROTECTED_EXTERNAL_COLUMNS
+    : PROTECTED_INTERNAL_COLUMNS
+
+  const tableFields = Object.keys(table.schema).filter(
+    f => table.schema[f].visible !== false
+  )
+  enriched = enriched.map((r: Row) =>
+    pick(r, [...tableFields, ...protectedColumns])
+  )
+
   return (wasArray ? enriched : enriched[0]) as T
 }