Merge branch 'master' into BUDI-8441/row-action-run-api

packages/backend-core/tests/core/utilities/jestUtils.ts

@@ -1,6 +1,6 @@
 import {
-  CONSTANT_EXTERNAL_ROW_COLS,
-  CONSTANT_INTERNAL_ROW_COLS,
+  PROTECTED_EXTERNAL_COLUMNS,
+  PROTECTED_INTERNAL_COLUMNS,
 } from "@budibase/shared-core"
 
 export function expectFunctionWasCalledTimesWith(
@@ -14,7 +14,7 @@ export function expectFunctionWasCalledTimesWith(
 }
 
 export const expectAnyInternalColsAttributes: {
-  [K in (typeof CONSTANT_INTERNAL_ROW_COLS)[number]]: any
+  [K in (typeof PROTECTED_INTERNAL_COLUMNS)[number]]: any
 } = {
   tableId: expect.anything(),
   type: expect.anything(),
@@ -25,7 +25,7 @@ export const expectAnyInternalColsAttributes: {
 }
 
 export const expectAnyExternalColsAttributes: {
-  [K in (typeof CONSTANT_EXTERNAL_ROW_COLS)[number]]: any
+  [K in (typeof PROTECTED_EXTERNAL_COLUMNS)[number]]: any
 } = {
   tableId: expect.anything(),
   _id: expect.anything(),

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -17,8 +17,8 @@
     SWITCHABLE_TYPES,
     ValidColumnNameRegex,
     helpers,
-    CONSTANT_INTERNAL_ROW_COLS,
-    CONSTANT_EXTERNAL_ROW_COLS,
+    PROTECTED_INTERNAL_COLUMNS,
+    PROTECTED_EXTERNAL_COLUMNS,
   } from "@budibase/shared-core"
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
@@ -489,8 +489,8 @@
     }
     const newError = {}
     const prohibited = externalTable
-      ? CONSTANT_EXTERNAL_ROW_COLS
-      : CONSTANT_INTERNAL_ROW_COLS
+      ? PROTECTED_EXTERNAL_COLUMNS
+      : PROTECTED_INTERNAL_COLUMNS
     if (!externalTable && fieldInfo.name?.startsWith("_")) {
       newError.name = `Column name cannot start with an underscore.`
     } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) {

packages/server/src/api/controllers/row/utils/basic.ts

@@ -1,8 +1,7 @@
 // need to handle table name + field or just field, depending on if relationships used
 import { FieldType, Row, Table } from "@budibase/types"
-import { helpers } from "@budibase/shared-core"
+import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
 import { generateRowIdField } from "../../../../integrations/utils"
-import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
 
 function extractFieldValue({
   row,
@@ -94,7 +93,7 @@ export function basicProcessing({
     thisRow._rev = "rev"
   } else {
     const columns = Object.keys(table.schema)
-    for (let internalColumn of [...CONSTANT_INTERNAL_ROW_COLS, ...columns]) {
+    for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) {
       thisRow[internalColumn] = extractFieldValue({
         row,
         tableName: table._id!,

packages/server/src/api/controllers/table/internal.ts

@@ -25,6 +25,8 @@ export async function save(
     sourceType: rest.sourceType || TableSourceType.INTERNAL,
   }
 
+  const isImport = !!rows
+
   if (!tableToSave.views) {
     tableToSave.views = {}
   }
@@ -35,6 +37,7 @@ export async function save(
       rowsToImport: rows,
       tableId: ctx.request.body._id,
       renaming,
+      isImport,
     })
 
     return table

packages/server/src/db/utils.ts

@@ -57,14 +57,6 @@ export const getUserMetadataParams = dbCore.getUserMetadataParams
 export const generateUserMetadataID = dbCore.generateUserMetadataID
 export const getGlobalIDFromUserMetadataID =
   dbCore.getGlobalIDFromUserMetadataID
-export const CONSTANT_INTERNAL_ROW_COLS = [
-  "_id",
-  "_rev",
-  "type",
-  "createdAt",
-  "updatedAt",
-  "tableId",
-]
 
 /**
  * Gets parameters for retrieving tables, this is a utility function for the getDocParams function.

packages/server/src/sdk/app/rows/search/external.ts

@@ -16,7 +16,7 @@ import {
   breakExternalTableId,
   breakRowIdField,
 } from "../../../../integrations/utils"
-import { utils, CONSTANT_EXTERNAL_ROW_COLS } from "@budibase/shared-core"
+import { utils, PROTECTED_EXTERNAL_COLUMNS } from "@budibase/shared-core"
 import { ExportRowsParams, ExportRowsResult } from "./types"
 import { HTTPError } from "@budibase/backend-core"
 import pick from "lodash/pick"
@@ -99,7 +99,7 @@ export async function search(
     }
 
     if (options.fields) {
-      const fields = [...options.fields, ...CONSTANT_EXTERNAL_ROW_COLS]
+      const fields = [...options.fields, ...PROTECTED_EXTERNAL_COLUMNS]
       rows = rows.map((r: any) => pick(r, fields))
     }
 

packages/server/src/sdk/app/rows/search/internal.ts

@@ -1,5 +1,5 @@
 import { context, HTTPError } from "@budibase/backend-core"
-import { CONSTANT_INTERNAL_ROW_COLS } from "@budibase/shared-core"
+import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
 import env from "../../../../environment"
 import { fullSearch, paginatedSearch } from "./utils"
 import { getRowParams, InternalTables } from "../../../../db/utils"
@@ -75,7 +75,7 @@ export async function search(
     }
 
     if (options.fields) {
-      const fields = [...options.fields, ...CONSTANT_INTERNAL_ROW_COLS]
+      const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS]
       response.rows = response.rows.map((r: any) => pick(r, fields))
     }
 

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -31,10 +31,7 @@ import {
   SQLITE_DESIGN_DOC_ID,
   SQS_DATASOURCE_INTERNAL,
 } from "@budibase/backend-core"
-import {
-  CONSTANT_INTERNAL_ROW_COLS,
-  generateJunctionTableID,
-} from "../../../../db/utils"
+import { generateJunctionTableID } from "../../../../db/utils"
 import AliasTables from "../sqlAlias"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
 import pick from "lodash/pick"
@@ -44,7 +41,7 @@ import {
   getRelationshipColumns,
   getTableIDList,
 } from "./filters"
-import { dataFilters } from "@budibase/shared-core"
+import { dataFilters, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
 
 const builder = new sql.Sql(SqlClient.SQL_LITE)
 const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`)
@@ -65,7 +62,7 @@ function buildInternalFieldList(
     })
   }
   fieldList = fieldList.concat(
-    CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`)
+    PROTECTED_INTERNAL_COLUMNS.map(col => `${table._id}.${col}`)
   )
   for (let col of Object.values(table.schema)) {
     const isRelationship = col.type === FieldType.LINK
@@ -355,7 +352,7 @@ export async function search(
 
     // check if we need to pick specific rows out
     if (options.fields) {
-      const fields = [...options.fields, ...CONSTANT_INTERNAL_ROW_COLS]
+      const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS]
       finalRows = finalRows.map((r: any) => pick(r, fields))
     }
 

packages/server/src/sdk/app/tables/internal/index.ts

@@ -31,6 +31,7 @@ export async function save(
     tableId?: string
     rowsToImport?: Row[]
     renaming?: RenameColumn
+    isImport?: boolean
   }
 ) {
   const db = context.getAppDB()
@@ -47,7 +48,9 @@ export async function save(
   }
 
   // check for case sensitivity - we don't want to allow duplicated columns
-  const duplicateColumn = findDuplicateInternalColumns(table)
+  const duplicateColumn = findDuplicateInternalColumns(table, {
+    ignoreProtectedColumnNames: !oldTable && !!opts?.isImport,
+  })
   if (duplicateColumn.length) {
     throw new Error(
       `Column(s) "${duplicateColumn.join(

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -10,12 +10,10 @@ import {
   Table,
 } from "@budibase/types"
 import tablesSdk from "../"
-import {
-  CONSTANT_INTERNAL_ROW_COLS,
-  generateJunctionTableID,
-} from "../../../../db/utils"
+import { generateJunctionTableID } from "../../../../db/utils"
 import { isEqual } from "lodash"
 import { DEFAULT_TABLES } from "../../../../db/defaultData/datasource_bb_default"
+import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
 
 const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.BOOLEAN]: SQLiteType.NUMERIC,
@@ -122,7 +120,7 @@ function mapTable(table: Table): SQLiteTables {
   }
   // there are some extra columns to map - add these in
   const constantMap: Record<string, SQLiteType> = {}
-  CONSTANT_INTERNAL_ROW_COLS.forEach(col => {
+  PROTECTED_INTERNAL_COLUMNS.forEach(col => {
     constantMap[col] = SQLiteType.TEXT
   })
   const thisTable: SQLiteTable = {

packages/server/src/sdk/app/views/index.ts

@@ -10,8 +10,8 @@ import { HTTPError } from "@budibase/backend-core"
 import { features } from "@budibase/pro"
 import {
   helpers,
-  CONSTANT_EXTERNAL_ROW_COLS,
-  CONSTANT_INTERNAL_ROW_COLS,
+  PROTECTED_EXTERNAL_COLUMNS,
+  PROTECTED_INTERNAL_COLUMNS,
 } from "@budibase/shared-core"
 import { cloneDeep } from "lodash/fp"
 
@@ -148,8 +148,8 @@ export function allowedFields(view: View | ViewV2) {
       const fieldSchema = view.schema![key]
       return fieldSchema.visible && !fieldSchema.readonly
     }),
-    ...CONSTANT_EXTERNAL_ROW_COLS,
-    ...CONSTANT_INTERNAL_ROW_COLS,
+    ...PROTECTED_EXTERNAL_COLUMNS,
+    ...PROTECTED_INTERNAL_COLUMNS,
   ]
 }
 

packages/shared-core/src/constants/rows.ts

@@ -1,4 +1,4 @@
-export const CONSTANT_INTERNAL_ROW_COLS = [
+export const PROTECTED_INTERNAL_COLUMNS = [
   "_id",
   "_rev",
   "type",
@@ -7,8 +7,8 @@ export const CONSTANT_INTERNAL_ROW_COLS = [
   "tableId",
 ] as const
 
-export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const
+export const PROTECTED_EXTERNAL_COLUMNS = ["_id", "_rev", "tableId"] as const
 
 export function isInternalColumnName(name: string): boolean {
-  return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name)
+  return (PROTECTED_INTERNAL_COLUMNS as readonly string[]).includes(name)
 }

packages/shared-core/src/table.ts

@@ -1,5 +1,5 @@
 import { FieldType, Table } from "@budibase/types"
-import { CONSTANT_INTERNAL_ROW_COLS } from "./constants"
+import { PROTECTED_INTERNAL_COLUMNS } from "./constants"
 
 const allowDisplayColumnByType: Record<FieldType, boolean> = {
   [FieldType.STRING]: true,
@@ -53,7 +53,10 @@ export function canBeSortColumn(type: FieldType): boolean {
   return !!allowSortColumnByType[type]
 }
 
-export function findDuplicateInternalColumns(table: Table): string[] {
+export function findDuplicateInternalColumns(
+  table: Table,
+  opts?: { ignoreProtectedColumnNames: boolean }
+): string[] {
   // maintains the case of keys
   const casedKeys = Object.keys(table.schema)
   // get the column names
@@ -69,9 +72,11 @@ export function findDuplicateInternalColumns(table: Table): string[] {
       }
     }
   }
-  for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) {
-    if (casedKeys.find(key => key === internalColumn)) {
-      duplicates.push(internalColumn)
+  if (!opts?.ignoreProtectedColumnNames) {
+    for (let internalColumn of PROTECTED_INTERNAL_COLUMNS) {
+      if (casedKeys.find(key => key === internalColumn)) {
+        duplicates.push(internalColumn)
+      }
     }
   }
   return duplicates