Merge branch 'master' into fix/couchdb-image-build

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.11.38",
+  "version": "2.11.39",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/frontend-core/src/api/relationships.js

@@ -9,7 +9,9 @@ export const buildRelationshipEndpoints = API => ({
     if (!tableId || !rowId) {
       return []
     }
-    const response = await API.get({ url: `/api/${tableId}/${rowId}/enrich` })
+    const response = await API.get({
+      url: `/api/${tableId}/${rowId}/enrich?field=${fieldName}`,
+    })
     if (!fieldName) {
       return response || []
     } else {

packages/server/src/api/controllers/datasource.ts

@@ -4,7 +4,6 @@ import {
   getQueryParams,
   getTableParams,
 } from "../../db/utils"
-import { destroy as tableDestroy } from "./table/internal"
 import { getIntegration } from "../../integrations"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { context, db as dbCore, events } from "@budibase/backend-core"
@@ -325,11 +324,7 @@ async function destroyInternalTablesBySourceId(datasourceId: string) {
 
   // Destroy the tables.
   for (const table of datasourceTableDocs) {
-    await tableDestroy({
+    await sdk.tables.internal.destroy(table)
-      params: {
-        tableId: table._id,
-      },
-    })
   }
 }
 

packages/server/src/api/controllers/row/index.ts

@@ -26,6 +26,7 @@ import { fixRow } from "../public/rows"
 import sdk from "../../../sdk"
 import * as exporters from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
+import { Format } from "../view/exporters"
 export * as views from "./views"
 
 function pickApi(tableId: any) {
@@ -267,7 +268,7 @@ export const exportRows = async (
     async () => {
       const { fileName, content } = await sdk.rows.exportRows({
         tableId,
-        format,
+        format: format as Format,
         rowIds: rows,
         columns,
         query,

packages/server/src/api/controllers/row/internal.ts

@@ -1,6 +1,7 @@
 import * as linkRows from "../../../db/linkedRows"
 import {
   generateRowID,
+  getMultiIDParams,
   getTableIDFromRowID,
   InternalTables,
 } from "../../../db/utils"
@@ -24,6 +25,8 @@ import {
   UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
+import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils"
+import { flatten } from "lodash"
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const tableId = utils.getTableId(ctx)
@@ -154,7 +157,7 @@ export async function destroy(ctx: UserCtx) {
   if (row.tableId !== tableId) {
     throw "Supplied tableId doesn't match the row's tableId"
   }
-  const table = await sdk.tables.getTable(row.tableId)
+  const table = await sdk.tables.getTable(tableId)
   // update the row to include full relationships before deleting them
   row = await outputProcessing(table, row, {
     squash: false,
@@ -164,7 +167,7 @@ export async function destroy(ctx: UserCtx) {
   await linkRows.updateLinks({
     eventType: linkRows.EventType.ROW_DELETE,
     row,
-    tableId: row.tableId,
+    tableId,
   })
   // remove any attachments that were on the row from object storage
   await cleanupAttachments(table, { row })
@@ -225,60 +228,52 @@ export async function bulkDestroy(ctx: UserCtx) {
 }
 
 export async function fetchEnrichedRow(ctx: UserCtx) {
+  const fieldName = ctx.request.query.field as string | undefined
   const db = context.getAppDB()
   const tableId = utils.getTableId(ctx)
-  const rowId = ctx.params.rowId
+  const rowId = ctx.params.rowId as string
-  // need table to work out where links go in row
+  // need table to work out where links go in row, as well as the link docs
-  let [table, row] = await Promise.all([
+  let response = await Promise.all([
     sdk.tables.getTable(tableId),
     utils.findRow(ctx, tableId, rowId),
+    linkRows.getLinkDocuments({ tableId, rowId, fieldName }),
   ])
-  // get the link docs
+  const table = response[0] as Table
-  const linkVals = (await linkRows.getLinkDocuments({
+  const row = response[1] as Row
-    tableId,
+  const linkVals = response[2] as LinkDocumentValue[]
-    rowId,
-  })) as LinkDocumentValue[]
   // look up the actual rows based on the ids
-  let response = (
+  const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
-    await db.allDocs({
+  let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc)
-      include_docs: true,
+
-      keys: linkVals.map(linkVal => linkVal.id),
+  // get the linked tables
-    })
+  const linkTableIds = getLinkedTableIDs(table as Table)
-  ).rows.map(row => row.doc)
+  const linkTables = await sdk.tables.getTables(linkTableIds)
-  // group responses by table
+
-  let groups: any = {},
+  // perform output processing
-    tables: Record<string, Table> = {}
+  let final: Promise<Row[]>[] = []
-  for (let row of response) {
+  for (let linkTable of linkTables) {
-    if (!row.tableId) {
+    const relatedRows = linkedRows.filter(row => row.tableId === linkTable._id)
-      row.tableId = getTableIDFromRowID(row._id)
+    // include the row being enriched for performance reasons, don't need to fetch it to include
-    }
+    final = final.concat(
-    const linkedTableId = row.tableId
+      outputProcessing(linkTable, relatedRows, {
-    if (groups[linkedTableId] == null) {
+        // have to clone to avoid JSON cycle
-      groups[linkedTableId] = [row]
+        fromRow: cloneDeep(row),
-      tables[linkedTableId] = await db.get(linkedTableId)
+        squash: true,
-    } else {
+      })
-      groups[linkedTableId].push(row)
-    }
-  }
-  let linkedRows: Row[] = []
-  for (let [tableId, rows] of Object.entries(groups)) {
-    // need to include the IDs in these rows for any links they may have
-    linkedRows = linkedRows.concat(
-      await outputProcessing(tables[tableId], rows as Row[])
     )
   }
+  // finalise the promises
+  linkedRows = flatten(await Promise.all(final))
 
   // insert the link rows in the correct place throughout the main row
   for (let fieldName of Object.keys(table.schema)) {
     let field = table.schema[fieldName]
     if (field.type === FieldTypes.LINK) {
-      // find the links that pertain to this field, get their indexes
+      // find the links that pertain to this field
-      const linkIndexes = linkVals
+      const links = linkVals.filter(link => link.fieldName === fieldName)
-        .filter(link => link.fieldName === fieldName)
-        .map(link => linkVals.indexOf(link))
       // find the rows that the links state are linked to this field
-      row[fieldName] = linkedRows.filter((linkRow, index) =>
+      row[fieldName] = linkedRows.filter(linkRow =>
-        linkIndexes.includes(index)
+        links.find(link => link.id === linkRow._id)
       )
     }
   }

packages/server/src/api/controllers/row/staticFormula.ts

@@ -149,7 +149,7 @@ export async function finaliseRow(
       await db.put(table)
     } catch (err: any) {
       if (err.status === 409) {
-        const updatedTable = await sdk.tables.getTable(table._id)
+        const updatedTable = await sdk.tables.getTable(table._id!)
         let response = processAutoColumn(null, updatedTable, row, {
           reprocessing: true,
         })

packages/server/src/api/controllers/row/utils.ts

@@ -17,20 +17,6 @@ import sdk from "../../../sdk"
 import validateJs from "validate.js"
 import { cloneDeep } from "lodash/fp"
 
-function isForeignKey(key: string, table: Table) {
-  const relationships = Object.values(table.schema).filter(
-    column => column.type === FieldType.LINK
-  )
-  return relationships.some(
-    relationship =>
-      (
-        relationship as
-          | OneToManyRelationshipFieldMetadata
-          | ManyToOneRelationshipFieldMetadata
-      ).foreignKey === key
-  )
-}
-
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
     return new Date(value).getTime()
@@ -60,7 +46,7 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   return row
 }
 
-export function getTableId(ctx: Ctx) {
+export function getTableId(ctx: Ctx): string {
   // top priority, use the URL first
   if (ctx.params?.sourceId) {
     return ctx.params.sourceId
@@ -77,112 +63,7 @@ export function getTableId(ctx: Ctx) {
   if (ctx.params?.viewName) {
     return ctx.params.viewName
   }
-}
+  throw new Error("Unable to find table ID in request")
-
-export async function validate({
-  tableId,
-  row,
-  table,
-}: {
-  tableId?: string
-  row: Row
-  table?: Table
-}) {
-  let fetchedTable: Table
-  if (!table) {
-    fetchedTable = await sdk.tables.getTable(tableId)
-  } else {
-    fetchedTable = table
-  }
-  const errors: any = {}
-  for (let fieldName of Object.keys(fetchedTable.schema)) {
-    const column = fetchedTable.schema[fieldName]
-    const constraints = cloneDeep(column.constraints)
-    const type = column.type
-    // foreign keys are likely to be enriched
-    if (isForeignKey(fieldName, fetchedTable)) {
-      continue
-    }
-    // formulas shouldn't validated, data will be deleted anyway
-    if (type === FieldTypes.FORMULA || column.autocolumn) {
-      continue
-    }
-    // special case for options, need to always allow unselected (empty)
-    if (type === FieldTypes.OPTIONS && constraints?.inclusion) {
-      constraints.inclusion.push(null as any, "")
-    }
-    let res
-
-    // Validate.js doesn't seem to handle array
-    if (type === FieldTypes.ARRAY && row[fieldName]) {
-      if (row[fieldName].length) {
-        if (!Array.isArray(row[fieldName])) {
-          row[fieldName] = row[fieldName].split(",")
-        }
-        row[fieldName].map((val: any) => {
-          if (
-            !constraints?.inclusion?.includes(val) &&
-            constraints?.inclusion?.length !== 0
-          ) {
-            errors[fieldName] = "Field not in list"
-          }
-        })
-      } else if (constraints?.presence && row[fieldName].length === 0) {
-        // non required MultiSelect creates an empty array, which should not throw errors
-        errors[fieldName] = [`${fieldName} is required`]
-      }
-    } else if (
-      (type === FieldTypes.ATTACHMENT || type === FieldTypes.JSON) &&
-      typeof row[fieldName] === "string"
-    ) {
-      // this should only happen if there is an error
-      try {
-        const json = JSON.parse(row[fieldName])
-        if (type === FieldTypes.ATTACHMENT) {
-          if (Array.isArray(json)) {
-            row[fieldName] = json
-          } else {
-            errors[fieldName] = [`Must be an array`]
-          }
-        }
-      } catch (err) {
-        errors[fieldName] = [`Contains invalid JSON`]
-      }
-    } else {
-      res = validateJs.single(row[fieldName], constraints)
-    }
-    if (res) errors[fieldName] = res
-  }
-  return { valid: Object.keys(errors).length === 0, errors }
-}
-
-// don't do a pure falsy check, as 0 is included
-// https://github.com/Budibase/budibase/issues/10118
-export function removeEmptyFilters(filters: SearchFilters) {
-  for (let filterField of NoEmptyFilterStrings) {
-    if (!filters[filterField]) {
-      continue
-    }
-
-    for (let filterType of Object.keys(filters)) {
-      if (filterType !== filterField) {
-        continue
-      }
-      // don't know which one we're checking, type could be anything
-      const value = filters[filterType] as unknown
-      if (typeof value === "object") {
-        for (let [key, value] of Object.entries(
-          filters[filterType] as object
-        )) {
-          if (value == null || value === "") {
-            // @ts-ignore
-            delete filters[filterField][key]
-          }
-        }
-      }
-    }
-  }
-  return filters
 }
 
 export function isUserMetadataTable(tableId: string) {

packages/server/src/api/controllers/table/ExternalRequest.ts

@@ -0,0 +1,36 @@
+import {
+  Datasource,
+  Operation,
+  QueryJson,
+  RenameColumn,
+  Table,
+} from "@budibase/types"
+import { makeExternalQuery } from "../../../integrations/base/query"
+
+export async function makeTableRequest(
+  datasource: Datasource,
+  operation: Operation,
+  table: Table,
+  tables: Record<string, Table>,
+  oldTable?: Table,
+  renamed?: RenameColumn
+) {
+  const json: QueryJson = {
+    endpoint: {
+      datasourceId: datasource._id!,
+      entityId: table._id!,
+      operation,
+    },
+    meta: {
+      tables,
+    },
+    table,
+  }
+  if (oldTable) {
+    json.meta!.table = oldTable
+  }
+  if (renamed) {
+    json.meta!.renamed = renamed
+  }
+  return makeExternalQuery(datasource, json)
+}

packages/server/src/api/controllers/table/external.ts

@@ -1,108 +1,20 @@
-import {
+import { breakExternalTableId } from "../../../integrations/utils"
-  breakExternalTableId,
-  buildExternalTableId,
-} from "../../../integrations/utils"
-import {
-  foreignKeyStructure,
-  generateForeignKey,
-  generateJunctionTableName,
-  hasTypeChanged,
-  setStaticSchemas,
-} from "./utils"
-import { FieldTypes } from "../../../constants"
-import { makeExternalQuery } from "../../../integrations/base/query"
 import { handleRequest } from "../row/external"
-import { context, events } from "@budibase/backend-core"
+import { events } from "@budibase/backend-core"
 import { isRows, isSchema, parse } from "../../../utilities/schema"
 import {
   BulkImportRequest,
   BulkImportResponse,
-  Datasource,
-  FieldSchema,
-  ManyToManyRelationshipFieldMetadata,
-  ManyToOneRelationshipFieldMetadata,
-  OneToManyRelationshipFieldMetadata,
   Operation,
-  QueryJson,
-  RelationshipFieldMetadata,
-  RelationshipType,
-  RenameColumn,
   SaveTableRequest,
   SaveTableResponse,
   Table,
   TableRequest,
   UserCtx,
-  ViewV2,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { builderSocket } from "../../../websockets"
 
-const { cloneDeep } = require("lodash/fp")
-
-async function makeTableRequest(
-  datasource: Datasource,
-  operation: Operation,
-  table: Table,
-  tables: Record<string, Table>,
-  oldTable?: Table,
-  renamed?: RenameColumn
-) {
-  const json: QueryJson = {
-    endpoint: {
-      datasourceId: datasource._id!,
-      entityId: table._id!,
-      operation,
-    },
-    meta: {
-      tables,
-    },
-    table,
-  }
-  if (oldTable) {
-    json.meta!.table = oldTable
-  }
-  if (renamed) {
-    json.meta!.renamed = renamed
-  }
-  return makeExternalQuery(datasource, json)
-}
-
-function cleanupRelationships(
-  table: Table,
-  tables: Record<string, Table>,
-  oldTable?: Table
-) {
-  const tableToIterate = oldTable ? oldTable : table
-  // clean up relationships in couch table schemas
-  for (let [key, schema] of Object.entries(tableToIterate.schema)) {
-    if (
-      schema.type === FieldTypes.LINK &&
-      (!oldTable || table.schema[key] == null)
-    ) {
-      const schemaTableId = schema.tableId
-      const relatedTable = Object.values(tables).find(
-        table => table._id === schemaTableId
-      )
-      const foreignKey =
-        schema.relationshipType !== RelationshipType.MANY_TO_MANY &&
-        schema.foreignKey
-      if (!relatedTable || !foreignKey) {
-        continue
-      }
-      for (let [relatedKey, relatedSchema] of Object.entries(
-        relatedTable.schema
-      )) {
-        if (
-          relatedSchema.type === FieldTypes.LINK &&
-          relatedSchema.fieldName === foreignKey
-        ) {
-          delete relatedTable.schema[relatedKey]
-        }
-      }
-    }
-  }
-}
-
 function getDatasourceId(table: Table) {
   if (!table) {
     throw "No table supplied"
@@ -113,247 +25,32 @@ function getDatasourceId(table: Table) {
   return breakExternalTableId(table._id).datasourceId
 }
 
-function otherRelationshipType(type?: string) {
-  if (type === RelationshipType.MANY_TO_MANY) {
-    return RelationshipType.MANY_TO_MANY
-  }
-  return type === RelationshipType.ONE_TO_MANY
-    ? RelationshipType.MANY_TO_ONE
-    : RelationshipType.ONE_TO_MANY
-}
-
-function generateManyLinkSchema(
-  datasource: Datasource,
-  column: ManyToManyRelationshipFieldMetadata,
-  table: Table,
-  relatedTable: Table
-): Table {
-  if (!table.primary || !relatedTable.primary) {
-    throw new Error("Unable to generate many link schema, no primary keys")
-  }
-  const primary = table.name + table.primary[0]
-  const relatedPrimary = relatedTable.name + relatedTable.primary[0]
-  const jcTblName = generateJunctionTableName(column, table, relatedTable)
-  // first create the new table
-  const junctionTable = {
-    _id: buildExternalTableId(datasource._id!, jcTblName),
-    name: jcTblName,
-    primary: [primary, relatedPrimary],
-    constrained: [primary, relatedPrimary],
-    schema: {
-      [primary]: foreignKeyStructure(primary, {
-        toTable: table.name,
-        toKey: table.primary[0],
-      }),
-      [relatedPrimary]: foreignKeyStructure(relatedPrimary, {
-        toTable: relatedTable.name,
-        toKey: relatedTable.primary[0],
-      }),
-    },
-  }
-  column.through = junctionTable._id
-  column.throughFrom = relatedPrimary
-  column.throughTo = primary
-  column.fieldName = relatedPrimary
-  return junctionTable
-}
-
-function generateLinkSchema(
-  column:
-    | OneToManyRelationshipFieldMetadata
-    | ManyToOneRelationshipFieldMetadata,
-  table: Table,
-  relatedTable: Table,
-  type: RelationshipType.ONE_TO_MANY | RelationshipType.MANY_TO_ONE
-) {
-  if (!table.primary || !relatedTable.primary) {
-    throw new Error("Unable to generate link schema, no primary keys")
-  }
-  const isOneSide = type === RelationshipType.ONE_TO_MANY
-  const primary = isOneSide ? relatedTable.primary[0] : table.primary[0]
-  // generate a foreign key
-  const foreignKey = generateForeignKey(column, relatedTable)
-  column.relationshipType = type
-  column.foreignKey = isOneSide ? foreignKey : primary
-  column.fieldName = isOneSide ? primary : foreignKey
-  return foreignKey
-}
-
-function generateRelatedSchema(
-  linkColumn: RelationshipFieldMetadata,
-  table: Table,
-  relatedTable: Table,
-  columnName: string
-) {
-  // generate column for other table
-  const relatedSchema = cloneDeep(linkColumn)
-  const isMany2Many =
-    linkColumn.relationshipType === RelationshipType.MANY_TO_MANY
-  // swap them from the main link
-  if (!isMany2Many && linkColumn.foreignKey) {
-    relatedSchema.fieldName = linkColumn.foreignKey
-    relatedSchema.foreignKey = linkColumn.fieldName
-  }
-  // is many to many
-  else if (isMany2Many) {
-    // don't need to copy through, already got it
-    relatedSchema.fieldName = linkColumn.throughTo
-    relatedSchema.throughTo = linkColumn.throughFrom
-    relatedSchema.throughFrom = linkColumn.throughTo
-  }
-  relatedSchema.relationshipType = otherRelationshipType(
-    linkColumn.relationshipType
-  )
-  relatedSchema.tableId = relatedTable._id
-  relatedSchema.name = columnName
-  table.schema[columnName] = relatedSchema
-}
-
-function isRelationshipSetup(column: RelationshipFieldMetadata) {
-  return (column as any).foreignKey || (column as any).through
-}
-
 export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   const inputs = ctx.request.body
-  const renamed = inputs?._rename
+  const renaming = inputs?._rename
   // can't do this right now
   delete inputs.rows
-  const datasourceId = getDatasourceId(ctx.request.body)!
+  const tableId = ctx.request.body._id
+  const datasourceId = getDatasourceId(ctx.request.body)
   // table doesn't exist already, note that it is created
   if (!inputs._id) {
     inputs.created = true
   }
-  let tableToSave: TableRequest = {
+  try {
-    type: "table",
+    const { datasource, table } = await sdk.tables.external.save(
-    _id: buildExternalTableId(datasourceId, inputs.name),
+      datasourceId!,
-    sourceId: datasourceId,
+      inputs,
-    ...inputs,
+      { tableId, renaming }
-  }
-
-  let oldTable: Table | undefined
-  if (ctx.request.body && ctx.request.body._id) {
-    oldTable = await sdk.tables.getTable(ctx.request.body._id)
-  }
-
-  if (hasTypeChanged(tableToSave, oldTable)) {
-    ctx.throw(400, "A column type has changed.")
-  }
-
-  for (let view in tableToSave.views) {
-    const tableView = tableToSave.views[view]
-    if (!tableView || !sdk.views.isV2(tableView)) continue
-
-    tableToSave.views[view] = sdk.views.syncSchema(
-      oldTable!.views![view] as ViewV2,
-      tableToSave.schema,
-      renamed
     )
-  }
+    builderSocket?.emitDatasourceUpdate(ctx, datasource)
-
+    return table
-  const db = context.getAppDB()
+  } catch (err: any) {
-  const datasource = await sdk.datasources.get(datasourceId)
+    if (err instanceof Error) {
-  if (!datasource.entities) {
+      ctx.throw(400, err.message)
-    datasource.entities = {}
-  }
-
-  // GSheets is a specific case - only ever has a static primary key
-  tableToSave = setStaticSchemas(datasource, tableToSave)
-
-  const oldTables = cloneDeep(datasource.entities)
-  const tables: Record<string, Table> = datasource.entities
-
-  const extraTablesToUpdate = []
-
-  // check if relations need setup
-  for (let schema of Object.values(tableToSave.schema)) {
-    if (schema.type !== FieldTypes.LINK || isRelationshipSetup(schema)) {
-      continue
-    }
-    const schemaTableId = schema.tableId
-    const relatedTable = Object.values(tables).find(
-      table => table._id === schemaTableId
-    )
-    if (!relatedTable) {
-      continue
-    }
-    const relatedColumnName = schema.fieldName!
-    const relationType = schema.relationshipType
-    if (relationType === RelationshipType.MANY_TO_MANY) {
-      const junctionTable = generateManyLinkSchema(
-        datasource,
-        schema,
-        tableToSave,
-        relatedTable
-      )
-      if (tables[junctionTable.name]) {
-        throw "Junction table already exists, cannot create another relationship."
-      }
-      tables[junctionTable.name] = junctionTable
-      extraTablesToUpdate.push(junctionTable)
     } else {
-      const fkTable =
+      ctx.throw(err.status || 500, err?.message || err)
-        relationType === RelationshipType.ONE_TO_MANY
-          ? tableToSave
-          : relatedTable
-      const foreignKey = generateLinkSchema(
-        schema,
-        tableToSave,
-        relatedTable,
-        relationType
-      )
-      fkTable.schema[foreignKey] = foreignKeyStructure(foreignKey)
-      if (fkTable.constrained == null) {
-        fkTable.constrained = []
-      }
-      if (fkTable.constrained.indexOf(foreignKey) === -1) {
-        fkTable.constrained.push(foreignKey)
-      }
-      // foreign key is in other table, need to save it to external
-      if (fkTable._id !== tableToSave._id) {
-        extraTablesToUpdate.push(fkTable)
-      }
     }
-    generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName)
-    schema.main = true
   }
-
-  cleanupRelationships(tableToSave, tables, oldTable)
-
-  const operation = oldTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE
-  await makeTableRequest(
-    datasource,
-    operation,
-    tableToSave,
-    tables,
-    oldTable,
-    renamed
-  )
-  // update any extra tables (like foreign keys in other tables)
-  for (let extraTable of extraTablesToUpdate) {
-    const oldExtraTable = oldTables[extraTable.name]
-    let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE
-    await makeTableRequest(datasource, op, extraTable, tables, oldExtraTable)
-  }
-
-  // make sure the constrained list, all still exist
-  if (Array.isArray(tableToSave.constrained)) {
-    tableToSave.constrained = tableToSave.constrained.filter(constraint =>
-      Object.keys(tableToSave.schema).includes(constraint)
-    )
-  }
-
-  // remove the rename prop
-  delete tableToSave._rename
-  // store it into couch now for budibase reference
-  datasource.entities[tableToSave.name] = tableToSave
-  await db.put(sdk.tables.populateExternalTableSchemas(datasource))
-
-  // Since tables are stored inside datasources, we need to notify clients
-  // that the datasource definition changed
-  const updatedDatasource = await sdk.datasources.get(datasource._id!)
-  builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource)
-
-  return tableToSave
 }
 
 export async function destroy(ctx: UserCtx) {
@@ -364,27 +61,20 @@ export async function destroy(ctx: UserCtx) {
     ctx.throw(400, "Cannot delete tables which weren't created in Budibase.")
   }
   const datasourceId = getDatasourceId(tableToDelete)
-
+  try {
-  const db = context.getAppDB()
+    const { datasource, table } = await sdk.tables.external.destroy(
-  const datasource = await sdk.datasources.get(datasourceId!)
+      datasourceId!,
-  const tables = datasource.entities
+      tableToDelete
-
+    )
-  const operation = Operation.DELETE_TABLE
+    builderSocket?.emitDatasourceUpdate(ctx, datasource)
-  if (tables) {
+    return table
-    await makeTableRequest(datasource, operation, tableToDelete, tables)
+  } catch (err: any) {
-    cleanupRelationships(tableToDelete, tables)
+    if (err instanceof Error) {
-    delete tables[tableToDelete.name]
+      ctx.throw(400, err.message)
-    datasource.entities = tables
+    } else {
+      ctx.throw(err.status || 500, err.message || err)
+    }
   }
-
-  await db.put(sdk.tables.populateExternalTableSchemas(datasource))
-
-  // Since tables are stored inside datasources, we need to notify clients
-  // that the datasource definition changed
-  const updatedDatasource = await sdk.datasources.get(datasource._id!)
-  builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource)
-
-  return tableToDelete
 }
 
 export async function bulkImport(

packages/server/src/api/controllers/table/index.ts

@@ -16,6 +16,7 @@ import {
   Table,
   TableResponse,
   UserCtx,
+  Row,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
@@ -139,8 +140,7 @@ export async function validateNewTableImport(ctx: UserCtx) {
 }
 
 export async function validateExistingTableImport(ctx: UserCtx) {
-  const { rows, tableId }: { rows: unknown; tableId: unknown } =
+  const { rows, tableId }: { rows: Row[]; tableId?: string } = ctx.request.body
-    ctx.request.body
 
   let schema = null
   if (tableId) {

packages/server/src/api/controllers/table/internal.ts

@@ -1,14 +1,5 @@
-import { updateLinks, EventType } from "../../../db/linkedRows"
+import { generateTableID } from "../../../db/utils"
-import { getRowParams, generateTableID } from "../../../db/utils"
+import { handleDataImport } from "./utils"
-import { FieldTypes } from "../../../constants"
-import { TableSaveFunctions, hasTypeChanged, handleDataImport } from "./utils"
-import { context } from "@budibase/backend-core"
-import env from "../../../environment"
-import {
-  cleanupAttachments,
-  fixAutoColumnSubType,
-} from "../../../utilities/rowProcessor"
-import { runStaticFormulaChecks } from "./bulkFormula"
 import {
   BulkImportRequest,
   BulkImportResponse,
@@ -17,195 +8,52 @@ import {
   SaveTableResponse,
   Table,
   UserCtx,
-  ViewStatisticsSchema,
-  ViewV2,
 } from "@budibase/types"
-import { quotas } from "@budibase/pro"
-import isEqual from "lodash/isEqual"
-import { cloneDeep } from "lodash/fp"
 import sdk from "../../../sdk"
 
-function checkAutoColumns(table: Table, oldTable?: Table) {
-  if (!table.schema) {
-    return table
-  }
-  for (let [key, schema] of Object.entries(table.schema)) {
-    if (!schema.autocolumn || schema.subtype) {
-      continue
-    }
-    const oldSchema = oldTable && oldTable.schema[key]
-    if (oldSchema && oldSchema.subtype) {
-      table.schema[key].subtype = oldSchema.subtype
-    } else {
-      table.schema[key] = fixAutoColumnSubType(schema)
-    }
-  }
-  return table
-}
-
 export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
-  const db = context.getAppDB()
   const { rows, ...rest } = ctx.request.body
   let tableToSave: Table & {
-    _rename?: { old: string; updated: string } | undefined
+    _rename?: RenameColumn
   } = {
     type: "table",
     _id: generateTableID(),
     views: {},
     ...rest,
   }
+  const renaming = tableToSave._rename
+  delete tableToSave._rename
 
-  // if the table obj had an _id then it will have been retrieved
-  let oldTable: Table | undefined
-  if (ctx.request.body && ctx.request.body._id) {
-    oldTable = await sdk.tables.getTable(ctx.request.body._id)
-  }
-
-  // check all types are correct
-  if (hasTypeChanged(tableToSave, oldTable)) {
-    ctx.throw(400, "A column type has changed.")
-  }
-  // check that subtypes have been maintained
-  tableToSave = checkAutoColumns(tableToSave, oldTable)
-
-  // saving a table is a complex operation, involving many different steps, this
-  // has been broken out into a utility to make it more obvious/easier to manipulate
-  const tableSaveFunctions = new TableSaveFunctions({
-    user: ctx.user,
-    oldTable,
-    importRows: rows,
-  })
-  tableToSave = await tableSaveFunctions.before(tableToSave)
-
-  // make sure that types don't change of a column, have to remove
-  // the column if you want to change the type
-  if (oldTable && oldTable.schema) {
-    for (const propKey of Object.keys(tableToSave.schema)) {
-      let oldColumn = oldTable.schema[propKey]
-      if (oldColumn && oldColumn.type === FieldTypes.INTERNAL) {
-        oldTable.schema[propKey].type = FieldTypes.AUTO
-      }
-    }
-  }
-
-  // Don't rename if the name is the same
-  let _rename: RenameColumn | undefined = tableToSave._rename
-  /* istanbul ignore next */
-  if (_rename && _rename.old === _rename.updated) {
-    _rename = undefined
-    delete tableToSave._rename
-  }
-
-  // rename row fields when table column is renamed
-  /* istanbul ignore next */
-  if (_rename && tableToSave.schema[_rename.updated].type === FieldTypes.LINK) {
-    ctx.throw(400, "Cannot rename a linked column.")
-  }
-
-  tableToSave = await tableSaveFunctions.mid(tableToSave)
-
-  // update schema of non-statistics views when new columns are added
-  for (let view in tableToSave.views) {
-    const tableView = tableToSave.views[view]
-    if (!tableView) continue
-
-    if (sdk.views.isV2(tableView)) {
-      tableToSave.views[view] = sdk.views.syncSchema(
-        oldTable!.views![view] as ViewV2,
-        tableToSave.schema,
-        _rename
-      )
-      continue
-    }
-
-    if (
-      (tableView.schema as ViewStatisticsSchema).group ||
-      tableView.schema.field
-    )
-      continue
-    tableView.schema = tableToSave.schema
-  }
-
-  // update linked rows
   try {
-    const linkResp: any = await updateLinks({
+    const { table } = await sdk.tables.internal.save(tableToSave, {
-      eventType: oldTable ? EventType.TABLE_UPDATED : EventType.TABLE_SAVE,
+      user: ctx.user,
-      table: tableToSave,
+      rowsToImport: rows,
-      oldTable: oldTable,
+      tableId: ctx.request.body._id,
+      renaming: renaming,
     })
-    if (linkResp != null && linkResp._rev) {
+
-      tableToSave._rev = linkResp._rev
+    return table
+  } catch (err: any) {
+    if (err instanceof Error) {
+      ctx.throw(400, err.message)
+    } else {
+      ctx.throw(err.status || 500, err.message || err)
     }
-  } catch (err) {
-    ctx.throw(400, err as string)
   }
-
-  // don't perform any updates until relationships have been
-  // checked by the updateLinks function
-  const updatedRows = tableSaveFunctions.getUpdatedRows()
-  if (updatedRows && updatedRows.length !== 0) {
-    await db.bulkDocs(updatedRows)
-  }
-  let result = await db.put(tableToSave)
-  tableToSave._rev = result.rev
-  const savedTable = cloneDeep(tableToSave)
-
-  tableToSave = await tableSaveFunctions.after(tableToSave)
-  // the table may be updated as part of the table save after functionality - need to write it
-  if (!isEqual(savedTable, tableToSave)) {
-    result = await db.put(tableToSave)
-    tableToSave._rev = result.rev
-  }
-  // has to run after, make sure it has _id
-  await runStaticFormulaChecks(tableToSave, { oldTable, deletion: false })
-  return tableToSave
 }
 
-export async function destroy(ctx: any) {
+export async function destroy(ctx: UserCtx) {
-  const db = context.getAppDB()
   const tableToDelete = await sdk.tables.getTable(ctx.params.tableId)
-
+  try {
-  // Delete all rows for that table
+    const { table } = await sdk.tables.internal.destroy(tableToDelete)
-  const rowsData = await db.allDocs(
+    return table
-    getRowParams(ctx.params.tableId, null, {
+  } catch (err: any) {
-      include_docs: true,
+    if (err instanceof Error) {
-    })
+      ctx.throw(400, err.message)
-  )
+    } else {
-  await db.bulkDocs(
+      ctx.throw(err.status || 500, err.message || err)
-    rowsData.rows.map((row: any) => ({ ...row.doc, _deleted: true }))
-  )
-  await quotas.removeRows(rowsData.rows.length, {
-    tableId: ctx.params.tableId,
-  })
-
-  // update linked rows
-  await updateLinks({
-    eventType: EventType.TABLE_DELETE,
-    table: tableToDelete,
-  })
-
-  // don't remove the table itself until very end
-  await db.remove(tableToDelete._id!, tableToDelete._rev)
-
-  // remove table search index
-  if (!env.isTest() || env.COUCH_DB_URL) {
-    const currentIndexes = await db.getIndexes()
-    const existingIndex = currentIndexes.indexes.find(
-      (existing: any) => existing.name === `search:${ctx.params.tableId}`
-    )
-    if (existingIndex) {
-      await db.deleteIndex(existingIndex)
     }
   }
-
-  // has to run after, make sure it has _id
-  await runStaticFormulaChecks(tableToDelete, {
-    deletion: true,
-  })
-  await cleanupAttachments(tableToDelete, {
-    rows: rowsData.rows.map((row: any) => row.doc),
-  })
-  return tableToDelete
 }
 
 export async function bulkImport(
@@ -213,6 +61,10 @@ export async function bulkImport(
 ) {
   const table = await sdk.tables.getTable(ctx.params.tableId)
   const { rows, identifierFields } = ctx.request.body
-  await handleDataImport(ctx.user, table, rows, identifierFields)
+  await handleDataImport(table, {
+    importRows: rows,
+    identifierFields,
+    user: ctx.user,
+  })
   return table
 }

packages/server/src/api/controllers/table/utils.ts

@@ -26,9 +26,16 @@ import {
   Row,
   SourceName,
   Table,
+  Database,
+  RenameColumn,
+  NumberFieldMetadata,
+  FieldSchema,
+  View,
+  RelationshipFieldMetadata,
+  FieldType,
 } from "@budibase/types"
 
-export async function clearColumns(table: any, columnNames: any) {
+export async function clearColumns(table: Table, columnNames: string[]) {
   const db = context.getAppDB()
   const rows = await db.allDocs(
     getRowParams(table._id, null, {
@@ -43,10 +50,13 @@ export async function clearColumns(table: any, columnNames: any) {
   )) as { id: string; _rev?: string }[]
 }
 
-export async function checkForColumnUpdates(oldTable: any, updatedTable: any) {
+export async function checkForColumnUpdates(
+  updatedTable: Table,
+  oldTable?: Table,
+  columnRename?: RenameColumn
+) {
   const db = context.getAppDB()
   let updatedRows = []
-  const rename = updatedTable._rename
   let deletedColumns: any = []
   if (oldTable && oldTable.schema && updatedTable.schema) {
     deletedColumns = Object.keys(oldTable.schema).filter(
@@ -54,7 +64,7 @@ export async function checkForColumnUpdates(oldTable: any, updatedTable: any) {
     )
   }
   // check for renaming of columns or deleted columns
-  if (rename || deletedColumns.length !== 0) {
+  if (columnRename || deletedColumns.length !== 0) {
     // Update all rows
     const rows = await db.allDocs(
       getRowParams(updatedTable._id, null, {
@@ -64,9 +74,9 @@ export async function checkForColumnUpdates(oldTable: any, updatedTable: any) {
     const rawRows = rows.rows.map(({ doc }: any) => doc)
     updatedRows = rawRows.map((row: any) => {
       row = cloneDeep(row)
-      if (rename) {
+      if (columnRename) {
-        row[rename.updated] = row[rename.old]
+        row[columnRename.updated] = row[columnRename.old]
-        delete row[rename.old]
+        delete row[columnRename.old]
       } else if (deletedColumns.length !== 0) {
         deletedColumns.forEach((colName: any) => delete row[colName])
       }
@@ -76,14 +86,13 @@ export async function checkForColumnUpdates(oldTable: any, updatedTable: any) {
     // cleanup any attachments from object storage for deleted attachment columns
     await cleanupAttachments(updatedTable, { oldTable, rows: rawRows })
     // Update views
-    await checkForViewUpdates(updatedTable, rename, deletedColumns)
+    await checkForViewUpdates(updatedTable, deletedColumns, columnRename)
-    delete updatedTable._rename
   }
   return { rows: updatedRows, table: updatedTable }
 }
 
 // makes sure the passed in table isn't going to reset the auto ID
-export function makeSureTableUpToDate(table: any, tableToSave: any) {
+export function makeSureTableUpToDate(table: Table, tableToSave: Table) {
   if (!table) {
     return tableToSave
   }
@@ -99,16 +108,17 @@ export function makeSureTableUpToDate(table: any, tableToSave: any) {
       column.subtype === AutoFieldSubTypes.AUTO_ID &&
       tableToSave.schema[field]
     ) {
-      tableToSave.schema[field].lastID = column.lastID
+      const tableCol = tableToSave.schema[field] as NumberFieldMetadata
+      tableCol.lastID = column.lastID
     }
   }
   return tableToSave
 }
 
 export async function importToRows(
-  data: any[],
+  data: Row[],
   table: Table,
-  user: ContextUser | null = null
+  user?: ContextUser
 ) {
   let originalTable = table
   let finalData: any = []
@@ -150,19 +160,20 @@ export async function importToRows(
 }
 
 export async function handleDataImport(
-  user: ContextUser,
   table: Table,
-  rows: Row[],
+  opts?: { identifierFields?: string[]; user?: ContextUser; importRows?: Row[] }
-  identifierFields: Array<string> = []
 ) {
   const schema = table.schema
+  const identifierFields = opts?.identifierFields || []
+  const user = opts?.user
+  const importRows = opts?.importRows
 
-  if (!rows || !isRows(rows) || !isSchema(schema)) {
+  if (!importRows || !isRows(importRows) || !isSchema(schema)) {
     return table
   }
 
   const db = context.getAppDB()
-  const data = parse(rows, schema)
+  const data = parse(importRows, schema)
 
   let finalData: any = await importToRows(data, table, user)
 
@@ -200,7 +211,7 @@ export async function handleDataImport(
   return table
 }
 
-export async function handleSearchIndexes(table: any) {
+export async function handleSearchIndexes(table: Table) {
   const db = context.getAppDB()
   // create relevant search indexes
   if (table.indexes && table.indexes.length > 0) {
@@ -244,13 +255,13 @@ export async function handleSearchIndexes(table: any) {
   return table
 }
 
-export function checkStaticTables(table: any) {
+export function checkStaticTables(table: Table) {
   // check user schema has all required elements
   if (table._id === InternalTables.USER_METADATA) {
     for (let [key, schema] of Object.entries(USERS_TABLE_SCHEMA.schema)) {
       // check if the schema exists on the table to be created/updated
       if (table.schema[key] == null) {
-        table.schema[key] = schema
+        table.schema[key] = schema as FieldSchema
       }
     }
   }
@@ -258,13 +269,21 @@ export function checkStaticTables(table: any) {
 }
 
 class TableSaveFunctions {
-  db: any
+  db: Database
-  user: any
+  user?: ContextUser
-  oldTable: any
+  oldTable?: Table
-  importRows: any
+  importRows?: Row[]
-  rows: any
+  rows: Row[]
 
-  constructor({ user, oldTable, importRows }: any) {
+  constructor({
+    user,
+    oldTable,
+    importRows,
+  }: {
+    user?: ContextUser
+    oldTable?: Table
+    importRows?: Row[]
+  }) {
     this.db = context.getAppDB()
     this.user = user
     this.oldTable = oldTable
@@ -274,7 +293,7 @@ class TableSaveFunctions {
   }
 
   // before anything is done
-  async before(table: any) {
+  async before(table: Table) {
     if (this.oldTable) {
       table = makeSureTableUpToDate(this.oldTable, table)
     }
@@ -283,16 +302,23 @@ class TableSaveFunctions {
   }
 
   // when confirmed valid
-  async mid(table: any) {
+  async mid(table: Table, columnRename?: RenameColumn) {
-    let response = await checkForColumnUpdates(this.oldTable, table)
+    let response = await checkForColumnUpdates(
+      table,
+      this.oldTable,
+      columnRename
+    )
     this.rows = this.rows.concat(response.rows)
     return table
   }
 
   // after saving
-  async after(table: any) {
+  async after(table: Table) {
     table = await handleSearchIndexes(table)
-    table = await handleDataImport(this.user, table, this.importRows)
+    table = await handleDataImport(table, {
+      importRows: this.importRows,
+      user: this.user,
+    })
     return table
   }
 
@@ -302,9 +328,9 @@ class TableSaveFunctions {
 }
 
 export async function checkForViewUpdates(
-  table: any,
+  table: Table,
-  rename: any,
+  deletedColumns: string[],
-  deletedColumns: any
+  columnRename?: RenameColumn
 ) {
   const views = await getViews()
   const tableViews = views.filter(view => view.meta.tableId === table._id)
@@ -314,30 +340,30 @@ export async function checkForViewUpdates(
     let needsUpdated = false
 
     // First check for renames, otherwise check for deletions
-    if (rename) {
+    if (columnRename) {
       // Update calculation field if required
-      if (view.meta.field === rename.old) {
+      if (view.meta.field === columnRename.old) {
-        view.meta.field = rename.updated
+        view.meta.field = columnRename.updated
         needsUpdated = true
       }
 
       // Update group by field if required
-      if (view.meta.groupBy === rename.old) {
+      if (view.meta.groupBy === columnRename.old) {
-        view.meta.groupBy = rename.updated
+        view.meta.groupBy = columnRename.updated
         needsUpdated = true
       }
 
       // Update filters if required
       if (view.meta.filters) {
         view.meta.filters.forEach((filter: any) => {
-          if (filter.key === rename.old) {
+          if (filter.key === columnRename.old) {
-            filter.key = rename.updated
+            filter.key = columnRename.updated
             needsUpdated = true
           }
         })
       }
     } else if (deletedColumns) {
-      deletedColumns.forEach((column: any) => {
+      deletedColumns.forEach((column: string) => {
         // Remove calculation statement if required
         if (view.meta.field === column) {
           delete view.meta.field
@@ -378,24 +404,29 @@ export async function checkForViewUpdates(
       if (!newViewTemplate.meta.schema) {
         newViewTemplate.meta.schema = table.schema
       }
-      table.views[view.name] = newViewTemplate.meta
+      if (table.views?.[view.name]) {
+        table.views[view.name] = newViewTemplate.meta as View
+      }
     }
   }
 }
 
-export function generateForeignKey(column: any, relatedTable: any) {
+export function generateForeignKey(
+  column: RelationshipFieldMetadata,
+  relatedTable: Table
+) {
   return `fk_${relatedTable.name}_${column.fieldName}`
 }
 
 export function generateJunctionTableName(
-  column: any,
+  column: RelationshipFieldMetadata,
-  table: any,
+  table: Table,
-  relatedTable: any
+  relatedTable: Table
 ) {
   return `jt_${table.name}_${relatedTable.name}_${column.name}_${column.fieldName}`
 }
 
-export function foreignKeyStructure(keyName: any, meta?: any) {
+export function foreignKeyStructure(keyName: string, meta?: any) {
   const structure: any = {
     type: FieldTypes.NUMBER,
     constraints: {},
@@ -407,7 +438,7 @@ export function foreignKeyStructure(keyName: any, meta?: any) {
   return structure
 }
 
-export function areSwitchableTypes(type1: any, type2: any) {
+export function areSwitchableTypes(type1: FieldType, type2: FieldType) {
   if (
     SwitchableTypes.indexOf(type1) === -1 &&
     SwitchableTypes.indexOf(type2) === -1

packages/server/src/api/controllers/user.ts

@@ -1,13 +1,11 @@
-import { generateUserFlagID } from "../../db/utils"
+import { generateUserFlagID, InternalTables } from "../../db/utils"
-import { InternalTables } from "../../db/utils"
 import { getFullUser } from "../../utilities/users"
 import { context } from "@budibase/backend-core"
 import { Ctx, UserCtx } from "@budibase/types"
 import sdk from "../../sdk"
 
 export async function fetchMetadata(ctx: Ctx) {
-  const users = await sdk.users.fetchMetadata()
+  ctx.body = await sdk.users.fetchMetadata()
-  ctx.body = users
 }
 
 export async function updateSelfMetadata(ctx: UserCtx) {

packages/server/src/api/routes/tests/misc.spec.js

@@ -12,14 +12,14 @@ describe("run misc tests", () => {
   })
 
   describe("/bbtel", () => {
-      it("check if analytics enabled", async () => {
+    it("check if analytics enabled", async () => {
-        const res = await request
+      const res = await request
-          .get(`/api/bbtel`)
+        .get(`/api/bbtel`)
-          .set(config.defaultHeaders())
+        .set(config.defaultHeaders())
-          .expect("Content-Type", /json/)
+        .expect("Content-Type", /json/)
-          .expect(200)
+        .expect(200)
-        expect(typeof res.body.enabled).toEqual("boolean")
+      expect(typeof res.body.enabled).toEqual("boolean")
-      })
+    })
   })
 
   describe("/health", () => {
@@ -37,7 +37,6 @@ describe("run misc tests", () => {
       } else {
         expect(text.split(".").length).toEqual(3)
       }
-      
     })
   })
 
@@ -93,77 +92,79 @@ describe("run misc tests", () => {
               constraints: {
                 type: "array",
                 presence: {
-                  "allowEmpty": true
+                  allowEmpty: true,
                 },
-                inclusion: [
+                inclusion: ["One", "Two", "Three"],
-                  "One",
-                  "Two",
-                  "Three",
-                ]
               },
               name: "Sample Tags",
-              sortable: false
+              sortable: false,
             },
             g: {
               type: "options",
               constraints: {
                 type: "string",
                 presence: false,
-                inclusion: [
+                inclusion: ["Alpha", "Beta", "Gamma"],
-                  "Alpha",
-                  "Beta",
-                  "Gamma"
-                ]
               },
-              name: "Sample Opts"
+              name: "Sample Opts",
-            }
+            },
           },
         })
-        
+
+        const importRows = [
+          { a: "1", b: "2", c: "3", d: "4", f: "['One']", g: "Alpha" },
+          { a: "5", b: "6", c: "7", d: "8", f: "[]", g: undefined },
+          { a: "9", b: "10", c: "11", d: "12", f: "['Two','Four']", g: "" },
+          { a: "13", b: "14", c: "15", d: "16", g: "Omega" },
+        ]
         // Shift specific row tests to the row spec
-        await tableUtils.handleDataImport(
+        await tableUtils.handleDataImport(table, {
-          { userId: "test" },
+          importRows,
-          table,
+          user: { userId: "test" },
-          [
+        })
-            { a: '1', b: '2', c: '3', d: '4', f: "['One']", g: "Alpha" },
-            { a: '5', b: '6', c: '7', d: '8', f: "[]", g: undefined},   
-            { a: '9', b: '10', c: '11', d: '12', f: "['Two','Four']", g: ""},
-            { a: '13', b: '14', c: '15', d: '16', g: "Omega"}
-          ]
-        )
 
         // 4 rows imported, the auto ID starts at 1
         // We expect the handleDataImport function to update the lastID
-        expect(table.schema.e.lastID).toEqual(4);
+        expect(table.schema.e.lastID).toEqual(4)
-        
+
         // Array/Multi - should have added a new value to the inclusion.
-        expect(table.schema.f.constraints.inclusion).toEqual(['Four','One','Three','Two']);
+        expect(table.schema.f.constraints.inclusion).toEqual([
+          "Four",
+          "One",
+          "Three",
+          "Two",
+        ])
 
         // Options - should have a new value in the inclusion
-        expect(table.schema.g.constraints.inclusion).toEqual(['Alpha','Beta','Gamma','Omega']);
+        expect(table.schema.g.constraints.inclusion).toEqual([
+          "Alpha",
+          "Beta",
+          "Gamma",
+          "Omega",
+        ])
 
         const rows = await config.getRows()
-        expect(rows.length).toEqual(4);
+        expect(rows.length).toEqual(4)
 
         const rowOne = rows.find(row => row.e === 1)
         expect(rowOne.a).toEqual("1")
-        expect(rowOne.f).toEqual(['One'])
+        expect(rowOne.f).toEqual(["One"])
-        expect(rowOne.g).toEqual('Alpha')
+        expect(rowOne.g).toEqual("Alpha")
 
         const rowTwo = rows.find(row => row.e === 2)
         expect(rowTwo.a).toEqual("5")
         expect(rowTwo.f).toEqual([])
         expect(rowTwo.g).toEqual(undefined)
-        
+
         const rowThree = rows.find(row => row.e === 3)
         expect(rowThree.a).toEqual("9")
-        expect(rowThree.f).toEqual(['Two','Four'])
+        expect(rowThree.f).toEqual(["Two", "Four"])
         expect(rowThree.g).toEqual(null)
 
         const rowFour = rows.find(row => row.e === 4)
         expect(rowFour.a).toEqual("13")
         expect(rowFour.f).toEqual(undefined)
-        expect(rowFour.g).toEqual('Omega')
+        expect(rowFour.g).toEqual("Omega")
       })
     })
   })

packages/server/src/db/linkedRows/index.ts

@@ -9,13 +9,13 @@ import {
   getLinkedTable,
 } from "./linkUtils"
 import flatten from "lodash/flatten"
-import { FieldTypes } from "../../constants"
 import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils"
 import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
 import { processFormulas } from "../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
-import { Table, Row, LinkDocumentValue } from "@budibase/types"
+import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types"
+import sdk from "../../sdk"
 
 export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils"
 
@@ -35,7 +35,7 @@ export const EventType = {
 
 function clearRelationshipFields(table: Table, rows: Row[]) {
   for (let [key, field] of Object.entries(table.schema)) {
-    if (field.type === FieldTypes.LINK) {
+    if (field.type === FieldType.LINK) {
       rows = rows.map(row => {
         delete row[key]
         return row
@@ -45,7 +45,7 @@ function clearRelationshipFields(table: Table, rows: Row[]) {
   return rows
 }
 
-async function getLinksForRows(rows: Row[]) {
+async function getLinksForRows(rows: Row[]): Promise<LinkDocumentValue[]> {
   const tableIds = [...new Set(rows.map(el => el.tableId))]
   // start by getting all the link values for performance reasons
   const promises = tableIds.map(tableId =>
@@ -146,32 +146,57 @@ export async function updateLinks(args: {
  * This is required for formula fields, this may only be utilised internally (for now).
  * @param table The table from which the rows originated.
  * @param rows The rows which are to be enriched.
+ * @param opts optional - options like passing in a base row to use for enrichment.
  * @return returns the rows with all of the enriched relationships on it.
  */
-export async function attachFullLinkedDocs(table: Table, rows: Row[]) {
+export async function attachFullLinkedDocs(
+  table: Table,
+  rows: Row[],
+  opts?: { fromRow?: Row }
+) {
   const linkedTableIds = getLinkedTableIDs(table)
   if (linkedTableIds.length === 0) {
     return rows
   }
-  // get all the links
+  // get tables and links
-  const links = (await getLinksForRows(rows)).filter(link =>
+  let response = await Promise.all([
+    getLinksForRows(rows),
+    sdk.tables.getTables(linkedTableIds),
+  ])
+  // find the links that pertain to one of the rows that is being enriched
+  const links = (response[0] as LinkDocumentValue[]).filter(link =>
     rows.some(row => row._id === link.thisId)
   )
+  // if fromRow has been passed in, then we don't need to fetch it (optimisation)
+  let linksWithoutFromRow = links
+  if (opts?.fromRow) {
+    linksWithoutFromRow = links.filter(link => link.id !== opts?.fromRow?._id)
+  }
+  const linkedTables = response[1] as Table[]
   // clear any existing links that could be dupe'd
   rows = clearRelationshipFields(table, rows)
   // now get the docs and combine into the rows
-  let linked = await getFullLinkedDocs(links)
+  let linked = []
-  const linkedTables: Table[] = []
+  if (linksWithoutFromRow.length > 0) {
+    linked = await getFullLinkedDocs(linksWithoutFromRow)
+  }
   for (let row of rows) {
     for (let link of links.filter(link => link.thisId === row._id)) {
       if (row[link.fieldName] == null) {
         row[link.fieldName] = []
       }
-      const linkedRow = linked.find(row => row._id === link.id)
+      let linkedRow: Row
+      if (opts?.fromRow && opts?.fromRow?._id === link.id) {
+        linkedRow = opts.fromRow!
+      } else {
+        linkedRow = linked.find(row => row._id === link.id)
+      }
       if (linkedRow) {
         const linkedTableId =
           linkedRow.tableId || getRelatedTableForField(table, link.fieldName)
-        const linkedTable = await getLinkedTable(linkedTableId, linkedTables)
+        const linkedTable = linkedTables.find(
+          table => table._id === linkedTableId
+        )
         if (linkedTable) {
           row[link.fieldName].push(processFormulas(linkedTable, linkedRow))
         }
@@ -199,13 +224,13 @@ export async function squashLinksToPrimaryDisplay(
     // this only fetches the table if its not already in array
     const rowTable = await getLinkedTable(row.tableId!, linkedTables)
     for (let [column, schema] of Object.entries(rowTable?.schema || {})) {
-      if (schema.type !== FieldTypes.LINK || !Array.isArray(row[column])) {
+      if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) {
         continue
       }
       const newLinks = []
       for (let link of row[column]) {
         const linkTblId = link.tableId || getRelatedTableForField(table, column)
-        const linkedTable = await getLinkedTable(linkTblId, linkedTables)
+        const linkedTable = await getLinkedTable(linkTblId!, linkedTables)
         const obj: any = { _id: link._id }
         if (linkedTable?.primaryDisplay && link[linkedTable.primaryDisplay]) {
           obj.primaryDisplay = link[linkedTable.primaryDisplay]

packages/server/src/db/linkedRows/linkUtils.ts

@@ -31,19 +31,22 @@ export const IncludeDocs = {
 export async function getLinkDocuments(args: {
   tableId?: string
   rowId?: string
-  includeDocs?: any
+  fieldName?: string
+  includeDocs?: boolean
 }): Promise<LinkDocumentValue[] | LinkDocument[]> {
-  const { tableId, rowId, includeDocs } = args
+  const { tableId, rowId, fieldName, includeDocs } = args
   const db = context.getAppDB()
   let params: any
-  if (rowId != null) {
+  if (rowId) {
     params = { key: [tableId, rowId] }
   }
   // only table is known
   else {
     params = { startKey: [tableId], endKey: [tableId, {}] }
   }
-  params.include_docs = !!includeDocs
+  if (includeDocs) {
+    params.include_docs = true
+  }
   try {
     let linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows
     // filter to get unique entries
@@ -63,6 +66,14 @@ export async function getLinkDocuments(args: {
       return unique
     })
 
+    // filter down to just the required field name
+    if (fieldName) {
+      linkRows = linkRows.filter(link => {
+        const value = link.value as LinkDocumentValue
+        return value.fieldName === fieldName
+      })
+    }
+    // return docs if docs requested, otherwise just the value information
     if (includeDocs) {
       return linkRows.map(row => row.doc) as LinkDocument[]
     } else {
@@ -87,7 +98,7 @@ export function getUniqueByProp(array: any[], prop: string) {
   })
 }
 
-export function getLinkedTableIDs(table: Table) {
+export function getLinkedTableIDs(table: Table): string[] {
   return Object.values(table.schema)
     .filter(isRelationshipColumn)
     .map(column => column.tableId)

packages/server/src/middleware/tests/trimViewRowInfo.spec.ts

@@ -16,6 +16,7 @@ jest.mock("../../sdk", () => ({
 import sdk from "../../sdk"
 import { Next } from "koa"
 
+const tableId = utils.generateTableID()
 const mockGetView = sdk.views.get as jest.MockedFunction<typeof sdk.views.get>
 const mockGetTable = sdk.tables.getTable as jest.MockedFunction<
   typeof sdk.tables.getTable
@@ -41,6 +42,7 @@ class TestConfiguration {
       body: ctxRequestBody,
     }
     this.params.viewId = viewId
+    this.params.sourceId = tableId
     return this.middleware(
       {
         request: this.request as any,
@@ -69,7 +71,7 @@ describe("trimViewRowInfo middleware", () => {
   })
 
   const table: Table = {
-    _id: utils.generateTableID(),
+    _id: tableId,
     name: generator.word(),
     type: "table",
     schema: {

packages/server/src/sdk/app/datasources/datasources.ts

@@ -23,10 +23,13 @@ import {
   getTableParams,
 } from "../../../db/utils"
 import sdk from "../../index"
+import datasource from "../../../api/routes/datasource"
 
 const ENV_VAR_PREFIX = "env."
 
-export async function fetch() {
+export async function fetch(opts?: {
+  enriched: boolean
+}): Promise<Datasource[]> {
   // Get internal tables
   const db = context.getAppDB()
   const internalTables = await db.allDocs(
@@ -44,7 +47,7 @@ export async function fetch() {
 
   const bbInternalDb = {
     ...BudibaseInternalDB,
-  }
+  } as Datasource
 
   // Get external datasources
   const datasources = (
@@ -66,7 +69,18 @@ export async function fetch() {
     }
   }
 
-  return [bbInternalDb, ...datasources]
+  if (opts?.enriched) {
+    const envVars = await getEnvironmentVariables()
+    const promises = datasources.map(datasource =>
+      enrichDatasourceWithValues(datasource, envVars)
+    )
+    const enriched = (await Promise.all(promises)).map(
+      result => result.datasource
+    )
+    return [bbInternalDb, ...enriched]
+  } else {
+    return [bbInternalDb, ...datasources]
+  }
 }
 
 export function areRESTVariablesValid(datasource: Datasource) {
@@ -107,9 +121,12 @@ export function checkDatasourceTypes(schema: Integration, config: any) {
   return config
 }
 
-async function enrichDatasourceWithValues(datasource: Datasource) {
+async function enrichDatasourceWithValues(
+  datasource: Datasource,
+  variables?: Record<string, string>
+) {
   const cloned = cloneDeep(datasource)
-  const env = await getEnvironmentVariables()
+  const env = variables ? variables : await getEnvironmentVariables()
   //Do not process entities, as we do not want to process formulas
   const { entities, ...clonedWithoutEntities } = cloned
   const processed = processObjectSync(
@@ -235,9 +252,9 @@ export function mergeConfigs(update: Datasource, old: Datasource) {
     if (value !== PASSWORD_REPLACEMENT) {
       continue
     }
-    if (old.config?.[key]) {
+    if (update.config && old.config && old.config?.[key]) {
       update.config[key] = old.config?.[key]
-    } else {
+    } else if (update.config) {
       delete update.config[key]
     }
   }

packages/server/src/sdk/app/rows/utils.ts

@@ -69,12 +69,15 @@ export async function validate({
   valid: boolean
   errors: Record<string, any>
 }> {
-  let fetchedTable: Table
+  let fetchedTable: Table | undefined
-  if (!table) {
+  if (!table && tableId) {
     fetchedTable = await sdk.tables.getTable(tableId)
-  } else {
+  } else if (table) {
     fetchedTable = table
   }
+  if (fetchedTable === undefined) {
+    throw new Error("Unable to fetch table for validation")
+  }
   const errors: Record<string, any> = {}
   for (let fieldName of Object.keys(fetchedTable.schema)) {
     const column = fetchedTable.schema[fieldName]

packages/server/src/sdk/app/tables/external/index.ts

@@ -0,0 +1,196 @@
+import {
+  Operation,
+  RelationshipType,
+  RenameColumn,
+  Table,
+  TableRequest,
+  ViewV2,
+} from "@budibase/types"
+import { context } from "@budibase/backend-core"
+import { buildExternalTableId } from "../../../../integrations/utils"
+import {
+  foreignKeyStructure,
+  hasTypeChanged,
+  setStaticSchemas,
+} from "../../../../api/controllers/table/utils"
+import { cloneDeep } from "lodash/fp"
+import { FieldTypes } from "../../../../constants"
+import { makeTableRequest } from "../../../../api/controllers/table/ExternalRequest"
+import {
+  isRelationshipSetup,
+  cleanupRelationships,
+  generateLinkSchema,
+  generateManyLinkSchema,
+  generateRelatedSchema,
+} from "./utils"
+
+import { getTable } from "../getters"
+import { populateExternalTableSchemas } from "../validation"
+import datasourceSdk from "../../datasources"
+import * as viewSdk from "../../views"
+
+export async function save(
+  datasourceId: string,
+  update: Table,
+  opts?: { tableId?: string; renaming?: RenameColumn }
+) {
+  let tableToSave: TableRequest = {
+    type: "table",
+    _id: buildExternalTableId(datasourceId, update.name),
+    sourceId: datasourceId,
+    ...update,
+  }
+
+  const tableId = opts?.tableId || update._id
+  let oldTable: Table | undefined
+  if (tableId) {
+    oldTable = await getTable(tableId)
+  }
+
+  if (hasTypeChanged(tableToSave, oldTable)) {
+    throw new Error("A column type has changed.")
+  }
+
+  for (let view in tableToSave.views) {
+    const tableView = tableToSave.views[view]
+    if (!tableView || !viewSdk.isV2(tableView)) continue
+
+    tableToSave.views[view] = viewSdk.syncSchema(
+      oldTable!.views![view] as ViewV2,
+      tableToSave.schema,
+      opts?.renaming
+    )
+  }
+
+  const db = context.getAppDB()
+  const datasource = await datasourceSdk.get(datasourceId)
+  if (!datasource.entities) {
+    datasource.entities = {}
+  }
+
+  // GSheets is a specific case - only ever has a static primary key
+  tableToSave = setStaticSchemas(datasource, tableToSave)
+
+  const oldTables = cloneDeep(datasource.entities)
+  const tables: Record<string, Table> = datasource.entities
+
+  const extraTablesToUpdate = []
+
+  // check if relations need setup
+  for (let schema of Object.values(tableToSave.schema)) {
+    if (schema.type !== FieldTypes.LINK || isRelationshipSetup(schema)) {
+      continue
+    }
+    const schemaTableId = schema.tableId
+    const relatedTable = Object.values(tables).find(
+      table => table._id === schemaTableId
+    )
+    if (!relatedTable) {
+      continue
+    }
+    const relatedColumnName = schema.fieldName!
+    const relationType = schema.relationshipType
+    if (relationType === RelationshipType.MANY_TO_MANY) {
+      const junctionTable = generateManyLinkSchema(
+        datasource,
+        schema,
+        tableToSave,
+        relatedTable
+      )
+      if (tables[junctionTable.name]) {
+        throw new Error(
+          "Junction table already exists, cannot create another relationship."
+        )
+      }
+      tables[junctionTable.name] = junctionTable
+      extraTablesToUpdate.push(junctionTable)
+    } else {
+      const fkTable =
+        relationType === RelationshipType.ONE_TO_MANY
+          ? tableToSave
+          : relatedTable
+      const foreignKey = generateLinkSchema(
+        schema,
+        tableToSave,
+        relatedTable,
+        relationType
+      )
+      if (fkTable.schema[foreignKey] != null) {
+        throw new Error(
+          `Unable to generate foreign key - column ${foreignKey} already in use.`
+        )
+      }
+      fkTable.schema[foreignKey] = foreignKeyStructure(foreignKey)
+      if (fkTable.constrained == null) {
+        fkTable.constrained = []
+      }
+      if (fkTable.constrained.indexOf(foreignKey) === -1) {
+        fkTable.constrained.push(foreignKey)
+      }
+      // foreign key is in other table, need to save it to external
+      if (fkTable._id !== tableToSave._id) {
+        extraTablesToUpdate.push(fkTable)
+      }
+    }
+    generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName)
+    schema.main = true
+  }
+
+  cleanupRelationships(tableToSave, tables, oldTable)
+
+  const operation = tableId ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE
+  await makeTableRequest(
+    datasource,
+    operation,
+    tableToSave,
+    tables,
+    oldTable,
+    opts?.renaming
+  )
+  // update any extra tables (like foreign keys in other tables)
+  for (let extraTable of extraTablesToUpdate) {
+    const oldExtraTable = oldTables[extraTable.name]
+    let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE
+    await makeTableRequest(datasource, op, extraTable, tables, oldExtraTable)
+  }
+
+  // make sure the constrained list, all still exist
+  if (Array.isArray(tableToSave.constrained)) {
+    tableToSave.constrained = tableToSave.constrained.filter(constraint =>
+      Object.keys(tableToSave.schema).includes(constraint)
+    )
+  }
+
+  // remove the rename prop
+  delete tableToSave._rename
+  // store it into couch now for budibase reference
+  datasource.entities[tableToSave.name] = tableToSave
+  await db.put(populateExternalTableSchemas(datasource))
+
+  // Since tables are stored inside datasources, we need to notify clients
+  // that the datasource definition changed
+  const updatedDatasource = await datasourceSdk.get(datasource._id!)
+
+  return { datasource: updatedDatasource, table: tableToSave }
+}
+
+export async function destroy(datasourceId: string, table: Table) {
+  const db = context.getAppDB()
+  const datasource = await datasourceSdk.get(datasourceId)
+  const tables = datasource.entities
+
+  const operation = Operation.DELETE_TABLE
+  if (tables) {
+    await makeTableRequest(datasource, operation, table, tables)
+    cleanupRelationships(table, tables)
+    delete tables[table.name]
+    datasource.entities = tables
+  }
+
+  await db.put(populateExternalTableSchemas(datasource))
+
+  // Since tables are stored inside datasources, we need to notify clients
+  // that the datasource definition changed
+  const updatedDatasource = await datasourceSdk.get(datasource._id!)
+  return { datasource: updatedDatasource, table }
+}

packages/server/src/sdk/app/tables/external/utils.ts

@@ -0,0 +1,161 @@
+import {
+  Datasource,
+  ManyToManyRelationshipFieldMetadata,
+  ManyToOneRelationshipFieldMetadata,
+  OneToManyRelationshipFieldMetadata,
+  RelationshipFieldMetadata,
+  RelationshipType,
+  Table,
+} from "@budibase/types"
+import { FieldTypes } from "../../../../constants"
+import {
+  foreignKeyStructure,
+  generateForeignKey,
+  generateJunctionTableName,
+} from "../../../../api/controllers/table/utils"
+import { buildExternalTableId } from "../../../../integrations/utils"
+import { cloneDeep } from "lodash/fp"
+
+export function cleanupRelationships(
+  table: Table,
+  tables: Record<string, Table>,
+  oldTable?: Table
+) {
+  const tableToIterate = oldTable ? oldTable : table
+  // clean up relationships in couch table schemas
+  for (let [key, schema] of Object.entries(tableToIterate.schema)) {
+    if (
+      schema.type === FieldTypes.LINK &&
+      (!oldTable || table.schema[key] == null)
+    ) {
+      const schemaTableId = schema.tableId
+      const relatedTable = Object.values(tables).find(
+        table => table._id === schemaTableId
+      )
+      const foreignKey =
+        schema.relationshipType !== RelationshipType.MANY_TO_MANY &&
+        schema.foreignKey
+      if (!relatedTable || !foreignKey) {
+        continue
+      }
+      for (let [relatedKey, relatedSchema] of Object.entries(
+        relatedTable.schema
+      )) {
+        if (
+          relatedSchema.type === FieldTypes.LINK &&
+          relatedSchema.fieldName === foreignKey
+        ) {
+          delete relatedTable.schema[relatedKey]
+        }
+      }
+    }
+  }
+}
+
+export function otherRelationshipType(type: RelationshipType) {
+  if (type === RelationshipType.MANY_TO_MANY) {
+    return RelationshipType.MANY_TO_MANY
+  }
+  return type === RelationshipType.ONE_TO_MANY
+    ? RelationshipType.MANY_TO_ONE
+    : RelationshipType.ONE_TO_MANY
+}
+
+export function generateManyLinkSchema(
+  datasource: Datasource,
+  column: ManyToManyRelationshipFieldMetadata,
+  table: Table,
+  relatedTable: Table
+): Table {
+  if (!table.primary || !relatedTable.primary) {
+    const noPrimaryName = !table.primary ? table.name : relatedTable.name
+    throw new Error(
+      `Unable to generate many link schema, "${noPrimaryName}" does not have a primary key`
+    )
+  }
+  const primary = table.name + table.primary[0]
+  const relatedPrimary = relatedTable.name + relatedTable.primary[0]
+  const jcTblName = generateJunctionTableName(column, table, relatedTable)
+  // first create the new table
+  const junctionTable = {
+    _id: buildExternalTableId(datasource._id!, jcTblName),
+    name: jcTblName,
+    primary: [primary, relatedPrimary],
+    constrained: [primary, relatedPrimary],
+    schema: {
+      [primary]: foreignKeyStructure(primary, {
+        toTable: table.name,
+        toKey: table.primary[0],
+      }),
+      [relatedPrimary]: foreignKeyStructure(relatedPrimary, {
+        toTable: relatedTable.name,
+        toKey: relatedTable.primary[0],
+      }),
+    },
+  }
+  column.through = junctionTable._id
+  column.throughFrom = relatedPrimary
+  column.throughTo = primary
+  column.fieldName = relatedPrimary
+  return junctionTable
+}
+
+export function generateLinkSchema(
+  column:
+    | OneToManyRelationshipFieldMetadata
+    | ManyToOneRelationshipFieldMetadata,
+  table: Table,
+  relatedTable: Table,
+  type: RelationshipType.ONE_TO_MANY | RelationshipType.MANY_TO_ONE
+) {
+  if (!table.primary || !relatedTable.primary) {
+    throw new Error("Unable to generate link schema, no primary keys")
+  }
+  const isOneSide = type === RelationshipType.ONE_TO_MANY
+  const primary = isOneSide ? relatedTable.primary[0] : table.primary[0]
+  // generate a foreign key
+  const foreignKey = generateForeignKey(column, relatedTable)
+  column.relationshipType = type
+  column.foreignKey = isOneSide ? foreignKey : primary
+  column.fieldName = isOneSide ? primary : foreignKey
+  return foreignKey
+}
+
+export function generateRelatedSchema(
+  linkColumn: RelationshipFieldMetadata,
+  table: Table,
+  relatedTable: Table,
+  columnName: string
+) {
+  // generate column for other table
+  let relatedSchema: RelationshipFieldMetadata
+  const isMany2Many =
+    linkColumn.relationshipType === RelationshipType.MANY_TO_MANY
+  // swap them from the main link
+  if (!isMany2Many && linkColumn.foreignKey) {
+    relatedSchema = cloneDeep(linkColumn) as
+      | OneToManyRelationshipFieldMetadata
+      | ManyToOneRelationshipFieldMetadata
+    relatedSchema.fieldName = linkColumn.foreignKey
+    relatedSchema.foreignKey = linkColumn.fieldName
+  }
+  // is many to many
+  else {
+    const manyToManyCol = linkColumn as ManyToManyRelationshipFieldMetadata
+    relatedSchema = cloneDeep(linkColumn) as ManyToManyRelationshipFieldMetadata
+    // don't need to copy through, already got it
+    relatedSchema.fieldName = manyToManyCol.throughTo!
+    relatedSchema.throughTo = manyToManyCol.throughFrom
+    relatedSchema.throughFrom = manyToManyCol.throughTo
+  }
+  relatedSchema.relationshipType = otherRelationshipType(
+    linkColumn.relationshipType
+  )
+  relatedSchema.tableId = relatedTable._id!
+  relatedSchema.name = columnName
+  table.schema[columnName] = relatedSchema
+}
+
+export function isRelationshipSetup(column: RelationshipFieldMetadata) {
+  return (column as any).foreignKey || (column as any).through
+}

packages/server/src/sdk/app/tables/getters.ts

@@ -0,0 +1,124 @@
+import { context } from "@budibase/backend-core"
+import {
+  BudibaseInternalDB,
+  getMultiIDParams,
+  getTableParams,
+} from "../../../db/utils"
+import {
+  breakExternalTableId,
+  isExternalTable,
+  isSQL,
+} from "../../../integrations/utils"
+import {
+  AllDocsResponse,
+  Database,
+  Table,
+  TableResponse,
+  TableViewsResponse,
+} from "@budibase/types"
+import datasources from "../datasources"
+import sdk from "../../../sdk"
+
+function processInternalTables(docs: AllDocsResponse<Table[]>): Table[] {
+  return docs.rows.map((tableDoc: any) => ({
+    ...tableDoc.doc,
+    type: "internal",
+    sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id,
+  }))
+}
+
+export async function getAllInternalTables(db?: Database): Promise<Table[]> {
+  if (!db) {
+    db = context.getAppDB()
+  }
+  const internalTables = await db.allDocs<Table[]>(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  return processInternalTables(internalTables)
+}
+
+async function getAllExternalTables(): Promise<Table[]> {
+  const datasources = await sdk.datasources.fetch({ enriched: true })
+  const allEntities = datasources.map(datasource => datasource.entities)
+  let final: Table[] = []
+  for (let entities of allEntities) {
+    if (entities) {
+      final = final.concat(Object.values(entities))
+    }
+  }
+  return final
+}
+
+export async function getExternalTable(
+  datasourceId: string,
+  tableName: string
+): Promise<Table> {
+  const entities = await getExternalTablesInDatasource(datasourceId)
+  return entities[tableName]
+}
+
+export async function getTable(tableId: string): Promise<Table> {
+  const db = context.getAppDB()
+  if (isExternalTable(tableId)) {
+    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    const datasource = await datasources.get(datasourceId!)
+    const table = await getExternalTable(datasourceId!, tableName!)
+    return { ...table, sql: isSQL(datasource) }
+  } else {
+    return db.get(tableId)
+  }
+}
+
+export async function getAllTables() {
+  const [internal, external] = await Promise.all([
+    getAllInternalTables(),
+    getAllExternalTables(),
+  ])
+  return [...internal, ...external]
+}
+
+export async function getExternalTablesInDatasource(
+  datasourceId: string
+): Promise<Record<string, Table>> {
+  const datasource = await datasources.get(datasourceId, { enriched: true })
+  if (!datasource || !datasource.entities) {
+    throw new Error("Datasource is not configured fully.")
+  }
+  return datasource.entities
+}
+
+export async function getTables(tableIds: string[]): Promise<Table[]> {
+  const externalTableIds = tableIds.filter(tableId => isExternalTable(tableId)),
+    internalTableIds = tableIds.filter(tableId => !isExternalTable(tableId))
+  let tables: Table[] = []
+  if (externalTableIds.length) {
+    const externalTables = await getAllExternalTables()
+    tables = tables.concat(
+      externalTables.filter(
+        table => externalTableIds.indexOf(table._id!) !== -1
+      )
+    )
+  }
+  if (internalTableIds.length) {
+    const db = context.getAppDB()
+    const internalTableDocs = await db.allDocs<Table[]>(
+      getMultiIDParams(internalTableIds)
+    )
+    tables = tables.concat(processInternalTables(internalTableDocs))
+  }
+  return tables
+}
+
+export function enrichViewSchemas(table: Table): TableResponse {
+  return {
+    ...table,
+    views: Object.values(table.views ?? [])
+      .map(v => sdk.views.enrichSchema(v, table.schema))
+      .reduce((p, v) => {
+        p[v.name!] = v
+        return p
+      }, {} as TableViewsResponse),
+  }
+}

packages/server/src/sdk/app/tables/index.ts

@@ -1,95 +1,11 @@
-import { context } from "@budibase/backend-core"
-import { BudibaseInternalDB, getTableParams } from "../../../db/utils"
-import {
-  breakExternalTableId,
-  isExternalTable,
-  isSQL,
-} from "../../../integrations/utils"
-import {
-  Database,
-  Table,
-  TableResponse,
-  TableViewsResponse,
-} from "@budibase/types"
-import datasources from "../datasources"
 import { populateExternalTableSchemas } from "./validation"
-import sdk from "../../../sdk"
+import * as getters from "./getters"
-
+import * as updates from "./update"
-async function getAllInternalTables(db?: Database): Promise<Table[]> {
+import * as utils from "./utils"
-  if (!db) {
-    db = context.getAppDB()
-  }
-  const internalTables = await db.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-  return internalTables.rows.map((tableDoc: any) => ({
-    ...tableDoc.doc,
-    type: "internal",
-    sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id,
-  }))
-}
-
-async function getAllExternalTables(
-  datasourceId: any
-): Promise<Record<string, Table>> {
-  const datasource = await datasources.get(datasourceId, { enriched: true })
-  if (!datasource || !datasource.entities) {
-    throw "Datasource is not configured fully."
-  }
-  return datasource.entities
-}
-
-async function getExternalTable(
-  datasourceId: any,
-  tableName: any
-): Promise<Table> {
-  const entities = await getAllExternalTables(datasourceId)
-  return entities[tableName]
-}
-
-async function getTable(tableId: any): Promise<Table> {
-  const db = context.getAppDB()
-  if (isExternalTable(tableId)) {
-    let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await datasources.get(datasourceId!)
-    const table = await getExternalTable(datasourceId, tableName)
-    return { ...table, sql: isSQL(datasource) }
-  } else {
-    return db.get(tableId)
-  }
-}
-
-function enrichViewSchemas(table: Table): TableResponse {
-  return {
-    ...table,
-    views: Object.values(table.views ?? [])
-      .map(v => sdk.views.enrichSchema(v, table.schema))
-      .reduce((p, v) => {
-        p[v.name] = v
-        return p
-      }, {} as TableViewsResponse),
-  }
-}
-
-async function saveTable(table: Table) {
-  const db = context.getAppDB()
-  if (isExternalTable(table._id!)) {
-    const datasource = await sdk.datasources.get(table.sourceId!)
-    datasource.entities![table.name] = table
-    await db.put(datasource)
-  } else {
-    await db.put(table)
-  }
-}
 
 export default {
-  getAllInternalTables,
-  getAllExternalTables,
-  getExternalTable,
-  getTable,
   populateExternalTableSchemas,
-  enrichViewSchemas,
+  ...updates,
-  saveTable,
+  ...getters,
+  ...utils,
 }

packages/server/src/sdk/app/tables/internal/index.ts

@@ -0,0 +1,172 @@
+import {
+  RenameColumn,
+  Table,
+  ViewStatisticsSchema,
+  ViewV2,
+  Row,
+  ContextUser,
+} from "@budibase/types"
+import {
+  hasTypeChanged,
+  TableSaveFunctions,
+} from "../../../../api/controllers/table/utils"
+import { FieldTypes } from "../../../../constants"
+import { EventType, updateLinks } from "../../../../db/linkedRows"
+import { cloneDeep } from "lodash/fp"
+import isEqual from "lodash/isEqual"
+import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula"
+import { context } from "@budibase/backend-core"
+import { getTable } from "../getters"
+import { checkAutoColumns } from "./utils"
+import * as viewsSdk from "../../views"
+import sdk from "../../../index"
+import { getRowParams } from "../../../../db/utils"
+import { quotas } from "@budibase/pro"
+import env from "../../../../environment"
+import { cleanupAttachments } from "../../../../utilities/rowProcessor"
+
+export async function save(
+  table: Table,
+  opts?: {
+    user?: ContextUser
+    tableId?: string
+    rowsToImport?: Row[]
+    renaming?: RenameColumn
+  }
+) {
+  const db = context.getAppDB()
+
+  // if the table obj had an _id then it will have been retrieved
+  let oldTable: Table | undefined
+  if (opts?.tableId) {
+    oldTable = await getTable(opts.tableId)
+  }
+
+  // check all types are correct
+  if (hasTypeChanged(table, oldTable)) {
+    throw new Error("A column type has changed.")
+  }
+  // check that subtypes have been maintained
+  table = checkAutoColumns(table, oldTable)
+
+  // saving a table is a complex operation, involving many different steps, this
+  // has been broken out into a utility to make it more obvious/easier to manipulate
+  const tableSaveFunctions = new TableSaveFunctions({
+    user: opts?.user,
+    oldTable,
+    importRows: opts?.rowsToImport,
+  })
+  table = await tableSaveFunctions.before(table)
+
+  let renaming = opts?.renaming
+  if (renaming && renaming.old === renaming.updated) {
+    renaming = undefined
+  }
+
+  // rename row fields when table column is renamed
+  if (renaming && table.schema[renaming.updated].type === FieldTypes.LINK) {
+    throw new Error("Cannot rename a linked column.")
+  }
+
+  table = await tableSaveFunctions.mid(table, renaming)
+
+  // update schema of non-statistics views when new columns are added
+  for (let view in table.views) {
+    const tableView = table.views[view]
+    if (!tableView) continue
+
+    if (viewsSdk.isV2(tableView)) {
+      table.views[view] = viewsSdk.syncSchema(
+        oldTable!.views![view] as ViewV2,
+        table.schema,
+        renaming
+      )
+      continue
+    }
+
+    if (
+      (tableView.schema as ViewStatisticsSchema).group ||
+      tableView.schema.field
+    )
+      continue
+    tableView.schema = table.schema
+  }
+
+  // update linked rows
+  const linkResp: any = await updateLinks({
+    eventType: oldTable ? EventType.TABLE_UPDATED : EventType.TABLE_SAVE,
+    table: table,
+    oldTable: oldTable,
+  })
+  if (linkResp != null && linkResp._rev) {
+    table._rev = linkResp._rev
+  }
+
+  // don't perform any updates until relationships have been
+  // checked by the updateLinks function
+  const updatedRows = tableSaveFunctions.getUpdatedRows()
+  if (updatedRows && updatedRows.length !== 0) {
+    await db.bulkDocs(updatedRows)
+  }
+  let result = await db.put(table)
+  table._rev = result.rev
+  const savedTable = cloneDeep(table)
+
+  table = await tableSaveFunctions.after(table)
+  // the table may be updated as part of the table save after functionality - need to write it
+  if (!isEqual(savedTable, table)) {
+    result = await db.put(table)
+    table._rev = result.rev
+  }
+  // has to run after, make sure it has _id
+  await runStaticFormulaChecks(table, { oldTable, deletion: false })
+  return { table }
+}
+
+export async function destroy(table: Table) {
+  const db = context.getAppDB()
+  const tableId = table._id!
+
+  // Delete all rows for that table
+  const rowsData = await db.allDocs(
+    getRowParams(tableId, null, {
+      include_docs: true,
+    })
+  )
+  await db.bulkDocs(
+    rowsData.rows.map((row: any) => ({ ...row.doc, _deleted: true }))
+  )
+  await quotas.removeRows(rowsData.rows.length, {
+    tableId,
+  })
+
+  // update linked rows
+  await updateLinks({
+    eventType: EventType.TABLE_DELETE,
+    table: table,
+  })
+
+  // don't remove the table itself until very end
+  await db.remove(tableId, table._rev)
+
+  // remove table search index
+  if (!env.isTest() || env.COUCH_DB_URL) {
+    const currentIndexes = await db.getIndexes()
+    const existingIndex = currentIndexes.indexes.find(
+      (existing: any) => existing.name === `search:${tableId}`
+    )
+    if (existingIndex) {
+      await db.deleteIndex(existingIndex)
+    }
+  }
+
+  // has to run after, make sure it has _id
+  await runStaticFormulaChecks(table, {
+    deletion: true,
+  })
+  await cleanupAttachments(table, {
+    rows: rowsData.rows.map((row: any) => row.doc),
+  })
+
+  return { table }
+}

packages/server/src/sdk/app/tables/internal/utils.ts

@@ -0,0 +1,20 @@
+import { Table } from "@budibase/types"
+import { fixAutoColumnSubType } from "../../../../utilities/rowProcessor"
+
+export function checkAutoColumns(table: Table, oldTable?: Table) {
+  if (!table.schema) {
+    return table
+  }
+  for (let [key, schema] of Object.entries(table.schema)) {
+    if (!schema.autocolumn || schema.subtype) {
+      continue
+    }
+    const oldSchema = oldTable && oldTable.schema[key]
+    if (oldSchema && oldSchema.subtype) {
+      table.schema[key].subtype = oldSchema.subtype
+    } else {
+      table.schema[key] = fixAutoColumnSubType(schema)
+    }
+  }
+  return table
+}

packages/server/src/sdk/app/tables/update.ts

@@ -0,0 +1,31 @@
+import { Table, RenameColumn } from "@budibase/types"
+import { isExternalTable } from "../../../integrations/utils"
+import sdk from "../../index"
+import { context } from "@budibase/backend-core"
+import { isExternal } from "./utils"
+
+import * as external from "./external"
+import * as internal from "./internal"
+export * as external from "./external"
+export * as internal from "./internal"
+
+export async function saveTable(table: Table) {
+  const db = context.getAppDB()
+  if (isExternalTable(table._id!)) {
+    const datasource = await sdk.datasources.get(table.sourceId!)
+    datasource.entities![table.name] = table
+    await db.put(datasource)
+  } else {
+    await db.put(table)
+  }
+}
+
+export async function update(table: Table, renaming?: RenameColumn) {
+  const tableId = table._id
+  if (isExternal({ table })) {
+    const datasourceId = table.sourceId!
+    await external.save(datasourceId, table, { tableId, renaming })
+  } else {
+    await internal.save(table, { tableId, renaming })
+  }
+}

packages/server/src/sdk/app/tables/utils.ts

@@ -0,0 +1,11 @@
+import { Table } from "@budibase/types"
+import { isExternalTable } from "../../../integrations/utils"
+
+export function isExternal(opts: { table?: Table; tableId?: string }): boolean {
+  if (opts.table && opts.table.type === "external") {
+    return true
+  } else if (opts.tableId && isExternalTable(opts.tableId)) {
+    return true
+  }
+  return false
+}

packages/server/src/sdk/app/views/external.ts

@@ -59,11 +59,10 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
   const existingView = Object.values(views).find(
     v => isV2(v) && v.id === view.id
   )
-  if (!existingView) {
+  if (!existingView || !existingView.name) {
     throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
   }
 
-  console.log("set to", view)
   delete views[existingView.name]
   views[view.name] = view
   await db.put(ds)

packages/server/src/sdk/app/views/internal.ts

@@ -51,11 +51,10 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
   const existingView = Object.values(table.views).find(
     v => isV2(v) && v.id === view.id
   )
-  if (!existingView) {
+  if (!existingView || !existingView.name) {
     throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
   }
 
-  console.log("set to", view)
   delete table.views[existingView.name]
   table.views[view.name] = view
   await db.put(table)

packages/server/src/utilities/rowProcessor/index.ts

@@ -210,6 +210,7 @@ export async function outputProcessing<T extends Row[] | Row>(
   opts: {
     squash?: boolean
     preserveLinks?: boolean
+    fromRow?: Row
     skipBBReferences?: boolean
   } = {
     squash: true,
@@ -227,7 +228,9 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
   // attach any linked row information
   let enriched = !opts.preserveLinks
-    ? await linkRows.attachFullLinkedDocs(table, safeRows)
+    ? await linkRows.attachFullLinkedDocs(table, safeRows, {
+        fromRow: opts?.fromRow,
+      })
     : safeRows
 
   // process complex types: attachements, bb references...

packages/types/src/documents/app/view.ts

@@ -2,7 +2,7 @@ import { SearchFilter, SortOrder, SortType } from "../../api"
 import { UIFieldMetadata } from "./table"
 
 export interface View {
-  name: string
+  name?: string
   tableId: string
   field?: string
   filters: ViewFilter[]