Fix to make foreign keys auto-columns when used in relationships as well as making sure that when fetching tables that they can be removed, using the old fetch mechanism (to be replaced, but needs to work for now).

packages/server/src/api/controllers/datasource.ts

@@ -183,9 +183,7 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
 
   let { tables, error } = await buildSchemaHelper(datasource)
   if (tablesFilter) {
-    if (!datasource.entities) {
     datasource.entities = {}
-    }
     for (let key in tables) {
       if (
         tablesFilter.some(
@@ -200,7 +198,7 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
   }
 
   setDefaultDisplayColumns(datasource)
-  const dbResp = await db.put(datasource)
+  const dbResp = await db.put(sdk.tables.checkExternalTableSchemas(datasource))
   datasource._rev = dbResp.rev
   const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource)
 
@@ -286,7 +284,9 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
     datasource.config!.auth = auth
   }
 
-  const response = await db.put(datasource)
+  const response = await db.put(
+    sdk.tables.checkExternalTableSchemas(datasource)
+  )
   await events.datasource.updated(datasource)
   datasource._rev = response.rev
 
@@ -327,7 +327,7 @@ export async function save(
     setDefaultDisplayColumns(datasource)
   }
 
-  const dbResp = await db.put(datasource)
+  const dbResp = await db.put(sdk.tables.checkExternalTableSchemas(datasource))
   await events.datasource.created(datasource)
   datasource._rev = dbResp.rev
 

packages/server/src/api/controllers/table/external.ts

@@ -1,32 +1,34 @@
 import {
-  buildExternalTableId,
   breakExternalTableId,
+  buildExternalTableId,
 } from "../../../integrations/utils"
 import {
+  foreignKeyStructure,
   generateForeignKey,
   generateJunctionTableName,
-  foreignKeyStructure,
   hasTypeChanged,
   setStaticSchemas,
 } from "./utils"
 import { FieldTypes } from "../../../constants"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { handleRequest } from "../row/external"
-import { events, context } from "@budibase/backend-core"
+import { context, events } from "@budibase/backend-core"
-import { parse, isRows, isSchema } from "../../../utilities/schema"
+import { isRows, isSchema, parse } from "../../../utilities/schema"
 import {
+  AutoReason,
   Datasource,
-  Table,
-  QueryJson,
-  Operation,
-  RenameColumn,
   FieldSchema,
-  UserCtx,
+  Operation,
-  TableRequest,
+  QueryJson,
   RelationshipTypes,
+  RenameColumn,
+  Table,
+  TableRequest,
+  UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { builderSocket } from "../../../websockets"
+
 const { cloneDeep } = require("lodash/fp")
 
 async function makeTableRequest(
@@ -317,7 +319,7 @@ export async function save(ctx: UserCtx) {
   delete tableToSave._rename
   // store it into couch now for budibase reference
   datasource.entities[tableToSave.name] = tableToSave
-  await db.put(datasource)
+  await db.put(sdk.tables.checkExternalTableSchemas(datasource))
 
   // Since tables are stored inside datasources, we need to notify clients
   // that the datasource definition changed
@@ -348,7 +350,7 @@ export async function destroy(ctx: UserCtx) {
     datasource.entities = tables
   }
 
-  await db.put(datasource)
+  await db.put(sdk.tables.checkExternalTableSchemas(datasource))
 
   // Since tables are stored inside datasources, we need to notify clients
   // that the datasource definition changed

packages/server/src/sdk/app/tables/index.ts

@@ -7,6 +7,7 @@ import {
 } from "../../../integrations/utils"
 import { Table, Database } from "@budibase/types"
 import datasources from "../datasources"
+import { checkExternalTableSchemas } from "./validation"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
   if (!db) {
@@ -60,4 +61,5 @@ export default {
   getAllExternalTables,
   getExternalTable,
   getTable,
+  checkExternalTableSchemas,
 }

packages/server/src/sdk/app/tables/validation.ts

@@ -0,0 +1,57 @@
+import {
+  Datasource,
+  FieldType,
+  RelationshipTypes,
+  AutoReason,
+} from "@budibase/types"
+
+function checkForeignKeysAreAutoColumns(datasource: Datasource) {
+  if (!datasource.entities) {
+    return datasource
+  }
+  const tables = Object.values(datasource.entities)
+  // make sure all foreign key columns are marked as auto columns
+  const foreignKeys: { tableId: string; key: string }[] = []
+  for (let table of tables) {
+    const relationships = Object.values(table.schema).filter(
+      column => column.type === FieldType.LINK
+    )
+    relationships.forEach(relationship => {
+      if (relationship.relationshipType === RelationshipTypes.MANY_TO_MANY) {
+        const tableId = relationship.through!
+        foreignKeys.push({ key: relationship.throughTo!, tableId })
+        foreignKeys.push({ key: relationship.throughFrom!, tableId })
+      } else {
+        const fk = relationship.foreignKey!
+        const oneSide =
+          relationship.relationshipType === RelationshipTypes.ONE_TO_MANY
+        foreignKeys.push({
+          tableId: oneSide ? table._id! : relationship.tableId!,
+          key: fk,
+        })
+      }
+    })
+  }
+
+  // now make sure schemas are all accurate
+  for (let table of tables) {
+    for (let column of Object.values(table.schema)) {
+      const shouldBeForeign = foreignKeys.find(
+        options => options.tableId === table._id && options.key === column.name
+      )
+      if (shouldBeForeign) {
+        column.autocolumn = true
+        column.autoReason = AutoReason.FOREIGN_KEY
+      } else if (column.autoReason === AutoReason.FOREIGN_KEY) {
+        delete column.autocolumn
+        delete column.autoReason
+      }
+    }
+  }
+
+  return datasource
+}
+
+export function checkExternalTableSchemas(datasource: Datasource) {
+  return checkForeignKeysAreAutoColumns(datasource)
+}

packages/types/src/documents/app/table.ts

@@ -9,6 +9,10 @@ export enum RelationshipTypes {
   MANY_TO_MANY = "many-to-many",
 }
 
+export enum AutoReason {
+  FOREIGN_KEY = "foreign_key",
+}
+
 export interface FieldSchema {
   type: FieldType
   externalType?: string
@@ -21,6 +25,7 @@ export interface FieldSchema {
   foreignKey?: string
   icon?: string
   autocolumn?: boolean
+  autoReason?: AutoReason
   subtype?: string
   throughFrom?: string
   throughTo?: string