Adding many to many support, generating junction table and setting up constraints.

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -93,10 +93,6 @@
     if (field.type === AUTO_TYPE) {
       field = buildAutoColumn($tables.draft.name, field.name, field.subtype)
     }
-    // for now you can't create other options, just many to many for SQL
-    if (field.type === LINK_TYPE && external) {
-      field.relationshipType = RelationshipTypes.ONE_TO_MANY
-    }
     await tables.saveField({
       originalName,
       field,
@@ -184,26 +180,23 @@
     }
     const thisName = truncate(table.name, { length: 14 }),
       linkName = truncate(linkTable.name, { length: 14 })
-    const options = [
+    return [
       {
         name: `Many ${thisName} rows → many ${linkName} rows`,
         alt: `Many ${table.name} rows → many ${linkTable.name} rows`,
         value: RelationshipTypes.MANY_TO_MANY,
       },
+      {
+        name: `One ${linkName} row → many ${thisName} rows`,
+        alt: `One ${linkTable.name} rows → many ${table.name} rows`,
+        value: RelationshipTypes.ONE_TO_MANY,
+      },
       {
         name: `One ${thisName} row → many ${linkName} rows`,
         alt: `One ${table.name} rows → many ${linkTable.name} rows`,
         value: RelationshipTypes.MANY_TO_ONE,
       },
     ]
-    if (!external) {
-      options.push({
-        name: `One ${linkName} row → many ${thisName} rows`,
-        alt: `One ${linkTable.name} rows → many ${table.name} rows`,
-        value: RelationshipTypes.ONE_TO_MANY,
-      })
-    }
-    return options
   }
 
   function getAllowedTypes() {

packages/server/src/api/controllers/datasource.js

@@ -157,9 +157,12 @@ const buildSchemaHelper = async datasource => {
     if (entity.primaryDisplay) {
       continue
     }
-    entity.primaryDisplay = Object.values(entity.schema).find(
+    const notAutoColumn = Object.values(entity.schema).find(
       schema => !schema.autocolumn
-    ).name
+    )
+    if (notAutoColumn) {
+      entity.primaryDisplay = notAutoColumn.name
+    }
   }
 
   const errors = connector.schemaErrors

packages/server/src/api/controllers/table/external.js

@@ -3,7 +3,12 @@ const {
   buildExternalTableId,
   breakExternalTableId,
 } = require("../../../integrations/utils")
-const { getTable } = require("./utils")
+const {
+  getTable,
+  generateForeignKey,
+  generateJunctionTableName,
+  foreignKeyStructure,
+} = require("./utils")
 const {
   DataSourceOperation,
   FieldTypes,
@@ -58,7 +63,7 @@ function cleanupRelationships(table, tables, oldTable = null) {
           relatedSchema.type === FieldTypes.LINK &&
           relatedSchema.fieldName === foreignKey
         ) {
-          delete relatedSchema[relatedKey]
+          delete relatedTable.schema[relatedKey]
         }
       }
     }
@@ -75,23 +80,78 @@ function getDatasourceId(table) {
   return breakExternalTableId(table._id).datasourceId
 }
 
-function generateRelatedSchema(linkColumn, table) {
-  // generate column for other table
-  const relatedSchema = cloneDeep(linkColumn)
-  relatedSchema.fieldName = linkColumn.foreignKey
-  relatedSchema.foreignKey = linkColumn.fieldName
-  relatedSchema.relationshipType = RelationshipTypes.MANY_TO_ONE
-  relatedSchema.tableId = table._id
-  delete relatedSchema.main
-  return relatedSchema
+function otherRelationshipType(type) {
+  if (type === RelationshipTypes.MANY_TO_MANY) {
+    return RelationshipTypes.MANY_TO_MANY
+  }
+  return type === RelationshipTypes.ONE_TO_MANY
+    ? RelationshipTypes.MANY_TO_ONE
+    : RelationshipTypes.ONE_TO_MANY
 }
 
-function oneToManyRelationshipNeedsSetup(column) {
-  return (
-    column.type === FieldTypes.LINK &&
-    column.relationshipType === RelationshipTypes.ONE_TO_MANY &&
-    !column.foreignKey
+function generateManyLinkSchema(datasource, column, table, relatedTable) {
+  const primary = table.name + table.primary[0]
+  const relatedPrimary = relatedTable.name + relatedTable.primary[0]
+  const jcTblName = generateJunctionTableName(column, table, relatedTable)
+  // first create the new table
+  const junctionTable = {
+    _id: buildExternalTableId(datasource._id, jcTblName),
+    name: jcTblName,
+    primary: [primary, relatedPrimary],
+    schema: {
+      [primary]: foreignKeyStructure(primary, {
+        toTable: table.name,
+        toKey: table.primary[0],
+      }),
+      [relatedPrimary]: foreignKeyStructure(relatedPrimary, {
+        toTable: relatedTable.name,
+        toKey: relatedTable.primary[0],
+      }),
+    },
+  }
+  column.through = junctionTable._id
+  column.throughFrom = primary
+  column.throughTo = relatedPrimary
+  column.fieldName = relatedPrimary
+  return junctionTable
+}
+
+function generateLinkSchema(column, table, relatedTable, type) {
+  const isOneSide = type === RelationshipTypes.ONE_TO_MANY
+  const primary = isOneSide ? relatedTable.primary[0] : table.primary[0]
+  // generate a foreign key
+  const foreignKey = generateForeignKey(column, relatedTable)
+  column.relationshipType = type
+  column.foreignKey = isOneSide ? foreignKey : primary
+  column.fieldName = isOneSide ? primary : foreignKey
+  return foreignKey
+}
+
+function generateRelatedSchema(linkColumn, table, relatedTable, columnName) {
+  // generate column for other table
+  const relatedSchema = cloneDeep(linkColumn)
+  // swap them from the main link
+  if (linkColumn.foreignKey) {
+    relatedSchema.fieldName = linkColumn.foreignKey
+    relatedSchema.foreignKey = linkColumn.fieldName
+  }
+  // is many to many
+  else {
+    // don't need to copy through, already got it
+    relatedSchema.fieldName = linkColumn.throughFrom
+    relatedSchema.throughTo = linkColumn.throughFrom
+    relatedSchema.throughFrom = linkColumn.throughTo
+  }
+  relatedSchema.relationshipType = otherRelationshipType(
+    linkColumn.relationshipType
   )
+  relatedSchema.tableId = relatedTable._id
+  relatedSchema.name = columnName
+  table.schema[columnName] = relatedSchema
+}
+
+function isRelationshipSetup(column) {
+  return column.foreignKey || column.through
 }
 
 exports.save = async function (ctx) {
@@ -113,32 +173,50 @@ exports.save = async function (ctx) {
 
   const db = new CouchDB(appId)
   const datasource = await db.get(datasourceId)
+  const oldTables = cloneDeep(datasource.entities)
   const tables = datasource.entities
 
+  const extraTablesToUpdate = []
+
   // check if relations need setup
   for (let schema of Object.values(tableToSave.schema)) {
-    // TODO: many to many handling
-    if (oneToManyRelationshipNeedsSetup(schema)) {
-      const relatedTable = Object.values(tables).find(
-        table => table._id === schema.tableId
+    if (schema.type !== FieldTypes.LINK || isRelationshipSetup(schema)) {
+      continue
+    }
+    const relatedTable = Object.values(tables).find(
+      table => table._id === schema.tableId
+    )
+    const relatedColumnName = schema.fieldName
+    const relationType = schema.relationshipType
+    if (relationType === RelationshipTypes.MANY_TO_MANY) {
+      const junctionTable = generateManyLinkSchema(
+        datasource,
+        schema,
+        table,
+        relatedTable
       )
-      // setup the schema in this table
-      const relatedField = schema.fieldName
-      const relatedPrimary = relatedTable.primary[0]
-      // generate a foreign key
-      const foreignKey = `fk_${relatedTable.name}_${schema.fieldName}`
-
-      schema.relationshipType = RelationshipTypes.ONE_TO_MANY
-      schema.foreignKey = foreignKey
-      schema.fieldName = relatedPrimary
-      schema.main = true
-
-      relatedTable.schema[relatedField] = generateRelatedSchema(schema, table)
-      tableToSave.schema[foreignKey] = {
-        type: FieldTypes.NUMBER,
-        constraints: {},
+      if (tables[junctionTable.name]) {
+        throw "Junction table already exists, cannot create another relationship."
+      }
+      tables[junctionTable.name] = junctionTable
+      extraTablesToUpdate.push(junctionTable)
+    } else {
+      const fkTable =
+        relationType === RelationshipTypes.ONE_TO_MANY ? table : relatedTable
+      const foreignKey = generateLinkSchema(
+        schema,
+        table,
+        relatedTable,
+        relationType
+      )
+      fkTable.schema[foreignKey] = foreignKeyStructure(foreignKey)
+      // foreign key is in other table, need to save it to external
+      if (fkTable._id !== table._id) {
+        extraTablesToUpdate.push(fkTable)
       }
     }
+    generateRelatedSchema(schema, relatedTable, table, relatedColumnName)
+    schema.main = true
   }
 
   cleanupRelationships(tableToSave, tables, oldTable)
@@ -147,6 +225,14 @@ exports.save = async function (ctx) {
     ? DataSourceOperation.UPDATE_TABLE
     : DataSourceOperation.CREATE_TABLE
   await makeTableRequest(datasource, operation, tableToSave, tables, oldTable)
+  // update any extra tables (like foreign keys in other tables)
+  for (let extraTable of extraTablesToUpdate) {
+    const oldExtraTable = oldTables[extraTable.name]
+    let op = oldExtraTable
+      ? DataSourceOperation.UPDATE_TABLE
+      : DataSourceOperation.CREATE_TABLE
+    await makeTableRequest(datasource, op, extraTable, tables, oldExtraTable)
+  }
 
   // store it into couch now for budibase reference
   datasource.entities[tableToSave.name] = tableToSave

packages/server/src/api/controllers/table/utils.js

@@ -315,4 +315,24 @@ exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
   }
 }
 
+exports.generateForeignKey = (column, relatedTable) => {
+  return `fk_${relatedTable.name}_${column.fieldName}`
+}
+
+exports.generateJunctionTableName = (column, table, relatedTable) => {
+  return `jt_${table.name}_${relatedTable.name}_${column.name}_${column.fieldName}`
+}
+
+exports.foreignKeyStructure = (keyName, meta = null) => {
+  const structure = {
+    type: FieldTypes.NUMBER,
+    constraints: {},
+    name: keyName,
+  }
+  if (meta) {
+    structure.meta = meta
+  }
+  return structure
+}
+
 exports.TableSaveFunctions = TableSaveFunctions

packages/server/src/definitions/common.ts

@@ -17,6 +17,11 @@ export interface FieldSchema {
   autocolumn?: boolean
   throughFrom?: string
   throughTo?: string
+  main?: boolean
+  meta?: {
+    toTable: string
+    toKey: string
+  }
   constraints?: {
     type?: string
     email?: boolean

packages/server/src/integrations/base/sqlTable.ts

@@ -8,9 +8,15 @@ const { FieldTypes, RelationshipTypes } = require("../../constants")
 
 function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record<string, Table>, oldTable: null | Table = null) {
   let primaryKey = table && table.primary ? table.primary[0] : null
+  const columns = Object.values(table.schema)
+  // all columns in a junction table will be meta
+  let metaCols = columns.filter(col => col.meta)
+  let isJunction = metaCols.length === columns.length
   // can't change primary once its set for now
-  if (primaryKey && !oldTable) {
+  if (primaryKey && !oldTable && !isJunction) {
     schema.increments(primaryKey).primary()
+  } else if (!oldTable && isJunction) {
+    schema.primary(metaCols.map(col => col.name))
   }
 
   // check if any columns need added
@@ -18,7 +24,7 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record
   for (let [key, column] of Object.entries(table.schema)) {
     // skip things that are already correct
     const oldColumn = oldTable ? oldTable.schema[key] : null
-    if ((oldColumn && oldColumn.type === column.type) || primaryKey === key) {
+    if ((oldColumn && oldColumn.type === column.type) || (primaryKey === key && !isJunction)) {
       continue
     }
     switch (column.type) {
@@ -26,7 +32,12 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record
         schema.string(key)
         break
       case FieldTypes.NUMBER:
-        if (foreignKeys.indexOf(key) === -1) {
+        // if meta is specified then this is a junction table entry
+        if (column.meta && column.meta.toKey && column.meta.toTable) {
+          const { toKey, toTable } = column.meta
+          schema.integer(key).unsigned()
+          schema.foreign(key).references(`${toTable}.${toKey}`)
+        } else if (foreignKeys.indexOf(key) === -1) {
           schema.float(key)
         }
         break
@@ -41,20 +52,23 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record
         break
       case FieldTypes.LINK:
         // this side of the relationship doesn't need any SQL work
-        if (column.relationshipType === RelationshipTypes.MANY_TO_ONE) {
-          break
+        if (
+          column.relationshipType !== RelationshipTypes.MANY_TO_ONE &&
+          column.relationshipType !== RelationshipTypes.MANY_TO_MANY
+        ) {
+          if (!column.foreignKey || !column.tableId) {
+            throw "Invalid relationship schema"
+          }
+          const { tableName } = breakExternalTableId(column.tableId)
+          // @ts-ignore
+          const relatedTable = tables[tableName]
+          if (!relatedTable) {
+            throw "Referenced table doesn't exist"
+          }
+          schema.integer(column.foreignKey).unsigned()
+          schema.foreign(column.foreignKey).references(`${tableName}.${relatedTable.primary[0]}`)
         }
-        if (!column.foreignKey || !column.tableId) {
-          throw "Invalid relationship schema"
-        }
-        const { tableName } = breakExternalTableId(column.tableId)
-        // @ts-ignore
-        const relatedTable = tables[tableName]
-        if (!relatedTable) {
-          throw "Referenced table doesn't exist"
-        }
-        schema.integer(column.foreignKey).unsigned()
-        schema.foreign(column.foreignKey).references(`${tableName}.${relatedTable.primary[0]}`)
+        break
     }
   }