From 0cf08df80f6c4263320e97024a0f3df49221f06d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 29 Oct 2021 18:37:29 +0100 Subject: [PATCH] Adding many to many support, generating junction table and setting up constraints. --- .../DataTable/modals/CreateEditColumn.svelte | 19 +-- .../server/src/api/controllers/datasource.js | 7 +- .../src/api/controllers/table/external.js | 156 ++++++++++++++---- .../server/src/api/controllers/table/utils.js | 20 +++ packages/server/src/definitions/common.ts | 5 + .../server/src/integrations/base/sqlTable.ts | 46 ++++-- 6 files changed, 187 insertions(+), 66 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 9af07fe28c..ebfea9cee6 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -93,10 +93,6 @@ if (field.type === AUTO_TYPE) { field = buildAutoColumn($tables.draft.name, field.name, field.subtype) } - // for now you can't create other options, just many to many for SQL - if (field.type === LINK_TYPE && external) { - field.relationshipType = RelationshipTypes.ONE_TO_MANY - } await tables.saveField({ originalName, field, @@ -184,26 +180,23 @@ } const thisName = truncate(table.name, { length: 14 }), linkName = truncate(linkTable.name, { length: 14 }) - const options = [ + return [ { name: `Many ${thisName} rows → many ${linkName} rows`, alt: `Many ${table.name} rows → many ${linkTable.name} rows`, value: RelationshipTypes.MANY_TO_MANY, }, + { + name: `One ${linkName} row → many ${thisName} rows`, + alt: `One ${linkTable.name} rows → many ${table.name} rows`, + value: RelationshipTypes.ONE_TO_MANY, + }, { name: `One ${thisName} row → many ${linkName} rows`, alt: `One ${table.name} rows → many ${linkTable.name} rows`, value: RelationshipTypes.MANY_TO_ONE, }, ] - if (!external) { - options.push({ - name: `One ${linkName} row → many ${thisName} rows`, - alt: `One ${linkTable.name} rows → many ${table.name} rows`, - value: RelationshipTypes.ONE_TO_MANY, - }) - } - return options } function getAllowedTypes() { diff --git a/packages/server/src/api/controllers/datasource.js b/packages/server/src/api/controllers/datasource.js index 6f33faf3d4..643e822a36 100644 --- a/packages/server/src/api/controllers/datasource.js +++ b/packages/server/src/api/controllers/datasource.js @@ -157,9 +157,12 @@ const buildSchemaHelper = async datasource => { if (entity.primaryDisplay) { continue } - entity.primaryDisplay = Object.values(entity.schema).find( + const notAutoColumn = Object.values(entity.schema).find( schema => !schema.autocolumn - ).name + ) + if (notAutoColumn) { + entity.primaryDisplay = notAutoColumn.name + } } const errors = connector.schemaErrors diff --git a/packages/server/src/api/controllers/table/external.js b/packages/server/src/api/controllers/table/external.js index d0ec01a1d8..7d332519ba 100644 --- a/packages/server/src/api/controllers/table/external.js +++ b/packages/server/src/api/controllers/table/external.js @@ -3,7 +3,12 @@ const { buildExternalTableId, breakExternalTableId, } = require("../../../integrations/utils") -const { getTable } = require("./utils") +const { + getTable, + generateForeignKey, + generateJunctionTableName, + foreignKeyStructure, +} = require("./utils") const { DataSourceOperation, FieldTypes, @@ -58,7 +63,7 @@ function cleanupRelationships(table, tables, oldTable = null) { relatedSchema.type === FieldTypes.LINK && relatedSchema.fieldName === foreignKey ) { - delete relatedSchema[relatedKey] + delete relatedTable.schema[relatedKey] } } } @@ -75,23 +80,78 @@ function getDatasourceId(table) { return breakExternalTableId(table._id).datasourceId } -function generateRelatedSchema(linkColumn, table) { - // generate column for other table - const relatedSchema = cloneDeep(linkColumn) - relatedSchema.fieldName = linkColumn.foreignKey - relatedSchema.foreignKey = linkColumn.fieldName - relatedSchema.relationshipType = RelationshipTypes.MANY_TO_ONE - relatedSchema.tableId = table._id - delete relatedSchema.main - return relatedSchema +function otherRelationshipType(type) { + if (type === RelationshipTypes.MANY_TO_MANY) { + return RelationshipTypes.MANY_TO_MANY + } + return type === RelationshipTypes.ONE_TO_MANY + ? RelationshipTypes.MANY_TO_ONE + : RelationshipTypes.ONE_TO_MANY } -function oneToManyRelationshipNeedsSetup(column) { - return ( - column.type === FieldTypes.LINK && - column.relationshipType === RelationshipTypes.ONE_TO_MANY && - !column.foreignKey +function generateManyLinkSchema(datasource, column, table, relatedTable) { + const primary = table.name + table.primary[0] + const relatedPrimary = relatedTable.name + relatedTable.primary[0] + const jcTblName = generateJunctionTableName(column, table, relatedTable) + // first create the new table + const junctionTable = { + _id: buildExternalTableId(datasource._id, jcTblName), + name: jcTblName, + primary: [primary, relatedPrimary], + schema: { + [primary]: foreignKeyStructure(primary, { + toTable: table.name, + toKey: table.primary[0], + }), + [relatedPrimary]: foreignKeyStructure(relatedPrimary, { + toTable: relatedTable.name, + toKey: relatedTable.primary[0], + }), + }, + } + column.through = junctionTable._id + column.throughFrom = primary + column.throughTo = relatedPrimary + column.fieldName = relatedPrimary + return junctionTable +} + +function generateLinkSchema(column, table, relatedTable, type) { + const isOneSide = type === RelationshipTypes.ONE_TO_MANY + const primary = isOneSide ? relatedTable.primary[0] : table.primary[0] + // generate a foreign key + const foreignKey = generateForeignKey(column, relatedTable) + column.relationshipType = type + column.foreignKey = isOneSide ? foreignKey : primary + column.fieldName = isOneSide ? primary : foreignKey + return foreignKey +} + +function generateRelatedSchema(linkColumn, table, relatedTable, columnName) { + // generate column for other table + const relatedSchema = cloneDeep(linkColumn) + // swap them from the main link + if (linkColumn.foreignKey) { + relatedSchema.fieldName = linkColumn.foreignKey + relatedSchema.foreignKey = linkColumn.fieldName + } + // is many to many + else { + // don't need to copy through, already got it + relatedSchema.fieldName = linkColumn.throughFrom + relatedSchema.throughTo = linkColumn.throughFrom + relatedSchema.throughFrom = linkColumn.throughTo + } + relatedSchema.relationshipType = otherRelationshipType( + linkColumn.relationshipType ) + relatedSchema.tableId = relatedTable._id + relatedSchema.name = columnName + table.schema[columnName] = relatedSchema +} + +function isRelationshipSetup(column) { + return column.foreignKey || column.through } exports.save = async function (ctx) { @@ -113,32 +173,50 @@ exports.save = async function (ctx) { const db = new CouchDB(appId) const datasource = await db.get(datasourceId) + const oldTables = cloneDeep(datasource.entities) const tables = datasource.entities + const extraTablesToUpdate = [] + // check if relations need setup for (let schema of Object.values(tableToSave.schema)) { - // TODO: many to many handling - if (oneToManyRelationshipNeedsSetup(schema)) { - const relatedTable = Object.values(tables).find( - table => table._id === schema.tableId + if (schema.type !== FieldTypes.LINK || isRelationshipSetup(schema)) { + continue + } + const relatedTable = Object.values(tables).find( + table => table._id === schema.tableId + ) + const relatedColumnName = schema.fieldName + const relationType = schema.relationshipType + if (relationType === RelationshipTypes.MANY_TO_MANY) { + const junctionTable = generateManyLinkSchema( + datasource, + schema, + table, + relatedTable ) - // setup the schema in this table - const relatedField = schema.fieldName - const relatedPrimary = relatedTable.primary[0] - // generate a foreign key - const foreignKey = `fk_${relatedTable.name}_${schema.fieldName}` - - schema.relationshipType = RelationshipTypes.ONE_TO_MANY - schema.foreignKey = foreignKey - schema.fieldName = relatedPrimary - schema.main = true - - relatedTable.schema[relatedField] = generateRelatedSchema(schema, table) - tableToSave.schema[foreignKey] = { - type: FieldTypes.NUMBER, - constraints: {}, + if (tables[junctionTable.name]) { + throw "Junction table already exists, cannot create another relationship." + } + tables[junctionTable.name] = junctionTable + extraTablesToUpdate.push(junctionTable) + } else { + const fkTable = + relationType === RelationshipTypes.ONE_TO_MANY ? table : relatedTable + const foreignKey = generateLinkSchema( + schema, + table, + relatedTable, + relationType + ) + fkTable.schema[foreignKey] = foreignKeyStructure(foreignKey) + // foreign key is in other table, need to save it to external + if (fkTable._id !== table._id) { + extraTablesToUpdate.push(fkTable) } } + generateRelatedSchema(schema, relatedTable, table, relatedColumnName) + schema.main = true } cleanupRelationships(tableToSave, tables, oldTable) @@ -147,6 +225,14 @@ exports.save = async function (ctx) { ? DataSourceOperation.UPDATE_TABLE : DataSourceOperation.CREATE_TABLE await makeTableRequest(datasource, operation, tableToSave, tables, oldTable) + // update any extra tables (like foreign keys in other tables) + for (let extraTable of extraTablesToUpdate) { + const oldExtraTable = oldTables[extraTable.name] + let op = oldExtraTable + ? DataSourceOperation.UPDATE_TABLE + : DataSourceOperation.CREATE_TABLE + await makeTableRequest(datasource, op, extraTable, tables, oldExtraTable) + } // store it into couch now for budibase reference datasource.entities[tableToSave.name] = tableToSave diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index 65c081f90e..1cca4c41d2 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -315,4 +315,24 @@ exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => { } } +exports.generateForeignKey = (column, relatedTable) => { + return `fk_${relatedTable.name}_${column.fieldName}` +} + +exports.generateJunctionTableName = (column, table, relatedTable) => { + return `jt_${table.name}_${relatedTable.name}_${column.name}_${column.fieldName}` +} + +exports.foreignKeyStructure = (keyName, meta = null) => { + const structure = { + type: FieldTypes.NUMBER, + constraints: {}, + name: keyName, + } + if (meta) { + structure.meta = meta + } + return structure +} + exports.TableSaveFunctions = TableSaveFunctions diff --git a/packages/server/src/definitions/common.ts b/packages/server/src/definitions/common.ts index b2ab203bee..4308aff0a9 100644 --- a/packages/server/src/definitions/common.ts +++ b/packages/server/src/definitions/common.ts @@ -17,6 +17,11 @@ export interface FieldSchema { autocolumn?: boolean throughFrom?: string throughTo?: string + main?: boolean + meta?: { + toTable: string + toKey: string + } constraints?: { type?: string email?: boolean diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts index 87d0f1e3b3..6eb6d02c3f 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/server/src/integrations/base/sqlTable.ts @@ -8,9 +8,15 @@ const { FieldTypes, RelationshipTypes } = require("../../constants") function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record, oldTable: null | Table = null) { let primaryKey = table && table.primary ? table.primary[0] : null + const columns = Object.values(table.schema) + // all columns in a junction table will be meta + let metaCols = columns.filter(col => col.meta) + let isJunction = metaCols.length === columns.length // can't change primary once its set for now - if (primaryKey && !oldTable) { + if (primaryKey && !oldTable && !isJunction) { schema.increments(primaryKey).primary() + } else if (!oldTable && isJunction) { + schema.primary(metaCols.map(col => col.name)) } // check if any columns need added @@ -18,7 +24,7 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record for (let [key, column] of Object.entries(table.schema)) { // skip things that are already correct const oldColumn = oldTable ? oldTable.schema[key] : null - if ((oldColumn && oldColumn.type === column.type) || primaryKey === key) { + if ((oldColumn && oldColumn.type === column.type) || (primaryKey === key && !isJunction)) { continue } switch (column.type) { @@ -26,7 +32,12 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record schema.string(key) break case FieldTypes.NUMBER: - if (foreignKeys.indexOf(key) === -1) { + // if meta is specified then this is a junction table entry + if (column.meta && column.meta.toKey && column.meta.toTable) { + const { toKey, toTable } = column.meta + schema.integer(key).unsigned() + schema.foreign(key).references(`${toTable}.${toKey}`) + } else if (foreignKeys.indexOf(key) === -1) { schema.float(key) } break @@ -41,20 +52,23 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record break case FieldTypes.LINK: // this side of the relationship doesn't need any SQL work - if (column.relationshipType === RelationshipTypes.MANY_TO_ONE) { - break + if ( + column.relationshipType !== RelationshipTypes.MANY_TO_ONE && + column.relationshipType !== RelationshipTypes.MANY_TO_MANY + ) { + if (!column.foreignKey || !column.tableId) { + throw "Invalid relationship schema" + } + const { tableName } = breakExternalTableId(column.tableId) + // @ts-ignore + const relatedTable = tables[tableName] + if (!relatedTable) { + throw "Referenced table doesn't exist" + } + schema.integer(column.foreignKey).unsigned() + schema.foreign(column.foreignKey).references(`${tableName}.${relatedTable.primary[0]}`) } - if (!column.foreignKey || !column.tableId) { - throw "Invalid relationship schema" - } - const { tableName } = breakExternalTableId(column.tableId) - // @ts-ignore - const relatedTable = tables[tableName] - if (!relatedTable) { - throw "Referenced table doesn't exist" - } - schema.integer(column.foreignKey).unsigned() - schema.foreign(column.foreignKey).references(`${tableName}.${relatedTable.primary[0]}`) + break } }