Merge branch 'feature/sql-relationships' of github.com:Budibase/budibase into feature/opinionated-relationships-ui

packages/server/src/api/controllers/row/external.js

@@ -1,6 +1,11 @@
 const { makeExternalQuery } = require("./utils")
-const { DataSourceOperation, SortDirection } = require("../../../constants")
-const { getExternalTable } = require("../table/utils")
+const {
+  DataSourceOperation,
+  SortDirection,
+  FieldTypes,
+  RelationshipTypes,
+} = require("../../../constants")
+const { getAllExternalTables } = require("../table/utils")
 const {
   breakExternalTableId,
   generateRowIdField,
@@ -35,17 +40,66 @@ function generateIdForRow(row, table) {
   return generateRowIdField(idParts)
 }
 
-function outputProcessing(rows, table) {
+function updateRelationshipColumns(rows, row, relationships, allTables) {
+  const columns = {}
+  for (let relationship of relationships) {
+    const linkedTable = allTables[relationship.tableName]
+    if (!linkedTable) {
+      continue
+    }
+    const display = linkedTable.primaryDisplay
+    const related = {}
+    if (display && row[display]) {
+      related.primaryDisplay = row[display]
+    }
+    related._id = row[relationship.to]
+    columns[relationship.from] = related
+  }
+  for (let [column, related] of Object.entries(columns)) {
+    if (!Array.isArray(rows[row._id][column])) {
+      rows[row._id][column] = []
+    }
+    rows[row._id][column].push(related)
+  }
+  return rows
+}
+
+function outputProcessing(rows, table, relationships, allTables) {
   // if no rows this is what is returned? Might be PG only
   if (rows[0].read === true) {
     return []
   }
+  let finalRows = {}
   for (let row of rows) {
     row._id = generateIdForRow(row, table)
-    row.tableId = table._id
-    row._rev = "rev"
+    // this is a relationship of some sort
+    if (finalRows[row._id]) {
+      finalRows = updateRelationshipColumns(
+        finalRows,
+        row,
+        relationships,
+        allTables
+      )
+      continue
     }
-  return rows
+    const thisRow = {}
+    // filter the row down to what is actually the row (not joined)
+    for (let fieldName of Object.keys(table.schema)) {
+      thisRow[fieldName] = row[fieldName]
+    }
+    thisRow._id = row._id
+    thisRow.tableId = table._id
+    thisRow._rev = "rev"
+    finalRows[thisRow._id] = thisRow
+    // do this at end once its been added to the final rows
+    finalRows = updateRelationshipColumns(
+      finalRows,
+      row,
+      relationships,
+      allTables
+    )
+  }
+  return Object.values(finalRows)
 }
 
 function buildFilters(id, filters, table) {
@@ -83,6 +137,26 @@ function buildFilters(id, filters, table) {
   }
 }
 
+function buildRelationships(table) {
+  const relationships = []
+  for (let [fieldName, field] of Object.entries(table.schema)) {
+    if (field.type !== FieldTypes.LINK) {
+      continue
+    }
+    // TODO: through field
+    if (field.relationshipType === RelationshipTypes.MANY_TO_MANY) {
+      continue
+    }
+    const broken = breakExternalTableId(field.tableId)
+    relationships.push({
+      from: fieldName,
+      to: field.fieldName,
+      tableName: broken.tableName,
+    })
+  }
+  return relationships
+}
+
 async function handleRequest(
   appId,
   operation,
@@ -90,12 +164,14 @@ async function handleRequest(
   { id, row, filters, sort, paginate } = {}
 ) {
   let { datasourceId, tableName } = breakExternalTableId(tableId)
-  const table = await getExternalTable(appId, datasourceId, tableName)
+  const tables = await getAllExternalTables(appId, datasourceId)
+  const table = tables[tableName]
   if (!table) {
     throw `Unable to process query, table "${tableName}" not defined.`
   }
   // clean up row on ingress using schema
   filters = buildFilters(id, filters, table)
+  const relationships = buildRelationships(table)
   row = inputProcessing(row, table)
   if (
     operation === DataSourceOperation.DELETE &&
@@ -116,6 +192,7 @@ async function handleRequest(
     filters,
     sort,
     paginate,
+    relationships,
     body: row,
     // pass an id filter into extra, purely for mysql/returning
     extra: {
@@ -126,9 +203,9 @@ async function handleRequest(
   const response = await makeExternalQuery(appId, json)
   // we searched for rows in someway
   if (operation === DataSourceOperation.READ && Array.isArray(response)) {
-    return outputProcessing(response, table)
+    return outputProcessing(response, table, relationships, tables)
   } else {
-    row = outputProcessing(response, table)[0]
+    row = outputProcessing(response, table, relationships, tables)[0]
     return { row, table }
   }
 }
@@ -270,7 +347,4 @@ exports.validate = async () => {
   return { valid: true }
 }
 
-exports.fetchEnrichedRow = async () => {
-  // TODO: How does this work
-  throw "Not Implemented"
-}
+exports.fetchEnrichedRow = async () => {}

packages/server/src/api/controllers/table/utils.js

@@ -204,15 +204,18 @@ class TableSaveFunctions {
   }
 }
 
-exports.getExternalTable = async (appId, datasourceId, tableName) => {
+exports.getAllExternalTables = async (appId, datasourceId) => {
   const db = new CouchDB(appId)
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
   }
-  return Object.values(datasource.entities).find(
-    entity => entity.name === tableName
-  )
+  return datasource.entities
+}
+
+exports.getExternalTable = async (appId, datasourceId, tableName) => {
+  const entities = await exports.getAllExternalTables(appId, datasourceId)
+  return entities[tableName]
 }
 
 exports.TableSaveFunctions = TableSaveFunctions

packages/server/src/integrations/base/definitions.ts

@@ -74,6 +74,17 @@ export interface SearchFilters {
   }
 }
 
+export interface RelationshipsJson {
+  through?: {
+    from: string
+    to: string
+    tableName: string
+  }
+  from: string
+  to: string
+  tableName: string
+}
+
 export interface QueryJson {
   endpoint: {
     datasourceId: string
@@ -92,9 +103,10 @@ export interface QueryJson {
     page: string | number
   }
   body?: object
-  extra: {
+  extra?: {
     idFilter?: SearchFilters
   }
+  relationships?: RelationshipsJson[]
 }
 
 export interface SqlQuery {

packages/server/src/integrations/base/sql.ts

@@ -6,12 +6,15 @@ import {
   QueryOptions,
   SortDirection,
   Operation,
+  RelationshipsJson,
 } from "./definitions"
 
+type KnexQuery = Knex.QueryBuilder | Knex
+
 function addFilters(
-  query: any,
+  query: KnexQuery,
   filters: SearchFilters | undefined
-): Knex.QueryBuilder {
+): KnexQuery {
   function iterate(
     structure: { [key: string]: any },
     fn: (key: string, value: any) => void
@@ -67,9 +70,38 @@ function addFilters(
   return query
 }
 
-function buildCreate(knex: Knex, json: QueryJson, opts: QueryOptions) {
+function addRelationships(
+  query: KnexQuery,
+  fromTable: string,
+  relationships: RelationshipsJson[] | undefined
+): KnexQuery {
+  if (!relationships) {
+    return query
+  }
+  for (let relationship of relationships) {
+    const from = `${fromTable}.${relationship.from}`
+    const to = `${relationship.tableName}.${relationship.to}`
+    if (!relationship.through) {
+      // @ts-ignore
+      query = query.innerJoin(relationship.tableName, from, to)
+    } else {
+      const through = relationship
+      query = query
+        // @ts-ignore
+        .innerJoin(through.tableName, from, through.from)
+        .innerJoin(relationship.tableName, to, through.to)
+    }
+  }
+  return query
+}
+
+function buildCreate(
+  knex: Knex,
+  json: QueryJson,
+  opts: QueryOptions
+): KnexQuery {
   const { endpoint, body } = json
-  let query = knex(endpoint.entityId)
+  let query: KnexQuery = knex(endpoint.entityId)
   // mysql can't use returning
   if (opts.disableReturning) {
     return query.insert(body)
@@ -78,9 +110,10 @@ function buildCreate(knex: Knex, json: QueryJson, opts: QueryOptions) {
   }
 }
 
-function buildRead(knex: Knex, json: QueryJson, limit: number) {
-  let { endpoint, resource, filters, sort, paginate } = json
-  let query: Knex.QueryBuilder = knex(endpoint.entityId)
+function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery {
+  let { endpoint, resource, filters, sort, paginate, relationships } = json
+  const tableName = endpoint.entityId
+  let query: KnexQuery = knex(tableName)
   // select all if not specified
   if (!resource) {
     resource = { fields: [] }
@@ -93,6 +126,8 @@ function buildRead(knex: Knex, json: QueryJson, limit: number) {
   }
   // handle where
   query = addFilters(query, filters)
+  // handle join
+  query = addRelationships(query, tableName, relationships)
   // handle sorting
   if (sort) {
     for (let [key, value] of Object.entries(sort)) {
@@ -114,9 +149,13 @@ function buildRead(knex: Knex, json: QueryJson, limit: number) {
   return query
 }
 
-function buildUpdate(knex: Knex, json: QueryJson, opts: QueryOptions) {
+function buildUpdate(
+  knex: Knex,
+  json: QueryJson,
+  opts: QueryOptions
+): KnexQuery {
   const { endpoint, body, filters } = json
-  let query = knex(endpoint.entityId)
+  let query: KnexQuery = knex(endpoint.entityId)
   query = addFilters(query, filters)
   // mysql can't use returning
   if (opts.disableReturning) {
@@ -126,9 +165,13 @@ function buildUpdate(knex: Knex, json: QueryJson, opts: QueryOptions) {
   }
 }
 
-function buildDelete(knex: Knex, json: QueryJson, opts: QueryOptions) {
+function buildDelete(
+  knex: Knex,
+  json: QueryJson,
+  opts: QueryOptions
+): KnexQuery {
   const { endpoint, filters } = json
-  let query = knex(endpoint.entityId)
+  let query: KnexQuery = knex(endpoint.entityId)
   query = addFilters(query, filters)
   // mysql can't use returning
   if (opts.disableReturning) {
@@ -180,6 +223,8 @@ class SqlQueryBuilder {
       default:
         throw `Operation type is not supported by SQL query builder`
     }
+
+    // @ts-ignore
     return query.toSQL().toNative()
   }
 }

packages/server/src/integrations/postgres.ts

@@ -174,6 +174,20 @@ module PostgresModule {
           name: columnName,
           type,
         }
+
+        // // TODO: hack for testing
+        // if (tableName === "persons") {
+        //   tables[tableName].primaryDisplay = "firstname"
+        // }
+        // if (columnName.toLowerCase() === "personid" && tableName === "tasks") {
+        //   tables[tableName].schema[columnName] = {
+        //     name: columnName,
+        //     type: "link",
+        //     tableId: buildExternalTableId(datasourceId, "persons"),
+        //     relationshipType: "one-to-many",
+        //     fieldName: "personid",
+        //   }
+        // }
       }
       this.tables = tables
     }