budibase/packages/server/src/api/controllers/row/ExternalRequest.ts

import {
  Operation,
  SearchFilters,
  SortJson,
  PaginationJson,
  RelationshipsJson,
} from "../../../definitions/datasource"
import { Row, Table, FieldSchema } from "../../../definitions/common"
import {
  breakRowIdField,
  generateRowIdField,
} from "../../../integrations/utils"

interface ManyRelationship {
  tableId?: string
  id?: string
  isUpdate?: boolean
  [key: string]: any
}

interface RunConfig {
  id: string
  row: Row
  filters: SearchFilters
  sort: SortJson
  paginate: PaginationJson
}

module External {
  const { makeExternalQuery } = require("./utils")
  const { DataSourceOperation, FieldTypes } = require("../../../constants")
  const { getAllExternalTables } = require("../table/utils")
  const { breakExternalTableId } = require("../../../integrations/utils")
  const { processObjectSync } = require("@budibase/string-templates")
  const { cloneDeep } = require("lodash/fp")

  function buildFilters(
    id: string | undefined,
    filters: SearchFilters,
    table: Table
  ) {
    const primary = table.primary
    // if passed in array need to copy for shifting etc
    let idCopy = cloneDeep(id)
    if (filters) {
      // need to map over the filters and make sure the _id field isn't present
      for (let filter of Object.values(filters)) {
        if (filter._id && primary) {
          const parts = breakRowIdField(filter._id)
          for (let field of primary) {
            filter[field] = parts.shift()
          }
        }
        // make sure this field doesn't exist on any filter
        delete filter._id
      }
    }
    // there is no id, just use the user provided filters
    if (!idCopy || !table) {
      return filters
    }
    // if used as URL parameter it will have been joined
    if (!Array.isArray(idCopy)) {
      idCopy = breakRowIdField(idCopy)
    }
    const equal: any = {}
    if (primary && idCopy) {
      for (let field of primary) {
        // work through the ID and get the parts
        equal[field] = idCopy.shift()
      }
    }
    return {
      equal,
    }
  }

  function generateIdForRow(row: Row, table: Table): string {
    const primary = table.primary
    if (!row || !primary) {
      return ""
    }
    // build id array
    let idParts = []
    for (let field of primary) {
      if (row[field]) {
        idParts.push(row[field])
      }
    }
    if (idParts.length === 0) {
      return ""
    }
    return generateRowIdField(idParts)
  }

  function basicProcessing(row: Row, table: Table) {
    const thisRow: { [key: string]: any } = {}
    // filter the row down to what is actually the row (not joined)
    for (let fieldName of Object.keys(table.schema)) {
      thisRow[fieldName] = row[fieldName]
    }
    thisRow._id = generateIdForRow(row, table)
    thisRow.tableId = table._id
    thisRow._rev = "rev"
    return thisRow
  }

  function isMany(field: FieldSchema) {
    return (
      field.relationshipType && field.relationshipType.split("-")[0] === "many"
    )
  }

  class ExternalRequest {
    private appId: string
    private operation: Operation
    private tableId: string
    private tables: { [key: string]: Table }

    constructor(
      appId: string,
      operation: Operation,
      tableId: string,
      tables: { [key: string]: Table }
    ) {
      this.appId = appId
      this.operation = operation
      this.tableId = tableId
      this.tables = tables
    }

    inputProcessing(row: Row, table: Table) {
      if (!row) {
        return { row, manyRelationships: [] }
      }
      // we don't really support composite keys for relationships, this is why [0] is used
      // @ts-ignore
      const tablePrimary: string = table.primary[0]
      let newRow: Row = {},
        manyRelationships: ManyRelationship[] = []
      for (let [key, field] of Object.entries(table.schema)) {
        // if set already, or not set just skip it
        if (!row[key] || newRow[key]) {
          continue
        }
        // if its not a link then just copy it over
        if (field.type !== FieldTypes.LINK) {
          newRow[key] = row[key]
          continue
        }
        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
        // table has to exist for many to many
        if (!this.tables[linkTableName]) {
          continue
        }
        const linkTable = this.tables[linkTableName]
        // @ts-ignore
        const linkTablePrimary = linkTable.primary[0]
        if (!isMany(field)) {
          newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(
            row[key][0]
          )[0]
        } else {
          // we're not inserting a doc, will be a bunch of update calls
          const isUpdate = !field.through
          const thisKey: string = isUpdate ? "id" : linkTablePrimary
          // @ts-ignore
          const otherKey: string = isUpdate ? field.foreignKey : tablePrimary
          row[key].map((relationship: any) => {
            // we don't really support composite keys for relationships, this is why [0] is used
            manyRelationships.push({
              tableId: field.through || field.tableId,
              isUpdate,
              [thisKey]: breakRowIdField(relationship)[0],
              // leave the ID for enrichment later
              [otherKey]: `{{ ${tablePrimary} }}`,
            })
          })
        }
      }
      // we return the relationships that may need to be created in the through table
      // we do this so that if the ID is generated by the DB it can be inserted
      // after the fact
      return { row: newRow, manyRelationships }
    }

    updateRelationshipColumns(
      row: Row,
      rows: { [key: string]: Row },
      relationships: RelationshipsJson[]
    ) {
      const columns: { [key: string]: any } = {}
      for (let relationship of relationships) {
        const linkedTable = this.tables[relationship.tableName]
        if (!linkedTable) {
          continue
        }
        let linked = basicProcessing(row, linkedTable)
        if (!linked._id) {
          continue
        }
        // if not returning full docs then get the minimal links out
        const display = linkedTable.primaryDisplay
        linked = {
          primaryDisplay: display ? linked[display] : undefined,
          _id: linked._id,
        }
        columns[relationship.column] = linked
      }
      for (let [column, related] of Object.entries(columns)) {
        if (!row._id) {
          continue
        }
        const rowId: string = row._id
        if (!Array.isArray(rows[rowId][column])) {
          rows[rowId][column] = []
        }
        // make sure relationship hasn't been found already
        if (
          !rows[rowId][column].find(
            (relation: Row) => relation._id === related._id
          )
        ) {
          rows[rowId][column].push(related)
        }
      }
      return rows
    }

    outputProcessing(
      rows: Row[],
      table: Table,
      relationships: RelationshipsJson[]
    ) {
      if (rows[0].read === true) {
        return []
      }
      let finalRows: { [key: string]: Row } = {}
      for (let row of rows) {
        const rowId = generateIdForRow(row, table)
        row._id = rowId
        // this is a relationship of some sort
        if (finalRows[rowId]) {
          finalRows = this.updateRelationshipColumns(
            row,
            finalRows,
            relationships
          )
          continue
        }
        const thisRow = basicProcessing(row, table)
        finalRows[thisRow._id] = thisRow
        // do this at end once its been added to the final rows
        finalRows = this.updateRelationshipColumns(
          row,
          finalRows,
          relationships
        )
      }
      return Object.values(finalRows)
    }

    buildRelationships(table: Table): RelationshipsJson[] {
      const relationships = []
      for (let [fieldName, field] of Object.entries(table.schema)) {
        if (field.type !== FieldTypes.LINK) {
          continue
        }
        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
        // no table to link to, this is not a valid relationships
        if (!this.tables[linkTableName]) {
          continue
        }
        const linkTable = this.tables[linkTableName]
        if (!table.primary || !linkTable.primary) {
          continue
        }
        const definition = {
          // if no foreign key specified then use the name of the field in other table
          from: field.foreignKey || table.primary[0],
          to: field.fieldName,
          tableName: linkTableName,
          through: undefined,
          // need to specify where to put this back into
          column: fieldName,
        }
        if (field.through) {
          const { tableName: throughTableName } = breakExternalTableId(
            field.through
          )
          definition.through = throughTableName
          // don't support composite keys for relationships
          definition.from = table.primary[0]
          definition.to = linkTable.primary[0]
        }
        relationships.push(definition)
      }
      return relationships
    }

    async handleManyRelationships(row: Row, relationships: ManyRelationship[]) {
      const { appId, tables } = this
      const promises = []
      for (let relationship of relationships) {
        const { tableId, isUpdate, id, ...rest } = relationship
        const { datasourceId, tableName } = breakExternalTableId(tableId)
        const linkedTable = tables[tableName]
        if (!linkedTable) {
          continue
        }
        const endpoint = {
          datasourceId,
          entityId: tableName,
          operation: isUpdate
            ? DataSourceOperation.UPDATE
            : DataSourceOperation.CREATE,
        }
        promises.push(
          makeExternalQuery(appId, {
            endpoint,
            // if we're doing many relationships then we're writing, only one response
            body: processObjectSync(rest, row),
            filters: buildFilters(id, {}, linkedTable),
          })
        )
      }
      await Promise.all(promises)
    }

    /**
     * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
     * you have column overlap in relationships, e.g. we join a few different tables and they all have the
     * concept of an ID, but for some of them it will be null (if they say don't have a relationship).
     * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
     * is more performant and has the added benefit of protecting against this scenario.
     */
    buildFields(table: Table) {
      function extractNonLinkFieldNames(table: Table, existing: string[] = []) {
        return Object.entries(table.schema)
          .filter(
            column =>
              column[1].type !== FieldTypes.LINK &&
              !existing.find((field: string) => field.includes(column[0]))
          )
          .map(column => `${table.name}.${column[0]}`)
      }
      let fields = extractNonLinkFieldNames(table)
      for (let field of Object.values(table.schema)) {
        if (field.type !== FieldTypes.LINK) {
          continue
        }
        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
        const linkTable = this.tables[linkTableName]
        if (linkTable) {
          const linkedFields = extractNonLinkFieldNames(linkTable, fields)
          fields = fields.concat(linkedFields)
        }
      }
      return fields
    }

    async run({ id, row, filters, sort, paginate }: RunConfig) {
      const { appId, operation, tableId } = this
      let { datasourceId, tableName } = breakExternalTableId(tableId)
      if (!this.tables) {
        this.tables = await getAllExternalTables(appId, datasourceId)
      }
      const table = this.tables[tableName]
      if (!table) {
        throw `Unable to process query, table "${tableName}" not defined.`
      }
      // clean up row on ingress using schema
      filters = buildFilters(id, filters, table)
      const relationships = this.buildRelationships(table)
      const processed = this.inputProcessing(row, table)
      row = processed.row
      if (
        operation === DataSourceOperation.DELETE &&
        (filters == null || Object.keys(filters).length === 0)
      ) {
        throw "Deletion must be filtered"
      }
      let json = {
        endpoint: {
          datasourceId,
          entityId: tableName,
          operation,
        },
        resource: {
          // have to specify the fields to avoid column overlap
          fields: this.buildFields(table),
        },
        filters,
        sort,
        paginate,
        relationships,
        body: row,
        // pass an id filter into extra, purely for mysql/returning
        extra: {
          idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
        },
      }
      // can't really use response right now
      const response = await makeExternalQuery(appId, json)
      // handle many to many relationships now if we know the ID (could be auto increment)
      if (processed.manyRelationships) {
        await this.handleManyRelationships(
          response[0],
          processed.manyRelationships
        )
      }
      const output = this.outputProcessing(response, table, relationships)
      // if reading it'll just be an array of rows, return whole thing
      return operation === DataSourceOperation.READ && Array.isArray(response)
        ? output
        : { row: output[0], table }
    }
  }

  module.exports = ExternalRequest
}