budibase/packages/server/src/api/controllers/table/utils.js

const csvParser = require("../../../utilities/csvParser")
const {
  getRowParams,
  generateRowID,
  InternalTables,
  getTableParams,
  BudibaseInternalDB,
} = require("../../../db/utils")
const { isEqual } = require("lodash")
const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
const {
  inputProcessing,
  cleanupAttachments,
} = require("../../../utilities/rowProcessor")
const {
  USERS_TABLE_SCHEMA,
  SwitchableTypes,
  CanSwitchTypes,
} = require("../../../constants")
const {
  isExternalTable,
  breakExternalTableId,
  isSQL,
} = require("../../../integrations/utils")
const { getViews, saveView } = require("../view/utils")
const viewTemplate = require("../view/viewBuilder")
const { quotas, StaticQuotaName, QuotaUsageType } = require("@budibase/pro")
const { getAppDB } = require("@budibase/backend-core/context")
const { cloneDeep } = require("lodash/fp")

exports.clearColumns = async (table, columnNames) => {
  const db = getAppDB()
  const rows = await db.allDocs(
    getRowParams(table._id, null, {
      include_docs: true,
    })
  )
  return db.bulkDocs(
    rows.rows.map(({ doc }) => {
      columnNames.forEach(colName => delete doc[colName])
      return doc
    })
  )
}

exports.checkForColumnUpdates = async (oldTable, updatedTable) => {
  const db = getAppDB()
  let updatedRows = []
  const rename = updatedTable._rename
  let deletedColumns = []
  if (oldTable && oldTable.schema && updatedTable.schema) {
    deletedColumns = Object.keys(oldTable.schema).filter(
      colName => updatedTable.schema[colName] == null
    )
  }
  // check for renaming of columns or deleted columns
  if (rename || deletedColumns.length !== 0) {
    // Update all rows
    const rows = await db.allDocs(
      getRowParams(updatedTable._id, null, {
        include_docs: true,
      })
    )
    const rawRows = rows.rows.map(({ doc }) => doc)
    updatedRows = rawRows.map(row => {
      row = cloneDeep(row)
      if (rename) {
        row[rename.updated] = row[rename.old]
        delete row[rename.old]
      } else if (deletedColumns.length !== 0) {
        deletedColumns.forEach(colName => delete row[colName])
      }
      return row
    })

    // cleanup any attachments from object storage for deleted attachment columns
    await cleanupAttachments(updatedTable, { oldTable, rows: rawRows })
    // Update views
    await exports.checkForViewUpdates(updatedTable, rename, deletedColumns)
    delete updatedTable._rename
  }
  return { rows: updatedRows, table: updatedTable }
}

// makes sure the passed in table isn't going to reset the auto ID
exports.makeSureTableUpToDate = (table, tableToSave) => {
  if (!table) {
    return tableToSave
  }
  // sure sure rev is up to date
  tableToSave._rev = table._rev
  // make sure auto IDs are always updated - these are internal
  // so the client may not know they have changed
  for (let [field, column] of Object.entries(table.schema)) {
    if (
      column.autocolumn &&
      column.subtype === AutoFieldSubTypes.AUTO_ID &&
      tableToSave.schema[field]
    ) {
      tableToSave.schema[field].lastID = column.lastID
    }
  }
  return tableToSave
}

exports.handleDataImport = async (user, table, dataImport) => {
  if (!dataImport || !dataImport.csvString) {
    return table
  }

  const db = getAppDB()
  // Populate the table with rows imported from CSV in a bulk update
  const data = await csvParser.transform({
    ...dataImport,
    existingTable: table,
  })

  let finalData = []
  for (let i = 0; i < data.length; i++) {
    let row = data[i]
    row._id = generateRowID(table._id)
    row.tableId = table._id
    const processed = inputProcessing(user, table, row, {
      noAutoRelationships: true,
    })
    table = processed.table
    row = processed.row

    for (let [fieldName, schema] of Object.entries(table.schema)) {
      // check whether the options need to be updated for inclusion as part of the data import
      if (
        schema.type === FieldTypes.OPTIONS &&
        (!schema.constraints.inclusion ||
          schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
      ) {
        schema.constraints.inclusion = [
          ...schema.constraints.inclusion,
          row[fieldName],
        ]
      }
    }

    finalData.push(row)
  }

  await quotas.updateUsage(
    finalData.length,
    StaticQuotaName.ROWS,
    QuotaUsageType.STATIC,
    {
      dryRun: true,
    }
  )
  await db.bulkDocs(finalData)
  await quotas.updateUsage(
    finalData.length,
    StaticQuotaName.ROWS,
    QuotaUsageType.STATIC
  )
  let response = await db.put(table)
  table._rev = response._rev
  return table
}

exports.handleSearchIndexes = async table => {
  const db = getAppDB()
  // create relevant search indexes
  if (table.indexes && table.indexes.length > 0) {
    const currentIndexes = await db.getIndexes()
    const indexName = `search:${table._id}`

    const existingIndex = currentIndexes.indexes.find(
      existing => existing.name === indexName
    )

    if (existingIndex) {
      const currentFields = existingIndex.def.fields.map(
        field => Object.keys(field)[0]
      )

      // if index fields have changed, delete the original index
      if (!isEqual(currentFields, table.indexes)) {
        await db.deleteIndex(existingIndex)
        // create/recreate the index with fields
        await db.createIndex({
          index: {
            fields: table.indexes,
            name: indexName,
            ddoc: "search_ddoc",
            type: "json",
          },
        })
      }
    } else {
      // create/recreate the index with fields
      await db.createIndex({
        index: {
          fields: table.indexes,
          name: indexName,
          ddoc: "search_ddoc",
          type: "json",
        },
      })
    }
  }
  return table
}

exports.checkStaticTables = table => {
  // check user schema has all required elements
  if (table._id === InternalTables.USER_METADATA) {
    for (let [key, schema] of Object.entries(USERS_TABLE_SCHEMA.schema)) {
      // check if the schema exists on the table to be created/updated
      if (table.schema[key] == null) {
        table.schema[key] = schema
      }
    }
  }
  return table
}

class TableSaveFunctions {
  constructor({ user, oldTable, dataImport }) {
    this.db = getAppDB()
    this.user = user
    this.oldTable = oldTable
    this.dataImport = dataImport
    // any rows that need updated
    this.rows = []
  }

  // before anything is done
  async before(table) {
    if (this.oldTable) {
      table = exports.makeSureTableUpToDate(this.oldTable, table)
    }
    table = exports.checkStaticTables(table)
    return table
  }

  // when confirmed valid
  async mid(table) {
    let response = await exports.checkForColumnUpdates(this.oldTable, table)
    this.rows = this.rows.concat(response.rows)
    return table
  }

  // after saving
  async after(table) {
    table = await exports.handleSearchIndexes(table)
    table = await exports.handleDataImport(this.user, table, this.dataImport)
    return table
  }

  getUpdatedRows() {
    return this.rows
  }
}

exports.getAllInternalTables = async () => {
  const db = getAppDB()
  const internalTables = await db.allDocs(
    getTableParams(null, {
      include_docs: true,
    })
  )
  return internalTables.rows.map(tableDoc => ({
    ...tableDoc.doc,
    type: "internal",
    sourceId: BudibaseInternalDB._id,
  }))
}

exports.getAllExternalTables = async datasourceId => {
  const db = getAppDB()
  const datasource = await db.get(datasourceId)
  if (!datasource || !datasource.entities) {
    throw "Datasource is not configured fully."
  }
  return datasource.entities
}

exports.getExternalTable = async (datasourceId, tableName) => {
  const entities = await exports.getAllExternalTables(datasourceId)
  return entities[tableName]
}

exports.getTable = async tableId => {
  const db = getAppDB()
  if (isExternalTable(tableId)) {
    let { datasourceId, tableName } = breakExternalTableId(tableId)
    const datasource = await db.get(datasourceId)
    const table = await exports.getExternalTable(datasourceId, tableName)
    return { ...table, sql: isSQL(datasource) }
  } else {
    return db.get(tableId)
  }
}

exports.checkForViewUpdates = async (table, rename, deletedColumns) => {
  const views = await getViews()
  const tableViews = views.filter(view => view.meta.tableId === table._id)

  // Check each table view to see if impacted by this table action
  for (let view of tableViews) {
    let needsUpdated = false

    // First check for renames, otherwise check for deletions
    if (rename) {
      // Update calculation field if required
      if (view.meta.field === rename.old) {
        view.meta.field = rename.updated
        needsUpdated = true
      }

      // Update group by field if required
      if (view.meta.groupBy === rename.old) {
        view.meta.groupBy = rename.updated
        needsUpdated = true
      }

      // Update filters if required
      if (view.meta.filters) {
        view.meta.filters.forEach(filter => {
          if (filter.key === rename.old) {
            filter.key = rename.updated
            needsUpdated = true
          }
        })
      }
    } else if (deletedColumns) {
      deletedColumns.forEach(column => {
        // Remove calculation statement if required
        if (view.meta.field === column) {
          delete view.meta.field
          delete view.meta.calculation
          delete view.meta.groupBy
          needsUpdated = true
        }

        // Remove group by field if required
        if (view.meta.groupBy === column) {
          delete view.meta.groupBy
          needsUpdated = true
        }

        // Remove filters referencing deleted field if required
        if (view.meta.filters && view.meta.filters.length) {
          const initialLength = view.meta.filters.length
          view.meta.filters = view.meta.filters.filter(filter => {
            return filter.key !== column
          })
          if (initialLength !== view.meta.filters.length) {
            needsUpdated = true
          }
        }
      })
    }

    // Update view if required
    if (needsUpdated) {
      const newViewTemplate = viewTemplate(view.meta)
      await saveView(null, view.name, newViewTemplate)
      if (!newViewTemplate.meta.schema) {
        newViewTemplate.meta.schema = table.schema
      }
      table.views[view.name] = newViewTemplate.meta
    }
  }
}

exports.generateForeignKey = (column, relatedTable) => {
  return `fk_${relatedTable.name}_${column.fieldName}`
}

exports.generateJunctionTableName = (column, table, relatedTable) => {
  return `jt_${table.name}_${relatedTable.name}_${column.name}_${column.fieldName}`
}

exports.foreignKeyStructure = (keyName, meta = null) => {
  const structure = {
    type: FieldTypes.NUMBER,
    constraints: {},
    name: keyName,
  }
  if (meta) {
    structure.meta = meta
  }
  return structure
}

exports.areSwitchableTypes = (type1, type2) => {
  if (
    SwitchableTypes.indexOf(type1) === -1 &&
    SwitchableTypes.indexOf(type2) === -1
  ) {
    return false
  }
  for (let option of CanSwitchTypes) {
    const index1 = option.indexOf(type1),
      index2 = option.indexOf(type2)
    if (index1 !== -1 && index2 !== -1 && index1 !== index2) {
      return true
    }
  }
  return false
}

exports.hasTypeChanged = (table, oldTable) => {
  if (!oldTable) {
    return false
  }
  for (let [key, field] of Object.entries(oldTable.schema)) {
    const oldType = field.type
    if (!table.schema[key]) {
      continue
    }
    const newType = table.schema[key].type
    if (oldType !== newType && !exports.areSwitchableTypes(oldType, newType)) {
      return true
    }
  }
  return false
}

exports.TableSaveFunctions = TableSaveFunctions