Adding mechanism for verifying the Budibase properties, such as required and min/max as part of the external API.

packages/server/src/api/controllers/row/external.ts

@@ -12,7 +12,7 @@ import * as exporters from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
   Operation,
-  BBContext,
+  UserCtx,
   Row,
   PaginationJson,
   Table,
@@ -21,6 +21,7 @@ import {
   SortJson,
 } from "@budibase/types"
 import sdk from "../../../sdk"
+import * as utils from "./utils"
 
 const { cleanExportRows } = require("./utils")
 
@@ -49,12 +50,19 @@ export async function handleRequest(
   )
 }
 
-export async function patch(ctx: BBContext) {
+export async function patch(ctx: UserCtx) {
   const inputs = ctx.request.body
   const tableId = ctx.params.tableId
   const id = inputs._id
   // don't save the ID to db
   delete inputs._id
+  const validateResult = await utils.validate({
+    row: inputs,
+    tableId,
+  })
+  if (!validateResult.valid) {
+    throw { validation: validateResult.errors }
+  }
   return handleRequest(Operation.UPDATE, tableId, {
     id: breakRowIdField(id),
     row: inputs,
@@ -62,16 +70,23 @@ export async function patch(ctx: BBContext) {
   })
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: UserCtx) {
   const inputs = ctx.request.body
   const tableId = ctx.params.tableId
+  const validateResult = await utils.validate({
+    row: inputs,
+    tableId,
+  })
+  if (!validateResult.valid) {
+    throw { validation: validateResult.errors }
+  }
   return handleRequest(Operation.CREATE, tableId, {
     row: inputs,
     includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })
 }
 
-export async function fetchView(ctx: BBContext) {
+export async function fetchView(ctx: UserCtx) {
   // there are no views in external datasources, shouldn't ever be called
   // for now just fetch
   const split = ctx.params.viewName.split("all_")
@@ -79,14 +94,14 @@ export async function fetchView(ctx: BBContext) {
   return fetch(ctx)
 }
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   const tableId = ctx.params.tableId
   return handleRequest(Operation.READ, tableId, {
     includeSqlRelationships: IncludeRelationship.INCLUDE,
   })
 }
 
-export async function find(ctx: BBContext) {
+export async function find(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
   const response = (await handleRequest(Operation.READ, tableId, {
@@ -96,7 +111,7 @@ export async function find(ctx: BBContext) {
   return response ? response[0] : response
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const tableId = ctx.params.tableId
   const id = ctx.request.body._id
   const { row } = (await handleRequest(Operation.DELETE, tableId, {
@@ -106,7 +121,7 @@ export async function destroy(ctx: BBContext) {
   return { response: { ok: true }, row }
 }
 
-export async function bulkDestroy(ctx: BBContext) {
+export async function bulkDestroy(ctx: UserCtx) {
   const { rows } = ctx.request.body
   const tableId = ctx.params.tableId
   let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
@@ -122,7 +137,7 @@ export async function bulkDestroy(ctx: BBContext) {
   return { response: { ok: true }, rows: responses.map(resp => resp.row) }
 }
 
-export async function search(ctx: BBContext) {
+export async function search(ctx: UserCtx) {
   const tableId = ctx.params.tableId
   const { paginate, query, ...params } = ctx.request.body
   let { bookmark, limit } = params
@@ -185,12 +200,7 @@ export async function search(ctx: BBContext) {
   }
 }
 
-export async function validate(ctx: BBContext) {
+export async function exportRows(ctx: UserCtx) {
-  // can't validate external right now - maybe in future
-  return { valid: true }
-}
-
-export async function exportRows(ctx: BBContext) {
   const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
   const format = ctx.query.format
   const { columns } = ctx.request.body
@@ -244,7 +254,7 @@ export async function exportRows(ctx: BBContext) {
   return apiFileReturn(exporter(headers, exportRows))
 }
 
-export async function fetchEnrichedRow(ctx: BBContext) {
+export async function fetchEnrichedRow(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
   const { datasourceId, tableName } = breakExternalTableId(tableId)

packages/server/src/api/controllers/row/index.ts

@@ -2,6 +2,8 @@ import { quotas } from "@budibase/pro"
 import * as internal from "./internal"
 import * as external from "./external"
 import { isExternalTable } from "../../../integrations/utils"
+import { Ctx } from "@budibase/types"
+import * as utils from "./utils"
 
 function pickApi(tableId: any) {
   if (isExternalTable(tableId)) {
@@ -129,9 +131,12 @@ export async function search(ctx: any) {
   })
 }
 
-export async function validate(ctx: any) {
+export async function validate(ctx: Ctx) {
   const tableId = getTableId(ctx)
-  ctx.body = await pickApi(tableId).validate(ctx)
+  ctx.body = await utils.validate({
+    row: ctx.request.body,
+    tableId,
+  })
 }
 
 export async function fetchEnrichedRow(ctx: any) {

packages/server/src/api/controllers/row/internal.ts

@@ -387,13 +387,6 @@ export async function search(ctx: Ctx) {
   return response
 }
 
-export async function validate(ctx: Ctx) {
-  return utils.validate({
-    tableId: ctx.params.tableId,
-    row: ctx.request.body,
-  })
-}
-
 export async function exportRows(ctx: Ctx) {
   const db = context.getAppDB()
   const table = await db.get(ctx.params.tableId)

packages/server/src/api/controllers/row/utils.ts

@@ -4,11 +4,11 @@ import { FieldTypes } from "../../../constants"
 import { context } from "@budibase/backend-core"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Row, Table } from "@budibase/types"
-const validateJs = require("validate.js")
-const { cloneDeep } = require("lodash/fp")
 import { Format } from "../view/exporters"
 import { Ctx } from "@budibase/types"
 import sdk from "../../../sdk"
+const validateJs = require("validate.js")
+const { cloneDeep } = require("lodash/fp")
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -56,8 +56,7 @@ export async function validate({
 }) {
   let fetchedTable: Table
   if (!table) {
-    const db = context.getAppDB()
+    fetchedTable = await sdk.tables.getTable(tableId)
-    fetchedTable = await db.get(tableId)
   } else {
     fetchedTable = table
   }

packages/server/src/api/controllers/table/external.ts

@@ -20,7 +20,7 @@ import {
   Operation,
   RenameColumn,
   FieldSchema,
-  BBContext,
+  UserCtx,
   TableRequest,
   RelationshipTypes,
 } from "@budibase/types"
@@ -194,7 +194,7 @@ function isRelationshipSetup(column: FieldSchema) {
   return column.foreignKey || column.through
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: UserCtx) {
   const table: TableRequest = ctx.request.body
   const renamed = table?._rename
   // can't do this right now
@@ -313,7 +313,7 @@ export async function save(ctx: BBContext) {
   return tableToSave
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const tableToDelete: TableRequest = await sdk.tables.getTable(
     ctx.params.tableId
   )
@@ -339,7 +339,7 @@ export async function destroy(ctx: BBContext) {
   return tableToDelete
 }
 
-export async function bulkImport(ctx: BBContext) {
+export async function bulkImport(ctx: UserCtx) {
   const table = await sdk.tables.getTable(ctx.params.tableId)
   const { rows }: { rows: unknown } = ctx.request.body
   const schema: unknown = table.schema
@@ -348,7 +348,7 @@ export async function bulkImport(ctx: BBContext) {
     ctx.throw(400, "Provided data import information is invalid.")
   }
 
-  const parsedRows = await parse(rows, schema)
+  const parsedRows = parse(rows, schema)
   await handleRequest(Operation.BULK_CREATE, table._id!, {
     rows: parsedRows,
   })

packages/server/src/integrations/googlesheets.ts

@@ -8,7 +8,6 @@ import {
   SearchFilters,
   SortJson,
   Table,
-  TableSchema,
 } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
 import { buildExternalTableId } from "./utils"
@@ -210,6 +209,26 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     }
   }
 
+  getTableSchema(title: string, headerValues: string[], id?: string) {
+    // base table
+    const table: Table = {
+      name: title,
+      primary: ["rowNumber"],
+      schema: {},
+    }
+    if (id) {
+      table._id = id
+    }
+    // build schema from headers
+    for (let header of headerValues) {
+      table.schema[header] = {
+        name: header,
+        type: FieldTypes.STRING,
+      }
+    }
+    return table
+  }
+
   async buildSchema(datasourceId: string) {
     await this.connect()
     const sheets = this.client.sheetsByIndex
@@ -217,26 +236,14 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     for (let sheet of sheets) {
       // must fetch rows to determine schema
       await sheet.getRows()
-      // build schema
-      const schema: TableSchema = {}
 
-      // build schema from headers
+      const id = buildExternalTableId(datasourceId, sheet.title)
-      for (let header of sheet.headerValues) {
+      tables[sheet.title] = this.getTableSchema(
-        schema[header] = {
+        sheet.title,
-          name: header,
+        sheet.headerValues,
-          type: FieldTypes.STRING,
+        id
-        }
+      )
-      }
-
-      // create tables
-      tables[sheet.title] = {
-        _id: buildExternalTableId(datasourceId, sheet.title),
-        name: sheet.title,
-        primary: ["rowNumber"],
-        schema,
-      }
     }
-
     this.tables = tables
   }
 
@@ -311,12 +318,19 @@ class GoogleSheetsIntegration implements DatasourcePlus {
       } else {
         const updatedHeaderValues = [...sheet.headerValues]
 
-        const newField = Object.keys(table.schema).find(
+        // add new column - doesn't currently exist
-          key => !sheet.headerValues.includes(key)
+        for (let key of Object.keys(table.schema)) {
-        )
+          if (!sheet.headerValues.includes(key)) {
+            updatedHeaderValues.push(key)
+          }
+        }
 
-        if (newField) {
+        // clear out deleted columns
-          updatedHeaderValues.push(newField)
+        for (let key of sheet.headerValues) {
+          if (!Object.keys(table.schema).includes(key)) {
+            const idx = updatedHeaderValues.indexOf(key)
+            updatedHeaderValues.splice(idx, 1)
+          }
         }
 
         await sheet.setHeaderRow(updatedHeaderValues)

packages/server/src/integrations/utils.ts

@@ -200,9 +200,9 @@ export function isIsoDateString(str: string) {
  * @param column The column to check, to see if it is a valid relationship.
  * @param tableIds The IDs of the tables which currently exist.
  */
-function shouldCopyRelationship(
+export function shouldCopyRelationship(
   column: { type: string; tableId?: string },
-  tableIds: [string]
+  tableIds: string[]
 ) {
   return (
     column.type === FieldTypes.LINK &&
@@ -219,7 +219,7 @@ function shouldCopyRelationship(
  * @param column The column to check for options or boolean type.
  * @param fetchedColumn The fetched column to check for the type in the external database.
  */
-function shouldCopySpecialColumn(
+export function shouldCopySpecialColumn(
   column: { type: string },
   fetchedColumn: { type: string } | undefined
 ) {