Merge branch 'labday/type_schema' into feature/multi-user-type-column

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -5,8 +5,11 @@ import {
   FieldType,
   FilterType,
   IncludeRelationship,
+  ManyToManyRelationshipFieldMetadata,
+  OneToManyRelationshipFieldMetadata,
   Operation,
   PaginationJson,
+  RelationshipFieldMetadata,
   RelationshipsJson,
   RelationshipType,
   Row,
@@ -254,12 +257,20 @@ function fixArrayTypes(row: Row, table: Table) {
   return row
 }
 
-function isOneSide(field: FieldSchema) {
+function isOneSide(
+  field: RelationshipFieldMetadata
+): field is OneToManyRelationshipFieldMetadata {
   return (
     field.relationshipType && field.relationshipType.split("-")[0] === "one"
   )
 }
 
+function isManyToMany(
+  field: RelationshipFieldMetadata
+): field is ManyToManyRelationshipFieldMetadata {
+  return !!(field as ManyToManyRelationshipFieldMetadata).through
+}
+
 function isEditableColumn(column: FieldSchema) {
   const isExternalAutoColumn =
     column.autocolumn &&
@@ -352,11 +363,11 @@ export class ExternalRequest<T extends Operation> {
         }
       }
       // many to many
-      else if (field.through) {
+      else if (isManyToMany(field)) {
         // we're not inserting a doc, will be a bunch of update calls
         const otherKey: string = field.throughFrom || linkTablePrimary
         const thisKey: string = field.throughTo || tablePrimary
-        row[key].forEach((relationship: any) => {
+        for (const relationship of row[key]) {
           manyRelationships.push({
             tableId: field.through || field.tableId,
             isUpdate: false,
@@ -365,14 +376,14 @@ export class ExternalRequest<T extends Operation> {
             // leave the ID for enrichment later
             [thisKey]: `{{ literal ${tablePrimary} }}`,
           })
-        })
+        }
       }
       // many to one
       else {
         const thisKey: string = "id"
         // @ts-ignore
         const otherKey: string = field.fieldName
-        row[key].forEach((relationship: any) => {
+        for (const relationship of row[key]) {
           manyRelationships.push({
             tableId: field.tableId,
             isUpdate: true,
@@ -381,7 +392,7 @@ export class ExternalRequest<T extends Operation> {
             // leave the ID for enrichment later
             [otherKey]: `{{ literal ${tablePrimary} }}`,
           })
-        })
+        }
       }
     }
     // we return the relationships that may need to be created in the through table
@@ -549,15 +560,12 @@ export class ExternalRequest<T extends Operation> {
       if (!table.primary || !linkTable.primary) {
         continue
       }
-      const definition: any = {
+      const definition: RelationshipsJson = {
-        // if no foreign key specified then use the name of the field in other table
-        from: field.foreignKey || table.primary[0],
-        to: field.fieldName,
         tableName: linkTableName,
         // need to specify where to put this back into
         column: fieldName,
       }
-      if (field.through) {
+      if (isManyToMany(field)) {
         const { tableName: throughTableName } = breakExternalTableId(
           field.through
         )
@@ -567,6 +575,10 @@ export class ExternalRequest<T extends Operation> {
         definition.to = field.throughFrom || linkTable.primary[0]
         definition.fromPrimary = table.primary[0]
         definition.toPrimary = linkTable.primary[0]
+      } else {
+        // if no foreign key specified then use the name of the field in other table
+        definition.from = field.foreignKey || table.primary[0]
+        definition.to = field.fieldName
       }
       relationships.push(definition)
     }
@@ -588,7 +600,7 @@ export class ExternalRequest<T extends Operation> {
     const primaryKey = table.primary[0]
     // make a new request to get the row with all its relationships
     // we need this to work out if any relationships need removed
-    for (let field of Object.values(table.schema)) {
+    for (const field of Object.values(table.schema)) {
       if (
         field.type !== FieldTypes.LINK ||
         !field.fieldName ||
@@ -601,9 +613,9 @@ export class ExternalRequest<T extends Operation> {
       const { tableName: relatedTableName } = breakExternalTableId(tableId)
       // @ts-ignore
       const linkPrimaryKey = this.tables[relatedTableName].primary[0]
-      const manyKey = field.throughTo || primaryKey
+
       const lookupField = isMany ? primaryKey : field.foreignKey
-      const fieldName = isMany ? manyKey : field.fieldName
+      const fieldName = isMany ? field.throughTo || primaryKey : field.fieldName
       if (!lookupField || !row[lookupField]) {
         continue
       }

packages/server/src/api/controllers/row/utils.ts

@@ -4,6 +4,8 @@ import { context } from "@budibase/backend-core"
 import {
   Ctx,
   FieldType,
+  ManyToOneRelationshipFieldMetadata,
+  OneToManyRelationshipFieldMetadata,
   Row,
   SearchFilters,
   Table,
@@ -19,7 +21,14 @@ function isForeignKey(key: string, table: Table) {
   const relationships = Object.values(table.schema).filter(
     column => column.type === FieldType.LINK
   )
-  return relationships.some(relationship => relationship.foreignKey === key)
+  return relationships.some(
+    relationship =>
+      (
+        relationship as
+          | OneToManyRelationshipFieldMetadata
+          | ManyToOneRelationshipFieldMetadata
+      ).foreignKey === key
+  )
 }
 
 validateJs.extend(validateJs.validators.datetime, {

packages/server/src/api/controllers/table/bulkFormula.ts

@@ -1,4 +1,4 @@
-import { FieldTypes, FormulaTypes } from "../../../constants"
+import { FormulaTypes } from "../../../constants"
 import { clearColumns } from "./utils"
 import { doesContainStrings } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
@@ -6,12 +6,20 @@ import isEqual from "lodash/isEqual"
 import uniq from "lodash/uniq"
 import { updateAllFormulasInTable } from "../row/staticFormula"
 import { context } from "@budibase/backend-core"
-import { FieldSchema, Table } from "@budibase/types"
+import {
+  FieldSchema,
+  FieldType,
+  FormulaFieldMetadata,
+  Table,
+} from "@budibase/types"
 import sdk from "../../../sdk"
+import { isRelationshipColumn } from "../../../db/utils"
 
-function isStaticFormula(column: FieldSchema) {
+function isStaticFormula(
+  column: FieldSchema
+): column is FormulaFieldMetadata & { formulaType: FormulaTypes.STATIC } {
   return (
-    column.type === FieldTypes.FORMULA &&
+    column.type === FieldType.FORMULA &&
     column.formulaType === FormulaTypes.STATIC
   )
 }
@@ -56,8 +64,9 @@ async function checkIfFormulaNeedsCleared(
   for (let removed of removedColumns) {
     let tableToUse: Table | undefined = table
     // if relationship, get the related table
-    if (removed.type === FieldTypes.LINK) {
+    if (removed.type === FieldType.LINK) {
-      tableToUse = tables.find(table => table._id === removed.tableId)
+      const removedTableId = removed.tableId
+      tableToUse = tables.find(table => table._id === removedTableId)
     }
     if (!tableToUse) {
       continue
@@ -73,17 +82,18 @@ async function checkIfFormulaNeedsCleared(
     }
     for (let relatedTableId of table.relatedFormula) {
       const relatedColumns = Object.values(table.schema).filter(
-        column => column.tableId === relatedTableId
+        column =>
+          column.type === FieldType.LINK && column.tableId === relatedTableId
       )
       const relatedTable = tables.find(table => table._id === relatedTableId)
       // look to see if the column was used in a relationship formula,
       // relationships won't be used for this
-      if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) {
+      if (relatedTable && relatedColumns && removed.type !== FieldType.LINK) {
         let relatedFormulaToRemove: string[] = []
         for (let column of relatedColumns) {
           relatedFormulaToRemove = relatedFormulaToRemove.concat(
             getFormulaThatUseColumn(relatedTable, [
-              column.fieldName!,
+              (column as any).fieldName!,
               removed.name,
             ])
           )
@@ -116,7 +126,7 @@ async function updateRelatedFormulaLinksOnTables(
   const initialTables = cloneDeep(tables)
   // first find the related column names
   const relatedColumns = Object.values(table.schema).filter(
-    col => col.type === FieldTypes.LINK
+    isRelationshipColumn
   )
   // we start by removing the formula field from all tables
   for (let otherTable of tables) {
@@ -135,6 +145,7 @@ async function updateRelatedFormulaLinksOnTables(
       if (!columns || columns.length === 0) {
         continue
       }
+
       const relatedTable = tables.find(
         related => related._id === relatedCol.tableId
       )

packages/server/src/api/controllers/table/external.ts

@@ -17,9 +17,12 @@ import { isRows, isSchema, parse } from "../../../utilities/schema"
 import {
   BulkImportRequest,
   Datasource,
-  FieldSchema,
+  ManyToManyRelationshipFieldMetadata,
+  ManyToOneRelationshipFieldMetadata,
+  OneToManyRelationshipFieldMetadata,
   Operation,
   QueryJson,
+  RelationshipFieldMetadata,
   RelationshipType,
   RenameColumn,
   SaveTableRequest,
@@ -74,10 +77,13 @@ function cleanupRelationships(
       schema.type === FieldTypes.LINK &&
       (!oldTable || table.schema[key] == null)
     ) {
+      const schemaTableId = schema.tableId
       const relatedTable = Object.values(tables).find(
-        table => table._id === schema.tableId
+        table => table._id === schemaTableId
       )
-      const foreignKey = schema.foreignKey
+      const foreignKey =
+        schema.relationshipType !== RelationshipType.MANY_TO_MANY &&
+        schema.foreignKey
       if (!relatedTable || !foreignKey) {
         continue
       }
@@ -116,7 +122,7 @@ function otherRelationshipType(type?: string) {
 
 function generateManyLinkSchema(
   datasource: Datasource,
-  column: FieldSchema,
+  column: ManyToManyRelationshipFieldMetadata,
   table: Table,
   relatedTable: Table
 ): Table {
@@ -151,10 +157,12 @@ function generateManyLinkSchema(
 }
 
 function generateLinkSchema(
-  column: FieldSchema,
+  column:
+    | OneToManyRelationshipFieldMetadata
+    | ManyToOneRelationshipFieldMetadata,
   table: Table,
   relatedTable: Table,
-  type: RelationshipType
+  type: RelationshipType.ONE_TO_MANY | RelationshipType.MANY_TO_ONE
 ) {
   if (!table.primary || !relatedTable.primary) {
     throw new Error("Unable to generate link schema, no primary keys")
@@ -170,20 +178,22 @@ function generateLinkSchema(
 }
 
 function generateRelatedSchema(
-  linkColumn: FieldSchema,
+  linkColumn: RelationshipFieldMetadata,
   table: Table,
   relatedTable: Table,
   columnName: string
 ) {
   // generate column for other table
   const relatedSchema = cloneDeep(linkColumn)
+  const isMany2Many =
+    linkColumn.relationshipType === RelationshipType.MANY_TO_MANY
   // swap them from the main link
-  if (linkColumn.foreignKey) {
+  if (!isMany2Many && linkColumn.foreignKey) {
     relatedSchema.fieldName = linkColumn.foreignKey
     relatedSchema.foreignKey = linkColumn.fieldName
   }
   // is many to many
-  else {
+  else if (isMany2Many) {
     // don't need to copy through, already got it
     relatedSchema.fieldName = linkColumn.throughTo
     relatedSchema.throughTo = linkColumn.throughFrom
@@ -197,8 +207,8 @@ function generateRelatedSchema(
   table.schema[columnName] = relatedSchema
 }
 
-function isRelationshipSetup(column: FieldSchema) {
+function isRelationshipSetup(column: RelationshipFieldMetadata) {
-  return column.foreignKey || column.through
+  return (column as any).foreignKey || (column as any).through
 }
 
 export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
@@ -257,14 +267,15 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
     if (schema.type !== FieldTypes.LINK || isRelationshipSetup(schema)) {
       continue
     }
+    const schemaTableId = schema.tableId
     const relatedTable = Object.values(tables).find(
-      table => table._id === schema.tableId
+      table => table._id === schemaTableId
     )
     if (!relatedTable) {
       continue
     }
     const relatedColumnName = schema.fieldName!
-    const relationType = schema.relationshipType!
+    const relationType = schema.relationshipType
     if (relationType === RelationshipType.MANY_TO_MANY) {
       const junctionTable = generateManyLinkSchema(
         datasource,

packages/server/src/api/controllers/table/internal.ts

@@ -79,10 +79,10 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   // make sure that types don't change of a column, have to remove
   // the column if you want to change the type
   if (oldTable && oldTable.schema) {
-    for (let propKey of Object.keys(tableToSave.schema)) {
+    for (const propKey of Object.keys(tableToSave.schema)) {
       let oldColumn = oldTable.schema[propKey]
       if (oldColumn && oldColumn.type === FieldTypes.INTERNAL) {
-        oldColumn.type = FieldTypes.AUTO
+        oldTable.schema[propKey].type = FieldTypes.AUTO
       }
     }
   }

packages/server/src/api/routes/tests/row.spec.ts

@@ -6,6 +6,8 @@ import * as setup from "./utilities"
 import { context, InternalTable, roles, tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
+  AutoFieldSubTypes,
+  FieldSchema,
   FieldType,
   FieldTypeSubtypes,
   MonthlyQuotaName,
@@ -171,7 +173,7 @@ describe.each([
             "Row ID": {
               name: "Row ID",
               type: FieldType.NUMBER,
-              subtype: "autoID",
+              subtype: AutoFieldSubTypes.AUTO_ID,
               icon: "ri-magic-line",
               autocolumn: true,
               constraints: {
@@ -272,27 +274,27 @@ describe.each([
 
     isInternal &&
       it("row values are coerced", async () => {
-        const str = {
+        const str: FieldSchema = {
           type: FieldType.STRING,
           name: "str",
           constraints: { type: "string", presence: false },
         }
-        const attachment = {
+        const attachment: FieldSchema = {
           type: FieldType.ATTACHMENT,
           name: "attachment",
           constraints: { type: "array", presence: false },
         }
-        const bool = {
+        const bool: FieldSchema = {
           type: FieldType.BOOLEAN,
           name: "boolean",
           constraints: { type: "boolean", presence: false },
         }
-        const number = {
+        const number: FieldSchema = {
           type: FieldType.NUMBER,
           name: "str",
           constraints: { type: "number", presence: false },
         }
-        const datetime = {
+        const datetime: FieldSchema = {
           type: FieldType.DATETIME,
           name: "datetime",
           constraints: {
@@ -301,7 +303,7 @@ describe.each([
             datetime: { earliest: "", latest: "" },
           },
         }
-        const arrayField = {
+        const arrayField: FieldSchema = {
           type: FieldType.ARRAY,
           constraints: {
             type: "array",
@@ -311,8 +313,7 @@ describe.each([
           name: "Sample Tags",
           sortable: false,
         }
-        const optsField = {
+        const optsField: FieldSchema = {
-          fieldName: "Sample Opts",
           name: "Sample Opts",
           type: FieldType.OPTIONS,
           constraints: {
@@ -1534,7 +1535,7 @@ describe.each([
   describe.each([
     [
       "relationship fields",
-      () => ({
+      (): Record<string, FieldSchema> => ({
         user: {
           name: "user",
           relationshipType: RelationshipType.ONE_TO_MANY,
@@ -1563,7 +1564,7 @@ describe.each([
     ],
     [
       "bb reference fields",
-      () => ({
+      (): Record<string, FieldSchema> => ({
         user: {
           name: "user",
           type: FieldType.BB_REFERENCE,

packages/server/src/api/routes/tests/table.spec.ts

@@ -3,6 +3,7 @@ import { events, context } from "@budibase/backend-core"
 import {
   FieldType,
   SaveTableRequest,
+  RelationshipType,
   Table,
   ViewCalculation,
 } from "@budibase/types"
@@ -357,9 +358,10 @@ describe("/tables", () => {
           },
           TestTable: {
             type: FieldType.LINK,
+            relationshipType: RelationshipType.ONE_TO_MANY,
             name: "TestTable",
             fieldName: "TestTable",
-            tableId: testTable._id,
+            tableId: testTable._id!,
             constraints: {
               type: "array",
             },

packages/server/src/constants/index.ts

@@ -1,6 +1,11 @@
 import { objectStore, roles, constants } from "@budibase/backend-core"
 import { FieldType as FieldTypes } from "@budibase/types"
-export { FieldType as FieldTypes, RelationshipType } from "@budibase/types"
+export {
+  FieldType as FieldTypes,
+  RelationshipType,
+  AutoFieldSubTypes,
+  FormulaTypes,
+} from "@budibase/types"
 
 export enum FilterTypes {
   STRING = "string",
@@ -39,11 +44,6 @@ export const SwitchableTypes = CanSwitchTypes.reduce((prev, current) =>
   prev ? prev.concat(current) : current
 )
 
-export enum FormulaTypes {
-  STATIC = "static",
-  DYNAMIC = "dynamic",
-}
-
 export enum AuthTypes {
   APP = "app",
   BUILDER = "builder",
@@ -132,14 +132,6 @@ export const USERS_TABLE_SCHEMA = {
   primaryDisplay: "email",
 }
 
-export enum AutoFieldSubTypes {
-  CREATED_BY = "createdBy",
-  CREATED_AT = "createdAt",
-  UPDATED_BY = "updatedBy",
-  UPDATED_AT = "updatedAt",
-  AUTO_ID = "autoID",
-}
-
 export enum AutoFieldDefaultNames {
   CREATED_BY = "Created By",
   CREATED_AT = "Created At",

packages/server/src/db/defaultData/datasource_bb_default.ts

@@ -7,7 +7,13 @@ import { employeeImport } from "./employeeImport"
 import { jobsImport } from "./jobsImport"
 import { expensesImport } from "./expensesImport"
 import { db as dbCore } from "@budibase/backend-core"
-import { Table, Row, RelationshipType } from "@budibase/types"
+import {
+  Table,
+  Row,
+  RelationshipType,
+  FieldType,
+  TableSchema,
+} from "@budibase/types"
 
 export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
 export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
@@ -28,7 +34,11 @@ export const DEFAULT_BB_DATASOURCE = defaultDatasource
 function syncLastIds(table: Table, rowCount: number) {
   Object.keys(table.schema).forEach(key => {
     const entry = table.schema[key]
-    if (entry.autocolumn && entry.subtype == "autoID") {
+    if (
+      entry.autocolumn &&
+      entry.type === FieldType.NUMBER &&
+      entry.subtype == AutoFieldSubTypes.AUTO_ID
+    ) {
       entry.lastID = rowCount
     }
   })
@@ -42,7 +52,7 @@ async function tableImport(table: Table, data: Row[]) {
 }
 
 // AUTO COLUMNS
-const AUTO_COLUMNS = {
+const AUTO_COLUMNS: TableSchema = {
   "Created At": {
     name: "Created At",
     type: FieldTypes.DATETIME,

packages/server/src/db/linkedRows/LinkController.ts

@@ -7,7 +7,9 @@ import LinkDocument from "./LinkDocument"
 import {
   Database,
   FieldSchema,
+  FieldType,
   LinkDocumentValue,
+  RelationshipFieldMetadata,
   RelationshipType,
   Row,
   Table,
@@ -133,7 +135,10 @@ class LinkController {
    * Given the link field of this table, and the link field of the linked table, this makes sure
    * the state of relationship type is accurate on both.
    */
-  handleRelationshipType(linkerField: FieldSchema, linkedField: FieldSchema) {
+  handleRelationshipType(
+    linkerField: RelationshipFieldMetadata,
+    linkedField: RelationshipFieldMetadata
+  ) {
     if (
       !linkerField.relationshipType ||
       linkerField.relationshipType === RelationshipType.MANY_TO_MANY
@@ -183,7 +188,7 @@ class LinkController {
 
         // if 1:N, ensure that this ID is not already attached to another record
         const linkedTable = await this._db.get<Table>(field.tableId)
-        const linkedSchema = linkedTable.schema[field.fieldName!]
+        const linkedSchema = linkedTable.schema[field.fieldName]
 
         // We need to map the global users to metadata in each app for relationships
         if (field.tableId === InternalTables.USER_METADATA) {
@@ -200,7 +205,10 @@ class LinkController {
 
         // iterate through the link IDs in the row field, see if any don't exist already
         for (let linkId of rowField) {
-          if (linkedSchema?.relationshipType === RelationshipType.ONE_TO_MANY) {
+          if (
+            linkedSchema?.type === FieldType.LINK &&
+            linkedSchema?.relationshipType === RelationshipType.ONE_TO_MANY
+          ) {
             let links = (
               (await getLinkDocuments({
                 tableId: field.tableId,
@@ -291,7 +299,7 @@ class LinkController {
    */
   async removeFieldFromTable(fieldName: string) {
     let oldTable = this._oldTable
-    let field = oldTable?.schema[fieldName] as FieldSchema
+    let field = oldTable?.schema[fieldName] as RelationshipFieldMetadata
     const linkDocs = await this.getTableLinkDocs()
     let toDelete = linkDocs.filter(linkDoc => {
       let correctFieldName =
@@ -351,9 +359,9 @@ class LinkController {
           name: field.fieldName,
           type: FieldTypes.LINK,
           // these are the props of the table that initiated the link
-          tableId: table._id,
+          tableId: table._id!,
           fieldName: fieldName,
-        })
+        } as RelationshipFieldMetadata)
 
         // update table schema after checking relationship types
         schema[fieldName] = fields.linkerField

packages/server/src/db/linkedRows/linkUtils.ts

@@ -1,13 +1,9 @@
-import { ViewName, getQueryIndex } from "../utils"
+import { ViewName, getQueryIndex, isRelationshipColumn } from "../utils"
 import { FieldTypes } from "../../constants"
 import { createLinkView } from "../views/staticViews"
 import { context, logging } from "@budibase/backend-core"
-import {
+import { LinkDocument, LinkDocumentValue, Table } from "@budibase/types"
-  FieldSchema,
+
-  LinkDocument,
-  LinkDocumentValue,
-  Table,
-} from "@budibase/types"
 export { createLinkView } from "../views/staticViews"
 
 /**
@@ -93,7 +89,7 @@ export function getUniqueByProp(array: any[], prop: string) {
 
 export function getLinkedTableIDs(table: Table) {
   return Object.values(table.schema)
-    .filter((column: FieldSchema) => column.type === FieldTypes.LINK)
+    .filter(isRelationshipColumn)
     .map(column => column.tableId)
 }
 
@@ -113,7 +109,7 @@ export async function getLinkedTable(id: string, tables: Table[]) {
 export function getRelatedTableForField(table: Table, fieldName: string) {
   // look to see if its on the table, straight in the schema
   const field = table.schema[fieldName]
-  if (field != null) {
+  if (field?.type === FieldTypes.LINK) {
     return field.tableId
   }
   for (let column of Object.values(table.schema)) {

packages/server/src/db/utils.ts

@@ -1,6 +1,12 @@
 import newid from "./newid"
 import { db as dbCore } from "@budibase/backend-core"
-import { DocumentType, VirtualDocumentType } from "@budibase/types"
+import {
+  DocumentType,
+  FieldSchema,
+  RelationshipFieldMetadata,
+  VirtualDocumentType,
+} from "@budibase/types"
+import { FieldTypes } from "../constants"
 export { DocumentType, VirtualDocumentType } from "@budibase/types"
 
 type Optional = string | null
@@ -307,3 +313,9 @@ export function extractViewInfoFromID(viewId: string) {
     tableId: res!.groups!["tableId"],
   }
 }
+
+export function isRelationshipColumn(
+  column: FieldSchema
+): column is RelationshipFieldMetadata {
+  return column.type === FieldTypes.LINK
+}

packages/server/src/integration-test/postgres.spec.ts

@@ -111,7 +111,7 @@ describe("postgres integrations", () => {
           fieldName: oneToManyRelationshipInfo.fieldName,
           name: "oneToManyRelation",
           relationshipType: RelationshipType.ONE_TO_MANY,
-          tableId: oneToManyRelationshipInfo.table._id,
+          tableId: oneToManyRelationshipInfo.table._id!,
           main: true,
         },
         manyToOneRelation: {
@@ -122,7 +122,7 @@ describe("postgres integrations", () => {
           fieldName: manyToOneRelationshipInfo.fieldName,
           name: "manyToOneRelation",
           relationshipType: RelationshipType.MANY_TO_ONE,
-          tableId: manyToOneRelationshipInfo.table._id,
+          tableId: manyToOneRelationshipInfo.table._id!,
           main: true,
         },
         manyToManyRelation: {
@@ -133,7 +133,7 @@ describe("postgres integrations", () => {
           fieldName: manyToManyRelationshipInfo.fieldName,
           name: "manyToManyRelation",
           relationshipType: RelationshipType.MANY_TO_MANY,
-          tableId: manyToManyRelationshipInfo.table._id,
+          tableId: manyToManyRelationshipInfo.table._id!,
           main: true,
         },
       },
@@ -250,6 +250,7 @@ describe("postgres integrations", () => {
         id: {
           name: "id",
           type: FieldType.AUTO,
+          autocolumn: true,
         },
       },
       sourceId: postgresDatasource._id,

packages/server/src/integrations/base/sqlTable.ts

@@ -1,6 +1,7 @@
 import { Knex, knex } from "knex"
 import {
   FieldSubtype,
+  NumberFieldMetadata,
   Operation,
   QueryJson,
   RenameColumn,
@@ -22,7 +23,7 @@ function generateSchema(
   let primaryKey = table && table.primary ? table.primary[0] : null
   const columns = Object.values(table.schema)
   // all columns in a junction table will be meta
-  let metaCols = columns.filter(col => col.meta)
+  let metaCols = columns.filter(col => (col as NumberFieldMetadata).meta)
   let isJunction = metaCols.length === columns.length
   // can't change primary once its set for now
   if (primaryKey && !oldTable && !isJunction) {
@@ -32,7 +33,9 @@ function generateSchema(
   }
 
   // check if any columns need added
-  const foreignKeys = Object.values(table.schema).map(col => col.foreignKey)
+  const foreignKeys = Object.values(table.schema).map(
+    col => (col as any).foreignKey
+  )
   for (let [key, column] of Object.entries(table.schema)) {
     // skip things that are already correct
     const oldColumn = oldTable ? oldTable.schema[key] : null

packages/server/src/integrations/oracle.ts

@@ -249,7 +249,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     )
   }
 
-  private internalConvertType(column: OracleColumn): { type: FieldTypes } {
+  private internalConvertType(column: OracleColumn) {
     if (this.isBooleanType(column)) {
       return { type: FieldTypes.BOOLEAN }
     }
@@ -307,6 +307,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
               },
               ...this.internalConvertType(oracleColumn),
             }
+
             table.schema[columnName] = fieldSchema
           }
 

packages/server/src/sdk/app/rows/utils.ts

@@ -1,10 +1,11 @@
 import cloneDeep from "lodash/cloneDeep"
 import validateJs from "validate.js"
-import { FieldType, Row, Table, TableSchema } from "@budibase/types"
+import { Row, Table, TableSchema } from "@budibase/types"
 import { FieldTypes } from "../../../constants"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
+import { isRelationshipColumn } from "../../../db/utils"
 
 export async function getDatasourceAndQuery(json: any) {
   const datasourceId = json.endpoint.datasourceId
@@ -50,10 +51,10 @@ export function cleanExportRows(
 }
 
 function isForeignKey(key: string, table: Table) {
-  const relationships = Object.values(table.schema).filter(
+  const relationships = Object.values(table.schema).filter(isRelationshipColumn)
-    column => column.type === FieldType.LINK
+  return relationships.some(
+    relationship => (relationship as any).foreignKey === key
   )
-  return relationships.some(relationship => relationship.foreignKey === key)
 }
 
 export async function validate({

packages/server/src/sdk/app/tables/tests/validation.spec.ts

@@ -1,6 +1,6 @@
 import { populateExternalTableSchemas } from "../validation"
 import { cloneDeep } from "lodash/fp"
-import { Datasource, Table } from "@budibase/types"
+import { AutoReason, Datasource, Table } from "@budibase/types"
 import { isEqual } from "lodash"
 
 const SCHEMA = {
@@ -109,7 +109,7 @@ describe("validation and update of external table schemas", () => {
     const response = populateExternalTableSchemas(cloneDeep(SCHEMA) as any)
     const foreignKey = getForeignKeyColumn(response)
     expect(foreignKey.autocolumn).toBe(true)
-    expect(foreignKey.autoReason).toBe("foreign_key")
+    expect(foreignKey.autoReason).toBe(AutoReason.FOREIGN_KEY)
     noOtherTableChanges(response)
   })
 

packages/server/src/sdk/app/tables/validation.ts

@@ -1,11 +1,9 @@
 import {
   AutoReason,
   Datasource,
-  FieldSchema,
   FieldType,
   RelationshipType,
 } from "@budibase/types"
-import { FieldTypes } from "../../../constants"
 
 function checkForeignKeysAreAutoColumns(datasource: Datasource) {
   if (!datasource.entities) {
@@ -15,10 +13,11 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) {
   // make sure all foreign key columns are marked as auto columns
   const foreignKeys: { tableId: string; key: string }[] = []
   for (let table of tables) {
-    const relationships = Object.values(table.schema).filter(
+    Object.values(table.schema).forEach(column => {
-      column => column.type === FieldType.LINK
+      if (column.type !== FieldType.LINK) {
-    )
+        return
-    relationships.forEach(relationship => {
+      }
+      const relationship = column
       if (relationship.relationshipType === RelationshipType.MANY_TO_MANY) {
         const tableId = relationship.through!
         foreignKeys.push({ key: relationship.throughTo!, tableId })
@@ -36,7 +35,7 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) {
   }
 
   // now make sure schemas are all accurate
-  for (let table of tables) {
+  for (const table of tables) {
     for (let column of Object.values(table.schema)) {
       const shouldBeForeign = foreignKeys.find(
         options => options.tableId === table._id && options.key === column.name

packages/server/src/sdk/app/views/tests/views.spec.ts

@@ -1,5 +1,11 @@
 import _ from "lodash"
-import { FieldType, Table, TableSchema, ViewV2 } from "@budibase/types"
+import {
+  FieldSchema,
+  FieldType,
+  Table,
+  TableSchema,
+  ViewV2,
+} from "@budibase/types"
 import { generator } from "@budibase/backend-core/tests"
 import { enrichSchema, syncSchema } from ".."
 
@@ -316,7 +322,7 @@ describe("table sdk", () => {
           ...basicView,
         }
 
-        const newTableSchema = {
+        const newTableSchema: TableSchema = {
           ...basicTable.schema,
           newField1: {
             type: FieldType.STRING,
@@ -403,7 +409,7 @@ describe("table sdk", () => {
           },
         }
 
-        const newTableSchema = {
+        const newTableSchema: TableSchema = {
           ...basicTable.schema,
           newField1: {
             type: FieldType.STRING,
@@ -531,7 +537,7 @@ describe("table sdk", () => {
             id: {
               ...basicTable.schema.id,
               type: FieldType.NUMBER,
-            },
+            } as FieldSchema,
           },
           undefined
         )

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -54,6 +54,7 @@ import {
   FieldType,
   RelationshipType,
   CreateViewRequest,
+  RelationshipFieldMetadata,
 } from "@budibase/types"
 
 import API from "./api"
@@ -584,10 +585,10 @@ class TestConfiguration {
       tableConfig.schema[link] = {
         type: FieldType.LINK,
         fieldName: link,
-        tableId: this.table._id,
+        tableId: this.table._id!,
         name: link,
         relationshipType,
-      }
+      } as RelationshipFieldMetadata
     }
 
     if (this.datasource && !tableConfig.sourceId) {

packages/server/src/utilities/rowProcessor/index.ts

@@ -5,7 +5,13 @@ import { ObjectStoreBuckets } from "../../constants"
 import { context, db as dbCore, objectStore } from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
-import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types"
+import {
+  AutoColumnFieldMetadata,
+  FieldSubtype,
+  Row,
+  RowAttachment,
+  Table,
+} from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import {
   processInputBBReferences,

packages/server/src/utilities/rowProcessor/tests/utils.spec.ts

@@ -4,10 +4,10 @@ import { FieldSchema, FieldType, RelationshipType } from "@budibase/types"
 
 describe("rowProcessor utility", () => {
   describe("fixAutoColumnSubType", () => {
-    let schema: FieldSchema = {
+    const schema: FieldSchema = {
       name: "",
       type: FieldType.LINK,
-      subtype: "", // missing subtype
+      subtype: undefined, // missing subtype
       icon: "ri-magic-line",
       autocolumn: true,
       constraints: { type: "array", presence: false },
@@ -22,31 +22,31 @@ describe("rowProcessor utility", () => {
       expect(fixAutoColumnSubType(schema).subtype).toEqual(
         AutoFieldSubTypes.CREATED_BY
       )
-      schema.subtype = ""
+      schema.subtype = undefined
 
       schema.name = AutoFieldDefaultNames.UPDATED_BY
       expect(fixAutoColumnSubType(schema).subtype).toEqual(
         AutoFieldSubTypes.UPDATED_BY
       )
-      schema.subtype = ""
+      schema.subtype = undefined
 
       schema.name = AutoFieldDefaultNames.CREATED_AT
       expect(fixAutoColumnSubType(schema).subtype).toEqual(
         AutoFieldSubTypes.CREATED_AT
       )
-      schema.subtype = ""
+      schema.subtype = undefined
 
       schema.name = AutoFieldDefaultNames.UPDATED_AT
       expect(fixAutoColumnSubType(schema).subtype).toEqual(
         AutoFieldSubTypes.UPDATED_AT
       )
-      schema.subtype = ""
+      schema.subtype = undefined
 
       schema.name = AutoFieldDefaultNames.AUTO_ID
       expect(fixAutoColumnSubType(schema).subtype).toEqual(
         AutoFieldSubTypes.AUTO_ID
       )
-      schema.subtype = ""
+      schema.subtype = undefined
     })
 
     it("returns the column if subtype exists", async () => {

packages/server/src/utilities/rowProcessor/utils.ts

@@ -5,13 +5,20 @@ import {
   FormulaTypes,
 } from "../../constants"
 import { processStringSync } from "@budibase/string-templates"
-import { FieldSchema, Row, Table } from "@budibase/types"
+import {
+  AutoColumnFieldMetadata,
+  FieldSchema,
+  Row,
+  Table,
+} from "@budibase/types"
 
 /**
  * If the subtype has been lost for any reason this works out what
  * subtype the auto column should be.
  */
-export function fixAutoColumnSubType(column: FieldSchema) {
+export function fixAutoColumnSubType(
+  column: FieldSchema
+): AutoColumnFieldMetadata | FieldSchema {
   if (!column.autocolumn || !column.name || column.subtype) {
     return column
   }
@@ -47,9 +54,13 @@ export function processFormulas(
     rowArray = rows
   }
   for (let [column, schema] of Object.entries(table.schema)) {
+    if (schema.type !== FieldTypes.FORMULA) {
+      continue
+    }
+
     const isStatic = schema.formulaType === FormulaTypes.STATIC
+
     if (
-      schema.type !== FieldTypes.FORMULA ||
       schema.formula == null ||
       (dynamic && isStatic) ||
       (!dynamic && !isStatic)

packages/types/src/documents/app/table/constants.ts

@@ -7,3 +7,16 @@ export enum RelationshipType {
 export enum AutoReason {
   FOREIGN_KEY = "foreign_key",
 }
+
+export enum AutoFieldSubTypes {
+  CREATED_BY = "createdBy",
+  CREATED_AT = "createdAt",
+  UPDATED_BY = "updatedBy",
+  UPDATED_AT = "updatedAt",
+  AUTO_ID = "autoID",
+}
+
+export enum FormulaTypes {
+  STATIC = "static",
+  DYNAMIC = "dynamic",
+}

packages/types/src/documents/app/table/schema.ts

@@ -1,7 +1,12 @@
 // all added by grid/table when defining the
 // column size, position and whether it can be viewed
-import { FieldType } from "../row"
+import { FieldSubtype, FieldType } from "../row"
-import { AutoReason, RelationshipType } from "./constants"
+import {
+  AutoFieldSubTypes,
+  AutoReason,
+  FormulaTypes,
+  RelationshipType,
+} from "./constants"
 
 export interface UIFieldMetadata {
   order?: number
@@ -10,28 +15,63 @@ export interface UIFieldMetadata {
   icon?: string
 }
 
-export interface RelationshipFieldMetadata {
+interface BaseRelationshipFieldMetadata
+  extends Omit<BaseFieldSchema, "subtype"> {
+  type: FieldType.LINK
   main?: boolean
-  fieldName?: string
+  fieldName: string
-  tableId?: string
+  tableId: string
-  // below is used for SQL relationships, needed to define the foreign keys
+  subtype?: AutoFieldSubTypes.CREATED_BY | AutoFieldSubTypes.UPDATED_BY
-  // or the tables used for many-to-many relationships (through)
-  relationshipType?: RelationshipType
-  through?: string
-  foreignKey?: string
-  throughFrom?: string
-  throughTo?: string
 }
 
-export interface AutoColumnFieldMetadata {
+// External tables use junction tables, internal tables don't require them
-  autocolumn?: boolean
+type ManyToManyJunctionTableMetadata =
-  subtype?: string
+  | {
+      through: string
+      throughFrom: string
+      throughTo: string
+    }
+  | {
+      through?: never
+      throughFrom?: never
+      throughTo?: never
+    }
+
+export type ManyToManyRelationshipFieldMetadata =
+  BaseRelationshipFieldMetadata & {
+    relationshipType: RelationshipType.MANY_TO_MANY
+  } & ManyToManyJunctionTableMetadata
+
+export interface OneToManyRelationshipFieldMetadata
+  extends BaseRelationshipFieldMetadata {
+  relationshipType: RelationshipType.ONE_TO_MANY
+  foreignKey?: string
+}
+export interface ManyToOneRelationshipFieldMetadata
+  extends BaseRelationshipFieldMetadata {
+  relationshipType: RelationshipType.MANY_TO_ONE
+  foreignKey?: string
+}
+export type RelationshipFieldMetadata =
+  | ManyToManyRelationshipFieldMetadata
+  | OneToManyRelationshipFieldMetadata
+  | ManyToOneRelationshipFieldMetadata
+
+export interface AutoColumnFieldMetadata
+  extends Omit<BaseFieldSchema, "subtype"> {
+  type: FieldType.AUTO
+  autocolumn: true
+  subtype?: AutoFieldSubTypes
   lastID?: number
   // if the column was turned to an auto-column for SQL, explains why (primary, foreign etc)
   autoReason?: AutoReason
 }
 
-export interface NumberFieldMetadata {
+export interface NumberFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
+  type: FieldType.NUMBER
+  subtype?: AutoFieldSubTypes.AUTO_ID
+  lastID?: number
+  autoReason?: AutoReason.FOREIGN_KEY
   // used specifically when Budibase generates external tables, this denotes if a number field
   // is a foreign key used for a many-to-many relationship
   meta?: {
@@ -40,18 +80,28 @@ export interface NumberFieldMetadata {
   }
 }
 
-export interface DateFieldMetadata {
+export interface DateFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
+  type: FieldType.DATETIME
   ignoreTimezones?: boolean
   timeOnly?: boolean
+  subtype?: AutoFieldSubTypes.CREATED_AT | AutoFieldSubTypes.UPDATED_AT
 }
 
-export interface StringFieldMetadata {
+export interface LongFormFieldMetadata extends BaseFieldSchema {
+  type: FieldType.LONGFORM
   useRichText?: boolean | null
 }
 
-export interface FormulaFieldMetadata {
+export interface FormulaFieldMetadata extends BaseFieldSchema {
-  formula?: string
+  type: FieldType.FORMULA
-  formulaType?: string
+  formula: string
+  formulaType?: FormulaTypes
+}
+
+export interface BBReferenceFieldMetadata
+  extends Omit<BaseFieldSchema, "subtype"> {
+  type: FieldType.BB_REFERENCE
+  subtype: FieldSubtype.USER
 }
 
 export interface FieldConstraints {
@@ -77,22 +127,40 @@ export interface FieldConstraints {
   }
 }
 
-export interface FieldSchema
+interface BaseFieldSchema extends UIFieldMetadata {
-  extends UIFieldMetadata,
-    DateFieldMetadata,
-    RelationshipFieldMetadata,
-    AutoColumnFieldMetadata,
-    StringFieldMetadata,
-    FormulaFieldMetadata,
-    NumberFieldMetadata {
   type: FieldType
   name: string
   sortable?: boolean
   // only used by external databases, to denote the real type
   externalType?: string
   constraints?: FieldConstraints
+  autocolumn?: boolean
+  autoReason?: AutoReason.FOREIGN_KEY
+  subtype?: never
 }
 
+interface OtherFieldMetadata extends BaseFieldSchema {
+  type: Exclude<
+    FieldType,
+    | FieldType.DATETIME
+    | FieldType.LINK
+    | FieldType.AUTO
+    | FieldType.FORMULA
+    | FieldType.NUMBER
+    | FieldType.LONGFORM
+  >
+}
+
+export type FieldSchema =
+  | OtherFieldMetadata
+  | DateFieldMetadata
+  | RelationshipFieldMetadata
+  | AutoColumnFieldMetadata
+  | FormulaFieldMetadata
+  | NumberFieldMetadata
+  | LongFormFieldMetadata
+  | BBReferenceFieldMetadata
+
 export interface TableSchema {
   [key: string]: FieldSchema
 }