Making sure the source ID is always set when creating a table - the frontend expects this to be set for every table so making the type represent this correctly.

packages/server/src/api/controllers/datasource.ts

@@ -12,7 +12,6 @@ import {
   CreateDatasourceResponse,
   Datasource,
   DatasourcePlus,
-  ExternalTable,
   FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
   IntegrationBase,
@@ -59,7 +58,7 @@ async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
   const connector = (await getConnector(datasource)) as DatasourcePlus
   return await connector.buildSchema(
     datasource._id!,
-    datasource.entities! as Record<string, ExternalTable>
+    datasource.entities! as Record<string, Table>
   )
 }
 

packages/server/src/api/controllers/table/index.ts

@@ -24,7 +24,6 @@ import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
 import { builderSocket } from "../../../websockets"
 import { cloneDeep, isEqual } from "lodash"
-import { processInternalTable } from "../../../sdk/app/tables/getters"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
@@ -50,7 +49,7 @@ export async function fetch(ctx: UserCtx<void, FetchTablesResponse>) {
       return Object.values(entities).map<Table>((entity: Table) => ({
         ...entity,
         type: "external",
-        sourceId: datasource._id,
+        sourceId: datasource._id!,
         sql: isSQL(datasource),
       }))
     } else {

packages/server/src/api/routes/tests/row.spec.ts

@@ -21,6 +21,7 @@ import {
   SortType,
   StaticQuotaName,
   Table,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import {
   expectAnyExternalColsAttributes,
@@ -65,6 +66,7 @@ describe.each([
       type: "table",
       primary: ["id"],
       primaryDisplay: "name",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         id: {
           type: FieldType.AUTO,
@@ -880,6 +882,7 @@ describe.each([
     async function userTable(): Promise<Table> {
       return {
         name: `users_${generator.word()}`,
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
         type: "table",
         primary: ["id"],
         schema: {
@@ -1062,6 +1065,7 @@ describe.each([
       async function userTable(): Promise<Table> {
         return {
           name: `users_${generator.word()}`,
+          sourceId: INTERNAL_TABLE_SOURCE_ID,
           type: "table",
           primary: ["id"],
           schema: {
@@ -1597,7 +1601,7 @@ describe.each([
       const tableConfig = generateTableConfig()
 
       if (config.datasource) {
-        tableConfig.sourceId = config.datasource._id
+        tableConfig.sourceId = config.datasource._id!
         if (config.datasource.plus) {
           tableConfig.type = "external"
         }

packages/server/src/api/routes/tests/table.spec.ts

@@ -9,6 +9,7 @@ import {
   InternalTable,
   FieldSubtype,
   Row,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
@@ -432,6 +433,7 @@ describe("/tables", () => {
       const table = await config.api.table.create({
         name: "table",
         type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
         schema: {
           "user relationship": {
             type: FieldType.LINK,
@@ -491,6 +493,7 @@ describe("/tables", () => {
       const table = await config.api.table.create({
         name: "table",
         type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
         schema: {
           "user relationship": {
             type: FieldType.LINK,
@@ -552,6 +555,7 @@ describe("/tables", () => {
       const table = await config.api.table.create({
         name: "table",
         type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
         schema: {
           "user relationship": {
             type: FieldType.LINK,

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -10,6 +10,7 @@ import {
   UIFieldMetadata,
   UpdateViewRequest,
   ViewV2,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import { generator } from "@budibase/backend-core/tests"
 import { generateDatasourceID } from "../../../db/utils"
@@ -18,6 +19,7 @@ function priceTable(): Table {
   return {
     name: "table",
     type: "table",
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
     schema: {
       Price: {
         type: FieldType.NUMBER,

packages/server/src/constants/index.ts

@@ -1,5 +1,9 @@
 import { objectStore, roles, constants } from "@budibase/backend-core"
-import { FieldType as FieldTypes } from "@budibase/types"
+import {
+  FieldType as FieldTypes,
+  Table,
+  INTERNAL_TABLE_SOURCE_ID,
+} from "@budibase/types"
 export {
   FieldType as FieldTypes,
   RelationshipType,
@@ -70,9 +74,10 @@ export enum SortDirection {
   DESCENDING = "DESCENDING",
 }
 
-export const USERS_TABLE_SCHEMA = {
+export const USERS_TABLE_SCHEMA: Table = {
   _id: "ta_users",
   type: "table",
+  sourceId: INTERNAL_TABLE_SOURCE_ID,
   views: {},
   name: "Users",
   // TODO: ADMIN PANEL - when implemented this doesn't need to be carried out
@@ -87,12 +92,10 @@ export const USERS_TABLE_SCHEMA = {
         },
         presence: true,
       },
-      fieldName: "email",
       name: "email",
     },
     firstName: {
       name: "firstName",
-      fieldName: "firstName",
       type: FieldTypes.STRING,
       constraints: {
         type: FieldTypes.STRING,
@@ -101,7 +104,6 @@ export const USERS_TABLE_SCHEMA = {
     },
     lastName: {
       name: "lastName",
-      fieldName: "lastName",
       type: FieldTypes.STRING,
       constraints: {
         type: FieldTypes.STRING,
@@ -109,7 +111,6 @@ export const USERS_TABLE_SCHEMA = {
       },
     },
     roleId: {
-      fieldName: "roleId",
       name: "roleId",
       type: FieldTypes.OPTIONS,
       constraints: {
@@ -119,7 +120,6 @@ export const USERS_TABLE_SCHEMA = {
       },
     },
     status: {
-      fieldName: "status",
       name: "status",
       type: FieldTypes.OPTIONS,
       constraints: {

packages/server/src/db/utils.ts

@@ -5,6 +5,7 @@ import {
   FieldSchema,
   RelationshipFieldMetadata,
   VirtualDocumentType,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import { FieldTypes } from "../constants"
 export { DocumentType, VirtualDocumentType } from "@budibase/types"
@@ -18,7 +19,7 @@ export const enum AppStatus {
 }
 
 export const BudibaseInternalDB = {
-  _id: "bb_internal",
+  _id: INTERNAL_TABLE_SOURCE_ID,
   type: dbCore.BUDIBASE_DATASOURCE_TYPE,
   name: "Budibase DB",
   source: "BUDIBASE",

packages/server/src/integrations/googlesheets.ts

@@ -12,7 +12,7 @@ import {
   Row,
   SearchFilters,
   SortJson,
-  ExternalTable,
+  Table,
   TableRequest,
   Schema,
 } from "@budibase/types"
@@ -262,7 +262,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     id?: string
   ) {
     // base table
-    const table: ExternalTable = {
+    const table: Table = {
       name: title,
       primary: [GOOGLE_SHEETS_PRIMARY_KEY],
       schema: {},
@@ -283,7 +283,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
     // not fully configured yet
     if (!this.config.auth) {
@@ -291,7 +291,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     }
     await this.connect()
     const sheets = this.client.sheetsByIndex
-    const tables: Record<string, ExternalTable> = {}
+    const tables: Record<string, Table> = {}
     let errors: Record<string, string> = {}
     await utils.parallelForeach(
       sheets,

packages/server/src/integrations/microsoftSqlServer.ts

@@ -2,7 +2,7 @@ import {
   DatasourceFieldType,
   Integration,
   Operation,
-  ExternalTable,
+  Table,
   TableSchema,
   QueryJson,
   QueryType,
@@ -380,7 +380,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
    */
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
     await this.connect()
     let tableInfo: MSSQLTablesResponse[] = await this.runSQL(this.TABLES_SQL)
@@ -394,7 +394,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
       .map((record: any) => record.TABLE_NAME)
       .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
 
-    const tables: Record<string, ExternalTable> = {}
+    const tables: Record<string, Table> = {}
     for (let tableName of tableNames) {
       // get the column definition (type)
       const definition = await this.runSQL(

packages/server/src/integrations/mysql.ts

@@ -4,7 +4,7 @@ import {
   QueryType,
   QueryJson,
   SqlQuery,
-  ExternalTable,
+  Table,
   TableSchema,
   DatasourcePlus,
   DatasourceFeature,
@@ -278,9 +278,9 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
 
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
-    const tables: { [key: string]: ExternalTable } = {}
+    const tables: { [key: string]: Table } = {}
     await this.connect()
 
     try {

packages/server/src/integrations/oracle.ts

@@ -5,7 +5,7 @@ import {
   QueryJson,
   QueryType,
   SqlQuery,
-  ExternalTable,
+  Table,
   DatasourcePlus,
   DatasourceFeature,
   ConnectionInfo,
@@ -263,14 +263,14 @@ class OracleIntegration extends Sql implements DatasourcePlus {
    */
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
     const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
       sql: this.COLUMNS_SQL,
     })
     const oracleTables = this.mapColumns(columnsResponse)
 
-    const tables: { [key: string]: ExternalTable } = {}
+    const tables: { [key: string]: Table } = {}
 
     // iterate each table
     Object.values(oracleTables).forEach(oracleTable => {

packages/server/src/integrations/postgres.ts

@@ -5,7 +5,7 @@ import {
   QueryType,
   QueryJson,
   SqlQuery,
-  ExternalTable,
+  Table,
   DatasourcePlus,
   DatasourceFeature,
   ConnectionInfo,
@@ -273,7 +273,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
    */
   async buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema> {
     let tableKeys: { [key: string]: string[] } = {}
     await this.openConnection()
@@ -300,7 +300,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
       const columnsResponse: { rows: PostgresColumn[] } =
         await this.client.query(this.COLUMNS_SQL)
 
-      const tables: { [key: string]: ExternalTable } = {}
+      const tables: { [key: string]: Table } = {}
 
       for (let column of columnsResponse.rows) {
         const tableName: string = column.table_name

packages/server/src/integrations/tests/googlesheets.spec.ts

@@ -31,6 +31,7 @@ import { structures } from "@budibase/backend-core/tests"
 import TestConfiguration from "../../tests/utilities/TestConfiguration"
 import GoogleSheetsIntegration from "../googlesheets"
 import { FieldType, Table, TableSchema } from "@budibase/types"
+import { generateDatasourceID } from "../../db/utils"
 
 describe("Google Sheets Integration", () => {
   let integration: any,
@@ -61,6 +62,7 @@ describe("Google Sheets Integration", () => {
   function createBasicTable(name: string, columns: string[]): Table {
     return {
       name,
+      sourceId: generateDatasourceID(),
       schema: {
         ...columns.reduce((p, c) => {
           p[c] = {

packages/server/src/integrations/utils.ts

@@ -4,7 +4,6 @@ import {
   SearchFilters,
   Datasource,
   FieldType,
-  ExternalTable,
 } from "@budibase/types"
 import { DocumentType, SEPARATOR } from "../db/utils"
 import { InvalidColumns, NoEmptyFilterStrings } from "../constants"
@@ -301,9 +300,9 @@ function copyExistingPropsOver(
  * @param entities The old list of tables, if there was any to look for definitions in.
  */
 export function finaliseExternalTables(
-  tables: Record<string, ExternalTable>,
-  entities: Record<string, ExternalTable>
-): Record<string, ExternalTable> {
+  tables: Record<string, Table>,
+  entities: Record<string, Table>
+): Record<string, Table> {
   let finalTables: Record<string, Table> = {}
   const tableIds = Object.values(tables).map(table => table._id!)
   for (let [name, table] of Object.entries(tables)) {
@@ -316,7 +315,7 @@ export function finaliseExternalTables(
 }
 
 export function checkExternalTables(
-  tables: Record<string, ExternalTable>
+  tables: Record<string, Table>
 ): Record<string, string> {
   const invalidColumns = Object.values(InvalidColumns) as string[]
   const errors: Record<string, string> = {}

packages/server/src/middleware/tests/trimViewRowInfo.spec.ts

@@ -1,5 +1,11 @@
 import { generator } from "@budibase/backend-core/tests"
-import { BBRequest, FieldType, Row, Table } from "@budibase/types"
+import {
+  BBRequest,
+  FieldType,
+  Row,
+  Table,
+  INTERNAL_TABLE_SOURCE_ID,
+} from "@budibase/types"
 import * as utils from "../../db/utils"
 import trimViewRowInfoMiddleware from "../trimViewRowInfo"
 
@@ -73,6 +79,7 @@ describe("trimViewRowInfo middleware", () => {
   const table: Table = {
     _id: tableId,
     name: generator.word(),
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
     type: "table",
     schema: {
       name: {

packages/server/src/sdk/app/rows/search/internal.ts

@@ -197,11 +197,7 @@ export async function fetchView(
     try {
       table = await sdk.tables.getTable(viewInfo.meta.tableId)
     } catch (err) {
-      /* istanbul ignore next */
-      table = {
-        name: "",
-        schema: {},
-      }
+      throw new Error("Unable to retrieve view table.")
     }
     rows = await outputProcessing(table, response.rows)
   }

packages/server/src/sdk/app/rows/search/tests/external.spec.ts

@@ -15,6 +15,7 @@ import {
   expectAnyExternalColsAttributes,
   generator,
 } from "@budibase/backend-core/tests"
+import datasource from "../../../../../api/routes/datasource"
 
 jest.unmock("mysql2/promise")
 
@@ -23,36 +24,7 @@ jest.setTimeout(30000)
 describe.skip("external", () => {
   const config = new TestConfiguration()
 
-  let externalDatasource: Datasource
-
-  const tableData: Table = {
-    name: generator.word(),
-    type: "external",
-    primary: ["id"],
-    schema: {
-      id: {
-        name: "id",
-        type: FieldType.AUTO,
-        autocolumn: true,
-      },
-      name: {
-        name: "name",
-        type: FieldType.STRING,
-      },
-      surname: {
-        name: "surname",
-        type: FieldType.STRING,
-      },
-      age: {
-        name: "age",
-        type: FieldType.NUMBER,
-      },
-      address: {
-        name: "address",
-        type: FieldType.STRING,
-      },
-    },
-  }
+  let externalDatasource: Datasource, tableData: Table
 
   beforeAll(async () => {
     const container = await new GenericContainer("mysql")
@@ -84,6 +56,36 @@ describe.skip("external", () => {
         },
       },
     })
+
+    tableData = {
+      name: generator.word(),
+      type: "external",
+      primary: ["id"],
+      sourceId: externalDatasource._id!,
+      schema: {
+        id: {
+          name: "id",
+          type: FieldType.AUTO,
+          autocolumn: true,
+        },
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+        },
+        surname: {
+          name: "surname",
+          type: FieldType.STRING,
+        },
+        age: {
+          name: "age",
+          type: FieldType.NUMBER,
+        },
+        address: {
+          name: "address",
+          type: FieldType.STRING,
+        },
+      },
+    }
   })
 
   describe("search", () => {

packages/server/src/sdk/app/rows/search/tests/internal.spec.ts

@@ -1,4 +1,10 @@
-import { FieldType, Row, Table, SearchParams } from "@budibase/types"
+import {
+  FieldType,
+  Row,
+  Table,
+  SearchParams,
+  INTERNAL_TABLE_SOURCE_ID,
+} from "@budibase/types"
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
 import { search } from "../internal"
 import {
@@ -12,6 +18,7 @@ describe("internal", () => {
   const tableData: Table = {
     name: generator.word(),
     type: "table",
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
     schema: {
       name: {
         name: "name",

packages/server/src/sdk/app/rows/search/tests/utils.spec.ts

@@ -5,12 +5,14 @@ import {
   FieldTypeSubtypes,
   Table,
   SearchParams,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 
 const tableId = "ta_a"
 const tableWithUserCol: Table = {
   _id: tableId,
   name: "table",
+  sourceId: INTERNAL_TABLE_SOURCE_ID,
   schema: {
     user: {
       name: "user",
@@ -23,6 +25,7 @@ const tableWithUserCol: Table = {
 const tableWithUsersCol: Table = {
   _id: tableId,
   name: "table",
+  sourceId: INTERNAL_TABLE_SOURCE_ID,
   schema: {
     user: {
       name: "user",

packages/server/src/sdk/app/tables/external/index.ts

@@ -35,10 +35,10 @@ export async function save(
   opts?: { tableId?: string; renaming?: RenameColumn }
 ) {
   let tableToSave: TableRequest = {
+    ...update,
     type: "table",
     _id: buildExternalTableId(datasourceId, update.name),
     sourceId: datasourceId,
-    ...update,
   }
 
   const tableId = opts?.tableId || update._id

packages/server/src/sdk/app/tables/external/utils.ts

@@ -76,12 +76,14 @@ export function generateManyLinkSchema(
   const primary = table.name + table.primary[0]
   const relatedPrimary = relatedTable.name + relatedTable.primary[0]
   const jcTblName = generateJunctionTableName(column, table, relatedTable)
+  const datasourceId = datasource._id!
   // first create the new table
   const junctionTable = {
-    _id: buildExternalTableId(datasource._id!, jcTblName),
+    _id: buildExternalTableId(datasourceId, jcTblName),
     name: jcTblName,
     primary: [primary, relatedPrimary],
     constrained: [primary, relatedPrimary],
+    sourceId: datasourceId,
     schema: {
       [primary]: foreignKeyStructure(primary, {
         toTable: table.name,

packages/server/src/sdk/app/tables/getters.ts

@@ -1,20 +1,16 @@
 import { context } from "@budibase/backend-core"
-import {
-  BudibaseInternalDB,
-  getMultiIDParams,
-  getTableParams,
-} from "../../../db/utils"
+import { getMultiIDParams, getTableParams } from "../../../db/utils"
 import {
   breakExternalTableId,
   isExternalTable,
   isSQL,
 } from "../../../integrations/utils"
 import {
-  AllDocsResponse,
   Database,
   Table,
   TableResponse,
   TableViewsResponse,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import datasources from "../datasources"
 import sdk from "../../../sdk"
@@ -27,7 +23,7 @@ export function processInternalTable(table: Table): Table {
   return {
     ...table,
     type: "internal",
-    sourceId: table.sourceId || BudibaseInternalDB._id,
+    sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID,
   }
 }
 

packages/server/src/sdk/app/tables/tests/tables.spec.ts

@@ -1,4 +1,9 @@
-import { FieldType, Table, ViewV2 } from "@budibase/types"
+import {
+  FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
+  Table,
+  ViewV2,
+} from "@budibase/types"
 import { generator } from "@budibase/backend-core/tests"
 import sdk from "../../.."
 
@@ -13,6 +18,7 @@ describe("table sdk", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,

packages/server/src/sdk/app/tables/tests/validation.spec.ts

@@ -1,36 +1,50 @@
 import { populateExternalTableSchemas } from "../validation"
 import { cloneDeep } from "lodash/fp"
-import { AutoReason, Datasource, Table } from "@budibase/types"
+import {
+  AutoReason,
+  Datasource,
+  FieldType,
+  RelationshipType,
+  SourceName,
+  Table,
+} from "@budibase/types"
 import { isEqual } from "lodash"
+import { generateDatasourceID } from "../../../../db/utils"
 
-const SCHEMA = {
+const datasourceId = generateDatasourceID()
+
+const SCHEMA: Datasource = {
+  source: SourceName.POSTGRES,
+  type: "datasource",
+  _id: datasourceId,
   entities: {
     client: {
       _id: "tableA",
       name: "client",
       primary: ["idC"],
       primaryDisplay: "Name",
+      sourceId: datasourceId,
       schema: {
         idC: {
           autocolumn: true,
           externalType: "int unsigned",
           name: "idC",
-          type: "number",
+          type: FieldType.NUMBER,
         },
         Name: {
           autocolumn: false,
           externalType: "varchar(255)",
           name: "Name",
-          type: "string",
+          type: FieldType.STRING,
         },
         project: {
           fieldName: "idC",
           foreignKey: "idC",
           main: true,
           name: "project",
-          relationshipType: "many-to-one",
+          relationshipType: RelationshipType.MANY_TO_ONE,
           tableId: "tableB",
-          type: "link",
+          type: FieldType.LINK,
         },
       },
     },
@@ -39,31 +53,32 @@ const SCHEMA = {
       name: "project",
       primary: ["idP"],
       primaryDisplay: "Name",
+      sourceId: datasourceId,
       schema: {
         idC: {
           externalType: "int unsigned",
           name: "idC",
-          type: "number",
+          type: FieldType.NUMBER,
         },
         idP: {
           autocolumn: true,
           externalType: "int unsigned",
           name: "idProject",
-          type: "number",
+          type: FieldType.NUMBER,
         },
         Name: {
           autocolumn: false,
           externalType: "varchar(255)",
           name: "Name",
-          type: "string",
+          type: FieldType.STRING,
         },
         client: {
           fieldName: "idC",
           foreignKey: "idC",
           name: "client",
-          relationshipType: "one-to-many",
+          relationshipType: RelationshipType.ONE_TO_MANY,
           tableId: "tableA",
-          type: "link",
+          type: FieldType.LINK,
         },
       },
       sql: true,
@@ -95,12 +110,12 @@ describe("validation and update of external table schemas", () => {
   function noOtherTableChanges(response: any) {
     checkOtherColumns(
       response.entities!.client!,
-      SCHEMA.entities.client as Table,
+      SCHEMA.entities!.client,
       OTHER_CLIENT_COLS
     )
     checkOtherColumns(
       response.entities!.project!,
-      SCHEMA.entities.project as Table,
+      SCHEMA.entities!.project,
       OTHER_PROJECT_COLS
     )
   }

packages/server/src/sdk/app/views/tests/views.spec.ts

@@ -2,6 +2,7 @@ import _ from "lodash"
 import {
   FieldSchema,
   FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
   Table,
   TableSchema,
   ViewV2,
@@ -14,6 +15,7 @@ describe("table sdk", () => {
     _id: generator.guid(),
     name: "TestTable",
     type: "table",
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
     schema: {
       name: {
         type: FieldType.STRING,

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -56,6 +56,7 @@ import {
   CreateViewRequest,
   RelationshipFieldMetadata,
   User,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 
 import API from "./api"
@@ -68,6 +69,10 @@ type DefaultUserValues = {
   csrfToken: string
 }
 
+interface TableToBuild extends Omit<Table, "sourceId"> {
+  sourceId?: string
+}
+
 class TestConfiguration {
   server: any
   request: supertest.SuperTest<supertest.Test> | undefined
@@ -538,10 +543,13 @@ class TestConfiguration {
   // TABLE
 
   async updateTable(
-    config?: Table,
+    config?: TableToBuild,
     { skipReassigning } = { skipReassigning: false }
   ): Promise<Table> {
     config = config || basicTable()
+    if (!config.sourceId) {
+      config.sourceId = INTERNAL_TABLE_SOURCE_ID
+    }
     const response = await this._req(config, null, controllers.table.save)
     if (!skipReassigning) {
       this.table = response
@@ -549,13 +557,19 @@ class TestConfiguration {
     return response
   }
 
-  async createTable(config?: Table, options = { skipReassigning: false }) {
+  async createTable(
+    config?: TableToBuild,
+    options = { skipReassigning: false }
+  ) {
     if (config != null && config._id) {
       delete config._id
     }
     config = config || basicTable()
+    if (!config.sourceId) {
+      config.sourceId = INTERNAL_TABLE_SOURCE_ID
+    }
     if (this.datasource && !config.sourceId) {
-      config.sourceId = this.datasource._id
+      config.sourceId = this.datasource._id || INTERNAL_TABLE_SOURCE_ID
       if (this.datasource.plus) {
         config.type = "external"
       }
@@ -572,12 +586,15 @@ class TestConfiguration {
   async createLinkedTable(
     relationshipType = RelationshipType.ONE_TO_MANY,
     links: any = ["link"],
-    config?: Table
+    config?: TableToBuild
   ) {
     if (!this.table) {
       throw "Must have created a table first."
     }
     const tableConfig = config || basicTable()
+    if (!tableConfig.sourceId) {
+      tableConfig.sourceId = INTERNAL_TABLE_SOURCE_ID
+    }
     tableConfig.primaryDisplay = "name"
     for (let link of links) {
       tableConfig.schema[link] = {
@@ -590,7 +607,7 @@ class TestConfiguration {
     }
 
     if (this.datasource && !tableConfig.sourceId) {
-      tableConfig.sourceId = this.datasource._id
+      tableConfig.sourceId = this.datasource._id || INTERNAL_TABLE_SOURCE_ID
       if (this.datasource.plus) {
         tableConfig.type = "external"
       }

packages/server/src/tests/utilities/structures.ts

@@ -19,12 +19,14 @@ import {
   FieldType,
   SourceName,
   Table,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 
 export function basicTable(): Table {
   return {
     name: "TestTable",
     type: "table",
+    sourceId: INTERNAL_TABLE_SOURCE_ID,
     schema: {
       name: {
         type: FieldType.STRING,

packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts

@@ -1,6 +1,11 @@
 import { inputProcessing } from ".."
 import { generator, structures } from "@budibase/backend-core/tests"
-import { FieldType, FieldTypeSubtypes, Table } from "@budibase/types"
+import {
+  FieldType,
+  FieldTypeSubtypes,
+  INTERNAL_TABLE_SOURCE_ID,
+  Table,
+} from "@budibase/types"
 import * as bbReferenceProcessor from "../bbReferenceProcessor"
 
 jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
@@ -20,6 +25,7 @@ describe("rowProcessor - inputProcessing", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,
@@ -70,6 +76,7 @@ describe("rowProcessor - inputProcessing", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,
@@ -110,6 +117,7 @@ describe("rowProcessor - inputProcessing", () => {
         _id: generator.guid(),
         name: "TestTable",
         type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
         schema: {
           name: {
             type: FieldType.STRING,
@@ -150,6 +158,7 @@ describe("rowProcessor - inputProcessing", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,

packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts

@@ -2,6 +2,7 @@ import {
   FieldSubtype,
   FieldType,
   FieldTypeSubtypes,
+  INTERNAL_TABLE_SOURCE_ID,
   Table,
 } from "@budibase/types"
 import { outputProcessing } from ".."
@@ -26,6 +27,7 @@ describe("rowProcessor - outputProcessing", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,
@@ -71,6 +73,7 @@ describe("rowProcessor - outputProcessing", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,
@@ -108,6 +111,7 @@ describe("rowProcessor - outputProcessing", () => {
       _id: generator.guid(),
       name: "TestTable",
       type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
       schema: {
         name: {
           type: FieldType.STRING,

packages/server/src/websockets/builder.ts

@@ -17,7 +17,7 @@ import { clearLock, updateLock } from "../utilities/redis"
 import { Socket } from "socket.io"
 import { BuilderSocketEvent } from "@budibase/shared-core"
 import { processInternalTable } from "../sdk/app/tables/getters"
-import { isExternalTable, isInternalTable } from "../integrations/utils"
+import { isInternalTable } from "../integrations/utils"
 
 export default class BuilderSocket extends BaseSocket {
   constructor(app: Koa, server: http.Server) {

packages/types/src/documents/app/table/table.ts

@@ -3,14 +3,16 @@ import { View, ViewV2 } from "../view"
 import { RenameColumn } from "../../../sdk"
 import { TableSchema } from "./schema"
 
+export const INTERNAL_TABLE_SOURCE_ID = "bb_internal"
+
 export interface Table extends Document {
   type?: string
   views?: { [key: string]: View | ViewV2 }
   name: string
+  sourceId: string
   primary?: string[]
   schema: TableSchema
   primaryDisplay?: string
-  sourceId?: string
   relatedFormula?: string[]
   constrained?: string[]
   sql?: boolean
@@ -19,10 +21,6 @@ export interface Table extends Document {
   rowHeight?: number
 }
 
-export interface ExternalTable extends Table {
-  sourceId: string
-}
-
 export interface TableRequest extends Table {
   _rename?: RenameColumn
   created?: boolean

packages/types/src/sdk/datasources.ts

@@ -1,4 +1,4 @@
-import { ExternalTable, Table } from "../documents"
+import { Table } from "../documents"
 
 export const PASSWORD_REPLACEMENT = "--secret-value--"
 
@@ -176,7 +176,7 @@ export interface IntegrationBase {
 }
 
 export interface Schema {
-  tables: Record<string, ExternalTable>
+  tables: Record<string, Table>
   errors: Record<string, string>
 }
 
@@ -187,7 +187,7 @@ export interface DatasourcePlus extends IntegrationBase {
   getStringConcat(parts: string[]): string
   buildSchema(
     datasourceId: string,
-    entities: Record<string, ExternalTable>
+    entities: Record<string, Table>
   ): Promise<Schema>
   getTableNames(): Promise<string[]>
 }