MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding external SDK, updating external controllers.

This commit is contained in:
mike12345567 2023-10-18 18:58:19 +01:00
parent 6bfa9d407b
commit 8d1163e50d
5 changed files with 107 additions and 445 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

368
packages/server/src/api/controllers/table/external.ts
View File

@ -1,108 +1,20 @@
import { import { breakExternalTableId } from "../../../integrations/utils"
breakExternalTableId,
buildExternalTableId,
} from "../../../integrations/utils"
import {
foreignKeyStructure,
generateForeignKey,
generateJunctionTableName,
hasTypeChanged,
setStaticSchemas,
} from "./utils"
import { FieldTypes } from "../../../constants"
import { makeExternalQuery } from "../../../integrations/base/query"
import { handleRequest } from "../row/external" import { handleRequest } from "../row/external"
import { context, events } from "@budibase/backend-core" import { events } from "@budibase/backend-core"
import { isRows, isSchema, parse } from "../../../utilities/schema" import { isRows, isSchema, parse } from "../../../utilities/schema"
import { import {
BulkImportRequest, BulkImportRequest,
BulkImportResponse, BulkImportResponse,
Datasource,
FieldSchema,
ManyToManyRelationshipFieldMetadata,
ManyToOneRelationshipFieldMetadata,
OneToManyRelationshipFieldMetadata,
Operation, Operation,
QueryJson,
RelationshipFieldMetadata,
RelationshipType,
RenameColumn,
SaveTableRequest, SaveTableRequest,
SaveTableResponse, SaveTableResponse,
Table, Table,
TableRequest, TableRequest,
UserCtx, UserCtx,
ViewV2,
} from "@budibase/types" } from "@budibase/types"
import sdk from "../../../sdk" import sdk from "../../../sdk"
import { builderSocket } from "../../../websockets" import { builderSocket } from "../../../websockets"
const { cloneDeep } = require("lodash/fp")
async function makeTableRequest(
datasource: Datasource,
operation: Operation,
table: Table,
tables: Record<string, Table>,
oldTable?: Table,
renamed?: RenameColumn
) {
const json: QueryJson = {
endpoint: {
datasourceId: datasource._id!,
entityId: table._id!,
operation,
},
meta: {
tables,
},
table,
}
if (oldTable) {
json.meta!.table = oldTable
}
if (renamed) {
json.meta!.renamed = renamed
}
return makeExternalQuery(datasource, json)
}
function cleanupRelationships(
table: Table,
tables: Record<string, Table>,
oldTable?: Table
) {
const tableToIterate = oldTable ? oldTable : table
// clean up relationships in couch table schemas
for (let [key, schema] of Object.entries(tableToIterate.schema)) {
if (
schema.type === FieldTypes.LINK &&
(!oldTable || table.schema[key] == null)
) {
const schemaTableId = schema.tableId
const relatedTable = Object.values(tables).find(
table => table._id === schemaTableId
)
const foreignKey =
schema.relationshipType !== RelationshipType.MANY_TO_MANY &&
schema.foreignKey
if (!relatedTable || !foreignKey) {
continue
}
for (let [relatedKey, relatedSchema] of Object.entries(
relatedTable.schema
)) {
if (
relatedSchema.type === FieldTypes.LINK &&
relatedSchema.fieldName === foreignKey
) {
delete relatedTable.schema[relatedKey]
}
}
}
}
}
function getDatasourceId(table: Table) { function getDatasourceId(table: Table) {
if (!table) { if (!table) {
throw "No table supplied" throw "No table supplied"
@ -113,247 +25,32 @@ function getDatasourceId(table: Table) {
return breakExternalTableId(table._id).datasourceId return breakExternalTableId(table._id).datasourceId
} }
function otherRelationshipType(type?: string) {
if (type === RelationshipType.MANY_TO_MANY) {
return RelationshipType.MANY_TO_MANY
}
return type === RelationshipType.ONE_TO_MANY
? RelationshipType.MANY_TO_ONE
: RelationshipType.ONE_TO_MANY
}
function generateManyLinkSchema(
datasource: Datasource,
column: ManyToManyRelationshipFieldMetadata,
table: Table,
relatedTable: Table
): Table {
if (!table.primary || !relatedTable.primary) {
throw new Error("Unable to generate many link schema, no primary keys")
}
const primary = table.name + table.primary[0]
const relatedPrimary = relatedTable.name + relatedTable.primary[0]
const jcTblName = generateJunctionTableName(column, table, relatedTable)
// first create the new table
const junctionTable = {
_id: buildExternalTableId(datasource._id!, jcTblName),
name: jcTblName,
primary: [primary, relatedPrimary],
constrained: [primary, relatedPrimary],
schema: {
[primary]: foreignKeyStructure(primary, {
toTable: table.name,
toKey: table.primary[0],
}),
[relatedPrimary]: foreignKeyStructure(relatedPrimary, {
toTable: relatedTable.name,
toKey: relatedTable.primary[0],
}),
},
}
column.through = junctionTable._id
column.throughFrom = relatedPrimary
column.throughTo = primary
column.fieldName = relatedPrimary
return junctionTable
}
function generateLinkSchema(
column:
| OneToManyRelationshipFieldMetadata
| ManyToOneRelationshipFieldMetadata,
table: Table,
relatedTable: Table,
type: RelationshipType.ONE_TO_MANY | RelationshipType.MANY_TO_ONE
) {
if (!table.primary || !relatedTable.primary) {
throw new Error("Unable to generate link schema, no primary keys")
}
const isOneSide = type === RelationshipType.ONE_TO_MANY
const primary = isOneSide ? relatedTable.primary[0] : table.primary[0]
// generate a foreign key
const foreignKey = generateForeignKey(column, relatedTable)
column.relationshipType = type
column.foreignKey = isOneSide ? foreignKey : primary
column.fieldName = isOneSide ? primary : foreignKey
return foreignKey
}
function generateRelatedSchema(
linkColumn: RelationshipFieldMetadata,
table: Table,
relatedTable: Table,
columnName: string
) {
// generate column for other table
const relatedSchema = cloneDeep(linkColumn)
const isMany2Many =
linkColumn.relationshipType === RelationshipType.MANY_TO_MANY
// swap them from the main link
if (!isMany2Many && linkColumn.foreignKey) {
relatedSchema.fieldName = linkColumn.foreignKey
relatedSchema.foreignKey = linkColumn.fieldName
}
// is many to many
else if (isMany2Many) {
// don't need to copy through, already got it
relatedSchema.fieldName = linkColumn.throughTo
relatedSchema.throughTo = linkColumn.throughFrom
relatedSchema.throughFrom = linkColumn.throughTo
}
relatedSchema.relationshipType = otherRelationshipType(
linkColumn.relationshipType
)
relatedSchema.tableId = relatedTable._id
relatedSchema.name = columnName
table.schema[columnName] = relatedSchema
}
function isRelationshipSetup(column: RelationshipFieldMetadata) {
return (column as any).foreignKey || (column as any).through
}
export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) { export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
const inputs = ctx.request.body const inputs = ctx.request.body
const renamed = inputs?._rename const renaming = inputs?._rename
// can't do this right now // can't do this right now
delete inputs.rows delete inputs.rows
const datasourceId = getDatasourceId(ctx.request.body)! const tableId = ctx.request.body._id
const datasourceId = getDatasourceId(ctx.request.body)
// table doesn't exist already, note that it is created // table doesn't exist already, note that it is created
if (!inputs._id) { if (!inputs._id) {
inputs.created = true inputs.created = true
} }
let tableToSave: TableRequest = { try {
type: "table", const { datasource, table } = await sdk.tables.external.save(
_id: buildExternalTableId(datasourceId, inputs.name), datasourceId!,
sourceId: datasourceId, inputs,
...inputs, { tableId, renaming }
}
let oldTable: Table | undefined
if (ctx.request.body && ctx.request.body._id) {
oldTable = await sdk.tables.getTable(ctx.request.body._id)
}
if (hasTypeChanged(tableToSave, oldTable)) {
ctx.throw(400, "A column type has changed.")
}
for (let view in tableToSave.views) {
const tableView = tableToSave.views[view]
if (!tableView || !sdk.views.isV2(tableView)) continue
tableToSave.views[view] = sdk.views.syncSchema(
oldTable!.views![view] as ViewV2,
tableToSave.schema,
renamed
) )
} builderSocket?.emitDatasourceUpdate(ctx, datasource)
return table
const db = context.getAppDB() } catch (err: any) {
const datasource = await sdk.datasources.get(datasourceId) if (err instanceof Error) {
if (!datasource.entities) { ctx.throw(400, err.message)
datasource.entities = {}
}
// GSheets is a specific case - only ever has a static primary key
tableToSave = setStaticSchemas(datasource, tableToSave)
const oldTables = cloneDeep(datasource.entities)
const tables: Record<string, Table> = datasource.entities
const extraTablesToUpdate = []
// check if relations need setup
for (let schema of Object.values(tableToSave.schema)) {
if (schema.type !== FieldTypes.LINK || isRelationshipSetup(schema)) {
continue
}
const schemaTableId = schema.tableId
const relatedTable = Object.values(tables).find(
table => table._id === schemaTableId
)
if (!relatedTable) {
continue
}
const relatedColumnName = schema.fieldName!
const relationType = schema.relationshipType
if (relationType === RelationshipType.MANY_TO_MANY) {
const junctionTable = generateManyLinkSchema(
datasource,
schema,
tableToSave,
relatedTable
)
if (tables[junctionTable.name]) {
throw "Junction table already exists, cannot create another relationship."
}
tables[junctionTable.name] = junctionTable
extraTablesToUpdate.push(junctionTable)
} else { } else {
const fkTable = ctx.throw(err.status || 500, err?.message || err)
relationType === RelationshipType.ONE_TO_MANY
? tableToSave
: relatedTable
const foreignKey = generateLinkSchema(
schema,
tableToSave,
relatedTable,
relationType
)
fkTable.schema[foreignKey] = foreignKeyStructure(foreignKey)
if (fkTable.constrained == null) {
fkTable.constrained = []
}
if (fkTable.constrained.indexOf(foreignKey) === -1) {
fkTable.constrained.push(foreignKey)
}
// foreign key is in other table, need to save it to external
if (fkTable._id !== tableToSave._id) {
extraTablesToUpdate.push(fkTable)
}
} }
generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName)
schema.main = true
} }
cleanupRelationships(tableToSave, tables, oldTable)
const operation = oldTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE
await makeTableRequest(
datasource,
operation,
tableToSave,
tables,
oldTable,
renamed
)
// update any extra tables (like foreign keys in other tables)
for (let extraTable of extraTablesToUpdate) {
const oldExtraTable = oldTables[extraTable.name]
let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE
await makeTableRequest(datasource, op, extraTable, tables, oldExtraTable)
}
// make sure the constrained list, all still exist
if (Array.isArray(tableToSave.constrained)) {
tableToSave.constrained = tableToSave.constrained.filter(constraint =>
Object.keys(tableToSave.schema).includes(constraint)
)
}
// remove the rename prop
delete tableToSave._rename
// store it into couch now for budibase reference
datasource.entities[tableToSave.name] = tableToSave
await db.put(sdk.tables.populateExternalTableSchemas(datasource))
// Since tables are stored inside datasources, we need to notify clients
// that the datasource definition changed
const updatedDatasource = await sdk.datasources.get(datasource._id!)
builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource)
return tableToSave
} }
export async function destroy(ctx: UserCtx) { export async function destroy(ctx: UserCtx) {
@ -364,27 +61,20 @@ export async function destroy(ctx: UserCtx) {
ctx.throw(400, "Cannot delete tables which weren't created in Budibase.") ctx.throw(400, "Cannot delete tables which weren't created in Budibase.")
} }
const datasourceId = getDatasourceId(tableToDelete) const datasourceId = getDatasourceId(tableToDelete)
try {
const db = context.getAppDB() const { datasource, table } = await sdk.tables.external.destroy(
const datasource = await sdk.datasources.get(datasourceId!) datasourceId!,
const tables = datasource.entities tableToDelete
)
const operation = Operation.DELETE_TABLE builderSocket?.emitDatasourceUpdate(ctx, datasource)
if (tables) { return table
await makeTableRequest(datasource, operation, tableToDelete, tables) } catch (err: any) {
cleanupRelationships(tableToDelete, tables) if (err instanceof Error) {
delete tables[tableToDelete.name] ctx.throw(400, err.message)
datasource.entities = tables } else {
ctx.throw(err.status || 500, err.message || err)
}
} }
await db.put(sdk.tables.populateExternalTableSchemas(datasource))
// Since tables are stored inside datasources, we need to notify clients
// that the datasource definition changed
const updatedDatasource = await sdk.datasources.get(datasource._id!)
builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource)
return tableToDelete
} }
export async function bulkImport( export async function bulkImport(

51
packages/server/src/sdk/app/tables/external/index.ts vendored
View File

@ -7,25 +7,29 @@ import {
ViewV2, ViewV2,
} from "@budibase/types" } from "@budibase/types"
import { context } from "@budibase/backend-core" import { context } from "@budibase/backend-core"
import { buildExternalTableId } from "../../../integrations/utils" import { buildExternalTableId } from "../../../../integrations/utils"
import sdk from "../../index"
import { import {
foreignKeyStructure, foreignKeyStructure,
hasTypeChanged, hasTypeChanged,
setStaticSchemas, setStaticSchemas,
} from "../../../api/controllers/table/utils" } from "../../../../api/controllers/table/utils"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import { FieldTypes } from "../../../constants" import { FieldTypes } from "../../../../constants"
import { makeTableRequest } from "../../../api/controllers/table/ExternalRequest" import { makeTableRequest } from "../../../../api/controllers/table/ExternalRequest"
import { import {
isRelationshipSetup, isRelationshipSetup,
cleanupRelationships, cleanupRelationships,
generateLinkSchema, generateLinkSchema,
generateManyLinkSchema, generateManyLinkSchema,
generateRelatedSchema, generateRelatedSchema,
} from "./externalUtils" } from "./utils"
export async function externalSave( import { getTable } from "../getters"
import { populateExternalTableSchemas } from "../validation"
import datasourceSdk from "../../datasources"
import * as viewSdk from "../../views"
export async function save(
datasourceId: string, datasourceId: string,
update: Table, update: Table,
opts?: { tableId?: string; renaming?: RenameColumn } opts?: { tableId?: string; renaming?: RenameColumn }
@ -39,7 +43,7 @@ export async function externalSave(
let oldTable: Table | undefined let oldTable: Table | undefined
if (opts?.tableId) { if (opts?.tableId) {
oldTable = await sdk.tables.getTable(opts.tableId) oldTable = await getTable(opts.tableId)
} }
if (hasTypeChanged(tableToSave, oldTable)) { if (hasTypeChanged(tableToSave, oldTable)) {
@ -48,9 +52,9 @@ export async function externalSave(
for (let view in tableToSave.views) { for (let view in tableToSave.views) {
const tableView = tableToSave.views[view] const tableView = tableToSave.views[view]
if (!tableView || !sdk.views.isV2(tableView)) continue if (!tableView || !viewSdk.isV2(tableView)) continue
tableToSave.views[view] = sdk.views.syncSchema( tableToSave.views[view] = viewSdk.syncSchema(
oldTable!.views![view] as ViewV2, oldTable!.views![view] as ViewV2,
tableToSave.schema, tableToSave.schema,
opts?.renaming opts?.renaming
@ -58,7 +62,7 @@ export async function externalSave(
} }
const db = context.getAppDB() const db = context.getAppDB()
const datasource = await sdk.datasources.get(datasourceId) const datasource = await datasourceSdk.get(datasourceId)
if (!datasource.entities) { if (!datasource.entities) {
datasource.entities = {} datasource.entities = {}
} }
@ -155,11 +159,32 @@ export async function externalSave(
delete tableToSave._rename delete tableToSave._rename
// store it into couch now for budibase reference // store it into couch now for budibase reference
datasource.entities[tableToSave.name] = tableToSave datasource.entities[tableToSave.name] = tableToSave
await db.put(sdk.tables.populateExternalTableSchemas(datasource)) await db.put(populateExternalTableSchemas(datasource))
// Since tables are stored inside datasources, we need to notify clients // Since tables are stored inside datasources, we need to notify clients
// that the datasource definition changed // that the datasource definition changed
const updatedDatasource = await sdk.datasources.get(datasource._id!) const updatedDatasource = await datasourceSdk.get(datasource._id!)
return { datasource: updatedDatasource, table: tableToSave } return { datasource: updatedDatasource, table: tableToSave }
} }
export async function destroy(datasourceId: string, table: Table) {
const db = context.getAppDB()
const datasource = await datasourceSdk.get(datasourceId)
const tables = datasource.entities
const operation = Operation.DELETE_TABLE
if (tables) {
await makeTableRequest(datasource, operation, table, tables)
cleanupRelationships(table, tables)
delete tables[table.name]
datasource.entities = tables
}
await db.put(populateExternalTableSchemas(datasource))
// Since tables are stored inside datasources, we need to notify clients
// that the datasource definition changed
const updatedDatasource = await datasourceSdk.get(datasource._id!)
return { datasource: updatedDatasource, table }
}

6
packages/server/src/sdk/app/tables/external/utils.ts vendored
View File

@ -7,13 +7,13 @@ import {
RelationshipType, RelationshipType,
Table, Table,
} from "@budibase/types" } from "@budibase/types"
import { FieldTypes } from "../../../constants" import { FieldTypes } from "../../../../constants"
import { import {
foreignKeyStructure, foreignKeyStructure,
generateForeignKey, generateForeignKey,
generateJunctionTableName, generateJunctionTableName,
} from "../../../api/controllers/table/utils" } from "../../../../api/controllers/table/utils"
import { buildExternalTableId } from "../../../integrations/utils" import { buildExternalTableId } from "../../../../integrations/utils"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
export function cleanupRelationships( export function cleanupRelationships(

96
packages/server/src/sdk/app/tables/index.ts
View File

@ -1,95 +1,11 @@
import { context } from "@budibase/backend-core"
import { BudibaseInternalDB, getTableParams } from "../../../db/utils"
import {
breakExternalTableId,
isExternalTable,
isSQL,
} from "../../../integrations/utils"
import {
Database,
Table,
TableResponse,
TableViewsResponse,
} from "@budibase/types"
import datasources from "../datasources"
import { populateExternalTableSchemas } from "./validation" import { populateExternalTableSchemas } from "./validation"
import sdk from "../../../sdk" import * as getters from "./getters"
import * as updates from "./update"
async function getAllInternalTables(db?: Database): Promise<Table[]> { import * as utils from "./utils"
if (!db) {
db = context.getAppDB()
}
const internalTables = await db.allDocs(
getTableParams(null, {
include_docs: true,
})
)
return internalTables.rows.map((tableDoc: any) => ({
...tableDoc.doc,
type: "internal",
sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id,
}))
}
async function getAllExternalTables(
datasourceId: any
): Promise<Record<string, Table>> {
const datasource = await datasources.get(datasourceId, { enriched: true })
if (!datasource || !datasource.entities) {
throw "Datasource is not configured fully."
}
return datasource.entities
}
async function getExternalTable(
datasourceId: any,
tableName: any
): Promise<Table> {
const entities = await getAllExternalTables(datasourceId)
return entities[tableName]
}
async function getTable(tableId: any): Promise<Table> {
const db = context.getAppDB()
if (isExternalTable(tableId)) {
let { datasourceId, tableName } = breakExternalTableId(tableId)
const datasource = await datasources.get(datasourceId!)
const table = await getExternalTable(datasourceId, tableName)
return { ...table, sql: isSQL(datasource) }
} else {
return db.get(tableId)
}
}
function enrichViewSchemas(table: Table): TableResponse {
return {
...table,
views: Object.values(table.views ?? [])
.map(v => sdk.views.enrichSchema(v, table.schema))
.reduce((p, v) => {
p[v.name] = v
return p
}, {} as TableViewsResponse),
}
}
async function saveTable(table: Table) {
const db = context.getAppDB()
if (isExternalTable(table._id!)) {
const datasource = await sdk.datasources.get(table.sourceId!)
datasource.entities![table.name] = table
await db.put(datasource)
} else {
await db.put(table)
}
}
export default { export default {
getAllInternalTables,
getAllExternalTables,
getExternalTable,
getTable,
populateExternalTableSchemas, populateExternalTableSchemas,
enrichViewSchemas, ...updates,
saveTable, ...getters,
...utils,
} }

31
packages/server/src/sdk/app/tables/update.ts
View File

@ -0,0 +1,31 @@
import { Table, RenameColumn } from "@budibase/types"
import { isExternalTable } from "../../../integrations/utils"
import sdk from "../../index"
import { context } from "@budibase/backend-core"
import { isExternal } from "./utils"
import * as external from "./external"
import * as internal from "./internal"
export * as external from "./external"
export * as internal from "./internal"
export async function saveTable(table: Table) {
const db = context.getAppDB()
if (isExternalTable(table._id!)) {
const datasource = await sdk.datasources.get(table.sourceId!)
datasource.entities![table.name] = table
await db.put(datasource)
} else {
await db.put(table)
}
}
export async function update(table: Table, renaming?: RenameColumn) {
const tableId = table._id
if (isExternal({ table })) {
const datasourceId = table.sourceId!
await external.save(datasourceId, table, { tableId, renaming })
} else {
await internal.save(table, { tableId, renaming })
}
}