Extract core from controller
This commit is contained in:
Adria Navarro
parent
c8fc01fa0c
commit
7f41d085f7
|
|
@ -1,200 +1,10 @@
|
|||
import * as uuid from "uuid"
|
||||
import {
|
||||
DocumentType,
|
||||
FieldType,
|
||||
GenerateTablesRequest,
|
||||
GenerateTablesResponse,
|
||||
SourceName,
|
||||
Table,
|
||||
TableSchema,
|
||||
TableSourceType,
|
||||
Upload,
|
||||
UserCtx,
|
||||
} from "@budibase/types"
|
||||
import { ai } from "@budibase/pro"
|
||||
import { context, objectStore, utils } from "@budibase/backend-core"
|
||||
import sdk from "../../sdk"
|
||||
import fs, { mkdirSync } from "fs"
|
||||
import path, { join } from "path"
|
||||
|
||||
import { pipeline } from "stream"
|
||||
import { promisify } from "util"
|
||||
import fetch from "node-fetch"
|
||||
import { ObjectStoreBuckets } from "../../constants"
|
||||
import { uploadUrl } from "../../utilities"
|
||||
import { uploadFile } from "../../utilities/fileUtils"
|
||||
|
||||
async function generateTablesDelegate(
|
||||
tables: { name: string; primaryDisplay: string; schema: TableSchema }[]
|
||||
) {
|
||||
const count = (await sdk.datasources.fetch()).length
|
||||
const { id: dsId } = await context.getAppDB().put({
|
||||
_id: `${DocumentType.DATASOURCE}_bb_internal_${utils.newid()}`,
|
||||
name: `Test ${count}`,
|
||||
type: "budibase",
|
||||
|
||||
source: SourceName.BUDIBASE,
|
||||
config: {},
|
||||
})
|
||||
|
||||
const createdTables: GenerateTablesResponse["createdTables"] = []
|
||||
const tableIds: Record<string, string> = {}
|
||||
|
||||
for (const table of tables) {
|
||||
const createdTable = await sdk.tables.create({
|
||||
...table,
|
||||
sourceId: dsId,
|
||||
schema: {},
|
||||
primaryDisplay: undefined,
|
||||
sourceType: TableSourceType.INTERNAL,
|
||||
type: "table",
|
||||
})
|
||||
|
||||
createdTables.push({ id: createdTable._id!, name: createdTable.name })
|
||||
tableIds[table.name] = createdTable._id!
|
||||
}
|
||||
|
||||
for (const table of tables) {
|
||||
for (const field of Object.values(table.schema)) {
|
||||
if (field.type === FieldType.LINK) {
|
||||
const linkedTable = createdTables.find(t => t.name === field.tableId)
|
||||
if (!linkedTable) {
|
||||
throw `Table ${field.tableId} not found in the json response.`
|
||||
}
|
||||
field.tableId = linkedTable.id
|
||||
} else if (field.type === FieldType.FORMULA) {
|
||||
field.formula = `{{ js "${btoa(field.formula)}" }}`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const table of tables) {
|
||||
const storedTable = await sdk.tables.getTable(tableIds[table.name])
|
||||
|
||||
await sdk.tables.update({
|
||||
...storedTable,
|
||||
schema: {
|
||||
...storedTable.schema,
|
||||
...table.schema,
|
||||
},
|
||||
primaryDisplay: table.primaryDisplay,
|
||||
})
|
||||
}
|
||||
|
||||
return createdTables
|
||||
}
|
||||
|
||||
async function generateDataDelegate(
|
||||
data: Record<string, Record<string, any>[]>,
|
||||
userId: string,
|
||||
tables: Record<string, Table>
|
||||
) {
|
||||
const createdData: Record<string, Record<string, string>> = {}
|
||||
const toUpdateLinks: {
|
||||
tableId: string
|
||||
rowId: string
|
||||
data: Record<string, { rowId: string[]; tableId: string }>
|
||||
}[] = []
|
||||
for (const tableName of Object.keys(data)) {
|
||||
const table = tables[tableName]
|
||||
const linksOverride: Record<string, null> = {}
|
||||
for (const field of Object.values(table.schema).filter(
|
||||
f => f.type === FieldType.LINK
|
||||
)) {
|
||||
linksOverride[field.name] = null
|
||||
}
|
||||
|
||||
const attachmentColumns = Object.values(table.schema).filter(f =>
|
||||
[FieldType.ATTACHMENTS, FieldType.ATTACHMENT_SINGLE].includes(f.type)
|
||||
)
|
||||
|
||||
for (const entry of data[tableName]) {
|
||||
const attachmentData: Record<string, any> = {}
|
||||
for (const column of attachmentColumns) {
|
||||
attachmentData[column.name] = []
|
||||
if (!Array.isArray(entry[column.name])) {
|
||||
entry[column.name] = [entry[column.name]]
|
||||
}
|
||||
for (const attachmentValue of entry[column.name]) {
|
||||
let attachment
|
||||
if (typeof attachmentValue === "object") {
|
||||
attachment = await uploadUrl(attachmentValue)
|
||||
} else {
|
||||
attachment = await uploadFile(attachmentValue)
|
||||
}
|
||||
if (attachment) {
|
||||
attachmentData[column.name].push(attachment)
|
||||
}
|
||||
}
|
||||
|
||||
if (column.type === FieldType.ATTACHMENT_SINGLE) {
|
||||
attachmentData[column.name] = attachmentData[column.name][0]
|
||||
}
|
||||
}
|
||||
|
||||
const tableId = tables[tableName]._id!
|
||||
const createdRow = await sdk.rows.save(
|
||||
tableId,
|
||||
{
|
||||
...entry,
|
||||
...linksOverride,
|
||||
...attachmentData,
|
||||
_id: undefined,
|
||||
},
|
||||
userId
|
||||
)
|
||||
|
||||
createdData[tableId] ??= {}
|
||||
createdData[tableId][entry._id] = createdRow.row._id!
|
||||
|
||||
const overridenLinks = Object.keys(linksOverride).reduce<
|
||||
Record<string, { rowId: string[]; tableId: string }>
|
||||
>((acc, l) => {
|
||||
if (entry[l]) {
|
||||
acc[l] = {
|
||||
tableId: (table.schema[l] as any).tableId,
|
||||
rowId: entry[l],
|
||||
}
|
||||
}
|
||||
return acc
|
||||
}, {})
|
||||
|
||||
if (Object.keys(overridenLinks).length) {
|
||||
toUpdateLinks.push({
|
||||
tableId: createdRow.table._id!,
|
||||
rowId: createdRow.row._id!,
|
||||
data: overridenLinks,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const data of toUpdateLinks) {
|
||||
const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
|
||||
|
||||
const updatedLinks = Object.keys(data.data).reduce<Record<string, any>>(
|
||||
(acc, d) => {
|
||||
acc[d] = [
|
||||
...(persistedRow[d] || []),
|
||||
...data.data[d].rowId.map(
|
||||
rid => createdData[data.data[d].tableId][rid]
|
||||
),
|
||||
]
|
||||
return acc
|
||||
},
|
||||
{}
|
||||
)
|
||||
|
||||
await sdk.rows.save(
|
||||
data.tableId,
|
||||
{
|
||||
...persistedRow,
|
||||
...updatedLinks,
|
||||
},
|
||||
userId
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export async function generateTables(
|
||||
ctx: UserCtx<GenerateTablesRequest, GenerateTablesResponse>
|
||||
|
|
@ -202,9 +12,9 @@ export async function generateTables(
|
|||
const { prompt, addData } = ctx.request.body
|
||||
|
||||
const tableGenerator = new ai.TableGeneration({
|
||||
generateTablesDelegate,
|
||||
generateTablesDelegate: sdk.ai.helpers.generateTables,
|
||||
getTablesDelegate: sdk.tables.getTables,
|
||||
generateDataDelegate,
|
||||
generateDataDelegate: sdk.ai.helpers.generateRows,
|
||||
})
|
||||
if (addData) {
|
||||
tableGenerator.withData(ctx.user._id || "")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
export * from "./table"
|
||||
export * from "./rows"
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
import { FieldType, Table } from "@budibase/types"
|
||||
import sdk from "../../.."
|
||||
import { uploadFile, uploadUrl } from "../../../../utilities"
|
||||
|
||||
export async function generateRows(
|
||||
data: Record<string, Record<string, any>[]>,
|
||||
userId: string,
|
||||
tables: Record<string, Table>
|
||||
) {
|
||||
const createdData: Record<string, Record<string, string>> = {}
|
||||
const toUpdateLinks: {
|
||||
tableId: string
|
||||
rowId: string
|
||||
data: Record<string, { rowId: string[]; tableId: string }>
|
||||
}[] = []
|
||||
for (const tableName of Object.keys(data)) {
|
||||
const table = tables[tableName]
|
||||
const linksOverride: Record<string, null> = {}
|
||||
for (const field of Object.values(table.schema).filter(
|
||||
f => f.type === FieldType.LINK
|
||||
)) {
|
||||
linksOverride[field.name] = null
|
||||
}
|
||||
|
||||
const attachmentColumns = Object.values(table.schema).filter(f =>
|
||||
[FieldType.ATTACHMENTS, FieldType.ATTACHMENT_SINGLE].includes(f.type)
|
||||
)
|
||||
|
||||
for (const entry of data[tableName]) {
|
||||
const attachmentData: Record<string, any> = {}
|
||||
for (const column of attachmentColumns) {
|
||||
attachmentData[column.name] = []
|
||||
if (!Array.isArray(entry[column.name])) {
|
||||
entry[column.name] = [entry[column.name]]
|
||||
}
|
||||
for (const attachmentValue of entry[column.name]) {
|
||||
let attachment
|
||||
if (typeof attachmentValue === "object") {
|
||||
attachment = await uploadUrl(attachmentValue)
|
||||
} else {
|
||||
attachment = await uploadFile(attachmentValue)
|
||||
}
|
||||
if (attachment) {
|
||||
attachmentData[column.name].push(attachment)
|
||||
}
|
||||
}
|
||||
|
||||
if (column.type === FieldType.ATTACHMENT_SINGLE) {
|
||||
attachmentData[column.name] = attachmentData[column.name][0]
|
||||
}
|
||||
}
|
||||
|
||||
const tableId = tables[tableName]._id!
|
||||
const createdRow = await sdk.rows.save(
|
||||
tableId,
|
||||
{
|
||||
...entry,
|
||||
...linksOverride,
|
||||
...attachmentData,
|
||||
_id: undefined,
|
||||
},
|
||||
userId
|
||||
)
|
||||
|
||||
createdData[tableId] ??= {}
|
||||
createdData[tableId][entry._id] = createdRow.row._id!
|
||||
|
||||
const overridenLinks = Object.keys(linksOverride).reduce<
|
||||
Record<string, { rowId: string[]; tableId: string }>
|
||||
>((acc, l) => {
|
||||
if (entry[l]) {
|
||||
acc[l] = {
|
||||
tableId: (table.schema[l] as any).tableId,
|
||||
rowId: entry[l],
|
||||
}
|
||||
}
|
||||
return acc
|
||||
}, {})
|
||||
|
||||
if (Object.keys(overridenLinks).length) {
|
||||
toUpdateLinks.push({
|
||||
tableId: createdRow.table._id!,
|
||||
rowId: createdRow.row._id!,
|
||||
data: overridenLinks,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const data of toUpdateLinks) {
|
||||
const persistedRow = await sdk.rows.find(data.tableId, data.rowId)
|
||||
|
||||
const updatedLinks = Object.keys(data.data).reduce<Record<string, any>>(
|
||||
(acc, d) => {
|
||||
acc[d] = [
|
||||
...(persistedRow[d] || []),
|
||||
...data.data[d].rowId.map(
|
||||
rid => createdData[data.data[d].tableId][rid]
|
||||
),
|
||||
]
|
||||
return acc
|
||||
},
|
||||
{}
|
||||
)
|
||||
|
||||
await sdk.rows.save(
|
||||
data.tableId,
|
||||
{
|
||||
...persistedRow,
|
||||
...updatedLinks,
|
||||
},
|
||||
userId
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
import {
|
||||
DocumentType,
|
||||
FieldType,
|
||||
GenerateTablesResponse,
|
||||
SourceName,
|
||||
TableSchema,
|
||||
TableSourceType,
|
||||
} from "@budibase/types"
|
||||
import sdk from "../../.."
|
||||
import { context, utils } from "@budibase/backend-core"
|
||||
|
||||
export async function generateTables(
|
||||
tables: { name: string; primaryDisplay: string; schema: TableSchema }[]
|
||||
) {
|
||||
const count = (await sdk.datasources.fetch()).length
|
||||
const { id: dsId } = await context.getAppDB().put({
|
||||
_id: `${DocumentType.DATASOURCE}_bb_internal_${utils.newid()}`,
|
||||
name: `Test ${count}`,
|
||||
type: "budibase",
|
||||
|
||||
source: SourceName.BUDIBASE,
|
||||
config: {},
|
||||
})
|
||||
|
||||
const createdTables: GenerateTablesResponse["createdTables"] = []
|
||||
const tableIds: Record<string, string> = {}
|
||||
|
||||
for (const table of tables) {
|
||||
const createdTable = await sdk.tables.create({
|
||||
...table,
|
||||
sourceId: dsId,
|
||||
schema: {},
|
||||
primaryDisplay: undefined,
|
||||
sourceType: TableSourceType.INTERNAL,
|
||||
type: "table",
|
||||
})
|
||||
|
||||
createdTables.push({ id: createdTable._id!, name: createdTable.name })
|
||||
tableIds[table.name] = createdTable._id!
|
||||
}
|
||||
|
||||
for (const table of tables) {
|
||||
for (const field of Object.values(table.schema)) {
|
||||
if (field.type === FieldType.LINK) {
|
||||
const linkedTable = createdTables.find(t => t.name === field.tableId)
|
||||
if (!linkedTable) {
|
||||
throw `Table ${field.tableId} not found in the json response.`
|
||||
}
|
||||
field.tableId = linkedTable.id
|
||||
} else if (field.type === FieldType.FORMULA) {
|
||||
field.formula = `{{ js "${btoa(field.formula)}" }}`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const table of tables) {
|
||||
const storedTable = await sdk.tables.getTable(tableIds[table.name])
|
||||
|
||||
await sdk.tables.update({
|
||||
...storedTable,
|
||||
schema: {
|
||||
...storedTable.schema,
|
||||
...table.schema,
|
||||
},
|
||||
primaryDisplay: table.primaryDisplay,
|
||||
})
|
||||
}
|
||||
|
||||
return createdTables
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
export * as helpers from "./helpers"
|
||||
|
|
@ -14,6 +14,7 @@ import * as rowActions from "./app/rowActions"
|
|||
import * as screens from "./app/screens"
|
||||
import * as common from "./app/common"
|
||||
import * as oauth2 from "./app/oauth2"
|
||||
import * as ai from "./app/ai"
|
||||
|
||||
const sdk = {
|
||||
backups,
|
||||
|
|
@ -32,6 +33,7 @@ const sdk = {
|
|||
rowActions,
|
||||
common,
|
||||
oauth2,
|
||||
ai,
|
||||
}
|
||||
|
||||
// default export for TS
|
||||
|
|
|
|||
Loading…
Reference in New Issue