From 069fd3396464346b3d8a9ad519cb403af7b6477e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Oct 2023 19:18:21 +0100 Subject: [PATCH 01/13] Some work towards improving performance of internal DB enrichment, there is a problem with cyclic enrichment due to the outputProcessing, need to decide how to handle formulas on enrichment. --- .../src/api/controllers/row/internal.ts | 67 +++++++---------- packages/server/src/api/controllers/user.ts | 6 +- packages/server/src/db/linkedRows/index.ts | 26 ++++--- .../server/src/db/linkedRows/linkUtils.ts | 4 +- .../src/sdk/app/datasources/datasources.ts | 31 ++++++-- packages/server/src/sdk/app/tables/index.ts | 75 +++++++++++++++---- 6 files changed, 134 insertions(+), 75 deletions(-) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index f33df4536f..f00621ba64 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -1,6 +1,7 @@ import * as linkRows from "../../../db/linkedRows" import { generateRowID, + getMultiIDParams, getTableIDFromRowID, InternalTables, } from "../../../db/utils" @@ -24,6 +25,8 @@ import { UserCtx, } from "@budibase/types" import sdk from "../../../sdk" +import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils" +import { flatten } from "lodash" export async function patch(ctx: UserCtx) { const tableId = utils.getTableId(ctx) @@ -224,57 +227,43 @@ export async function fetchEnrichedRow(ctx: UserCtx) { const db = context.getAppDB() const tableId = utils.getTableId(ctx) const rowId = ctx.params.rowId - // need table to work out where links go in row - let [table, row] = await Promise.all([ + // need table to work out where links go in row, as well as the link docs + let response = await Promise.all([ sdk.tables.getTable(tableId), utils.findRow(ctx, tableId, rowId), + linkRows.getLinkDocuments({ tableId, rowId }), ]) - // get the link docs - const linkVals = (await linkRows.getLinkDocuments({ - tableId, - rowId, - })) as LinkDocumentValue[] + const table = response[0] as Table + const row = response[1] as Row + const linkVals = response[2] as LinkDocumentValue[] // look up the actual rows based on the ids - let response = ( - await db.allDocs({ - include_docs: true, - keys: linkVals.map(linkVal => linkVal.id), - }) - ).rows.map(row => row.doc) - // group responses by table - let groups: any = {}, - tables: Record = {} - for (let row of response) { - if (!row.tableId) { - row.tableId = getTableIDFromRowID(row._id) - } - const linkedTableId = row.tableId - if (groups[linkedTableId] == null) { - groups[linkedTableId] = [row] - tables[linkedTableId] = await db.get(linkedTableId) - } else { - groups[linkedTableId].push(row) - } - } - let linkedRows: Row[] = [] - for (let [tableId, rows] of Object.entries(groups)) { - // need to include the IDs in these rows for any links they may have - linkedRows = linkedRows.concat( - await outputProcessing(tables[tableId], rows as Row[]) + const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id)) + let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc) + + // get the linked tables + const linkTableIds = getLinkedTableIDs(table as Table) + const linkTables = await sdk.tables.getTables(linkTableIds) + + // perform output processing + let final = [] + for (let linkTable of linkTables) { + const relatedRows = linkedRows.filter(row => row.tableId === linkTable._id) + final = final.concat( + outputProcessing(linkTable, relatedRows, { preserveLinks: true }) ) } + // finalise the promises + linkedRows = flatten(await Promise.all(final)) // insert the link rows in the correct place throughout the main row for (let fieldName of Object.keys(table.schema)) { let field = table.schema[fieldName] if (field.type === FieldTypes.LINK) { - // find the links that pertain to this field, get their indexes - const linkIndexes = linkVals - .filter(link => link.fieldName === fieldName) - .map(link => linkVals.indexOf(link)) + // find the links that pertain to this field + const links = linkVals.filter(link => link.fieldName === fieldName) // find the rows that the links state are linked to this field - row[fieldName] = linkedRows.filter((linkRow, index) => - linkIndexes.includes(index) + row[fieldName] = linkedRows.filter(linkRow => + links.find(link => link.id === linkRow._id) ) } } diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index dbbfc5c586..b6c3e7c6bd 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -1,13 +1,11 @@ -import { generateUserFlagID } from "../../db/utils" -import { InternalTables } from "../../db/utils" +import { generateUserFlagID, InternalTables } from "../../db/utils" import { getFullUser } from "../../utilities/users" import { context } from "@budibase/backend-core" import { Ctx, UserCtx } from "@budibase/types" import sdk from "../../sdk" export async function fetchMetadata(ctx: Ctx) { - const users = await sdk.users.fetchMetadata() - ctx.body = users + ctx.body = await sdk.users.fetchMetadata() } export async function updateSelfMetadata(ctx: UserCtx) { diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 3c5379ac97..1b64e615b0 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -9,13 +9,13 @@ import { getLinkedTable, } from "./linkUtils" import flatten from "lodash/flatten" -import { FieldTypes } from "../../constants" import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils" import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" import { processFormulas } from "../../utilities/rowProcessor" import { context } from "@budibase/backend-core" -import { Table, Row, LinkDocumentValue } from "@budibase/types" +import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types" +import sdk from "../../sdk" export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils" @@ -35,7 +35,7 @@ export const EventType = { function clearRelationshipFields(table: Table, rows: Row[]) { for (let [key, field] of Object.entries(table.schema)) { - if (field.type === FieldTypes.LINK) { + if (field.type === FieldType.LINK) { rows = rows.map(row => { delete row[key] return row @@ -45,7 +45,7 @@ function clearRelationshipFields(table: Table, rows: Row[]) { return rows } -async function getLinksForRows(rows: Row[]) { +async function getLinksForRows(rows: Row[]): Promise { const tableIds = [...new Set(rows.map(el => el.tableId))] // start by getting all the link values for performance reasons const promises = tableIds.map(tableId => @@ -153,15 +153,19 @@ export async function attachFullLinkedDocs(table: Table, rows: Row[]) { if (linkedTableIds.length === 0) { return rows } - // get all the links - const links = (await getLinksForRows(rows)).filter(link => + // get tables and links + let response = await Promise.all([ + getLinksForRows(rows), + sdk.tables.getTables(linkedTableIds), + ]) + const links = (response[0] as LinkDocumentValue[]).filter(link => rows.some(row => row._id === link.thisId) ) + const linkedTables = response[1] as Table[] // clear any existing links that could be dupe'd rows = clearRelationshipFields(table, rows) // now get the docs and combine into the rows let linked = await getFullLinkedDocs(links) - const linkedTables: Table[] = [] for (let row of rows) { for (let link of links.filter(link => link.thisId === row._id)) { if (row[link.fieldName] == null) { @@ -171,7 +175,9 @@ export async function attachFullLinkedDocs(table: Table, rows: Row[]) { if (linkedRow) { const linkedTableId = linkedRow.tableId || getRelatedTableForField(table, link.fieldName) - const linkedTable = await getLinkedTable(linkedTableId, linkedTables) + const linkedTable = linkedTables.find( + table => table._id === linkedTableId + ) if (linkedTable) { row[link.fieldName].push(processFormulas(linkedTable, linkedRow)) } @@ -199,13 +205,13 @@ export async function squashLinksToPrimaryDisplay( // this only fetches the table if its not already in array const rowTable = await getLinkedTable(row.tableId!, linkedTables) for (let [column, schema] of Object.entries(rowTable?.schema || {})) { - if (schema.type !== FieldTypes.LINK || !Array.isArray(row[column])) { + if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) { continue } const newLinks = [] for (let link of row[column]) { const linkTblId = link.tableId || getRelatedTableForField(table, column) - const linkedTable = await getLinkedTable(linkTblId, linkedTables) + const linkedTable = await getLinkedTable(linkTblId!, linkedTables) const obj: any = { _id: link._id } if (linkedTable?.primaryDisplay && link[linkedTable.primaryDisplay]) { obj.primaryDisplay = link[linkedTable.primaryDisplay] diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index c7db7d522a..3c81843565 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -91,10 +91,10 @@ export function getUniqueByProp(array: any[], prop: string) { }) } -export function getLinkedTableIDs(table: Table) { +export function getLinkedTableIDs(table: Table): string[] { return Object.values(table.schema) .filter((column: FieldSchema) => column.type === FieldTypes.LINK) - .map(column => column.tableId) + .map(column => column.tableId!) } export async function getLinkedTable(id: string, tables: Table[]) { diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index 35107fd6b8..fb5d04b03e 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -23,10 +23,13 @@ import { getTableParams, } from "../../../db/utils" import sdk from "../../index" +import datasource from "../../../api/routes/datasource" const ENV_VAR_PREFIX = "env." -export async function fetch() { +export async function fetch(opts?: { + enriched: boolean +}): Promise { // Get internal tables const db = context.getAppDB() const internalTables = await db.allDocs( @@ -44,7 +47,7 @@ export async function fetch() { const bbInternalDb = { ...BudibaseInternalDB, - } + } as Datasource // Get external datasources const datasources = ( @@ -66,7 +69,18 @@ export async function fetch() { } } - return [bbInternalDb, ...datasources] + if (opts?.enriched) { + const envVars = await getEnvironmentVariables() + const promises = datasources.map(datasource => + enrichDatasourceWithValues(datasource, envVars) + ) + const enriched = (await Promise.all(promises)).map( + result => result.datasource + ) + return [bbInternalDb, ...enriched] + } else { + return [bbInternalDb, ...datasources] + } } export function areRESTVariablesValid(datasource: Datasource) { @@ -107,9 +121,12 @@ export function checkDatasourceTypes(schema: Integration, config: any) { return config } -async function enrichDatasourceWithValues(datasource: Datasource) { +async function enrichDatasourceWithValues( + datasource: Datasource, + variables?: Record +) { const cloned = cloneDeep(datasource) - const env = await getEnvironmentVariables() + const env = variables ? variables : await getEnvironmentVariables() //Do not process entities, as we do not want to process formulas const { entities, ...clonedWithoutEntities } = cloned const processed = processObjectSync( @@ -235,9 +252,9 @@ export function mergeConfigs(update: Datasource, old: Datasource) { if (value !== PASSWORD_REPLACEMENT) { continue } - if (old.config?.[key]) { + if (update.config && old.config && old.config?.[key]) { update.config[key] = old.config?.[key] - } else { + } else if (update.config) { delete update.config[key] } } diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index 64fcde4bff..f02f46e156 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -1,11 +1,16 @@ import { context } from "@budibase/backend-core" -import { BudibaseInternalDB, getTableParams } from "../../../db/utils" +import { + BudibaseInternalDB, + getMultiIDParams, + getTableParams, +} from "../../../db/utils" import { breakExternalTableId, isExternalTable, isSQL, } from "../../../integrations/utils" import { + AllDocsResponse, Database, Table, TableResponse, @@ -15,6 +20,14 @@ import datasources from "../datasources" import { populateExternalTableSchemas } from "./validation" import sdk from "../../../sdk" +function processInternalTables(docs: AllDocsResponse): Table[] { + return docs.rows.map((tableDoc: any) => ({ + ...tableDoc.doc, + type: "internal", + sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id, + })) +} + async function getAllInternalTables(db?: Database): Promise { if (!db) { db = context.getAppDB() @@ -24,15 +37,49 @@ async function getAllInternalTables(db?: Database): Promise { include_docs: true, }) ) - return internalTables.rows.map((tableDoc: any) => ({ - ...tableDoc.doc, - type: "internal", - sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id, - })) + return processInternalTables(internalTables) } -async function getAllExternalTables( - datasourceId: any +async function getAllExternalTables(): Promise { + const datasources = await sdk.datasources.fetch({ enriched: true }) + const allEntities = datasources.map(datasource => datasource.entities) + let final = [] + for (let entities of allEntities) { + final = final.concat(Object.values(entities)) + } + return final +} + +async function getAllTables() { + const [internal, external] = await Promise.all([ + getAllInternalTables(), + getAllExternalTables(), + ]) + return [...internal, external] +} + +async function getTables(tableIds: string[]): Promise { + const externalTableIds = tableIds.filter(tableId => isExternalTable(tableId)), + internalTableIds = tableIds.filter(tableId => !isExternalTable(tableId)) + let tables = [] + if (externalTableIds.length) { + const externalTables = await getAllExternalTables() + tables = tables.concat( + externalTables.filter( + table => externalTableIds.indexOf(table._id!) !== -1 + ) + ) + } + if (internalTableIds.length) { + const db = context.getAppDB() + const internalTables = await db.allDocs(getMultiIDParams(internalTableIds)) + tables = tables.concat(processInternalTables(internalTables)) + } + return tables +} + +async function getExternalTablesInDatasource( + datasourceId: string ): Promise> { const datasource = await datasources.get(datasourceId, { enriched: true }) if (!datasource || !datasource.entities) { @@ -42,14 +89,14 @@ async function getAllExternalTables( } async function getExternalTable( - datasourceId: any, - tableName: any + datasourceId: string, + tableName: string ): Promise { - const entities = await getAllExternalTables(datasourceId) + const entities = await getExternalTablesInDatasource(datasourceId) return entities[tableName] } -async function getTable(tableId: any): Promise
{ +async function getTable(tableId: string): Promise
{ const db = context.getAppDB() if (isExternalTable(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) @@ -86,9 +133,11 @@ async function saveTable(table: Table) { export default { getAllInternalTables, - getAllExternalTables, + getExternalTablesInDatasource, getExternalTable, getTable, + getAllTables, + getTables, populateExternalTableSchemas, enrichViewSchemas, saveTable, From 3e2f9dfc4ee9677f4686d9dafe0626d622f8668a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 5 Oct 2023 18:23:18 +0100 Subject: [PATCH 02/13] Further enhancement, client library sends up the column it wants enriched and then we can ignore everything else, makes a big difference for enriching users (with a lot of relationships). --- .../frontend-core/src/api/relationships.js | 4 ++- .../src/api/controllers/row/internal.ts | 12 ++++++--- packages/server/src/db/linkedRows/index.ts | 25 ++++++++++++++++--- .../server/src/db/linkedRows/linkUtils.ts | 16 +++++++++--- packages/server/src/sdk/app/tables/index.ts | 2 +- .../src/utilities/rowProcessor/index.ts | 6 +++-- 6 files changed, 51 insertions(+), 14 deletions(-) diff --git a/packages/frontend-core/src/api/relationships.js b/packages/frontend-core/src/api/relationships.js index fbc727f8e1..45595750a8 100644 --- a/packages/frontend-core/src/api/relationships.js +++ b/packages/frontend-core/src/api/relationships.js @@ -9,7 +9,9 @@ export const buildRelationshipEndpoints = API => ({ if (!tableId || !rowId) { return [] } - const response = await API.get({ url: `/api/${tableId}/${rowId}/enrich` }) + const response = await API.get({ + url: `/api/${tableId}/${rowId}/enrich?field=${fieldName}`, + }) if (!fieldName) { return response || [] } else { diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index f00621ba64..b4ea80e72b 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -224,14 +224,15 @@ export async function bulkDestroy(ctx: UserCtx) { } export async function fetchEnrichedRow(ctx: UserCtx) { + const fieldName = ctx.request.query.field as string | undefined const db = context.getAppDB() const tableId = utils.getTableId(ctx) - const rowId = ctx.params.rowId + const rowId = ctx.params.rowId as string // need table to work out where links go in row, as well as the link docs let response = await Promise.all([ sdk.tables.getTable(tableId), utils.findRow(ctx, tableId, rowId), - linkRows.getLinkDocuments({ tableId, rowId }), + linkRows.getLinkDocuments({ tableId, rowId, fieldName }), ]) const table = response[0] as Table const row = response[1] as Row @@ -248,8 +249,13 @@ export async function fetchEnrichedRow(ctx: UserCtx) { let final = [] for (let linkTable of linkTables) { const relatedRows = linkedRows.filter(row => row.tableId === linkTable._id) + // include the row being enriched for performance reasons, don't need to fetch it to include final = final.concat( - outputProcessing(linkTable, relatedRows, { preserveLinks: true }) + outputProcessing(linkTable, relatedRows, { + // have to clone to avoid JSON cycle + fromRow: cloneDeep(row), + squash: true, + }) ) } // finalise the promises diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 1b64e615b0..85d64dda82 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -146,9 +146,14 @@ export async function updateLinks(args: { * This is required for formula fields, this may only be utilised internally (for now). * @param {object} table The table from which the rows originated. * @param {array} rows The rows which are to be enriched. + * @param {object} opts optional - options like passing in a base row to use for enrichment. * @return {Promise<*>} returns the rows with all of the enriched relationships on it. */ -export async function attachFullLinkedDocs(table: Table, rows: Row[]) { +export async function attachFullLinkedDocs( + table: Table, + rows: Row[], + opts?: { fromRow?: Row } +) { const linkedTableIds = getLinkedTableIDs(table) if (linkedTableIds.length === 0) { return rows @@ -158,20 +163,34 @@ export async function attachFullLinkedDocs(table: Table, rows: Row[]) { getLinksForRows(rows), sdk.tables.getTables(linkedTableIds), ]) + // find the links that pertain to one of the rows that is being enriched const links = (response[0] as LinkDocumentValue[]).filter(link => rows.some(row => row._id === link.thisId) ) + // if fromRow has been passed in, then we don't need to fetch it (optimisation) + let linksWithoutFromRow = links + if (opts?.fromRow) { + linksWithoutFromRow = links.filter(link => link.id !== opts?.fromRow?._id) + } const linkedTables = response[1] as Table[] // clear any existing links that could be dupe'd rows = clearRelationshipFields(table, rows) // now get the docs and combine into the rows - let linked = await getFullLinkedDocs(links) + let linked = [] + if (linksWithoutFromRow.length > 0) { + linked = await getFullLinkedDocs(linksWithoutFromRow) + } for (let row of rows) { for (let link of links.filter(link => link.thisId === row._id)) { if (row[link.fieldName] == null) { row[link.fieldName] = [] } - const linkedRow = linked.find(row => row._id === link.id) + let linkedRow: Row + if (opts?.fromRow && opts?.fromRow?._id === link.id) { + linkedRow = opts.fromRow! + } else { + linkedRow = linked.find(row => row._id === link.id) + } if (linkedRow) { const linkedTableId = linkedRow.tableId || getRelatedTableForField(table, link.fieldName) diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index 3c81843565..1a15246f85 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -35,19 +35,22 @@ export const IncludeDocs = { export async function getLinkDocuments(args: { tableId?: string rowId?: string - includeDocs?: any + fieldName?: string + includeDocs?: boolean }): Promise { - const { tableId, rowId, includeDocs } = args + const { tableId, rowId, fieldName, includeDocs } = args const db = context.getAppDB() let params: any - if (rowId != null) { + if (rowId) { params = { key: [tableId, rowId] } } // only table is known else { params = { startKey: [tableId], endKey: [tableId, {}] } } - params.include_docs = !!includeDocs + if (includeDocs) { + params.include_docs = true + } try { let linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows // filter to get unique entries @@ -67,6 +70,11 @@ export async function getLinkDocuments(args: { return unique }) + // filter down to just the required field name + if (fieldName) { + linkRows = linkRows.filter(link => link.value.fieldName === fieldName) + } + // return docs if docs requested, otherwise just the value information if (includeDocs) { return linkRows.map(row => row.doc) as LinkDocument[] } else { diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index f02f46e156..d2b1fe005a 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -104,7 +104,7 @@ async function getTable(tableId: string): Promise
{ const table = await getExternalTable(datasourceId, tableName) return { ...table, sql: isSQL(datasource) } } else { - return db.get(tableId) + return db.get
(tableId) } } diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 0bdaaa393e..b3e780491d 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -201,7 +201,7 @@ export async function inputProcessing( export async function outputProcessing( table: Table, rows: T, - opts: { squash?: boolean; preserveLinks?: boolean } = { + opts: { squash?: boolean; preserveLinks?: boolean; fromRow?: Row } = { squash: true, preserveLinks: false, } @@ -216,7 +216,9 @@ export async function outputProcessing( } // attach any linked row information let enriched = !opts.preserveLinks - ? await linkRows.attachFullLinkedDocs(table, safeRows) + ? await linkRows.attachFullLinkedDocs(table, safeRows, { + fromRow: opts?.fromRow, + }) : safeRows // process formulas From ee4a042204acd369c3445309266f1d8d45e3c68e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 5 Oct 2023 18:40:56 +0100 Subject: [PATCH 03/13] Typing fixes - unsure why all of these came up suddenly. --- .../server/src/api/controllers/row/index.ts | 3 +- .../src/api/controllers/row/internal.ts | 6 +- .../src/api/controllers/row/staticFormula.ts | 2 +- .../server/src/api/controllers/row/utils.ts | 128 +----------------- .../server/src/api/controllers/table/index.ts | 4 +- .../server/src/db/linkedRows/linkUtils.ts | 5 +- packages/server/src/sdk/app/rows/utils.ts | 9 +- packages/server/src/sdk/app/tables/index.ts | 20 +-- 8 files changed, 33 insertions(+), 144 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 6e0a6d979e..0ccbf5cacf 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -26,6 +26,7 @@ import { fixRow } from "../public/rows" import sdk from "../../../sdk" import * as exporters from "../view/exporters" import { apiFileReturn } from "../../../utilities/fileSystem" +import { Format } from "../view/exporters" export * as views from "./views" function pickApi(tableId: any) { @@ -267,7 +268,7 @@ export const exportRows = async ( async () => { const { fileName, content } = await sdk.rows.exportRows({ tableId, - format, + format: format as Format, rowIds: rows, columns, query, diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index b4ea80e72b..bc5162d6e8 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -157,14 +157,14 @@ export async function destroy(ctx: UserCtx) { if (row.tableId !== tableId) { throw "Supplied tableId doesn't match the row's tableId" } - const table = await sdk.tables.getTable(row.tableId) + const table = await sdk.tables.getTable(tableId) // update the row to include full relationships before deleting them row = await outputProcessing(table, row, { squash: false }) // now remove the relationships await linkRows.updateLinks({ eventType: linkRows.EventType.ROW_DELETE, row, - tableId: row.tableId, + tableId, }) // remove any attachments that were on the row from object storage await cleanupAttachments(table, { row }) @@ -246,7 +246,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { const linkTables = await sdk.tables.getTables(linkTableIds) // perform output processing - let final = [] + let final: Promise[] = [] for (let linkTable of linkTables) { const relatedRows = linkedRows.filter(row => row.tableId === linkTable._id) // include the row being enriched for performance reasons, don't need to fetch it to include diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index efe6f8719c..6f426c6fa0 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -149,7 +149,7 @@ export async function finaliseRow( await db.put(table) } catch (err: any) { if (err.status === 409) { - const updatedTable = await sdk.tables.getTable(table._id) + const updatedTable = await sdk.tables.getTable(table._id!) let response = processAutoColumn(null, updatedTable, row, { reprocessing: true, }) diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index 5f10fd9ad4..ed6ccd4c53 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -1,26 +1,9 @@ import { InternalTables } from "../../../db/utils" import * as userController from "../user" import { context } from "@budibase/backend-core" -import { - Ctx, - FieldType, - Row, - SearchFilters, - Table, - UserCtx, -} from "@budibase/types" -import { FieldTypes, NoEmptyFilterStrings } from "../../../constants" -import sdk from "../../../sdk" +import { Ctx, Row, UserCtx } from "@budibase/types" import validateJs from "validate.js" -import { cloneDeep } from "lodash/fp" - -function isForeignKey(key: string, table: Table) { - const relationships = Object.values(table.schema).filter( - column => column.type === FieldType.LINK - ) - return relationships.some(relationship => relationship.foreignKey === key) -} validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { @@ -51,7 +34,7 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) { return row } -export function getTableId(ctx: Ctx) { +export function getTableId(ctx: Ctx): string { // top priority, use the URL first if (ctx.params?.sourceId) { return ctx.params.sourceId @@ -68,112 +51,7 @@ export function getTableId(ctx: Ctx) { if (ctx.params?.viewName) { return ctx.params.viewName } -} - -export async function validate({ - tableId, - row, - table, -}: { - tableId?: string - row: Row - table?: Table -}) { - let fetchedTable: Table - if (!table) { - fetchedTable = await sdk.tables.getTable(tableId) - } else { - fetchedTable = table - } - const errors: any = {} - for (let fieldName of Object.keys(fetchedTable.schema)) { - const column = fetchedTable.schema[fieldName] - const constraints = cloneDeep(column.constraints) - const type = column.type - // foreign keys are likely to be enriched - if (isForeignKey(fieldName, fetchedTable)) { - continue - } - // formulas shouldn't validated, data will be deleted anyway - if (type === FieldTypes.FORMULA || column.autocolumn) { - continue - } - // special case for options, need to always allow unselected (empty) - if (type === FieldTypes.OPTIONS && constraints?.inclusion) { - constraints.inclusion.push(null as any, "") - } - let res - - // Validate.js doesn't seem to handle array - if (type === FieldTypes.ARRAY && row[fieldName]) { - if (row[fieldName].length) { - if (!Array.isArray(row[fieldName])) { - row[fieldName] = row[fieldName].split(",") - } - row[fieldName].map((val: any) => { - if ( - !constraints?.inclusion?.includes(val) && - constraints?.inclusion?.length !== 0 - ) { - errors[fieldName] = "Field not in list" - } - }) - } else if (constraints?.presence && row[fieldName].length === 0) { - // non required MultiSelect creates an empty array, which should not throw errors - errors[fieldName] = [`${fieldName} is required`] - } - } else if ( - (type === FieldTypes.ATTACHMENT || type === FieldTypes.JSON) && - typeof row[fieldName] === "string" - ) { - // this should only happen if there is an error - try { - const json = JSON.parse(row[fieldName]) - if (type === FieldTypes.ATTACHMENT) { - if (Array.isArray(json)) { - row[fieldName] = json - } else { - errors[fieldName] = [`Must be an array`] - } - } - } catch (err) { - errors[fieldName] = [`Contains invalid JSON`] - } - } else { - res = validateJs.single(row[fieldName], constraints) - } - if (res) errors[fieldName] = res - } - return { valid: Object.keys(errors).length === 0, errors } -} - -// don't do a pure falsy check, as 0 is included -// https://github.com/Budibase/budibase/issues/10118 -export function removeEmptyFilters(filters: SearchFilters) { - for (let filterField of NoEmptyFilterStrings) { - if (!filters[filterField]) { - continue - } - - for (let filterType of Object.keys(filters)) { - if (filterType !== filterField) { - continue - } - // don't know which one we're checking, type could be anything - const value = filters[filterType] as unknown - if (typeof value === "object") { - for (let [key, value] of Object.entries( - filters[filterType] as object - )) { - if (value == null || value === "") { - // @ts-ignore - delete filters[filterField][key] - } - } - } - } - } - return filters + throw new Error("Unable to find table ID in request") } export function isUserMetadataTable(tableId: string) { diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index e7c6ae57b0..d61f593346 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -14,6 +14,7 @@ import { Table, TableResponse, UserCtx, + Row, } from "@budibase/types" import sdk from "../../../sdk" import { jsonFromCsvString } from "../../../utilities/csv" @@ -129,8 +130,7 @@ export async function validateNewTableImport(ctx: UserCtx) { } export async function validateExistingTableImport(ctx: UserCtx) { - const { rows, tableId }: { rows: unknown; tableId: unknown } = - ctx.request.body + const { rows, tableId }: { rows: Row[]; tableId?: string } = ctx.request.body let schema = null if (tableId) { diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index 1a15246f85..4579454531 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -72,7 +72,10 @@ export async function getLinkDocuments(args: { // filter down to just the required field name if (fieldName) { - linkRows = linkRows.filter(link => link.value.fieldName === fieldName) + linkRows = linkRows.filter(link => { + const value = link.value as LinkDocumentValue + return value.fieldName === fieldName + }) } // return docs if docs requested, otherwise just the value information if (includeDocs) { diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 51e418c324..d9d2c745d3 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -68,12 +68,15 @@ export async function validate({ valid: boolean errors: Record }> { - let fetchedTable: Table - if (!table) { + let fetchedTable: Table | undefined + if (!table && tableId) { fetchedTable = await sdk.tables.getTable(tableId) - } else { + } else if (table) { fetchedTable = table } + if (fetchedTable === undefined) { + throw new Error("Unable to fetch table for validation") + } const errors: Record = {} for (let fieldName of Object.keys(fetchedTable.schema)) { const column = fetchedTable.schema[fieldName] diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index d2b1fe005a..96cda05396 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -32,20 +32,22 @@ async function getAllInternalTables(db?: Database): Promise { if (!db) { db = context.getAppDB() } - const internalTables = await db.allDocs( + const internalTableDocs = await db.allDocs( getTableParams(null, { include_docs: true, }) ) - return processInternalTables(internalTables) + return processInternalTables(internalTableDocs) } async function getAllExternalTables(): Promise { const datasources = await sdk.datasources.fetch({ enriched: true }) const allEntities = datasources.map(datasource => datasource.entities) - let final = [] + let final: Table[] = [] for (let entities of allEntities) { - final = final.concat(Object.values(entities)) + if (entities) { + final = final.concat(Object.values(entities)) + } } return final } @@ -61,7 +63,7 @@ async function getAllTables() { async function getTables(tableIds: string[]): Promise { const externalTableIds = tableIds.filter(tableId => isExternalTable(tableId)), internalTableIds = tableIds.filter(tableId => !isExternalTable(tableId)) - let tables = [] + let tables: Table[] = [] if (externalTableIds.length) { const externalTables = await getAllExternalTables() tables = tables.concat( @@ -72,8 +74,10 @@ async function getTables(tableIds: string[]): Promise { } if (internalTableIds.length) { const db = context.getAppDB() - const internalTables = await db.allDocs(getMultiIDParams(internalTableIds)) - tables = tables.concat(processInternalTables(internalTables)) + const internalTableDocs = await db.allDocs( + getMultiIDParams(internalTableIds) + ) + tables = tables.concat(processInternalTables(internalTableDocs)) } return tables } @@ -101,7 +105,7 @@ async function getTable(tableId: string): Promise
{ if (isExternalTable(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) const datasource = await datasources.get(datasourceId!) - const table = await getExternalTable(datasourceId, tableName) + const table = await getExternalTable(datasourceId!, tableName!) return { ...table, sql: isSQL(datasource) } } else { return db.get
(tableId) From 797677284222bb397a4e83600945d20c9a1e0ede Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 5 Oct 2023 23:19:11 +0100 Subject: [PATCH 04/13] Fixing test case. --- packages/server/src/middleware/tests/trimViewRowInfo.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts index 17b4cc7b93..bf717d5828 100644 --- a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts +++ b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts @@ -16,6 +16,7 @@ jest.mock("../../sdk", () => ({ import sdk from "../../sdk" import { Next } from "koa" +const tableId = utils.generateTableID() const mockGetView = sdk.views.get as jest.MockedFunction const mockGetTable = sdk.tables.getTable as jest.MockedFunction< typeof sdk.tables.getTable @@ -41,6 +42,7 @@ class TestConfiguration { body: ctxRequestBody, } this.params.viewId = viewId + this.params.sourceId = tableId return this.middleware( { request: this.request as any, @@ -69,7 +71,7 @@ describe("trimViewRowInfo middleware", () => { }) const table: Table = { - _id: utils.generateTableID(), + _id: tableId, name: generator.word(), type: "table", schema: { From fefd5fa0dcbd81d60e4f9b68d4acb8d3a723a400 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 16 Oct 2023 17:41:20 +0100 Subject: [PATCH 05/13] Quick fix for drag and drop behaviour of relationship cells, appears empty cells were causing things to break in the re-render. --- .../grid/cells/RelationshipCell.svelte | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte index 925c840478..e6d83e0bea 100644 --- a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte @@ -260,29 +260,31 @@ class:wrap={editable || contentLines > 1} on:wheel={e => (focused ? e.stopPropagation() : null)} > - {#each value || [] as relationship} - {#if relationship[primaryDisplay] || relationship.primaryDisplay} -
- showRelationship(relationship._id) - : null} - > - {readable( - relationship[primaryDisplay] || relationship.primaryDisplay - )} - - {#if editable} - toggleRow(relationship)} - /> - {/if} -
- {/if} - {/each} + {#if Array.isArray(value) && value.length} + {#each value as relationship} + {#if relationship[primaryDisplay] || relationship.primaryDisplay} +
+ showRelationship(relationship._id) + : null} + > + {readable( + relationship[primaryDisplay] || relationship.primaryDisplay + )} + + {#if editable} + toggleRow(relationship)} + /> + {/if} +
+ {/if} + {/each} + {/if} {#if editable}
@@ -318,7 +320,7 @@
- {:else if searchResults?.length} + {:else if Array.isArray(searchResults) && searchResults.length}
{#each searchResults as row, idx}
Date: Wed, 18 Oct 2023 09:20:06 +0100 Subject: [PATCH 06/13] Added testimonial check to reset and forgot pages. --- packages/builder/src/pages/builder/auth/forgot.svelte | 2 +- packages/builder/src/pages/builder/auth/reset.svelte | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/pages/builder/auth/forgot.svelte b/packages/builder/src/pages/builder/auth/forgot.svelte index 2ea8bf7a94..9df7196cfe 100644 --- a/packages/builder/src/pages/builder/auth/forgot.svelte +++ b/packages/builder/src/pages/builder/auth/forgot.svelte @@ -43,7 +43,7 @@ }) - + logo diff --git a/packages/builder/src/pages/builder/auth/reset.svelte b/packages/builder/src/pages/builder/auth/reset.svelte index 19bc1a1b7d..becc30d9a4 100644 --- a/packages/builder/src/pages/builder/auth/reset.svelte +++ b/packages/builder/src/pages/builder/auth/reset.svelte @@ -53,7 +53,7 @@ }) - + {#if loaded} logo From 68e8630d85bd54cdd90f8edae63f9c66f423654f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 18 Oct 2023 10:57:04 +0100 Subject: [PATCH 07/13] Improving the typing around the ExternalRequest object, which has implications throughout the row API and SDK, cleaning up where possible based on it. --- .../api/controllers/row/ExternalRequest.ts | 26 +++++++------------ .../src/api/controllers/row/external.ts | 14 +++++----- packages/server/src/sdk/app/rows/external.ts | 4 +-- packages/server/src/sdk/app/rows/search.ts | 6 ++--- .../src/sdk/app/rows/search/external.ts | 24 ++++++++++------- .../src/sdk/app/rows/search/internal.ts | 17 ++++++------ 6 files changed, 44 insertions(+), 47 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 71532c37d5..c3c5468840 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -280,17 +280,8 @@ function isEditableColumn(column: FieldSchema) { return !(isExternalAutoColumn || isFormula) } -export type ExternalRequestReturnType = T extends Operation.READ - ? - | Row[] - | { - row: Row - table: Table - } - : { - row: Row - table: Table - } +export type ExternalRequestReturnType = + T extends Operation.READ ? Row[] : { row: Row; table: Table } export class ExternalRequest { private readonly operation: T @@ -857,11 +848,12 @@ export class ExternalRequest { } const output = this.outputProcessing(response, table, relationships) // if reading it'll just be an array of rows, return whole thing - const result = ( - operation === Operation.READ && Array.isArray(response) - ? output - : { row: output[0], table } - ) as ExternalRequestReturnType - return result + if (operation === Operation.READ) { + return ( + Array.isArray(output) ? output : [output] + ) as ExternalRequestReturnType + } else { + return { row: output[0], table } as ExternalRequestReturnType + } } } diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index ddc63e5790..0515b6b97e 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -44,7 +44,7 @@ export async function handleRequest( return [] as any } - return new ExternalRequest(operation, tableId, opts?.datasource).run( + return new ExternalRequest(operation, tableId, opts?.datasource).run( opts || {} ) } @@ -148,17 +148,17 @@ export async function find(ctx: UserCtx): Promise { export async function destroy(ctx: UserCtx) { const tableId = utils.getTableId(ctx) const _id = ctx.request.body._id - const { row } = (await handleRequest(Operation.DELETE, tableId, { + const { row } = await handleRequest(Operation.DELETE, tableId, { id: breakRowIdField(_id), includeSqlRelationships: IncludeRelationship.EXCLUDE, - })) as { row: Row } + }) return { response: { ok: true, id: _id }, row } } export async function bulkDestroy(ctx: UserCtx) { const { rows } = ctx.request.body const tableId = utils.getTableId(ctx) - let promises: Promise[] = [] + let promises: Promise<{ row: Row; table: Table }>[] = [] for (let row of rows) { promises.push( handleRequest(Operation.DELETE, tableId, { @@ -167,7 +167,7 @@ export async function bulkDestroy(ctx: UserCtx) { }) ) } - const responses = (await Promise.all(promises)) as { row: Row }[] + const responses = await Promise.all(promises) return { response: { ok: true }, rows: responses.map(resp => resp.row) } } @@ -183,11 +183,11 @@ export async function fetchEnrichedRow(ctx: UserCtx) { ctx.throw(400, "Datasource has not been configured for plus API.") } const tables = datasource.entities - const response = (await handleRequest(Operation.READ, tableId, { + const response = await handleRequest(Operation.READ, tableId, { id, datasource, includeSqlRelationships: IncludeRelationship.INCLUDE, - })) as Row[] + }) const table: Table = tables[tableName] const row = response[0] // this seems like a lot of work, but basically we need to dig deeper for the enrich diff --git a/packages/server/src/sdk/app/rows/external.ts b/packages/server/src/sdk/app/rows/external.ts index 568bd07e9d..8bcf89a3f5 100644 --- a/packages/server/src/sdk/app/rows/external.ts +++ b/packages/server/src/sdk/app/rows/external.ts @@ -7,11 +7,11 @@ export async function getRow( rowId: string, opts?: { relationships?: boolean } ) { - const response = (await handleRequest(Operation.READ, tableId, { + const response = await handleRequest(Operation.READ, tableId, { id: breakRowIdField(rowId), includeSqlRelationships: opts?.relationships ? IncludeRelationship.INCLUDE : IncludeRelationship.EXCLUDE, - })) as Row[] + }) return response ? response[0] : response } diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index f75bd07437..ced35db9be 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,4 +1,4 @@ -import { SearchFilters, SearchParams } from "@budibase/types" +import { SearchFilters, SearchParams, Row } from "@budibase/types" import { isExternalTable } from "../../../integrations/utils" import * as internal from "./search/internal" import * as external from "./search/external" @@ -45,7 +45,7 @@ export async function exportRows( return pickApi(options.tableId).exportRows(options) } -export async function fetch(tableId: string) { +export async function fetch(tableId: string): Promise { return pickApi(tableId).fetch(tableId) } @@ -53,6 +53,6 @@ export async function fetchView( tableId: string, viewName: string, params: ViewParams -) { +): Promise { return pickApi(tableId).fetchView(viewName, params) } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 8dd141f8ef..c41efad171 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -55,15 +55,15 @@ export async function search(options: SearchParams) { try { const table = await sdk.tables.getTable(tableId) options = searchInputMapping(table, options) - let rows = (await handleRequest(Operation.READ, tableId, { + let rows = await handleRequest(Operation.READ, tableId, { filters: query, sort, paginate: paginateObj as PaginationJson, includeSqlRelationships: IncludeRelationship.INCLUDE, - })) as Row[] + }) let hasNextPage = false if (paginate && rows.length === limit) { - const nextRows = (await handleRequest(Operation.READ, tableId, { + const nextRows = await handleRequest(Operation.READ, tableId, { filters: query, sort, paginate: { @@ -71,7 +71,7 @@ export async function search(options: SearchParams) { page: bookmark! * limit + 1, }, includeSqlRelationships: IncludeRelationship.INCLUDE, - })) as Row[] + }) hasNextPage = nextRows.length > 0 } @@ -172,12 +172,18 @@ export async function exportRows( } } -export async function fetch(tableId: string) { - const response = await handleRequest(Operation.READ, tableId, { - includeSqlRelationships: IncludeRelationship.INCLUDE, - }) +export async function fetch(tableId: string): Promise { + const response = await handleRequest( + Operation.READ, + tableId, + { + includeSqlRelationships: IncludeRelationship.INCLUDE, + } + ) const table = await sdk.tables.getTable(tableId) - return await outputProcessing(table, response, { preserveLinks: true }) + return await outputProcessing(table, response, { + preserveLinks: true, + }) } export async function fetchView(viewName: string) { diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index d78c0213b3..779ff5f777 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -6,26 +6,26 @@ import { import env from "../../../../environment" import { fullSearch, paginatedSearch } from "./internalSearch" import { - InternalTables, - getRowParams, DocumentType, + getRowParams, + InternalTables, } from "../../../../db/utils" import { getGlobalUsersFromMetadata } from "../../../../utilities/global" import { outputProcessing } from "../../../../utilities/rowProcessor" -import { Database, Row, Table, SearchParams } from "@budibase/types" +import { Database, Row, SearchParams, Table } from "@budibase/types" import { cleanExportRows } from "../utils" import { - Format, csv, + Format, json, jsonWithSchema, } from "../../../../api/controllers/view/exporters" import * as inMemoryViews from "../../../../db/inMemoryView" import { - migrateToInMemoryView, - migrateToDesignView, getFromDesignDoc, getFromMemoryDoc, + migrateToDesignView, + migrateToInMemoryView, } from "../../../../api/controllers/view/utils" import sdk from "../../../../sdk" import { ExportRowsParams, ExportRowsResult } from "../search" @@ -139,13 +139,12 @@ export async function exportRows( } } -export async function fetch(tableId: string) { +export async function fetch(tableId: string): Promise { const db = context.getAppDB() const table = await sdk.tables.getTable(tableId) const rows = await getRawTableData(db, tableId) - const result = await outputProcessing(table, rows) - return result + return await outputProcessing(table, rows) } async function getRawTableData(db: Database, tableId: string) { From aee955cdf2c524a4c5c6c641ea01f2c31e3b58b8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 18 Oct 2023 12:11:24 +0100 Subject: [PATCH 08/13] Switching pro back to master. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 044bec6447..570d14aa44 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 044bec6447066b215932d6726c437e7ec5a9e42e +Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76 From ad3083ede6b60cddd01313555ca232244f0464e3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 18 Oct 2023 12:36:22 +0100 Subject: [PATCH 09/13] Switching to pro master before recent creator changes. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 570d14aa44..044bec6447 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76 +Subproject commit 044bec6447066b215932d6726c437e7ec5a9e42e From 5b2f55a5922c80092513ebf0e70ee1193ebff678 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Wed, 18 Oct 2023 13:36:34 +0200 Subject: [PATCH 10/13] Per user per creator changes --- .../backend-core/src/cache/writethrough.ts | 4 +- packages/backend-core/src/users/db.ts | 113 ++++++++++-------- packages/backend-core/src/users/users.ts | 18 ++- packages/backend-core/src/users/utils.ts | 1 + .../tests/core/users/users.spec.js | 54 +++++++++ .../core/utilities/structures/licenses.ts | 13 ++ .../tests/core/utilities/structures/quotas.ts | 5 +- packages/pro | 2 +- .../src/migrations/functions/syncQuotas.ts | 2 + .../functions/usageQuotas/syncCreators.ts | 13 ++ .../usageQuotas/tests/syncCreators.spec.ts | 26 ++++ .../shared-core/src/sdk/documents/users.ts | 25 ++++ packages/types/src/documents/global/quotas.ts | 1 + packages/types/src/sdk/featureFlag.ts | 3 + packages/types/src/sdk/licensing/billing.ts | 7 ++ packages/types/src/sdk/licensing/plan.ts | 4 + packages/types/src/sdk/licensing/quota.ts | 2 + 17 files changed, 237 insertions(+), 56 deletions(-) create mode 100644 packages/backend-core/tests/core/users/users.spec.js create mode 100644 packages/server/src/migrations/functions/usageQuotas/syncCreators.ts create mode 100644 packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index e64c116663..c331d791a6 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -119,8 +119,8 @@ export class Writethrough { this.writeRateMs = writeRateMs } - async put(doc: any) { - return put(this.db, doc, this.writeRateMs) + async put(doc: any, writeRateMs: number = this.writeRateMs) { + return put(this.db, doc, writeRateMs) } async get(id: string) { diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 1d02bebc32..8bb6300d4e 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -21,17 +21,21 @@ import { User, UserStatus, UserGroup, - ContextUser, } from "@budibase/types" import { getAccountHolderFromUserIds, isAdmin, + isCreator, validateUniqueUser, } from "./utils" import { searchExistingEmails } from "./lookup" import { hash } from "../utils" -type QuotaUpdateFn = (change: number, cb?: () => Promise) => Promise +type QuotaUpdateFn = ( + change: number, + creatorsChange: number, + cb?: () => Promise +) => Promise type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise type FeatureFn = () => Promise type GroupGetFn = (ids: string[]) => Promise @@ -135,7 +139,7 @@ export class UserDB { if (!fullUser.roles) { fullUser.roles = {} } - // add the active status to a user if its not provided + // add the active status to a user if it's not provided if (fullUser.status == null) { fullUser.status = UserStatus.ACTIVE } @@ -246,7 +250,8 @@ export class UserDB { } const change = dbUser ? 0 : 1 // no change if there is existing user - return UserDB.quotas.addUsers(change, async () => { + const creatorsChange = isCreator(dbUser) !== isCreator(user) ? 1 : 0 + return UserDB.quotas.addUsers(change, creatorsChange, async () => { await validateUniqueUser(email, tenantId) let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser) @@ -308,6 +313,7 @@ export class UserDB { let usersToSave: any[] = [] let newUsers: any[] = [] + let newCreators: any[] = [] const emails = newUsersRequested.map((user: User) => user.email) const existingEmails = await searchExistingEmails(emails) @@ -328,59 +334,66 @@ export class UserDB { } newUser.userGroups = groups newUsers.push(newUser) + if (isCreator(newUser)) { + newCreators.push(newUser) + } } const account = await accountSdk.getAccountByTenantId(tenantId) - return UserDB.quotas.addUsers(newUsers.length, async () => { - // create the promises array that will be called by bulkDocs - newUsers.forEach((user: any) => { - usersToSave.push( - UserDB.buildUser( - user, - { - hashPassword: true, - requirePassword: user.requirePassword, - }, - tenantId, - undefined, // no dbUser - account + return UserDB.quotas.addUsers( + newUsers.length, + newCreators.length, + async () => { + // create the promises array that will be called by bulkDocs + newUsers.forEach((user: any) => { + usersToSave.push( + UserDB.buildUser( + user, + { + hashPassword: true, + requirePassword: user.requirePassword, + }, + tenantId, + undefined, // no dbUser + account + ) ) - ) - }) + }) - const usersToBulkSave = await Promise.all(usersToSave) - await usersCore.bulkUpdateGlobalUsers(usersToBulkSave) + const usersToBulkSave = await Promise.all(usersToSave) + await usersCore.bulkUpdateGlobalUsers(usersToBulkSave) - // Post-processing of bulk added users, e.g. events and cache operations - for (const user of usersToBulkSave) { - // TODO: Refactor to bulk insert users into the info db - // instead of relying on looping tenant creation - await platform.users.addUser(tenantId, user._id, user.email) - await eventHelpers.handleSaveEvents(user, undefined) - } + // Post-processing of bulk added users, e.g. events and cache operations + for (const user of usersToBulkSave) { + // TODO: Refactor to bulk insert users into the info db + // instead of relying on looping tenant creation + await platform.users.addUser(tenantId, user._id, user.email) + await eventHelpers.handleSaveEvents(user, undefined) + } + + const saved = usersToBulkSave.map(user => { + return { + _id: user._id, + email: user.email, + } + }) + + // now update the groups + if (Array.isArray(saved) && groups) { + const groupPromises = [] + const createdUserIds = saved.map(user => user._id) + for (let groupId of groups) { + groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds)) + } + await Promise.all(groupPromises) + } - const saved = usersToBulkSave.map(user => { return { - _id: user._id, - email: user.email, + successful: saved, + unsuccessful, } - }) - - // now update the groups - if (Array.isArray(saved) && groups) { - const groupPromises = [] - const createdUserIds = saved.map(user => user._id) - for (let groupId of groups) { - groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds)) - } - await Promise.all(groupPromises) } - - return { - successful: saved, - unsuccessful, - } - }) + ) } static async bulkDelete(userIds: string[]): Promise { @@ -420,11 +433,12 @@ export class UserDB { _deleted: true, })) const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete) + const creatorsToDelete = usersToDelete.filter(isCreator) - await UserDB.quotas.removeUsers(toDelete.length) for (let user of usersToDelete) { await bulkDeleteProcessing(user) } + await UserDB.quotas.removeUsers(toDelete.length, creatorsToDelete.length) // Build Response // index users by id @@ -473,7 +487,8 @@ export class UserDB { await db.remove(userId, dbUser._rev) - await UserDB.quotas.removeUsers(1) + const creatorsToDelete = isCreator(dbUser) ? 1 : 0 + await UserDB.quotas.removeUsers(1, creatorsToDelete) await eventHelpers.handleDeleteEvents(dbUser) await cache.user.invalidateUser(userId) await sessions.invalidateSessions(userId, { reason: "deletion" }) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index b087a6b538..a64997224e 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -14,14 +14,15 @@ import { } from "../db" import { BulkDocsResponse, - ContextUser, SearchQuery, SearchQueryOperators, SearchUsersRequest, User, + ContextUser, } from "@budibase/types" -import * as context from "../context" import { getGlobalDB } from "../context" +import * as context from "../context" +import { isCreator } from "./utils" type GetOpts = { cleanup?: boolean } @@ -283,6 +284,19 @@ export async function getUserCount() { return response.total_rows } +export async function getCreatorCount() { + let creators = 0 + async function iterate(startPage?: string) { + const page = await paginatedUsers({ bookmark: startPage }) + creators += page.data.filter(isCreator).length + if (page.hasNextPage) { + await iterate(page.nextPage) + } + } + await iterate() + return creators +} + // used to remove the builder/admin permissions, for processing the // user as an app user (they may have some specific role/group export function removePortalUserPermissions(user: User | ContextUser) { diff --git a/packages/backend-core/src/users/utils.ts b/packages/backend-core/src/users/utils.ts index af0e8e10c7..0ef4b77998 100644 --- a/packages/backend-core/src/users/utils.ts +++ b/packages/backend-core/src/users/utils.ts @@ -10,6 +10,7 @@ import { getAccountByTenantId } from "../accounts" // extract from shared-core to make easily accessible from backend-core export const isBuilder = sdk.users.isBuilder export const isAdmin = sdk.users.isAdmin +export const isCreator = sdk.users.isCreator export const isGlobalBuilder = sdk.users.isGlobalBuilder export const isAdminOrBuilder = sdk.users.isAdminOrBuilder export const hasAdminPermissions = sdk.users.hasAdminPermissions diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.js new file mode 100644 index 0000000000..ae7109344a --- /dev/null +++ b/packages/backend-core/tests/core/users/users.spec.js @@ -0,0 +1,54 @@ +const _ = require('lodash/fp') +const {structures} = require("../../../tests") + +jest.mock("../../../src/context") +jest.mock("../../../src/db") + +const context = require("../../../src/context") +const db = require("../../../src/db") + +const {getCreatorCount} = require('../../../src/users/users') + +describe("Users", () => { + + let getGlobalDBMock + let getGlobalUserParamsMock + let paginationMock + + beforeEach(() => { + jest.resetAllMocks() + + getGlobalDBMock = jest.spyOn(context, "getGlobalDB") + getGlobalUserParamsMock = jest.spyOn(db, "getGlobalUserParams") + paginationMock = jest.spyOn(db, "pagination") + }) + + it("Retrieves the number of creators", async () => { + const getUsers = (offset, limit, creators = false) => { + const range = _.range(offset, limit) + const opts = creators ? {builder: {global: true}} : undefined + return range.map(() => structures.users.user(opts)) + } + const page1Data = getUsers(0, 8) + const page2Data = getUsers(8, 12, true) + getGlobalDBMock.mockImplementation(() => ({ + name : "fake-db", + allDocs: () => ({ + rows: [...page1Data, ...page2Data] + }) + })) + paginationMock.mockImplementationOnce(() => ({ + data: page1Data, + hasNextPage: true, + nextPage: "1" + })) + paginationMock.mockImplementation(() => ({ + data: page2Data, + hasNextPage: false, + nextPage: undefined + })) + const creatorsCount = await getCreatorCount() + expect(creatorsCount).toBe(4) + expect(paginationMock).toHaveBeenCalledTimes(2) + }) +}) diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts index 5cce84edfd..bb452f9ad5 100644 --- a/packages/backend-core/tests/core/utilities/structures/licenses.ts +++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts @@ -72,6 +72,11 @@ export function quotas(): Quotas { value: 1, triggers: [], }, + creators: { + name: "Creators", + value: 1, + triggers: [], + }, userGroups: { name: "User Groups", value: 1, @@ -118,6 +123,10 @@ export function customer(): Customer { export function subscription(): Subscription { return { amount: 10000, + amounts: { + user: 10000, + creator: 0, + }, cancelAt: undefined, currency: "usd", currentPeriodEnd: 0, @@ -126,6 +135,10 @@ export function subscription(): Subscription { duration: PriceDuration.MONTHLY, pastDueAt: undefined, quantity: 0, + quantities: { + user: 0, + creator: 0, + }, status: "active", } } diff --git a/packages/backend-core/tests/core/utilities/structures/quotas.ts b/packages/backend-core/tests/core/utilities/structures/quotas.ts index e82117053f..8d0b05fe1e 100644 --- a/packages/backend-core/tests/core/utilities/structures/quotas.ts +++ b/packages/backend-core/tests/core/utilities/structures/quotas.ts @@ -1,6 +1,6 @@ import { MonthlyQuotaName, QuotaUsage } from "@budibase/types" -export const usage = (): QuotaUsage => { +export const usage = (users: number = 0, creators: number = 0): QuotaUsage => { return { _id: "usage_quota", quotaReset: new Date().toISOString(), @@ -58,7 +58,8 @@ export const usage = (): QuotaUsage => { usageQuota: { apps: 0, plugins: 0, - users: 0, + users, + creators, userGroups: 0, rows: 0, triggers: {}, diff --git a/packages/pro b/packages/pro index 044bec6447..570d14aa44 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 044bec6447066b215932d6726c437e7ec5a9e42e +Subproject commit 570d14aa44aa88f4d053856322210f0008ba5c76 diff --git a/packages/server/src/migrations/functions/syncQuotas.ts b/packages/server/src/migrations/functions/syncQuotas.ts index 67f38ba929..83a7670e78 100644 --- a/packages/server/src/migrations/functions/syncQuotas.ts +++ b/packages/server/src/migrations/functions/syncQuotas.ts @@ -3,6 +3,7 @@ import * as syncApps from "./usageQuotas/syncApps" import * as syncRows from "./usageQuotas/syncRows" import * as syncPlugins from "./usageQuotas/syncPlugins" import * as syncUsers from "./usageQuotas/syncUsers" +import * as syncCreators from "./usageQuotas/syncCreators" /** * Synchronise quotas to the state of the db. @@ -13,5 +14,6 @@ export const run = async () => { await syncRows.run() await syncPlugins.run() await syncUsers.run() + await syncCreators.run() }) } diff --git a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts b/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts new file mode 100644 index 0000000000..ce53be925a --- /dev/null +++ b/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts @@ -0,0 +1,13 @@ +import { users } from "@budibase/backend-core" +import { quotas } from "@budibase/pro" +import { QuotaUsageType, StaticQuotaName } from "@budibase/types" + +export const run = async () => { + const creatorCount = await users.getCreatorCount() + console.log(`Syncing creator count: ${creatorCount}`) + await quotas.setUsage( + creatorCount, + StaticQuotaName.CREATORS, + QuotaUsageType.STATIC + ) +} diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts new file mode 100644 index 0000000000..75fa9f217e --- /dev/null +++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts @@ -0,0 +1,26 @@ +import TestConfig from "../../../../tests/utilities/TestConfiguration" +import * as syncCreators from "../syncCreators" +import { quotas } from "@budibase/pro" + +describe("syncCreators", () => { + let config = new TestConfig(false) + + beforeEach(async () => { + await config.init() + }) + + afterAll(config.end) + + it("syncs creators", async () => { + return config.doInContext(null, async () => { + await config.createUser({ admin: true }) + + await syncCreators.run() + + const usageDoc = await quotas.getQuotaUsage() + // default + additional creator + const creatorsCount = 2 + expect(usageDoc.usageQuota.creators).toBe(creatorsCount) + }) + }) +}) diff --git a/packages/shared-core/src/sdk/documents/users.ts b/packages/shared-core/src/sdk/documents/users.ts index 03d86daa85..b58994aa46 100644 --- a/packages/shared-core/src/sdk/documents/users.ts +++ b/packages/shared-core/src/sdk/documents/users.ts @@ -6,6 +6,7 @@ import { InternalTable, } from "@budibase/types" import { getProdAppID } from "./applications" +import * as _ from "lodash/fp" // checks if a user is specifically a builder, given an app ID export function isBuilder(user: User | ContextUser, appId?: string): boolean { @@ -58,6 +59,18 @@ export function hasAppBuilderPermissions(user?: User | ContextUser): boolean { return !isGlobalBuilder && appLength != null && appLength > 0 } +export function hasAppCreatorPermissions(user?: User | ContextUser): boolean { + if (!user) { + return false + } + return _.flow( + _.get("roles"), + _.values, + _.find(x => ["CREATOR", "ADMIN"].includes(x)), + x => !!x + )(user) +} + // checks if a user is capable of building any app export function hasBuilderPermissions(user?: User | ContextUser): boolean { if (!user) { @@ -74,6 +87,18 @@ export function hasAdminPermissions(user?: User | ContextUser): boolean { return !!user.admin?.global } +export function isCreator(user?: User | ContextUser): boolean { + if (!user) { + return false + } + return ( + isGlobalBuilder(user) || + hasAdminPermissions(user) || + hasAppBuilderPermissions(user) || + hasAppCreatorPermissions(user) + ) +} + export function getGlobalUserID(userId?: string): string | undefined { if (typeof userId !== "string") { return userId diff --git a/packages/types/src/documents/global/quotas.ts b/packages/types/src/documents/global/quotas.ts index 61410f7435..4eb1168f7d 100644 --- a/packages/types/src/documents/global/quotas.ts +++ b/packages/types/src/documents/global/quotas.ts @@ -32,6 +32,7 @@ export interface StaticUsage { [StaticQuotaName.APPS]: number [StaticQuotaName.PLUGINS]: number [StaticQuotaName.USERS]: number + [StaticQuotaName.CREATORS]: number [StaticQuotaName.USER_GROUPS]: number [StaticQuotaName.ROWS]: number triggers: { diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts index 53aa4842c4..e3935bc7ee 100644 --- a/packages/types/src/sdk/featureFlag.ts +++ b/packages/types/src/sdk/featureFlag.ts @@ -1,5 +1,8 @@ export enum FeatureFlag { LICENSING = "LICENSING", + // Feature IDs in Posthog + PER_CREATOR_PER_USER_PRICE = "18873", + PER_CREATOR_PER_USER_PRICE_ALERT = "18530", } export interface TenantFeatureFlags { diff --git a/packages/types/src/sdk/licensing/billing.ts b/packages/types/src/sdk/licensing/billing.ts index 35f366c811..bcbc7abd18 100644 --- a/packages/types/src/sdk/licensing/billing.ts +++ b/packages/types/src/sdk/licensing/billing.ts @@ -5,10 +5,17 @@ export interface Customer { currency: string | null | undefined } +export interface SubscriptionItems { + user: number | undefined + creator: number | undefined +} + export interface Subscription { amount: number + amounts: SubscriptionItems | undefined currency: string quantity: number + quantities: SubscriptionItems | undefined duration: PriceDuration cancelAt: number | null | undefined currentPeriodStart: number diff --git a/packages/types/src/sdk/licensing/plan.ts b/packages/types/src/sdk/licensing/plan.ts index 3e214a01ff..1604dfb8af 100644 --- a/packages/types/src/sdk/licensing/plan.ts +++ b/packages/types/src/sdk/licensing/plan.ts @@ -4,7 +4,9 @@ export enum PlanType { PRO = "pro", /** @deprecated */ TEAM = "team", + /** @deprecated */ PREMIUM = "premium", + PREMIUM_PLUS = "premium_plus", BUSINESS = "business", ENTERPRISE = "enterprise", } @@ -26,10 +28,12 @@ export interface AvailablePrice { currency: string duration: PriceDuration priceId: string + type?: string } export enum PlanModel { PER_USER = "perUser", + PER_CREATOR_PER_USER = "per_creator_per_user", DAY_PASS = "dayPass", } diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 73afa1ed05..85700f167b 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -14,6 +14,7 @@ export enum StaticQuotaName { ROWS = "rows", APPS = "apps", USERS = "users", + CREATORS = "creators", USER_GROUPS = "userGroups", PLUGINS = "plugins", } @@ -67,6 +68,7 @@ export type StaticQuotas = { [StaticQuotaName.ROWS]: Quota [StaticQuotaName.APPS]: Quota [StaticQuotaName.USERS]: Quota + [StaticQuotaName.CREATORS]: Quota [StaticQuotaName.USER_GROUPS]: Quota [StaticQuotaName.PLUGINS]: Quota } From 4e742f0c0766902b58fe6b951bdebf0cdeb3683e Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 18 Oct 2023 12:42:55 +0000 Subject: [PATCH 12/13] Bump version to 2.11.38 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 36e24a19de..9cdf56cbbb 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.37", + "version": "2.11.38", "npmClient": "yarn", "packages": [ "packages/*" From 41e5bba533b9f8cc916eeb70f7eb78b780410502 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 19 Oct 2023 06:53:47 +0000 Subject: [PATCH 13/13] Bump version to 2.11.39 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 9cdf56cbbb..e01e5ae03e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.38", + "version": "2.11.39", "npmClient": "yarn", "packages": [ "packages/*"