Ensure that the DB always returns Documents.

This commit is contained in:
Sam Rose 2023-11-07 18:14:52 +00:00
parent 86cfd76b5a
commit fdfda100c1
No known key found for this signature in database
24 changed files with 140 additions and 122 deletions

View File

@ -175,12 +175,14 @@ export class DatabaseImpl implements Database {
return this.updateOutput(() => db.bulk({ docs: documents }))
}
async allDocs<T>(params: DatabaseQueryOpts): Promise<AllDocsResponse<T>> {
async allDocs<T extends Document>(
params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>> {
const db = await this.checkSetup()
return this.updateOutput(() => db.list(params))
}
async query<T>(
async query<T extends Document>(
viewName: string,
params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>> {

View File

@ -7,7 +7,12 @@ import {
} from "../constants"
import { getGlobalDB } from "../context"
import { doWithDB } from "./"
import { AllDocsResponse, Database, DatabaseQueryOpts } from "@budibase/types"
import {
AllDocsResponse,
Database,
DatabaseQueryOpts,
Document,
} from "@budibase/types"
import env from "../environment"
const DESIGN_DB = "_design/database"
@ -109,7 +114,7 @@ export interface QueryViewOptions {
arrayResponse?: boolean
}
export async function queryViewRaw<T>(
export async function queryViewRaw<T extends Document>(
viewName: ViewName,
params: DatabaseQueryOpts,
db: Database,
@ -137,18 +142,16 @@ export async function queryViewRaw<T>(
}
}
export const queryView = async <T>(
export const queryView = async <T extends Document>(
viewName: ViewName,
params: DatabaseQueryOpts,
db: Database,
createFunc: any,
opts?: QueryViewOptions
): Promise<T[] | T | undefined> => {
): Promise<T[] | T> => {
const response = await queryViewRaw<T>(viewName, params, db, createFunc, opts)
const rows = response.rows
const docs = rows.map((row: any) =>
params.include_docs ? row.doc : row.value
)
const docs = rows.map(row => (params.include_docs ? row.doc! : row.value))
// if arrayResponse has been requested, always return array regardless of length
if (opts?.arrayResponse) {
@ -198,11 +201,11 @@ export const createPlatformUserView = async () => {
await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
}
export const queryPlatformView = async <T>(
export const queryPlatformView = async <T extends Document>(
viewName: ViewName,
params: DatabaseQueryOpts,
opts?: QueryViewOptions
): Promise<T[] | T | undefined> => {
): Promise<T[] | T> => {
const CreateFuncByName: any = {
[ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView,
[ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
@ -220,7 +223,7 @@ const CreateFuncByName: any = {
[ViewName.USER_BY_APP]: createUserAppView,
}
export const queryGlobalView = async <T>(
export const queryGlobalView = async <T extends Document>(
viewName: ViewName,
params: DatabaseQueryOpts,
db?: Database,
@ -231,10 +234,10 @@ export const queryGlobalView = async <T>(
db = getGlobalDB()
}
const createFn = CreateFuncByName[viewName]
return queryView(viewName, params, db!, createFn, opts)
return queryView<T>(viewName, params, db!, createFn, opts)
}
export async function queryGlobalViewRaw<T>(
export async function queryGlobalViewRaw<T extends Document>(
viewName: ViewName,
params: DatabaseQueryOpts,
opts?: QueryViewOptions

View File

@ -413,15 +413,13 @@ export class UserDB {
}
// Get users and delete
const allDocsResponse: AllDocsResponse<User> = await db.allDocs({
const allDocsResponse = await db.allDocs<User>({
include_docs: true,
keys: userIds,
})
const usersToDelete: User[] = allDocsResponse.rows.map(
(user: RowResponse<User>) => {
return user.doc
}
)
const usersToDelete = allDocsResponse.rows.map(user => {
return user.doc!
})
// Delete from DB
const toDelete = usersToDelete.map(user => ({

View File

@ -151,7 +151,7 @@ export const searchGlobalUsersByApp = async (
include_docs: true,
})
params.startkey = opts && opts.startkey ? opts.startkey : params.startkey
let response = await queryGlobalView(ViewName.USER_BY_APP, params)
let response = await queryGlobalView<User>(ViewName.USER_BY_APP, params)
if (!response) {
response = []

View File

@ -30,12 +30,12 @@ export async function destroy(ctx: BBContext) {
layoutRev = ctx.params.layoutRev
const layoutsUsedByScreens = (
await db.allDocs(
await db.allDocs<any>(
getScreenParams(null, {
include_docs: true,
})
)
).rows.map(element => element.doc.layoutId)
).rows.map(element => element.doc!.layoutId)
if (layoutsUsedByScreens.includes(layoutId)) {
ctx.throw(400, "Cannot delete a layout that's being used by a screen")
}

View File

@ -25,12 +25,12 @@ const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS
// utility function to stop this repetition - permissions always stored under roles
async function getAllDBRoles(db: Database) {
const body = await db.allDocs(
const body = await db.allDocs<Role>(
getRoleParams(null, {
include_docs: true,
})
)
return body.rows.map(row => row.doc)
return body.rows.map(row => row.doc!)
}
async function updatePermissionOnRole(
@ -79,7 +79,7 @@ async function updatePermissionOnRole(
) {
rolePermissions[resourceId] =
typeof rolePermissions[resourceId] === "string"
? [rolePermissions[resourceId]]
? [rolePermissions[resourceId] as unknown as string]
: []
}
// handle the removal/updating the role which has this permission first

View File

@ -6,7 +6,13 @@ import {
Header,
} from "@budibase/backend-core"
import { getUserMetadataParams, InternalTables } from "../../db/utils"
import { Database, Role, UserCtx, UserRoles } from "@budibase/types"
import {
Database,
Role,
UserCtx,
UserMetadata,
UserRoles,
} from "@budibase/types"
import { sdk as sharedSdk } from "@budibase/shared-core"
import sdk from "../../sdk"
@ -115,12 +121,12 @@ export async function destroy(ctx: UserCtx) {
const role = await db.get<Role>(roleId)
// first check no users actively attached to role
const users = (
await db.allDocs(
await db.allDocs<UserMetadata>(
getUserMetadataParams(undefined, {
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map(row => row.doc!)
const usersWithRole = users.filter(user => user.roleId === roleId)
if (usersWithRole.length !== 0) {
ctx.throw(400, "Cannot delete role when it is in use.")

View File

@ -233,17 +233,16 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
const tableId = utils.getTableId(ctx)
const rowId = ctx.params.rowId as string
// need table to work out where links go in row, as well as the link docs
let response = await Promise.all([
const [table, row, links] = await Promise.all([
sdk.tables.getTable(tableId),
utils.findRow(ctx, tableId, rowId),
linkRows.getLinkDocuments({ tableId, rowId, fieldName }),
])
const table = response[0] as Table
const row = response[1] as Row
const linkVals = response[2] as LinkDocumentValue[]
const linkVals = links as LinkDocumentValue[]
// look up the actual rows based on the ids
const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc)
let linkedRows = (await db.allDocs<Row>(params)).rows.map(row => row.doc!)
// get the linked tables
const linkTableIds = getLinkedTableIDs(table as Table)

View File

@ -86,12 +86,12 @@ export async function updateAllFormulasInTable(table: Table) {
const db = context.getAppDB()
// start by getting the raw rows (which will be written back to DB after update)
let rows = (
await db.allDocs(
await db.allDocs<Row>(
getRowParams(table._id, null, {
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map(row => row.doc!)
// now enrich the rows, note the clone so that we have the base state of the
// rows so that we don't write any of the enriched information back
let enrichedRows = await outputProcessing(table, cloneDeep(rows), {

View File

@ -53,12 +53,12 @@ export async function getViews() {
}
} else {
const views = (
await db.allDocs(
await db.allDocs<any>(
getMemoryViewParams({
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map(row => row.doc!)
for (let viewDoc of views) {
response.push({
name: viewDoc.name,

View File

@ -20,10 +20,10 @@ const JOB_OPTS = {
async function getAllAutomations() {
const db = context.getAppDB()
let automations = await db.allDocs(
let automations = await db.allDocs<Automation>(
getAutomationParams(null, { include_docs: true })
)
return automations.rows.map(row => row.doc)
return automations.rows.map(row => row.doc!)
}
async function queueRelevantRowAutomations(
@ -45,19 +45,19 @@ async function queueRelevantRowAutomations(
for (let automation of automations) {
let automationDef = automation.definition
let automationTrigger = automationDef ? automationDef.trigger : {}
let automationTrigger = automationDef?.trigger
// don't queue events which are for dev apps, only way to test automations is
// running tests on them, in production the test flag will never
// be checked due to lazy evaluation (first always false)
if (
!env.ALLOW_DEV_AUTOMATIONS &&
isDevAppID(event.appId) &&
!(await checkTestFlag(automation._id))
!(await checkTestFlag(automation._id!))
) {
continue
}
if (
automationTrigger.inputs &&
automationTrigger?.inputs &&
automationTrigger.inputs.tableId === event.row.tableId
) {
await automationQueue.add({ automation, event }, JOB_OPTS)

View File

@ -14,7 +14,14 @@ import partition from "lodash/partition"
import { getGlobalUsersFromMetadata } from "../../utilities/global"
import { processFormulas } from "../../utilities/rowProcessor"
import { context } from "@budibase/backend-core"
import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types"
import {
Table,
Row,
LinkDocumentValue,
FieldType,
LinkDocument,
ContextUser,
} from "@budibase/types"
import sdk from "../../sdk"
export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils"
@ -73,18 +80,18 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) {
const db = context.getAppDB()
const linkedRowIds = links.map(link => link.id)
const uniqueRowIds = [...new Set(linkedRowIds)]
let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map(
row => row.doc
let dbRows = (await db.allDocs<Row>(getMultiIDParams(uniqueRowIds))).rows.map(
row => row.doc!
)
// convert the unique db rows back to a full list of linked rows
const linked = linkedRowIds
.map(id => dbRows.find(row => row && row._id === id))
.filter(row => row != null)
.filter(row => row != null) as Row[]
// need to handle users as specific cases
let [users, other] = partition(linked, linkRow =>
linkRow._id.startsWith(USER_METDATA_PREFIX)
linkRow._id!.startsWith(USER_METDATA_PREFIX)
)
users = await getGlobalUsersFromMetadata(users)
users = await getGlobalUsersFromMetadata(users as ContextUser[])
return [...other, ...users]
}
@ -176,7 +183,7 @@ export async function attachFullLinkedDocs(
// clear any existing links that could be dupe'd
rows = clearRelationshipFields(table, rows)
// now get the docs and combine into the rows
let linked = []
let linked: Row[] = []
if (linksWithoutFromRow.length > 0) {
linked = await getFullLinkedDocs(linksWithoutFromRow)
}
@ -189,7 +196,7 @@ export async function attachFullLinkedDocs(
if (opts?.fromRow && opts?.fromRow?._id === link.id) {
linkedRow = opts.fromRow!
} else {
linkedRow = linked.find(row => row._id === link.id)
linkedRow = linked.find(row => row._id === link.id)!
}
if (linkedRow) {
const linkedTableId =

View File

@ -91,7 +91,7 @@ async function getImportableDocuments(db: Database) {
// map the responses to the document itself
let documents: Document[] = []
for (let response of await Promise.all(docPromises)) {
documents = documents.concat(response.rows.map(row => row.doc))
documents = documents.concat(response.rows.map(row => row.doc!))
}
// remove the _rev, stops it being written
documents.forEach(doc => {

View File

@ -3,7 +3,11 @@ import { db as dbCore, context, logging, roles } from "@budibase/backend-core"
import { User, ContextUser, UserGroup } from "@budibase/types"
import { sdk as proSdk } from "@budibase/pro"
import sdk from "../../"
import { getGlobalUsers, processUser } from "../../../utilities/global"
import {
getGlobalUsers,
getRawGlobalUsers,
processUser,
} from "../../../utilities/global"
import { generateUserMetadataID, InternalTables } from "../../../db/utils"
type DeletedUser = { _id: string; deleted: boolean }
@ -77,9 +81,7 @@ async function syncUsersToApp(
export async function syncUsersToAllApps(userIds: string[]) {
// list of users, if one has been deleted it will be undefined in array
const users = (await getGlobalUsers(userIds, {
noProcessing: true,
})) as User[]
const users = (await getRawGlobalUsers(userIds)) as User[]
const groups = await proSdk.groups.fetch()
const finalUsers: (User | DeletedUser)[] = []
for (let userId of userIds) {

View File

@ -51,12 +51,12 @@ export async function fetch(opts?: {
// Get external datasources
const datasources = (
await db.allDocs(
await db.allDocs<Datasource>(
getDatasourceParams(null, {
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map(row => row.doc!)
const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
bbInternalDb,
@ -271,5 +271,5 @@ export async function getExternalDatasources(): Promise<Datasource[]> {
})
)
return externalDatasources.rows.map(r => r.doc)
return externalDatasources.rows.map(r => r.doc!)
}

View File

@ -48,7 +48,7 @@ export async function getAllInternalTables(db?: Database): Promise<Table[]> {
if (!db) {
db = context.getAppDB()
}
const internalTables = await db.allDocs<Table[]>(
const internalTables = await db.allDocs<Table>(
getTableParams(null, {
include_docs: true,
})
@ -124,7 +124,7 @@ export async function getTables(tableIds: string[]): Promise<Table[]> {
}
if (internalTableIds.length) {
const db = context.getAppDB()
const internalTableDocs = await db.allDocs<Table[]>(
const internalTableDocs = await db.allDocs<Table>(
getMultiIDParams(internalTableIds)
)
tables = tables.concat(internalTableDocs.rows.map(row => row.doc!))

View File

@ -7,12 +7,17 @@ import {
InternalTables,
} from "../../db/utils"
import isEqual from "lodash/isEqual"
import { ContextUser, UserMetadata, User, Database } from "@budibase/types"
import {
ContextUser,
UserMetadata,
Database,
ContextUserMetadata,
} from "@budibase/types"
export function combineMetadataAndUser(
user: ContextUser,
metadata: UserMetadata | UserMetadata[]
) {
): ContextUserMetadata | null {
const metadataId = generateUserMetadataID(user._id!)
const found = Array.isArray(metadata)
? metadata.find(doc => doc._id === metadataId)
@ -51,33 +56,33 @@ export function combineMetadataAndUser(
return null
}
export async function rawUserMetadata(db?: Database) {
export async function rawUserMetadata(db?: Database): Promise<UserMetadata[]> {
if (!db) {
db = context.getAppDB()
}
return (
await db.allDocs(
await db.allDocs<UserMetadata>(
getUserMetadataParams(null, {
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map(row => row.doc!)
}
export async function fetchMetadata() {
export async function fetchMetadata(): Promise<ContextUserMetadata[]> {
const global = await getGlobalUsers()
const metadata = await rawUserMetadata()
const users = []
const users: ContextUserMetadata[] = []
for (let user of global) {
// find the metadata that matches up to the global ID
const info = metadata.find(meta => meta._id.includes(user._id))
const info = metadata.find(meta => meta._id!.includes(user._id!))
// remove these props, not for the correct DB
users.push({
...user,
...info,
tableId: InternalTables.USER_METADATA,
// make sure the ID is always a local ID, not a global one
_id: generateUserMetadataID(user._id),
_id: generateUserMetadataID(user._id!),
})
}
return users
@ -90,9 +95,10 @@ export async function syncGlobalUsers() {
if (!(await db.exists())) {
continue
}
const resp = await Promise.all([getGlobalUsers(), rawUserMetadata(db)])
const users = resp[0] as User[]
const metadata = resp[1] as UserMetadata[]
const [users, metadata] = await Promise.all([
getGlobalUsers(),
rawUserMetadata(db),
])
const toWrite = []
for (let user of users) {
const combined = combineMetadataAndUser(user, metadata)

View File

@ -71,69 +71,67 @@ export async function processUser(
return user
}
export async function getCachedSelf(ctx: UserCtx, appId: string) {
export async function getCachedSelf(
ctx: UserCtx,
appId: string
): Promise<ContextUser> {
// this has to be tenant aware, can't depend on the context to find it out
// running some middlewares before the tenancy causes context to break
const user = await cache.user.getUser(ctx.user?._id!)
return processUser(user, { appId })
}
export async function getRawGlobalUser(userId: string) {
export async function getRawGlobalUser(userId: string): Promise<User> {
const db = tenancy.getGlobalDB()
return db.get<User>(getGlobalIDFromUserMetadataID(userId))
}
export async function getGlobalUser(userId: string) {
export async function getGlobalUser(userId: string): Promise<ContextUser> {
const appId = context.getAppId()
let user = await getRawGlobalUser(userId)
return processUser(user, { appId })
}
export async function getGlobalUsers(
userIds?: string[],
opts?: { noProcessing?: boolean }
) {
const appId = context.getAppId()
export async function getRawGlobalUsers(userIds?: string[]): Promise<User[]> {
const db = tenancy.getGlobalDB()
let globalUsers
let globalUsers: User[]
if (userIds) {
globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map(
row => row.doc
globalUsers = (await db.allDocs<User>(getMultiIDParams(userIds))).rows.map(
row => row.doc!
)
} else {
globalUsers = (
await db.allDocs(
await db.allDocs<User>(
dbCore.getGlobalUserParams(null, {
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map(row => row.doc!)
}
globalUsers = globalUsers
return globalUsers
.filter(user => user != null)
.map(user => {
delete user.password
delete user.forceResetPassword
return user
})
}
if (opts?.noProcessing || !appId) {
return globalUsers
} else {
// pass in the groups, meaning we don't actually need to retrieve them for
// each user individually
const allGroups = await groups.fetch()
return Promise.all(
globalUsers.map(user => processUser(user, { groups: allGroups }))
)
}
export async function getGlobalUsers(
userIds?: string[]
): Promise<ContextUser[]> {
const users = await getRawGlobalUsers(userIds)
const allGroups = await groups.fetch()
return Promise.all(
users.map(user => processUser(user, { groups: allGroups }))
)
}
export async function getGlobalUsersFromMetadata(users: ContextUser[]) {
const globalUsers = await getGlobalUsers(users.map(user => user._id!))
return users.map(user => {
const globalUser = globalUsers.find(
globalUser => globalUser && user._id?.includes(globalUser._id)
globalUser => globalUser && user._id?.includes(globalUser._id!)
)
return {
...globalUser,

View File

@ -1,9 +1,9 @@
import { createRoutingView } from "../../db/views/staticViews"
import { ViewName, getQueryIndex, UNICODE_MAX } from "../../db/utils"
import { context } from "@budibase/backend-core"
import { ScreenRouting } from "@budibase/types"
import { ScreenRouting, Document } from "@budibase/types"
type ScreenRoutesView = {
interface ScreenRoutesView extends Document {
id: string
routing: ScreenRouting
}

View File

@ -1,6 +1,6 @@
import { Document } from "../document"
import { User } from "../global"
import { Row } from "./row"
import { ContextUser } from "../../sdk"
export interface UserMetadata extends Document {
roleId: string
email?: string
}
export type UserMetadata = User & Row
export type ContextUserMetadata = ContextUser & Row

View File

@ -1,17 +1,19 @@
import { Document } from "../"
export interface RowValue {
rev: string
deleted: boolean
}
export interface RowResponse<T> {
export interface RowResponse<T extends Document> {
id: string
key: string
error: string
value: T | RowValue
doc?: T | any
doc?: T
}
export interface AllDocsResponse<T> {
export interface AllDocsResponse<T extends Document> {
offset: number
total_rows: number
rows: RowResponse<T>[]

View File

@ -101,8 +101,10 @@ export interface Database {
opts?: DatabasePutOpts
): Promise<Nano.DocumentInsertResponse>
bulkDocs(documents: AnyDocument[]): Promise<Nano.DocumentBulkResponse[]>
allDocs<T>(params: DatabaseQueryOpts): Promise<AllDocsResponse<T>>
query<T>(
allDocs<T extends Document>(
params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>>
query<T extends Document>(
viewName: string,
params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>>

View File

@ -56,12 +56,12 @@ export async function getTemplates({
id,
}: { ownerId?: string; type?: string; id?: string } = {}) {
const db = tenancy.getGlobalDB()
const response = await db.allDocs(
const response = await db.allDocs<Template>(
dbCore.getTemplateParams(ownerId || GLOBAL_OWNER, id, {
include_docs: true,
})
)
let templates = response.rows.map(row => row.doc)
let templates = response.rows.map(row => row.doc!)
// should only be one template with ID
if (id) {
return templates[0]
@ -73,6 +73,6 @@ export async function getTemplates({
}
export async function getTemplateByPurpose(type: string, purpose: string) {
const templates = await getTemplates({ type })
const templates = (await getTemplates({ type })) as Template[]
return templates.find((template: Template) => template.purpose === purpose)
}

View File

@ -99,8 +99,6 @@ async function buildEmail(
if (!base || !body || !core) {
throw "Unable to build email, missing base components"
}
base = base.contents
body = body.contents
let name = user ? user.name : undefined
if (user && !name && user.firstName) {
@ -114,15 +112,10 @@ async function buildEmail(
user: user || {},
}
// Prepend the core template
const fullBody = core + body
body = await processString(fullBody, context)
// this should now be the core email HTML
return processString(base, {
return processString(base.contents, {
...context,
body,
body: await processString(core + body?.contents, context),
})
}