MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13603 from Budibase/budi-8123/bbreference-process-single 

Clean BBReferenceProcessor code
This commit is contained in:
Adria Navarro 2024-05-03 16:49:26 +02:00 committed by GitHub
parent 849cd9599f 23d6c0dc3a
commit f44d8442ce
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 403 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
packages/server/src/api/controllers/row/utils/basic.ts
View File

@ -104,7 +104,10 @@ export function basicProcessing({
export function fixArrayTypes(row: Row, table: Table) {
for (let [fieldName, schema] of Object.entries(table.schema)) {
if (schema.type === FieldType.ARRAY && typeof row[fieldName] === "string") {
if (
[FieldType.ARRAY, FieldType.BB_REFERENCE].includes(schema.type) &&
typeof row[fieldName] === "string"
) {
try {
row[fieldName] = JSON.parse(row[fieldName])
} catch (err) {

197
packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
View File

@ -1,7 +1,6 @@
import { cache, db as dbCore } from "@budibase/backend-core"
import { utils } from "@budibase/shared-core"
import {
FieldType,
BBReferenceFieldSubType,
DocumentType,
SEPARATOR,
@ -10,39 +9,57 @@ import { InvalidBBRefError } from "./errors"
const ROW_PREFIX = DocumentType.ROW + SEPARATOR
export function processInputBBReferences(
export async function processInputBBReference(
value: string | { _id: string },
type: FieldType.BB_REFERENCE_SINGLE
): Promise<string | null>
export function processInputBBReferences(
value: string | string[] | { _id: string } | { _id: string }[],
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType
): Promise<string | null>
subtype: BBReferenceFieldSubType.USER
): Promise<string | null> {
if (value && Array.isArray(value)) {
throw "BB_REFERENCE_SINGLE cannot be an array"
}
let id = typeof value === "string" ? value : value?._id
if (!id) {
return null
}
switch (subtype) {
case BBReferenceFieldSubType.USER: {
if (id.startsWith(ROW_PREFIX)) {
id = dbCore.getGlobalIDFromUserMetadataID(id)
}
try {
await cache.user.getUser(id)
return id
} catch (e: any) {
if (e.statusCode === 404) {
throw new InvalidBBRefError(id, BBReferenceFieldSubType.USER)
}
throw e
}
}
default:
throw utils.unreachable(subtype)
}
}
export async function processInputBBReferences(
value: string | string[] | { _id: string } | { _id: string }[],
type: FieldType.BB_REFERENCE | FieldType.BB_REFERENCE_SINGLE,
subtype?: BBReferenceFieldSubType
): Promise<string | string[] | null> {
switch (type) {
case FieldType.BB_REFERENCE: {
let referenceIds: string[] = []
value: string | string[] | { _id: string }[],
subtype: BBReferenceFieldSubType
): Promise<string[] | null> {
if (!value || !value[0]) {
return null
}
if (Array.isArray(value)) {
referenceIds.push(
...value.map(idOrDoc =>
typeof idOrDoc === "string" ? idOrDoc : idOrDoc._id
)
)
} else if (typeof value !== "string") {
referenceIds.push(value._id)
} else {
referenceIds.push(
...value
let referenceIds
if (typeof value === "string") {
referenceIds = value
.split(",")
.filter(x => x)
.map((id: string) => id.trim())
.map(u => u.trim())
.filter(u => !!u)
} else {
referenceIds = value.map(idOrDoc =>
typeof idOrDoc === "string" ? idOrDoc : idOrDoc._id
)
}
@ -75,73 +92,67 @@ export async function processInputBBReferences(
return null
}
if (subtype === BBReferenceFieldSubType.USERS) {
return referenceIds
}
return referenceIds.join(",")
}
default:
throw utils.unreachable(subtype)
}
}
case FieldType.BB_REFERENCE_SINGLE: {
if (value && Array.isArray(value)) {
throw "BB_REFERENCE_SINGLE cannot be an array"
}
const id = typeof value === "string" ? value : value._id
const user = await cache.user.getUser(id)
if (!user) {
throw new InvalidBBRefError(id, BBReferenceFieldSubType.USER)
}
return user._id!
}
default:
throw utils.unreachable(type)
}
}
interface UserReferenceInfo {
_id: string
primaryDisplay: string
email: string
firstName: string
lastName: string
firstName?: string
lastName?: string
}
export function processOutputBBReferences(
value: string,
type: FieldType.BB_REFERENCE_SINGLE
): Promise<UserReferenceInfo | undefined>
export function processOutputBBReferences(
value: string,
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType
): Promise<UserReferenceInfo[] | undefined>
export async function processOutputBBReferences(
value: string | string[],
type: FieldType.BB_REFERENCE | FieldType.BB_REFERENCE_SINGLE,
subtype?: BBReferenceFieldSubType
) {
if (value === null || value === undefined) {
// Already processed or nothing to process
return value || undefined
export async function processOutputBBReference(
value: string | null | undefined,
subtype: BBReferenceFieldSubType.USER
): Promise<UserReferenceInfo | undefined> {
if (!value) {
return undefined
}
switch (type) {
case FieldType.BB_REFERENCE: {
switch (subtype) {
case BBReferenceFieldSubType.USER: {
let user
try {
user = await cache.user.getUser(value as string)
} catch (err: any) {
if (err.statusCode !== 404) {
throw err
}
}
if (!user) {
return undefined
}
return {
_id: user._id!,
primaryDisplay: user.email,
email: user.email,
firstName: user.firstName,
lastName: user.lastName,
}
}
default:
throw utils.unreachable(subtype)
}
}
export async function processOutputBBReferences(
value: string | null | undefined,
subtype: BBReferenceFieldSubType
): Promise<UserReferenceInfo[] | undefined> {
if (!value) {
return undefined
}
const ids =
typeof value === "string" ? value.split(",").filter(id => !!id) : value
switch (subtype) {
case undefined:
throw "Subtype must be defined"
case BBReferenceFieldSubType.USER:
case BBReferenceFieldSubType.USERS: {
const { users } = await cache.user.getUsers(ids)
@ -150,7 +161,7 @@ export async function processOutputBBReferences(
}
return users.map(u => ({
_id: u._id,
_id: u._id!,
primaryDisplay: u.email,
email: u.email,
firstName: u.firstName,
@ -160,34 +171,4 @@ export async function processOutputBBReferences(
default:
throw utils.unreachable(subtype)
}
}
case FieldType.BB_REFERENCE_SINGLE: {
if (!value) {
return undefined
}
let user
try {
user = await cache.user.getUser(value as string)
} catch (err: any) {
if (err.code !== 404) {
throw err
}
}
if (!user) {
return undefined
}
return {
_id: user._id,
primaryDisplay: user.email,
email: user.email,
firstName: user.firstName,
lastName: user.lastName,
}
}
default:
throw utils.unreachable(type)
}
}

15
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -12,7 +12,9 @@ import {
} from "@budibase/types"
import { cloneDeep } from "lodash/fp"
import {
processInputBBReference,
processInputBBReferences,
processOutputBBReference,
processOutputBBReferences,
} from "./bbReferenceProcessor"
import { isExternalTableID } from "../../integrations/utils"
@ -161,13 +163,9 @@ export async function inputProcessing(
delete clonedRow[key].url
}
} else if (field.type === FieldType.BB_REFERENCE && value) {
clonedRow[key] = await processInputBBReferences(
value,
field.type,
field.subtype
)
clonedRow[key] = await processInputBBReferences(value, field.subtype)
} else if (field.type === FieldType.BB_REFERENCE_SINGLE && value) {
clonedRow[key] = await processInputBBReferences(value, field.type)
clonedRow[key] = await processInputBBReference(value, field.subtype)
}
}
@ -253,7 +251,6 @@ export async function outputProcessing<T extends Row[] | Row>(
for (let row of enriched) {
row[property] = await processOutputBBReferences(
row[property],
column.type,
column.subtype
)
}
@ -262,9 +259,9 @@ export async function outputProcessing<T extends Row[] | Row>(
column.type == FieldType.BB_REFERENCE_SINGLE
) {
for (let row of enriched) {
row[property] = await processOutputBBReferences(
row[property] = await processOutputBBReference(
row[property],
column.type
column.subtype
)
}
}

217
packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
View File

@ -1,8 +1,10 @@
import _ from "lodash"
import * as backendCore from "@budibase/backend-core"
import { BBReferenceFieldSubType, FieldType, User } from "@budibase/types"
import { BBReferenceFieldSubType, User } from "@budibase/types"
import {
processInputBBReference,
processInputBBReferences,
processOutputBBReference,
processOutputBBReferences,
} from "../bbReferenceProcessor"
import {
@ -22,6 +24,7 @@ jest.mock("@budibase/backend-core", (): typeof backendCore => {
...actual.cache,
user: {
...actual.cache.user,
getUser: jest.fn(actual.cache.user.getUser),
getUsers: jest.fn(actual.cache.user.getUsers),
},
},
@ -31,6 +34,9 @@ jest.mock("@budibase/backend-core", (): typeof backendCore => {
const config = new DBTestConfiguration()
describe("bbReferenceProcessor", () => {
const cacheGetUserSpy = backendCore.cache.user.getUser as jest.MockedFunction<
typeof backendCore.cache.user.getUser
>
const cacheGetUsersSpy = backendCore.cache.user
.getUsers as jest.MockedFunction<typeof backendCore.cache.user.getUsers>
@ -56,6 +62,64 @@ describe("bbReferenceProcessor", () => {
jest.clearAllMocks()
})
describe("processInputBBReference", () => {
describe("subtype user", () => {
it("validate valid string id", async () => {
const user = _.sample(users)
const userId = user!._id!
const result = await config.doInTenant(() =>
processInputBBReference(userId, BBReferenceFieldSubType.USER)
)
expect(result).toEqual(userId)
expect(cacheGetUserSpy).toHaveBeenCalledTimes(1)
expect(cacheGetUserSpy).toHaveBeenCalledWith(userId)
})
it("throws an error given an invalid id", async () => {
const userId = generator.guid()
await expect(
config.doInTenant(() =>
processInputBBReference(userId, BBReferenceFieldSubType.USER)
)
).rejects.toThrow(
new InvalidBBRefError(userId, BBReferenceFieldSubType.USER)
)
})
it("validate valid user object", async () => {
const userId = _.sample(users)!._id!
const result = await config.doInTenant(() =>
processInputBBReference({ _id: userId }, BBReferenceFieldSubType.USER)
)
expect(result).toEqual(userId)
expect(cacheGetUserSpy).toHaveBeenCalledTimes(1)
expect(cacheGetUserSpy).toHaveBeenCalledWith(userId)
})
it("empty strings will return null", async () => {
const result = await config.doInTenant(() =>
processInputBBReference("", BBReferenceFieldSubType.USER)
)
expect(result).toEqual(null)
})
it("should convert user medata IDs to global IDs", async () => {
const userId = _.sample(users)!._id!
const userMetadataId = backendCore.db.generateUserMetadataID(userId)
const result = await config.doInTenant(() =>
processInputBBReference(userMetadataId, BBReferenceFieldSubType.USER)
)
expect(result).toBe(userId)
})
})
})
describe("processInputBBReferences", () => {
describe("subtype user", () => {
it("validate valid string id", async () => {
@ -63,14 +127,10 @@ describe("bbReferenceProcessor", () => {
const userId = user!._id!
const result = await config.doInTenant(() =>
processInputBBReferences(
userId,
FieldType.BB_REFERENCE,
BBReferenceFieldSubType.USER
)
processInputBBReferences(userId, BBReferenceFieldSubType.USER)
)
expect(result).toEqual(userId)
expect(result).toEqual([userId])
expect(cacheGetUsersSpy).toHaveBeenCalledTimes(1)
expect(cacheGetUsersSpy).toHaveBeenCalledWith([userId])
})
@ -80,11 +140,7 @@ describe("bbReferenceProcessor", () => {
await expect(
config.doInTenant(() =>
processInputBBReferences(
userId,
FieldType.BB_REFERENCE,
BBReferenceFieldSubType.USER
)
processInputBBReferences(userId, BBReferenceFieldSubType.USER)
)
).rejects.toThrow(
new InvalidBBRefError(userId, BBReferenceFieldSubType.USER)
@ -98,14 +154,10 @@ describe("bbReferenceProcessor", () => {
const userIdCsv = userIds.join(" , ")
const result = await config.doInTenant(() =>
processInputBBReferences(
userIdCsv,
FieldType.BB_REFERENCE,
BBReferenceFieldSubType.USER
)
processInputBBReferences(userIdCsv, BBReferenceFieldSubType.USER)
)
expect(result).toEqual(userIds.join(","))
expect(result).toEqual(userIds)
expect(cacheGetUsersSpy).toHaveBeenCalledTimes(1)
expect(cacheGetUsersSpy).toHaveBeenCalledWith(userIds)
})
@ -122,45 +174,22 @@ describe("bbReferenceProcessor", () => {
await expect(
config.doInTenant(() =>
processInputBBReferences(
userIdCsv,
FieldType.BB_REFERENCE,
BBReferenceFieldSubType.USER
)
processInputBBReferences(userIdCsv, BBReferenceFieldSubType.USER)
)
).rejects.toThrow(
new InvalidBBRefError(wrongId, BBReferenceFieldSubType.USER)
)
})
it("validate valid user object", async () => {
const userId = _.sample(users)!._id!
const result = await config.doInTenant(() =>
processInputBBReferences(
{ _id: userId },
FieldType.BB_REFERENCE,
BBReferenceFieldSubType.USER
)
)
expect(result).toEqual(userId)
expect(cacheGetUsersSpy).toHaveBeenCalledTimes(1)
expect(cacheGetUsersSpy).toHaveBeenCalledWith([userId])
})
it("validate valid user object array", async () => {
const userIds = _.sampleSize(users, 3).map(x => x._id!)
const inputUsers = _.sampleSize(users, 3).map(u => ({ _id: u._id! }))
const userIds = inputUsers.map(u => u._id)
const result = await config.doInTenant(() =>
processInputBBReferences(
userIds,
FieldType.BB_REFERENCE,
BBReferenceFieldSubType.USER
)
processInputBBReferences(inputUsers, BBReferenceFieldSubType.USER)
)
expect(result).toEqual(userIds.join(","))
expect(result).toEqual(userIds)
expect(cacheGetUsersSpy).toHaveBeenCalledTimes(1)
expect(cacheGetUsersSpy).toHaveBeenCalledWith(userIds)
})
@ -169,7 +198,7 @@ describe("bbReferenceProcessor", () => {
const result = await config.doInTenant(() =>
processInputBBReferences(
"",
FieldType.BB_REFERENCE,
BBReferenceFieldSubType.USER
)
)
@ -179,11 +208,7 @@ describe("bbReferenceProcessor", () => {
it("empty arrays will return null", async () => {
const result = await config.doInTenant(() =>
processInputBBReferences(
[],
FieldType.BB_REFERENCE,
BBReferenceFieldSubType.USER
)
processInputBBReferences([], BBReferenceFieldSubType.USER)
)
expect(result).toEqual(null)
@ -193,13 +218,44 @@ describe("bbReferenceProcessor", () => {
const userId = _.sample(users)!._id!
const userMetadataId = backendCore.db.generateUserMetadataID(userId)
const result = await config.doInTenant(() =>
processInputBBReferences(
userMetadataId,
FieldType.BB_REFERENCE,
BBReferenceFieldSubType.USER
processInputBBReferences(userMetadataId, BBReferenceFieldSubType.USER)
)
expect(result).toEqual([userId])
})
})
})
describe("processOutputBBReference", () => {
describe("subtype user", () => {
it("fetches user given a valid string id", async () => {
const user = _.sample(users)!
const userId = user._id!
const result = await config.doInTenant(() =>
processOutputBBReference(userId, BBReferenceFieldSubType.USER)
)
expect(result).toBe(userId)
expect(result).toEqual({
_id: user._id,
primaryDisplay: user.email,
email: user.email,
firstName: user.firstName,
lastName: user.lastName,
})
expect(cacheGetUserSpy).toHaveBeenCalledTimes(1)
expect(cacheGetUserSpy).toHaveBeenCalledWith(userId)
})
it("returns undefined given an unexisting user", async () => {
const userId = generator.guid()
const result = await config.doInTenant(() =>
processOutputBBReference(userId, BBReferenceFieldSubType.USER)
)
expect(result).toBeUndefined()
expect(cacheGetUserSpy).toHaveBeenCalledTimes(1)
expect(cacheGetUserSpy).toHaveBeenCalledWith(userId)
})
})
})
@ -211,11 +267,7 @@ describe("bbReferenceProcessor", () => {
const userId = user._id!
const result = await config.doInTenant(() =>
processOutputBBReferences(
userId,
FieldType.BB_REFERENCE,
BBReferenceFieldSubType.USER
)
processOutputBBReferences(userId, BBReferenceFieldSubType.USER)
)
expect(result).toEqual([
@ -239,7 +291,6 @@ describe("bbReferenceProcessor", () => {
const result = await config.doInTenant(() =>
processOutputBBReferences(
[userId1, userId2].join(","),
FieldType.BB_REFERENCE,
BBReferenceFieldSubType.USER
)
)
@ -259,6 +310,46 @@ describe("bbReferenceProcessor", () => {
expect(cacheGetUsersSpy).toHaveBeenCalledTimes(1)
expect(cacheGetUsersSpy).toHaveBeenCalledWith([userId1, userId2])
})
it("trims unexisting users user given a valid string id csv", async () => {
const [user1, user2] = _.sampleSize(users, 2)
const userId1 = user1._id!
const userId2 = user2._id!
const unexistingUserId1 = generator.guid()
const unexistingUserId2 = generator.guid()
const input = [
unexistingUserId1,
userId1,
unexistingUserId2,
userId2,
].join(",")
const result = await config.doInTenant(() =>
processOutputBBReferences(input, BBReferenceFieldSubType.USER)
)
expect(result).toHaveLength(2)
expect(result).toEqual(
expect.arrayContaining(
[user1, user2].map(u => ({
_id: u._id,
primaryDisplay: u.email,
email: u.email,
firstName: u.firstName,
lastName: u.lastName,
}))
)
)
expect(cacheGetUsersSpy).toHaveBeenCalledTimes(1)
expect(cacheGetUsersSpy).toHaveBeenCalledWith([
unexistingUserId1,
userId1,
unexistingUserId2,
userId2,
])
})
})
})
})

66
packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts
View File

@ -10,7 +10,9 @@ import {
import * as bbReferenceProcessor from "../bbReferenceProcessor"
jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
processInputBBReference: jest.fn(),
processInputBBReferences: jest.fn(),
processOutputBBReference: jest.fn(),
processOutputBBReferences: jest.fn(),
}))
@ -19,7 +21,64 @@ describe("rowProcessor - inputProcessing", () => {
jest.resetAllMocks()
})
it("processes BB references if on the schema and it's populated", async () => {
const processInputBBReferenceMock =
bbReferenceProcessor.processInputBBReference as jest.Mock
const processInputBBReferencesMock =
bbReferenceProcessor.processInputBBReferences as jest.Mock
it("processes single BB references if on the schema and it's populated", async () => {
const userId = generator.guid()
const table: Table = {
_id: generator.guid(),
name: "TestTable",
type: "table",
sourceId: INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: {
presence: true,
type: "string",
},
},
user: {
type: FieldType.BB_REFERENCE_SINGLE,
subtype: BBReferenceFieldSubType.USER,
name: "user",
constraints: {
presence: true,
type: "string",
},
},
},
}
const newRow = {
name: "Jack",
user: "123",
}
const user = structures.users.user()
processInputBBReferenceMock.mockResolvedValue(user)
const { row } = await inputProcessing(userId, table, newRow)
expect(bbReferenceProcessor.processInputBBReference).toHaveBeenCalledTimes(
1
)
expect(bbReferenceProcessor.processInputBBReference).toHaveBeenCalledWith(
"123",
"user"
)
expect(row).toEqual({ ...newRow, user })
})
it("processes multiple BB references if on the schema and it's populated", async () => {
const userId = generator.guid()
const table: Table = {
@ -56,9 +115,7 @@ describe("rowProcessor - inputProcessing", () => {
const user = structures.users.user()
;(
bbReferenceProcessor.processInputBBReferences as jest.Mock
).mockResolvedValue(user)
processInputBBReferencesMock.mockResolvedValue(user)
const { row } = await inputProcessing(userId, table, newRow)
@ -67,7 +124,6 @@ describe("rowProcessor - inputProcessing", () => {
)
expect(bbReferenceProcessor.processInputBBReferences).toHaveBeenCalledWith(
"123",
"bb_reference",
"user"
)

60
packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
View File

@ -11,7 +11,9 @@ import { generator, structures } from "@budibase/backend-core/tests"
import * as bbReferenceProcessor from "../bbReferenceProcessor"
jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
processInputBBReference: jest.fn(),
processInputBBReferences: jest.fn(),
processOutputBBReference: jest.fn(),
processOutputBBReferences: jest.fn(),
}))
@ -20,10 +22,12 @@ describe("rowProcessor - outputProcessing", () => {
jest.resetAllMocks()
})
const processOutputBBReferenceMock =
bbReferenceProcessor.processOutputBBReference as jest.Mock
const processOutputBBReferencesMock =
bbReferenceProcessor.processOutputBBReferences as jest.Mock
it("fetches bb user references given a populated field", async () => {
it("fetches single user references given a populated field", async () => {
const table: Table = {
_id: generator.guid(),
name: "TestTable",
@ -40,7 +44,7 @@ describe("rowProcessor - outputProcessing", () => {
},
},
user: {
type: FieldType.BB_REFERENCE,
type: FieldType.BB_REFERENCE_SINGLE,
subtype: BBReferenceFieldSubType.USER,
name: "user",
constraints: {
@ -57,18 +61,66 @@ describe("rowProcessor - outputProcessing", () => {
}
const user = structures.users.user()
processOutputBBReferencesMock.mockResolvedValue(user)
processOutputBBReferenceMock.mockResolvedValue(user)
const result = await outputProcessing(table, row, { squash: false })
expect(result).toEqual({ name: "Jack", user })
expect(bbReferenceProcessor.processOutputBBReference).toHaveBeenCalledTimes(
1
)
expect(bbReferenceProcessor.processOutputBBReference).toHaveBeenCalledWith(
"123",
BBReferenceFieldSubType.USER
)
})
it("fetches users references given a populated field", async () => {
const table: Table = {
_id: generator.guid(),
name: "TestTable",
type: "table",
sourceId: INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: {
presence: true,
type: "string",
},
},
users: {
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USER,
name: "users",
constraints: {
presence: false,
type: "string",
},
},
},
}
const row = {
name: "Jack",
users: "123",
}
const users = [structures.users.user()]
processOutputBBReferencesMock.mockResolvedValue(users)
const result = await outputProcessing(table, row, { squash: false })
expect(result).toEqual({ name: "Jack", users })
expect(
bbReferenceProcessor.processOutputBBReferences
).toHaveBeenCalledTimes(1)
expect(bbReferenceProcessor.processOutputBBReferences).toHaveBeenCalledWith(
"123",
FieldType.BB_REFERENCE,
BBReferenceFieldSubType.USER
)
})