MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'update-pro-2' of github.com:budibase/budibase into update-pro-2

This commit is contained in:
Sam Rose 2024-08-28 16:28:55 +01:00
parent c1c6c4203e 7c5ae1546d
commit b798930514
No known key found for this signature in database
17 changed files with 114 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
packages/server/src/api/controllers/row/external.ts
View File

@ -38,7 +38,7 @@ export async function handleRequest<T extends Operation>(
}
export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const { _id, ...rowData } = ctx.request.body
const table = await sdk.tables.getTable(tableId)
@ -93,7 +93,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
}
export async function destroy(ctx: UserCtx) {
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const _id = ctx.request.body._id
const { row } = await handleRequest(Operation.DELETE, tableId, {
id: breakRowIdField(_id),
@ -104,7 +104,7 @@ export async function destroy(ctx: UserCtx) {
export async function bulkDestroy(ctx: UserCtx) {
const { rows } = ctx.request.body
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
let promises: Promise<{ row: Row; table: Table }>[] = []
for (let row of rows) {
promises.push(
@ -123,7 +123,7 @@ export async function bulkDestroy(ctx: UserCtx) {
export async function fetchEnrichedRow(ctx: UserCtx) {
const id = ctx.params.rowId
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const { datasourceId, tableName } = breakExternalTableId(tableId)
const datasource: Datasource = await sdk.datasources.get(datasourceId)
if (!datasource || !datasource.entities) {

29
packages/server/src/api/controllers/row/index.ts
View File

@ -47,7 +47,7 @@ export async function patch(
ctx: UserCtx<PatchRowRequest, PatchRowResponse>
): Promise<any> {
const appId = ctx.appId
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const body = ctx.request.body
// if it doesn't have an _id then its save
@ -72,7 +72,7 @@ export async function patch(
export const save = async (ctx: UserCtx<Row, Row>) => {
const appId = ctx.appId
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const body = ctx.request.body
// user metadata doesn't exist yet - don't allow creation
@ -97,13 +97,12 @@ export const save = async (ctx: UserCtx<Row, Row>) => {
gridSocket?.emitRowUpdate(ctx, row || squashed)
}
export async function fetchView(ctx: any) {
const tableId = utils.getTableId(ctx)
export async function fetchLegacyView(ctx: any) {
const viewName = decodeURIComponent(ctx.params.viewName)
const { calculation, group, field } = ctx.query
ctx.body = await sdk.rows.fetchView(tableId, viewName, {
ctx.body = await sdk.rows.fetchLegacyView(viewName, {
calculation,
group: calculation ? group : null,
field,
@ -111,12 +110,12 @@ export async function fetchView(ctx: any) {
}
export async function fetch(ctx: any) {
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
ctx.body = await sdk.rows.fetch(tableId)
}
export async function find(ctx: UserCtx<void, GetRowResponse>) {
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const rowId = ctx.params.rowId
ctx.body = await sdk.rows.find(tableId, rowId)
@ -132,7 +131,7 @@ function isDeleteRow(input: any): input is DeleteRow {
async function processDeleteRowsRequest(ctx: UserCtx<DeleteRowRequest>) {
let request = ctx.request.body as DeleteRows
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const processedRows = request.rows.map(row => {
let processedRow: Row = typeof row == "string" ? { _id: row } : row
@ -148,7 +147,7 @@ async function processDeleteRowsRequest(ctx: UserCtx<DeleteRowRequest>) {
}
async function deleteRows(ctx: UserCtx<DeleteRowRequest>) {
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const appId = ctx.appId
let deleteRequest = ctx.request.body as DeleteRows
@ -170,7 +169,7 @@ async function deleteRows(ctx: UserCtx<DeleteRowRequest>) {
async function deleteRow(ctx: UserCtx<DeleteRowRequest>) {
const appId = ctx.appId
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const resp = await pickApi(tableId).destroy(ctx)
if (!tableId.includes("datasource_plus")) {
@ -204,7 +203,7 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
}
export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
await context.ensureSnippetContext(true)
@ -226,7 +225,7 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
}
export async function validate(ctx: Ctx<Row, ValidateResponse>) {
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
// external tables are hard to validate currently
if (isExternalTableID(tableId)) {
ctx.body = { valid: true, errors: {} }
@ -239,14 +238,14 @@ export async function validate(ctx: Ctx<Row, ValidateResponse>) {
}
export async function fetchEnrichedRow(ctx: UserCtx<void, Row>) {
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
ctx.body = await pickApi(tableId).fetchEnrichedRow(ctx)
}
export const exportRows = async (
ctx: Ctx<ExportRowsRequest, ExportRowsResponse>
) => {
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const format = ctx.query.format
@ -279,7 +278,7 @@ export const exportRows = async (
export async function downloadAttachment(ctx: UserCtx) {
const { columnName } = ctx.params
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const rowId = ctx.params.rowId
const row = await sdk.rows.find(tableId, rowId)

11
packages/server/src/api/controllers/row/internal.ts
View File

@ -16,7 +16,6 @@ import {
PatchRowRequest,
PatchRowResponse,
Row,
Table,
UserCtx,
} from "@budibase/types"
import sdk from "../../../sdk"
@ -24,7 +23,7 @@ import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils"
import { flatten } from "lodash"
export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const inputs = ctx.request.body
const isUserTable = tableId === InternalTables.USER_METADATA
let oldRow
@ -98,7 +97,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
export async function destroy(ctx: UserCtx) {
const db = context.getAppDB()
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const { _id } = ctx.request.body
let row = await db.get<Row>(_id)
let _rev = ctx.request.body._rev || row._rev
@ -137,7 +136,7 @@ export async function destroy(ctx: UserCtx) {
}
export async function bulkDestroy(ctx: UserCtx) {
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const table = await sdk.tables.getTable(tableId)
let { rows } = ctx.request.body
@ -179,7 +178,7 @@ export async function bulkDestroy(ctx: UserCtx) {
export async function fetchEnrichedRow(ctx: UserCtx) {
const fieldName = ctx.request.query.field as string | undefined
const db = context.getAppDB()
const tableId = utils.getTableId(ctx)
const { tableId } = utils.getSourceId(ctx)
const rowId = ctx.params.rowId as string
// need table to work out where links go in row, as well as the link docs
const [table, links] = await Promise.all([
@ -197,7 +196,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
)
// get the linked tables
const linkTableIds = getLinkedTableIDs(table as Table)
const linkTableIds = getLinkedTableIDs(table.schema)
const linkTables = await sdk.tables.getTables(linkTableIds)
// perform output processing

25
packages/server/src/api/controllers/row/utils/utils.ts
View File

@ -1,4 +1,4 @@
import { InternalTables } from "../../../../db/utils"
import * as utils from "../../../../db/utils"
import { context } from "@budibase/backend-core"
import {
@ -67,7 +67,7 @@ export async function findRow(tableId: string, rowId: string) {
const db = context.getAppDB()
let row: Row
// TODO remove special user case in future
if (tableId === InternalTables.USER_METADATA) {
if (tableId === utils.InternalTables.USER_METADATA) {
row = await getFullUser(rowId)
} else {
row = await db.get(rowId)
@ -78,22 +78,25 @@ export async function findRow(tableId: string, rowId: string) {
return row
}
export function getTableId(ctx: Ctx): string {
export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } {
// top priority, use the URL first
if (ctx.params?.sourceId) {
return ctx.params.sourceId
const { sourceId } = ctx.params
if (utils.isViewID(sourceId)) {
return {
tableId: utils.extractViewInfoFromID(sourceId).tableId,
viewId: sourceId,
}
}
return { tableId: ctx.params.sourceId }
}
// now check for old way of specifying table ID
if (ctx.params?.tableId) {
return ctx.params.tableId
return { tableId: ctx.params.tableId }
}
// check body for a table ID
if (ctx.request.body?.tableId) {
return ctx.request.body.tableId
}
// now check if a specific view name
if (ctx.params?.viewName) {
return ctx.params.viewName
return { tableId: ctx.request.body.tableId }
}
throw new Error("Unable to find table ID in request")
}
@ -198,7 +201,7 @@ export async function sqlOutputProcessing(
}
export function isUserMetadataTable(tableId: string) {
return tableId === InternalTables.USER_METADATA
return tableId === utils.InternalTables.USER_METADATA
}
export async function enrichArrayContext(

4
packages/server/src/api/controllers/view/views.ts
View File

@ -2,7 +2,7 @@ import viewTemplate from "./viewBuilder"
import { apiFileReturn } from "../../../utilities/fileSystem"
import { csv, json, jsonWithSchema, Format, isFormat } from "./exporters"
import { deleteView, getView, getViews, saveView } from "./utils"
import { fetchView } from "../row"
import { fetchLegacyView } from "../row"
import { context, events } from "@budibase/backend-core"
import sdk from "../../../sdk"
import {
@ -170,7 +170,7 @@ export async function exportView(ctx: Ctx) {
ctx.params.viewName = viewName
}
await fetchView(ctx)
await fetchLegacyView(ctx)
let rows = ctx.body as Row[]
let schema: TableSchema = view && view.meta && view.meta.schema

26
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -9,7 +9,13 @@ import {
import tk from "timekeeper"
import emitter from "../../../../src/events"
import { outputProcessing } from "../../../utilities/rowProcessor"
import { context, InternalTable, tenancy } from "@budibase/backend-core"
import {
context,
InternalTable,
tenancy,
withEnv as withCoreEnv,
setEnv as setCoreEnv,
} from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import {
AttachmentFieldMetadata,
@ -69,6 +75,7 @@ async function waitForEvent(
describe.each([
["internal", undefined],
["sqs", undefined],
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
@ -76,6 +83,8 @@ describe.each([
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("/rows (%s)", (providerType, dsProvider) => {
const isInternal = dsProvider === undefined
const isLucene = providerType === "lucene"
const isSqs = providerType === "sqs"
const isMSSQL = providerType === DatabaseName.SQL_SERVER
const isOracle = providerType === DatabaseName.ORACLE
const config = setup.getConfig()
@ -83,9 +92,17 @@ describe.each([
let table: Table
let datasource: Datasource | undefined
let client: Knex | undefined
let envCleanup: (() => void) | undefined
beforeAll(async () => {
await config.init()
await withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, () => config.init())
if (isSqs) {
envCleanup = setCoreEnv({
SQS_SEARCH_ENABLE: "true",
SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()],
})
}
if (dsProvider) {
const rawDatasource = await dsProvider
datasource = await config.createDatasource({
@ -97,6 +114,9 @@ describe.each([
afterAll(async () => {
setup.afterAll()
if (envCleanup) {
envCleanup()
}
})
function saveTableRequest(
@ -346,7 +366,7 @@ describe.each([
expect(ids).toEqual(expect.arrayContaining(sequence))
})
isInternal &&
isLucene &&
it("row values are coerced", async () => {
const str: FieldSchema = {
type: FieldType.STRING,

2
packages/server/src/api/routes/view.ts
View File

@ -46,7 +46,7 @@ router
permissions.PermissionType.TABLE,
permissions.PermissionLevel.READ
),
rowController.fetchView
rowController.fetchLegacyView
)
.get("/api/views", authorized(permissions.BUILDER), viewController.v1.fetch)
.delete(

16
packages/server/src/db/linkedRows/index.ts
View File

@ -18,6 +18,7 @@ import {
LinkDocumentValue,
Row,
Table,
TableSchema,
} from "@budibase/types"
import sdk from "../../sdk"
@ -46,8 +47,8 @@ export const EventType = {
TABLE_DELETE: "table:delete",
}
function clearRelationshipFields(table: Table, rows: Row[]) {
for (let [key, field] of Object.entries(table.schema)) {
function clearRelationshipFields(schema: TableSchema, rows: Row[]) {
for (let [key, field] of Object.entries(schema)) {
if (field.type === FieldType.LINK) {
rows = rows.map(row => {
delete row[key]
@ -158,11 +159,11 @@ export async function updateLinks(args: {
* @return returns the rows with all of the enriched relationships on it.
*/
export async function attachFullLinkedDocs(
table: Table,
schema: TableSchema,
rows: Row[],
opts?: { fromRow?: Row }
) {
const linkedTableIds = getLinkedTableIDs(table)
const linkedTableIds = getLinkedTableIDs(schema)
if (linkedTableIds.length === 0) {
return rows
}
@ -182,7 +183,7 @@ export async function attachFullLinkedDocs(
}
const linkedTables = response[1] as Table[]
// clear any existing links that could be dupe'd
rows = clearRelationshipFields(table, rows)
rows = clearRelationshipFields(schema, rows)
// now get the docs and combine into the rows
let linked: Row[] = []
if (linksWithoutFromRow.length > 0) {
@ -201,7 +202,7 @@ export async function attachFullLinkedDocs(
}
if (linkedRow) {
const linkedTableId =
linkedRow.tableId || getRelatedTableForField(table, link.fieldName)
linkedRow.tableId || getRelatedTableForField(schema, link.fieldName)
const linkedTable = linkedTables.find(
table => table._id === linkedTableId
)
@ -263,7 +264,8 @@ export async function squashLinksToPrimaryDisplay(
}
const newLinks = []
for (let link of row[column]) {
const linkTblId = link.tableId || getRelatedTableForField(table, column)
const linkTblId =
link.tableId || getRelatedTableForField(table.schema, column)
const linkedTable = await getLinkedTable(linkTblId!, linkedTables)
const obj: any = { _id: link._id }
obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable)

14
packages/server/src/db/linkedRows/linkUtils.ts
View File

@ -7,6 +7,7 @@ import {
LinkDocument,
LinkDocumentValue,
Table,
TableSchema,
} from "@budibase/types"
import sdk from "../../sdk"
@ -121,8 +122,8 @@ export function getUniqueByProp(array: any[], prop: string) {
return filteredArray
}
export function getLinkedTableIDs(table: Table): string[] {
return Object.values(table.schema)
export function getLinkedTableIDs(schema: TableSchema): string[] {
return Object.values(schema)
.filter(isRelationshipColumn)
.map(column => column.tableId)
}
@ -139,13 +140,16 @@ export async function getLinkedTable(id: string, tables: Table[]) {
return linkedTable
}
export function getRelatedTableForField(table: Table, fieldName: string) {
export function getRelatedTableForField(
schema: TableSchema,
fieldName: string
) {
// look to see if its on the table, straight in the schema
const field = table.schema[fieldName]
const field = schema[fieldName]
if (field?.type === FieldType.LINK) {
return field.tableId
}
for (let column of Object.values(table.schema)) {
for (let column of Object.values(schema)) {
if (column.type === FieldType.LINK && column.fieldName === fieldName) {
return column.tableId
}

8
packages/server/src/db/tests/linkTests.spec.ts
View File

@ -34,7 +34,7 @@ describe("test link functionality", () => {
})
describe("getRelatedTableForField", () => {
let link = basicTable()
const link = basicTable()
link.schema.link = {
name: "link",
relationshipType: RelationshipType.ONE_TO_MANY,
@ -44,11 +44,13 @@ describe("test link functionality", () => {
}
it("should get the field from the table directly", () => {
expect(linkUtils.getRelatedTableForField(link, "link")).toBe("tableID")
expect(linkUtils.getRelatedTableForField(link.schema, "link")).toBe(
"tableID"
)
})
it("should get the field from the link", () => {
expect(linkUtils.getRelatedTableForField(link, "otherLink")).toBe(
expect(linkUtils.getRelatedTableForField(link.schema, "otherLink")).toBe(
"tableID"
)
})

16
packages/server/src/middleware/trimViewRowInfo.ts
View File

@ -1,33 +1,23 @@
import { Ctx, Row } from "@budibase/types"
import * as utils from "../db/utils"
import sdk from "../sdk"
import { Next } from "koa"
import { getTableId } from "../api/controllers/row/utils"
import { getSourceId } from "../api/controllers/row/utils"
export default async (ctx: Ctx<Row>, next: Next) => {
const { body } = ctx.request
let { _viewId: viewId } = body
const possibleViewId = getTableId(ctx)
if (utils.isViewID(possibleViewId)) {
viewId = possibleViewId
}
const viewId = getSourceId(ctx).viewId ?? body._viewId
// nothing to do, it is not a view (just a table ID)
if (!viewId) {
return next()
}
const { tableId } = utils.extractViewInfoFromID(viewId)
// don't need to trim delete requests
if (ctx?.method?.toLowerCase() !== "delete") {
await trimViewFields(ctx.request.body, viewId)
}
ctx.params.sourceId = tableId
ctx.params.viewId = viewId
return next()
}

5
packages/server/src/sdk/app/rows/search.ts
View File

@ -121,10 +121,9 @@ export async function fetchRaw(tableId: string): Promise<Row[]> {
return pickApi(tableId).fetchRaw(tableId)
}
export async function fetchView(
tableId: string,
export async function fetchLegacyView(
viewName: string,
params: ViewParams
): Promise<Row[]> {
return pickApi(tableId).fetchView(viewName, params)
return internal.fetchLegacyView(viewName, params)
}

8
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -272,11 +272,3 @@ export async function fetchRaw(tableId: string): Promise<Row[]> {
})
return response.rows
}
export async function fetchView(viewName: string) {
// there are no views in external datasources, shouldn't ever be called
// for now just fetch
const split = viewName.split("all_")
const tableId = split[1] ? split[1] : split[0]
return fetch(tableId)
}

2
packages/server/src/sdk/app/rows/search/internal/internal.ts
View File

@ -145,7 +145,7 @@ export async function fetchRaw(tableId: string): Promise<Row[]> {
return rows as Row[]
}
export async function fetchView(
export async function fetchLegacyView(
viewName: string,
options: { calculation: string; group: string; field: string }
): Promise<Row[]> {

13
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -3,6 +3,7 @@ import { fixAutoColumnSubType, processFormulas } from "./utils"
import {
cache,
context,
db,
HTTPError,
objectStore,
utils,
@ -262,7 +263,7 @@ export async function outputProcessing<T extends Row[] | Row>(
}
// attach any linked row information
let enriched = !opts.preserveLinks
? await linkRows.attachFullLinkedDocs(table, safeRows, {
? await linkRows.attachFullLinkedDocs(table.schema, safeRows, {
fromRow: opts?.fromRow,
})
: safeRows
@ -349,11 +350,19 @@ export async function outputProcessing<T extends Row[] | Row>(
}
// remove null properties to match internal API
const isExternal = isExternalTableID(table._id!)
if (isExternal) {
if (isExternal || db.isSqsEnabledForTenant()) {
for (const row of enriched) {
for (const key of Object.keys(row)) {
if (row[key] === null) {
delete row[key]
} else if (row[key] && table.schema[key]?.type === FieldType.LINK) {
for (const link of row[key] || []) {
for (const linkKey of Object.keys(link)) {
if (link[linkKey] === null) {
delete link[linkKey]
}
}
}
}
}
}

8
packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
View File

@ -10,6 +10,14 @@ import { outputProcessing } from ".."
import { generator, structures } from "@budibase/backend-core/tests"
import * as bbReferenceProcessor from "../bbReferenceProcessor"
jest.mock("@budibase/backend-core", () => ({
...jest.requireActual("@budibase/backend-core"),
db: {
...jest.requireActual("@budibase/backend-core").db,
isSqsEnabledForTenant: () => true,
},
}))
jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
processInputBBReference: jest.fn(),
processInputBBReferences: jest.fn(),

6
packages/server/src/websockets/grid.ts
View File

@ -4,7 +4,7 @@ import { BaseSocket } from "./websocket"
import { auth, permissions } from "@budibase/backend-core"
import http from "http"
import Koa from "koa"
import { getTableId } from "../api/controllers/row/utils"
import { getSourceId } from "../api/controllers/row/utils"
import { Row, Table, View, ViewV2 } from "@budibase/types"
import { Socket } from "socket.io"
import { GridSocketEvent } from "@budibase/shared-core"
@ -80,7 +80,7 @@ export default class GridSocket extends BaseSocket {
}
emitRowUpdate(ctx: any, row: Row) {
const resourceId = ctx.params?.viewId || getTableId(ctx)
const resourceId = ctx.params?.viewId || getSourceId(ctx)
const room = `${ctx.appId}-${resourceId}`
this.emitToRoom(ctx, room, GridSocketEvent.RowChange, {
id: row._id,
@ -89,7 +89,7 @@ export default class GridSocket extends BaseSocket {
}
emitRowDeletion(ctx: any, row: Row) {
const resourceId = ctx.params?.viewId || getTableId(ctx)
const resourceId = ctx.params?.viewId || getSourceId(ctx)
const room = `${ctx.appId}-${resourceId}`
this.emitToRoom(ctx, room, GridSocketEvent.RowChange, {
id: row._id,