MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11250 from Budibase/BUDI-7189/clean_ctx_dependencies_from_sdk 

Clean ctx dependencies from rows SDK
This commit is contained in:
Adria Navarro 2023-07-17 19:01:40 +01:00 committed by GitHub
parent fa0a296012 deb256a013
commit 37c1c1c9b0
8 changed files with 254 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/server/src/api/controllers/row/external.ts
View File

@ -49,6 +49,7 @@ export async function handleRequest(
}
}
}
return new ExternalRequest(operation, tableId, opts?.datasource).run(
opts || {}
)

63
packages/server/src/api/controllers/row/index.ts
View File

@ -6,6 +6,8 @@ import { Ctx } from "@budibase/types"
import * as utils from "./utils"
import { gridSocket } from "../../../websockets"
import sdk from "../../../sdk"
import * as exporters from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
function pickApi(tableId: any) {
if (isExternalTable(tableId)) {
@ -65,14 +67,26 @@ export const save = async (ctx: any) => {
}
export async function fetchView(ctx: any) {
const tableId = utils.getTableId(ctx)
ctx.body = await quotas.addQuery(() => sdk.rows.fetchView(tableId, ctx), {
datasourceId: tableId,
})
const viewName = decodeURIComponent(ctx.params.viewName)
const { calculation, group, field } = ctx.query
ctx.body = await quotas.addQuery(
() =>
sdk.rows.fetchView(tableId, viewName, {
calculation,
group,
field,
}),
{
datasourceId: tableId,
}
)
}
export async function fetch(ctx: any) {
const tableId = utils.getTableId(ctx)
ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId, ctx), {
ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId), {
datasourceId: tableId,
})
}
@ -120,9 +134,14 @@ export async function destroy(ctx: any) {
export async function search(ctx: any) {
const tableId = utils.getTableId(ctx)
ctx.status = 200
ctx.body = await quotas.addQuery(() => sdk.rows.search(tableId, ctx), {
const searchParams = {
...ctx.request.body,
tableId,
}
ctx.status = 200
ctx.body = await quotas.addQuery(() => sdk.rows.search(searchParams), {
datasourceId: tableId,
})
}
@ -152,7 +171,33 @@ export async function fetchEnrichedRow(ctx: any) {
export const exportRows = async (ctx: any) => {
const tableId = utils.getTableId(ctx)
ctx.body = await quotas.addQuery(() => sdk.rows.exportRows(tableId, ctx), {
datasourceId: tableId,
})
const format = ctx.query.format
const { rows, columns, query } = ctx.request.body
if (typeof format !== "string" || !exporters.isFormat(format)) {
ctx.throw(
400,
`Format ${format} not valid. Valid values: ${Object.values(
exporters.Format
)}`
)
}
ctx.body = await quotas.addQuery(
async () => {
const { fileName, content } = await sdk.rows.exportRows({
tableId,
format,
rowIds: rows,
columns,
query,
})
ctx.attachment(fileName)
return apiFileReturn(content)
},
{
datasourceId: tableId,
}
)
}

5
packages/server/src/api/controllers/view/index.ts
View File

@ -162,7 +162,10 @@ export async function exportView(ctx: Ctx) {
let rows = ctx.body as Row[]
let schema: TableSchema = view && view.meta && view.meta.schema
const tableId = ctx.params.tableId || view.meta.tableId
const tableId =
ctx.params.tableId ||
view?.meta?.tableId ||
(viewName.startsWith(DocumentType.TABLE) && viewName)
const table: Table = await sdk.tables.getTable(tableId)
if (!schema) {
schema = table.schema

54
packages/server/src/sdk/app/rows/search.ts
View File

@ -1,17 +1,26 @@
import { Ctx, SearchFilters } from "@budibase/types"
import { SearchFilters } from "@budibase/types"
import { isExternalTable } from "../../../integrations/utils"
import * as internal from "./search/internal"
import * as external from "./search/external"
import { Format } from "../../../api/controllers/view/exporters"
export interface SearchParams {
tableId: string
paginate: boolean
query?: SearchFilters
bookmark?: number
limit: number
paginate?: boolean
query: SearchFilters
bookmark?: string
limit?: number
sort?: string
sortOrder?: string
sortType?: string
version?: string
disableEscaping?: boolean
}
export interface ViewParams {
calculation: string
group: string
field: string
}
function pickApi(tableId: any) {
@ -21,18 +30,37 @@ function pickApi(tableId: any) {
return internal
}
export async function search(tableId: string, ctx: Ctx) {
return pickApi(tableId).search(ctx)
export async function search(options: SearchParams) {
return pickApi(options.tableId).search(options)
}
export async function exportRows(tableId: string, ctx: Ctx) {
return pickApi(tableId).exportRows(ctx)
export interface ExportRowsParams {
tableId: string
format: Format
rowIds?: string[]
columns?: string[]
query: SearchFilters
}
export async function fetch(tableId: string, ctx: Ctx) {
return pickApi(tableId).fetch(ctx)
export interface ExportRowsResult {
fileName: string
content: string
}
export async function fetchView(tableId: string, ctx: Ctx) {
return pickApi(tableId).fetchView(ctx)
export async function exportRows(
options: ExportRowsParams
): Promise<ExportRowsResult> {
return pickApi(options.tableId).exportRows(options)
}
export async function fetch(tableId: string) {
return pickApi(tableId).fetch(tableId)
}
export async function fetchView(
tableId: string,
viewName: string,
params: ViewParams
) {
return pickApi(tableId).fetchView(viewName, params)
}

107
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -5,21 +5,23 @@ import {
PaginationJson,
IncludeRelationship,
Row,
Ctx,
SearchFilters,
} from "@budibase/types"
import * as exporters from "../../../../api/controllers/view/exporters"
import sdk from "../../../../sdk"
import { handleRequest } from "../../../../api/controllers/row/external"
import { breakExternalTableId } from "../../../../integrations/utils"
import { cleanExportRows } from "../utils"
import { apiFileReturn } from "../../../../utilities/fileSystem"
import { utils } from "@budibase/shared-core"
import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
import { HTTPError } from "@budibase/backend-core"
export async function search(ctx: Ctx) {
const tableId = ctx.params.tableId
const { paginate, query, ...params } = ctx.request.body
let { bookmark, limit } = params
if (!bookmark && paginate) {
export async function search(options: SearchParams) {
const { tableId } = options
const { paginate, query, ...params } = options
const { limit } = params
let bookmark = (params.bookmark && parseInt(params.bookmark)) || null
if (paginate && !bookmark) {
bookmark = 1
}
let paginateObj = {}
@ -59,14 +61,14 @@ export async function search(ctx: Ctx) {
sort,
paginate: {
limit: 1,
page: bookmark * limit + 1,
page: bookmark! * limit + 1,
},
includeSqlRelationships: IncludeRelationship.INCLUDE,
})) as Row[]
hasNextPage = nextRows.length > 0
}
// need wrapper object for bookmarks etc when paginating
return { rows, hasNextPage, bookmark: bookmark + 1 }
return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
} catch (err: any) {
if (err.message && err.message.includes("does not exist")) {
throw new Error(
@ -78,43 +80,38 @@ export async function search(ctx: Ctx) {
}
}
export async function exportRows(ctx: Ctx) {
const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
const format = ctx.query.format as string
const { columns } = ctx.request.body
const datasource = await sdk.datasources.get(datasourceId!)
if (!datasource || !datasource.entities) {
ctx.throw(400, "Datasource has not been configured for plus API.")
}
export async function exportRows(
options: ExportRowsParams
): Promise<ExportRowsResult> {
const { tableId, format, columns, rowIds } = options
const { datasourceId, tableName } = breakExternalTableId(tableId)
if (!exporters.isFormat(format)) {
ctx.throw(
400,
`Format ${format} not valid. Valid values: ${Object.values(
exporters.Format
)}`
)
}
if (ctx.request.body.rows) {
ctx.request.body = {
query: {
oneOf: {
_id: ctx.request.body.rows.map((row: string) => {
const ids = JSON.parse(
decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
let query: SearchFilters = {}
if (rowIds?.length) {
query = {
oneOf: {
_id: rowIds.map((row: string) => {
const ids = JSON.parse(
decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
)
if (ids.length > 1) {
throw new HTTPError(
"Export data does not support composite keys.",
400
)
if (ids.length > 1) {
ctx.throw(400, "Export data does not support composite keys.")
}
return ids[0]
}),
},
}
return ids[0]
}),
},
}
}
let result = await search(ctx)
const datasource = await sdk.datasources.get(datasourceId!)
if (!datasource || !datasource.entities) {
throw new HTTPError("Datasource has not been configured for plus API.", 400)
}
let result = await search({ tableId, query })
let rows: Row[] = []
// Filter data to only specified columns if required
@ -131,14 +128,14 @@ export async function exportRows(ctx: Ctx) {
}
if (!tableName) {
ctx.throw(400, "Could not find table name.")
throw new HTTPError("Could not find table name.", 400)
}
let schema = datasource.entities[tableName].schema
const schema = datasource.entities[tableName].schema
let exportRows = cleanExportRows(rows, schema, format, columns)
let headers = Object.keys(schema)
let content
let content: string
switch (format) {
case exporters.Format.CSV:
content = exporters.csv(headers, exportRows)
@ -150,28 +147,26 @@ export async function exportRows(ctx: Ctx) {
content = exporters.jsonWithSchema(schema, exportRows)
break
default:
utils.unreachable(format)
break
throw utils.unreachable(format)
}
const filename = `export.${format}`
// send down the file
ctx.attachment(filename)
return apiFileReturn(content)
const fileName = `export.${format}`
return {
fileName,
content,
}
}
export async function fetch(ctx: Ctx) {
const tableId = ctx.params.tableId
export async function fetch(tableId: string) {
return handleRequest(Operation.READ, tableId, {
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
}
export async function fetchView(ctx: Ctx) {
export async function fetchView(viewName: string) {
// there are no views in external datasources, shouldn't ever be called
// for now just fetch
const split = ctx.params.viewName.split("all_")
ctx.params.tableId = split[1] ? split[1] : split[0]
return fetch(ctx)
const split = viewName.split("all_")
const tableId = split[1] ? split[1] : split[0]
return fetch(tableId)
}

84
packages/server/src/sdk/app/rows/search/internal.ts
View File

@ -1,4 +1,7 @@
import { context } from "@budibase/backend-core"
import {
context,
SearchParams as InternalSearchParams,
} from "@budibase/backend-core"
import env from "../../../../environment"
import { fullSearch, paginatedSearch } from "./internalSearch"
import {
@ -8,7 +11,7 @@ import {
} from "../../../../db/utils"
import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import { Ctx, Database, Row } from "@budibase/types"
import { Database, Row } from "@budibase/types"
import { cleanExportRows } from "../utils"
import {
Format,
@ -16,7 +19,6 @@ import {
json,
jsonWithSchema,
} from "../../../../api/controllers/view/exporters"
import { apiFileReturn } from "../../../../utilities/fileSystem"
import * as inMemoryViews from "../../../../db/inMemoryView"
import {
migrateToInMemoryView,
@ -25,25 +27,36 @@ import {
getFromMemoryDoc,
} from "../../../../api/controllers/view/utils"
import sdk from "../../../../sdk"
import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
export async function search(options: SearchParams) {
const { tableId } = options
export async function search(ctx: Ctx) {
// Fetch the whole table when running in cypress, as search doesn't work
if (!env.COUCH_DB_URL && env.isCypress()) {
return { rows: await fetch(ctx) }
return { rows: await fetch(tableId) }
}
const { tableId } = ctx.params
const db = context.getAppDB()
const { paginate, query, ...params } = ctx.request.body
params.version = ctx.version
params.tableId = tableId
const { paginate, query } = options
const params: InternalSearchParams<any> = {
tableId: options.tableId,
sort: options.sort,
sortOrder: options.sortOrder,
sortType: options.sortType,
limit: options.limit,
bookmark: options.bookmark,
version: options.version,
disableEscaping: options.disableEscaping,
}
let table
if (params.sort && !params.sortType) {
table = await db.get(tableId)
const schema = table.schema
const sortField = schema[params.sort]
params.sortType = sortField.type == "number" ? "number" : "string"
params.sortType = sortField.type === "number" ? "number" : "string"
}
let response
@ -66,15 +79,12 @@ export async function search(ctx: Ctx) {
return response
}
export async function exportRows(ctx: Ctx) {
export async function exportRows(
options: ExportRowsParams
): Promise<ExportRowsResult> {
const { tableId, format, rowIds, columns, query } = options
const db = context.getAppDB()
const table = await db.get(ctx.params.tableId)
const rowIds = ctx.request.body.rows
let format = ctx.query.format
if (typeof format !== "string") {
ctx.throw(400, "Format parameter is not valid")
}
const { columns, query } = ctx.request.body
const table = await db.get(tableId)
let result
if (rowIds) {
@ -87,7 +97,7 @@ export async function exportRows(ctx: Ctx) {
result = await outputProcessing(table, response)
} else if (query) {
let searchResponse = await search(ctx)
let searchResponse = await search({ tableId, query })
result = searchResponse.rows
}
@ -108,26 +118,32 @@ export async function exportRows(ctx: Ctx) {
let exportRows = cleanExportRows(rows, schema, format, columns)
if (format === Format.CSV) {
ctx.attachment("export.csv")
return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
return {
fileName: "export.csv",
content: csv(Object.keys(rows[0]), exportRows),
}
} else if (format === Format.JSON) {
ctx.attachment("export.json")
return apiFileReturn(json(exportRows))
return {
fileName: "export.json",
content: json(exportRows),
}
} else if (format === Format.JSON_WITH_SCHEMA) {
ctx.attachment("export.json")
return apiFileReturn(jsonWithSchema(schema, exportRows))
return {
fileName: "export.json",
content: jsonWithSchema(schema, exportRows),
}
} else {
throw "Format not recognised"
}
}
export async function fetch(ctx: Ctx) {
export async function fetch(tableId: string) {
const db = context.getAppDB()
const tableId = ctx.params.tableId
let table = await db.get(tableId)
let rows = await getRawTableData(db, tableId)
return outputProcessing(table, rows)
const result = await outputProcessing(table, rows)
return result
}
async function getRawTableData(db: Database, tableId: string) {
@ -145,17 +161,17 @@ async function getRawTableData(db: Database, tableId: string) {
return rows as Row[]
}
export async function fetchView(ctx: Ctx) {
const viewName = decodeURIComponent(ctx.params.viewName)
export async function fetchView(
viewName: string,
options: { calculation: string; group: string; field: string }
) {
// if this is a table view being looked for just transfer to that
if (viewName.startsWith(DocumentType.TABLE)) {
ctx.params.tableId = viewName
return fetch(ctx)
return fetch(viewName)
}
const db = context.getAppDB()
const { calculation, group, field } = ctx.query
const { calculation, group, field } = options
const viewInfo = await getView(db, viewName)
let response
if (env.SELF_HOSTED) {

5
packages/server/src/sdk/app/rows/utils.ts
View File

@ -1,3 +1,4 @@
import { TableSchema } from "@budibase/types"
import { FieldTypes } from "../../../constants"
import { makeExternalQuery } from "../../../integrations/base/query"
import { Format } from "../../../api/controllers/view/exporters"
@ -11,9 +12,9 @@ export async function getDatasourceAndQuery(json: any) {
export function cleanExportRows(
rows: any[],
schema: any,
schema: TableSchema,
format: string,
columns: string[]
columns?: string[]
) {
let cleanRows = [...rows]

115
packages/server/src/sdk/tests/rows/row.spec.ts
View File

@ -1,6 +1,10 @@
import { exportRows } from "../../app/rows/search/external"
import sdk from "../.."
import { ExternalRequest } from "../../../api/controllers/row/ExternalRequest"
import { ExportRowsParams } from "../../app/rows/search"
import { Format } from "../../../api/controllers/view/exporters"
import { HTTPError } from "@budibase/backend-core"
import { Operation } from "@budibase/types"
const mockDatasourcesGet = jest.fn()
sdk.datasources.get = mockDatasourcesGet
@ -16,30 +20,21 @@ jest.mock("../../../api/controllers/view/exporters", () => ({
}))
jest.mock("../../../utilities/fileSystem")
function getUserCtx() {
return {
params: {
tableId: "datasource__tablename",
},
query: {
format: "csv",
},
request: {
body: {},
},
throw: jest.fn(() => {
throw "Err"
}),
attachment: jest.fn(),
} as any
}
describe("external row controller", () => {
describe("external row sdk", () => {
describe("exportRows", () => {
function getExportOptions(): ExportRowsParams {
return {
tableId: "datasource__tablename",
format: Format.CSV,
query: {},
}
}
const externalRequestCall = jest.fn()
beforeAll(() => {
jest
.spyOn(ExternalRequest.prototype, "run")
.mockImplementation(() => Promise.resolve([]))
.mockImplementation(externalRequestCall.mockResolvedValue([]))
})
afterEach(() => {
@ -47,15 +42,10 @@ describe("external row controller", () => {
})
it("should throw a 400 if no datasource entities are present", async () => {
let userCtx = getUserCtx()
try {
await exportRows(userCtx)
} catch (e) {
expect(userCtx.throw).toHaveBeenCalledWith(
400,
"Datasource has not been configured for plus API."
)
}
const exportOptions = getExportOptions()
await expect(exportRows(exportOptions)).rejects.toThrowError(
new HTTPError("Datasource has not been configured for plus API.", 400)
)
})
it("should handle single quotes from a row ID", async () => {
@ -66,51 +56,46 @@ describe("external row controller", () => {
},
},
}))
let userCtx = getUserCtx()
userCtx.request.body = {
rows: ["['d001']"],
}
const exportOptions = getExportOptions()
exportOptions.rowIds = ["['d001']"]
await exportRows(userCtx)
await exportRows(exportOptions)
expect(userCtx.request.body).toEqual({
query: {
oneOf: {
_id: ["d001"],
expect(ExternalRequest).toBeCalledTimes(1)
expect(ExternalRequest).toBeCalledWith(
Operation.READ,
exportOptions.tableId,
undefined
)
expect(externalRequestCall).toBeCalledTimes(1)
expect(externalRequestCall).toBeCalledWith(
expect.objectContaining({
filters: {
oneOf: {
_id: ["d001"],
},
},
},
})
})
)
})
it("should throw a 400 if any composite keys are present", async () => {
let userCtx = getUserCtx()
userCtx.request.body = {
rows: ["[123]", "['d001'%2C'10111']"],
}
try {
await exportRows(userCtx)
} catch (e) {
expect(userCtx.throw).toHaveBeenCalledWith(
400,
"Export data does not support composite keys."
)
}
const exportOptions = getExportOptions()
exportOptions.rowIds = ["[123]", "['d001'%2C'10111']"]
await expect(exportRows(exportOptions)).rejects.toThrowError(
new HTTPError("Export data does not support composite keys.", 400)
)
})
it("should throw a 400 if no table name was found", async () => {
let userCtx = getUserCtx()
userCtx.params.tableId = "datasource__"
userCtx.request.body = {
rows: ["[123]"],
}
try {
await exportRows(userCtx)
} catch (e) {
expect(userCtx.throw).toHaveBeenCalledWith(
400,
"Could not find table name."
)
}
const exportOptions = getExportOptions()
exportOptions.tableId = "datasource__"
exportOptions.rowIds = ["[123]"]
await expect(exportRows(exportOptions)).rejects.toThrowError(
new HTTPError("Could not find table name.", 400)
)
})
})
})