MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' of github.com:Budibase/budibase into feature/BUDI-7052

This commit is contained in:
mike12345567 2023-07-18 11:32:38 +01:00
parent a04443c059 fc592342c3
commit e30509c4f9
80 changed files with 1361 additions and 1081 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/workflows/budibase_ci.yml vendored
View File

@ -12,9 +12,6 @@ on:
- master
- develop
pull_request:
branches:
- master
- develop
workflow_dispatch:
env:

2
.tool-versions
View File

@ -1,2 +1,2 @@
nodejs 14.20.1
nodejs 14.21.3
python 3.10.0

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.8.12-alpha.0",
"version": "2.8.12-alpha.3",
"npmClient": "yarn",
"packages": [
"packages/*"

2
package.json
View File

@ -53,7 +53,7 @@
"kill-all": "yarn run kill-builder && yarn run kill-server",
"dev": "yarn run kill-all && lerna run --stream dev:builder --stream",
"dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
"dev:server": "yarn run kill-server && yarn build --projects=@budibase/client && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
"dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
"dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
"dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
"test": "lerna run --stream test --stream",

2
packages/backend-core/src/auth/auth.ts
View File

@ -159,7 +159,7 @@ export async function updateUserOAuth(userId: string, oAuthConfig: any) {
try {
const db = getGlobalDB()
const dbUser = await db.get(userId)
const dbUser = await db.get<any>(userId)
//Do not overwrite the refresh token if a valid one is not provided.
if (typeof details.refreshToken !== "string") {

2
packages/backend-core/src/cache/user.ts vendored
View File

@ -12,7 +12,7 @@ const EXPIRY_SECONDS = 3600
*/
async function populateFromDB(userId: string, tenantId: string) {
const db = tenancy.getTenantDB(tenantId)
const user = await db.get(userId)
const user = await db.get<any>(userId)
user.budibaseAccess = true
if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
const account = await accounts.getAccount(user.email)

2
packages/backend-core/src/db/searchIndexes/searchIndexes.ts
View File

@ -5,7 +5,7 @@ export async function createUserIndex() {
const db = getGlobalDB()
let designDoc
try {
designDoc = await db.get("_design/database")
designDoc = await db.get<any>("_design/database")
} catch (err: any) {
if (err.status === 404) {
designDoc = { _id: "_design/database" }

4
packages/backend-core/src/users.ts
View File

@ -67,9 +67,9 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => {
export async function getById(id: string, opts?: GetOpts): Promise<User> {
const db = context.getGlobalDB()
let user = await db.get(id)
let user = await db.get<User>(id)
if (opts?.cleanup) {
user = removeUserPassword(user)
user = removeUserPassword(user) as User
}
return user
}

3
packages/builder/src/components/backend/TableNavigator/popovers/EditViewPopover.svelte
View File

@ -35,9 +35,8 @@
try {
const isSelected =
decodeURIComponent($params.viewName) === $views.selectedViewName
const name = view.name
const id = view.tableId
await views.delete(name)
await views.delete(view)
notifications.success("View deleted")
if (isSelected) {
$goto(`./table/${id}`)

2
packages/builder/src/components/common/NavItem.svelte
View File

@ -2,7 +2,7 @@
import { Icon } from "@budibase/bbui"
import { createEventDispatcher, getContext } from "svelte"
import { helpers } from "@budibase/shared-core"
import UserAvatars from "../../pages/builder/app/[application]/_components/UserAvatars.svelte"
import { UserAvatars } from "@budibase/frontend-core"
export let icon
export let withArrow = false

6
packages/builder/src/components/start/AppRow.svelte
View File

@ -2,12 +2,12 @@
import { Heading, Body, Button, Icon } from "@budibase/bbui"
import { processStringSync } from "@budibase/string-templates"
import { goto } from "@roxi/routify"
import { UserAvatar } from "@budibase/frontend-core"
import { UserAvatars } from "@budibase/frontend-core"
export let app
export let lockedAction
$: editing = app?.lockedBy != null
$: editing = app.sessions?.length
const handleDefaultClick = () => {
if (window.innerWidth < 640) {
@ -41,7 +41,7 @@
<div class="updated">
{#if editing}
Currently editing
<UserAvatar user={app.lockedBy} />
<UserAvatars users={app.sessions} />
{:else if app.updatedAt}
{processStringSync("Updated {{ duration time 'millisecond' }} ago", {
time: new Date().getTime() - new Date(app.updatedAt).getTime(),

2
packages/builder/src/pages/builder/app/[application]/_layout.svelte
View File

@ -26,7 +26,7 @@
import TourWrap from "components/portal/onboarding/TourWrap.svelte"
import TourPopover from "components/portal/onboarding/TourPopover.svelte"
import BuilderSidePanel from "./_components/BuilderSidePanel.svelte"
import UserAvatars from "./_components/UserAvatars.svelte"
import { UserAvatars } from "@budibase/frontend-core"
import { TOUR_KEYS, TOURS } from "components/portal/onboarding/tours.js"
import PreviewOverlay from "./_components/PreviewOverlay.svelte"

6
packages/builder/src/stores/backend/views.js
View File

@ -26,14 +26,12 @@ export function createViewsStore() {
}
const deleteView = async view => {
await API.deleteView(view)
await API.deleteView(view.name)
// Update tables
tables.update(state => {
const table = state.list.find(table => table._id === view.tableId)
if (table) {
delete table.views[view.name]
}
delete table.views[view.name]
return { ...state }
})
}

0
packages/builder/src/pages/builder/app/[application]/_components/UserAvatars.svelte → packages/frontend-core/src/components/UserAvatars.svelte
View File

1
packages/frontend-core/src/components/index.js
View File

@ -2,4 +2,5 @@ export { default as SplitPage } from "./SplitPage.svelte"
export { default as TestimonialPage } from "./TestimonialPage.svelte"
export { default as Testimonial } from "./Testimonial.svelte"
export { default as UserAvatar } from "./UserAvatar.svelte"
export { default as UserAvatars } from "./UserAvatars.svelte"
export { Grid } from "./grid"

2
packages/pro

@ -1 +1 @@
Subproject commit 544c7e067de69832469cde673e59501480d6d98a
Subproject commit 9c564edb37cb619cb5971e10c4317fa6e7c5bb00

4
packages/server/src/api/controllers/analytics.ts
View File

@ -1,5 +1,5 @@
import { events } from "@budibase/backend-core"
import { AnalyticsPingRequest, PingSource } from "@budibase/types"
import { AnalyticsPingRequest, App, PingSource } from "@budibase/types"
import { DocumentType, isDevAppID } from "../../db/utils"
import { context } from "@budibase/backend-core"
@ -16,7 +16,7 @@ export const ping = async (ctx: any) => {
switch (body.source) {
case PingSource.APP: {
const db = context.getAppDB({ skip_setup: true })
const appInfo = await db.get(DocumentType.APP_METADATA)
const appInfo = await db.get<App>(DocumentType.APP_METADATA)
let appId = context.getAppId()
if (isDevAppID(appId)) {

2
packages/server/src/api/controllers/apikeys.ts
View File

@ -6,7 +6,7 @@ const KEYS_DOC = dbCore.StaticDatabases.GLOBAL.docs.apiKeys
async function getBuilderMainDoc() {
const db = tenancy.getGlobalDB()
try {
return await db.get(KEYS_DOC)
return await db.get<any>(KEYS_DOC)
} catch (err) {
// doesn't exist yet, nothing to get
return {

17
packages/server/src/api/controllers/application.ts
View File

@ -49,6 +49,7 @@ import {
MigrationType,
PlanType,
Screen,
SocketSession,
UserCtx,
} from "@budibase/types"
import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
@ -183,6 +184,7 @@ export async function fetch(ctx: UserCtx) {
const appIds = apps
.filter((app: any) => app.status === "development")
.map((app: any) => app.appId)
// get the locks for all the dev apps
if (dev || all) {
const locks = await getLocksById(appIds)
@ -197,7 +199,10 @@ export async function fetch(ctx: UserCtx) {
}
}
ctx.body = await checkAppMetadata(apps)
// Enrich apps with all builder user sessions
const enrichedApps = await sdk.users.sessions.enrichApps(apps)
ctx.body = await checkAppMetadata(enrichedApps)
}
export async function fetchAppDefinition(ctx: UserCtx) {
@ -217,7 +222,7 @@ export async function fetchAppDefinition(ctx: UserCtx) {
export async function fetchAppPackage(ctx: UserCtx) {
const db = context.getAppDB()
let application = await db.get(DocumentType.APP_METADATA)
let application = await db.get<any>(DocumentType.APP_METADATA)
const layouts = await getLayouts()
let screens = await getScreens()
const license = await licensing.cache.getCachedLicense()
@ -453,7 +458,7 @@ export async function update(ctx: UserCtx) {
export async function updateClient(ctx: UserCtx) {
// Get current app version
const db = context.getAppDB()
const application = await db.get(DocumentType.APP_METADATA)
const application = await db.get<App>(DocumentType.APP_METADATA)
const currentVersion = application.version
// Update client library and manifest
@ -477,7 +482,7 @@ export async function updateClient(ctx: UserCtx) {
export async function revertClient(ctx: UserCtx) {
// Check app can be reverted
const db = context.getAppDB()
const application = await db.get(DocumentType.APP_METADATA)
const application = await db.get<App>(DocumentType.APP_METADATA)
if (!application.revertableVersion) {
ctx.throw(400, "There is no version to revert to")
}
@ -530,7 +535,7 @@ async function destroyApp(ctx: UserCtx) {
const db = dbCore.getDB(devAppId)
// standard app deletion flow
const app = await db.get(DocumentType.APP_METADATA)
const app = await db.get<App>(DocumentType.APP_METADATA)
const result = await db.destroy()
await quotas.removeApp()
await events.app.deleted(app)
@ -593,7 +598,7 @@ export async function sync(ctx: UserCtx) {
export async function updateAppPackage(appPackage: any, appId: any) {
return context.doInAppContext(appId, async () => {
const db = context.getAppDB()
const application = await db.get(DocumentType.APP_METADATA)
const application = await db.get<App>(DocumentType.APP_METADATA)
const newAppPackage = { ...application, ...appPackage }
if (appPackage._rev !== application._rev) {

3
packages/server/src/api/controllers/auth.ts
View File

@ -4,6 +4,7 @@ import { getFullUser } from "../../utilities/users"
import { roles, context } from "@budibase/backend-core"
import { groups } from "@budibase/pro"
import { ContextUser, User, Row, UserCtx } from "@budibase/types"
import sdk from "../../sdk"
const PUBLIC_ROLE = roles.BUILTIN_ROLE_IDS.PUBLIC
@ -41,7 +42,7 @@ export async function fetchSelf(ctx: UserCtx) {
// remove the full roles structure
delete user.roles
try {
const userTable = await db.get(InternalTables.USER_METADATA)
const userTable = await sdk.tables.getTable(InternalTables.USER_METADATA)
// specifically needs to make sure is enriched
ctx.body = await outputProcessing(userTable, user as Row)
} catch (err: any) {

15
packages/server/src/api/controllers/automation.ts
View File

@ -16,6 +16,7 @@ import { setTestFlag, clearTestFlag } from "../../utilities/redis"
import { context, cache, events } from "@budibase/backend-core"
import { automations, features } from "@budibase/pro"
import {
App,
Automation,
AutomationActionStepId,
AutomationResults,
@ -152,7 +153,7 @@ export async function update(ctx: BBContext) {
return
}
const oldAutomation = await db.get(automation._id)
const oldAutomation = await db.get<Automation>(automation._id)
automation = cleanAutomationInputs(automation)
automation = await checkForWebhooks({
oldAuto: oldAutomation,
@ -210,7 +211,7 @@ export async function find(ctx: BBContext) {
export async function destroy(ctx: BBContext) {
const db = context.getAppDB()
const automationId = ctx.params.id
const oldAutomation = await db.get(automationId)
const oldAutomation = await db.get<Automation>(automationId)
await checkForWebhooks({
oldAuto: oldAutomation,
})
@ -229,7 +230,7 @@ export async function clearLogError(ctx: BBContext) {
const { automationId, appId } = ctx.request.body
await context.doInAppContext(appId, async () => {
const db = context.getProdAppDB()
const metadata = await db.get(DocumentType.APP_METADATA)
const metadata = await db.get<App>(DocumentType.APP_METADATA)
if (!automationId) {
delete metadata.automationErrors
} else if (
@ -267,7 +268,7 @@ export async function getDefinitionList(ctx: BBContext) {
export async function trigger(ctx: BBContext) {
const db = context.getAppDB()
let automation = await db.get(ctx.params.id)
let automation = await db.get<Automation>(ctx.params.id)
let hasCollectStep = sdk.automations.utils.checkForCollectStep(automation)
if (hasCollectStep && (await features.isSyncAutomationsEnabled())) {
@ -312,8 +313,8 @@ function prepareTestInput(input: any) {
export async function test(ctx: BBContext) {
const db = context.getAppDB()
let automation = await db.get(ctx.params.id)
await setTestFlag(automation._id)
let automation = await db.get<Automation>(ctx.params.id)
await setTestFlag(automation._id!)
const testInput = prepareTestInput(ctx.request.body)
const response = await triggers.externalTrigger(
automation,
@ -328,7 +329,7 @@ export async function test(ctx: BBContext) {
...ctx.request.body,
occurredAt: new Date().getTime(),
})
await clearTestFlag(automation._id)
await clearTestFlag(automation._id!)
ctx.body = response
await events.automation.tested(automation)
}

4
packages/server/src/api/controllers/backup.ts
View File

@ -1,7 +1,7 @@
import sdk from "../../sdk"
import { events, context, db } from "@budibase/backend-core"
import { DocumentType } from "../../db/utils"
import { Ctx } from "@budibase/types"
import { App, Ctx } from "@budibase/types"
interface ExportAppDumpRequest {
excludeRows: boolean
@ -29,7 +29,7 @@ export async function exportAppDump(ctx: Ctx<ExportAppDumpRequest>) {
await context.doInAppContext(appId, async () => {
const appDb = context.getAppDB()
const app = await appDb.get(DocumentType.APP_METADATA)
const app = await appDb.get<App>(DocumentType.APP_METADATA)
await events.app.exported(app)
})
}

4
packages/server/src/api/controllers/component.ts
View File

@ -1,5 +1,5 @@
import { DocumentType } from "../../db/utils"
import { Plugin } from "@budibase/types"
import { App, Plugin } from "@budibase/types"
import { db as dbCore, context, tenancy } from "@budibase/backend-core"
import { getComponentLibraryManifest } from "../../utilities/fileSystem"
import { UserCtx } from "@budibase/types"
@ -7,7 +7,7 @@ import { UserCtx } from "@budibase/types"
export async function fetchAppComponentDefinitions(ctx: UserCtx) {
try {
const db = context.getAppDB()
const app = await db.get(DocumentType.APP_METADATA)
const app = await db.get<App>(DocumentType.APP_METADATA)
let componentManifests = await Promise.all(
app.componentLibraries.map(async (library: any) => {

9
packages/server/src/api/controllers/datasource.ts
View File

@ -9,7 +9,6 @@ import {
import { destroy as tableDestroy } from "./table/internal"
import { BuildSchemaErrors, InvalidColumns } from "../../constants"
import { getIntegration } from "../../integrations"
import { getDatasourceAndQuery } from "./row/utils"
import { invalidateDynamicVariables } from "../../threads/utils"
import { db as dbCore, context, events } from "@budibase/backend-core"
import {
@ -433,8 +432,7 @@ export async function destroy(ctx: UserCtx) {
}
export async function find(ctx: UserCtx) {
const db = context.getAppDB()
const datasource = await db.get(ctx.params.datasourceId)
const datasource = await sdk.datasources.get(ctx.params.datasourceId)
ctx.body = await sdk.datasources.removeSecretSingle(datasource)
}
@ -442,15 +440,14 @@ export async function find(ctx: UserCtx) {
export async function query(ctx: UserCtx) {
const queryJson = ctx.request.body
try {
ctx.body = await getDatasourceAndQuery(queryJson)
ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson)
} catch (err: any) {
ctx.throw(400, err)
}
}
export async function getExternalSchema(ctx: UserCtx) {
const db = context.getAppDB()
const datasource = await db.get(ctx.params.datasourceId)
const datasource = await sdk.datasources.get(ctx.params.datasourceId)
const enrichedDatasource = await getAndMergeDatasource(datasource)
const connector = await getConnector(enrichedDatasource)

10
packages/server/src/api/controllers/deploy/index.ts
View File

@ -7,7 +7,7 @@ import {
enableCronTrigger,
} from "../../../automations/utils"
import { backups } from "@budibase/pro"
import { AppBackupTrigger } from "@budibase/types"
import { App, AppBackupTrigger } from "@budibase/types"
import sdk from "../../../sdk"
import { builderSocket } from "../../../websockets"
@ -44,7 +44,7 @@ async function storeDeploymentHistory(deployment: any) {
let deploymentDoc
try {
// theres only one deployment doc per app database
deploymentDoc = await db.get(DocumentType.DEPLOYMENTS)
deploymentDoc = await db.get<any>(DocumentType.DEPLOYMENTS)
} catch (err) {
deploymentDoc = { _id: DocumentType.DEPLOYMENTS, history: {} }
}
@ -113,7 +113,7 @@ export async function fetchDeployments(ctx: any) {
export async function deploymentProgress(ctx: any) {
try {
const db = context.getAppDB()
const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS)
const deploymentDoc = await db.get<any>(DocumentType.DEPLOYMENTS)
ctx.body = deploymentDoc[ctx.params.deploymentId]
} catch (err) {
ctx.throw(
@ -165,9 +165,9 @@ export const publishApp = async function (ctx: any) {
// app metadata is excluded as it is likely to be in conflict
// replicate the app metadata document manually
const db = context.getProdAppDB()
const appDoc = await devDb.get(DocumentType.APP_METADATA)
const appDoc = await devDb.get<App>(DocumentType.APP_METADATA)
try {
const prodAppDoc = await db.get(DocumentType.APP_METADATA)
const prodAppDoc = await db.get<App>(DocumentType.APP_METADATA)
appDoc._rev = prodAppDoc._rev
} catch (err) {
delete appDoc._rev

5
packages/server/src/api/controllers/dev.ts
View File

@ -6,6 +6,7 @@ import { clearLock as redisClearLock } from "../../utilities/redis"
import { DocumentType } from "../../db/utils"
import { context, env as envCore } from "@budibase/backend-core"
import { events, db as dbCore, cache } from "@budibase/backend-core"
import { App } from "@budibase/types"
async function redirect(ctx: any, method: string, path: string = "global") {
const { devPath } = ctx.params
@ -81,7 +82,7 @@ export async function revert(ctx: any) {
if (!exists) {
throw new Error("App must be deployed to be reverted.")
}
const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS)
const deploymentDoc = await db.get<any>(DocumentType.DEPLOYMENTS)
if (
!deploymentDoc.history ||
Object.keys(deploymentDoc.history).length === 0
@ -104,7 +105,7 @@ export async function revert(ctx: any) {
// update appID in reverted app to be dev version again
const db = context.getAppDB()
const appDoc = await db.get(DocumentType.APP_METADATA)
const appDoc = await db.get<App>(DocumentType.APP_METADATA)
appDoc.appId = appId
appDoc.instance._id = appId
await db.put(appDoc)

2
packages/server/src/api/controllers/public/utils.ts
View File

@ -14,7 +14,7 @@ export async function addRev(
id = DocumentType.APP_METADATA
}
const db = context.getAppDB()
const dbDoc = await db.get(id)
const dbDoc = await db.get<any>(id)
body._rev = dbDoc._rev
// update ID in case it is an app ID
body._id = id

7
packages/server/src/api/controllers/query/index.ts
View File

@ -9,6 +9,7 @@ import { quotas } from "@budibase/pro"
import { events, context, utils, constants } from "@budibase/backend-core"
import sdk from "../../../sdk"
import { QueryEvent } from "../../../threads/definitions"
import { Query } from "@budibase/types"
const Runner = new Thread(ThreadType.QUERY, {
timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000,
@ -206,7 +207,7 @@ async function execute(
) {
const db = context.getAppDB()
const query = await db.get(ctx.params.queryId)
const query = await db.get<Query>(ctx.params.queryId)
const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
query.datasourceId
)
@ -275,7 +276,7 @@ export async function executeV2(
const removeDynamicVariables = async (queryId: any) => {
const db = context.getAppDB()
const query = await db.get(queryId)
const query = await db.get<Query>(queryId)
const datasource = await sdk.datasources.get(query.datasourceId)
const dynamicVariables = datasource.config?.dynamicVariables as any[]
@ -298,7 +299,7 @@ export async function destroy(ctx: any) {
const db = context.getAppDB()
const queryId = ctx.params.queryId
await removeDynamicVariables(queryId)
const query = await db.get(queryId)
const query = await db.get<Query>(queryId)
const datasource = await sdk.datasources.get(query.datasourceId)
await db.remove(ctx.params.queryId, ctx.params.revId)
ctx.message = `Query deleted.`

17
packages/server/src/api/controllers/role.ts
View File

@ -1,6 +1,7 @@
import { roles, context, events, db as dbCore } from "@budibase/backend-core"
import { getUserMetadataParams, InternalTables } from "../../db/utils"
import { UserCtx, Database } from "@budibase/types"
import { UserCtx, Database, UserRoles, Role } from "@budibase/types"
import sdk from "../../sdk"
const UpdateRolesOptions = {
CREATED: "created",
@ -13,23 +14,23 @@ async function updateRolesOnUserTable(
updateOption: string,
roleVersion: string | undefined
) {
const table = await db.get(InternalTables.USER_METADATA)
const table = await sdk.tables.getTable(InternalTables.USER_METADATA)
const schema = table.schema
const remove = updateOption === UpdateRolesOptions.REMOVED
let updated = false
for (let prop of Object.keys(schema)) {
if (prop === "roleId") {
updated = true
const constraints = schema[prop].constraints
const constraints = schema[prop].constraints!
const updatedRoleId =
roleVersion === roles.RoleIDVersion.NAME
? roles.getExternalRoleID(roleId, roleVersion)
: roleId
const indexOfRoleId = constraints.inclusion.indexOf(updatedRoleId)
const indexOfRoleId = constraints.inclusion!.indexOf(updatedRoleId)
if (remove && indexOfRoleId !== -1) {
constraints.inclusion.splice(indexOfRoleId, 1)
constraints.inclusion!.splice(indexOfRoleId, 1)
} else if (!remove && indexOfRoleId === -1) {
constraints.inclusion.push(updatedRoleId)
constraints.inclusion!.push(updatedRoleId)
}
break
}
@ -69,7 +70,7 @@ export async function save(ctx: UserCtx) {
let dbRole
if (!isCreate) {
dbRole = await db.get(_id)
dbRole = await db.get<UserRoles>(_id)
}
if (dbRole && dbRole.name !== name && isNewVersion) {
ctx.throw(400, "Cannot change custom role name")
@ -105,7 +106,7 @@ export async function destroy(ctx: UserCtx) {
// make sure has the prefix (if it has it then it won't be added)
roleId = dbCore.generateRoleID(roleId)
}
const role = await db.get(roleId)
const role = await db.get<Role>(roleId)
// first check no users actively attached to role
const users = (
await db.allDocs(

3
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -23,14 +23,13 @@ import {
isRowId,
isSQL,
} from "../../../integrations/utils"
import { getDatasourceAndQuery } from "./utils"
import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
import { FieldTypes } from "../../../constants"
import { processObjectSync } from "@budibase/string-templates"
import { cloneDeep } from "lodash/fp"
import { processDates, processFormulas } from "../../../utilities/rowProcessor"
import { db as dbCore } from "@budibase/backend-core"
import sdk from "../../../sdk"
import { isEditableColumn } from "../../../sdk/app/tables/validation"
export interface ManyRelationship {
tableId?: string

151
packages/server/src/api/controllers/row/external.ts
View File

@ -1,30 +1,20 @@
import {
FieldTypes,
NoEmptyFilterStrings,
SortDirection,
} from "../../../constants"
import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
import {
breakExternalTableId,
breakRowIdField,
} from "../../../integrations/utils"
import { ExternalRequest, RunConfig } from "./ExternalRequest"
import * as exporters from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
import {
Datasource,
IncludeRelationship,
Operation,
PaginationJson,
Row,
SortJson,
Table,
UserCtx,
} from "@budibase/types"
import sdk from "../../../sdk"
import * as utils from "./utils"
const { cleanExportRows } = require("./utils")
async function getRow(
tableId: string,
rowId: string,
@ -59,6 +49,7 @@ export async function handleRequest(
}
}
}
return new ExternalRequest(operation, tableId, opts?.datasource).run(
opts || {}
)
@ -114,21 +105,6 @@ export async function save(ctx: UserCtx) {
}
}
export async function fetchView(ctx: UserCtx) {
// there are no views in external datasources, shouldn't ever be called
// for now just fetch
const split = ctx.params.viewName.split("all_")
ctx.params.tableId = split[1] ? split[1] : split[0]
return fetch(ctx)
}
export async function fetch(ctx: UserCtx) {
const tableId = ctx.params.tableId
return handleRequest(Operation.READ, tableId, {
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
}
export async function find(ctx: UserCtx) {
const id = ctx.params.rowId
const tableId = ctx.params.tableId
@ -161,129 +137,6 @@ export async function bulkDestroy(ctx: UserCtx) {
return { response: { ok: true }, rows: responses.map(resp => resp.row) }
}
export async function search(ctx: UserCtx) {
const tableId = ctx.params.tableId
const { paginate, query, ...params } = ctx.request.body
let { bookmark, limit } = params
if (!bookmark && paginate) {
bookmark = 1
}
let paginateObj = {}
if (paginate) {
paginateObj = {
// add one so we can track if there is another page
limit: limit,
page: bookmark,
}
} else if (params && limit) {
paginateObj = {
limit: limit,
}
}
let sort: SortJson | undefined
if (params.sort) {
const direction =
params.sortOrder === "descending"
? SortDirection.DESCENDING
: SortDirection.ASCENDING
sort = {
[params.sort]: { direction },
}
}
try {
const rows = (await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
paginate: paginateObj as PaginationJson,
includeSqlRelationships: IncludeRelationship.INCLUDE,
})) as Row[]
let hasNextPage = false
if (paginate && rows.length === limit) {
const nextRows = (await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
paginate: {
limit: 1,
page: bookmark * limit + 1,
},
includeSqlRelationships: IncludeRelationship.INCLUDE,
})) as Row[]
hasNextPage = nextRows.length > 0
}
// need wrapper object for bookmarks etc when paginating
return { rows, hasNextPage, bookmark: bookmark + 1 }
} catch (err: any) {
if (err.message && err.message.includes("does not exist")) {
throw new Error(
`Table updated externally, please re-fetch - ${err.message}`
)
} else {
throw err
}
}
}
export async function exportRows(ctx: UserCtx) {
const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
const format = ctx.query.format
const { columns } = ctx.request.body
const datasource = await sdk.datasources.get(datasourceId!)
if (!datasource || !datasource.entities) {
ctx.throw(400, "Datasource has not been configured for plus API.")
}
if (ctx.request.body.rows) {
ctx.request.body = {
query: {
oneOf: {
_id: ctx.request.body.rows.map((row: string) => {
const ids = JSON.parse(
decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
)
if (ids.length > 1) {
ctx.throw(400, "Export data does not support composite keys.")
}
return ids[0]
}),
},
},
}
}
let result = await search(ctx)
let rows: Row[] = []
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.rows.length; i++) {
rows[i] = {}
for (let column of columns) {
rows[i][column] = result.rows[i][column]
}
}
} else {
rows = result.rows
}
if (!tableName) {
ctx.throw(400, "Could not find table name.")
}
let schema = datasource.entities[tableName].schema
let exportRows = cleanExportRows(rows, schema, format, columns)
let headers = Object.keys(schema)
// @ts-ignore
const exporter = exporters[format]
const filename = `export.${format}`
// send down the file
ctx.attachment(filename)
return apiFileReturn(exporter(headers, exportRows))
}
export async function fetchEnrichedRow(ctx: UserCtx) {
const id = ctx.params.rowId
const tableId = ctx.params.tableId

63
packages/server/src/api/controllers/row/index.ts
View File

@ -5,6 +5,9 @@ import { isExternalTable } from "../../../integrations/utils"
import { Ctx } from "@budibase/types"
import * as utils from "./utils"
import { gridSocket } from "../../../websockets"
import sdk from "../../../sdk"
import * as exporters from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
function pickApi(tableId: any) {
if (isExternalTable(tableId)) {
@ -64,14 +67,26 @@ export const save = async (ctx: any) => {
}
export async function fetchView(ctx: any) {
const tableId = utils.getTableId(ctx)
ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx), {
datasourceId: tableId,
})
const viewName = decodeURIComponent(ctx.params.viewName)
const { calculation, group, field } = ctx.query
ctx.body = await quotas.addQuery(
() =>
sdk.rows.fetchView(tableId, viewName, {
calculation,
group,
field,
}),
{
datasourceId: tableId,
}
)
}
export async function fetch(ctx: any) {
const tableId = utils.getTableId(ctx)
ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx), {
ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId), {
datasourceId: tableId,
})
}
@ -119,8 +134,14 @@ export async function destroy(ctx: any) {
export async function search(ctx: any) {
const tableId = utils.getTableId(ctx)
const searchParams = {
...ctx.request.body,
tableId,
}
ctx.status = 200
ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx), {
ctx.body = await quotas.addQuery(() => sdk.rows.search(searchParams), {
datasourceId: tableId,
})
}
@ -150,7 +171,33 @@ export async function fetchEnrichedRow(ctx: any) {
export const exportRows = async (ctx: any) => {
const tableId = utils.getTableId(ctx)
ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx), {
datasourceId: tableId,
})
const format = ctx.query.format
const { rows, columns, query } = ctx.request.body
if (typeof format !== "string" || !exporters.isFormat(format)) {
ctx.throw(
400,
`Format ${format} not valid. Valid values: ${Object.values(
exporters.Format
)}`
)
}
ctx.body = await quotas.addQuery(
async () => {
const { fileName, content } = await sdk.rows.exportRows({
tableId,
format,
rowIds: rows,
columns,
query,
})
ctx.attachment(fileName)
return apiFileReturn(content)
},
{
datasourceId: tableId,
}
)
}

260
packages/server/src/api/controllers/row/internal.ts
View File

@ -1,9 +1,7 @@
import * as linkRows from "../../../db/linkedRows"
import {
generateRowID,
getRowParams,
getTableIDFromRowID,
DocumentType,
InternalTables,
} from "../../../db/utils"
import * as userController from "../user"
@ -14,78 +12,11 @@ import {
} from "../../../utilities/rowProcessor"
import { FieldTypes } from "../../../constants"
import * as utils from "./utils"
import { fullSearch, paginatedSearch } from "./internalSearch"
import { getGlobalUsersFromMetadata } from "../../../utilities/global"
import * as inMemoryViews from "../../../db/inMemoryView"
import env from "../../../environment"
import {
migrateToInMemoryView,
migrateToDesignView,
getFromDesignDoc,
getFromMemoryDoc,
} from "../view/utils"
import { cloneDeep } from "lodash/fp"
import { context, db as dbCore } from "@budibase/backend-core"
import { finaliseRow, updateRelatedFormula } from "./staticFormula"
import { csv, json, jsonWithSchema, Format } from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
import {
UserCtx,
Database,
LinkDocumentValue,
Row,
Table,
} from "@budibase/types"
import { cleanExportRows } from "./utils"
const CALCULATION_TYPES = {
SUM: "sum",
COUNT: "count",
STATS: "stats",
}
async function getView(db: Database, viewName: string) {
let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
let viewInfo,
migrate = false
try {
viewInfo = await mainGetter(db, viewName)
} catch (err: any) {
// check if it can be retrieved from design doc (needs migrated)
if (err.status !== 404) {
viewInfo = null
} else {
viewInfo = await secondaryGetter(db, viewName)
migrate = !!viewInfo
}
}
if (migrate) {
await migration(db, viewName)
}
if (!viewInfo) {
throw "View does not exist."
}
return viewInfo
}
async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) {
let rows
if (tableId === InternalTables.USER_METADATA) {
await userController.fetchMetadata(ctx)
rows = ctx.body
} else {
const response = await db.allDocs(
getRowParams(tableId, null, {
include_docs: true,
})
)
rows = response.rows.map(row => row.doc)
}
return rows as Row[]
}
import { UserCtx, LinkDocumentValue, Row, Table } from "@budibase/types"
import sdk from "../../../sdk"
export async function patch(ctx: UserCtx) {
const db = context.getAppDB()
@ -94,7 +25,7 @@ export async function patch(ctx: UserCtx) {
const isUserTable = tableId === InternalTables.USER_METADATA
let oldRow
try {
let dbTable = await db.get(tableId)
let dbTable = await sdk.tables.getTable(tableId)
oldRow = await outputProcessing(
dbTable,
await utils.findRow(ctx, tableId, inputs._id)
@ -110,7 +41,7 @@ export async function patch(ctx: UserCtx) {
throw "Row does not exist"
}
}
let dbTable = await db.get(tableId)
let dbTable = await sdk.tables.getTable(tableId)
// need to build up full patch fields before coerce
let combinedRow: any = cloneDeep(oldRow)
for (let key of Object.keys(inputs)) {
@ -165,7 +96,7 @@ export async function save(ctx: UserCtx) {
}
// this returns the table and row incase they have been updated
const dbTable = await db.get(inputs.tableId)
const dbTable = await sdk.tables.getTable(inputs.tableId)
// need to copy the table so it can be differenced on way out
const tableClone = cloneDeep(dbTable)
@ -195,85 +126,9 @@ export async function save(ctx: UserCtx) {
})
}
export async function fetchView(ctx: UserCtx) {
const viewName = decodeURIComponent(ctx.params.viewName)
// if this is a table view being looked for just transfer to that
if (viewName.startsWith(DocumentType.TABLE)) {
ctx.params.tableId = viewName
return fetch(ctx)
}
const db = context.getAppDB()
const { calculation, group, field } = ctx.query
const viewInfo = await getView(db, viewName)
let response
if (env.SELF_HOSTED) {
response = await db.query(`database/${viewName}`, {
include_docs: !calculation,
group: !!group,
})
} else {
const tableId = viewInfo.meta.tableId
const data = await getRawTableData(ctx, db, tableId)
response = await inMemoryViews.runView(
viewInfo,
calculation as string,
!!group,
data
)
}
let rows
if (!calculation) {
response.rows = response.rows.map(row => row.doc)
let table
try {
table = await db.get(viewInfo.meta.tableId)
} catch (err) {
/* istanbul ignore next */
table = {
schema: {},
}
}
rows = await outputProcessing(table, response.rows)
}
if (calculation === CALCULATION_TYPES.STATS) {
response.rows = response.rows.map(row => ({
group: row.key,
field,
...row.value,
avg: row.value.sum / row.value.count,
}))
rows = response.rows
}
if (
calculation === CALCULATION_TYPES.COUNT ||
calculation === CALCULATION_TYPES.SUM
) {
rows = response.rows.map(row => ({
group: row.key,
field,
value: row.value,
}))
}
return rows
}
export async function fetch(ctx: UserCtx) {
const db = context.getAppDB()
const tableId = ctx.params.tableId
let table = await db.get(tableId)
let rows = await getRawTableData(ctx, db, tableId)
return outputProcessing(table, rows)
}
export async function find(ctx: UserCtx) {
const db = dbCore.getDB(ctx.appId)
const table = await db.get(ctx.params.tableId)
const table = await sdk.tables.getTable(ctx.params.tableId)
let row = await utils.findRow(ctx, ctx.params.tableId, ctx.params.rowId)
row = await outputProcessing(table, row)
return row
@ -282,13 +137,13 @@ export async function find(ctx: UserCtx) {
export async function destroy(ctx: UserCtx) {
const db = context.getAppDB()
const { _id } = ctx.request.body
let row = await db.get(_id)
let row = await db.get<Row>(_id)
let _rev = ctx.request.body._rev || row._rev
if (row.tableId !== ctx.params.tableId) {
throw "Supplied tableId doesn't match the row's tableId"
}
const table = await db.get(row.tableId)
const table = await sdk.tables.getTable(row.tableId)
// update the row to include full relationships before deleting them
row = await outputProcessing(table, row, { squash: false })
// now remove the relationships
@ -318,7 +173,7 @@ export async function destroy(ctx: UserCtx) {
export async function bulkDestroy(ctx: UserCtx) {
const db = context.getAppDB()
const tableId = ctx.params.tableId
const table = await db.get(tableId)
const table = await sdk.tables.getTable(tableId)
let { rows } = ctx.request.body
// before carrying out any updates, make sure the rows are ready to be returned
@ -354,108 +209,13 @@ export async function bulkDestroy(ctx: UserCtx) {
return { response: { ok: true }, rows: processedRows }
}
export async function search(ctx: UserCtx) {
// Fetch the whole table when running in cypress, as search doesn't work
if (!env.COUCH_DB_URL && env.isCypress()) {
return { rows: await fetch(ctx) }
}
const { tableId } = ctx.params
const db = context.getAppDB()
const { paginate, query, ...params } = ctx.request.body
params.version = ctx.version
params.tableId = tableId
let table
if (params.sort && !params.sortType) {
table = await db.get(tableId)
const schema = table.schema
const sortField = schema[params.sort]
params.sortType = sortField.type == "number" ? "number" : "string"
}
let response
if (paginate) {
response = await paginatedSearch(query, params)
} else {
response = await fullSearch(query, params)
}
// Enrich search results with relationships
if (response.rows && response.rows.length) {
// enrich with global users if from users table
if (tableId === InternalTables.USER_METADATA) {
response.rows = await getGlobalUsersFromMetadata(response.rows)
}
table = table || (await db.get(tableId))
response.rows = await outputProcessing(table, response.rows)
}
return response
}
export async function exportRows(ctx: UserCtx) {
const db = context.getAppDB()
const table = await db.get(ctx.params.tableId)
const rowIds = ctx.request.body.rows
let format = ctx.query.format
if (typeof format !== "string") {
ctx.throw(400, "Format parameter is not valid")
}
const { columns, query } = ctx.request.body
let result
if (rowIds) {
let response = (
await db.allDocs({
include_docs: true,
keys: rowIds,
})
).rows.map(row => row.doc)
result = await outputProcessing(table, response)
} else if (query) {
let searchResponse = await search(ctx)
result = searchResponse.rows
}
let rows: Row[] = []
let schema = table.schema
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.length; i++) {
rows[i] = {}
for (let column of columns) {
rows[i][column] = result[i][column]
}
}
} else {
rows = result
}
let exportRows = cleanExportRows(rows, schema, format, columns)
if (format === Format.CSV) {
ctx.attachment("export.csv")
return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
} else if (format === Format.JSON) {
ctx.attachment("export.json")
return apiFileReturn(json(exportRows))
} else if (format === Format.JSON_WITH_SCHEMA) {
ctx.attachment("export.json")
return apiFileReturn(jsonWithSchema(schema, exportRows))
} else {
throw "Format not recognised"
}
}
export async function fetchEnrichedRow(ctx: UserCtx) {
const db = context.getAppDB()
const tableId = ctx.params.tableId
const rowId = ctx.params.rowId
// need table to work out where links go in row
let [table, row] = await Promise.all([
db.get(tableId),
sdk.tables.getTable(tableId),
utils.findRow(ctx, tableId, rowId),
])
// get the link docs

7
packages/server/src/api/controllers/row/staticFormula.ts
View File

@ -8,8 +8,9 @@ import { FieldTypes, FormulaTypes } from "../../../constants"
import { context } from "@budibase/backend-core"
import { Table, Row } from "@budibase/types"
import * as linkRows from "../../../db/linkedRows"
const { isEqual } = require("lodash")
const { cloneDeep } = require("lodash/fp")
import sdk from "../../../sdk"
import { isEqual } from "lodash"
import { cloneDeep } from "lodash/fp"
/**
* This function runs through a list of enriched rows, looks at the rows which
@ -148,7 +149,7 @@ export async function finaliseRow(
await db.put(table)
} catch (err: any) {
if (err.status === 409) {
const updatedTable = await db.get(table._id)
const updatedTable = await sdk.tables.getTable(table._id)
let response = processAutoColumn(null, updatedTable, row, {
reprocessing: true,
})

101
packages/server/src/api/controllers/row/utils.ts
View File

@ -1,14 +1,19 @@
import { InternalTables } from "../../../db/utils"
import * as userController from "../user"
import { FieldTypes } from "../../../constants"
import { context } from "@budibase/backend-core"
import { makeExternalQuery } from "../../../integrations/base/query"
import { FieldType, Row, Table, UserCtx } from "@budibase/types"
import { Format } from "../view/exporters"
import { Ctx, FieldType, Row, Table, UserCtx } from "@budibase/types"
import { FieldTypes } from "../../../constants"
import sdk from "../../../sdk"
const validateJs = require("validate.js")
const { cloneDeep } = require("lodash/fp")
import validateJs from "validate.js"
import { cloneDeep } from "lodash/fp"
function isForeignKey(key: string, table: Table) {
const relationships = Object.values(table.schema).filter(
column => column.type === FieldType.LINK
)
return relationships.some(relationship => relationship.foreignKey === key)
}
validateJs.extend(validateJs.validators.datetime, {
parse: function (value: string) {
@ -20,19 +25,6 @@ validateJs.extend(validateJs.validators.datetime, {
},
})
function isForeignKey(key: string, table: Table) {
const relationships = Object.values(table.schema).filter(
column => column.type === FieldType.LINK
)
return relationships.some(relationship => relationship.foreignKey === key)
}
export async function getDatasourceAndQuery(json: any) {
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
return makeExternalQuery(datasource, json)
}
export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
const db = context.getAppDB()
let row
@ -52,6 +44,18 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
return row
}
export function getTableId(ctx: Ctx) {
if (ctx.request.body && ctx.request.body.tableId) {
return ctx.request.body.tableId
}
if (ctx.params && ctx.params.tableId) {
return ctx.params.tableId
}
if (ctx.params && ctx.params.viewName) {
return ctx.params.viewName
}
}
export async function validate({
tableId,
row,
@ -81,8 +85,8 @@ export async function validate({
continue
}
// special case for options, need to always allow unselected (empty)
if (type === FieldTypes.OPTIONS && constraints.inclusion) {
constraints.inclusion.push(null, "")
if (type === FieldTypes.OPTIONS && constraints?.inclusion) {
constraints.inclusion.push(null as any, "")
}
let res
@ -94,13 +98,13 @@ export async function validate({
}
row[fieldName].map((val: any) => {
if (
!constraints.inclusion.includes(val) &&
constraints.inclusion.length !== 0
!constraints?.inclusion?.includes(val) &&
constraints?.inclusion?.length !== 0
) {
errors[fieldName] = "Field not in list"
}
})
} else if (constraints.presence && row[fieldName].length === 0) {
} else if (constraints?.presence && row[fieldName].length === 0) {
// non required MultiSelect creates an empty array, which should not throw errors
errors[fieldName] = [`${fieldName} is required`]
}
@ -128,52 +132,3 @@ export async function validate({
}
return { valid: Object.keys(errors).length === 0, errors }
}
export function cleanExportRows(
rows: any[],
schema: any,
format: string,
columns: string[]
) {
let cleanRows = [...rows]
const relationships = Object.entries(schema)
.filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
.map(entry => entry[0])
relationships.forEach(column => {
cleanRows.forEach(row => {
delete row[column]
})
delete schema[column]
})
if (format === Format.CSV) {
// Intended to append empty values in export
const schemaKeys = Object.keys(schema)
for (let key of schemaKeys) {
if (columns?.length && columns.indexOf(key) > 0) {
continue
}
for (let row of cleanRows) {
if (row[key] == null) {
row[key] = undefined
}
}
}
}
return cleanRows
}
export function getTableId(ctx: any) {
if (ctx.request.body && ctx.request.body.tableId) {
return ctx.request.body.tableId
}
if (ctx.params && ctx.params.tableId) {
return ctx.params.tableId
}
if (ctx.params && ctx.params.viewName) {
return ctx.params.viewName
}
}

6
packages/server/src/api/controllers/screen.ts
View File

@ -7,7 +7,7 @@ import {
roles,
} from "@budibase/backend-core"
import { updateAppPackage } from "./application"
import { Plugin, ScreenProps, BBContext } from "@budibase/types"
import { Plugin, ScreenProps, BBContext, Screen } from "@budibase/types"
import { builderSocket } from "../../websockets"
export async function fetch(ctx: BBContext) {
@ -64,7 +64,7 @@ export async function save(ctx: BBContext) {
})
// Update the app metadata
const application = await db.get(DocumentType.APP_METADATA)
const application = await db.get<any>(DocumentType.APP_METADATA)
let usedPlugins = application.usedPlugins || []
requiredPlugins.forEach((plugin: Plugin) => {
@ -104,7 +104,7 @@ export async function save(ctx: BBContext) {
export async function destroy(ctx: BBContext) {
const db = context.getAppDB()
const id = ctx.params.screenId
const screen = await db.get(id)
const screen = await db.get<Screen>(id)
await db.remove(id, ctx.params.screenRev)

7
packages/server/src/api/controllers/static/index.ts
View File

@ -1,7 +1,7 @@
require("svelte/register")
import { join } from "../../../utilities/centralPath"
const uuid = require("uuid")
import uuid from "uuid"
import { ObjectStoreBuckets } from "../../../constants"
import { processString } from "@budibase/string-templates"
import {
@ -16,6 +16,7 @@ import AWS from "aws-sdk"
import fs from "fs"
import sdk from "../../../sdk"
import * as pro from "@budibase/pro"
import { App } from "@budibase/types"
const send = require("koa-send")
@ -110,7 +111,7 @@ export const serveApp = async function (ctx: any) {
let db
try {
db = context.getAppDB({ skip_setup: true })
const appInfo = await db.get(DocumentType.APP_METADATA)
const appInfo = await db.get<any>(DocumentType.APP_METADATA)
let appId = context.getAppId()
if (!env.isJest()) {
@ -177,7 +178,7 @@ export const serveApp = async function (ctx: any) {
export const serveBuilderPreview = async function (ctx: any) {
const db = context.getAppDB({ skip_setup: true })
const appInfo = await db.get(DocumentType.APP_METADATA)
const appInfo = await db.get<App>(DocumentType.APP_METADATA)
if (!env.isJest()) {
let appId = context.getAppId()

4
packages/server/src/api/controllers/table/external.ts
View File

@ -323,7 +323,7 @@ export async function save(ctx: UserCtx) {
// Since tables are stored inside datasources, we need to notify clients
// that the datasource definition changed
const updatedDatasource = await db.get(datasource._id)
const updatedDatasource = await sdk.datasources.get(datasource._id!)
builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource)
return tableToSave
@ -354,7 +354,7 @@ export async function destroy(ctx: UserCtx) {
// Since tables are stored inside datasources, we need to notify clients
// that the datasource definition changed
const updatedDatasource = await db.get(datasource._id)
const updatedDatasource = await sdk.datasources.get(datasource._id!)
builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource)
return tableToDelete

13
packages/server/src/api/controllers/table/internal.ts
View File

@ -15,7 +15,7 @@ import { isEqual } from "lodash"
import { cloneDeep } from "lodash/fp"
import sdk from "../../../sdk"
function checkAutoColumns(table: Table, oldTable: Table) {
function checkAutoColumns(table: Table, oldTable?: Table) {
if (!table.schema) {
return table
}
@ -46,7 +46,7 @@ export async function save(ctx: any) {
// if the table obj had an _id then it will have been retrieved
let oldTable
if (ctx.request.body && ctx.request.body._id) {
oldTable = await db.get(ctx.request.body._id)
oldTable = await sdk.tables.getTable(ctx.request.body._id)
}
// check all types are correct
@ -70,8 +70,8 @@ export async function save(ctx: any) {
if (oldTable && oldTable.schema) {
for (let propKey of Object.keys(tableToSave.schema)) {
let oldColumn = oldTable.schema[propKey]
if (oldColumn && oldColumn.type === "internal") {
oldColumn.type = "auto"
if (oldColumn && oldColumn.type === FieldTypes.INTERNAL) {
oldColumn.type = FieldTypes.AUTO
}
}
}
@ -138,7 +138,7 @@ export async function save(ctx: any) {
export async function destroy(ctx: any) {
const db = context.getAppDB()
const tableToDelete = await db.get(ctx.params.tableId)
const tableToDelete = await sdk.tables.getTable(ctx.params.tableId)
// Delete all rows for that table
const rowsData = await db.allDocs(
@ -160,7 +160,7 @@ export async function destroy(ctx: any) {
})
// don't remove the table itself until very end
await db.remove(tableToDelete._id, tableToDelete._rev)
await db.remove(tableToDelete._id!, tableToDelete._rev)
// remove table search index
if (!env.isTest() || env.COUCH_DB_URL) {
@ -184,7 +184,6 @@ export async function destroy(ctx: any) {
}
export async function bulkImport(ctx: any) {
const db = context.getAppDB()
const table = await sdk.tables.getTable(ctx.params.tableId)
const { rows, identifierFields } = ctx.request.body
await handleDataImport(ctx.user, table, rows, identifierFields)

10
packages/server/src/api/controllers/table/utils.ts
View File

@ -20,16 +20,10 @@ import viewTemplate from "../view/viewBuilder"
import { cloneDeep } from "lodash/fp"
import { quotas } from "@budibase/pro"
import { events, context } from "@budibase/backend-core"
import {
ContextUser,
Database,
Datasource,
SourceName,
Table,
} from "@budibase/types"
import { ContextUser, Datasource, SourceName, Table } from "@budibase/types"
export async function clearColumns(table: any, columnNames: any) {
const db: Database = context.getAppDB()
const db = context.getAppDB()
const rows = await db.allDocs(
getRowParams(table._id, null, {
include_docs: true,

120
packages/server/src/api/controllers/tests/row.spec.ts
View File

@ -1,120 +0,0 @@
import { exportRows } from "../row/external"
import sdk from "../../../sdk"
import { ExternalRequest } from "../row/ExternalRequest"
// @ts-ignore
sdk.datasources = {
get: jest.fn(),
}
jest.mock("../row/ExternalRequest")
jest.mock("../view/exporters", () => ({
csv: jest.fn(),
Format: {
CSV: "csv",
},
}))
jest.mock("../../../utilities/fileSystem")
function getUserCtx() {
return {
params: {
tableId: "datasource__tablename",
},
query: {
format: "csv",
},
request: {
body: {},
},
throw: jest.fn(() => {
throw "Err"
}),
attachment: jest.fn(),
}
}
describe("external row controller", () => {
describe("exportRows", () => {
beforeAll(() => {
//@ts-ignore
jest.spyOn(ExternalRequest.prototype, "run").mockImplementation(() => [])
})
afterEach(() => {
jest.clearAllMocks()
})
it("should throw a 400 if no datasource entities are present", async () => {
let userCtx = getUserCtx()
try {
//@ts-ignore
await exportRows(userCtx)
} catch (e) {
expect(userCtx.throw).toHaveBeenCalledWith(
400,
"Datasource has not been configured for plus API."
)
}
})
it("should handle single quotes from a row ID", async () => {
//@ts-ignore
sdk.datasources.get.mockImplementation(() => ({
entities: {
tablename: {
schema: {},
},
},
}))
let userCtx = getUserCtx()
userCtx.request.body = {
rows: ["['d001']"],
}
//@ts-ignore
await exportRows(userCtx)
expect(userCtx.request.body).toEqual({
query: {
oneOf: {
_id: ["d001"],
},
},
})
})
it("should throw a 400 if any composite keys are present", async () => {
let userCtx = getUserCtx()
userCtx.request.body = {
rows: ["[123]", "['d001'%2C'10111']"],
}
try {
//@ts-ignore
await exportRows(userCtx)
} catch (e) {
expect(userCtx.throw).toHaveBeenCalledWith(
400,
"Export data does not support composite keys."
)
}
})
it("should throw a 400 if no table name was found", async () => {
let userCtx = getUserCtx()
userCtx.params.tableId = "datasource__"
userCtx.request.body = {
rows: ["[123]"],
}
try {
//@ts-ignore
await exportRows(userCtx)
} catch (e) {
expect(userCtx.throw).toHaveBeenCalledWith(
400,
"Could not find table name."
)
}
})
})
})

26
packages/server/src/api/controllers/user.ts
View File

@ -3,25 +3,11 @@ import { InternalTables } from "../../db/utils"
import { getGlobalUsers } from "../../utilities/global"
import { getFullUser } from "../../utilities/users"
import { context } from "@budibase/backend-core"
import { UserCtx } from "@budibase/types"
import { Ctx, UserCtx } from "@budibase/types"
import sdk from "../../sdk"
export async function fetchMetadata(ctx: UserCtx) {
const global = await getGlobalUsers()
const metadata = await sdk.users.rawUserMetadata()
const users = []
for (let user of global) {
// find the metadata that matches up to the global ID
const info = metadata.find(meta => meta._id.includes(user._id))
// remove these props, not for the correct DB
users.push({
...user,
...info,
tableId: InternalTables.USER_METADATA,
// make sure the ID is always a local ID, not a global one
_id: generateUserMetadataID(user._id),
})
}
export async function fetchMetadata(ctx: Ctx) {
const users = await sdk.users.fetchMetadata()
ctx.body = users
}
@ -50,8 +36,8 @@ export async function updateMetadata(ctx: UserCtx) {
export async function destroyMetadata(ctx: UserCtx) {
const db = context.getAppDB()
try {
const dbUser = await db.get(ctx.params.id)
await db.remove(dbUser._id, dbUser._rev)
const dbUser = await sdk.users.get(ctx.params.id)
await db.remove(dbUser._id!, dbUser._rev)
} catch (err) {
// error just means the global user has no config in this app
}
@ -74,7 +60,7 @@ export async function setFlag(ctx: UserCtx) {
const db = context.getAppDB()
let doc
try {
doc = await db.get(flagDocId)
doc = await db.get<any>(flagDocId)
} catch (err) {
doc = { _id: flagDocId }
}

15
packages/server/src/api/controllers/view/index.ts
View File

@ -15,7 +15,6 @@ import {
TableSchema,
View,
} from "@budibase/types"
import { cleanExportRows } from "../row/utils"
import { builderSocket } from "../../../websockets"
const { cloneDeep, isEqual } = require("lodash")
@ -28,7 +27,8 @@ export async function save(ctx: Ctx) {
const db = context.getAppDB()
const { originalName, ...viewToSave } = ctx.request.body
const existingTable = await db.get(ctx.request.body.tableId)
const existingTable = await sdk.tables.getTable(ctx.request.body.tableId)
existingTable.views ??= {}
const table = cloneDeep(existingTable)
const groupByField: any = Object.values(table.schema).find(
@ -121,8 +121,8 @@ export async function destroy(ctx: Ctx) {
const db = context.getAppDB()
const viewName = decodeURIComponent(ctx.params.viewName)
const view = await deleteView(viewName)
const table = await db.get(view.meta.tableId)
delete table.views[viewName]
const table = await sdk.tables.getTable(view.meta.tableId)
delete table.views![viewName]
await db.put(table)
await events.view.deleted(view)
@ -163,13 +163,16 @@ export async function exportView(ctx: Ctx) {
let rows = ctx.body as Row[]
let schema: TableSchema = view && view.meta && view.meta.schema
const tableId = ctx.params.tableId || view.meta.tableId
const tableId =
ctx.params.tableId ||
view?.meta?.tableId ||
(viewName.startsWith(DocumentType.TABLE) && viewName)
const table: Table = await sdk.tables.getTable(tableId)
if (!schema) {
schema = table.schema
}
let exportRows = cleanExportRows(rows, schema, format, [])
let exportRows = sdk.rows.utils.cleanExportRows(rows, schema, format, [])
if (format === Format.CSV) {
ctx.attachment(`${viewName}.csv`)

26
packages/server/src/api/controllers/view/utils.ts
View File

@ -13,7 +13,7 @@ import { Database } from "@budibase/types"
export async function getView(viewName: string) {
const db = context.getAppDB()
if (env.SELF_HOSTED) {
const designDoc = await db.get("_design/database")
const designDoc = await db.get<any>("_design/database")
return designDoc.views[viewName]
} else {
// This is a table view, don't read the view from the DB
@ -22,7 +22,7 @@ export async function getView(viewName: string) {
}
try {
const viewDoc = await db.get(generateMemoryViewID(viewName))
const viewDoc = await db.get<any>(generateMemoryViewID(viewName))
return viewDoc.view
} catch (err: any) {
// Return null when PouchDB doesn't found the view
@ -39,7 +39,7 @@ export async function getViews() {
const db = context.getAppDB()
const response = []
if (env.SELF_HOSTED) {
const designDoc = await db.get("_design/database")
const designDoc = await db.get<any>("_design/database")
for (let name of Object.keys(designDoc.views)) {
// Only return custom views, not built ins
const viewNames = Object.values(ViewName) as string[]
@ -76,7 +76,7 @@ export async function saveView(
) {
const db = context.getAppDB()
if (env.SELF_HOSTED) {
const designDoc = await db.get("_design/database")
const designDoc = await db.get<any>("_design/database")
designDoc.views = {
...designDoc.views,
[viewName]: viewTemplate,
@ -96,9 +96,9 @@ export async function saveView(
tableId: viewTemplate.meta.tableId,
}
try {
const old = await db.get(id)
const old = await db.get<any>(id)
if (originalId) {
const originalDoc = await db.get(originalId)
const originalDoc = await db.get<any>(originalId)
await db.remove(originalDoc._id, originalDoc._rev)
}
if (old && old._rev) {
@ -114,14 +114,14 @@ export async function saveView(
export async function deleteView(viewName: string) {
const db = context.getAppDB()
if (env.SELF_HOSTED) {
const designDoc = await db.get("_design/database")
const designDoc = await db.get<any>("_design/database")
const view = designDoc.views[viewName]
delete designDoc.views[viewName]
await db.put(designDoc)
return view
} else {
const id = generateMemoryViewID(viewName)
const viewDoc = await db.get(id)
const viewDoc = await db.get<any>(id)
await db.remove(viewDoc._id, viewDoc._rev)
return viewDoc.view
}
@ -129,7 +129,7 @@ export async function deleteView(viewName: string) {
export async function migrateToInMemoryView(db: Database, viewName: string) {
// delete the view initially
const designDoc = await db.get("_design/database")
const designDoc = await db.get<any>("_design/database")
// run the view back through the view builder to update it
const view = viewBuilder(designDoc.views[viewName].meta)
delete designDoc.views[viewName]
@ -138,15 +138,15 @@ export async function migrateToInMemoryView(db: Database, viewName: string) {
}
export async function migrateToDesignView(db: Database, viewName: string) {
let view = await db.get(generateMemoryViewID(viewName))
const designDoc = await db.get("_design/database")
let view = await db.get<any>(generateMemoryViewID(viewName))
const designDoc = await db.get<any>("_design/database")
designDoc.views[viewName] = viewBuilder(view.view.meta)
await db.put(designDoc)
await db.remove(view._id, view._rev)
}
export async function getFromDesignDoc(db: Database, viewName: string) {
const designDoc = await db.get("_design/database")
const designDoc = await db.get<any>("_design/database")
let view = designDoc.views[viewName]
if (view == null) {
throw { status: 404, message: "Unable to get view" }
@ -155,7 +155,7 @@ export async function getFromDesignDoc(db: Database, viewName: string) {
}
export async function getFromMemoryDoc(db: Database, viewName: string) {
let view = await db.get(generateMemoryViewID(viewName))
let view = await db.get<any>(generateMemoryViewID(viewName))
if (view) {
view = view.view
} else {

2
packages/server/src/api/controllers/webhook.ts
View File

@ -77,7 +77,7 @@ export async function trigger(ctx: BBContext) {
if (webhook.bodySchema) {
validate(ctx.request.body, webhook.bodySchema)
}
const target = await db.get(webhook.action.target)
const target = await db.get<Automation>(webhook.action.target)
if (webhook.action.type === WebhookActionType.AUTOMATION) {
// trigger with both the pure request and then expand it
// incase the user has produced a schema to bind to

197
packages/server/src/api/routes/tests/internalSearch.spec.js
View File

@ -1,197 +0,0 @@
const fetch = require("node-fetch")
fetch.mockSearch()
const search = require("../../controllers/row/internalSearch")
// this will be mocked out for _search endpoint
const PARAMS = {
tableId: "ta_12345679abcdef",
version: "1",
bookmark: null,
sort: null,
sortOrder: "ascending",
sortType: "string",
}
function checkLucene(resp, expected, params = PARAMS) {
const query = resp.rows[0].query
const json = JSON.parse(query)
if (PARAMS.sort) {
expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
}
if (PARAMS.bookmark) {
expect(json.bookmark).toBe(PARAMS.bookmark)
}
expect(json.include_docs).toBe(true)
expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
expect(json.limit).toBe(params.limit || 50)
}
describe("internal search", () => {
it("default query", async () => {
const response = await search.paginatedSearch({
}, PARAMS)
checkLucene(response, `*:*`)
})
it("test equal query", async () => {
const response = await search.paginatedSearch({
equal: {
"column": "1",
}
}, PARAMS)
checkLucene(response, `*:* AND column:"1"`)
})
it("test notEqual query", async () => {
const response = await search.paginatedSearch({
notEqual: {
"column": "1",
}
}, PARAMS)
checkLucene(response, `*:* AND !column:"1"`)
})
it("test OR query", async () => {
const response = await search.paginatedSearch({
allOr: true,
equal: {
"column": "2",
},
notEqual: {
"column": "1",
}
}, PARAMS)
checkLucene(response, `(column:"2" OR !column:"1")`)
})
it("test AND query", async () => {
const response = await search.paginatedSearch({
equal: {
"column": "2",
},
notEqual: {
"column": "1",
}
}, PARAMS)
checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
})
it("test pagination query", async () => {
const updatedParams = {
...PARAMS,
limit: 100,
bookmark: "awd",
sort: "column",
}
const response = await search.paginatedSearch({
string: {
"column": "2",
},
}, updatedParams)
checkLucene(response, `*:* AND column:2*`, updatedParams)
})
it("test range query", async () => {
const response = await search.paginatedSearch({
range: {
"column": { low: 1, high: 2 },
},
}, PARAMS)
checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
})
it("test empty query", async () => {
const response = await search.paginatedSearch({
empty: {
"column": "",
},
}, PARAMS)
checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
})
it("test notEmpty query", async () => {
const response = await search.paginatedSearch({
notEmpty: {
"column": "",
},
}, PARAMS)
checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
})
it("test oneOf query", async () => {
const response = await search.paginatedSearch({
oneOf: {
"column": ["a", "b"],
},
}, PARAMS)
checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
})
it("test contains query", async () => {
const response = await search.paginatedSearch({
contains: {
"column": "a",
"colArr": [1, 2, 3],
},
}, PARAMS)
checkLucene(response, `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`, PARAMS)
})
it("test multiple of same column", async () => {
const response = await search.paginatedSearch({
allOr: true,
equal: {
"1:column": "a",
"2:column": "b",
"3:column": "c",
},
}, PARAMS)
checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
})
it("check a weird case for lucene building", async () => {
const response = await search.paginatedSearch({
equal: {
"1:1:column": "a",
},
}, PARAMS)
checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
})
it("test containsAny query", async () => {
const response = await search.paginatedSearch({
containsAny: {
"column": ["a", "b", "c"]
},
}, PARAMS)
checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
})
it("test notContains query", async () => {
const response = await search.paginatedSearch({
notContains: {
"column": ["a", "b", "c"]
},
}, PARAMS)
checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
})
it("test equal without version query", async () => {
PARAMS.version = null
const response = await search.paginatedSearch({
equal: {
"column": "1",
}
}, PARAMS)
const query = response.rows[0].query
const json = JSON.parse(query)
if (PARAMS.sort) {
expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
}
if (PARAMS.bookmark) {
expect(json.bookmark).toBe(PARAMS.bookmark)
}
expect(json.include_docs).toBe(true)
expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
})
})

248
packages/server/src/api/routes/tests/internalSearch.spec.ts Normal file
View File

@ -0,0 +1,248 @@
const nodeFetch = require("node-fetch")
nodeFetch.mockSearch()
import { SearchParams } from "@budibase/backend-core"
import * as search from "../../../sdk/app/rows/search/internalSearch"
import { Row } from "@budibase/types"
// this will be mocked out for _search endpoint
const PARAMS: SearchParams<Row> = {
tableId: "ta_12345679abcdef",
version: "1",
bookmark: undefined,
sort: undefined,
sortOrder: "ascending",
sortType: "string",
}
function checkLucene(resp: any, expected: any, params = PARAMS) {
const query = resp.rows[0].query
const json = JSON.parse(query)
if (PARAMS.sort) {
expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
}
if (PARAMS.bookmark) {
expect(json.bookmark).toBe(PARAMS.bookmark)
}
expect(json.include_docs).toBe(true)
expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
expect(json.limit).toBe(params.limit || 50)
}
describe("internal search", () => {
it("default query", async () => {
const response = await search.paginatedSearch({}, PARAMS)
checkLucene(response, `*:*`)
})
it("test equal query", async () => {
const response = await search.paginatedSearch(
{
equal: {
column: "1",
},
},
PARAMS
)
checkLucene(response, `*:* AND column:"1"`)
})
it("test notEqual query", async () => {
const response = await search.paginatedSearch(
{
notEqual: {
column: "1",
},
},
PARAMS
)
checkLucene(response, `*:* AND !column:"1"`)
})
it("test OR query", async () => {
const response = await search.paginatedSearch(
{
allOr: true,
equal: {
column: "2",
},
notEqual: {
column: "1",
},
},
PARAMS
)
checkLucene(response, `(column:"2" OR !column:"1")`)
})
it("test AND query", async () => {
const response = await search.paginatedSearch(
{
equal: {
column: "2",
},
notEqual: {
column: "1",
},
},
PARAMS
)
checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
})
it("test pagination query", async () => {
const updatedParams = {
...PARAMS,
limit: 100,
bookmark: "awd",
sort: "column",
}
const response = await search.paginatedSearch(
{
string: {
column: "2",
},
},
updatedParams
)
checkLucene(response, `*:* AND column:2*`, updatedParams)
})
it("test range query", async () => {
const response = await search.paginatedSearch(
{
range: {
column: { low: 1, high: 2 },
},
},
PARAMS
)
checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
})
it("test empty query", async () => {
const response = await search.paginatedSearch(
{
empty: {
column: "",
},
},
PARAMS
)
checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
})
it("test notEmpty query", async () => {
const response = await search.paginatedSearch(
{
notEmpty: {
column: "",
},
},
PARAMS
)
checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
})
it("test oneOf query", async () => {
const response = await search.paginatedSearch(
{
oneOf: {
column: ["a", "b"],
},
},
PARAMS
)
checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
})
it("test contains query", async () => {
const response = await search.paginatedSearch(
{
contains: {
column: "a",
colArr: [1, 2, 3],
},
},
PARAMS
)
checkLucene(
response,
`(*:* AND column:a AND colArr:(1 AND 2 AND 3))`,
PARAMS
)
})
it("test multiple of same column", async () => {
const response = await search.paginatedSearch(
{
allOr: true,
equal: {
"1:column": "a",
"2:column": "b",
"3:column": "c",
},
},
PARAMS
)
checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
})
it("check a weird case for lucene building", async () => {
const response = await search.paginatedSearch(
{
equal: {
"1:1:column": "a",
},
},
PARAMS
)
checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
})
it("test containsAny query", async () => {
const response = await search.paginatedSearch(
{
containsAny: {
column: ["a", "b", "c"],
},
},
PARAMS
)
checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
})
it("test notContains query", async () => {
const response = await search.paginatedSearch(
{
notContains: {
column: ["a", "b", "c"],
},
},
PARAMS
)
checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
})
it("test equal without version query", async () => {
PARAMS.version = undefined
const response = await search.paginatedSearch(
{
equal: {
column: "1",
},
},
PARAMS
)
const query = response.rows[0].query
const json = JSON.parse(query)
if (PARAMS.sort) {
expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
}
if (PARAMS.bookmark) {
expect(json.bookmark).toBe(PARAMS.bookmark)
}
expect(json.include_docs).toBe(true)
expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
})
})

221
packages/server/src/api/routes/tests/row.spec.js → packages/server/src/api/routes/tests/row.spec.ts
View File

@ -1,27 +1,27 @@
const tk = require( "timekeeper")
import tk from "timekeeper"
const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
tk.freeze(timestamp)
const { outputProcessing } = require("../../../utilities/rowProcessor")
const setup = require("./utilities")
import { outputProcessing } from "../../../utilities/rowProcessor"
import * as setup from "./utilities"
const { basicRow } = setup.structures
const { context, tenancy } = require("@budibase/backend-core")
const {
quotas,
} = require("@budibase/pro")
const {
import { context, tenancy } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import {
QuotaUsageType,
StaticQuotaName,
MonthlyQuotaName,
} = require("@budibase/types")
const { structures } = require("@budibase/backend-core/tests");
Row,
Table,
FieldType,
} from "@budibase/types"
import { structures } from "@budibase/backend-core/tests"
describe("/rows", () => {
let request = setup.getRequest()
let config = setup.getConfig()
let table
let row
let table: Table
let row: Row
afterAll(setup.afterAll)
@ -29,12 +29,12 @@ describe("/rows", () => {
await config.init()
})
beforeEach(async()=>{
beforeEach(async () => {
table = await config.createTable()
row = basicRow(table._id)
row = basicRow(table._id!)
})
const loadRow = async (id, tbl_Id, status = 200) =>
const loadRow = async (id: string, tbl_Id: string, status = 200) =>
await request
.get(`/api/${tbl_Id}/rows/${id}`)
.set(config.defaultHeaders())
@ -42,21 +42,28 @@ describe("/rows", () => {
.expect(status)
const getRowUsage = async () => {
const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS))
const { total } = await config.doInContext(null, () =>
quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS)
)
return total
}
const getQueryUsage = async () => {
const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.MONTHLY, MonthlyQuotaName.QUERIES))
const { total } = await config.doInContext(null, () =>
quotas.getCurrentUsageValues(
QuotaUsageType.MONTHLY,
MonthlyQuotaName.QUERIES
)
)
return total
}
const assertRowUsage = async expected => {
const assertRowUsage = async (expected: number) => {
const usage = await getRowUsage()
expect(usage).toBe(expected)
}
const assertQueryUsage = async expected => {
const assertQueryUsage = async (expected: number) => {
const usage = await getQueryUsage()
expect(usage).toBe(expected)
}
@ -70,9 +77,11 @@ describe("/rows", () => {
.post(`/api/${row.tableId}/rows`)
.send(row)
.set(config.defaultHeaders())
.expect('Content-Type', /json/)
.expect("Content-Type", /json/)
.expect(200)
expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`)
expect((res as any).res.statusMessage).toEqual(
`${table.name} saved successfully`
)
expect(res.body.name).toEqual("Test Contact")
expect(res.body._rev).toBeDefined()
await assertRowUsage(rowUsage + 1)
@ -86,12 +95,11 @@ describe("/rows", () => {
const newTable = await config.createTable({
name: "TestTableAuto",
type: "table",
key: "name",
schema: {
...table.schema,
"Row ID": {
name: "Row ID",
type: "number",
type: FieldType.NUMBER,
subtype: "autoID",
icon: "ri-magic-line",
autocolumn: true,
@ -104,28 +112,30 @@ describe("/rows", () => {
},
},
},
}
},
})
const ids = [1,2,3]
const ids = [1, 2, 3]
// Performing several create row requests should increment the autoID fields accordingly
const createRow = async (id) => {
const createRow = async (id: number) => {
const res = await request
.post(`/api/${newTable._id}/rows`)
.send({
name: "row_" + id
name: "row_" + id,
})
.set(config.defaultHeaders())
.expect('Content-Type', /json/)
.expect("Content-Type", /json/)
.expect(200)
expect(res.res.statusMessage).toEqual(`${newTable.name} saved successfully`)
expect((res as any).res.statusMessage).toEqual(
`${newTable.name} saved successfully`
)
expect(res.body.name).toEqual("row_" + id)
expect(res.body._rev).toBeDefined()
expect(res.body["Row ID"]).toEqual(id)
}
for (let i=0; i<ids.length; i++ ){
for (let i = 0; i < ids.length; i++) {
await createRow(ids[i])
}
@ -150,7 +160,7 @@ describe("/rows", () => {
.expect("Content-Type", /json/)
.expect(200)
expect(res.res.statusMessage).toEqual(
expect((res as any).res.statusMessage).toEqual(
`${table.name} updated successfully.`
)
expect(res.body.name).toEqual("Updated Name")
@ -196,8 +206,8 @@ describe("/rows", () => {
.expect(200)
expect(res.body.length).toBe(2)
expect(res.body.find(r => r.name === newRow.name)).toBeDefined()
expect(res.body.find(r => r.name === row.name)).toBeDefined()
expect(res.body.find((r: Row) => r.name === newRow.name)).toBeDefined()
expect(res.body.find((r: Row) => r.name === row.name)).toBeDefined()
await assertQueryUsage(queryUsage + 1)
})
@ -215,92 +225,91 @@ describe("/rows", () => {
it("row values are coerced", async () => {
const str = {
type: "string",
type: FieldType.STRING,
name: "str",
constraints: { type: "string", presence: false },
}
const attachment = {
type: "attachment",
type: FieldType.ATTACHMENT,
name: "attachment",
constraints: { type: "array", presence: false },
}
const bool = {
type: "boolean",
type: FieldType.BOOLEAN,
name: "boolean",
constraints: { type: "boolean", presence: false },
}
const number = {
type: "number",
type: FieldType.NUMBER,
name: "str",
constraints: { type: "number", presence: false },
}
const datetime = {
type: "datetime",
type: FieldType.DATETIME,
name: "datetime",
constraints: {
type: "string",
presence: false,
datetime: { earliest: "", latest: "" },
}
},
}
const arrayField = {
type: "array",
type: FieldType.ARRAY,
constraints: {
type: "array",
presence: false,
inclusion: [
"One",
"Two",
"Three",
]
inclusion: ["One", "Two", "Three"],
},
name: "Sample Tags",
sortable: false
sortable: false,
}
const optsField = {
fieldName: "Sample Opts",
name: "Sample Opts",
type: "options",
constraints: {
type: "string",
presence: false,
inclusion: [ "Alpha", "Beta", "Gamma" ]
fieldName: "Sample Opts",
name: "Sample Opts",
type: FieldType.OPTIONS,
constraints: {
type: "string",
presence: false,
inclusion: ["Alpha", "Beta", "Gamma"],
},
},
},
table = await config.createTable({
name: "TestTable2",
type: "table",
key: "name",
schema: {
name: str,
stringUndefined: str,
stringNull: str,
stringString: str,
numberEmptyString: number,
numberNull: number,
numberUndefined: number,
numberString: number,
numberNumber: number,
datetimeEmptyString: datetime,
datetimeNull: datetime,
datetimeUndefined: datetime,
datetimeString: datetime,
datetimeDate: datetime,
boolNull: bool,
boolEmpty: bool,
boolUndefined: bool,
boolString: bool,
boolBool: bool,
attachmentNull: attachment,
attachmentUndefined: attachment,
attachmentEmpty: attachment,
attachmentEmptyArrayStr: attachment,
arrayFieldEmptyArrayStr: arrayField,
arrayFieldArrayStrKnown: arrayField,
arrayFieldNull: arrayField,
arrayFieldUndefined: arrayField,
optsFieldEmptyStr: optsField,
optsFieldUndefined: optsField,
optsFieldNull: optsField,
optsFieldStrKnown: optsField
},
})
table = await config.createTable({
name: "TestTable2",
type: "table",
schema: {
name: str,
stringUndefined: str,
stringNull: str,
stringString: str,
numberEmptyString: number,
numberNull: number,
numberUndefined: number,
numberString: number,
numberNumber: number,
datetimeEmptyString: datetime,
datetimeNull: datetime,
datetimeUndefined: datetime,
datetimeString: datetime,
datetimeDate: datetime,
boolNull: bool,
boolEmpty: bool,
boolUndefined: bool,
boolString: bool,
boolBool: bool,
attachmentNull: attachment,
attachmentUndefined: attachment,
attachmentEmpty: attachment,
attachmentEmptyArrayStr: attachment,
arrayFieldEmptyArrayStr: arrayField,
arrayFieldArrayStrKnown: arrayField,
arrayFieldNull: arrayField,
arrayFieldUndefined: arrayField,
optsFieldEmptyStr: optsField,
optsFieldUndefined: optsField,
optsFieldNull: optsField,
optsFieldStrKnown: optsField,
},
})
row = {
name: "Test Row",
@ -334,13 +343,13 @@ describe("/rows", () => {
optsFieldEmptyStr: "",
optsFieldUndefined: undefined,
optsFieldNull: null,
optsFieldStrKnown: 'Alpha'
optsFieldStrKnown: "Alpha",
}
const createdRow = await config.createRow(row);
const id = createdRow._id
const createdRow = await config.createRow(row)
const id = createdRow._id!
const saved = (await loadRow(id, table._id)).body
const saved = (await loadRow(id, table._id!)).body
expect(saved.stringUndefined).toBe(undefined)
expect(saved.stringNull).toBe("")
@ -365,15 +374,15 @@ describe("/rows", () => {
expect(saved.attachmentNull).toEqual([])
expect(saved.attachmentUndefined).toBe(undefined)
expect(saved.attachmentEmpty).toEqual([])
expect(saved.attachmentEmptyArrayStr).toEqual([])
expect(saved.attachmentEmptyArrayStr).toEqual([])
expect(saved.arrayFieldEmptyArrayStr).toEqual([])
expect(saved.arrayFieldNull).toEqual([])
expect(saved.arrayFieldUndefined).toEqual(undefined)
expect(saved.optsFieldEmptyStr).toEqual(null)
expect(saved.optsFieldUndefined).toEqual(undefined)
expect(saved.optsFieldNull).toEqual(null)
expect(saved.arrayFieldArrayStrKnown).toEqual(['One'])
expect(saved.optsFieldStrKnown).toEqual('Alpha')
expect(saved.arrayFieldArrayStrKnown).toEqual(["One"])
expect(saved.optsFieldStrKnown).toEqual("Alpha")
})
})
@ -396,13 +405,13 @@ describe("/rows", () => {
.expect("Content-Type", /json/)
.expect(200)
expect(res.res.statusMessage).toEqual(
expect((res as any).res.statusMessage).toEqual(
`${table.name} updated successfully.`
)
expect(res.body.name).toEqual("Updated Name")
expect(res.body.description).toEqual(existing.description)
const savedRow = await loadRow(res.body._id, table._id)
const savedRow = await loadRow(res.body._id, table._id!)
expect(savedRow.body.description).toEqual(existing.description)
expect(savedRow.body.name).toEqual("Updated Name")
@ -504,7 +513,7 @@ describe("/rows", () => {
.expect(200)
expect(res.body.length).toEqual(2)
await loadRow(row1._id, table._id, 404)
await loadRow(row1._id!, table._id!, 404)
await assertRowUsage(rowUsage - 2)
await assertQueryUsage(queryUsage + 1)
})
@ -562,7 +571,7 @@ describe("/rows", () => {
describe("fetchEnrichedRows", () => {
it("should allow enriching some linked rows", async () => {
const { table, firstRow, secondRow } = await tenancy.doInTenant(
setup.structures.TENANT_ID,
config.getTenantId(),
async () => {
const table = await config.createLinkedTable()
const firstRow = await config.createRow({
@ -624,7 +633,7 @@ describe("/rows", () => {
await setup.switchToSelfHosted(async () => {
context.doInAppContext(config.getAppId(), async () => {
const enriched = await outputProcessing(table, [row])
expect(enriched[0].attachment[0].url).toBe(
expect((enriched as Row[])[0].attachment[0].url).toBe(
`/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
)
})

13
packages/server/src/automations/tests/queryRows.spec.js → packages/server/src/automations/tests/queryRows.spec.ts
View File

@ -1,6 +1,6 @@
// lucene searching not supported in test due to use of PouchDB
let rows = []
jest.mock("../../api/controllers/row/internalSearch", () => ({
let rows: Row[] = []
jest.mock("../../sdk/app/rows/search/internalSearch", () => ({
fullSearch: jest.fn(() => {
return {
rows,
@ -8,12 +8,13 @@ jest.mock("../../api/controllers/row/internalSearch", () => ({
}),
paginatedSearch: jest.fn(),
}))
const setup = require("./utilities")
import { Row, Table } from "@budibase/types"
import * as setup from "./utilities"
const NAME = "Test"
describe("Test a query step automation", () => {
let table
let table: Table
let config = setup.getConfig()
beforeAll(async () => {
@ -87,8 +88,8 @@ describe("Test a query step automation", () => {
filters: {},
"filters-def": [
{
value: null
}
value: null,
},
],
sortColumn: "name",
sortOrder: "ascending",

13
packages/server/src/automations/utils.ts
View File

@ -8,7 +8,12 @@ import { db as dbCore, context } from "@budibase/backend-core"
import { getAutomationMetadataParams } from "../db/utils"
import { cloneDeep } from "lodash/fp"
import { quotas } from "@budibase/pro"
import { Automation, AutomationJob, WebhookActionType } from "@budibase/types"
import {
Automation,
AutomationJob,
Webhook,
WebhookActionType,
} from "@budibase/types"
import sdk from "../sdk"
const REBOOT_CRON = "@reboot"
@ -204,15 +209,15 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
oldTrigger.webhookId
) {
try {
let db = context.getAppDB()
const db = context.getAppDB()
// need to get the webhook to get the rev
const webhook = await db.get(oldTrigger.webhookId)
const webhook = await db.get<Webhook>(oldTrigger.webhookId)
// might be updating - reset the inputs to remove the URLs
if (newTrigger) {
delete newTrigger.webhookId
newTrigger.inputs = {}
}
await sdk.automations.webhook.destroy(webhook._id, webhook._rev)
await sdk.automations.webhook.destroy(webhook._id!, webhook._rev!)
} catch (err) {
// don't worry about not being able to delete, if it doesn't exist all good
}

8
packages/server/src/db/linkedRows/LinkController.ts
View File

@ -182,7 +182,7 @@ class LinkController {
})
// if 1:N, ensure that this ID is not already attached to another record
const linkedTable = await this._db.get(field.tableId)
const linkedTable = await this._db.get<Table>(field.tableId)
const linkedSchema = linkedTable.schema[field.fieldName!]
// We need to map the global users to metadata in each app for relationships
@ -311,7 +311,7 @@ class LinkController {
})
)
// remove schema from other table
let linkedTable = await this._db.get(field.tableId)
let linkedTable = await this._db.get<Table>(field.tableId)
if (field.fieldName) {
delete linkedTable.schema[field.fieldName]
}
@ -337,7 +337,7 @@ class LinkController {
// table for some reason
let linkedTable
try {
linkedTable = await this._db.get(field.tableId)
linkedTable = await this._db.get<Table>(field.tableId)
} catch (err) {
/* istanbul ignore next */
continue
@ -416,7 +416,7 @@ class LinkController {
const field = schema[fieldName]
try {
if (field.type === FieldTypes.LINK && field.fieldName) {
const linkedTable = await this._db.get(field.tableId)
const linkedTable = await this._db.get<Table>(field.tableId)
delete linkedTable.schema[field.fieldName]
await this._db.put(linkedTable)
}

6
packages/server/src/db/views/staticViews.ts
View File

@ -22,7 +22,7 @@ const SCREEN_PREFIX = DocumentType.SCREEN + SEPARATOR
*/
export async function createLinkView() {
const db = context.getAppDB()
const designDoc = await db.get("_design/database")
const designDoc = await db.get<any>("_design/database")
const view = {
map: function (doc: LinkDocument) {
// everything in this must remain constant as its going to Pouch, no external variables
@ -58,7 +58,7 @@ export async function createLinkView() {
export async function createRoutingView() {
const db = context.getAppDB()
const designDoc = await db.get("_design/database")
const designDoc = await db.get<any>("_design/database")
const view = {
// if using variables in a map function need to inject them before use
map: `function(doc) {
@ -79,7 +79,7 @@ export async function createRoutingView() {
async function searchIndex(indexName: string, fnString: string) {
const db = context.getAppDB()
const designDoc = await db.get("_design/database")
const designDoc = await db.get<any>("_design/database")
designDoc.indexes = {
[indexName]: {
index: fnString,

2
packages/server/src/integrations/tests/mysql.spec.ts
View File

@ -102,7 +102,7 @@ describe("MySQL Integration", () => {
)
})
it("parses strings matching a valid date format", async () => {
it.skip("parses strings matching a valid date format", async () => {
const sql = "select * from users;"
await config.integration.read({
sql,

4
packages/server/src/middleware/builder.ts
View File

@ -10,7 +10,7 @@ import {
setDebounce,
} from "../utilities/redis"
import { db as dbCore, cache } from "@budibase/backend-core"
import { UserCtx, Database } from "@budibase/types"
import { UserCtx, Database, App } from "@budibase/types"
const DEBOUNCE_TIME_SEC = 30
@ -51,7 +51,7 @@ async function updateAppUpdatedAt(ctx: UserCtx) {
}
await dbCore.doWithDB(appId, async (db: Database) => {
try {
const metadata = await db.get(DocumentType.APP_METADATA)
const metadata = await db.get<any>(DocumentType.APP_METADATA)
metadata.updatedAt = new Date().toISOString()
metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user?.userId!)

2
packages/server/src/sdk/app/applications/sync.ts
View File

@ -37,7 +37,7 @@ async function syncUsersToApp(
let metadata
try {
metadata = await db.get(metadataId)
metadata = await db.get<any>(metadataId)
} catch (err: any) {
if (err.status !== 404) {
throw err

4
packages/server/src/sdk/app/datasources/datasources.ts
View File

@ -62,7 +62,7 @@ export async function get(
opts?: { enriched: boolean }
): Promise<Datasource> {
const appDb = context.getAppDB()
const datasource = await appDb.get(datasourceId)
const datasource = await appDb.get<Datasource>(datasourceId)
if (opts?.enriched) {
return (await enrichDatasourceWithValues(datasource)).datasource
} else {
@ -72,7 +72,7 @@ export async function get(
export async function getWithEnvVars(datasourceId: string) {
const appDb = context.getAppDB()
const datasource = await appDb.get(datasourceId)
const datasource = await appDb.get<Datasource>(datasourceId)
return enrichDatasourceWithValues(datasource)
}

4
packages/server/src/sdk/app/rows/index.ts
View File

@ -1,7 +1,11 @@
import * as attachments from "./attachments"
import * as rows from "./rows"
import * as search from "./search"
import * as utils from "./utils"
export default {
...attachments,
...rows,
...search,
utils: utils,
}

66
packages/server/src/sdk/app/rows/search.ts Normal file
View File

@ -0,0 +1,66 @@
import { SearchFilters } from "@budibase/types"
import { isExternalTable } from "../../../integrations/utils"
import * as internal from "./search/internal"
import * as external from "./search/external"
import { Format } from "../../../api/controllers/view/exporters"
export interface SearchParams {
tableId: string
paginate?: boolean
query: SearchFilters
bookmark?: string
limit?: number
sort?: string
sortOrder?: string
sortType?: string
version?: string
disableEscaping?: boolean
}
export interface ViewParams {
calculation: string
group: string
field: string
}
function pickApi(tableId: any) {
if (isExternalTable(tableId)) {
return external
}
return internal
}
export async function search(options: SearchParams) {
return pickApi(options.tableId).search(options)
}
export interface ExportRowsParams {
tableId: string
format: Format
rowIds?: string[]
columns?: string[]
query: SearchFilters
}
export interface ExportRowsResult {
fileName: string
content: string
}
export async function exportRows(
options: ExportRowsParams
): Promise<ExportRowsResult> {
return pickApi(options.tableId).exportRows(options)
}
export async function fetch(tableId: string) {
return pickApi(tableId).fetch(tableId)
}
export async function fetchView(
tableId: string,
viewName: string,
params: ViewParams
) {
return pickApi(tableId).fetchView(viewName, params)
}

172
packages/server/src/sdk/app/rows/search/external.ts Normal file
View File

@ -0,0 +1,172 @@
import {
SortJson,
SortDirection,
Operation,
PaginationJson,
IncludeRelationship,
Row,
SearchFilters,
} from "@budibase/types"
import * as exporters from "../../../../api/controllers/view/exporters"
import sdk from "../../../../sdk"
import { handleRequest } from "../../../../api/controllers/row/external"
import { breakExternalTableId } from "../../../../integrations/utils"
import { cleanExportRows } from "../utils"
import { utils } from "@budibase/shared-core"
import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
import { HTTPError } from "@budibase/backend-core"
export async function search(options: SearchParams) {
const { tableId } = options
const { paginate, query, ...params } = options
const { limit } = params
let bookmark = (params.bookmark && parseInt(params.bookmark)) || null
if (paginate && !bookmark) {
bookmark = 1
}
let paginateObj = {}
if (paginate) {
paginateObj = {
// add one so we can track if there is another page
limit: limit,
page: bookmark,
}
} else if (params && limit) {
paginateObj = {
limit: limit,
}
}
let sort: SortJson | undefined
if (params.sort) {
const direction =
params.sortOrder === "descending"
? SortDirection.DESCENDING
: SortDirection.ASCENDING
sort = {
[params.sort]: { direction },
}
}
try {
const rows = (await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
paginate: paginateObj as PaginationJson,
includeSqlRelationships: IncludeRelationship.INCLUDE,
})) as Row[]
let hasNextPage = false
if (paginate && rows.length === limit) {
const nextRows = (await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
paginate: {
limit: 1,
page: bookmark! * limit + 1,
},
includeSqlRelationships: IncludeRelationship.INCLUDE,
})) as Row[]
hasNextPage = nextRows.length > 0
}
// need wrapper object for bookmarks etc when paginating
return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
} catch (err: any) {
if (err.message && err.message.includes("does not exist")) {
throw new Error(
`Table updated externally, please re-fetch - ${err.message}`
)
} else {
throw err
}
}
}
export async function exportRows(
options: ExportRowsParams
): Promise<ExportRowsResult> {
const { tableId, format, columns, rowIds } = options
const { datasourceId, tableName } = breakExternalTableId(tableId)
let query: SearchFilters = {}
if (rowIds?.length) {
query = {
oneOf: {
_id: rowIds.map((row: string) => {
const ids = JSON.parse(
decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
)
if (ids.length > 1) {
throw new HTTPError(
"Export data does not support composite keys.",
400
)
}
return ids[0]
}),
},
}
}
const datasource = await sdk.datasources.get(datasourceId!)
if (!datasource || !datasource.entities) {
throw new HTTPError("Datasource has not been configured for plus API.", 400)
}
let result = await search({ tableId, query })
let rows: Row[] = []
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.rows.length; i++) {
rows[i] = {}
for (let column of columns) {
rows[i][column] = result.rows[i][column]
}
}
} else {
rows = result.rows
}
if (!tableName) {
throw new HTTPError("Could not find table name.", 400)
}
const schema = datasource.entities[tableName].schema
let exportRows = cleanExportRows(rows, schema, format, columns)
let headers = Object.keys(schema)
let content: string
switch (format) {
case exporters.Format.CSV:
content = exporters.csv(headers, exportRows)
break
case exporters.Format.JSON:
content = exporters.json(exportRows)
break
case exporters.Format.JSON_WITH_SCHEMA:
content = exporters.jsonWithSchema(schema, exportRows)
break
default:
throw utils.unreachable(format)
}
const fileName = `export.${format}`
return {
fileName,
content,
}
}
export async function fetch(tableId: string) {
return handleRequest(Operation.READ, tableId, {
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
}
export async function fetchView(viewName: string) {
// there are no views in external datasources, shouldn't ever be called
// for now just fetch
const split = viewName.split("all_")
const tableId = split[1] ? split[1] : split[0]
return fetch(tableId)
}

261
packages/server/src/sdk/app/rows/search/internal.ts Normal file
View File

@ -0,0 +1,261 @@
import {
context,
SearchParams as InternalSearchParams,
} from "@budibase/backend-core"
import env from "../../../../environment"
import { fullSearch, paginatedSearch } from "./internalSearch"
import {
InternalTables,
getRowParams,
DocumentType,
} from "../../../../db/utils"
import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import { Database, Row, Table } from "@budibase/types"
import { cleanExportRows } from "../utils"
import {
Format,
csv,
json,
jsonWithSchema,
} from "../../../../api/controllers/view/exporters"
import * as inMemoryViews from "../../../../db/inMemoryView"
import {
migrateToInMemoryView,
migrateToDesignView,
getFromDesignDoc,
getFromMemoryDoc,
} from "../../../../api/controllers/view/utils"
import sdk from "../../../../sdk"
import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
export async function search(options: SearchParams) {
const { tableId } = options
// Fetch the whole table when running in cypress, as search doesn't work
if (!env.COUCH_DB_URL && env.isCypress()) {
return { rows: await fetch(tableId) }
}
const { paginate, query } = options
const params: InternalSearchParams<any> = {
tableId: options.tableId,
sort: options.sort,
sortOrder: options.sortOrder,
sortType: options.sortType,
limit: options.limit,
bookmark: options.bookmark,
version: options.version,
disableEscaping: options.disableEscaping,
}
let table
if (params.sort && !params.sortType) {
table = await sdk.tables.getTable(tableId)
const schema = table.schema
const sortField = schema[params.sort]
params.sortType = sortField.type === "number" ? "number" : "string"
}
let response
if (paginate) {
response = await paginatedSearch(query, params)
} else {
response = await fullSearch(query, params)
}
// Enrich search results with relationships
if (response.rows && response.rows.length) {
// enrich with global users if from users table
if (tableId === InternalTables.USER_METADATA) {
response.rows = await getGlobalUsersFromMetadata(response.rows)
}
table = table || (await sdk.tables.getTable(tableId))
response.rows = await outputProcessing(table, response.rows)
}
return response
}
export async function exportRows(
options: ExportRowsParams
): Promise<ExportRowsResult> {
const { tableId, format, rowIds, columns, query } = options
const db = context.getAppDB()
const table = await sdk.tables.getTable(tableId)
let result
if (rowIds) {
let response = (
await db.allDocs({
include_docs: true,
keys: rowIds,
})
).rows.map(row => row.doc)
result = await outputProcessing(table, response)
} else if (query) {
let searchResponse = await search({ tableId, query })
result = searchResponse.rows
}
let rows: Row[] = []
let schema = table.schema
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.length; i++) {
rows[i] = {}
for (let column of columns) {
rows[i][column] = result[i][column]
}
}
} else {
rows = result
}
let exportRows = cleanExportRows(rows, schema, format, columns)
if (format === Format.CSV) {
return {
fileName: "export.csv",
content: csv(Object.keys(rows[0]), exportRows),
}
} else if (format === Format.JSON) {
return {
fileName: "export.json",
content: json(exportRows),
}
} else if (format === Format.JSON_WITH_SCHEMA) {
return {
fileName: "export.json",
content: jsonWithSchema(schema, exportRows),
}
} else {
throw "Format not recognised"
}
}
export async function fetch(tableId: string) {
const db = context.getAppDB()
let table = await sdk.tables.getTable(tableId)
let rows = await getRawTableData(db, tableId)
const result = await outputProcessing(table, rows)
return result
}
async function getRawTableData(db: Database, tableId: string) {
let rows
if (tableId === InternalTables.USER_METADATA) {
rows = await sdk.users.fetchMetadata()
} else {
const response = await db.allDocs(
getRowParams(tableId, null, {
include_docs: true,
})
)
rows = response.rows.map(row => row.doc)
}
return rows as Row[]
}
export async function fetchView(
viewName: string,
options: { calculation: string; group: string; field: string }
) {
// if this is a table view being looked for just transfer to that
if (viewName.startsWith(DocumentType.TABLE)) {
return fetch(viewName)
}
const db = context.getAppDB()
const { calculation, group, field } = options
const viewInfo = await getView(db, viewName)
let response
if (env.SELF_HOSTED) {
response = await db.query(`database/${viewName}`, {
include_docs: !calculation,
group: !!group,
})
} else {
const tableId = viewInfo.meta.tableId
const data = await getRawTableData(db, tableId)
response = await inMemoryViews.runView(
viewInfo,
calculation as string,
!!group,
data
)
}
let rows
if (!calculation) {
response.rows = response.rows.map(row => row.doc)
let table: Table
try {
table = await sdk.tables.getTable(viewInfo.meta.tableId)
} catch (err) {
/* istanbul ignore next */
table = {
name: "",
schema: {},
}
}
rows = await outputProcessing(table, response.rows)
}
if (calculation === CALCULATION_TYPES.STATS) {
response.rows = response.rows.map(row => ({
group: row.key,
field,
...row.value,
avg: row.value.sum / row.value.count,
}))
rows = response.rows
}
if (
calculation === CALCULATION_TYPES.COUNT ||
calculation === CALCULATION_TYPES.SUM
) {
rows = response.rows.map(row => ({
group: row.key,
field,
value: row.value,
}))
}
return rows
}
const CALCULATION_TYPES = {
SUM: "sum",
COUNT: "count",
STATS: "stats",
}
async function getView(db: Database, viewName: string) {
let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
let viewInfo,
migrate = false
try {
viewInfo = await mainGetter(db, viewName)
} catch (err: any) {
// check if it can be retrieved from design doc (needs migrated)
if (err.status !== 404) {
viewInfo = null
} else {
viewInfo = await secondaryGetter(db, viewName)
migrate = !!viewInfo
}
}
if (migrate) {
await migration(db, viewName)
}
if (!viewInfo) {
throw "View does not exist."
}
return viewInfo
}

0
packages/server/src/api/controllers/row/internalSearch.ts → packages/server/src/sdk/app/rows/search/internalSearch.ts
View File

48
packages/server/src/sdk/app/rows/utils.ts Normal file
View File

@ -0,0 +1,48 @@
import { TableSchema } from "@budibase/types"
import { FieldTypes } from "../../../constants"
import { makeExternalQuery } from "../../../integrations/base/query"
import { Format } from "../../../api/controllers/view/exporters"
import sdk from "../.."
export async function getDatasourceAndQuery(json: any) {
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
return makeExternalQuery(datasource, json)
}
export function cleanExportRows(
rows: any[],
schema: TableSchema,
format: string,
columns?: string[]
) {
let cleanRows = [...rows]
const relationships = Object.entries(schema)
.filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
.map(entry => entry[0])
relationships.forEach(column => {
cleanRows.forEach(row => {
delete row[column]
})
delete schema[column]
})
if (format === Format.CSV) {
// Intended to append empty values in export
const schemaKeys = Object.keys(schema)
for (let key of schemaKeys) {
if (columns?.length && columns.indexOf(key) > 0) {
continue
}
for (let row of cleanRows) {
if (row[key] == null) {
row[key] = undefined
}
}
}
}
return cleanRows
}

1
packages/server/src/sdk/app/tables/index.ts
View File

@ -28,7 +28,6 @@ async function getAllInternalTables(db?: Database): Promise<Table[]> {
async function getAllExternalTables(
datasourceId: any
): Promise<Record<string, Table>> {
const db = context.getAppDB()
const datasource = await datasources.get(datasourceId, { enriched: true })
if (!datasource || !datasource.entities) {
throw "Datasource is not configured fully."

101
packages/server/src/sdk/tests/rows/row.spec.ts Normal file
View File

@ -0,0 +1,101 @@
import { exportRows } from "../../app/rows/search/external"
import sdk from "../.."
import { ExternalRequest } from "../../../api/controllers/row/ExternalRequest"
import { ExportRowsParams } from "../../app/rows/search"
import { Format } from "../../../api/controllers/view/exporters"
import { HTTPError } from "@budibase/backend-core"
import { Operation } from "@budibase/types"
const mockDatasourcesGet = jest.fn()
sdk.datasources.get = mockDatasourcesGet
jest.mock("../../../api/controllers/row/ExternalRequest")
jest.mock("../../../api/controllers/view/exporters", () => ({
...jest.requireActual("../../../api/controllers/view/exporters"),
csv: jest.fn(),
Format: {
CSV: "csv",
},
}))
jest.mock("../../../utilities/fileSystem")
describe("external row sdk", () => {
describe("exportRows", () => {
function getExportOptions(): ExportRowsParams {
return {
tableId: "datasource__tablename",
format: Format.CSV,
query: {},
}
}
const externalRequestCall = jest.fn()
beforeAll(() => {
jest
.spyOn(ExternalRequest.prototype, "run")
.mockImplementation(externalRequestCall.mockResolvedValue([]))
})
afterEach(() => {
jest.clearAllMocks()
})
it("should throw a 400 if no datasource entities are present", async () => {
const exportOptions = getExportOptions()
await expect(exportRows(exportOptions)).rejects.toThrowError(
new HTTPError("Datasource has not been configured for plus API.", 400)
)
})
it("should handle single quotes from a row ID", async () => {
mockDatasourcesGet.mockImplementation(async () => ({
entities: {
tablename: {
schema: {},
},
},
}))
const exportOptions = getExportOptions()
exportOptions.rowIds = ["['d001']"]
await exportRows(exportOptions)
expect(ExternalRequest).toBeCalledTimes(1)
expect(ExternalRequest).toBeCalledWith(
Operation.READ,
exportOptions.tableId,
undefined
)
expect(externalRequestCall).toBeCalledTimes(1)
expect(externalRequestCall).toBeCalledWith(
expect.objectContaining({
filters: {
oneOf: {
_id: ["d001"],
},
},
})
)
})
it("should throw a 400 if any composite keys are present", async () => {
const exportOptions = getExportOptions()
exportOptions.rowIds = ["[123]", "['d001'%2C'10111']"]
await expect(exportRows(exportOptions)).rejects.toThrowError(
new HTTPError("Export data does not support composite keys.", 400)
)
})
it("should throw a 400 if no table name was found", async () => {
const exportOptions = getExportOptions()
exportOptions.tableId = "datasource__"
exportOptions.rowIds = ["[123]"]
await expect(exportRows(exportOptions)).rejects.toThrowError(
new HTTPError("Could not find table name.", 400)
)
})
})
})

7
packages/server/src/sdk/users/crud.ts Normal file
View File

@ -0,0 +1,7 @@
import { context } from "@budibase/backend-core"
import { User } from "@budibase/types"
export function get(userId: string) {
const db = context.getAppDB()
return db.get<User>(userId)
}

4
packages/server/src/sdk/users/index.ts
View File

@ -1,5 +1,9 @@
import * as utils from "./utils"
import * as sessions from "./sessions"
import * as crud from "./crud"
export default {
...utils,
...crud,
sessions,
}

38
packages/server/src/sdk/users/sessions.ts Normal file
View File

@ -0,0 +1,38 @@
import { builderSocket } from "../../websockets"
import { App, SocketSession } from "@budibase/types"
export const enrichApps = async (apps: App[]) => {
// Sessions can only exist for dev app IDs
const devAppIds = apps
.filter((app: any) => app.status === "development")
.map((app: any) => app.appId)
// Get all sessions for all apps and enrich app list
const sessions = await builderSocket?.getRoomSessions(devAppIds)
if (sessions?.length) {
let appSessionMap: Record<string, SocketSession[]> = {}
sessions.forEach(session => {
const room = session.room
if (!room) {
return
}
if (!appSessionMap[room]) {
appSessionMap[room] = []
}
appSessionMap[room].push(session)
})
return apps.map(app => {
// Shallow clone to avoid mutating original reference
let enriched = { ...app }
const sessions = appSessionMap[app.appId]
if (sessions?.length) {
enriched.sessions = sessions
} else {
delete enriched.sessions
}
return enriched
})
} else {
return apps
}
}

19
packages/server/src/sdk/users/utils.ts
View File

@ -64,6 +64,25 @@ export async function rawUserMetadata(db?: Database) {
).rows.map(row => row.doc)
}
export async function fetchMetadata() {
const global = await getGlobalUsers()
const metadata = await rawUserMetadata()
const users = []
for (let user of global) {
// find the metadata that matches up to the global ID
const info = metadata.find(meta => meta._id.includes(user._id))
// remove these props, not for the correct DB
users.push({
...user,
...info,
tableId: InternalTables.USER_METADATA,
// make sure the ID is always a local ID, not a global one
_id: generateUserMetadataID(user._id),
})
}
return users
}
export async function syncGlobalUsers() {
// sync user metadata
const dbs = [context.getDevAppDB(), context.getProdAppDB()]

5
packages/server/src/tests/utilities/TestConfiguration.ts
View File

@ -135,7 +135,10 @@ class TestConfiguration {
}
}
async doInContext(appId: string | null, task: any) {
async doInContext<T>(
appId: string | null,
task: () => Promise<T>
): Promise<T> {
if (!appId) {
appId = this.appId
}

3
packages/server/src/threads/query.ts
View File

@ -11,6 +11,7 @@ import { cloneDeep } from "lodash/fp"
import { isSQL } from "../integrations/utils"
import { interpolateSQL } from "../integrations/queries/sql"
import { Query } from "@budibase/types"
class QueryRunner {
datasource: any
@ -167,7 +168,7 @@ class QueryRunner {
async runAnotherQuery(queryId: string, parameters: any) {
const db = context.getAppDB()
const query = await db.get(queryId)
const query = await db.get<Query>(queryId)
const datasource = await sdk.datasources.get(query.datasourceId, {
enriched: true,
})

2
packages/server/src/utilities/global.ts
View File

@ -90,7 +90,7 @@ export async function getCachedSelf(ctx: UserCtx, appId: string) {
export async function getRawGlobalUser(userId: string) {
const db = tenancy.getGlobalDB()
return db.get(getGlobalIDFromUserMetadataID(userId))
return db.get<User>(getGlobalIDFromUserMetadataID(userId))
}
export async function getGlobalUser(userId: string) {

4
packages/server/src/utilities/index.ts
View File

@ -41,7 +41,7 @@ export async function updateEntityMetadata(
// read it to see if it exists, we'll overwrite it no matter what
let rev, metadata: Document
try {
const oldMetadata = await db.get(id)
const oldMetadata = await db.get<any>(id)
rev = oldMetadata._rev
metadata = updateFn(oldMetadata)
} catch (err) {
@ -75,7 +75,7 @@ export async function deleteEntityMetadata(type: string, entityId: string) {
const id = generateMetadataID(type, entityId)
let rev
try {
const metadata = await db.get(id)
const metadata = await db.get<any>(id)
if (metadata) {
rev = metadata._rev
}

17
packages/server/src/websockets/websocket.ts
View File

@ -125,9 +125,18 @@ export class BaseSocket {
}
// Gets an array of all redis keys of users inside a certain room
async getRoomSessionIds(room: string): Promise<string[]> {
const keys = await this.redisClient?.get(this.getRoomKey(room))
return keys || []
async getRoomSessionIds(room: string | string[]): Promise<string[]> {
if (Array.isArray(room)) {
const roomKeys = room.map(this.getRoomKey.bind(this))
const roomSessionIdMap = await this.redisClient?.bulkGet(roomKeys)
let sessionIds: any[] = []
Object.values(roomSessionIdMap || {}).forEach(roomSessionIds => {
sessionIds = sessionIds.concat(roomSessionIds)
})
return sessionIds
} else {
return (await this.redisClient?.get(this.getRoomKey(room))) || []
}
}
// Sets the list of redis keys for users inside a certain room.
@ -137,7 +146,7 @@ export class BaseSocket {
}
// Gets a list of all users inside a certain room
async getRoomSessions(room?: string): Promise<SocketSession[]> {
async getRoomSessions(room?: string | string[]): Promise<SocketSession[]> {
if (room) {
const sessionIds = await this.getRoomSessionIds(room)
const keys = sessionIds.map(this.getSessionKey.bind(this))

2
packages/types/src/documents/app/app.ts
View File

@ -1,4 +1,5 @@
import { User, Document } from "../"
import { SocketSession } from "../../sdk"
export type AppMetadataErrors = { [key: string]: string[] }
@ -17,6 +18,7 @@ export interface App extends Document {
customTheme?: AppCustomTheme
revertableVersion?: string
lockedBy?: User
sessions?: SocketSession[]
navigation?: AppNavigation
automationErrors?: AppMetadataErrors
icon?: AppIcon

2
packages/types/src/sdk/db.ts
View File

@ -89,7 +89,7 @@ export interface Database {
exists(): Promise<boolean>
checkSetup(): Promise<Nano.DocumentScope<any>>
get<T>(id?: string): Promise<T | any>
get<T>(id?: string): Promise<T>
remove(
id: string | Document,
rev?: string

2
packages/types/src/sdk/search.ts
View File

@ -32,7 +32,7 @@ export interface SearchFilters {
[key: string]: any[]
}
contains?: {
[key: string]: any[]
[key: string]: any[] | any
}
notContains?: {
[key: string]: any[]

6
packages/worker/src/api/controllers/global/email.ts
View File

@ -1,6 +1,6 @@
import { sendEmail as sendEmailFn } from "../../../utilities/email"
import { tenancy } from "@budibase/backend-core"
import { BBContext } from "@budibase/types"
import { BBContext, User } from "@budibase/types"
export async function sendEmail(ctx: BBContext) {
let {
@ -16,10 +16,10 @@ export async function sendEmail(ctx: BBContext) {
automation,
invite,
} = ctx.request.body
let user
let user: any
if (userId) {
const db = tenancy.getGlobalDB()
user = await db.get(userId)
user = await db.get<User>(userId)
}
const response = await sendEmailFn(email, purpose, {
workspaceId,

2
packages/worker/src/api/controllers/global/roles.ts
View File

@ -35,7 +35,7 @@ export async function find(ctx: BBContext) {
const appId = ctx.params.appId
await context.doInAppContext(dbCore.getDevAppID(appId), async () => {
const db = context.getAppDB()
const app = await db.get(dbCore.DocumentType.APP_METADATA)
const app = await db.get<App>(dbCore.DocumentType.APP_METADATA)
ctx.body = {
roles: await roles.getAllRoles(),
name: app.name,

2
packages/worker/src/api/controllers/global/self.ts
View File

@ -44,7 +44,7 @@ export async function generateAPIKey(ctx: any) {
const id = dbCore.generateDevInfoID(userId)
let devInfo
try {
devInfo = await db.get(id)
devInfo = await db.get<any>(id)
} catch (err) {
devInfo = { _id: id, userId }
}

2
packages/worker/src/api/controllers/global/users.ts
View File

@ -414,7 +414,7 @@ export const inviteAccept = async (
const saved = await userSdk.save(request)
const db = tenancy.getGlobalDB()
const user = await db.get(saved._id)
const user = await db.get<User>(saved._id)
await events.user.inviteAccepted(user)
return saved
})