MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into fix/export-data-ui

This commit is contained in:
Dean 2023-11-09 08:47:08 +00:00
parent 856fe14ae9 8c230f649e
commit 0520a30fed
76 changed files with 995 additions and 647 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hosting/couchdb/build-target-paths.sh
View File

@ -2,8 +2,8 @@
echo ${TARGETBUILD} > /buildtarget.txt
if [[ "${TARGETBUILD}" = "aas" ]]; then
# Azure AppService uses /home for persisent data & SSH on port 2222
DATA_DIR=/home
# Azure AppService uses /home for persistent data & SSH on port 2222
DATA_DIR="${DATA_DIR:-/home}"
WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
mkdir -p $DATA_DIR/{search,minio,couch}
mkdir -p $DATA_DIR/couch/{dbs,views}

4
hosting/scripts/build-target-paths.sh
View File

@ -2,8 +2,8 @@
echo ${TARGETBUILD} > /buildtarget.txt
if [[ "${TARGETBUILD}" = "aas" ]]; then
# Azure AppService uses /home for persisent data & SSH on port 2222
DATA_DIR=/home
# Azure AppService uses /home for persistent data & SSH on port 2222
DATA_DIR="${DATA_DIR:-/home}"
WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
mkdir -p $DATA_DIR/{search,minio,couch}
mkdir -p $DATA_DIR/couch/{dbs,views}

2
hosting/single/runner.sh
View File

@ -22,7 +22,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME
# Azure App Service customisations
if [[ "${TARGETBUILD}" = "aas" ]]; then
DATA_DIR=/home
DATA_DIR="${DATA_DIR:-/home}"
WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
/etc/init.d/ssh start
else

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.13.2",
"version": "2.13.5",
"npmClient": "yarn",
"packages": [
"packages/*"

2
packages/backend-core/src/constants/db.ts
View File

@ -28,7 +28,7 @@ export enum ViewName {
APP_BACKUP_BY_TRIGGER = "by_trigger",
}
export const DeprecatedViews = {
export const DeprecatedViews: Record<string, string[]> = {
[ViewName.USER_BY_EMAIL]: [
// removed due to inaccuracy in view doc filter logic
"by_email",

6
packages/backend-core/src/db/couch/DatabaseImpl.ts
View File

@ -175,12 +175,14 @@ export class DatabaseImpl implements Database {
return this.updateOutput(() => db.bulk({ docs: documents }))
}
async allDocs<T>(params: DatabaseQueryOpts): Promise<AllDocsResponse<T>> {
async allDocs<T extends Document>(
params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>> {
const db = await this.checkSetup()
return this.updateOutput(() => db.list(params))
}
async query<T>(
async query<T extends Document>(
viewName: string,
params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>> {

45
packages/backend-core/src/db/views.ts
View File

@ -7,12 +7,19 @@ import {
} from "../constants"
import { getGlobalDB } from "../context"
import { doWithDB } from "./"
import { AllDocsResponse, Database, DatabaseQueryOpts } from "@budibase/types"
import {
AllDocsResponse,
Database,
DatabaseQueryOpts,
Document,
DesignDocument,
DBView,
} from "@budibase/types"
import env from "../environment"
const DESIGN_DB = "_design/database"
function DesignDoc() {
function DesignDoc(): DesignDocument {
return {
_id: DESIGN_DB,
// view collation information, read before writing any complex views:
@ -21,20 +28,14 @@ function DesignDoc() {
}
}
interface DesignDocument {
views: any
}
async function removeDeprecated(db: Database, viewName: ViewName) {
// @ts-ignore
if (!DeprecatedViews[viewName]) {
return
}
try {
const designDoc = await db.get<DesignDocument>(DESIGN_DB)
// @ts-ignore
for (let deprecatedNames of DeprecatedViews[viewName]) {
delete designDoc.views[deprecatedNames]
delete designDoc.views?.[deprecatedNames]
}
await db.put(designDoc)
} catch (err) {
@ -43,18 +44,18 @@ async function removeDeprecated(db: Database, viewName: ViewName) {
}
export async function createView(
db: any,
db: Database,
viewJs: string,
viewName: string
): Promise<void> {
let designDoc
try {
designDoc = (await db.get(DESIGN_DB)) as DesignDocument
designDoc = await db.get<DesignDocument>(DESIGN_DB)
} catch (err) {
// no design doc, make one
designDoc = DesignDoc()
}
const view = {
const view: DBView = {
map: viewJs,
}
designDoc.views = {
@ -109,7 +110,7 @@ export interface QueryViewOptions {
arrayResponse?: boolean
}
export async function queryViewRaw<T>(
export async function queryViewRaw<T extends Document>(
viewName: ViewName,
params: DatabaseQueryOpts,
db: Database,
@ -137,18 +138,16 @@ export async function queryViewRaw<T>(
}
}
export const queryView = async <T>(
export const queryView = async <T extends Document>(
viewName: ViewName,
params: DatabaseQueryOpts,
db: Database,
createFunc: any,
opts?: QueryViewOptions
): Promise<T[] | T | undefined> => {
): Promise<T[] | T> => {
const response = await queryViewRaw<T>(viewName, params, db, createFunc, opts)
const rows = response.rows
const docs = rows.map((row: any) =>
params.include_docs ? row.doc : row.value
)
const docs = rows.map(row => (params.include_docs ? row.doc! : row.value))
// if arrayResponse has been requested, always return array regardless of length
if (opts?.arrayResponse) {
@ -198,11 +197,11 @@ export const createPlatformUserView = async () => {
await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
}
export const queryPlatformView = async <T>(
export const queryPlatformView = async <T extends Document>(
viewName: ViewName,
params: DatabaseQueryOpts,
opts?: QueryViewOptions
): Promise<T[] | T | undefined> => {
): Promise<T[] | T> => {
const CreateFuncByName: any = {
[ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView,
[ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
@ -220,7 +219,7 @@ const CreateFuncByName: any = {
[ViewName.USER_BY_APP]: createUserAppView,
}
export const queryGlobalView = async <T>(
export const queryGlobalView = async <T extends Document>(
viewName: ViewName,
params: DatabaseQueryOpts,
db?: Database,
@ -231,10 +230,10 @@ export const queryGlobalView = async <T>(
db = getGlobalDB()
}
const createFn = CreateFuncByName[viewName]
return queryView(viewName, params, db!, createFn, opts)
return queryView<T>(viewName, params, db!, createFn, opts)
}
export async function queryGlobalViewRaw<T>(
export async function queryGlobalViewRaw<T extends Document>(
viewName: ViewName,
params: DatabaseQueryOpts,
opts?: QueryViewOptions

10
packages/backend-core/src/users/db.ts
View File

@ -413,15 +413,13 @@ export class UserDB {
}
// Get users and delete
const allDocsResponse: AllDocsResponse<User> = await db.allDocs({
const allDocsResponse = await db.allDocs<User>({
include_docs: true,
keys: userIds,
})
const usersToDelete: User[] = allDocsResponse.rows.map(
(user: RowResponse<User>) => {
return user.doc
}
)
const usersToDelete = allDocsResponse.rows.map(user => {
return user.doc!
})
// Delete from DB
const toDelete = usersToDelete.map(user => ({

2
packages/backend-core/src/users/users.ts
View File

@ -151,7 +151,7 @@ export const searchGlobalUsersByApp = async (
include_docs: true,
})
params.startkey = opts && opts.startkey ? opts.startkey : params.startkey
let response = await queryGlobalView(ViewName.USER_BY_APP, params)
let response = await queryGlobalView<User>(ViewName.USER_BY_APP, params)
if (!response) {
response = []

13
packages/backend-core/src/utils/tests/utils.spec.ts
View File

@ -188,4 +188,17 @@ describe("utils", () => {
expectResult(false)
})
})
describe("hasCircularStructure", () => {
it("should detect a circular structure", () => {
const a: any = { b: "b" }
const b = { a }
a.b = b
expect(utils.hasCircularStructure(b)).toBe(true)
})
it("should allow none circular structures", () => {
expect(utils.hasCircularStructure({ a: "b" })).toBe(false)
})
})
})

14
packages/backend-core/src/utils/utils.ts
View File

@ -237,3 +237,17 @@ export function timeout(timeMs: number) {
export function isAudited(event: Event) {
return !!AuditedEventFriendlyName[event]
}
export function hasCircularStructure(json: any) {
if (typeof json !== "object") {
return false
}
try {
JSON.stringify(json)
} catch (err) {
if (err instanceof Error && err?.message.includes("circular structure")) {
return true
}
}
return false
}

15
packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte
View File

@ -48,15 +48,14 @@
<UndoRedoControl store={automationHistoryStore} />
</div>
<div class="controls">
<div class="buttons">
<div
on:click={() => {
testDataModal.show()
}}
class="buttons"
>
<Icon hoverable size="M" name="Play" />
<div
on:click={() => {
testDataModal.show()
}}
>
Run test
</div>
<div>Run test</div>
</div>
<div class="buttons">
<Icon

68
packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte
View File

@ -13,13 +13,13 @@
export let idx
export let addLooping
export let deleteStep
export let enableNaming = true
let validRegex = /^[A-Za-z0-9_\s]+$/
let typing = false
const dispatch = createEventDispatcher()
$: stepNames = $selectedAutomation.definition.stepNames
$: stepNames = $selectedAutomation?.definition.stepNames
$: automationName = stepNames?.[block.id] || block?.name || ""
$: automationNameError = getAutomationNameError(automationName)
$: status = updateStatus(testResult, isTrigger)
@ -32,7 +32,7 @@
)?.[0]
}
}
$: loopBlock = $selectedAutomation.definition.steps.find(
$: loopBlock = $selectedAutomation?.definition.steps.find(
x => x.blockToLoop === block?.id
)
@ -126,24 +126,33 @@
<Body size="XS"><b>Step {idx}</b></Body>
</div>
{/if}
<input
placeholder="Enter some text"
name="name"
autocomplete="off"
value={automationName}
on:input={e => {
automationName = e.target.value.trim()
}}
on:click={startTyping}
on:blur={async () => {
typing = false
if (automationNameError) {
automationName = stepNames[block.id] || block?.name
} else {
await saveName()
}
}}
/>
{#if enableNaming}
<input
class="input-text"
disabled={!enableNaming}
placeholder="Enter some text"
name="name"
autocomplete="off"
value={automationName}
on:input={e => {
automationName = e.target.value.trim()
}}
on:click={startTyping}
on:blur={async () => {
typing = false
if (automationNameError) {
automationName = stepNames[block.id] || block?.name
} else {
await saveName()
}
}}
/>
{:else}
<div class="input-text">
{automationName}
</div>
{/if}
</div>
</div>
<div class="blockTitle">
@ -178,9 +187,11 @@
<Icon on:click={addLooping} hoverable name="RotateCW" />
</AbsTooltip>
{/if}
<AbsTooltip type="negative" text="Delete step">
<Icon on:click={deleteStep} hoverable name="DeleteOutline" />
</AbsTooltip>
{#if !isHeaderTrigger}
<AbsTooltip type="negative" text="Delete step">
<Icon on:click={deleteStep} hoverable name="DeleteOutline" />
</AbsTooltip>
{/if}
{/if}
{#if !showTestStatus}
<Icon
@ -244,18 +255,21 @@
display: none;
}
input {
font-family: var(--font-sans);
color: var(--ink);
background-color: transparent;
border: 1px solid transparent;
font-size: var(--spectrum-alias-font-size-default);
width: 230px;
box-sizing: border-box;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.input-text {
font-size: var(--spectrum-alias-font-size-default);
font-family: var(--font-sans);
text-overflow: ellipsis;
}
input:focus {
outline: none;
}

1
packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte
View File

@ -48,6 +48,7 @@
<div class="block" style={width ? `width: ${width}` : ""}>
{#if block.stepId !== ActionStepID.LOOP}
<FlowItemHeader
enableNaming={false}
open={!!openBlocks[block.id]}
on:toggle={() => (openBlocks[block.id] = !openBlocks[block.id])}
isTrigger={idx === 0}

14
packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte
View File

@ -27,7 +27,7 @@
$: if (value?.queryId == null) value = { queryId: "" }
</script>
<div class="schema-fields">
<div class="schema-field">
<Label>Query</Label>
<div class="field-width">
<Select
@ -41,8 +41,8 @@
</div>
{#if parameters.length}
<div class="schema-fields">
{#each parameters as field}
{#each parameters as field}
<div class="schema-field">
<Label>{field.name}</Label>
<div class="field-width">
<DrawerBindableInput
@ -56,8 +56,8 @@
updateOnChange={false}
/>
</div>
{/each}
</div>
</div>
{/each}
{/if}
<style>
@ -65,7 +65,7 @@
width: 320px;
}
.schema-fields {
.schema-field {
display: flex;
justify-content: space-between;
align-items: center;
@ -76,7 +76,7 @@
margin-bottom: 10px;
}
.schema-fields :global(label) {
.schema-field :global(label) {
text-transform: capitalize;
}
</style>

32
packages/builder/src/components/automation/SetupPanel/RowSelector.svelte
View File

@ -114,10 +114,10 @@
</div>
{#if schemaFields.length}
{#each schemaFields as [field, schema]}
<div class="schema-fields">
<Label>{field}</Label>
<div class="field-width">
{#if !schema.autocolumn && schema.type !== "attachment"}
{#if !schema.autocolumn && schema.type !== "attachment"}
<div class="schema-fields">
<Label>{field}</Label>
<div class="field-width">
{#if isTestModal}
<RowSelectorTypes
{isTestModal}
@ -151,20 +151,20 @@
/>
</DrawerBindableSlot>
{/if}
{/if}
{#if isUpdateRow && schema.type === "link"}
<div class="checkbox-field">
<Checkbox
value={meta.fields?.[field]?.clearRelationships}
text={"Clear relationships if empty?"}
size={"S"}
on:change={e => onChangeSetting(e, field)}
/>
</div>
{/if}
{#if isUpdateRow && schema.type === "link"}
<div class="checkbox-field">
<Checkbox
value={meta.fields?.[field]?.clearRelationships}
text={"Clear relationships if empty?"}
size={"S"}
on:change={e => onChangeSetting(e, field)}
/>
</div>
{/if}
</div>
</div>
</div>
{/if}
{/each}
{/if}

1
packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte
View File

@ -67,6 +67,7 @@
bind:linkedRows={value[field]}
{schema}
on:change={e => onChange(e, field)}
useLabel={false}
/>
{:else if schema.type === "string" || schema.type === "number"}
<svelte:component

4
packages/builder/src/components/common/LinkedRowSelector.svelte
View File

@ -7,7 +7,7 @@
export let schema
export let linkedRows = []
export let useLabel = true
const dispatch = createEventDispatcher()
let rows = []
@ -51,7 +51,7 @@
linkedIds = e.detail ? [e.detail] : []
dispatch("change", linkedIds)
}}
{label}
label={useLabel ? label : null}
sort
/>
{:else}

2
packages/builder/src/pages/builder/app/[application]/settings/automation-history/_components/HistoryDetailsPanel.svelte
View File

@ -56,7 +56,7 @@
{/if}
{#key history}
<div class="history">
<TestDisplay testResults={history} width="100%" />
<TestDisplay testResults={history} width="320px" />
</div>
{/key}
</Layout>

2
packages/pro

@ -1 +1 @@
Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8
Subproject commit ad9a0085bee0c4f3184acd86cadd872ea9917e88

2
packages/server/src/api/controllers/auth.ts
View File

@ -26,7 +26,7 @@ export async function fetchSelf(ctx: UserCtx) {
}
const appId = context.getAppId()
let user: ContextUser = await getFullUser(ctx, userId)
let user: ContextUser = await getFullUser(userId)
// this shouldn't be returned by the app self
delete user.roles
// forward the csrf token from the session

1
packages/server/src/api/controllers/deploy/index.ts
View File

@ -106,7 +106,6 @@ export async function fetchDeployments(ctx: any) {
}
ctx.body = Object.values(deployments.history).reverse()
} catch (err) {
console.error(err)
ctx.body = []
}
}

6
packages/server/src/api/controllers/layout.ts
View File

@ -1,7 +1,7 @@
import { EMPTY_LAYOUT } from "../../constants/layouts"
import { generateLayoutID, getScreenParams } from "../../db/utils"
import { events, context } from "@budibase/backend-core"
import { BBContext } from "@budibase/types"
import { BBContext, Layout } from "@budibase/types"
export async function save(ctx: BBContext) {
const db = context.getAppDB()
@ -30,12 +30,12 @@ export async function destroy(ctx: BBContext) {
layoutRev = ctx.params.layoutRev
const layoutsUsedByScreens = (
await db.allDocs(
await db.allDocs<Layout>(
getScreenParams(null, {
include_docs: true,
})
)
).rows.map(element => element.doc.layoutId)
).rows.map(element => element.doc!.layoutId)
if (layoutsUsedByScreens.includes(layoutId)) {
ctx.throw(400, "Cannot delete a layout that's being used by a screen")
}

6
packages/server/src/api/controllers/permission.ts
View File

@ -25,12 +25,12 @@ const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS
// utility function to stop this repetition - permissions always stored under roles
async function getAllDBRoles(db: Database) {
const body = await db.allDocs(
const body = await db.allDocs<Role>(
getRoleParams(null, {
include_docs: true,
})
)
return body.rows.map(row => row.doc)
return body.rows.map(row => row.doc!)
}
async function updatePermissionOnRole(
@ -79,7 +79,7 @@ async function updatePermissionOnRole(
) {
rolePermissions[resourceId] =
typeof rolePermissions[resourceId] === "string"
? [rolePermissions[resourceId]]
? [rolePermissions[resourceId] as unknown as string]
: []
}
// handle the removal/updating the role which has this permission first

12
packages/server/src/api/controllers/role.ts
View File

@ -6,7 +6,13 @@ import {
Header,
} from "@budibase/backend-core"
import { getUserMetadataParams, InternalTables } from "../../db/utils"
import { Database, Role, UserCtx, UserRoles } from "@budibase/types"
import {
Database,
Role,
UserCtx,
UserMetadata,
UserRoles,
} from "@budibase/types"
import { sdk as sharedSdk } from "@budibase/shared-core"
import sdk from "../../sdk"
@ -115,12 +121,12 @@ export async function destroy(ctx: UserCtx) {
const role = await db.get<Role>(roleId)
// first check no users actively attached to role
const users = (
await db.allDocs(
await db.allDocs<UserMetadata>(
getUserMetadataParams(undefined, {
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map(row => row.doc!)
const usersWithRole = users.filter(user => user.roleId === roleId)
if (usersWithRole.length !== 0) {
ctx.throw(400, "Cannot delete role when it is in use.")

33
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -23,7 +23,6 @@ import {
breakRowIdField,
convertRowId,
generateRowIdField,
getPrimaryDisplay,
isRowId,
isSQL,
} from "../../../integrations/utils"
@ -237,7 +236,7 @@ function basicProcessing({
thisRow._id = generateIdForRow(row, table, isLinked)
thisRow.tableId = table._id
thisRow._rev = "rev"
return processFormulas(table, thisRow)
return thisRow
}
function fixArrayTypes(row: Row, table: Table) {
@ -392,7 +391,7 @@ export class ExternalRequest<T extends Operation> {
return { row: newRow, manyRelationships }
}
squashRelationshipColumns(
processRelationshipFields(
table: Table,
row: Row,
relationships: RelationshipsJson[]
@ -402,7 +401,6 @@ export class ExternalRequest<T extends Operation> {
if (!linkedTable || !row[relationship.column]) {
continue
}
const display = linkedTable.primaryDisplay
for (let key of Object.keys(row[relationship.column])) {
let relatedRow: Row = row[relationship.column][key]
// add this row as context for the relationship
@ -411,15 +409,10 @@ export class ExternalRequest<T extends Operation> {
relatedRow[col.name] = [row]
}
}
// process additional types
relatedRow = processDates(table, relatedRow)
relatedRow = processFormulas(linkedTable, relatedRow)
let relatedDisplay
if (display) {
relatedDisplay = getPrimaryDisplay(relatedRow[display])
}
row[relationship.column][key] = {
primaryDisplay: relatedDisplay || "Invalid display column",
_id: relatedRow._id,
}
row[relationship.column][key] = relatedRow
}
}
return row
@ -521,14 +514,14 @@ export class ExternalRequest<T extends Operation> {
)
}
// Process some additional data types
let finalRowArray = Object.values(finalRows)
finalRowArray = processDates(table, finalRowArray)
finalRowArray = processFormulas(table, finalRowArray) as Row[]
return finalRowArray.map((row: Row) =>
this.squashRelationshipColumns(table, row, relationships)
// make sure all related rows are correct
let finalRowArray = Object.values(finalRows).map(row =>
this.processRelationshipFields(table, row, relationships)
)
// process some additional types
finalRowArray = processDates(table, finalRowArray)
return finalRowArray
}
/**
@ -663,7 +656,7 @@ export class ExternalRequest<T extends Operation> {
linkPrimary,
linkSecondary,
}: {
row: { [key: string]: any }
row: Row
linkPrimary: string
linkSecondary?: string
}) {

14
packages/server/src/api/controllers/row/external.ts
View File

@ -76,6 +76,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
relationships: true,
})
const enrichedRow = await outputProcessing(table, row, {
squash: true,
preserveLinks: true,
})
return {
@ -119,7 +120,10 @@ export async function save(ctx: UserCtx) {
})
return {
...response,
row: await outputProcessing(table, row, { preserveLinks: true }),
row: await outputProcessing(table, row, {
preserveLinks: true,
squash: true,
}),
}
} else {
return response
@ -140,7 +144,7 @@ export async function find(ctx: UserCtx): Promise<Row> {
const table = await sdk.tables.getTable(tableId)
// Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case
return await outputProcessing(table, row, {
squash: false,
squash: true,
preserveLinks: true,
})
}
@ -207,7 +211,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
// don't support composite keys right now
const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0])
const primaryLink = linkedTable.primary?.[0] as string
row[fieldName] = await handleRequest(Operation.READ, linkedTableId!, {
const relatedRows = await handleRequest(Operation.READ, linkedTableId!, {
tables,
filters: {
oneOf: {
@ -216,6 +220,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
},
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
row[fieldName] = await outputProcessing(linkedTable, relatedRows, {
squash: true,
preserveLinks: true,
})
}
return row
}

4
packages/server/src/api/controllers/row/index.ts
View File

@ -254,7 +254,7 @@ export const exportRows = async (
const format = ctx.query.format
const { rows, columns, query } = ctx.request.body
const { rows, columns, query, sort, sortOrder } = ctx.request.body
if (typeof format !== "string" || !exporters.isFormat(format)) {
ctx.throw(
400,
@ -272,6 +272,8 @@ export const exportRows = async (
rowIds: rows,
columns,
query,
sort,
sortOrder,
})
ctx.attachment(fileName)
return apiFileReturn(content)

12
packages/server/src/api/controllers/row/internal.ts
View File

@ -2,7 +2,6 @@ import * as linkRows from "../../../db/linkedRows"
import {
generateRowID,
getMultiIDParams,
getTableIDFromRowID,
InternalTables,
} from "../../../db/utils"
import * as userController from "../user"
@ -89,7 +88,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
if (isUserTable) {
// the row has been updated, need to put it into the ctx
ctx.request.body = row as any
await userController.updateMetadata(ctx)
await userController.updateMetadata(ctx as any)
return { row: ctx.body as Row, table }
}
@ -233,17 +232,16 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
const tableId = utils.getTableId(ctx)
const rowId = ctx.params.rowId as string
// need table to work out where links go in row, as well as the link docs
let response = await Promise.all([
const [table, row, links] = await Promise.all([
sdk.tables.getTable(tableId),
utils.findRow(ctx, tableId, rowId),
linkRows.getLinkDocuments({ tableId, rowId, fieldName }),
])
const table = response[0] as Table
const row = response[1] as Row
const linkVals = response[2] as LinkDocumentValue[]
const linkVals = links as LinkDocumentValue[]
// look up the actual rows based on the ids
const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc)
let linkedRows = (await db.allDocs<Row>(params)).rows.map(row => row.doc!)
// get the linked tables
const linkTableIds = getLinkedTableIDs(table as Table)

14
packages/server/src/api/controllers/row/staticFormula.ts
View File

@ -86,12 +86,12 @@ export async function updateAllFormulasInTable(table: Table) {
const db = context.getAppDB()
// start by getting the raw rows (which will be written back to DB after update)
let rows = (
await db.allDocs(
await db.allDocs<Row>(
getRowParams(table._id, null, {
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map(row => row.doc!)
// now enrich the rows, note the clone so that we have the base state of the
// rows so that we don't write any of the enriched information back
let enrichedRows = await outputProcessing(table, cloneDeep(rows), {
@ -101,12 +101,12 @@ export async function updateAllFormulasInTable(table: Table) {
for (let row of rows) {
// find the enriched row, if found process the formulas
const enrichedRow = enrichedRows.find(
(enriched: any) => enriched._id === row._id
(enriched: Row) => enriched._id === row._id
)
if (enrichedRow) {
const processed = processFormulas(table, cloneDeep(row), {
dynamic: false,
contextRows: enrichedRow,
contextRows: [enrichedRow],
})
// values have changed, need to add to bulk docs to update
if (!isEqual(processed, row)) {
@ -139,7 +139,7 @@ export async function finaliseRow(
// use enriched row to generate formulas for saving, specifically only use as context
row = processFormulas(table, row, {
dynamic: false,
contextRows: enrichedRow,
contextRows: [enrichedRow],
})
// don't worry about rev, tables handle rev/lastID updates
// if another row has been written since processing this will
@ -163,7 +163,9 @@ export async function finaliseRow(
const response = await db.put(row)
// for response, calculate the formulas for the enriched row
enrichedRow._rev = response.rev
enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
enrichedRow = processFormulas(table, enrichedRow, {
dynamic: false,
})
// this updates the related formulas in other rows based on the relations to this row
if (updateFormula) {
await updateRelatedFormula(table, enrichedRow)

51
packages/server/src/api/controllers/table/utils.ts
View File

@ -333,29 +333,33 @@ export async function checkForViewUpdates(
columnRename?: RenameColumn
) {
const views = await getViews()
const tableViews = views.filter(view => view.meta.tableId === table._id)
const tableViews = views.filter(view => view.meta?.tableId === table._id)
// Check each table view to see if impacted by this table action
for (let view of tableViews) {
let needsUpdated = false
const viewMetadata = view.meta as any
if (!viewMetadata) {
continue
}
// First check for renames, otherwise check for deletions
if (columnRename) {
// Update calculation field if required
if (view.meta.field === columnRename.old) {
view.meta.field = columnRename.updated
if (viewMetadata.field === columnRename.old) {
viewMetadata.field = columnRename.updated
needsUpdated = true
}
// Update group by field if required
if (view.meta.groupBy === columnRename.old) {
view.meta.groupBy = columnRename.updated
if (viewMetadata.groupBy === columnRename.old) {
viewMetadata.groupBy = columnRename.updated
needsUpdated = true
}
// Update filters if required
if (view.meta.filters) {
view.meta.filters.forEach((filter: any) => {
if (viewMetadata.filters) {
viewMetadata.filters.forEach((filter: any) => {
if (filter.key === columnRename.old) {
filter.key = columnRename.updated
needsUpdated = true
@ -365,26 +369,26 @@ export async function checkForViewUpdates(
} else if (deletedColumns) {
deletedColumns.forEach((column: string) => {
// Remove calculation statement if required
if (view.meta.field === column) {
delete view.meta.field
delete view.meta.calculation
delete view.meta.groupBy
if (viewMetadata.field === column) {
delete viewMetadata.field
delete viewMetadata.calculation
delete viewMetadata.groupBy
needsUpdated = true
}
// Remove group by field if required
if (view.meta.groupBy === column) {
delete view.meta.groupBy
if (viewMetadata.groupBy === column) {
delete viewMetadata.groupBy
needsUpdated = true
}
// Remove filters referencing deleted field if required
if (view.meta.filters && view.meta.filters.length) {
const initialLength = view.meta.filters.length
view.meta.filters = view.meta.filters.filter((filter: any) => {
if (viewMetadata.filters && viewMetadata.filters.length) {
const initialLength = viewMetadata.filters.length
viewMetadata.filters = viewMetadata.filters.filter((filter: any) => {
return filter.key !== column
})
if (initialLength !== view.meta.filters.length) {
if (initialLength !== viewMetadata.filters.length) {
needsUpdated = true
}
}
@ -397,15 +401,16 @@ export async function checkForViewUpdates(
(field: any) => field.name == view.groupBy
)
const newViewTemplate = viewTemplate(
view.meta,
viewMetadata,
groupByField?.type === FieldTypes.ARRAY
)
await saveView(null, view.name, newViewTemplate)
if (!newViewTemplate.meta.schema) {
newViewTemplate.meta.schema = table.schema
const viewName = view.name!
await saveView(null, viewName, newViewTemplate)
if (!newViewTemplate.meta?.schema) {
newViewTemplate.meta!.schema = table.schema
}
if (table.views?.[view.name]) {
table.views[view.name] = newViewTemplate.meta as View
if (table.views?.[viewName]) {
table.views[viewName] = newViewTemplate.meta as View
}
}
}

50
packages/server/src/api/controllers/user.ts
View File

@ -1,14 +1,26 @@
import { generateUserFlagID, InternalTables } from "../../db/utils"
import { getFullUser } from "../../utilities/users"
import { context } from "@budibase/backend-core"
import { Ctx, UserCtx } from "@budibase/types"
import {
ContextUserMetadata,
Ctx,
FetchUserMetadataResponse,
FindUserMetadataResponse,
Flags,
SetFlagRequest,
UserCtx,
UserMetadata,
} from "@budibase/types"
import sdk from "../../sdk"
import { DocumentInsertResponse } from "@budibase/nano"
export async function fetchMetadata(ctx: Ctx) {
export async function fetchMetadata(ctx: Ctx<void, FetchUserMetadataResponse>) {
ctx.body = await sdk.users.fetchMetadata()
}
export async function updateSelfMetadata(ctx: UserCtx) {
export async function updateSelfMetadata(
ctx: UserCtx<UserMetadata, DocumentInsertResponse>
) {
// overwrite the ID with current users
ctx.request.body._id = ctx.user?._id
// make sure no stale rev
@ -18,19 +30,21 @@ export async function updateSelfMetadata(ctx: UserCtx) {
await updateMetadata(ctx)
}
export async function updateMetadata(ctx: UserCtx) {
export async function updateMetadata(
ctx: UserCtx<UserMetadata, DocumentInsertResponse>
) {
const db = context.getAppDB()
const user = ctx.request.body
// this isn't applicable to the user
delete user.roles
const metadata = {
const metadata: ContextUserMetadata = {
tableId: InternalTables.USER_METADATA,
...user,
}
// this isn't applicable to the user
delete metadata.roles
ctx.body = await db.put(metadata)
}
export async function destroyMetadata(ctx: UserCtx) {
export async function destroyMetadata(ctx: UserCtx<void, { message: string }>) {
const db = context.getAppDB()
try {
const dbUser = await sdk.users.get(ctx.params.id)
@ -43,11 +57,15 @@ export async function destroyMetadata(ctx: UserCtx) {
}
}
export async function findMetadata(ctx: UserCtx) {
ctx.body = await getFullUser(ctx, ctx.params.id)
export async function findMetadata(
ctx: UserCtx<void, FindUserMetadataResponse>
) {
ctx.body = await getFullUser(ctx.params.id)
}
export async function setFlag(ctx: UserCtx) {
export async function setFlag(
ctx: UserCtx<SetFlagRequest, { message: string }>
) {
const userId = ctx.user?._id
const { flag, value } = ctx.request.body
if (!flag) {
@ -55,9 +73,9 @@ export async function setFlag(ctx: UserCtx) {
}
const flagDocId = generateUserFlagID(userId!)
const db = context.getAppDB()
let doc
let doc: Flags
try {
doc = await db.get<any>(flagDocId)
doc = await db.get<Flags>(flagDocId)
} catch (err) {
doc = { _id: flagDocId }
}
@ -66,13 +84,13 @@ export async function setFlag(ctx: UserCtx) {
ctx.body = { message: "Flag set successfully" }
}
export async function getFlags(ctx: UserCtx) {
export async function getFlags(ctx: UserCtx<void, Flags>) {
const userId = ctx.user?._id
const docId = generateUserFlagID(userId!)
const db = context.getAppDB()
let doc
let doc: Flags
try {
doc = await db.get(docId)
doc = await db.get<Flags>(docId)
} catch (err) {
doc = { _id: docId }
}

96
packages/server/src/api/controllers/view/utils.ts
View File

@ -8,13 +8,13 @@ import {
import env from "../../../environment"
import { context } from "@budibase/backend-core"
import viewBuilder from "./viewBuilder"
import { Database } from "@budibase/types"
import { Database, DBView, DesignDocument, InMemoryView } from "@budibase/types"
export async function getView(viewName: string) {
const db = context.getAppDB()
if (env.SELF_HOSTED) {
const designDoc = await db.get<any>("_design/database")
return designDoc.views[viewName]
const designDoc = await db.get<DesignDocument>("_design/database")
return designDoc.views?.[viewName]
} else {
// This is a table view, don't read the view from the DB
if (viewName.startsWith(DocumentType.TABLE + SEPARATOR)) {
@ -22,7 +22,7 @@ export async function getView(viewName: string) {
}
try {
const viewDoc = await db.get<any>(generateMemoryViewID(viewName))
const viewDoc = await db.get<InMemoryView>(generateMemoryViewID(viewName))
return viewDoc.view
} catch (err: any) {
// Return null when PouchDB doesn't found the view
@ -35,30 +35,33 @@ export async function getView(viewName: string) {
}
}
export async function getViews() {
export async function getViews(): Promise<DBView[]> {
const db = context.getAppDB()
const response = []
const response: DBView[] = []
if (env.SELF_HOSTED) {
const designDoc = await db.get<any>("_design/database")
for (let name of Object.keys(designDoc.views)) {
const designDoc = await db.get<DesignDocument>("_design/database")
for (let name of Object.keys(designDoc.views || {})) {
// Only return custom views, not built ins
const viewNames = Object.values(ViewName) as string[]
if (viewNames.indexOf(name) !== -1) {
continue
}
response.push({
name,
...designDoc.views[name],
})
const view = designDoc.views?.[name]
if (view) {
response.push({
name,
...view,
})
}
}
} else {
const views = (
await db.allDocs(
await db.allDocs<InMemoryView>(
getMemoryViewParams({
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map(row => row.doc!)
for (let viewDoc of views) {
response.push({
name: viewDoc.name,
@ -72,11 +75,11 @@ export async function getViews() {
export async function saveView(
originalName: string | null,
viewName: string,
viewTemplate: any
viewTemplate: DBView
) {
const db = context.getAppDB()
if (env.SELF_HOSTED) {
const designDoc = await db.get<any>("_design/database")
const designDoc = await db.get<DesignDocument>("_design/database")
designDoc.views = {
...designDoc.views,
[viewName]: viewTemplate,
@ -89,17 +92,17 @@ export async function saveView(
} else {
const id = generateMemoryViewID(viewName)
const originalId = originalName ? generateMemoryViewID(originalName) : null
const viewDoc: any = {
const viewDoc: InMemoryView = {
_id: id,
view: viewTemplate,
name: viewName,
tableId: viewTemplate.meta.tableId,
tableId: viewTemplate.meta!.tableId,
}
try {
const old = await db.get<any>(id)
const old = await db.get<InMemoryView>(id)
if (originalId) {
const originalDoc = await db.get<any>(originalId)
await db.remove(originalDoc._id, originalDoc._rev)
const originalDoc = await db.get<InMemoryView>(originalId)
await db.remove(originalDoc._id!, originalDoc._rev)
}
if (old && old._rev) {
viewDoc._rev = old._rev
@ -114,52 +117,65 @@ export async function saveView(
export async function deleteView(viewName: string) {
const db = context.getAppDB()
if (env.SELF_HOSTED) {
const designDoc = await db.get<any>("_design/database")
const view = designDoc.views[viewName]
delete designDoc.views[viewName]
const designDoc = await db.get<DesignDocument>("_design/database")
const view = designDoc.views?.[viewName]
delete designDoc.views?.[viewName]
await db.put(designDoc)
return view
} else {
const id = generateMemoryViewID(viewName)
const viewDoc = await db.get<any>(id)
await db.remove(viewDoc._id, viewDoc._rev)
const viewDoc = await db.get<InMemoryView>(id)
await db.remove(viewDoc._id!, viewDoc._rev)
return viewDoc.view
}
}
export async function migrateToInMemoryView(db: Database, viewName: string) {
// delete the view initially
const designDoc = await db.get<any>("_design/database")
const designDoc = await db.get<DesignDocument>("_design/database")
const meta = designDoc.views?.[viewName].meta
if (!meta) {
throw new Error("Unable to migrate view - no metadata")
}
// run the view back through the view builder to update it
const view = viewBuilder(designDoc.views[viewName].meta)
delete designDoc.views[viewName]
const view = viewBuilder(meta)
delete designDoc.views?.[viewName]
await db.put(designDoc)
await exports.saveView(db, null, viewName, view)
await saveView(null, viewName, view)
}
export async function migrateToDesignView(db: Database, viewName: string) {
let view = await db.get<any>(generateMemoryViewID(viewName))
const designDoc = await db.get<any>("_design/database")
designDoc.views[viewName] = viewBuilder(view.view.meta)
let view = await db.get<InMemoryView>(generateMemoryViewID(viewName))
const designDoc = await db.get<DesignDocument>("_design/database")
const meta = view.view.meta
if (!meta) {
throw new Error("Unable to migrate view - no metadata")
}
if (!designDoc.views) {
designDoc.views = {}
}
designDoc.views[viewName] = viewBuilder(meta)
await db.put(designDoc)
await db.remove(view._id, view._rev)
await db.remove(view._id!, view._rev)
}
export async function getFromDesignDoc(db: Database, viewName: string) {
const designDoc = await db.get<any>("_design/database")
let view = designDoc.views[viewName]
const designDoc = await db.get<DesignDocument>("_design/database")
let view = designDoc.views?.[viewName]
if (view == null) {
throw { status: 404, message: "Unable to get view" }
}
return view
}
export async function getFromMemoryDoc(db: Database, viewName: string) {
let view = await db.get<any>(generateMemoryViewID(viewName))
export async function getFromMemoryDoc(
db: Database,
viewName: string
): Promise<DBView> {
let view = await db.get<InMemoryView>(generateMemoryViewID(viewName))
if (view) {
view = view.view
return view.view
} else {
throw { status: 404, message: "Unable to get view" }
}
return view
}

13
packages/server/src/api/controllers/view/viewBuilder.ts
View File

@ -1,13 +1,4 @@
import { ViewFilter } from "@budibase/types"
type ViewTemplateOpts = {
field: string
tableId: string
groupBy: string
filters: ViewFilter[]
calculation: string
groupByMulti: boolean
}
import { ViewFilter, ViewTemplateOpts, DBView } from "@budibase/types"
const TOKEN_MAP: Record<string, string> = {
EQUALS: "===",
@ -146,7 +137,7 @@ function parseEmitExpression(field: string, groupBy: string) {
export default function (
{ field, tableId, groupBy, filters = [], calculation }: ViewTemplateOpts,
groupByMulti?: boolean
) {
): DBView {
// first filter can't have a conjunction
if (filters && filters.length > 0 && filters[0].conjunction) {
delete filters[0].conjunction

14
packages/server/src/api/controllers/view/views.ts
View File

@ -47,8 +47,11 @@ export async function save(ctx: Ctx) {
// add views to table document
if (!table.views) table.views = {}
if (!view.meta.schema) {
view.meta.schema = table.schema
if (!view.meta?.schema) {
view.meta = {
...view.meta!,
schema: table.schema,
}
}
table.views[viewName] = { ...view.meta, name: viewName }
if (originalName) {
@ -125,10 +128,13 @@ export async function destroy(ctx: Ctx) {
const db = context.getAppDB()
const viewName = decodeURIComponent(ctx.params.viewName)
const view = await deleteView(viewName)
if (!view || !view.meta) {
ctx.throw(400, "Unable to delete view - no metadata/view not found.")
}
const table = await sdk.tables.getTable(view.meta.tableId)
delete table.views![viewName]
await db.put(table)
await events.view.deleted(view)
await events.view.deleted(view as View)
ctx.body = view
builderSocket?.emitTableUpdate(ctx, table)
@ -147,7 +153,7 @@ export async function exportView(ctx: Ctx) {
)
}
if (view) {
if (view && view.meta) {
ctx.params.viewName = viewName
// Fetch view rows
ctx.query = {

72
packages/server/src/api/routes/public/index.ts
View File

@ -27,51 +27,59 @@ interface KoaRateLimitOptions {
}
const PREFIX = "/api/public/v1"
// allow a lot more requests when in test
const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10
function getApiLimitPerSecond(): number {
if (!env.API_REQ_LIMIT_PER_SEC) {
return DEFAULT_API_REQ_LIMIT_PER_SEC
}
return parseInt(env.API_REQ_LIMIT_PER_SEC)
}
// type can't be known - untyped libraries
let limiter: any, rateLimitStore: any
if (!env.DISABLE_RATE_LIMITING) {
// allow a lot more requests when in test
const DEFAULT_API_REQ_LIMIT_PER_SEC = env.isTest() ? 100 : 10
let rateLimitStore: any = null
if (!env.isTest()) {
const { password, host, port } = redis.utils.getRedisConnectionDetails()
let options: KoaRateLimitOptions = {
socket: {
host: host,
port: port,
},
function getApiLimitPerSecond(): number {
if (!env.API_REQ_LIMIT_PER_SEC) {
return DEFAULT_API_REQ_LIMIT_PER_SEC
}
return parseInt(env.API_REQ_LIMIT_PER_SEC)
}
if (password) {
options.password = password
}
if (!env.isTest()) {
const { password, host, port } = redis.utils.getRedisConnectionDetails()
let options: KoaRateLimitOptions = {
socket: {
host: host,
port: port,
},
}
if (!env.REDIS_CLUSTERED) {
// Can't set direct redis db in clustered env
options.database = SelectableDatabase.RATE_LIMITING
if (password) {
options.password = password
}
if (!env.REDIS_CLUSTERED) {
// Can't set direct redis db in clustered env
options.database = SelectableDatabase.RATE_LIMITING
}
rateLimitStore = new Stores.Redis(options)
RateLimit.defaultOptions({
store: rateLimitStore,
})
}
rateLimitStore = new Stores.Redis(options)
RateLimit.defaultOptions({
store: rateLimitStore,
// rate limiting, allows for 2 requests per second
limiter = RateLimit.middleware({
interval: { sec: 1 },
// per ip, per interval
max: getApiLimitPerSecond(),
})
} else {
console.log("**** PUBLIC API RATE LIMITING DISABLED ****")
}
// rate limiting, allows for 2 requests per second
const limiter = RateLimit.middleware({
interval: { sec: 1 },
// per ip, per interval
max: getApiLimitPerSecond(),
})
const publicRouter = new Router({
prefix: PREFIX,
})
publicRouter.use(limiter)
if (limiter) {
publicRouter.use(limiter)
}
function addMiddleware(
endpoints: any,

63
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -10,6 +10,7 @@ import {
FieldSchema,
FieldType,
FieldTypeSubtypes,
FormulaTypes,
INTERNAL_TABLE_SOURCE_ID,
MonthlyQuotaName,
PermissionLevel,
@ -32,6 +33,7 @@ import {
structures,
} from "@budibase/backend-core/tests"
import _ from "lodash"
import * as uuid from "uuid"
const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
tk.freeze(timestamp)
@ -68,7 +70,7 @@ describe.each([
const generateTableConfig: () => SaveTableRequest = () => {
return {
name: generator.word(),
name: uuid.v4(),
type: "table",
primary: ["id"],
primaryDisplay: "name",
@ -481,7 +483,7 @@ describe.each([
})
const createViewResponse = await config.createView({
name: generator.word(),
name: uuid.v4(),
schema: {
Country: {
visible: true,
@ -816,7 +818,8 @@ describe.each([
RelationshipType.ONE_TO_MANY,
["link"],
{
name: generator.word(),
// Making sure that the combined table name + column name is within postgres limits
name: uuid.v4().replace(/-/g, "").substring(0, 16),
type: "table",
primary: ["id"],
primaryDisplay: "id",
@ -949,7 +952,7 @@ describe.each([
describe("view 2.0", () => {
async function userTable(): Promise<Table> {
return {
name: `users_${generator.word()}`,
name: `users_${uuid.v4()}`,
sourceId: INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
type: "table",
@ -1133,7 +1136,7 @@ describe.each([
const viewSchema = { age: { visible: true }, name: { visible: true } }
async function userTable(): Promise<Table> {
return {
name: `users_${generator.word()}`,
name: `users_${uuid.v4()}`,
sourceId: INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
type: "table",
@ -1630,7 +1633,7 @@ describe.each([
}),
(tableId: string) =>
config.api.row.save(tableId, {
name: generator.word(),
name: uuid.v4(),
description: generator.paragraph(),
tableId,
}),
@ -1998,4 +2001,52 @@ describe.each([
})
})
})
describe("Formula fields", () => {
let relationshipTable: Table, tableId: string, relatedRow: Row
beforeAll(async () => {
const otherTableId = config.table!._id!
const cfg = generateTableConfig()
relationshipTable = await config.createLinkedTable(
RelationshipType.ONE_TO_MANY,
["links"],
{
...cfg,
// needs to be a short name
name: "b",
schema: {
...cfg.schema,
formula: {
name: "formula",
type: FieldType.FORMULA,
formula: "{{ links.0.name }}",
formulaType: FormulaTypes.DYNAMIC,
},
},
}
)
tableId = relationshipTable._id!
relatedRow = await config.api.row.save(otherTableId, {
name: generator.word(),
description: generator.paragraph(),
})
await config.api.row.save(tableId, {
name: generator.word(),
description: generator.paragraph(),
tableId,
links: [relatedRow._id],
})
})
it("should be able to search for rows containing formulas", async () => {
const { rows } = await config.api.row.search(tableId)
expect(rows.length).toBe(1)
expect(rows[0].links.length).toBe(1)
const row = rows[0]
expect(row.formula).toBe(relatedRow.name)
})
})
})

208
packages/server/src/api/routes/tests/user.spec.js
View File

@ -1,208 +0,0 @@
const { roles, utils } = require("@budibase/backend-core")
const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
const setup = require("./utilities")
const { BUILTIN_ROLE_IDS } = roles
jest.setTimeout(30000)
jest.mock("../../../utilities/workerRequests", () => ({
getGlobalUsers: jest.fn(() => {
return {}
}),
getGlobalSelf: jest.fn(() => {
return {}
}),
deleteGlobalUser: jest.fn(),
}))
describe("/users", () => {
let request = setup.getRequest()
let config = setup.getConfig()
afterAll(setup.afterAll)
beforeAll(async () => {
await config.init()
})
describe("fetch", () => {
it("returns a list of users from an instance db", async () => {
await config.createUser({ id: "uuidx" })
await config.createUser({ id: "uuidy" })
const res = await request
.get(`/api/users/metadata`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body.length).toBe(3)
expect(res.body.find(u => u._id === `ro_ta_users_us_uuidx`)).toBeDefined()
expect(res.body.find(u => u._id === `ro_ta_users_us_uuidy`)).toBeDefined()
})
it("should apply authorization to endpoint", async () => {
await config.createUser()
await checkPermissionsEndpoint({
config,
request,
method: "GET",
url: `/api/users/metadata`,
passRole: BUILTIN_ROLE_IDS.ADMIN,
failRole: BUILTIN_ROLE_IDS.PUBLIC,
})
})
})
describe("update", () => {
it("should be able to update the user", async () => {
const user = await config.createUser({ id: `us_update${utils.newid()}` })
user.roleId = BUILTIN_ROLE_IDS.BASIC
delete user._rev
const res = await request
.put(`/api/users/metadata`)
.set(config.defaultHeaders())
.send(user)
.expect(200)
.expect("Content-Type", /json/)
expect(res.body.ok).toEqual(true)
})
it("should be able to update the user multiple times", async () => {
const user = await config.createUser()
delete user._rev
const res1 = await request
.put(`/api/users/metadata`)
.set(config.defaultHeaders())
.send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
.expect(200)
.expect("Content-Type", /json/)
const res = await request
.put(`/api/users/metadata`)
.set(config.defaultHeaders())
.send({ ...user, _rev: res1.body.rev, roleId: BUILTIN_ROLE_IDS.POWER })
.expect(200)
.expect("Content-Type", /json/)
expect(res.body.ok).toEqual(true)
})
it("should require the _rev field for multiple updates", async () => {
const user = await config.createUser()
delete user._rev
await request
.put(`/api/users/metadata`)
.set(config.defaultHeaders())
.send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
.expect(200)
.expect("Content-Type", /json/)
await request
.put(`/api/users/metadata`)
.set(config.defaultHeaders())
.send({ ...user, roleId: BUILTIN_ROLE_IDS.POWER })
.expect(409)
.expect("Content-Type", /json/)
})
})
describe("destroy", () => {
it("should be able to delete the user", async () => {
const user = await config.createUser()
const res = await request
.delete(`/api/users/metadata/${user._id}`)
.set(config.defaultHeaders())
.expect(200)
.expect("Content-Type", /json/)
expect(res.body.message).toBeDefined()
})
})
describe("find", () => {
it("should be able to find the user", async () => {
const user = await config.createUser()
const res = await request
.get(`/api/users/metadata/${user._id}`)
.set(config.defaultHeaders())
.expect(200)
.expect("Content-Type", /json/)
expect(res.body._id).toEqual(user._id)
expect(res.body.roleId).toEqual(BUILTIN_ROLE_IDS.ADMIN)
expect(res.body.tableId).toBeDefined()
})
})
describe("setFlag", () => {
it("should throw an error if a flag is not provided", async () => {
await config.createUser()
const res = await request
.post(`/api/users/flags`)
.set(config.defaultHeaders())
.send({ value: "test" })
.expect(400)
.expect("Content-Type", /json/)
expect(res.body.message).toEqual(
"Must supply a 'flag' field in request body."
)
})
it("should be able to set a flag on the user", async () => {
await config.createUser()
const res = await request
.post(`/api/users/flags`)
.set(config.defaultHeaders())
.send({ value: "test", flag: "test" })
.expect(200)
.expect("Content-Type", /json/)
expect(res.body.message).toEqual("Flag set successfully")
})
})
describe("getFlags", () => {
it("should get flags for a specific user", async () => {
let flagData = { value: "test", flag: "test" }
await config.createUser()
await request
.post(`/api/users/flags`)
.set(config.defaultHeaders())
.send(flagData)
.expect(200)
.expect("Content-Type", /json/)
const res = await request
.get(`/api/users/flags`)
.set(config.defaultHeaders())
.expect(200)
.expect("Content-Type", /json/)
expect(res.body[flagData.value]).toEqual(flagData.flag)
})
})
describe("setFlag", () => {
it("should throw an error if a flag is not provided", async () => {
await config.createUser()
const res = await request
.post(`/api/users/flags`)
.set(config.defaultHeaders())
.send({ value: "test" })
.expect(400)
.expect("Content-Type", /json/)
expect(res.body.message).toEqual(
"Must supply a 'flag' field in request body."
)
})
it("should be able to set a flag on the user", async () => {
await config.createUser()
const res = await request
.post(`/api/users/flags`)
.set(config.defaultHeaders())
.send({ value: "test", flag: "test" })
.expect(200)
.expect("Content-Type", /json/)
expect(res.body.message).toEqual("Flag set successfully")
})
})
})

144
packages/server/src/api/routes/tests/user.spec.ts Normal file
View File

@ -0,0 +1,144 @@
import { roles, utils } from "@budibase/backend-core"
import { checkPermissionsEndpoint } from "./utilities/TestFunctions"
import * as setup from "./utilities"
import { UserMetadata } from "@budibase/types"
jest.setTimeout(30000)
jest.mock("../../../utilities/workerRequests", () => ({
getGlobalUsers: jest.fn(() => {
return {}
}),
getGlobalSelf: jest.fn(() => {
return {}
}),
deleteGlobalUser: jest.fn(),
}))
describe("/users", () => {
let request = setup.getRequest()
let config = setup.getConfig()
afterAll(setup.afterAll)
beforeAll(async () => {
await config.init()
})
describe("fetch", () => {
it("returns a list of users from an instance db", async () => {
await config.createUser({ id: "uuidx" })
await config.createUser({ id: "uuidy" })
const res = await config.api.user.fetch()
expect(res.length).toBe(3)
const ids = res.map(u => u._id)
expect(ids).toContain(`ro_ta_users_us_uuidx`)
expect(ids).toContain(`ro_ta_users_us_uuidy`)
})
it("should apply authorization to endpoint", async () => {
await config.createUser()
await checkPermissionsEndpoint({
config,
request,
method: "GET",
url: `/api/users/metadata`,
passRole: roles.BUILTIN_ROLE_IDS.ADMIN,
failRole: roles.BUILTIN_ROLE_IDS.PUBLIC,
})
})
})
describe("update", () => {
it("should be able to update the user", async () => {
const user: UserMetadata = await config.createUser({
id: `us_update${utils.newid()}`,
})
user.roleId = roles.BUILTIN_ROLE_IDS.BASIC
delete user._rev
const res = await config.api.user.update(user)
expect(res.ok).toEqual(true)
})
it("should be able to update the user multiple times", async () => {
const user = await config.createUser()
delete user._rev
const res1 = await config.api.user.update({
...user,
roleId: roles.BUILTIN_ROLE_IDS.BASIC,
})
const res2 = await config.api.user.update({
...user,
_rev: res1.rev,
roleId: roles.BUILTIN_ROLE_IDS.POWER,
})
expect(res2.ok).toEqual(true)
})
it("should require the _rev field for multiple updates", async () => {
const user = await config.createUser()
delete user._rev
await config.api.user.update({
...user,
roleId: roles.BUILTIN_ROLE_IDS.BASIC,
})
await config.api.user.update(
{ ...user, roleId: roles.BUILTIN_ROLE_IDS.POWER },
{ expectStatus: 409 }
)
})
})
describe("destroy", () => {
it("should be able to delete the user", async () => {
const user = await config.createUser()
const res = await config.api.user.destroy(user._id!)
expect(res.message).toBeDefined()
})
})
describe("find", () => {
it("should be able to find the user", async () => {
const user = await config.createUser()
const res = await config.api.user.find(user._id!)
expect(res._id).toEqual(user._id)
expect(res.roleId).toEqual(roles.BUILTIN_ROLE_IDS.ADMIN)
expect(res.tableId).toBeDefined()
})
})
describe("setFlag", () => {
it("should throw an error if a flag is not provided", async () => {
await config.createUser()
const res = await request
.post(`/api/users/flags`)
.set(config.defaultHeaders())
.send({ value: "test" })
.expect(400)
.expect("Content-Type", /json/)
expect(res.body.message).toEqual(
"Must supply a 'flag' field in request body."
)
})
it("should be able to set a flag on the user", async () => {
await config.createUser()
const res = await config.api.user.setFlag("test", true)
expect(res.message).toEqual("Flag set successfully")
})
})
describe("getFlags", () => {
it("should get flags for a specific user", async () => {
await config.createUser()
await config.api.user.setFlag("test", "test")
const res = await config.api.user.getFlags()
expect(res.test).toEqual("test")
})
})
})

10
packages/server/src/automations/triggers.ts
View File

@ -20,10 +20,10 @@ const JOB_OPTS = {
async function getAllAutomations() {
const db = context.getAppDB()
let automations = await db.allDocs(
let automations = await db.allDocs<Automation>(
getAutomationParams(null, { include_docs: true })
)
return automations.rows.map(row => row.doc)
return automations.rows.map(row => row.doc!)
}
async function queueRelevantRowAutomations(
@ -45,19 +45,19 @@ async function queueRelevantRowAutomations(
for (let automation of automations) {
let automationDef = automation.definition
let automationTrigger = automationDef ? automationDef.trigger : {}
let automationTrigger = automationDef?.trigger
// don't queue events which are for dev apps, only way to test automations is
// running tests on them, in production the test flag will never
// be checked due to lazy evaluation (first always false)
if (
!env.ALLOW_DEV_AUTOMATIONS &&
isDevAppID(event.appId) &&
!(await checkTestFlag(automation._id))
!(await checkTestFlag(automation._id!))
) {
continue
}
if (
automationTrigger.inputs &&
automationTrigger?.inputs &&
automationTrigger.inputs.tableId === event.row.tableId
) {
await automationQueue.add({ automation, event }, JOB_OPTS)

4
packages/server/src/db/inMemoryView.ts
View File

@ -1,5 +1,5 @@
import newid from "./newid"
import { Row, View, Document } from "@budibase/types"
import { Row, Document, DBView } from "@budibase/types"
// bypass the main application db config
// use in memory pouchdb directly
@ -7,7 +7,7 @@ import { db as dbCore } from "@budibase/backend-core"
const Pouch = dbCore.getPouch({ inMemory: true })
export async function runView(
view: View,
view: DBView,
calculation: string,
group: boolean,
data: Row[]

24
packages/server/src/db/linkedRows/index.ts
View File

@ -2,7 +2,6 @@ import LinkController from "./LinkController"
import {
IncludeDocs,
getLinkDocuments,
createLinkView,
getUniqueByProp,
getRelatedTableForField,
getLinkedTableIDs,
@ -14,7 +13,14 @@ import partition from "lodash/partition"
import { getGlobalUsersFromMetadata } from "../../utilities/global"
import { processFormulas } from "../../utilities/rowProcessor"
import { context } from "@budibase/backend-core"
import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types"
import {
Table,
Row,
LinkDocumentValue,
FieldType,
LinkDocument,
ContextUser,
} from "@budibase/types"
import sdk from "../../sdk"
export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils"
@ -73,18 +79,18 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) {
const db = context.getAppDB()
const linkedRowIds = links.map(link => link.id)
const uniqueRowIds = [...new Set(linkedRowIds)]
let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map(
row => row.doc
let dbRows = (await db.allDocs<Row>(getMultiIDParams(uniqueRowIds))).rows.map(
row => row.doc!
)
// convert the unique db rows back to a full list of linked rows
const linked = linkedRowIds
.map(id => dbRows.find(row => row && row._id === id))
.filter(row => row != null)
.filter(row => row != null) as Row[]
// need to handle users as specific cases
let [users, other] = partition(linked, linkRow =>
linkRow._id.startsWith(USER_METDATA_PREFIX)
linkRow._id!.startsWith(USER_METDATA_PREFIX)
)
users = await getGlobalUsersFromMetadata(users)
users = await getGlobalUsersFromMetadata(users as ContextUser[])
return [...other, ...users]
}
@ -176,7 +182,7 @@ export async function attachFullLinkedDocs(
// clear any existing links that could be dupe'd
rows = clearRelationshipFields(table, rows)
// now get the docs and combine into the rows
let linked = []
let linked: Row[] = []
if (linksWithoutFromRow.length > 0) {
linked = await getFullLinkedDocs(linksWithoutFromRow)
}
@ -189,7 +195,7 @@ export async function attachFullLinkedDocs(
if (opts?.fromRow && opts?.fromRow?._id === link.id) {
linkedRow = opts.fromRow!
} else {
linkedRow = linked.find(row => row._id === link.id)
linkedRow = linked.find(row => row._id === link.id)!
}
if (linkedRow) {
const linkedTableId =

4
packages/server/src/db/linkedRows/linkUtils.ts
View File

@ -8,6 +8,7 @@ import {
LinkDocumentValue,
Table,
} from "@budibase/types"
import sdk from "../../sdk"
export { createLinkView } from "../views/staticViews"
@ -110,12 +111,11 @@ export function getLinkedTableIDs(table: Table): string[] {
}
export async function getLinkedTable(id: string, tables: Table[]) {
const db = context.getAppDB()
let linkedTable = tables.find(table => table._id === id)
if (linkedTable) {
return linkedTable
}
linkedTable = await db.get(id)
linkedTable = await sdk.tables.getTable(id)
if (linkedTable) {
tables.push(linkedTable)
}

1
packages/server/src/environment.ts
View File

@ -61,6 +61,7 @@ const environment = {
ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
DISABLE_THREADING: process.env.DISABLE_THREADING,
DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS,
DISABLE_RATE_LIMITING: process.env.DISABLE_RATE_LIMITING,
MULTI_TENANCY: process.env.MULTI_TENANCY,
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
SELF_HOSTED: process.env.SELF_HOSTED,

2
packages/server/src/integration-test/postgres.spec.ts
View File

@ -923,7 +923,6 @@ describe("postgres integrations", () => {
[m2mFieldName]: [
{
_id: row._id,
primaryDisplay: "Invalid display column",
},
],
})
@ -932,7 +931,6 @@ describe("postgres integrations", () => {
[m2mFieldName]: [
{
_id: row._id,
primaryDisplay: "Invalid display column",
},
],
})

2
packages/server/src/sdk/app/applications/import.ts
View File

@ -91,7 +91,7 @@ async function getImportableDocuments(db: Database) {
// map the responses to the document itself
let documents: Document[] = []
for (let response of await Promise.all(docPromises)) {
documents = documents.concat(response.rows.map(row => row.doc))
documents = documents.concat(response.rows.map(row => row.doc!))
}
// remove the _rev, stops it being written
documents.forEach(doc => {

10
packages/server/src/sdk/app/applications/sync.ts
View File

@ -3,7 +3,11 @@ import { db as dbCore, context, logging, roles } from "@budibase/backend-core"
import { User, ContextUser, UserGroup } from "@budibase/types"
import { sdk as proSdk } from "@budibase/pro"
import sdk from "../../"
import { getGlobalUsers, processUser } from "../../../utilities/global"
import {
getGlobalUsers,
getRawGlobalUsers,
processUser,
} from "../../../utilities/global"
import { generateUserMetadataID, InternalTables } from "../../../db/utils"
type DeletedUser = { _id: string; deleted: boolean }
@ -77,9 +81,7 @@ async function syncUsersToApp(
export async function syncUsersToAllApps(userIds: string[]) {
// list of users, if one has been deleted it will be undefined in array
const users = (await getGlobalUsers(userIds, {
noProcessing: true,
})) as User[]
const users = await getRawGlobalUsers(userIds)
const groups = await proSdk.groups.fetch()
const finalUsers: (User | DeletedUser)[] = []
for (let userId of userIds) {

6
packages/server/src/sdk/app/datasources/datasources.ts
View File

@ -51,12 +51,12 @@ export async function fetch(opts?: {
// Get external datasources
const datasources = (
await db.allDocs(
await db.allDocs<Datasource>(
getDatasourceParams(null, {
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map(row => row.doc!)
const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
bbInternalDb,
@ -271,5 +271,5 @@ export async function getExternalDatasources(): Promise<Datasource[]> {
})
)
return externalDatasources.rows.map(r => r.doc)
return externalDatasources.rows.map(r => r.doc!)
}

10
packages/server/src/sdk/app/rows/search.ts
View File

@ -1,4 +1,10 @@
import { Row, SearchFilters, SearchParams } from "@budibase/types"
import {
Row,
SearchFilters,
SearchParams,
SortOrder,
SortType,
} from "@budibase/types"
import { isExternalTableID } from "../../../integrations/utils"
import * as internal from "./search/internal"
import * as external from "./search/external"
@ -32,6 +38,8 @@ export interface ExportRowsParams {
rowIds?: string[]
columns?: string[]
query?: SearchFilters
sort?: string
sortOrder?: SortOrder
}
export interface ExportRowsResult {

16
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -80,7 +80,10 @@ export async function search(options: SearchParams) {
rows = rows.map((r: any) => pick(r, fields))
}
rows = await outputProcessing(table, rows, { preserveLinks: true })
rows = await outputProcessing(table, rows, {
preserveLinks: true,
squash: true,
})
// need wrapper object for bookmarks etc when paginating
return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
@ -98,12 +101,12 @@ export async function search(options: SearchParams) {
export async function exportRows(
options: ExportRowsParams
): Promise<ExportRowsResult> {
const { tableId, format, columns, rowIds } = options
const { tableId, format, columns, rowIds, query, sort, sortOrder } = options
const { datasourceId, tableName } = breakExternalTableId(tableId)
let query: SearchFilters = {}
let requestQuery: SearchFilters = {}
if (rowIds?.length) {
query = {
requestQuery = {
oneOf: {
_id: rowIds.map((row: string) => {
const ids = JSON.parse(
@ -119,6 +122,8 @@ export async function exportRows(
}),
},
}
} else {
requestQuery = query || {}
}
const datasource = await sdk.datasources.get(datasourceId!)
@ -126,7 +131,7 @@ export async function exportRows(
throw new HTTPError("Datasource has not been configured for plus API.", 400)
}
let result = await search({ tableId, query })
let result = await search({ tableId, query: requestQuery, sort, sortOrder })
let rows: Row[] = []
// Filter data to only specified columns if required
@ -183,6 +188,7 @@ export async function fetch(tableId: string): Promise<Row[]> {
const table = await sdk.tables.getTable(tableId)
return await outputProcessing<Row[]>(table, response, {
preserveLinks: true,
squash: true,
})
}

15
packages/server/src/sdk/app/rows/search/internal.ts
View File

@ -84,7 +84,7 @@ export async function search(options: SearchParams) {
export async function exportRows(
options: ExportRowsParams
): Promise<ExportRowsResult> {
const { tableId, format, rowIds, columns, query } = options
const { tableId, format, rowIds, columns, query, sort, sortOrder } = options
const db = context.getAppDB()
const table = await sdk.tables.getTable(tableId)
@ -99,7 +99,12 @@ export async function exportRows(
result = await outputProcessing(table, response)
} else if (query) {
let searchResponse = await search({ tableId, query })
let searchResponse = await search({
tableId,
query,
sort,
sortOrder,
})
result = searchResponse.rows
}
@ -180,8 +185,8 @@ export async function fetchView(
group: !!group,
})
} else {
const tableId = viewInfo.meta.tableId
const data = await fetchRaw(tableId)
const tableId = viewInfo.meta!.tableId
const data = await fetchRaw(tableId!)
response = await inMemoryViews.runView(
viewInfo,
calculation as string,
@ -195,7 +200,7 @@ export async function fetchView(
response.rows = response.rows.map(row => row.doc)
let table: Table
try {
table = await sdk.tables.getTable(viewInfo.meta.tableId)
table = await sdk.tables.getTable(viewInfo.meta!.tableId)
} catch (err) {
throw new Error("Unable to retrieve view table.")
}

4
packages/server/src/sdk/app/tables/getters.ts
View File

@ -48,7 +48,7 @@ export async function getAllInternalTables(db?: Database): Promise<Table[]> {
if (!db) {
db = context.getAppDB()
}
const internalTables = await db.allDocs<Table[]>(
const internalTables = await db.allDocs<Table>(
getTableParams(null, {
include_docs: true,
})
@ -124,7 +124,7 @@ export async function getTables(tableIds: string[]): Promise<Table[]> {
}
if (internalTableIds.length) {
const db = context.getAppDB()
const internalTableDocs = await db.allDocs<Table[]>(
const internalTableDocs = await db.allDocs<Table>(
getMultiIDParams(internalTableIds)
)
tables = tables.concat(internalTableDocs.rows.map(row => row.doc!))

30
packages/server/src/sdk/users/utils.ts
View File

@ -7,12 +7,17 @@ import {
InternalTables,
} from "../../db/utils"
import isEqual from "lodash/isEqual"
import { ContextUser, UserMetadata, User, Database } from "@budibase/types"
import {
ContextUser,
UserMetadata,
Database,
ContextUserMetadata,
} from "@budibase/types"
export function combineMetadataAndUser(
user: ContextUser,
metadata: UserMetadata | UserMetadata[]
) {
): ContextUserMetadata | null {
const metadataId = generateUserMetadataID(user._id!)
const found = Array.isArray(metadata)
? metadata.find(doc => doc._id === metadataId)
@ -51,33 +56,33 @@ export function combineMetadataAndUser(
return null
}
export async function rawUserMetadata(db?: Database) {
export async function rawUserMetadata(db?: Database): Promise<UserMetadata[]> {
if (!db) {
db = context.getAppDB()
}
return (
await db.allDocs(
await db.allDocs<UserMetadata>(
getUserMetadataParams(null, {
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map(row => row.doc!)
}
export async function fetchMetadata() {
export async function fetchMetadata(): Promise<ContextUserMetadata[]> {
const global = await getGlobalUsers()
const metadata = await rawUserMetadata()
const users = []
const users: ContextUserMetadata[] = []
for (let user of global) {
// find the metadata that matches up to the global ID
const info = metadata.find(meta => meta._id.includes(user._id))
const info = metadata.find(meta => meta._id!.includes(user._id!))
// remove these props, not for the correct DB
users.push({
...user,
...info,
tableId: InternalTables.USER_METADATA,
// make sure the ID is always a local ID, not a global one
_id: generateUserMetadataID(user._id),
_id: generateUserMetadataID(user._id!),
})
}
return users
@ -90,9 +95,10 @@ export async function syncGlobalUsers() {
if (!(await db.exists())) {
continue
}
const resp = await Promise.all([getGlobalUsers(), rawUserMetadata(db)])
const users = resp[0] as User[]
const metadata = resp[1] as UserMetadata[]
const [users, metadata] = await Promise.all([
getGlobalUsers(),
rawUserMetadata(db),
])
const toWrite = []
for (let user of users) {
const combined = combineMetadataAndUser(user, metadata)

2
packages/server/src/tests/utilities/TestConfiguration.ts
View File

@ -264,7 +264,7 @@ class TestConfiguration {
admin = false,
email = this.defaultUserValues.email,
roles,
}: any = {}) {
}: any = {}): Promise<User> {
const db = tenancy.getTenantDB(this.getTenantId())
let existing
try {

3
packages/server/src/tests/utilities/api/index.ts
View File

@ -9,6 +9,7 @@ import { ScreenAPI } from "./screen"
import { ApplicationAPI } from "./application"
import { BackupAPI } from "./backup"
import { AttachmentAPI } from "./attachment"
import { UserAPI } from "./user"
export default class API {
table: TableAPI
@ -21,6 +22,7 @@ export default class API {
application: ApplicationAPI
backup: BackupAPI
attachment: AttachmentAPI
user: UserAPI
constructor(config: TestConfiguration) {
this.table = new TableAPI(config)
@ -33,5 +35,6 @@ export default class API {
this.application = new ApplicationAPI(config)
this.backup = new BackupAPI(config)
this.attachment = new AttachmentAPI(config)
this.user = new UserAPI(config)
}
}

3
packages/server/src/tests/utilities/api/row.ts
View File

@ -6,6 +6,7 @@ import {
ExportRowsRequest,
BulkImportRequest,
BulkImportResponse,
SearchRowResponse,
} from "@budibase/types"
import TestConfiguration from "../TestConfiguration"
import { TestAPI } from "./base"
@ -154,7 +155,7 @@ export class RowAPI extends TestAPI {
search = async (
sourceId: string,
{ expectStatus } = { expectStatus: 200 }
): Promise<Row[]> => {
): Promise<SearchRowResponse> => {
const request = this.request
.post(`/api/${sourceId}/search`)
.set(this.config.defaultHeaders())

157
packages/server/src/tests/utilities/api/user.ts Normal file
View File

@ -0,0 +1,157 @@
import {
FetchUserMetadataResponse,
FindUserMetadataResponse,
Flags,
UserMetadata,
} from "@budibase/types"
import TestConfiguration from "../TestConfiguration"
import { TestAPI } from "./base"
import { DocumentInsertResponse } from "@budibase/nano"
export class UserAPI extends TestAPI {
constructor(config: TestConfiguration) {
super(config)
}
fetch = async (
{ expectStatus } = { expectStatus: 200 }
): Promise<FetchUserMetadataResponse> => {
const res = await this.request
.get(`/api/users/metadata`)
.set(this.config.defaultHeaders())
.expect("Content-Type", /json/)
if (res.status !== expectStatus) {
throw new Error(
`Expected status ${expectStatus} but got ${
res.status
} with body ${JSON.stringify(res.body)}`
)
}
return res.body
}
find = async (
id: string,
{ expectStatus } = { expectStatus: 200 }
): Promise<FindUserMetadataResponse> => {
const res = await this.request
.get(`/api/users/metadata/${id}`)
.set(this.config.defaultHeaders())
.expect("Content-Type", /json/)
if (res.status !== expectStatus) {
throw new Error(
`Expected status ${expectStatus} but got ${
res.status
} with body ${JSON.stringify(res.body)}`
)
}
return res.body
}
update = async (
user: UserMetadata,
{ expectStatus } = { expectStatus: 200 }
): Promise<DocumentInsertResponse> => {
const res = await this.request
.put(`/api/users/metadata`)
.set(this.config.defaultHeaders())
.send(user)
.expect("Content-Type", /json/)
if (res.status !== expectStatus) {
throw new Error(
`Expected status ${expectStatus} but got ${
res.status
} with body ${JSON.stringify(res.body)}`
)
}
return res.body as DocumentInsertResponse
}
updateSelf = async (
user: UserMetadata,
{ expectStatus } = { expectStatus: 200 }
): Promise<DocumentInsertResponse> => {
const res = await this.request
.post(`/api/users/metadata/self`)
.set(this.config.defaultHeaders())
.send(user)
.expect("Content-Type", /json/)
if (res.status !== expectStatus) {
throw new Error(
`Expected status ${expectStatus} but got ${
res.status
} with body ${JSON.stringify(res.body)}`
)
}
return res.body as DocumentInsertResponse
}
destroy = async (
id: string,
{ expectStatus } = { expectStatus: 200 }
): Promise<{ message: string }> => {
const res = await this.request
.delete(`/api/users/metadata/${id}`)
.set(this.config.defaultHeaders())
.expect("Content-Type", /json/)
if (res.status !== expectStatus) {
throw new Error(
`Expected status ${expectStatus} but got ${
res.status
} with body ${JSON.stringify(res.body)}`
)
}
return res.body as { message: string }
}
setFlag = async (
flag: string,
value: any,
{ expectStatus } = { expectStatus: 200 }
): Promise<{ message: string }> => {
const res = await this.request
.post(`/api/users/flags`)
.set(this.config.defaultHeaders())
.send({ flag, value })
.expect("Content-Type", /json/)
if (res.status !== expectStatus) {
throw new Error(
`Expected status ${expectStatus} but got ${
res.status
} with body ${JSON.stringify(res.body)}`
)
}
return res.body as { message: string }
}
getFlags = async (
{ expectStatus } = { expectStatus: 200 }
): Promise<Flags> => {
const res = await this.request
.get(`/api/users/flags`)
.set(this.config.defaultHeaders())
.expect("Content-Type", /json/)
if (res.status !== expectStatus) {
throw new Error(
`Expected status ${expectStatus} but got ${
res.status
} with body ${JSON.stringify(res.body)}`
)
}
return res.body as Flags
}
}

48
packages/server/src/utilities/global.ts
View File

@ -71,69 +71,67 @@ export async function processUser(
return user
}
export async function getCachedSelf(ctx: UserCtx, appId: string) {
export async function getCachedSelf(
ctx: UserCtx,
appId: string
): Promise<ContextUser> {
// this has to be tenant aware, can't depend on the context to find it out
// running some middlewares before the tenancy causes context to break
const user = await cache.user.getUser(ctx.user?._id!)
return processUser(user, { appId })
}
export async function getRawGlobalUser(userId: string) {
export async function getRawGlobalUser(userId: string): Promise<User> {
const db = tenancy.getGlobalDB()
return db.get<User>(getGlobalIDFromUserMetadataID(userId))
}
export async function getGlobalUser(userId: string) {
export async function getGlobalUser(userId: string): Promise<ContextUser> {
const appId = context.getAppId()
let user = await getRawGlobalUser(userId)
return processUser(user, { appId })
}
export async function getGlobalUsers(
userIds?: string[],
opts?: { noProcessing?: boolean }
) {
const appId = context.getAppId()
export async function getRawGlobalUsers(userIds?: string[]): Promise<User[]> {
const db = tenancy.getGlobalDB()
let globalUsers
let globalUsers: User[]
if (userIds) {
globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map(
row => row.doc
globalUsers = (await db.allDocs<User>(getMultiIDParams(userIds))).rows.map(
row => row.doc!
)
} else {
globalUsers = (
await db.allDocs(
await db.allDocs<User>(
dbCore.getGlobalUserParams(null, {
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map(row => row.doc!)
}
globalUsers = globalUsers
return globalUsers
.filter(user => user != null)
.map(user => {
delete user.password
delete user.forceResetPassword
return user
})
}
if (opts?.noProcessing || !appId) {
return globalUsers
} else {
// pass in the groups, meaning we don't actually need to retrieve them for
// each user individually
const allGroups = await groups.fetch()
return Promise.all(
globalUsers.map(user => processUser(user, { groups: allGroups }))
)
}
export async function getGlobalUsers(
userIds?: string[]
): Promise<ContextUser[]> {
const users = await getRawGlobalUsers(userIds)
const allGroups = await groups.fetch()
return Promise.all(
users.map(user => processUser(user, { groups: allGroups }))
)
}
export async function getGlobalUsersFromMetadata(users: ContextUser[]) {
const globalUsers = await getGlobalUsers(users.map(user => user._id!))
return users.map(user => {
const globalUser = globalUsers.find(
globalUser => globalUser && user._id?.includes(globalUser._id)
globalUser => globalUser && user._id?.includes(globalUser._id!)
)
return {
...globalUser,

4
packages/server/src/utilities/routing/index.ts
View File

@ -1,9 +1,9 @@
import { createRoutingView } from "../../db/views/staticViews"
import { ViewName, getQueryIndex, UNICODE_MAX } from "../../db/utils"
import { context } from "@budibase/backend-core"
import { ScreenRouting } from "@budibase/types"
import { ScreenRouting, Document } from "@budibase/types"
type ScreenRoutesView = {
interface ScreenRoutesView extends Document {
id: string
routing: ScreenRouting
}

22
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -2,16 +2,15 @@ import * as linkRows from "../../db/linkedRows"
import { FieldTypes, AutoFieldSubTypes } from "../../constants"
import { processFormulas, fixAutoColumnSubType } from "./utils"
import { ObjectStoreBuckets } from "../../constants"
import { context, db as dbCore, objectStore } from "@budibase/backend-core"
import {
context,
db as dbCore,
objectStore,
utils,
} from "@budibase/backend-core"
import { InternalTables } from "../../db/utils"
import { TYPE_TRANSFORM_MAP } from "./map"
import {
AutoColumnFieldMetadata,
FieldSubtype,
Row,
RowAttachment,
Table,
} from "@budibase/types"
import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types"
import { cloneDeep } from "lodash/fp"
import {
processInputBBReferences,
@ -233,6 +232,11 @@ export async function outputProcessing<T extends Row[] | Row>(
})
: safeRows
// make sure squash is enabled if needed
if (!opts.squash && utils.hasCircularStructure(rows)) {
opts.squash = true
}
// process complex types: attachements, bb references...
for (let [property, column] of Object.entries(table.schema)) {
if (column.type === FieldTypes.ATTACHMENT) {
@ -258,7 +262,7 @@ export async function outputProcessing<T extends Row[] | Row>(
}
// process formulas after the complex types had been processed
enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
enriched = processFormulas(table, enriched, { dynamic: true })
if (opts.squash) {
enriched = (await linkRows.squashLinksToPrimaryDisplay(

76
packages/server/src/utilities/rowProcessor/utils.ts
View File

@ -12,6 +12,11 @@ import {
Table,
} from "@budibase/types"
interface FormulaOpts {
dynamic?: boolean
contextRows?: Row[]
}
/**
* If the subtype has been lost for any reason this works out what
* subtype the auto column should be.
@ -40,52 +45,50 @@ export function fixAutoColumnSubType(
/**
* Looks through the rows provided and finds formulas - which it then processes.
*/
export function processFormulas(
export function processFormulas<T extends Row | Row[]>(
table: Table,
rows: Row[] | Row,
{ dynamic, contextRows }: any = { dynamic: true }
) {
const single = !Array.isArray(rows)
let rowArray: Row[]
if (single) {
rowArray = [rows]
contextRows = contextRows ? [contextRows] : contextRows
} else {
rowArray = rows
}
for (let [column, schema] of Object.entries(table.schema)) {
if (schema.type !== FieldTypes.FORMULA) {
continue
}
inputRows: T,
{ dynamic, contextRows }: FormulaOpts = { dynamic: true }
): T {
const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
if (rows)
for (let [column, schema] of Object.entries(table.schema)) {
if (schema.type !== FieldTypes.FORMULA) {
continue
}
const isStatic = schema.formulaType === FormulaTypes.STATIC
const isStatic = schema.formulaType === FormulaTypes.STATIC
if (
schema.formula == null ||
(dynamic && isStatic) ||
(!dynamic && !isStatic)
) {
continue
}
// iterate through rows and process formula
for (let i = 0; i < rowArray.length; i++) {
let row = rowArray[i]
let context = contextRows ? contextRows[i] : row
rowArray[i] = {
...row,
[column]: processStringSync(schema.formula, context),
if (
schema.formula == null ||
(dynamic && isStatic) ||
(!dynamic && !isStatic)
) {
continue
}
// iterate through rows and process formula
for (let i = 0; i < rows.length; i++) {
let row = rows[i]
let context = contextRows ? contextRows[i] : row
rows[i] = {
...row,
[column]: processStringSync(schema.formula, context),
}
}
}
}
return single ? rowArray[0] : rowArray
return Array.isArray(inputRows) ? rows : rows[0]
}
/**
* Processes any date columns and ensures that those without the ignoreTimezones
* flag set are parsed as UTC rather than local time.
*/
export function processDates(table: Table, rows: Row[]) {
let datesWithTZ = []
export function processDates<T extends Row | Row[]>(
table: Table,
inputRows: T
): T {
let rows = Array.isArray(inputRows) ? inputRows : [inputRows]
let datesWithTZ: string[] = []
for (let [column, schema] of Object.entries(table.schema)) {
if (schema.type !== FieldTypes.DATETIME) {
continue
@ -102,5 +105,6 @@ export function processDates(table: Table, rows: Row[]) {
}
}
}
return rows
return Array.isArray(inputRows) ? rows : rows[0]
}

12
packages/server/src/utilities/users.ts
View File

@ -1,11 +1,13 @@
import { InternalTables } from "../db/utils"
import { getGlobalUser } from "./global"
import { context, roles } from "@budibase/backend-core"
import { UserCtx } from "@budibase/types"
import { ContextUserMetadata, UserCtx, UserMetadata } from "@budibase/types"
export async function getFullUser(ctx: UserCtx, userId: string) {
export async function getFullUser(
userId: string
): Promise<ContextUserMetadata> {
const global = await getGlobalUser(userId)
let metadata: any = {}
let metadata: UserMetadata | undefined = undefined
// always prefer the user metadata _id and _rev
delete global._id
@ -14,11 +16,11 @@ export async function getFullUser(ctx: UserCtx, userId: string) {
try {
// this will throw an error if the db doesn't exist, or there is no appId
const db = context.getAppDB()
metadata = await db.get(userId)
metadata = await db.get<UserMetadata>(userId)
delete metadata.csrfToken
} catch (err) {
// it is fine if there is no user metadata yet
}
delete metadata.csrfToken
return {
...metadata,
...global,

1
packages/types/src/api/web/app/index.ts
View File

@ -6,3 +6,4 @@ export * from "./rows"
export * from "./table"
export * from "./permission"
export * from "./attachment"
export * from "./user"

3
packages/types/src/api/web/app/rows.ts
View File

@ -1,5 +1,6 @@
import { SearchFilters, SearchParams } from "../../../sdk"
import { Row } from "../../../documents"
import { SortOrder } from "../../../api"
import { ReadStream } from "fs"
export interface SaveRowRequest extends Row {}
@ -34,6 +35,8 @@ export interface ExportRowsRequest {
rows: string[]
columns?: string[]
query?: SearchFilters
sort?: string
sortOrder?: SortOrder
}
export type ExportRowsResponse = ReadStream

9
packages/types/src/api/web/app/user.ts Normal file
View File

@ -0,0 +1,9 @@
import { ContextUserMetadata } from "../../../"
export type FetchUserMetadataResponse = ContextUserMetadata[]
export type FindUserMetadataResponse = ContextUserMetadata
export interface SetFlagRequest {
flag: string
value: any
}

5
packages/types/src/documents/account/flag.ts Normal file
View File

@ -0,0 +1,5 @@
import { Document } from "../../"
export interface Flags extends Document {
[key: string]: any
}

1
packages/types/src/documents/account/index.ts
View File

@ -1,2 +1,3 @@
export * from "./account"
export * from "./user"
export * from "./flag"

1
packages/types/src/documents/app/layout.ts
View File

@ -2,4 +2,5 @@ import { Document } from "../document"
export interface Layout extends Document {
props: any
layoutId?: string
}

10
packages/types/src/documents/app/user.ts
View File

@ -1,6 +1,6 @@
import { Document } from "../document"
import { User } from "../global"
import { Row } from "./row"
import { ContextUser } from "../../sdk"
export interface UserMetadata extends Document {
roleId: string
email?: string
}
export type UserMetadata = User & Row
export type ContextUserMetadata = ContextUser & Row

21
packages/types/src/documents/app/view.ts
View File

@ -1,5 +1,24 @@
import { SearchFilter, SortOrder, SortType } from "../../api"
import { UIFieldMetadata } from "./table"
import { Document } from "../document"
import { DBView } from "../../sdk"
export type ViewTemplateOpts = {
field: string
tableId: string
groupBy: string
filters: ViewFilter[]
schema: any
calculation: string
groupByMulti?: boolean
}
export interface InMemoryView extends Document {
view: DBView
name: string
tableId: string
groupBy?: string
}
export interface View {
name?: string
@ -10,7 +29,7 @@ export interface View {
calculation?: ViewCalculation
map?: string
reduce?: any
meta?: Record<string, any>
meta?: ViewTemplateOpts
}
export interface ViewV2 {

8
packages/types/src/documents/pouch.ts
View File

@ -1,17 +1,19 @@
import { Document } from "../"
export interface RowValue {
rev: string
deleted: boolean
}
export interface RowResponse<T> {
export interface RowResponse<T extends Document> {
id: string
key: string
error: string
value: T | RowValue
doc?: T | any
doc?: T
}
export interface AllDocsResponse<T> {
export interface AllDocsResponse<T extends Document> {
offset: number
total_rows: number
rows: RowResponse<T>[]

39
packages/types/src/sdk/db.ts
View File

@ -1,5 +1,5 @@
import Nano from "@budibase/nano"
import { AllDocsResponse, AnyDocument, Document } from "../"
import { AllDocsResponse, AnyDocument, Document, ViewTemplateOpts } from "../"
import { Writable } from "stream"
export enum SearchIndex {
@ -20,6 +20,37 @@ export enum SortOption {
DESCENDING = "desc",
}
export type IndexAnalyzer = {
name: string
default?: string
fields?: Record<string, string>
}
export type DBView = {
name?: string
map: string
reduce?: string
meta?: ViewTemplateOpts
groupBy?: string
}
export interface DesignDocument extends Document {
// we use this static reference for all design documents
_id: "_design/database"
language?: string
// CouchDB views
views?: {
[viewName: string]: DBView
}
// Lucene indexes
indexes?: {
[indexName: string]: {
index: string
analyzer?: string | IndexAnalyzer
}
}
}
export type CouchFindOptions = {
selector: PouchDB.Find.Selector
fields?: string[]
@ -101,8 +132,10 @@ export interface Database {
opts?: DatabasePutOpts
): Promise<Nano.DocumentInsertResponse>
bulkDocs(documents: AnyDocument[]): Promise<Nano.DocumentBulkResponse[]>
allDocs<T>(params: DatabaseQueryOpts): Promise<AllDocsResponse<T>>
query<T>(
allDocs<T extends Document>(
params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>>
query<T extends Document>(
viewName: string,
params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>>

6
packages/worker/src/constants/templates/index.ts
View File

@ -56,12 +56,12 @@ export async function getTemplates({
id,
}: { ownerId?: string; type?: string; id?: string } = {}) {
const db = tenancy.getGlobalDB()
const response = await db.allDocs(
const response = await db.allDocs<Template>(
dbCore.getTemplateParams(ownerId || GLOBAL_OWNER, id, {
include_docs: true,
})
)
let templates = response.rows.map(row => row.doc)
let templates = response.rows.map(row => row.doc!)
// should only be one template with ID
if (id) {
return templates[0]
@ -73,6 +73,6 @@ export async function getTemplates({
}
export async function getTemplateByPurpose(type: string, purpose: string) {
const templates = await getTemplates({ type })
const templates = (await getTemplates({ type })) as Template[]
return templates.find((template: Template) => template.purpose === purpose)
}

11
packages/worker/src/utilities/email.ts
View File

@ -99,8 +99,6 @@ async function buildEmail(
if (!base || !body || !core) {
throw "Unable to build email, missing base components"
}
base = base.contents
body = body.contents
let name = user ? user.name : undefined
if (user && !name && user.firstName) {
@ -114,15 +112,10 @@ async function buildEmail(
user: user || {},
}
// Prepend the core template
const fullBody = core + body
body = await processString(fullBody, context)
// this should now be the core email HTML
return processString(base, {
return processString(base.contents, {
...context,
body,
body: await processString(core + body?.contents, context),
})
}

2
pull_request_template.md
View File

@ -1,7 +1,7 @@
## Description
_Describe the problem or feature in addition to a link to the relevant github issues._
### Addresses:
## Addresses
- `<Enter the Link to the issue(s) this PR addresses>`
- ...more if required

2
qa-core/package.json
View File

@ -20,7 +20,7 @@
"test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.licensing\\.",
"serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci",
"serve": "start-server-and-test dev:built http://localhost:4001/health",
"dev:built": "cd ../ && yarn dev:built"
"dev:built": "cd ../ && DISABLE_RATE_LIMITING=1 yarn dev:built"
},
"devDependencies": {
"@budibase/types": "^2.3.17",