MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into BUDI-8986/validate-datasource-setting-on-components

This commit is contained in:
Adria Navarro 2025-01-22 11:55:13 +01:00 committed by GitHub
parent 4d85006c35 5a023df0e0
commit 79539ab992
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
143 changed files with 1972 additions and 3980 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "3.2.46",
"version": "3.2.47",
"npmClient": "yarn",
"concurrency": 20,
"command": {

1
packages/backend-core/src/index.ts
View File

@ -1,6 +1,5 @@
export * as configs from "./configs"
export * as events from "./events"
export * as migrations from "./migrations"
export * as users from "./users"
export * as userUtils from "./users/utils"
export * as roles from "./security/roles"

40
packages/backend-core/src/migrations/definitions.ts
View File

@ -1,40 +0,0 @@
import {
MigrationType,
MigrationName,
MigrationDefinition,
} from "@budibase/types"
export const DEFINITIONS: MigrationDefinition[] = [
{
type: MigrationType.GLOBAL,
name: MigrationName.USER_EMAIL_VIEW_CASING,
},
{
type: MigrationType.GLOBAL,
name: MigrationName.SYNC_QUOTAS,
},
{
type: MigrationType.APP,
name: MigrationName.APP_URLS,
},
{
type: MigrationType.APP,
name: MigrationName.EVENT_APP_BACKFILL,
},
{
type: MigrationType.APP,
name: MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS,
},
{
type: MigrationType.GLOBAL,
name: MigrationName.EVENT_GLOBAL_BACKFILL,
},
{
type: MigrationType.INSTALLATION,
name: MigrationName.EVENT_INSTALLATION_BACKFILL,
},
{
type: MigrationType.GLOBAL,
name: MigrationName.GLOBAL_INFO_SYNC_USERS,
},
]

2
packages/backend-core/src/migrations/index.ts
View File

@ -1,2 +0,0 @@
export * from "./migrations"
export * from "./definitions"

186
packages/backend-core/src/migrations/migrations.ts
View File

@ -1,186 +0,0 @@
import { DEFAULT_TENANT_ID } from "../constants"
import {
DocumentType,
StaticDatabases,
getAllApps,
getGlobalDBName,
getDB,
} from "../db"
import environment from "../environment"
import * as platform from "../platform"
import * as context from "../context"
import { DEFINITIONS } from "."
import {
Migration,
MigrationOptions,
MigrationType,
MigrationNoOpOptions,
App,
} from "@budibase/types"
export const getMigrationsDoc = async (db: any) => {
// get the migrations doc
try {
return await db.get(DocumentType.MIGRATIONS)
} catch (err: any) {
if (err.status && err.status === 404) {
return { _id: DocumentType.MIGRATIONS }
} else {
throw err
}
}
}
export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => {
// filter migrations to the type and populate a no-op migration
const migrations: Migration[] = DEFINITIONS.filter(
def => def.type === opts.type
).map(d => ({ ...d, fn: async () => {} }))
await runMigrations(migrations, { noOp: opts })
}
export const runMigration = async (
migration: Migration,
options: MigrationOptions = {}
) => {
const migrationType = migration.type
const migrationName = migration.name
const silent = migration.silent
const log = (message: string) => {
if (!silent) {
console.log(message)
}
}
// get the db to store the migration in
let dbNames: string[]
if (migrationType === MigrationType.GLOBAL) {
dbNames = [getGlobalDBName()]
} else if (migrationType === MigrationType.APP) {
if (options.noOp) {
if (!options.noOp.appId) {
throw new Error("appId is required for noOp app migration")
}
dbNames = [options.noOp.appId]
} else {
const apps = (await getAllApps(migration.appOpts)) as App[]
dbNames = apps.map(app => app.appId)
}
} else if (migrationType === MigrationType.INSTALLATION) {
dbNames = [StaticDatabases.PLATFORM_INFO.name]
} else {
throw new Error(`Unrecognised migration type [${migrationType}]`)
}
const length = dbNames.length
let count = 0
// run the migration against each db
for (const dbName of dbNames) {
count++
const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
const db = getDB(dbName)
try {
const doc = await getMigrationsDoc(db)
// the migration has already been run
if (doc[migrationName]) {
// check for force
if (
options.force &&
options.force[migrationType] &&
options.force[migrationType].includes(migrationName)
) {
log(`[Migration: ${migrationName}] [DB: ${dbName}] Forcing`)
} else {
// no force, exit
return
}
}
// check if the migration is not a no-op
if (!options.noOp) {
log(
`[Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
)
if (migration.preventRetry) {
// eagerly set the completion date
// so that we never run this migration twice even upon failure
doc[migrationName] = Date.now()
const response = await db.put(doc)
doc._rev = response.rev
}
// run the migration
if (migrationType === MigrationType.APP) {
await context.doInAppContext(db.name, async () => {
await migration.fn(db)
})
} else {
await migration.fn(db)
}
log(`[Migration: ${migrationName}] [DB: ${dbName}] Complete`)
}
// mark as complete
doc[migrationName] = Date.now()
await db.put(doc)
} catch (err) {
console.error(
`[Migration: ${migrationName}] [DB: ${dbName}] Error: `,
err
)
throw err
}
}
}
export const runMigrations = async (
migrations: Migration[],
options: MigrationOptions = {}
) => {
let tenantIds
if (environment.MULTI_TENANCY) {
if (options.noOp) {
tenantIds = [options.noOp.tenantId]
} else if (!options.tenantIds || !options.tenantIds.length) {
// run for all tenants
tenantIds = await platform.tenants.getTenantIds()
} else {
tenantIds = options.tenantIds
}
} else {
// single tenancy
tenantIds = [DEFAULT_TENANT_ID]
}
if (tenantIds.length > 1) {
console.log(`Checking migrations for ${tenantIds.length} tenants`)
} else {
console.log("Checking migrations")
}
let count = 0
// for all tenants
for (const tenantId of tenantIds) {
count++
if (tenantIds.length > 1) {
console.log(`Progress [${count}/${tenantIds.length}]`)
}
// for all migrations
for (const migration of migrations) {
// run the migration
await context.doInTenant(
tenantId,
async () => await runMigration(migration, options)
)
}
}
console.log("Migrations complete")
}

11
packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap
View File

@ -1,11 +0,0 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`migrations should match snapshot 1`] = `
{
"_id": "migrations",
"_rev": "1-2f64479842a0513aa8b97f356b0b9127",
"createdAt": "2020-01-01T00:00:00.000Z",
"test": 1577836800000,
"updatedAt": "2020-01-01T00:00:00.000Z",
}
`;

64
packages/backend-core/src/migrations/tests/migrations.spec.ts
View File

@ -1,64 +0,0 @@
import { testEnv, DBTestConfiguration } from "../../../tests/extra"
import * as migrations from "../index"
import * as context from "../../context"
import { MigrationType } from "@budibase/types"
testEnv.multiTenant()
describe("migrations", () => {
const config = new DBTestConfiguration()
const migrationFunction = jest.fn()
const MIGRATIONS = [
{
type: MigrationType.GLOBAL,
name: "test" as any,
fn: migrationFunction,
},
]
beforeEach(() => {
config.newTenant()
})
afterEach(async () => {
jest.clearAllMocks()
})
const migrate = () => {
return migrations.runMigrations(MIGRATIONS, {
tenantIds: [config.tenantId],
})
}
it("should run a new migration", async () => {
await config.doInTenant(async () => {
await migrate()
expect(migrationFunction).toHaveBeenCalled()
const db = context.getGlobalDB()
const doc = await migrations.getMigrationsDoc(db)
expect(doc.test).toBeDefined()
})
})
it("should match snapshot", async () => {
await config.doInTenant(async () => {
await migrate()
const doc = await migrations.getMigrationsDoc(context.getGlobalDB())
expect(doc).toMatchSnapshot()
})
})
it("should skip a previously run migration", async () => {
await config.doInTenant(async () => {
const db = context.getGlobalDB()
await migrate()
const previousDoc = await migrations.getMigrationsDoc(db)
await migrate()
const currentDoc = await migrations.getMigrationsDoc(db)
expect(migrationFunction).toHaveBeenCalledTimes(1)
expect(currentDoc.test).toBe(previousDoc.test)
})
})
})

5
packages/bbui/src/bbui.css
View File

@ -45,6 +45,11 @@
--purple: #806fde;
--purple-dark: #130080;
--error-bg: rgba(226, 109, 105, 0.3);
--warning-bg: rgba(255, 210, 106, 0.3);
--error-content: rgba(226, 109, 105, 0.6);
--warning-content: rgba(255, 210, 106, 0.6);
--rounded-small: 4px;
--rounded-medium: 8px;
--rounded-large: 16px;

9
packages/builder/src/components/common/bindings/BindingPanel.svelte
View File

@ -12,7 +12,7 @@
decodeJSBinding,
encodeJSBinding,
processObjectSync,
processStringSync,
processStringWithLogsSync,
} from "@budibase/string-templates"
import { readableToRuntimeBinding } from "@/dataBinding"
import CodeEditor from "../CodeEditor/CodeEditor.svelte"
@ -41,6 +41,7 @@
InsertAtPositionFn,
JSONValue,
} from "@budibase/types"
import type { Log } from "@budibase/string-templates"
import type { CompletionContext } from "@codemirror/autocomplete"
const dispatch = createEventDispatcher()
@ -66,6 +67,7 @@
let insertAtPos: InsertAtPositionFn | undefined
let targetMode: BindingMode | null = null
let expressionResult: string | undefined
let expressionLogs: Log[] | undefined
let expressionError: string | undefined
let evaluating = false
@ -157,7 +159,7 @@
(expression: string | null, context: any, snippets: Snippet[]) => {
try {
expressionError = undefined
expressionResult = processStringSync(
const output = processStringWithLogsSync(
expression || "",
{
...context,
@ -167,6 +169,8 @@
noThrow: false,
}
)
expressionResult = output.result
expressionLogs = output.logs
} catch (err: any) {
expressionResult = undefined
expressionError = err
@ -421,6 +425,7 @@
<EvaluationSidePanel
{expressionResult}
{expressionError}
{expressionLogs}
{evaluating}
expression={editorValue ? editorValue : ""}
/>

70
packages/builder/src/components/common/bindings/EvaluationSidePanel.svelte
View File

@ -4,11 +4,13 @@
import { Helpers } from "@budibase/bbui"
import { fade } from "svelte/transition"
import { UserScriptError } from "@budibase/string-templates"
import type { Log } from "@budibase/string-templates"
import type { JSONValue } from "@budibase/types"
// this can be essentially any primitive response from the JS function
export let expressionResult: JSONValue | undefined = undefined
export let expressionError: string | undefined = undefined
export let expressionLogs: Log[] = []
export let evaluating = false
export let expression: string | null = null
@ -16,6 +18,11 @@
$: empty = expression == null || expression?.trim() === ""
$: success = !error && !empty
$: highlightedResult = highlight(expressionResult)
$: highlightedLogs = expressionLogs.map(l => ({
log: highlight(l.log.join(", ")),
line: l.line,
type: l.type,
}))
const formatError = (err: any) => {
if (err.code === UserScriptError.code) {
@ -25,14 +32,14 @@
}
// json can be any primitive type
const highlight = (json?: any | null) => {
const highlight = (json?: JSONValue | null) => {
if (json == null) {
return ""
}
// Attempt to parse and then stringify, in case this is valid result
try {
json = JSON.stringify(JSON.parse(json), null, 2)
json = JSON.stringify(JSON.parse(json as any), null, 2)
} catch (err) {
// couldn't parse/stringify, just treat it as the raw input
}
@ -61,7 +68,7 @@
<div class="header" class:success class:error>
<div class="header-content">
{#if error}
<Icon name="Alert" color="var(--spectrum-global-color-red-600)" />
<Icon name="Alert" color="var(--error-content)" />
<div>Error</div>
{#if evaluating}
<div transition:fade|local={{ duration: 130 }}>
@ -90,8 +97,36 @@
{:else if error}
{formatError(expressionError)}
{:else}
<!-- eslint-disable-next-line svelte/no-at-html-tags-->
{@html highlightedResult}
<div class="output-lines">
{#each highlightedLogs as logLine}
<div
class="line"
class:error-log={logLine.type === "error"}
class:warn-log={logLine.type === "warn"}
>
<div class="icon-log">
{#if logLine.type === "error"}
<Icon
size="XS"
name="CloseCircle"
color="var(--error-content)"
/>
{:else if logLine.type === "warn"}
<Icon size="XS" name="Alert" color="var(--warning-content)" />
{/if}
<!-- eslint-disable-next-line svelte/no-at-html-tags-->
<span>{@html logLine.log}</span>
</div>
{#if logLine.line}
<span style="color: var(--blue)">:{logLine.line}</span>
{/if}
</div>
{/each}
<div class="line">
<!-- eslint-disable-next-line svelte/no-at-html-tags-->
{@html highlightedResult}
</div>
</div>
{/if}
</div>
</div>
@ -130,20 +165,37 @@
height: 100%;
z-index: 1;
position: absolute;
opacity: 10%;
}
.header.error::before {
background: var(--spectrum-global-color-red-400);
background: var(--error-bg);
}
.body {
flex: 1 1 auto;
padding: var(--spacing-m) var(--spacing-l);
font-family: var(--font-mono);
font-size: 12px;
overflow-y: scroll;
overflow-y: auto;
overflow-x: hidden;
white-space: pre-wrap;
white-space: pre-line;
word-wrap: break-word;
height: 0;
}
.output-lines {
display: flex;
flex-direction: column;
gap: var(--spacing-xs);
}
.line {
border-bottom: var(--border-light);
display: flex;
flex-direction: row;
justify-content: space-between;
align-items: end;
padding: var(--spacing-s);
}
.icon-log {
display: flex;
gap: var(--spacing-s);
align-items: start;
}
</style>

9
packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte
View File

@ -1,4 +1,5 @@
<script>
import { datasources } from "@/stores/builder"
import { Divider, Heading } from "@budibase/bbui"
export let dividerState
@ -6,6 +7,8 @@
export let dataSet
export let value
export let onSelect
$: displayDatasourceName = $datasources.list.length > 1
</script>
{#if dividerState}
@ -21,7 +24,7 @@
{#each dataSet as data}
<li
class="spectrum-Menu-item"
class:is-selected={value?.label === data.label &&
class:is-selected={value?.resourceId === data.resourceId &&
value?.type === data.type}
role="option"
aria-selected="true"
@ -29,7 +32,9 @@
on:click={() => onSelect(data)}
>
<span class="spectrum-Menu-itemLabel">
{data.datasourceName ? `${data.datasourceName} - ` : ""}{data.label}
{data.datasourceName && displayDatasourceName
? `${data.datasourceName} - `
: ""}{data.label}
</span>
<svg
class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"

18
packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte
View File

@ -34,7 +34,7 @@
import ClientBindingPanel from "@/components/common/bindings/ClientBindingPanel.svelte"
import DataSourceCategory from "@/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte"
import { API } from "@/api"
import { datasourceSelect as format } from "@/helpers/data/format"
import { sortAndFormat } from "@/helpers/data/format"
export let value = {}
export let otherSources
@ -51,25 +51,13 @@
let modal
$: text = value?.label ?? "Choose an option"
$: tables = $tablesStore.list
.map(table => format.table(table, $datasources.list))
.sort((a, b) => {
// sort tables alphabetically, grouped by datasource
const dsA = a.datasourceName ?? ""
const dsB = b.datasourceName ?? ""
const dsComparison = dsA.localeCompare(dsB)
if (dsComparison !== 0) {
return dsComparison
}
return a.label.localeCompare(b.label)
})
$: tables = sortAndFormat.tables($tablesStore.list, $datasources.list)
$: viewsV1 = $viewsStore.list.map(view => ({
...view,
label: view.name,
type: "view",
}))
$: viewsV2 = $viewsV2Store.list.map(format.viewV2)
$: viewsV2 = sortAndFormat.viewsV2($viewsV2Store.list, $datasources.list)
$: views = [...(viewsV1 || []), ...(viewsV2 || [])]
$: queries = $queriesStore.list
.filter(q => showAllQueries || q.queryVerb === "read" || q.readable)

73
packages/builder/src/components/design/settings/controls/TableSelect.svelte
View File

@ -1,22 +1,32 @@
<script>
import { Select } from "@budibase/bbui"
import { Popover, Select } from "@budibase/bbui"
import { createEventDispatcher, onMount } from "svelte"
import { tables as tablesStore, viewsV2 } from "@/stores/builder"
import { tableSelect as format } from "@/helpers/data/format"
import {
tables as tableStore,
datasources as datasourceStore,
viewsV2 as viewsV2Store,
} from "@/stores/builder"
import DataSourceCategory from "./DataSourceSelect/DataSourceCategory.svelte"
import { sortAndFormat } from "@/helpers/data/format"
export let value
let anchorRight, dropdownRight
const dispatch = createEventDispatcher()
$: tables = $tablesStore.list.map(format.table)
$: views = $viewsV2.list.map(format.viewV2)
$: tables = sortAndFormat.tables($tableStore.list, $datasourceStore.list)
$: views = sortAndFormat.viewsV2($viewsV2Store.list, $datasourceStore.list)
$: options = [...(tables || []), ...(views || [])]
$: text = value?.label ?? "Choose an option"
const onChange = e => {
dispatch(
"change",
options.find(x => x.resourceId === e.detail)
options.find(x => x.resourceId === e.resourceId)
)
dropdownRight.hide()
}
onMount(() => {
@ -29,10 +39,47 @@
})
</script>
<Select
on:change={onChange}
value={value?.resourceId}
{options}
getOptionValue={x => x.resourceId}
getOptionLabel={x => x.label}
/>
<div class="container" bind:this={anchorRight}>
<Select
readonly
value={text}
options={[text]}
on:click={dropdownRight.show}
/>
</div>
<Popover bind:this={dropdownRight} anchor={anchorRight}>
<div class="dropdown">
<DataSourceCategory
heading="Tables"
dataSet={tables}
{value}
onSelect={onChange}
/>
{#if views?.length}
<DataSourceCategory
dividerState={true}
heading="Views"
dataSet={views}
{value}
onSelect={onChange}
/>
{/if}
</div>
</Popover>
<style>
.container {
display: flex;
flex-direction: row;
justify-content: flex-start;
align-items: center;
}
.container :global(:first-child) {
flex: 1 1 auto;
}
.dropdown {
padding: var(--spacing-m) 0;
z-index: 99999999;
}
</style>

50
packages/builder/src/helpers/data/format.js
View File

@ -9,11 +9,18 @@ export const datasourceSelect = {
datasourceName: datasource?.name,
}
},
viewV2: view => ({
...view,
label: view.name,
type: "viewV2",
}),
viewV2: (view, datasources) => {
const datasource = datasources
.filter(f => f.entities)
.flatMap(d => d.entities)
.find(ds => ds._id === view.tableId)
return {
...view,
label: view.name,
type: "viewV2",
datasourceName: datasource?.name,
}
},
}
export const tableSelect = {
@ -31,3 +38,36 @@ export const tableSelect = {
resourceId: view.id,
}),
}
export const sortAndFormat = {
tables: (tables, datasources) => {
return tables
.map(table => {
const formatted = datasourceSelect.table(table, datasources)
return {
...formatted,
resourceId: table._id,
}
})
.sort((a, b) => {
// sort tables alphabetically, grouped by datasource
const dsA = a.datasourceName ?? ""
const dsB = b.datasourceName ?? ""
const dsComparison = dsA.localeCompare(dsB)
if (dsComparison !== 0) {
return dsComparison
}
return a.label.localeCompare(b.label)
})
},
viewsV2: (views, datasources) => {
return views.map(view => {
const formatted = datasourceSelect.viewV2(view, datasources)
return {
...formatted,
resourceId: view.id,
}
})
},
}

10
packages/builder/src/pages/builder/apps/index.svelte
View File

@ -15,6 +15,7 @@
import {
appsStore,
organisation,
admin,
auth,
groups,
licensing,
@ -42,6 +43,7 @@
app => app.status === AppStatus.DEPLOYED
)
$: userApps = getUserApps(publishedApps, userGroups, $auth.user)
$: isOwner = $auth.accountPortalAccess && $admin.cloud
function getUserApps(publishedApps, userGroups, user) {
if (sdk.users.isAdmin(user)) {
@ -111,7 +113,13 @@
</MenuItem>
<MenuItem
icon="LockClosed"
on:click={() => changePasswordModal.show()}
on:click={() => {
if (isOwner) {
window.location.href = `${$admin.accountPortalUrl}/portal/account`
} else {
changePasswordModal.show()
}
}}
>
Update password
</MenuItem>

6
packages/builder/src/pages/builder/auth/reset.svelte
View File

@ -30,10 +30,16 @@
try {
loading = true
if (forceResetPassword) {
const email = $auth.user.email
const tenantId = $auth.user.tenantId
await auth.updateSelf({
password,
forceResetPassword: false,
})
if (!$auth.user) {
// Update self will clear the platform user, so need to login
await auth.login(email, password, tenantId)
}
$goto("../portal/")
} else {
await auth.resetPassword(password, resetCode)

15
packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte
View File

@ -1,5 +1,5 @@
<script>
import { auth } from "@/stores/portal"
import { admin, auth } from "@/stores/portal"
import { ActionMenu, MenuItem, Icon, Modal } from "@budibase/bbui"
import { goto } from "@roxi/routify"
import ProfileModal from "@/components/settings/ProfileModal.svelte"
@ -13,6 +13,8 @@
let updatePasswordModal
let apiKeyModal
$: isOwner = $auth.accountPortalAccess && $admin.cloud
const logout = async () => {
try {
await auth.logout()
@ -32,7 +34,16 @@
</MenuItem>
<MenuItem icon="Moon" on:click={() => themeModal.show()}>Theme</MenuItem>
{#if !$auth.isSSO}
<MenuItem icon="LockClosed" on:click={() => updatePasswordModal.show()}>
<MenuItem
icon="LockClosed"
on:click={() => {
if (isOwner) {
window.location.href = `${$admin.accountPortalUrl}/portal/account`
} else {
updatePasswordModal.show()
}
}}
>
Update password
</MenuItem>
{/if}

4
packages/builder/src/stores/portal/auth.ts
View File

@ -121,8 +121,8 @@ class AuthStore extends BudiStore<PortalAuthStore> {
}
}
async login(username: string, password: string) {
const tenantId = get(this.store).tenantId
async login(username: string, password: string, targetTenantId?: string) {
const tenantId = targetTenantId || get(this.store).tenantId
await API.logIn(tenantId, username, password)
await this.getSelf()
}

5
packages/client/src/components/app/DataProvider.svelte
View File

@ -14,7 +14,6 @@
GroupUserDatasource,
DataFetchOptions,
} from "@budibase/types"
import { SDK, Component } from "../../index"
type ProviderDatasource = Exclude<
DataFetchDatasource,
@ -29,8 +28,8 @@
export let paginate: boolean
export let autoRefresh: number
const { styleable, Provider, ActionTypes, API } = getContext<SDK>("sdk")
const component = getContext<Component>("component")
const { styleable, Provider, ActionTypes, API } = getContext("sdk")
const component = getContext("component")
let interval: ReturnType<typeof setInterval>
let queryExtensions: Record<string, any> = {}

7
packages/client/src/components/app/blocks/form/FormBlock.svelte
View File

@ -4,7 +4,6 @@
import { Utils } from "@budibase/frontend-core"
import FormBlockWrapper from "./FormBlockWrapper.svelte"
import { get } from "svelte/store"
import { Component, Context, SDK } from "../../../../index"
import { TableSchema, UIDatasource } from "@budibase/types"
type Field = { name: string; active: boolean }
@ -34,9 +33,9 @@
export let saveButtonLabel: boolean
export let deleteButtonLabel: boolean
const { fetchDatasourceSchema, generateGoldenSample } = getContext<SDK>("sdk")
const component = getContext<Component>("component")
const context = getContext<Context>("context")
const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk")
const component = getContext("component")
const context = getContext("context")
let schema: TableSchema

5
packages/client/src/components/error-states/ComponentErrorState.svelte
View File

@ -3,7 +3,6 @@
import { Icon } from "@budibase/bbui"
import MissingRequiredSetting from "./MissingRequiredSetting.svelte"
import MissingRequiredAncestor from "./MissingRequiredAncestor.svelte"
import { Component, SDK } from "../../index"
export let missingRequiredSettings:
| { key: string; label: string }[]
@ -11,8 +10,8 @@
export let missingRequiredAncestors: string[] | undefined
export let invalidSettings: string[] | undefined
const component = getContext<Component>("component")
const { styleable, builderStore } = getContext<SDK>("sdk")
const component = getContext("component")
const { styleable, builderStore } = getContext("sdk")
$: styles = { ...$component.styles, normal: {}, custom: null, empty: true }
$: requiredSetting = missingRequiredSettings?.[0]

7
packages/client/src/context.d.ts vendored Normal file
View File

@ -0,0 +1,7 @@
import { Component, Context, SDK } from "."
declare module "svelte" {
export function getContext(key: "sdk"): SDK
export function getContext(key: "component"): Component
export function getContext(key: "context"): Context
}

2
packages/client/src/index.ts
View File

@ -20,4 +20,4 @@ export type Component = Readable<{
errorState: boolean
}>
export type Context = Readable<any>
export type Context = Readable<{}>

4
packages/frontend-core/src/api/migrations.ts
View File

@ -1,8 +1,8 @@
import { GetOldMigrationStatus } from "@budibase/types"
import { GetMigrationStatus } from "@budibase/types"
import { BaseAPIClient } from "./types"
export interface MigrationEndpoints {
getMigrationStatus: () => Promise<GetOldMigrationStatus>
getMigrationStatus: () => Promise<GetMigrationStatus>
}
export const buildMigrationEndpoints = (

1
packages/server/scripts/dev/manage.js
View File

@ -43,7 +43,6 @@ async function init() {
BB_ADMIN_USER_EMAIL: "",
BB_ADMIN_USER_PASSWORD: "",
PLUGINS_DIR: "",
HTTP_MIGRATIONS: "0",
HTTP_LOGGING: "0",
VERSION: "0.0.0+local",
PASSWORD_MIN_LENGTH: "1",

9
packages/server/src/api/controllers/application.ts
View File

@ -27,7 +27,6 @@ import {
env as envCore,
ErrorCode,
events,
migrations,
objectStore,
roles,
tenancy,
@ -43,7 +42,6 @@ import { groups, licensing, quotas } from "@budibase/pro"
import {
App,
Layout,
MigrationType,
PlanType,
Screen,
UserCtx,
@ -488,13 +486,6 @@ async function creationEvents(request: BBRequest<CreateAppRequest>, app: App) {
}
async function appPostCreate(ctx: UserCtx<CreateAppRequest, App>, app: App) {
const tenantId = tenancy.getTenantId()
await migrations.backPopulateMigrations({
type: MigrationType.APP,
tenantId,
appId: app.appId,
})
await creationEvents(ctx.request, app)
// app import, template creation and duplication

28
packages/server/src/api/controllers/migrations.ts
View File

@ -1,35 +1,11 @@
import { context } from "@budibase/backend-core"
import { migrate as migrationImpl, MIGRATIONS } from "../../migrations"
import {
Ctx,
FetchOldMigrationResponse,
GetOldMigrationStatus,
RuneOldMigrationResponse,
RunOldMigrationRequest,
} from "@budibase/types"
import { Ctx, GetMigrationStatus } from "@budibase/types"
import {
getAppMigrationVersion,
getLatestEnabledMigrationId,
} from "../../appMigrations"
export async function migrate(
ctx: Ctx<RunOldMigrationRequest, RuneOldMigrationResponse>
) {
const options = ctx.request.body
// don't await as can take a while, just return
migrationImpl(options)
ctx.body = { message: "Migration started." }
}
export async function fetchDefinitions(
ctx: Ctx<void, FetchOldMigrationResponse>
) {
ctx.body = MIGRATIONS
}
export async function getMigrationStatus(
ctx: Ctx<void, GetOldMigrationStatus>
) {
export async function getMigrationStatus(ctx: Ctx<void, GetMigrationStatus>) {
const appId = context.getAppId()
if (!appId) {

147
packages/server/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap
View File

@ -1,147 +0,0 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`viewBuilder Calculate and filter creates a view with the calculation statistics and filter schema 1`] = `
{
"map": "function (doc) {
if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !(
doc["myField"] === undefined ||
doc["myField"] === null ||
doc["myField"] === "" ||
(Array.isArray(doc["myField"]) && doc["myField"].length === 0)
)) && (doc["age"] > 17)) {
emit(doc["_id"], doc["myField"]);
}
}",
"meta": {
"calculation": "stats",
"field": "myField",
"filters": [
{
"condition": "MT",
"key": "age",
"value": 17,
},
],
"groupBy": undefined,
"schema": {
"avg": {
"type": "number",
},
"count": {
"type": "number",
},
"field": {
"type": "string",
},
"max": {
"type": "number",
},
"min": {
"type": "number",
},
"sum": {
"type": "number",
},
"sumsqr": {
"type": "number",
},
},
"tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
},
"reduce": "_stats",
}
`;
exports[`viewBuilder Calculate creates a view with the calculation statistics schema 1`] = `
{
"map": "function (doc) {
if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !(
doc["myField"] === undefined ||
doc["myField"] === null ||
doc["myField"] === "" ||
(Array.isArray(doc["myField"]) && doc["myField"].length === 0)
)) ) {
emit(doc["_id"], doc["myField"]);
}
}",
"meta": {
"calculation": "stats",
"field": "myField",
"filters": [],
"groupBy": undefined,
"schema": {
"avg": {
"type": "number",
},
"count": {
"type": "number",
},
"field": {
"type": "string",
},
"max": {
"type": "number",
},
"min": {
"type": "number",
},
"sum": {
"type": "number",
},
"sumsqr": {
"type": "number",
},
},
"tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
},
"reduce": "_stats",
}
`;
exports[`viewBuilder Filter creates a view with multiple filters and conjunctions 1`] = `
{
"map": "function (doc) {
if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && (doc["Name"] === "Test" || doc["Yes"] > "Value")) {
emit(doc["_id"], doc["undefined"]);
}
}",
"meta": {
"calculation": undefined,
"field": undefined,
"filters": [
{
"condition": "EQUALS",
"key": "Name",
"value": "Test",
},
{
"condition": "MT",
"conjunction": "OR",
"key": "Yes",
"value": "Value",
},
],
"groupBy": undefined,
"schema": null,
"tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
},
}
`;
exports[`viewBuilder Group By creates a view emitting the group by field 1`] = `
{
"map": "function (doc) {
if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" ) {
emit(doc["age"], doc["score"]);
}
}",
"meta": {
"calculation": undefined,
"field": "score",
"filters": [],
"groupBy": "age",
"schema": null,
"tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
},
}
`;

75
packages/server/src/api/controllers/view/tests/viewBuilder.spec.js
View File

@ -1,75 +0,0 @@
const viewTemplate = require("../viewBuilder").default
describe("viewBuilder", () => {
describe("Filter", () => {
it("creates a view with multiple filters and conjunctions", () => {
expect(
viewTemplate({
name: "Test View",
tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
filters: [
{
value: "Test",
condition: "EQUALS",
key: "Name",
},
{
value: "Value",
condition: "MT",
key: "Yes",
conjunction: "OR",
},
],
})
).toMatchSnapshot()
})
})
describe("Calculate", () => {
it("creates a view with the calculation statistics schema", () => {
expect(
viewTemplate({
name: "Calculate View",
field: "myField",
calculation: "stats",
tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
filters: [],
})
).toMatchSnapshot()
})
})
describe("Group By", () => {
it("creates a view emitting the group by field", () => {
expect(
viewTemplate({
name: "Test Scores Grouped By Age",
tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
groupBy: "age",
field: "score",
filters: [],
})
).toMatchSnapshot()
})
})
describe("Calculate and filter", () => {
it("creates a view with the calculation statistics and filter schema", () => {
expect(
viewTemplate({
name: "Calculate View",
field: "myField",
calculation: "stats",
tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
filters: [
{
value: 17,
condition: "MT",
key: "age",
},
],
})
).toMatchSnapshot()
})
})
})

174
packages/server/src/api/controllers/view/tests/viewBuilder.spec.ts Normal file
View File

@ -0,0 +1,174 @@
import viewTemplate from "../viewBuilder"
describe("viewBuilder", () => {
describe("Filter", () => {
it("creates a view with multiple filters and conjunctions", () => {
expect(
viewTemplate({
field: "myField",
tableId: "tableId",
filters: [
{
value: "Test",
condition: "EQUALS",
key: "Name",
},
{
value: "Value",
condition: "MT",
key: "Yes",
conjunction: "OR",
},
],
})
).toEqual({
map: `function (doc) {
if (doc.tableId === "tableId" && (doc["Name"] === "Test" || doc["Yes"] > "Value")) {
emit(doc["_id"], doc["myField"]);
}
}`,
meta: {
calculation: undefined,
field: "myField",
filters: [
{
condition: "EQUALS",
key: "Name",
value: "Test",
},
{
condition: "MT",
conjunction: "OR",
key: "Yes",
value: "Value",
},
],
groupBy: undefined,
schema: null,
tableId: "tableId",
},
})
})
})
describe("Calculate", () => {
it("creates a view with the calculation statistics schema", () => {
expect(
viewTemplate({
field: "myField",
calculation: "stats",
tableId: "tableId",
filters: [],
})
).toEqual({
map: `function (doc) {
if ((doc.tableId === "tableId" && !(
doc["myField"] === undefined ||
doc["myField"] === null ||
doc["myField"] === "" ||
(Array.isArray(doc["myField"]) && doc["myField"].length === 0)
)) ) {
emit(doc["_id"], doc["myField"]);
}
}`,
meta: {
calculation: "stats",
field: "myField",
filters: [],
groupBy: undefined,
schema: {
min: { type: "number" },
max: { type: "number" },
avg: { type: "number" },
count: { type: "number" },
sumsqr: { type: "number" },
sum: { type: "number" },
field: { type: "string" },
},
tableId: "tableId",
},
reduce: "_stats",
})
})
})
describe("Group By", () => {
it("creates a view emitting the group by field", () => {
expect(
viewTemplate({
tableId: "tableId",
groupBy: "age",
field: "score",
filters: [],
})
).toEqual({
map: `function (doc) {
if (doc.tableId === "tableId" ) {
emit(doc["age"], doc["score"]);
}
}`,
meta: {
calculation: undefined,
field: "score",
filters: [],
groupBy: "age",
schema: null,
tableId: "tableId",
},
})
})
})
describe("Calculate and filter", () => {
it("creates a view with the calculation statistics and filter schema", () => {
expect(
viewTemplate({
field: "myField",
calculation: "stats",
tableId: "tableId",
filters: [
{
value: 17,
condition: "MT",
key: "age",
},
],
})
).toEqual({
map: `function (doc) {
if ((doc.tableId === "tableId" && !(
doc["myField"] === undefined ||
doc["myField"] === null ||
doc["myField"] === "" ||
(Array.isArray(doc["myField"]) && doc["myField"].length === 0)
)) && (doc["age"] > 17)) {
emit(doc["_id"], doc["myField"]);
}
}`,
meta: {
calculation: "stats",
field: "myField",
filters: [
{
condition: "MT",
key: "age",
value: 17,
},
],
groupBy: undefined,
schema: {
min: { type: "number" },
max: { type: "number" },
avg: { type: "number" },
count: { type: "number" },
sumsqr: { type: "number" },
sum: { type: "number" },
field: { type: "string" },
},
tableId: "tableId",
},
reduce: "_stats",
})
})
})
})

20
packages/server/src/api/controllers/view/viewBuilder.ts
View File

@ -1,4 +1,4 @@
import { ViewFilter, ViewTemplateOpts, DBView } from "@budibase/types"
import { ViewFilter, DBView } from "@budibase/types"
const TOKEN_MAP: Record<string, string> = {
EQUALS: "===",
@ -120,7 +120,7 @@ function parseFilterExpression(filters: ViewFilter[]) {
* @param groupBy - field to group calculation results on, if any
*/
function parseEmitExpression(field: string, groupBy: string) {
return `emit(doc["${groupBy || "_id"}"], doc["${field}"]);`
return `emit(doc["${groupBy}"], doc["${field}"]);`
}
/**
@ -135,7 +135,19 @@ function parseEmitExpression(field: string, groupBy: string) {
* calculation: an optional calculation to be performed over the view data.
*/
export default function (
{ field, tableId, groupBy, filters = [], calculation }: ViewTemplateOpts,
{
field,
tableId,
groupBy,
filters = [],
calculation,
}: {
field: string
tableId: string
groupBy?: string
filters?: ViewFilter[]
calculation?: string
},
groupByMulti?: boolean
): DBView {
// first filter can't have a conjunction
@ -168,7 +180,7 @@ export default function (
const parsedFilters = parseFilterExpression(filters)
const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
const emitExpression = parseEmitExpression(field, groupBy)
const emitExpression = parseEmitExpression(field, groupBy || "_id")
const tableExpression = `doc.tableId === "${tableId}"`
const coreExpression = statFilter
? `(${tableExpression} && ${statFilter})`

10
packages/server/src/api/routes/migrations.ts
View File

@ -1,16 +1,8 @@
import Router from "@koa/router"
import * as migrationsController from "../controllers/migrations"
import { auth } from "@budibase/backend-core"
const router: Router = new Router()
router
.post("/api/migrations/run", auth.internalApi, migrationsController.migrate)
.get(
"/api/migrations/definitions",
auth.internalApi,
migrationsController.fetchDefinitions
)
.get("/api/migrations/status", migrationsController.getMigrationStatus)
router.get("/api/migrations/status", migrationsController.getMigrationStatus)
export default router

4
packages/server/src/api/routes/tests/automation.spec.ts
View File

@ -19,9 +19,11 @@ import {
Table,
} from "@budibase/types"
import { mocks } from "@budibase/backend-core/tests"
import { FilterConditions } from "../../../automations/steps/filter"
import { removeDeprecated } from "../../../automations/utils"
import { createAutomationBuilder } from "../../../automations/tests/utilities/AutomationTestBuilder"
import { automations } from "@budibase/shared-core"
const FilterConditions = automations.steps.filter.FilterConditions
const MAX_RETRIES = 4
let {

55
packages/server/src/automations/actions.ts
View File

@ -1,3 +1,4 @@
import { automations } from "@budibase/shared-core"
import * as sendSmtpEmail from "./steps/sendSmtpEmail"
import * as createRow from "./steps/createRow"
import * as updateRow from "./steps/updateRow"
@ -14,11 +15,10 @@ import * as make from "./steps/make"
import * as filter from "./steps/filter"
import * as delay from "./steps/delay"
import * as queryRow from "./steps/queryRows"
import * as loop from "./steps/loop"
import * as collect from "./steps/collect"
import * as branch from "./steps/branch"
import * as triggerAutomationRun from "./steps/triggerAutomationRun"
import * as openai from "./steps/openai"
import * as bash from "./steps/bash"
import env from "../environment"
import {
PluginType,
@ -62,42 +62,39 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<
string,
AutomationStepDefinition
> = {
SEND_EMAIL_SMTP: sendSmtpEmail.definition,
CREATE_ROW: createRow.definition,
UPDATE_ROW: updateRow.definition,
DELETE_ROW: deleteRow.definition,
OUTGOING_WEBHOOK: outgoingWebhook.definition,
EXECUTE_SCRIPT: executeScript.definition,
EXECUTE_QUERY: executeQuery.definition,
SERVER_LOG: serverLog.definition,
DELAY: delay.definition,
FILTER: filter.definition,
QUERY_ROWS: queryRow.definition,
LOOP: loop.definition,
COLLECT: collect.definition,
TRIGGER_AUTOMATION_RUN: triggerAutomationRun.definition,
BRANCH: branch.definition,
SEND_EMAIL_SMTP: automations.steps.sendSmtpEmail.definition,
CREATE_ROW: automations.steps.createRow.definition,
UPDATE_ROW: automations.steps.updateRow.definition,
DELETE_ROW: automations.steps.deleteRow.definition,
OUTGOING_WEBHOOK: automations.steps.outgoingWebhook.definition,
EXECUTE_SCRIPT: automations.steps.executeScript.definition,
EXECUTE_QUERY: automations.steps.executeQuery.definition,
SERVER_LOG: automations.steps.serverLog.definition,
DELAY: automations.steps.delay.definition,
FILTER: automations.steps.filter.definition,
QUERY_ROWS: automations.steps.queryRows.definition,
LOOP: automations.steps.loop.definition,
COLLECT: automations.steps.collect.definition,
TRIGGER_AUTOMATION_RUN: automations.steps.triggerAutomationRun.definition,
BRANCH: automations.steps.branch.definition,
// these used to be lowercase step IDs, maintain for backwards compat
discord: discord.definition,
slack: slack.definition,
zapier: zapier.definition,
integromat: make.definition,
n8n: n8n.definition,
discord: automations.steps.discord.definition,
slack: automations.steps.slack.definition,
zapier: automations.steps.zapier.definition,
integromat: automations.steps.make.definition,
n8n: automations.steps.n8n.definition,
}
// don't add the bash script/definitions unless in self host
// the fact this isn't included in any definitions means it cannot be
// ran at all
if (env.SELF_HOSTED) {
const bash = require("./steps/bash")
// @ts-ignore
// @ts-expect-error
ACTION_IMPLS["EXECUTE_BASH"] = bash.run
// @ts-ignore
BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = automations.steps.bash.definition
if (env.isTest()) {
BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition
}
}
@ -105,7 +102,7 @@ export async function getActionDefinitions(): Promise<
Record<keyof typeof AutomationActionStepId, AutomationStepDefinition>
> {
if (env.SELF_HOSTED) {
BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition
}
const actionDefinitions = BUILTIN_ACTION_DEFINITIONS

50
packages/server/src/automations/steps/bash.ts
View File

@ -2,55 +2,7 @@ import { execSync } from "child_process"
import { processStringSync } from "@budibase/string-templates"
import * as automationUtils from "../automationUtils"
import environment from "../../environment"
import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
BashStepInputs,
BashStepOutputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Bash Scripting",
tagline: "Execute a bash command",
icon: "JourneyEvent",
description: "Run a bash script",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.EXECUTE_BASH,
inputs: {},
schema: {
inputs: {
properties: {
code: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.CODE,
title: "Code",
},
},
required: ["code"],
},
outputs: {
properties: {
stdout: {
type: AutomationIOType.STRING,
description: "Standard output of your bash command or script",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the command was successful",
},
},
required: ["stdout"],
},
},
}
import { BashStepInputs, BashStepOutputs } from "@budibase/types"
export async function run({
inputs,

46
packages/server/src/automations/steps/collect.ts
View File

@ -1,48 +1,4 @@
import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
CollectStepInputs,
CollectStepOutputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Collect Data",
tagline: "Collect data to be sent to design",
icon: "Collection",
description:
"Collects specified data so it can be provided to the design section",
type: AutomationStepType.ACTION,
internal: true,
features: {},
stepId: AutomationActionStepId.COLLECT,
inputs: {},
schema: {
inputs: {
properties: {
collection: {
type: AutomationIOType.STRING,
title: "What to Collect",
},
},
required: ["collection"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
value: {
type: AutomationIOType.STRING,
description: "Collected data",
},
},
required: ["success", "value"],
},
},
}
import { CollectStepInputs, CollectStepOutputs } from "@budibase/types"
export async function run({
inputs,

70
packages/server/src/automations/steps/createRow.ts
View File

@ -5,77 +5,9 @@ import {
sendAutomationAttachmentsToStorage,
} from "../automationUtils"
import { buildCtx } from "./utils"
import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
CreateRowStepInputs,
CreateRowStepOutputs,
} from "@budibase/types"
import { CreateRowStepInputs, CreateRowStepOutputs } from "@budibase/types"
import { EventEmitter } from "events"
export const definition: AutomationStepDefinition = {
name: "Create Row",
tagline: "Create a {{inputs.enriched.table.name}} row",
icon: "TableRowAddBottom",
description: "Add a row to your database",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.CREATE_ROW,
inputs: {},
schema: {
inputs: {
properties: {
row: {
type: AutomationIOType.OBJECT,
properties: {
tableId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.TABLE,
},
},
customType: AutomationCustomIOType.ROW,
title: "Table",
required: ["tableId"],
},
},
required: ["row"],
},
outputs: {
properties: {
row: {
type: AutomationIOType.OBJECT,
customType: AutomationCustomIOType.ROW,
description: "The new row",
},
response: {
type: AutomationIOType.OBJECT,
description: "The response from the table",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the row creation was successful",
},
id: {
type: AutomationIOType.STRING,
description: "The identifier of the new row",
},
revision: {
type: AutomationIOType.STRING,
description: "The revision of the new row",
},
},
required: ["success", "id", "revision"],
},
},
}
export async function run({
inputs,
appId,

41
packages/server/src/automations/steps/delay.ts
View File

@ -1,44 +1,5 @@
import { wait } from "../../utilities"
import {
AutomationActionStepId,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
DelayStepInputs,
DelayStepOutputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Delay",
icon: "Clock",
tagline: "Delay for {{inputs.time}} milliseconds",
description: "Delay the automation until an amount of time has passed",
stepId: AutomationActionStepId.DELAY,
internal: true,
features: {},
inputs: {},
schema: {
inputs: {
properties: {
time: {
type: AutomationIOType.NUMBER,
title: "Delay in milliseconds",
},
},
required: ["time"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the delay was successful",
},
},
required: ["success"],
},
},
type: AutomationStepType.LOGIC,
}
import { DelayStepInputs, DelayStepOutputs } from "@budibase/types"
export async function run({
inputs,

59
packages/server/src/automations/steps/deleteRow.ts
View File

@ -2,64 +2,7 @@ import { EventEmitter } from "events"
import { destroy } from "../../api/controllers/row"
import { buildCtx } from "./utils"
import { getError } from "../automationUtils"
import {
AutomationActionStepId,
AutomationStepType,
AutomationIOType,
AutomationCustomIOType,
AutomationFeature,
DeleteRowStepInputs,
DeleteRowStepOutputs,
AutomationStepDefinition,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
description: "Delete a row from your database",
icon: "TableRowRemoveCenter",
name: "Delete Row",
tagline: "Delete a {{inputs.enriched.table.name}} row",
type: AutomationStepType.ACTION,
stepId: AutomationActionStepId.DELETE_ROW,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
tableId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.TABLE,
title: "Table",
},
id: {
type: AutomationIOType.STRING,
title: "Row ID",
},
},
required: ["tableId", "id"],
},
outputs: {
properties: {
row: {
type: AutomationIOType.OBJECT,
customType: AutomationCustomIOType.ROW,
description: "The deleted row",
},
response: {
type: AutomationIOType.OBJECT,
description: "The response from the table",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the deletion was successful",
},
},
required: ["row", "success"],
},
},
}
import { DeleteRowStepInputs, DeleteRowStepOutputs } from "@budibase/types"
export async function run({
inputs,

63
packages/server/src/automations/steps/discord.ts
View File

@ -1,71 +1,10 @@
import fetch from "node-fetch"
import { getFetchResponse } from "./utils"
import {
AutomationActionStepId,
AutomationStepType,
AutomationIOType,
AutomationFeature,
ExternalAppStepOutputs,
DiscordStepInputs,
AutomationStepDefinition,
} from "@budibase/types"
import { ExternalAppStepOutputs, DiscordStepInputs } from "@budibase/types"
const DEFAULT_USERNAME = "Budibase Automate"
const DEFAULT_AVATAR_URL = "https://i.imgur.com/a1cmTKM.png"
export const definition: AutomationStepDefinition = {
name: "Discord Message",
tagline: "Send a message to a Discord server",
description: "Send a message to a Discord server",
icon: "ri-discord-line",
stepId: AutomationActionStepId.discord,
type: AutomationStepType.ACTION,
internal: false,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
url: {
type: AutomationIOType.STRING,
title: "Discord Webhook URL",
},
username: {
type: AutomationIOType.STRING,
title: "Bot Name",
},
avatar_url: {
type: AutomationIOType.STRING,
title: "Bot Avatar URL",
},
content: {
type: AutomationIOType.STRING,
title: "Message",
},
},
required: ["url", "content"],
},
outputs: {
properties: {
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code of the request",
},
response: {
type: AutomationIOType.STRING,
description: "The response from the Discord Webhook",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the message sent successfully",
},
},
},
},
}
export async function run({
inputs,
}: {

57
packages/server/src/automations/steps/executeQuery.ts
View File

@ -3,67 +3,10 @@ import * as queryController from "../../api/controllers/query"
import { buildCtx } from "./utils"
import * as automationUtils from "../automationUtils"
import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
ExecuteQueryStepInputs,
ExecuteQueryStepOutputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "External Data Connector",
tagline: "Execute Data Connector",
icon: "Data",
description: "Execute a query in an external data connector",
type: AutomationStepType.ACTION,
stepId: AutomationActionStepId.EXECUTE_QUERY,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
query: {
type: AutomationIOType.OBJECT,
properties: {
queryId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.QUERY,
},
},
customType: AutomationCustomIOType.QUERY_PARAMS,
title: "Parameters",
required: ["queryId"],
},
},
required: ["query"],
},
outputs: {
properties: {
response: {
type: AutomationIOType.OBJECT,
description: "The response from the datasource execution",
},
info: {
type: AutomationIOType.OBJECT,
description:
"Some query types may return extra data, like headers from a REST query",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
},
required: ["response", "success"],
},
},
}
export async function run({
inputs,
appId,

45
packages/server/src/automations/steps/executeScript.ts
View File

@ -2,56 +2,11 @@ import * as scriptController from "../../api/controllers/script"
import { buildCtx } from "./utils"
import * as automationUtils from "../automationUtils"
import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
ExecuteScriptStepInputs,
ExecuteScriptStepOutputs,
} from "@budibase/types"
import { EventEmitter } from "events"
export const definition: AutomationStepDefinition = {
name: "JS Scripting",
tagline: "Execute JavaScript Code",
icon: "Code",
description: "Run a piece of JavaScript code in your automation",
type: AutomationStepType.ACTION,
internal: true,
stepId: AutomationActionStepId.EXECUTE_SCRIPT,
inputs: {},
features: {
[AutomationFeature.LOOPING]: true,
},
schema: {
inputs: {
properties: {
code: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.CODE,
title: "Code",
},
},
required: ["code"],
},
outputs: {
properties: {
value: {
type: AutomationIOType.STRING,
description: "The result of the return statement",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
},
required: ["success"],
},
},
}
export async function run({
inputs,
appId,

73
packages/server/src/automations/steps/filter.ts
View File

@ -1,74 +1,7 @@
import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
FilterStepInputs,
FilterStepOutputs,
} from "@budibase/types"
import { FilterStepInputs, FilterStepOutputs } from "@budibase/types"
import { automations } from "@budibase/shared-core"
export const FilterConditions = {
EQUAL: "EQUAL",
NOT_EQUAL: "NOT_EQUAL",
GREATER_THAN: "GREATER_THAN",
LESS_THAN: "LESS_THAN",
}
export const PrettyFilterConditions = {
[FilterConditions.EQUAL]: "Equals",
[FilterConditions.NOT_EQUAL]: "Not equals",
[FilterConditions.GREATER_THAN]: "Greater than",
[FilterConditions.LESS_THAN]: "Less than",
}
export const definition: AutomationStepDefinition = {
name: "Condition",
tagline: "{{inputs.field}} {{inputs.condition}} {{inputs.value}}",
icon: "Branch2",
description:
"Conditionally halt automations which do not meet certain conditions",
type: AutomationStepType.LOGIC,
internal: true,
features: {},
stepId: AutomationActionStepId.FILTER,
inputs: {
condition: FilterConditions.EQUAL,
},
schema: {
inputs: {
properties: {
field: {
type: AutomationIOType.STRING,
title: "Reference Value",
},
condition: {
type: AutomationIOType.STRING,
title: "Condition",
enum: Object.values(FilterConditions),
pretty: Object.values(PrettyFilterConditions),
},
value: {
type: AutomationIOType.STRING,
title: "Comparison Value",
},
},
required: ["field", "condition", "value"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
result: {
type: AutomationIOType.BOOLEAN,
description: "Whether the logic block passed",
},
},
required: ["success", "result"],
},
},
}
const FilterConditions = automations.steps.filter.FilterConditions
export async function run({
inputs,

58
packages/server/src/automations/steps/make.ts
View File

@ -1,62 +1,6 @@
import fetch from "node-fetch"
import { getFetchResponse } from "./utils"
import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
AutomationFeature,
ExternalAppStepOutputs,
MakeIntegrationInputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Make Integration",
stepTitle: "Make",
tagline: "Trigger a Make scenario",
description:
"Performs a webhook call to Make and gets the response (if configured)",
icon: "ri-shut-down-line",
stepId: AutomationActionStepId.integromat,
type: AutomationStepType.ACTION,
internal: false,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
url: {
type: AutomationIOType.STRING,
title: "Webhook URL",
},
body: {
type: AutomationIOType.JSON,
title: "Payload",
},
},
required: ["url", "body"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether call was successful",
},
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code returned",
},
response: {
type: AutomationIOType.OBJECT,
description: "The webhook response - this can have properties",
},
},
required: ["success", "response"],
},
},
}
import { ExternalAppStepOutputs, MakeIntegrationInputs } from "@budibase/types"
export async function run({
inputs,

62
packages/server/src/automations/steps/n8n.ts
View File

@ -1,73 +1,11 @@
import fetch, { HeadersInit } from "node-fetch"
import { getFetchResponse } from "./utils"
import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
AutomationFeature,
HttpMethod,
ExternalAppStepOutputs,
n8nStepInputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "n8n Integration",
stepTitle: "n8n",
tagline: "Trigger an n8n workflow",
description:
"Performs a webhook call to n8n and gets the response (if configured)",
icon: "ri-shut-down-line",
stepId: AutomationActionStepId.n8n,
type: AutomationStepType.ACTION,
internal: false,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
url: {
type: AutomationIOType.STRING,
title: "Webhook URL",
},
method: {
type: AutomationIOType.STRING,
title: "Method",
enum: Object.values(HttpMethod),
},
authorization: {
type: AutomationIOType.STRING,
title: "Authorization",
},
body: {
type: AutomationIOType.JSON,
title: "Payload",
},
},
required: ["url", "method"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether call was successful",
},
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code returned",
},
response: {
type: AutomationIOType.OBJECT,
description: "The webhook response - this can have properties",
},
},
required: ["success", "response"],
},
},
}
export async function run({
inputs,
}: {

59
packages/server/src/automations/steps/openai.ts
View File

@ -1,67 +1,10 @@
import { OpenAI } from "openai"
import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
OpenAIStepInputs,
OpenAIStepOutputs,
} from "@budibase/types"
import { OpenAIStepInputs, OpenAIStepOutputs } from "@budibase/types"
import { env } from "@budibase/backend-core"
import * as automationUtils from "../automationUtils"
import * as pro from "@budibase/pro"
enum Model {
GPT_4O_MINI = "gpt-4o-mini",
GPT_4O = "gpt-4o",
GPT_4 = "gpt-4",
GPT_35_TURBO = "gpt-3.5-turbo",
}
export const definition: AutomationStepDefinition = {
name: "OpenAI",
tagline: "Send prompts to ChatGPT",
icon: "Algorithm",
description: "Interact with the OpenAI ChatGPT API.",
type: AutomationStepType.ACTION,
internal: true,
features: {},
stepId: AutomationActionStepId.OPENAI,
inputs: {
prompt: "",
},
schema: {
inputs: {
properties: {
prompt: {
type: AutomationIOType.STRING,
title: "Prompt",
},
model: {
type: AutomationIOType.STRING,
title: "Model",
enum: Object.values(Model),
},
},
required: ["prompt", "model"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
response: {
type: AutomationIOType.STRING,
description: "What was output",
},
},
required: ["success", "response"],
},
},
}
/**
* Maintains backward compatibility with automation steps created before the introduction
* of custom configurations and Budibase AI

69
packages/server/src/automations/steps/outgoingWebhook.ts
View File

@ -2,12 +2,6 @@ import fetch from "node-fetch"
import { getFetchResponse } from "./utils"
import * as automationUtils from "../automationUtils"
import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
ExternalAppStepOutputs,
OutgoingWebhookStepInputs,
} from "@budibase/types"
@ -26,69 +20,6 @@ const BODY_REQUESTS = [RequestType.POST, RequestType.PUT, RequestType.PATCH]
* NOTE: this functionality is deprecated - it no longer should be used.
*/
export const definition: AutomationStepDefinition = {
deprecated: true,
name: "Outgoing webhook",
tagline: "Send a {{inputs.requestMethod}} request",
icon: "Send",
description: "Send a request of specified method to a URL",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.OUTGOING_WEBHOOK,
inputs: {
requestMethod: "POST",
url: "http://",
requestBody: "{}",
headers: "{}",
},
schema: {
inputs: {
properties: {
requestMethod: {
type: AutomationIOType.STRING,
enum: Object.values(RequestType),
title: "Request method",
},
url: {
type: AutomationIOType.STRING,
title: "URL",
},
requestBody: {
type: AutomationIOType.STRING,
title: "JSON Body",
customType: AutomationCustomIOType.WIDE,
},
headers: {
type: AutomationIOType.STRING,
title: "Headers",
customType: AutomationCustomIOType.WIDE,
},
},
required: ["requestMethod", "url"],
},
outputs: {
properties: {
response: {
type: AutomationIOType.OBJECT,
description: "The response from the webhook",
},
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code returned",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
},
required: ["response", "success"],
},
},
}
export async function run({
inputs,
}: {

72
packages/server/src/automations/steps/queryRows.ts
View File

@ -4,84 +4,12 @@ import { buildCtx } from "./utils"
import * as automationUtils from "../automationUtils"
import {
FieldType,
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
EmptyFilterOption,
SortOrder,
QueryRowsStepInputs,
QueryRowsStepOutputs,
} from "@budibase/types"
const SortOrderPretty = {
[SortOrder.ASCENDING]: "Ascending",
[SortOrder.DESCENDING]: "Descending",
}
export const definition: AutomationStepDefinition = {
description: "Query rows from the database",
icon: "Search",
name: "Query rows",
tagline: "Query rows from {{inputs.enriched.table.name}} table",
type: AutomationStepType.ACTION,
stepId: AutomationActionStepId.QUERY_ROWS,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
tableId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.TABLE,
title: "Table",
},
filters: {
type: AutomationIOType.OBJECT,
customType: AutomationCustomIOType.FILTERS,
title: "Filtering",
},
sortColumn: {
type: AutomationIOType.STRING,
title: "Sort Column",
customType: AutomationCustomIOType.COLUMN,
},
sortOrder: {
type: AutomationIOType.STRING,
title: "Sort Order",
enum: Object.values(SortOrder),
pretty: Object.values(SortOrderPretty),
},
limit: {
type: AutomationIOType.NUMBER,
title: "Limit",
customType: AutomationCustomIOType.QUERY_LIMIT,
},
},
required: ["tableId"],
},
outputs: {
properties: {
rows: {
type: AutomationIOType.ARRAY,
customType: AutomationCustomIOType.ROWS,
description: "The rows that were found",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the query was successful",
},
},
required: ["rows", "success"],
},
},
}
async function getTable(appId: string, tableId: string) {
const ctx: any = buildCtx(appId, null, {
params: {

98
packages/server/src/automations/steps/sendSmtpEmail.ts
View File

@ -1,102 +1,6 @@
import { sendSmtpEmail } from "../../utilities/workerRequests"
import * as automationUtils from "../automationUtils"
import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
AutomationFeature,
AutomationCustomIOType,
SmtpEmailStepInputs,
BaseAutomationOutputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
description: "Send an email using SMTP",
tagline: "Send SMTP email to {{inputs.to}}",
icon: "Email",
name: "Send Email (SMTP)",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.SEND_EMAIL_SMTP,
inputs: {},
schema: {
inputs: {
properties: {
to: {
type: AutomationIOType.STRING,
title: "Send To",
},
from: {
type: AutomationIOType.STRING,
title: "Send From",
},
cc: {
type: AutomationIOType.STRING,
title: "CC",
},
bcc: {
type: AutomationIOType.STRING,
title: "BCC",
},
subject: {
type: AutomationIOType.STRING,
title: "Email Subject",
},
contents: {
type: AutomationIOType.STRING,
title: "HTML Contents",
},
addInvite: {
type: AutomationIOType.BOOLEAN,
title: "Add calendar invite",
},
startTime: {
type: AutomationIOType.DATE,
title: "Start Time",
dependsOn: "addInvite",
},
endTime: {
type: AutomationIOType.DATE,
title: "End Time",
dependsOn: "addInvite",
},
summary: {
type: AutomationIOType.STRING,
title: "Meeting Summary",
dependsOn: "addInvite",
},
location: {
type: AutomationIOType.STRING,
title: "Location",
dependsOn: "addInvite",
},
attachments: {
type: AutomationIOType.ATTACHMENT,
customType: AutomationCustomIOType.MULTI_ATTACHMENTS,
title: "Attachments",
},
},
required: ["to", "from", "subject", "contents"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the email was sent",
},
response: {
type: AutomationIOType.OBJECT,
description: "A response from the email client, this may be an error",
},
},
required: ["success"],
},
},
}
import { SmtpEmailStepInputs, BaseAutomationOutputs } from "@budibase/types"
export async function run({
inputs,

56
packages/server/src/automations/steps/serverLog.ts
View File

@ -1,58 +1,4 @@
import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
AutomationFeature,
ServerLogStepInputs,
ServerLogStepOutputs,
} from "@budibase/types"
/**
* Note, there is some functionality in this that is not currently exposed as it
* is complex and maybe better to be opinionated here.
* GET/DELETE requests cannot handle body elements so they will not be sent if configured.
*/
export const definition: AutomationStepDefinition = {
name: "Backend log",
tagline: "Console log a value in the backend",
icon: "Monitoring",
description: "Logs the given text to the server (using console.log)",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.SERVER_LOG,
inputs: {
text: "",
},
schema: {
inputs: {
properties: {
text: {
type: AutomationIOType.STRING,
title: "Log",
},
},
required: ["text"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
message: {
type: AutomationIOType.STRING,
description: "What was output",
},
},
required: ["success", "message"],
},
},
}
import { ServerLogStepInputs, ServerLogStepOutputs } from "@budibase/types"
export async function run({
inputs,

55
packages/server/src/automations/steps/slack.ts
View File

@ -1,59 +1,6 @@
import fetch from "node-fetch"
import { getFetchResponse } from "./utils"
import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
AutomationFeature,
ExternalAppStepOutputs,
SlackStepInputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Slack Message",
tagline: "Send a message to Slack",
description: "Send a message to Slack",
icon: "ri-slack-line",
stepId: AutomationActionStepId.slack,
type: AutomationStepType.ACTION,
internal: false,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
url: {
type: AutomationIOType.STRING,
title: "Incoming Webhook URL",
},
text: {
type: AutomationIOType.STRING,
title: "Message",
},
},
required: ["url", "text"],
},
outputs: {
properties: {
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code of the request",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the message sent successfully",
},
response: {
type: AutomationIOType.STRING,
description: "The response from the Slack Webhook",
},
},
},
},
}
import { ExternalAppStepOutputs, SlackStepInputs } from "@budibase/types"
export async function run({
inputs,

53
packages/server/src/automations/steps/triggerAutomationRun.ts
View File

@ -1,10 +1,5 @@
import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
Automation,
AutomationCustomIOType,
TriggerAutomationStepInputs,
TriggerAutomationStepOutputs,
} from "@budibase/types"
@ -13,54 +8,6 @@ import { context } from "@budibase/backend-core"
import { features } from "@budibase/pro"
import env from "../../environment"
export const definition: AutomationStepDefinition = {
name: "Trigger an automation",
tagline: "Triggers an automation synchronously",
icon: "Sync",
description: "Triggers an automation synchronously",
type: AutomationStepType.ACTION,
internal: true,
features: {},
stepId: AutomationActionStepId.TRIGGER_AUTOMATION_RUN,
inputs: {},
schema: {
inputs: {
properties: {
automation: {
type: AutomationIOType.OBJECT,
properties: {
automationId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.AUTOMATION,
},
},
customType: AutomationCustomIOType.AUTOMATION_FIELDS,
title: "automatioFields",
required: ["automationId"],
},
timeout: {
type: AutomationIOType.NUMBER,
title: "Timeout (ms)",
},
},
required: ["automationId"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the automation was successful",
},
value: {
type: AutomationIOType.OBJECT,
description: "Automation Result",
},
},
required: ["success", "value"],
},
},
}
export async function run({
inputs,
}: {

70
packages/server/src/automations/steps/updateRow.ts
View File

@ -2,76 +2,8 @@ import { EventEmitter } from "events"
import * as rowController from "../../api/controllers/row"
import * as automationUtils from "../automationUtils"
import { buildCtx } from "./utils"
import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
UpdateRowStepInputs,
UpdateRowStepOutputs,
} from "@budibase/types"
import { UpdateRowStepInputs, UpdateRowStepOutputs } from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Update Row",
tagline: "Update a {{inputs.enriched.table.name}} row",
icon: "Refresh",
description: "Update a row in your database",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.UPDATE_ROW,
inputs: {},
schema: {
inputs: {
properties: {
meta: {
type: AutomationIOType.OBJECT,
title: "Field settings",
},
row: {
type: AutomationIOType.OBJECT,
customType: AutomationCustomIOType.ROW,
title: "Table",
},
rowId: {
type: AutomationIOType.STRING,
title: "Row ID",
},
},
required: ["row", "rowId"],
},
outputs: {
properties: {
row: {
type: AutomationIOType.OBJECT,
customType: AutomationCustomIOType.ROW,
description: "The updated row",
},
response: {
type: AutomationIOType.OBJECT,
description: "The response from the table",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
id: {
type: AutomationIOType.STRING,
description: "The identifier of the updated row",
},
revision: {
type: AutomationIOType.STRING,
description: "The revision of the updated row",
},
},
required: ["success", "id", "revision"],
},
},
}
export async function run({
inputs,
appId,

51
packages/server/src/automations/steps/zapier.ts
View File

@ -1,55 +1,6 @@
import fetch from "node-fetch"
import { getFetchResponse } from "./utils"
import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
AutomationFeature,
ZapierStepInputs,
ZapierStepOutputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Zapier Webhook",
stepId: AutomationActionStepId.zapier,
type: AutomationStepType.ACTION,
internal: false,
features: {
[AutomationFeature.LOOPING]: true,
},
description: "Trigger a Zapier Zap via webhooks",
tagline: "Trigger a Zapier Zap",
icon: "ri-flashlight-line",
inputs: {},
schema: {
inputs: {
properties: {
url: {
type: AutomationIOType.STRING,
title: "Webhook URL",
},
body: {
type: AutomationIOType.JSON,
title: "Payload",
},
},
required: ["url"],
},
outputs: {
properties: {
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code of the request",
},
response: {
type: AutomationIOType.STRING,
description: "The response from Zapier",
},
},
},
},
}
import { ZapierStepInputs, ZapierStepOutputs } from "@budibase/types"
export async function run({
inputs,

4
packages/server/src/automations/tests/filter.spec.ts
View File

@ -1,5 +1,7 @@
import * as setup from "./utilities"
import { FilterConditions } from "../steps/filter"
import { automations } from "@budibase/shared-core"
const FilterConditions = automations.steps.filter.FilterConditions
describe("test the filter logic", () => {
const config = setup.getConfig()

4
packages/server/src/automations/tests/scenarios/scenarios.spec.ts
View File

@ -6,9 +6,11 @@ import {
DatabaseName,
datasourceDescribe,
} from "../../../integrations/tests/utils"
import { FilterConditions } from "../../../automations/steps/filter"
import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests"
import { automations } from "@budibase/shared-core"
const FilterConditions = automations.steps.filter.FilterConditions
describe("Automation Scenarios", () => {
let config = setup.getConfig()

4
packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts
View File

@ -42,7 +42,7 @@ import {
} from "@budibase/types"
import TestConfiguration from "../../../tests/utilities/TestConfiguration"
import * as setup from "../utilities"
import { definition } from "../../../automations/steps/branch"
import { automations } from "@budibase/shared-core"
type TriggerOutputs =
| RowCreatedTriggerOutputs
@ -103,7 +103,7 @@ class BaseStepBuilder {
branchStepInputs.children![branchId] = stepBuilder.build()
})
const branchStep: AutomationStep = {
...definition,
...automations.steps.branch.definition,
id: uuidv4(),
stepId: AutomationActionStepId.BRANCH,
inputs: branchStepInputs,

1
packages/server/src/environment.ts
View File

@ -54,7 +54,6 @@ const environment = {
REDIS_URL: process.env.REDIS_URL,
REDIS_PASSWORD: process.env.REDIS_PASSWORD,
REDIS_CLUSTERED: process.env.REDIS_CLUSTERED,
HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
CLUSTER_MODE: process.env.CLUSTER_MODE,
API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

5
packages/server/src/jsRunner/index.ts
View File

@ -4,12 +4,17 @@ import {
JsTimeoutError,
setJSRunner,
setOnErrorLog,
setTestingBackendJS,
} from "@budibase/string-templates"
import { context, logging } from "@budibase/backend-core"
import tracer from "dd-trace"
import { IsolatedVM } from "./vm"
export function init() {
// enforce that if we're using isolated-VM runner then we are running backend JS
if (env.isTest()) {
setTestingBackendJS()
}
setJSRunner((js: string, ctx: Record<string, any>) => {
return tracer.trace("runJS", {}, () => {
try {

27
packages/server/src/migrations/functions/appUrls.ts
View File

@ -1,27 +0,0 @@
import { db as dbCore } from "@budibase/backend-core"
import sdk from "../../sdk"
/**
* Date:
* January 2022
*
* Description:
* Add the url to the app metadata if it doesn't exist
*/
export const run = async (appDb: any) => {
let metadata
try {
metadata = await appDb.get(dbCore.DocumentType.APP_METADATA)
} catch (e) {
// sometimes the metadata document doesn't exist
// exit early instead of failing the migration
console.error("Error retrieving app metadata. Skipping", e)
return
}
if (!metadata.url) {
metadata.url = sdk.applications.getAppUrl({ name: metadata.name })
console.log(`Adding url to app: ${metadata.url}`)
await appDb.put(metadata)
}
}

149
packages/server/src/migrations/functions/backfill/app.ts
View File

@ -1,149 +0,0 @@
import * as automations from "./app/automations"
import * as datasources from "./app/datasources"
import * as layouts from "./app/layouts"
import * as queries from "./app/queries"
import * as roles from "./app/roles"
import * as tables from "./app/tables"
import * as screens from "./app/screens"
import * as global from "./global"
import { App, AppBackfillSucceededEvent, Event } from "@budibase/types"
import { db as dbUtils, events } from "@budibase/backend-core"
import env from "../../../environment"
import { DEFAULT_TIMESTAMP } from "."
const failGraceful = env.SELF_HOSTED && !env.isDev()
const handleError = (e: any, errors?: any) => {
if (failGraceful) {
if (errors) {
errors.push(e)
}
return
}
console.trace(e)
throw e
}
const EVENTS = [
Event.AUTOMATION_CREATED,
Event.AUTOMATION_STEP_CREATED,
Event.DATASOURCE_CREATED,
Event.LAYOUT_CREATED,
Event.QUERY_CREATED,
Event.ROLE_CREATED,
Event.SCREEN_CREATED,
Event.TABLE_CREATED,
Event.VIEW_CREATED,
Event.VIEW_CALCULATION_CREATED,
Event.VIEW_FILTER_CREATED,
Event.APP_PUBLISHED,
Event.APP_CREATED,
]
/**
* Date:
* May 2022
*
* Description:
* Backfill app events.
*/
export const run = async (appDb: any) => {
try {
if (await global.isComplete()) {
// make sure new apps aren't backfilled
// return if the global migration for this tenant is complete
// which runs after the app migrations
return
}
// tell the event pipeline to start caching
// events for this tenant
await events.backfillCache.start(EVENTS)
let timestamp: string | number = DEFAULT_TIMESTAMP
const app: App = await appDb.get(dbUtils.DocumentType.APP_METADATA)
if (app.createdAt) {
timestamp = app.createdAt as string
}
if (dbUtils.isProdAppID(app.appId)) {
await events.app.published(app, timestamp)
}
const totals: any = {}
const errors: any = []
if (dbUtils.isDevAppID(app.appId)) {
await events.app.created(app, timestamp)
try {
totals.automations = await automations.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.datasources = await datasources.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.layouts = await layouts.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.queries = await queries.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.roles = await roles.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.screens = await screens.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.tables = await tables.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
}
const properties: AppBackfillSucceededEvent = {
appId: app.appId,
automations: totals.automations,
datasources: totals.datasources,
layouts: totals.layouts,
queries: totals.queries,
roles: totals.roles,
tables: totals.tables,
screens: totals.screens,
}
if (errors.length) {
properties.errors = errors.map((e: any) =>
JSON.stringify(e, Object.getOwnPropertyNames(e))
)
properties.errorCount = errors.length
} else {
properties.errorCount = 0
}
await events.backfill.appSucceeded(properties)
// tell the event pipeline to stop caching events for this tenant
await events.backfillCache.end()
} catch (e) {
handleError(e)
await events.backfill.appFailed(e)
}
}

26
packages/server/src/migrations/functions/backfill/app/automations.ts
View File

@ -1,26 +0,0 @@
import { events } from "@budibase/backend-core"
import { getAutomationParams } from "../../../../db/utils"
import { Automation } from "@budibase/types"
const getAutomations = async (appDb: any): Promise<Automation[]> => {
const response = await appDb.allDocs(
getAutomationParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const automations = await getAutomations(appDb)
for (const automation of automations) {
await events.automation.created(automation, timestamp)
for (const step of automation.definition.steps) {
await events.automation.stepCreated(automation, step, timestamp)
}
}
return automations.length
}

22
packages/server/src/migrations/functions/backfill/app/datasources.ts
View File

@ -1,22 +0,0 @@
import { events } from "@budibase/backend-core"
import { getDatasourceParams } from "../../../../db/utils"
import { Datasource } from "@budibase/types"
const getDatasources = async (appDb: any): Promise<Datasource[]> => {
const response = await appDb.allDocs(
getDatasourceParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const datasources: Datasource[] = await getDatasources(appDb)
for (const datasource of datasources) {
await events.datasource.created(datasource, timestamp)
}
return datasources.length
}

29
packages/server/src/migrations/functions/backfill/app/layouts.ts
View File

@ -1,29 +0,0 @@
import { events } from "@budibase/backend-core"
import { getLayoutParams } from "../../../../db/utils"
import { Layout } from "@budibase/types"
const getLayouts = async (appDb: any): Promise<Layout[]> => {
const response = await appDb.allDocs(
getLayoutParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const layouts: Layout[] = await getLayouts(appDb)
for (const layout of layouts) {
// exclude default layouts
if (
layout._id === "layout_private_master" ||
layout._id === "layout_public_master"
) {
continue
}
await events.layout.created(layout, timestamp)
}
return layouts.length
}

47
packages/server/src/migrations/functions/backfill/app/queries.ts
View File

@ -1,47 +0,0 @@
import { events } from "@budibase/backend-core"
import { getQueryParams } from "../../../../db/utils"
import { Query, Datasource, SourceName } from "@budibase/types"
const getQueries = async (appDb: any): Promise<Query[]> => {
const response = await appDb.allDocs(
getQueryParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
const getDatasource = async (
appDb: any,
datasourceId: string
): Promise<Datasource> => {
return appDb.get(datasourceId)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const queries: Query[] = await getQueries(appDb)
for (const query of queries) {
let datasource: Datasource
try {
datasource = await getDatasource(appDb, query.datasourceId)
} catch (e: any) {
// handle known bug where a datasource has been deleted
// and the query has not
if (e.status === 404) {
datasource = {
type: "unknown",
_id: query.datasourceId,
source: "unknown" as SourceName,
}
} else {
throw e
}
}
await events.query.created(datasource, query, timestamp)
}
return queries.length
}

22
packages/server/src/migrations/functions/backfill/app/roles.ts
View File

@ -1,22 +0,0 @@
import { events } from "@budibase/backend-core"
import { getRoleParams } from "../../../../db/utils"
import { Role } from "@budibase/types"
const getRoles = async (appDb: any): Promise<Role[]> => {
const response = await appDb.allDocs(
getRoleParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const roles = await getRoles(appDb)
for (const role of roles) {
await events.role.created(role, timestamp)
}
return roles.length
}

22
packages/server/src/migrations/functions/backfill/app/screens.ts
View File

@ -1,22 +0,0 @@
import { events } from "@budibase/backend-core"
import { getScreenParams } from "../../../../db/utils"
import { Screen } from "@budibase/types"
const getScreens = async (appDb: any): Promise<Screen[]> => {
const response = await appDb.allDocs(
getScreenParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const screens = await getScreens(appDb)
for (const screen of screens) {
await events.screen.created(screen, timestamp)
}
return screens.length
}

13
packages/server/src/migrations/functions/backfill/app/tables.ts
View File

@ -1,13 +0,0 @@
import { events } from "@budibase/backend-core"
import { Database } from "@budibase/types"
import sdk from "../../../../sdk"
export const backfill = async (appDb: Database, timestamp: string | number) => {
const tables = await sdk.tables.getAllInternalTables(appDb)
for (const table of tables) {
await events.table.created(table, timestamp)
}
return tables.length
}

214
packages/server/src/migrations/functions/backfill/global.ts
View File

@ -1,214 +0,0 @@
import * as users from "./global/users"
import * as configs from "./global/configs"
import * as quotas from "./global/quotas"
import {
tenancy,
events,
migrations,
accounts,
db as dbUtils,
} from "@budibase/backend-core"
import {
App,
CloudAccount,
Event,
Hosting,
QuotaUsage,
TenantBackfillSucceededEvent,
User,
} from "@budibase/types"
import env from "../../../environment"
import { DEFAULT_TIMESTAMP } from "."
const failGraceful = env.SELF_HOSTED && !env.isDev()
const handleError = (e: any, errors?: any) => {
if (failGraceful) {
if (errors) {
errors.push(e)
}
return
}
throw e
}
const formatUsage = (usage: QuotaUsage) => {
let maxAutomations = 0
let maxQueries = 0
let rows = 0
if (usage) {
if (usage.usageQuota) {
rows = usage.usageQuota.rows
}
if (usage.monthly) {
for (const value of Object.values(usage.monthly)) {
if (value.automations > maxAutomations) {
maxAutomations = value.automations
}
if (value.queries > maxQueries) {
maxQueries = value.queries
}
}
}
}
return {
maxAutomations,
maxQueries,
rows,
}
}
const EVENTS = [
Event.EMAIL_SMTP_CREATED,
Event.AUTH_SSO_CREATED,
Event.AUTH_SSO_ACTIVATED,
Event.ORG_NAME_UPDATED,
Event.ORG_LOGO_UPDATED,
Event.ORG_PLATFORM_URL_UPDATED,
Event.USER_CREATED,
Event.USER_PERMISSION_ADMIN_ASSIGNED,
Event.USER_PERMISSION_BUILDER_ASSIGNED,
Event.ROLE_ASSIGNED,
Event.ROWS_CREATED,
Event.QUERIES_RUN,
Event.AUTOMATIONS_RUN,
]
/**
* Date:
* May 2022
*
* Description:
* Backfill global events.
*/
export const run = async (db: any) => {
try {
const tenantId = tenancy.getTenantId()
let timestamp: string | number = DEFAULT_TIMESTAMP
const totals: any = {}
const errors: any = []
let allUsers: User[] = []
try {
allUsers = await users.getUsers(db)
} catch (e: any) {
handleError(e, errors)
}
if (!allUsers || allUsers.length === 0) {
// first time startup - we don't need to backfill anything
// tenant will be identified when admin user is created
if (env.SELF_HOSTED) {
await events.installation.firstStartup()
}
return
}
try {
const installTimestamp = await getInstallTimestamp(db, allUsers)
if (installTimestamp) {
timestamp = installTimestamp
}
} catch (e) {
handleError(e, errors)
}
let account: CloudAccount | undefined
if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
account = await accounts.getAccountByTenantId(tenantId)
}
try {
await events.identification.identifyTenantGroup(
tenantId,
env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD,
timestamp
)
} catch (e) {
handleError(e, errors)
}
// tell the event pipeline to start caching
// events for this tenant
await events.backfillCache.start(EVENTS)
try {
await configs.backfill(db, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.users = await users.backfill(db, account)
} catch (e) {
handleError(e, errors)
}
try {
const allApps = (await dbUtils.getAllApps({ dev: true })) as App[]
totals.apps = allApps.length
totals.usage = await quotas.backfill(allApps)
} catch (e) {
handleError(e, errors)
}
const properties: TenantBackfillSucceededEvent = {
apps: totals.apps,
users: totals.users,
...formatUsage(totals.usage),
usage: totals.usage,
}
if (errors.length) {
properties.errors = errors.map((e: any) =>
JSON.stringify(e, Object.getOwnPropertyNames(e))
)
properties.errorCount = errors.length
} else {
properties.errorCount = 0
}
await events.backfill.tenantSucceeded(properties)
// tell the event pipeline to stop caching events for this tenant
await events.backfillCache.end()
} catch (e) {
handleError(e)
await events.backfill.tenantFailed(e)
}
}
export const isComplete = async (): Promise<boolean> => {
const globalDb = tenancy.getGlobalDB()
const migrationsDoc = await migrations.getMigrationsDoc(globalDb)
return !!migrationsDoc.event_global_backfill
}
export const getInstallTimestamp = async (
globalDb: any,
allUsers?: User[]
): Promise<number | undefined> => {
if (!allUsers) {
allUsers = await users.getUsers(globalDb)
}
// get the oldest user timestamp
if (allUsers) {
const timestamps = allUsers
.map(user => user.createdAt)
.filter(timestamp => !!timestamp)
.sort(
(a, b) =>
new Date(a as number).getTime() - new Date(b as number).getTime()
)
if (timestamps.length) {
return timestamps[0]
}
}
}

74
packages/server/src/migrations/functions/backfill/global/configs.ts
View File

@ -1,74 +0,0 @@
import {
events,
DocumentType,
SEPARATOR,
UNICODE_MAX,
} from "@budibase/backend-core"
import {
Config,
isSMTPConfig,
isGoogleConfig,
isOIDCConfig,
isSettingsConfig,
ConfigType,
DatabaseQueryOpts,
} from "@budibase/types"
import env from "./../../../../environment"
export function getConfigParams(): DatabaseQueryOpts {
return {
include_docs: true,
startkey: `${DocumentType.CONFIG}${SEPARATOR}`,
endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`,
}
}
const getConfigs = async (globalDb: any): Promise<Config[]> => {
const response = await globalDb.allDocs(getConfigParams())
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (
globalDb: any,
timestamp: string | number | undefined
) => {
const configs = await getConfigs(globalDb)
for (const config of configs) {
if (isSMTPConfig(config)) {
await events.email.SMTPCreated(timestamp)
}
if (isGoogleConfig(config)) {
await events.auth.SSOCreated(ConfigType.GOOGLE, timestamp)
if (config.config.activated) {
await events.auth.SSOActivated(ConfigType.GOOGLE, timestamp)
}
}
if (isOIDCConfig(config)) {
await events.auth.SSOCreated(ConfigType.OIDC, timestamp)
if (config.config.configs[0].activated) {
await events.auth.SSOActivated(ConfigType.OIDC, timestamp)
}
}
if (isSettingsConfig(config)) {
const company = config.config.company
if (company && company !== "Budibase") {
await events.org.nameUpdated(timestamp)
}
const logoUrl = config.config.logoUrl
if (logoUrl) {
await events.org.logoUpdated(timestamp)
}
const platformUrl = config.config.platformUrl
if (
platformUrl &&
platformUrl !== "http://localhost:10000" &&
env.SELF_HOSTED
) {
await events.org.platformURLUpdated(timestamp)
}
}
}
}

60
packages/server/src/migrations/functions/backfill/global/quotas.ts
View File

@ -1,60 +0,0 @@
import { DEFAULT_TIMESTAMP } from "./../index"
import { events } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { App } from "@budibase/types"
const getOldestCreatedAt = (allApps: App[]): string | undefined => {
const timestamps = allApps
.filter(app => !!app.createdAt)
.map(app => app.createdAt as string)
.sort((a, b) => new Date(a).getTime() - new Date(b).getTime())
if (timestamps.length) {
return timestamps[0]
}
}
const getMonthTimestamp = (monthString: string): number => {
const parts = monthString.split("-")
const month = parseInt(parts[0]) - 1 // we already do +1 in month string calculation
const year = parseInt(parts[1])
// using 0 as the day in next month gives us last day in previous month
const date = new Date(year, month + 1, 0).getTime()
const now = new Date().getTime()
if (date > now) {
return now
} else {
return date
}
}
export const backfill = async (allApps: App[]) => {
const usage = await quotas.getQuotaUsage()
const rows = usage.usageQuota.rows
let timestamp: string | number = DEFAULT_TIMESTAMP
const oldestAppTimestamp = getOldestCreatedAt(allApps)
if (oldestAppTimestamp) {
timestamp = oldestAppTimestamp
}
await events.rows.created(rows, timestamp)
for (const [monthString, quotas] of Object.entries(usage.monthly)) {
if (monthString === "current") {
continue
}
const monthTimestamp = getMonthTimestamp(monthString)
const queries = quotas.queries
await events.query.run(queries, monthTimestamp)
const automations = quotas.automations
await events.automation.run(automations, monthTimestamp)
}
return usage
}

53
packages/server/src/migrations/functions/backfill/global/users.ts
View File

@ -1,53 +0,0 @@
import {
events,
db as dbUtils,
users as usersCore,
} from "@budibase/backend-core"
import { User, CloudAccount } from "@budibase/types"
import { DEFAULT_TIMESTAMP } from ".."
// manually define user doc params - normally server doesn't read users from the db
const getUserParams = (props: any) => {
return dbUtils.getDocParams(dbUtils.DocumentType.USER, null, props)
}
export const getUsers = async (globalDb: any): Promise<User[]> => {
const response = await globalDb.allDocs(
getUserParams({
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (
globalDb: any,
account: CloudAccount | undefined
) => {
const users = await getUsers(globalDb)
for (const user of users) {
let timestamp: string | number = DEFAULT_TIMESTAMP
if (user.createdAt) {
timestamp = user.createdAt
}
await events.identification.identifyUser(user, account, timestamp)
await events.user.created(user, timestamp)
if (usersCore.hasAdminPermissions(user)) {
await events.user.permissionAdminAssigned(user, timestamp)
}
if (usersCore.hasBuilderPermissions(user)) {
await events.user.permissionBuilderAssigned(user, timestamp)
}
if (user.roles) {
for (const [, role] of Object.entries(user.roles)) {
await events.role.assigned(user, role, timestamp)
}
}
}
return users.length
}

7
packages/server/src/migrations/functions/backfill/index.ts
View File

@ -1,7 +0,0 @@
export * as app from "./app"
export * as global from "./global"
export * as installation from "./installation"
// historical events are free in posthog - make sure we default to a
// historical time if no other can be found
export const DEFAULT_TIMESTAMP = new Date(2022, 0, 1).getTime()

50
packages/server/src/migrations/functions/backfill/installation.ts
View File

@ -1,50 +0,0 @@
import { DEFAULT_TIMESTAMP } from "./index"
import { events, tenancy, installation } from "@budibase/backend-core"
import { Installation } from "@budibase/types"
import * as global from "./global"
import env from "../../../environment"
const failGraceful = env.SELF_HOSTED
const handleError = (e: any, errors?: any) => {
if (failGraceful) {
if (errors) {
errors.push(e)
}
return
}
throw e
}
/**
* Date:
* May 2022
*
* Description:
* Backfill installation events.
*/
export const run = async () => {
try {
// need to use the default tenant to try to get the installation time
await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => {
const db = tenancy.getGlobalDB()
let timestamp: string | number = DEFAULT_TIMESTAMP
const installTimestamp = await global.getInstallTimestamp(db)
if (installTimestamp) {
timestamp = installTimestamp
}
const install: Installation = await installation.getInstall()
await events.identification.identifyInstallationGroup(
install.installId,
timestamp
)
})
await events.backfill.installationSucceeded()
} catch (e) {
handleError(e)
await events.backfill.installationFailed(e)
}
}

19
packages/server/src/migrations/functions/syncQuotas.ts
View File

@ -1,19 +0,0 @@
import { runQuotaMigration } from "./usageQuotas"
import * as syncApps from "./usageQuotas/syncApps"
import * as syncRows from "./usageQuotas/syncRows"
import * as syncPlugins from "./usageQuotas/syncPlugins"
import * as syncUsers from "./usageQuotas/syncUsers"
import * as syncCreators from "./usageQuotas/syncCreators"
/**
* Synchronise quotas to the state of the db.
*/
export const run = async () => {
await runQuotaMigration(async () => {
await syncApps.run()
await syncRows.run()
await syncPlugins.run()
await syncUsers.run()
await syncCreators.run()
})
}

145
packages/server/src/migrations/functions/tableSettings.ts
View File

@ -1,145 +0,0 @@
import { getScreenParams } from "../../db/utils"
import { Screen } from "@budibase/types"
import { makePropSafe as safe } from "@budibase/string-templates"
/**
* Date:
* November 2022
*
* Description:
* Update table settings to use actions instead of links. We do not remove the
* legacy values here as we cannot guarantee that their apps are up-t-date.
* It is safe to simply save both the new and old structure in the definition.
*
* Migration 1:
* Legacy "linkRows", "linkURL", "linkPeek" and "linkColumn" settings on tables
* and table blocks are migrated into a "Navigate To" action under the new
* "onClick" setting.
*
* Migration 2:
* Legacy "titleButtonURL" and "titleButtonPeek" settings on table blocks are
* migrated into a "Navigate To" action under the new "onClickTitleButton"
* setting.
*/
export const run = async (appDb: any) => {
// Get all app screens
let screens: Screen[]
try {
screens = (
await appDb.allDocs(
getScreenParams(null, {
include_docs: true,
})
)
).rows.map((row: any) => row.doc)
} catch (e) {
// sometimes the metadata document doesn't exist
// exit early instead of failing the migration
console.error("Error retrieving app metadata. Skipping", e)
return
}
// Recursively update any relevant components and mutate the screen docs
for (let screen of screens) {
const changed = migrateTableSettings(screen.props)
// Save screen if we updated it
if (changed) {
await appDb.put(screen)
console.log(
`Screen ${screen.routing?.route} contained table settings which were migrated`
)
}
}
}
// Recursively searches and mutates a screen doc to migrate table component
// and table block settings
const migrateTableSettings = (component: any) => {
let changed = false
if (!component) {
return changed
}
// Migration 1: migrate table row click settings
if (
component._component.endsWith("/table") ||
component._component.endsWith("/tableblock")
) {
const { linkRows, linkURL, linkPeek, linkColumn, onClick } = component
if (linkRows && !onClick) {
const column = linkColumn || "_id"
const action = convertLinkSettingToAction(linkURL, !!linkPeek, column)
if (action) {
changed = true
component.onClick = action
if (component._component.endsWith("/tableblock")) {
component.clickBehaviour = "actions"
}
}
}
}
// Migration 2: migrate table block title button settings
if (component._component.endsWith("/tableblock")) {
const {
showTitleButton,
titleButtonURL,
titleButtonPeek,
onClickTitleButton,
} = component
if (showTitleButton && !onClickTitleButton) {
const action = convertLinkSettingToAction(
titleButtonURL,
!!titleButtonPeek
)
if (action) {
changed = true
component.onClickTitleButton = action
component.titleButtonClickBehaviour = "actions"
}
}
}
// Recurse down the tree as needed
component._children?.forEach((child: any) => {
const childChanged = migrateTableSettings(child)
changed = changed || childChanged
})
return changed
}
// Util ti convert the legacy settings into a navigation action structure
const convertLinkSettingToAction = (
linkURL: string,
linkPeek: boolean,
linkColumn?: string
) => {
// Sanity check we have a URL
if (!linkURL) {
return null
}
// Default URL to the old URL setting
let url = linkURL
// If we enriched the old URL with a column, update the url
if (linkColumn && linkURL.includes("/:")) {
// Convert old link URL setting, which is a screen URL, into a valid
// binding using the new clicked row binding
const split = linkURL.split("/:")
const col = linkColumn || "_id"
const binding = `{{ ${safe("eventContext")}.${safe("row")}.${safe(col)} }}`
url = `${split[0]}/${binding}`
}
// Create action structure
return [
{
"##eventHandlerType": "Navigate To",
parameters: {
url,
peek: linkPeek,
},
},
]
}

26
packages/server/src/migrations/functions/tests/appUrls.spec.js
View File

@ -1,26 +0,0 @@
const { db: dbCore } = require("@budibase/backend-core")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
const migration = require("../appUrls")
describe("run", () => {
let config = new TestConfig(false)
beforeAll(async () => {
await config.init()
})
afterAll(config.end)
it("runs successfully", async () => {
const app = await config.createApp("testApp")
const metadata = await dbCore.doWithDB(app.appId, async db => {
const metadataDoc = await db.get(dbCore.DocumentType.APP_METADATA)
delete metadataDoc.url
await db.put(metadataDoc)
await migration.run(db)
return await db.get(dbCore.DocumentType.APP_METADATA)
})
expect(metadata.url).toEqual("/testapp")
})
})

144
packages/server/src/migrations/functions/tests/tableSettings.spec.ts
View File

@ -1,144 +0,0 @@
import { App, Screen } from "@budibase/types"
import { db as dbCore } from "@budibase/backend-core"
import TestConfig from "../../../tests/utilities/TestConfiguration"
import { run as runMigration } from "../tableSettings"
describe("run", () => {
const config = new TestConfig(false)
let app: App
let screen: Screen
beforeAll(async () => {
await config.init()
app = await config.createApp("testApp")
screen = await config.createScreen()
})
afterAll(config.end)
it("migrates table block row on click settings", async () => {
// Add legacy table block as first child
screen.props._children = [
{
_instanceName: "Table Block",
_styles: {},
_component: "@budibase/standard-components/tableblock",
_id: "foo",
linkRows: true,
linkURL: "/rows/:id",
linkPeek: true,
linkColumn: "name",
},
]
await config.createScreen(screen)
// Run migration
screen = await dbCore.doWithDB(app.appId, async (db: any) => {
await runMigration(db)
return await db.get(screen._id)
})
// Verify new "onClick" setting
const onClick = screen.props._children?.[0].onClick
expect(onClick).toBeDefined()
expect(onClick.length).toBe(1)
expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
expect(onClick[0].parameters.url).toBe(
`/rows/{{ [eventContext].[row].[name] }}`
)
expect(onClick[0].parameters.peek).toBeTruthy()
})
it("migrates table row on click settings", async () => {
// Add legacy table block as first child
screen.props._children = [
{
_instanceName: "Table",
_styles: {},
_component: "@budibase/standard-components/table",
_id: "foo",
linkRows: true,
linkURL: "/rows/:id",
linkPeek: true,
linkColumn: "name",
},
]
await config.createScreen(screen)
// Run migration
screen = await dbCore.doWithDB(app.appId, async (db: any) => {
await runMigration(db)
return await db.get(screen._id)
})
// Verify new "onClick" setting
const onClick = screen.props._children?.[0].onClick
expect(onClick).toBeDefined()
expect(onClick.length).toBe(1)
expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
expect(onClick[0].parameters.url).toBe(
`/rows/{{ [eventContext].[row].[name] }}`
)
expect(onClick[0].parameters.peek).toBeTruthy()
})
it("migrates table block title button settings", async () => {
// Add legacy table block as first child
screen.props._children = [
{
_instanceName: "Table Block",
_styles: {},
_component: "@budibase/standard-components/tableblock",
_id: "foo",
showTitleButton: true,
titleButtonURL: "/url",
titleButtonPeek: true,
},
]
await config.createScreen(screen)
// Run migration
screen = await dbCore.doWithDB(app.appId, async (db: any) => {
await runMigration(db)
return await db.get(screen._id)
})
// Verify new "onClickTitleButton" setting
const onClick = screen.props._children?.[0].onClickTitleButton
expect(onClick).toBeDefined()
expect(onClick.length).toBe(1)
expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
expect(onClick[0].parameters.url).toBe("/url")
expect(onClick[0].parameters.peek).toBeTruthy()
})
it("ignores components that have already been migrated", async () => {
// Add legacy table block as first child
screen.props._children = [
{
_instanceName: "Table Block",
_styles: {},
_component: "@budibase/standard-components/tableblock",
_id: "foo",
linkRows: true,
linkURL: "/rows/:id",
linkPeek: true,
linkColumn: "name",
onClick: "foo",
},
]
const initialDefinition = JSON.stringify(screen.props._children?.[0])
await config.createScreen(screen)
// Run migration
screen = await dbCore.doWithDB(app.appId, async (db: any) => {
await runMigration(db)
return await db.get(screen._id)
})
// Verify new "onClick" setting
const newDefinition = JSON.stringify(screen.props._children?.[0])
expect(initialDefinition).toEqual(newDefinition)
})
})

34
packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js
View File

@ -1,34 +0,0 @@
jest.mock("@budibase/backend-core", () => {
const core = jest.requireActual("@budibase/backend-core")
return {
...core,
db: {
...core.db,
createNewUserEmailView: jest.fn(),
},
}
})
const { context, db: dbCore } = require("@budibase/backend-core")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
// mock email view creation
const migration = require("../userEmailViewCasing")
describe("run", () => {
let config = new TestConfig(false)
beforeAll(async () => {
await config.init()
})
afterAll(config.end)
it("runs successfully", async () => {
await config.doInTenant(async () => {
const globalDb = context.getGlobalDB()
await migration.run(globalDb)
expect(dbCore.createNewUserEmailView).toHaveBeenCalledTimes(1)
})
})
})

3
packages/server/src/migrations/functions/usageQuotas/index.ts
View File

@ -1,3 +0,0 @@
export const runQuotaMigration = async (migration: () => Promise<void>) => {
await migration()
}

13
packages/server/src/migrations/functions/usageQuotas/syncApps.ts
View File

@ -1,13 +0,0 @@
import { db as dbCore } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
export const run = async () => {
// get app count
const devApps = await dbCore.getAllApps({ dev: true })
const appCount = devApps ? devApps.length : 0
// sync app count
console.log(`Syncing app count: ${appCount}`)
await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
}

13
packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
View File

@ -1,13 +0,0 @@
import { users } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
export const run = async () => {
const creatorCount = await users.getCreatorCount()
console.log(`Syncing creator count: ${creatorCount}`)
await quotas.setUsage(
creatorCount,
StaticQuotaName.CREATORS,
QuotaUsageType.STATIC
)
}

10
packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts
View File

@ -1,10 +0,0 @@
import { logging } from "@budibase/backend-core"
import { plugins } from "@budibase/pro"
export const run = async () => {
try {
await plugins.checkPluginQuotas()
} catch (err) {
logging.logAlert("Failed to update plugin quotas", err)
}
}

27
packages/server/src/migrations/functions/usageQuotas/syncRows.ts
View File

@ -1,27 +0,0 @@
import { db as dbCore } from "@budibase/backend-core"
import { getUniqueRows } from "../../../utilities/usageQuota/rows"
import { quotas } from "@budibase/pro"
import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types"
export const run = async () => {
// get all rows in all apps
const allApps = (await dbCore.getAllApps({ all: true })) as App[]
const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
const { appRows } = await getUniqueRows(appIds)
// get the counts per app
const counts: { [key: string]: number } = {}
let rowCount = 0
Object.entries(appRows).forEach(([appId, rows]) => {
counts[appId] = rows.length
rowCount += rows.length
})
// sync row count
console.log(`Syncing row count: ${rowCount}`)
await quotas.setUsagePerApp(
counts,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
}

9
packages/server/src/migrations/functions/usageQuotas/syncUsers.ts
View File

@ -1,9 +0,0 @@
import { users } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
export const run = async () => {
const userCount = await users.getUserCount()
console.log(`Syncing user count: ${userCount}`)
await quotas.setUsage(userCount, StaticQuotaName.USERS, QuotaUsageType.STATIC)
}

35
packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts
View File

@ -1,35 +0,0 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import * as syncApps from "../syncApps"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
describe("syncApps", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
})
afterAll(config.end)
it("runs successfully", async () => {
return config.doInContext(undefined, async () => {
// create the usage quota doc and mock usages
await quotas.getQuotaUsage()
await quotas.setUsage(3, StaticQuotaName.APPS, QuotaUsageType.STATIC)
let usageDoc = await quotas.getQuotaUsage()
expect(usageDoc.usageQuota.apps).toEqual(3)
// create an extra app to test the migration
await config.createApp("quota-test")
// migrate
await syncApps.run()
// assert the migration worked
usageDoc = await quotas.getQuotaUsage()
expect(usageDoc.usageQuota.apps).toEqual(2)
})
})
})

26
packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts
View File

@ -1,26 +0,0 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import * as syncCreators from "../syncCreators"
import { quotas } from "@budibase/pro"
describe("syncCreators", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
})
afterAll(config.end)
it("syncs creators", async () => {
return config.doInContext(undefined, async () => {
await config.createUser({ admin: { global: true } })
await syncCreators.run()
const usageDoc = await quotas.getQuotaUsage()
// default + additional creator
const creatorsCount = 2
expect(usageDoc.usageQuota.creators).toBe(creatorsCount)
})
})
})

53
packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
View File

@ -1,53 +0,0 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import * as syncRows from "../syncRows"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
import { db as dbCore, context } from "@budibase/backend-core"
describe("syncRows", () => {
const config = new TestConfig()
beforeEach(async () => {
await config.init()
})
afterAll(config.end)
it("runs successfully", async () => {
return config.doInContext(undefined, async () => {
// create the usage quota doc and mock usages
await quotas.getQuotaUsage()
await quotas.setUsage(300, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
let usageDoc = await quotas.getQuotaUsage()
expect(usageDoc.usageQuota.rows).toEqual(300)
// app 1
const app1 = config.app
await context.doInAppContext(app1!.appId, async () => {
await config.createTable()
await config.createRow()
})
// app 2
const app2 = await config.createApp("second-app")
await context.doInAppContext(app2.appId, async () => {
await config.createTable()
await config.createRow()
await config.createRow()
})
// migrate
await syncRows.run()
// assert the migration worked
usageDoc = await quotas.getQuotaUsage()
expect(usageDoc.usageQuota.rows).toEqual(3)
expect(
usageDoc.apps?.[dbCore.getProdAppID(app1!.appId)].usageQuota.rows
).toEqual(1)
expect(
usageDoc.apps?.[dbCore.getProdAppID(app2.appId)].usageQuota.rows
).toEqual(2)
})
})
})

26
packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts
View File

@ -1,26 +0,0 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import * as syncUsers from "../syncUsers"
import { quotas } from "@budibase/pro"
describe("syncUsers", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
})
afterAll(config.end)
it("syncs users", async () => {
return config.doInContext(undefined, async () => {
await config.createUser()
await syncUsers.run()
const usageDoc = await quotas.getQuotaUsage()
// default + additional user
const userCount = 2
expect(usageDoc.usageQuota.users).toBe(userCount)
})
})
})

13
packages/server/src/migrations/functions/userEmailViewCasing.ts
View File

@ -1,13 +0,0 @@
import { db as dbCore } from "@budibase/backend-core"
/**
* Date:
* October 2021
*
* Description:
* Recreate the user email view to include latest changes i.e. lower casing the email address
*/
export const run = async () => {
await dbCore.createNewUserEmailView()
}

115
packages/server/src/migrations/index.ts
View File

@ -1,115 +0,0 @@
import { locks, migrations } from "@budibase/backend-core"
import {
Migration,
MigrationOptions,
MigrationName,
LockType,
LockName,
} from "@budibase/types"
import env from "../environment"
// migration functions
import * as userEmailViewCasing from "./functions/userEmailViewCasing"
import * as syncQuotas from "./functions/syncQuotas"
import * as appUrls from "./functions/appUrls"
import * as tableSettings from "./functions/tableSettings"
import * as backfill from "./functions/backfill"
/**
* Populate the migration function and additional configuration from
* the static migration definitions.
*/
export const buildMigrations = () => {
const definitions = migrations.DEFINITIONS
const serverMigrations: Migration[] = []
for (const definition of definitions) {
switch (definition.name) {
case MigrationName.USER_EMAIL_VIEW_CASING: {
serverMigrations.push({
...definition,
fn: userEmailViewCasing.run,
})
break
}
case MigrationName.SYNC_QUOTAS: {
serverMigrations.push({
...definition,
fn: syncQuotas.run,
})
break
}
case MigrationName.APP_URLS: {
serverMigrations.push({
...definition,
appOpts: { all: true },
fn: appUrls.run,
})
break
}
case MigrationName.EVENT_APP_BACKFILL: {
serverMigrations.push({
...definition,
appOpts: { all: true },
fn: backfill.app.run,
silent: !!env.SELF_HOSTED, // reduce noisy logging
preventRetry: !!env.SELF_HOSTED, // only ever run once
})
break
}
case MigrationName.EVENT_GLOBAL_BACKFILL: {
serverMigrations.push({
...definition,
fn: backfill.global.run,
silent: !!env.SELF_HOSTED, // reduce noisy logging
preventRetry: !!env.SELF_HOSTED, // only ever run once
})
break
}
case MigrationName.EVENT_INSTALLATION_BACKFILL: {
serverMigrations.push({
...definition,
fn: backfill.installation.run,
silent: !!env.SELF_HOSTED, // reduce noisy logging
preventRetry: !!env.SELF_HOSTED, // only ever run once
})
break
}
case MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS: {
serverMigrations.push({
...definition,
appOpts: { dev: true },
fn: tableSettings.run,
})
break
}
}
}
return serverMigrations
}
export const MIGRATIONS = buildMigrations()
export const migrate = async (options?: MigrationOptions) => {
if (env.SELF_HOSTED) {
// self host runs migrations on startup
// make sure only a single instance runs them
await migrateWithLock(options)
} else {
await migrations.runMigrations(MIGRATIONS, options)
}
}
const migrateWithLock = async (options?: MigrationOptions) => {
await locks.doWithLock(
{
type: LockType.TRY_ONCE,
name: LockName.MIGRATIONS,
ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
systemLock: true,
},
async () => {
await migrations.runMigrations(MIGRATIONS, options)
}
)
}

40
packages/server/src/migrations/tests/helpers.ts
View File

@ -1,40 +0,0 @@
// Mimic configs test configuration from worker, creation configs directly in database
import * as structures from "./structures"
import { configs } from "@budibase/backend-core"
import { Config } from "@budibase/types"
export const saveSettingsConfig = async (globalDb: any) => {
const config = structures.settings()
await saveConfig(config, globalDb)
}
export const saveGoogleConfig = async (globalDb: any) => {
const config = structures.google()
await saveConfig(config, globalDb)
}
export const saveOIDCConfig = async (globalDb: any) => {
const config = structures.oidc()
await saveConfig(config, globalDb)
}
export const saveSmtpConfig = async (globalDb: any) => {
const config = structures.smtp()
await saveConfig(config, globalDb)
}
const saveConfig = async (config: Config, globalDb: any) => {
config._id = configs.generateConfigID(config.type)
let response
try {
response = await globalDb.get(config._id)
config._rev = response._rev
await globalDb.put(config)
} catch (e: any) {
if (e.status === 404) {
await globalDb.put(config)
}
}
}

137
packages/server/src/migrations/tests/index.spec.ts
View File

@ -1,137 +0,0 @@
import {
events,
migrations,
tenancy,
DocumentType,
context,
} from "@budibase/backend-core"
import TestConfig from "../../tests/utilities/TestConfiguration"
import * as structures from "../../tests/utilities/structures"
import { MIGRATIONS } from "../"
import * as helpers from "./helpers"
import tk from "timekeeper"
import { View } from "@budibase/types"
const timestamp = new Date().toISOString()
tk.freeze(timestamp)
const clearMigrations = async () => {
const dbs = [context.getDevAppDB(), context.getProdAppDB()]
for (const db of dbs) {
const doc = await db.get<any>(DocumentType.MIGRATIONS)
const newDoc = { _id: doc._id, _rev: doc._rev }
await db.put(newDoc)
}
}
describe("migrations", () => {
const config = new TestConfig()
beforeAll(async () => {
await config.init()
})
afterAll(() => {
config.end()
})
describe("backfill", () => {
it("runs app db migration", async () => {
await config.doInContext(undefined, async () => {
await clearMigrations()
await config.createAutomation()
await config.createAutomation(structures.newAutomation())
await config.createDatasource()
await config.createDatasource()
await config.createLayout()
await config.createQuery()
await config.createQuery()
await config.createRole()
await config.createRole()
await config.createTable()
await config.createLegacyView()
await config.createTable()
await config.createLegacyView(
structures.view(config.table!._id!) as View
)
await config.createScreen()
await config.createScreen()
jest.clearAllMocks()
const migration = MIGRATIONS.filter(
m => m.name === "event_app_backfill"
)[0]
await migrations.runMigration(migration)
expect(events.app.created).toHaveBeenCalledTimes(1)
expect(events.app.published).toHaveBeenCalledTimes(1)
expect(events.automation.created).toHaveBeenCalledTimes(2)
expect(events.automation.stepCreated).toHaveBeenCalledTimes(1)
expect(events.datasource.created).toHaveBeenCalledTimes(2)
expect(events.layout.created).toHaveBeenCalledTimes(1)
expect(events.query.created).toHaveBeenCalledTimes(2)
expect(events.role.created).toHaveBeenCalledTimes(3) // created roles + admin (created on table creation)
expect(events.table.created).toHaveBeenCalledTimes(3)
expect(events.backfill.appSucceeded).toHaveBeenCalledTimes(2)
// to make sure caching is working as expected
expect(
events.processors.analyticsProcessor.processEvent
).toHaveBeenCalledTimes(20) // Addition of of the events above
})
})
})
it("runs global db migration", async () => {
await config.doInContext(undefined, async () => {
await clearMigrations()
const appId = config.getProdAppId()
const roles = { [appId]: "role_12345" }
await config.createUser({
builder: { global: false },
admin: { global: true },
roles,
}) // admin only
await config.createUser({
builder: { global: false },
admin: { global: false },
roles,
}) // non admin non builder
await config.createTable()
await config.createRow()
await config.createRow()
const db = tenancy.getGlobalDB()
await helpers.saveGoogleConfig(db)
await helpers.saveOIDCConfig(db)
await helpers.saveSettingsConfig(db)
await helpers.saveSmtpConfig(db)
jest.clearAllMocks()
const migration = MIGRATIONS.filter(
m => m.name === "event_global_backfill"
)[0]
await migrations.runMigration(migration)
expect(events.user.created).toHaveBeenCalledTimes(3)
expect(events.role.assigned).toHaveBeenCalledTimes(2)
expect(events.user.permissionBuilderAssigned).toHaveBeenCalledTimes(1) // default test user
expect(events.user.permissionAdminAssigned).toHaveBeenCalledTimes(1) // admin from above
expect(events.rows.created).toHaveBeenCalledTimes(1)
expect(events.rows.created).toHaveBeenCalledWith(2, timestamp)
expect(events.email.SMTPCreated).toHaveBeenCalledTimes(1)
expect(events.auth.SSOCreated).toHaveBeenCalledTimes(2)
expect(events.auth.SSOActivated).toHaveBeenCalledTimes(2)
expect(events.org.logoUpdated).toHaveBeenCalledTimes(1)
expect(events.org.nameUpdated).toHaveBeenCalledTimes(1)
expect(events.org.platformURLUpdated).toHaveBeenCalledTimes(1)
expect(events.backfill.tenantSucceeded).toHaveBeenCalledTimes(1)
// to make sure caching is working as expected
expect(
events.processors.analyticsProcessor.processEvent
).toHaveBeenCalledTimes(19)
})
})
})

67
packages/server/src/migrations/tests/structures.ts
View File

@ -1,67 +0,0 @@
import { utils } from "@budibase/backend-core"
import {
SMTPConfig,
OIDCConfig,
GoogleConfig,
SettingsConfig,
ConfigType,
} from "@budibase/types"
export const oidc = (conf?: OIDCConfig): OIDCConfig => {
return {
type: ConfigType.OIDC,
config: {
configs: [
{
configUrl: "http://someconfigurl",
clientID: "clientId",
clientSecret: "clientSecret",
logo: "Microsoft",
name: "Active Directory",
uuid: utils.newid(),
activated: true,
scopes: [],
...conf,
},
],
},
}
}
export const google = (conf?: GoogleConfig): GoogleConfig => {
return {
type: ConfigType.GOOGLE,
config: {
clientID: "clientId",
clientSecret: "clientSecret",
activated: true,
...conf,
},
}
}
export const smtp = (conf?: SMTPConfig): SMTPConfig => {
return {
type: ConfigType.SMTP,
config: {
port: 12345,
host: "smtptesthost.com",
from: "testfrom@example.com",
subject: "Hello!",
secure: false,
...conf,
},
}
}
export const settings = (conf?: SettingsConfig): SettingsConfig => {
return {
type: ConfigType.SETTINGS,
config: {
platformUrl: "http://mycustomdomain.com",
logoUrl: "http://mylogourl,com",
company: "mycompany",
...conf,
},
}
}

13
packages/server/src/startup/index.ts
View File

@ -15,7 +15,6 @@ import { watch } from "../watch"
import * as automations from "../automations"
import * as fileSystem from "../utilities/fileSystem"
import { default as eventEmitter, init as eventInit } from "../events"
import * as migrations from "../migrations"
import * as bullboard from "../automations/bullboard"
import * as appMigrations from "../appMigrations/queue"
import * as pro from "@budibase/pro"
@ -106,18 +105,6 @@ export async function startup(
initialiseWebsockets(app, server)
}
// run migrations on startup if not done via http
// not recommended in a clustered environment
if (!env.HTTP_MIGRATIONS && !env.isTest()) {
console.log("Running migrations")
try {
await migrations.migrate()
} catch (e) {
logging.logAlert("Error performing migrations. Exiting.", e)
shutdown(server)
}
}
// monitor plugin directory if required
if (
env.SELF_HOSTED &&

1
packages/shared-core/src/automations/index.ts Normal file
View File

@ -0,0 +1 @@
export * as steps from "./steps/index"

47
packages/shared-core/src/automations/steps/bash.ts Normal file
View File

@ -0,0 +1,47 @@
import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Bash Scripting",
tagline: "Execute a bash command",
icon: "JourneyEvent",
description: "Run a bash script",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.EXECUTE_BASH,
inputs: {},
schema: {
inputs: {
properties: {
code: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.CODE,
title: "Code",
},
},
required: ["code"],
},
outputs: {
properties: {
stdout: {
type: AutomationIOType.STRING,
description: "Standard output of your bash command or script",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the command was successful",
},
},
required: ["stdout"],
},
},
}

0
packages/server/src/automations/steps/branch.ts → packages/shared-core/src/automations/steps/branch.ts
View File

43
packages/shared-core/src/automations/steps/collect.ts Normal file
View File

@ -0,0 +1,43 @@
import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Collect Data",
tagline: "Collect data to be sent to design",
icon: "Collection",
description:
"Collects specified data so it can be provided to the design section",
type: AutomationStepType.ACTION,
internal: true,
features: {},
stepId: AutomationActionStepId.COLLECT,
inputs: {},
schema: {
inputs: {
properties: {
collection: {
type: AutomationIOType.STRING,
title: "What to Collect",
},
},
required: ["collection"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
value: {
type: AutomationIOType.STRING,
description: "Collected data",
},
},
required: ["success", "value"],
},
},
}

67
packages/shared-core/src/automations/steps/createRow.ts Normal file
View File

@ -0,0 +1,67 @@
import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Create Row",
tagline: "Create a {{inputs.enriched.table.name}} row",
icon: "TableRowAddBottom",
description: "Add a row to your database",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.CREATE_ROW,
inputs: {},
schema: {
inputs: {
properties: {
row: {
type: AutomationIOType.OBJECT,
properties: {
tableId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.TABLE,
},
},
customType: AutomationCustomIOType.ROW,
title: "Table",
required: ["tableId"],
},
},
required: ["row"],
},
outputs: {
properties: {
row: {
type: AutomationIOType.OBJECT,
customType: AutomationCustomIOType.ROW,
description: "The new row",
},
response: {
type: AutomationIOType.OBJECT,
description: "The response from the table",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the row creation was successful",
},
id: {
type: AutomationIOType.STRING,
description: "The identifier of the new row",
},
revision: {
type: AutomationIOType.STRING,
description: "The revision of the new row",
},
},
required: ["success", "id", "revision"],
},
},
}

Some files were not shown because too many files have changed in this diff Show More