MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into BUDI-8986/convert-screen-store

This commit is contained in:
Adria Navarro 2025-01-20 23:31:28 +01:00 committed by GitHub
parent 4671633bfe 1211811302
commit 048d56456c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
128 changed files with 1545 additions and 3920 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/backend-core/src/index.ts
View File

@ -1,6 +1,5 @@
export * as configs from "./configs" export * as configs from "./configs"
export * as events from "./events" export * as events from "./events"
export * as migrations from "./migrations"
export * as users from "./users" export * as users from "./users"
export * as userUtils from "./users/utils" export * as userUtils from "./users/utils"
export * as roles from "./security/roles" export * as roles from "./security/roles"

40
packages/backend-core/src/migrations/definitions.ts
View File

@ -1,40 +0,0 @@
import {
MigrationType,
MigrationName,
MigrationDefinition,
} from "@budibase/types"
export const DEFINITIONS: MigrationDefinition[] = [
{
type: MigrationType.GLOBAL,
name: MigrationName.USER_EMAIL_VIEW_CASING,
},
{
type: MigrationType.GLOBAL,
name: MigrationName.SYNC_QUOTAS,
},
{
type: MigrationType.APP,
name: MigrationName.APP_URLS,
},
{
type: MigrationType.APP,
name: MigrationName.EVENT_APP_BACKFILL,
},
{
type: MigrationType.APP,
name: MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS,
},
{
type: MigrationType.GLOBAL,
name: MigrationName.EVENT_GLOBAL_BACKFILL,
},
{
type: MigrationType.INSTALLATION,
name: MigrationName.EVENT_INSTALLATION_BACKFILL,
},
{
type: MigrationType.GLOBAL,
name: MigrationName.GLOBAL_INFO_SYNC_USERS,
},
]

2
packages/backend-core/src/migrations/index.ts
View File

@ -1,2 +0,0 @@
export * from "./migrations"
export * from "./definitions"

186
packages/backend-core/src/migrations/migrations.ts
View File

@ -1,186 +0,0 @@
import { DEFAULT_TENANT_ID } from "../constants"
import {
DocumentType,
StaticDatabases,
getAllApps,
getGlobalDBName,
getDB,
} from "../db"
import environment from "../environment"
import * as platform from "../platform"
import * as context from "../context"
import { DEFINITIONS } from "."
import {
Migration,
MigrationOptions,
MigrationType,
MigrationNoOpOptions,
App,
} from "@budibase/types"
export const getMigrationsDoc = async (db: any) => {
// get the migrations doc
try {
return await db.get(DocumentType.MIGRATIONS)
} catch (err: any) {
if (err.status && err.status === 404) {
return { _id: DocumentType.MIGRATIONS }
} else {
throw err
}
}
}
export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => {
// filter migrations to the type and populate a no-op migration
const migrations: Migration[] = DEFINITIONS.filter(
def => def.type === opts.type
).map(d => ({ ...d, fn: async () => {} }))
await runMigrations(migrations, { noOp: opts })
}
export const runMigration = async (
migration: Migration,
options: MigrationOptions = {}
) => {
const migrationType = migration.type
const migrationName = migration.name
const silent = migration.silent
const log = (message: string) => {
if (!silent) {
console.log(message)
}
}
// get the db to store the migration in
let dbNames: string[]
if (migrationType === MigrationType.GLOBAL) {
dbNames = [getGlobalDBName()]
} else if (migrationType === MigrationType.APP) {
if (options.noOp) {
if (!options.noOp.appId) {
throw new Error("appId is required for noOp app migration")
}
dbNames = [options.noOp.appId]
} else {
const apps = (await getAllApps(migration.appOpts)) as App[]
dbNames = apps.map(app => app.appId)
}
} else if (migrationType === MigrationType.INSTALLATION) {
dbNames = [StaticDatabases.PLATFORM_INFO.name]
} else {
throw new Error(`Unrecognised migration type [${migrationType}]`)
}
const length = dbNames.length
let count = 0
// run the migration against each db
for (const dbName of dbNames) {
count++
const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
const db = getDB(dbName)
try {
const doc = await getMigrationsDoc(db)
// the migration has already been run
if (doc[migrationName]) {
// check for force
if (
options.force &&
options.force[migrationType] &&
options.force[migrationType].includes(migrationName)
) {
log(`[Migration: ${migrationName}] [DB: ${dbName}] Forcing`)
} else {
// no force, exit
return
}
}
// check if the migration is not a no-op
if (!options.noOp) {
log(
`[Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
)
if (migration.preventRetry) {
// eagerly set the completion date
// so that we never run this migration twice even upon failure
doc[migrationName] = Date.now()
const response = await db.put(doc)
doc._rev = response.rev
}
// run the migration
if (migrationType === MigrationType.APP) {
await context.doInAppContext(db.name, async () => {
await migration.fn(db)
})
} else {
await migration.fn(db)
}
log(`[Migration: ${migrationName}] [DB: ${dbName}] Complete`)
}
// mark as complete
doc[migrationName] = Date.now()
await db.put(doc)
} catch (err) {
console.error(
`[Migration: ${migrationName}] [DB: ${dbName}] Error: `,
err
)
throw err
}
}
}
export const runMigrations = async (
migrations: Migration[],
options: MigrationOptions = {}
) => {
let tenantIds
if (environment.MULTI_TENANCY) {
if (options.noOp) {
tenantIds = [options.noOp.tenantId]
} else if (!options.tenantIds || !options.tenantIds.length) {
// run for all tenants
tenantIds = await platform.tenants.getTenantIds()
} else {
tenantIds = options.tenantIds
}
} else {
// single tenancy
tenantIds = [DEFAULT_TENANT_ID]
}
if (tenantIds.length > 1) {
console.log(`Checking migrations for ${tenantIds.length} tenants`)
} else {
console.log("Checking migrations")
}
let count = 0
// for all tenants
for (const tenantId of tenantIds) {
count++
if (tenantIds.length > 1) {
console.log(`Progress [${count}/${tenantIds.length}]`)
}
// for all migrations
for (const migration of migrations) {
// run the migration
await context.doInTenant(
tenantId,
async () => await runMigration(migration, options)
)
}
}
console.log("Migrations complete")
}

11
packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap
View File

@ -1,11 +0,0 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`migrations should match snapshot 1`] = `
{
"_id": "migrations",
"_rev": "1-2f64479842a0513aa8b97f356b0b9127",
"createdAt": "2020-01-01T00:00:00.000Z",
"test": 1577836800000,
"updatedAt": "2020-01-01T00:00:00.000Z",
}
`;

64
packages/backend-core/src/migrations/tests/migrations.spec.ts
View File

@ -1,64 +0,0 @@
import { testEnv, DBTestConfiguration } from "../../../tests/extra"
import * as migrations from "../index"
import * as context from "../../context"
import { MigrationType } from "@budibase/types"
testEnv.multiTenant()
describe("migrations", () => {
const config = new DBTestConfiguration()
const migrationFunction = jest.fn()
const MIGRATIONS = [
{
type: MigrationType.GLOBAL,
name: "test" as any,
fn: migrationFunction,
},
]
beforeEach(() => {
config.newTenant()
})
afterEach(async () => {
jest.clearAllMocks()
})
const migrate = () => {
return migrations.runMigrations(MIGRATIONS, {
tenantIds: [config.tenantId],
})
}
it("should run a new migration", async () => {
await config.doInTenant(async () => {
await migrate()
expect(migrationFunction).toHaveBeenCalled()
const db = context.getGlobalDB()
const doc = await migrations.getMigrationsDoc(db)
expect(doc.test).toBeDefined()
})
})
it("should match snapshot", async () => {
await config.doInTenant(async () => {
await migrate()
const doc = await migrations.getMigrationsDoc(context.getGlobalDB())
expect(doc).toMatchSnapshot()
})
})
it("should skip a previously run migration", async () => {
await config.doInTenant(async () => {
const db = context.getGlobalDB()
await migrate()
const previousDoc = await migrations.getMigrationsDoc(db)
await migrate()
const currentDoc = await migrations.getMigrationsDoc(db)
expect(migrationFunction).toHaveBeenCalledTimes(1)
expect(currentDoc.test).toBe(previousDoc.test)
})
})
})

22
packages/bbui/src/Icon/Icon.svelte
View File

@ -1,23 +1,23 @@
<script> <script lang="ts">
import { import {
default as AbsTooltip, default as AbsTooltip,
TooltipPosition, TooltipPosition,
TooltipType, TooltipType,
} from "../Tooltip/AbsTooltip.svelte" } from "../Tooltip/AbsTooltip.svelte"
export let name = "Add" export let name: string = "Add"
export let hidden = false export let hidden: boolean = false
export let size = "M" export let size = "M"
export let hoverable = false export let hoverable: boolean = false
export let disabled = false export let disabled: boolean = false
export let color = undefined export let color: string | undefined = undefined
export let hoverColor = undefined export let hoverColor: string | undefined = undefined
export let tooltip = undefined export let tooltip: string | undefined = undefined
export let tooltipPosition = TooltipPosition.Bottom export let tooltipPosition = TooltipPosition.Bottom
export let tooltipType = TooltipType.Default export let tooltipType = TooltipType.Default
export let tooltipColor = undefined export let tooltipColor: string | undefined = undefined
export let tooltipWrap = true export let tooltipWrap: boolean = true
export let newStyles = false export let newStyles: boolean = false
</script> </script>
<AbsTooltip <AbsTooltip

2
packages/bbui/src/Tooltip/AbsTooltip.svelte
View File

@ -23,7 +23,7 @@
export let type = TooltipType.Default export let type = TooltipType.Default
export let text = "" export let text = ""
export let fixed = false export let fixed = false
export let color = null export let color = ""
export let noWrap = false export let noWrap = false
let wrapper let wrapper

5
packages/client/src/components/app/DataProvider.svelte
View File

@ -14,7 +14,6 @@
GroupUserDatasource, GroupUserDatasource,
DataFetchOptions, DataFetchOptions,
} from "@budibase/types" } from "@budibase/types"
import { SDK, Component } from "../../index"
type ProviderDatasource = Exclude< type ProviderDatasource = Exclude<
DataFetchDatasource, DataFetchDatasource,
@ -29,8 +28,8 @@
export let paginate: boolean export let paginate: boolean
export let autoRefresh: number export let autoRefresh: number
const { styleable, Provider, ActionTypes, API } = getContext<SDK>("sdk") const { styleable, Provider, ActionTypes, API } = getContext("sdk")
const component = getContext<Component>("component") const component = getContext("component")
let interval: ReturnType<typeof setInterval> let interval: ReturnType<typeof setInterval>
let queryExtensions: Record<string, any> = {} let queryExtensions: Record<string, any> = {}

60
packages/client/src/components/app/blocks/form/FormBlock.svelte
View File

@ -1,37 +1,43 @@
<script> <script lang="ts">
import { getContext } from "svelte" import { getContext } from "svelte"
import InnerFormBlock from "./InnerFormBlock.svelte" import InnerFormBlock from "./InnerFormBlock.svelte"
import { Utils } from "@budibase/frontend-core" import { Utils } from "@budibase/frontend-core"
import FormBlockWrapper from "./FormBlockWrapper.svelte" import FormBlockWrapper from "./FormBlockWrapper.svelte"
import { get } from "svelte/store" import { get } from "svelte/store"
import { TableSchema, UIDatasource } from "@budibase/types"
export let actionType type Field = { name: string; active: boolean }
export let dataSource
export let size export let actionType: string
export let disabled export let dataSource: UIDatasource
export let fields export let size: string
export let buttons export let disabled: boolean
export let buttonPosition export let fields: (Field | string)[]
export let title export let buttons: {
export let description "##eventHandlerType": string
export let rowId parameters: Record<string, string>
export let actionUrl }[]
export let noRowsMessage export let buttonPosition: "top" | "bottom"
export let notificationOverride export let title: string
export let buttonsCollapsed export let description: string
export let buttonsCollapsedText export let rowId: string
export let actionUrl: string
export let noRowsMessage: string
export let notificationOverride: boolean
export let buttonsCollapsed: boolean
export let buttonsCollapsedText: string
// Legacy // Legacy
export let showDeleteButton export let showDeleteButton: boolean
export let showSaveButton export let showSaveButton: boolean
export let saveButtonLabel export let saveButtonLabel: boolean
export let deleteButtonLabel export let deleteButtonLabel: boolean
const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk") const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk")
const component = getContext("component") const component = getContext("component")
const context = getContext("context") const context = getContext("context")
let schema let schema: TableSchema
$: fetchSchema(dataSource) $: fetchSchema(dataSource)
$: id = $component.id $: id = $component.id
@ -61,7 +67,7 @@
} }
} }
const convertOldFieldFormat = fields => { const convertOldFieldFormat = (fields: (Field | string)[]): Field[] => {
if (!fields) { if (!fields) {
return [] return []
} }
@ -82,11 +88,11 @@
}) })
} }
const getDefaultFields = (fields, schema) => { const getDefaultFields = (fields: Field[], schema: TableSchema) => {
if (!schema) { if (!schema) {
return [] return []
} }
let defaultFields = [] let defaultFields: Field[] = []
if (!fields || fields.length === 0) { if (!fields || fields.length === 0) {
Object.values(schema) Object.values(schema)
@ -101,15 +107,14 @@
return [...fields, ...defaultFields].filter(field => field.active) return [...fields, ...defaultFields].filter(field => field.active)
} }
const fetchSchema = async () => { const fetchSchema = async (datasource: UIDatasource) => {
schema = (await fetchDatasourceSchema(dataSource)) || {} schema = (await fetchDatasourceSchema(datasource)) || {}
} }
</script> </script>
<FormBlockWrapper {actionType} {dataSource} {rowId} {noRowsMessage}> <FormBlockWrapper {actionType} {dataSource} {rowId} {noRowsMessage}>
<InnerFormBlock <InnerFormBlock
{dataSource} {dataSource}
{actionUrl}
{actionType} {actionType}
{size} {size}
{disabled} {disabled}
@ -117,7 +122,6 @@
{title} {title}
{description} {description}
{schema} {schema}
{notificationOverride}
buttons={buttonsOrDefault} buttons={buttonsOrDefault}
buttonPosition={buttons ? buttonPosition : "top"} buttonPosition={buttons ? buttonPosition : "top"}
{buttonsCollapsed} {buttonsCollapsed}

8
packages/client/src/components/error-states/ComponentErrorState.svelte
View File

@ -1,11 +1,13 @@
<script> <script lang="ts">
import { getContext } from "svelte" import { getContext } from "svelte"
import { Icon } from "@budibase/bbui" import { Icon } from "@budibase/bbui"
import MissingRequiredSetting from "./MissingRequiredSetting.svelte" import MissingRequiredSetting from "./MissingRequiredSetting.svelte"
import MissingRequiredAncestor from "./MissingRequiredAncestor.svelte" import MissingRequiredAncestor from "./MissingRequiredAncestor.svelte"
export let missingRequiredSettings export let missingRequiredSettings:
export let missingRequiredAncestors | { key: string; label: string }[]
| undefined
export let missingRequiredAncestors: string[] | undefined
const component = getContext("component") const component = getContext("component")
const { styleable, builderStore } = getContext("sdk") const { styleable, builderStore } = getContext("sdk")

7
packages/client/src/context.d.ts vendored Normal file
View File

@ -0,0 +1,7 @@
import { Component, Context, SDK } from "."
declare module "svelte" {
export function getContext(key: "sdk"): SDK
export function getContext(key: "component"): Component
export function getContext(key: "context"): Context
}

8
packages/client/src/index.ts
View File

@ -7,9 +7,17 @@ export interface SDK {
styleable: any styleable: any
Provider: any Provider: any
ActionTypes: typeof ActionTypes ActionTypes: typeof ActionTypes
fetchDatasourceSchema: any
generateGoldenSample: any
builderStore: Readable<{
inBuilder: boolean
}>
} }
export type Component = Readable<{ export type Component = Readable<{
id: string id: string
styles: any styles: any
errorState: boolean
}> }>
export type Context = Readable<{}>

15
packages/client/src/utils/schema.ts
View File

@ -1,5 +1,6 @@
import { API } from "api" import { API } from "api"
import { DataFetchMap, DataFetchType } from "@budibase/frontend-core" import { DataFetchMap, DataFetchType } from "@budibase/frontend-core"
import { FieldType, TableSchema } from "@budibase/types"
/** /**
* Constructs a fetch instance for a given datasource. * Constructs a fetch instance for a given datasource.
@ -42,14 +43,14 @@ export const fetchDatasourceSchema = async <
} }
// Get the normal schema as long as we aren't wanting a form schema // Get the normal schema as long as we aren't wanting a form schema
let schema: any let schema: TableSchema | undefined
if (datasource?.type !== "query" || !options?.formSchema) { if (datasource?.type !== "query" || !options?.formSchema) {
schema = instance.getSchema(definition as any) schema = instance.getSchema(definition as any) as TableSchema
} else if ("parameters" in definition && definition.parameters?.length) { } else if ("parameters" in definition && definition.parameters?.length) {
schema = {} schema = {}
definition.parameters.forEach(param => { for (const param of definition.parameters) {
schema[param.name] = { ...param, type: "string" } schema[param.name] = { ...param, type: FieldType.STRING }
}) }
} }
if (!schema) { if (!schema) {
return null return null
@ -57,11 +58,11 @@ export const fetchDatasourceSchema = async <
// Strip hidden fields from views // Strip hidden fields from views
if (datasource.type === "viewV2") { if (datasource.type === "viewV2") {
Object.keys(schema).forEach(field => { for (const field of Object.keys(schema)) {
if (!schema[field].visible) { if (!schema[field].visible) {
delete schema[field] delete schema[field]
} }
}) }
} }
// Enrich schema with relationships if required // Enrich schema with relationships if required

4
packages/frontend-core/src/api/migrations.ts
View File

@ -1,8 +1,8 @@
import { GetOldMigrationStatus } from "@budibase/types" import { GetMigrationStatus } from "@budibase/types"
import { BaseAPIClient } from "./types" import { BaseAPIClient } from "./types"
export interface MigrationEndpoints { export interface MigrationEndpoints {
getMigrationStatus: () => Promise<GetOldMigrationStatus> getMigrationStatus: () => Promise<GetMigrationStatus>
} }
export const buildMigrationEndpoints = ( export const buildMigrationEndpoints = (

1
packages/server/scripts/dev/manage.js
View File

@ -43,7 +43,6 @@ async function init() {
BB_ADMIN_USER_EMAIL: "", BB_ADMIN_USER_EMAIL: "",
BB_ADMIN_USER_PASSWORD: "", BB_ADMIN_USER_PASSWORD: "",
PLUGINS_DIR: "", PLUGINS_DIR: "",
HTTP_MIGRATIONS: "0",
HTTP_LOGGING: "0", HTTP_LOGGING: "0",
VERSION: "0.0.0+local", VERSION: "0.0.0+local",
PASSWORD_MIN_LENGTH: "1", PASSWORD_MIN_LENGTH: "1",

9
packages/server/src/api/controllers/application.ts
View File

@ -27,7 +27,6 @@ import {
env as envCore, env as envCore,
ErrorCode, ErrorCode,
events, events,
migrations,
objectStore, objectStore,
roles, roles,
tenancy, tenancy,
@ -43,7 +42,6 @@ import { groups, licensing, quotas } from "@budibase/pro"
import { import {
App, App,
Layout, Layout,
MigrationType,
PlanType, PlanType,
Screen, Screen,
UserCtx, UserCtx,
@ -488,13 +486,6 @@ async function creationEvents(request: BBRequest<CreateAppRequest>, app: App) {
} }
async function appPostCreate(ctx: UserCtx<CreateAppRequest, App>, app: App) { async function appPostCreate(ctx: UserCtx<CreateAppRequest, App>, app: App) {
const tenantId = tenancy.getTenantId()
await migrations.backPopulateMigrations({
type: MigrationType.APP,
tenantId,
appId: app.appId,
})
await creationEvents(ctx.request, app) await creationEvents(ctx.request, app)
// app import, template creation and duplication // app import, template creation and duplication

28
packages/server/src/api/controllers/migrations.ts
View File

@ -1,35 +1,11 @@
import { context } from "@budibase/backend-core" import { context } from "@budibase/backend-core"
import { migrate as migrationImpl, MIGRATIONS } from "../../migrations" import { Ctx, GetMigrationStatus } from "@budibase/types"
import {
Ctx,
FetchOldMigrationResponse,
GetOldMigrationStatus,
RuneOldMigrationResponse,
RunOldMigrationRequest,
} from "@budibase/types"
import { import {
getAppMigrationVersion, getAppMigrationVersion,
getLatestEnabledMigrationId, getLatestEnabledMigrationId,
} from "../../appMigrations" } from "../../appMigrations"
export async function migrate( export async function getMigrationStatus(ctx: Ctx<void, GetMigrationStatus>) {
ctx: Ctx<RunOldMigrationRequest, RuneOldMigrationResponse>
) {
const options = ctx.request.body
// don't await as can take a while, just return
migrationImpl(options)
ctx.body = { message: "Migration started." }
}
export async function fetchDefinitions(
ctx: Ctx<void, FetchOldMigrationResponse>
) {
ctx.body = MIGRATIONS
}
export async function getMigrationStatus(
ctx: Ctx<void, GetOldMigrationStatus>
) {
const appId = context.getAppId() const appId = context.getAppId()
if (!appId) { if (!appId) {

147
packages/server/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap
View File

@ -1,147 +0,0 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`viewBuilder Calculate and filter creates a view with the calculation statistics and filter schema 1`] = `
{
"map": "function (doc) {
if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !(
doc["myField"] === undefined ||
doc["myField"] === null ||
doc["myField"] === "" ||
(Array.isArray(doc["myField"]) && doc["myField"].length === 0)
)) && (doc["age"] > 17)) {
emit(doc["_id"], doc["myField"]);
}
}",
"meta": {
"calculation": "stats",
"field": "myField",
"filters": [
{
"condition": "MT",
"key": "age",
"value": 17,
},
],
"groupBy": undefined,
"schema": {
"avg": {
"type": "number",
},
"count": {
"type": "number",
},
"field": {
"type": "string",
},
"max": {
"type": "number",
},
"min": {
"type": "number",
},
"sum": {
"type": "number",
},
"sumsqr": {
"type": "number",
},
},
"tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
},
"reduce": "_stats",
}
`;
exports[`viewBuilder Calculate creates a view with the calculation statistics schema 1`] = `
{
"map": "function (doc) {
if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !(
doc["myField"] === undefined ||
doc["myField"] === null ||
doc["myField"] === "" ||
(Array.isArray(doc["myField"]) && doc["myField"].length === 0)
)) ) {
emit(doc["_id"], doc["myField"]);
}
}",
"meta": {
"calculation": "stats",
"field": "myField",
"filters": [],
"groupBy": undefined,
"schema": {
"avg": {
"type": "number",
},
"count": {
"type": "number",
},
"field": {
"type": "string",
},
"max": {
"type": "number",
},
"min": {
"type": "number",
},
"sum": {
"type": "number",
},
"sumsqr": {
"type": "number",
},
},
"tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
},
"reduce": "_stats",
}
`;
exports[`viewBuilder Filter creates a view with multiple filters and conjunctions 1`] = `
{
"map": "function (doc) {
if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && (doc["Name"] === "Test" || doc["Yes"] > "Value")) {
emit(doc["_id"], doc["undefined"]);
}
}",
"meta": {
"calculation": undefined,
"field": undefined,
"filters": [
{
"condition": "EQUALS",
"key": "Name",
"value": "Test",
},
{
"condition": "MT",
"conjunction": "OR",
"key": "Yes",
"value": "Value",
},
],
"groupBy": undefined,
"schema": null,
"tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
},
}
`;
exports[`viewBuilder Group By creates a view emitting the group by field 1`] = `
{
"map": "function (doc) {
if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" ) {
emit(doc["age"], doc["score"]);
}
}",
"meta": {
"calculation": undefined,
"field": "score",
"filters": [],
"groupBy": "age",
"schema": null,
"tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
},
}
`;

75
packages/server/src/api/controllers/view/tests/viewBuilder.spec.js
View File

@ -1,75 +0,0 @@
const viewTemplate = require("../viewBuilder").default
describe("viewBuilder", () => {
describe("Filter", () => {
it("creates a view with multiple filters and conjunctions", () => {
expect(
viewTemplate({
name: "Test View",
tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
filters: [
{
value: "Test",
condition: "EQUALS",
key: "Name",
},
{
value: "Value",
condition: "MT",
key: "Yes",
conjunction: "OR",
},
],
})
).toMatchSnapshot()
})
})
describe("Calculate", () => {
it("creates a view with the calculation statistics schema", () => {
expect(
viewTemplate({
name: "Calculate View",
field: "myField",
calculation: "stats",
tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
filters: [],
})
).toMatchSnapshot()
})
})
describe("Group By", () => {
it("creates a view emitting the group by field", () => {
expect(
viewTemplate({
name: "Test Scores Grouped By Age",
tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
groupBy: "age",
field: "score",
filters: [],
})
).toMatchSnapshot()
})
})
describe("Calculate and filter", () => {
it("creates a view with the calculation statistics and filter schema", () => {
expect(
viewTemplate({
name: "Calculate View",
field: "myField",
calculation: "stats",
tableId: "14f1c4e94d6a47b682ce89d35d4c78b0",
filters: [
{
value: 17,
condition: "MT",
key: "age",
},
],
})
).toMatchSnapshot()
})
})
})

174
packages/server/src/api/controllers/view/tests/viewBuilder.spec.ts Normal file
View File

@ -0,0 +1,174 @@
import viewTemplate from "../viewBuilder"
describe("viewBuilder", () => {
describe("Filter", () => {
it("creates a view with multiple filters and conjunctions", () => {
expect(
viewTemplate({
field: "myField",
tableId: "tableId",
filters: [
{
value: "Test",
condition: "EQUALS",
key: "Name",
},
{
value: "Value",
condition: "MT",
key: "Yes",
conjunction: "OR",
},
],
})
).toEqual({
map: `function (doc) {
if (doc.tableId === "tableId" && (doc["Name"] === "Test" || doc["Yes"] > "Value")) {
emit(doc["_id"], doc["myField"]);
}
}`,
meta: {
calculation: undefined,
field: "myField",
filters: [
{
condition: "EQUALS",
key: "Name",
value: "Test",
},
{
condition: "MT",
conjunction: "OR",
key: "Yes",
value: "Value",
},
],
groupBy: undefined,
schema: null,
tableId: "tableId",
},
})
})
})
describe("Calculate", () => {
it("creates a view with the calculation statistics schema", () => {
expect(
viewTemplate({
field: "myField",
calculation: "stats",
tableId: "tableId",
filters: [],
})
).toEqual({
map: `function (doc) {
if ((doc.tableId === "tableId" && !(
doc["myField"] === undefined ||
doc["myField"] === null ||
doc["myField"] === "" ||
(Array.isArray(doc["myField"]) && doc["myField"].length === 0)
)) ) {
emit(doc["_id"], doc["myField"]);
}
}`,
meta: {
calculation: "stats",
field: "myField",
filters: [],
groupBy: undefined,
schema: {
min: { type: "number" },
max: { type: "number" },
avg: { type: "number" },
count: { type: "number" },
sumsqr: { type: "number" },
sum: { type: "number" },
field: { type: "string" },
},
tableId: "tableId",
},
reduce: "_stats",
})
})
})
describe("Group By", () => {
it("creates a view emitting the group by field", () => {
expect(
viewTemplate({
tableId: "tableId",
groupBy: "age",
field: "score",
filters: [],
})
).toEqual({
map: `function (doc) {
if (doc.tableId === "tableId" ) {
emit(doc["age"], doc["score"]);
}
}`,
meta: {
calculation: undefined,
field: "score",
filters: [],
groupBy: "age",
schema: null,
tableId: "tableId",
},
})
})
})
describe("Calculate and filter", () => {
it("creates a view with the calculation statistics and filter schema", () => {
expect(
viewTemplate({
field: "myField",
calculation: "stats",
tableId: "tableId",
filters: [
{
value: 17,
condition: "MT",
key: "age",
},
],
})
).toEqual({
map: `function (doc) {
if ((doc.tableId === "tableId" && !(
doc["myField"] === undefined ||
doc["myField"] === null ||
doc["myField"] === "" ||
(Array.isArray(doc["myField"]) && doc["myField"].length === 0)
)) && (doc["age"] > 17)) {
emit(doc["_id"], doc["myField"]);
}
}`,
meta: {
calculation: "stats",
field: "myField",
filters: [
{
condition: "MT",
key: "age",
value: 17,
},
],
groupBy: undefined,
schema: {
min: { type: "number" },
max: { type: "number" },
avg: { type: "number" },
count: { type: "number" },
sumsqr: { type: "number" },
sum: { type: "number" },
field: { type: "string" },
},
tableId: "tableId",
},
reduce: "_stats",
})
})
})
})

20
packages/server/src/api/controllers/view/viewBuilder.ts
View File

@ -1,4 +1,4 @@
import { ViewFilter, ViewTemplateOpts, DBView } from "@budibase/types" import { ViewFilter, DBView } from "@budibase/types"
const TOKEN_MAP: Record<string, string> = { const TOKEN_MAP: Record<string, string> = {
EQUALS: "===", EQUALS: "===",
@ -120,7 +120,7 @@ function parseFilterExpression(filters: ViewFilter[]) {
* @param groupBy - field to group calculation results on, if any * @param groupBy - field to group calculation results on, if any
*/ */
function parseEmitExpression(field: string, groupBy: string) { function parseEmitExpression(field: string, groupBy: string) {
return `emit(doc["${groupBy || "_id"}"], doc["${field}"]);` return `emit(doc["${groupBy}"], doc["${field}"]);`
} }
/** /**
@ -135,7 +135,19 @@ function parseEmitExpression(field: string, groupBy: string) {
* calculation: an optional calculation to be performed over the view data. * calculation: an optional calculation to be performed over the view data.
*/ */
export default function ( export default function (
{ field, tableId, groupBy, filters = [], calculation }: ViewTemplateOpts, {
field,
tableId,
groupBy,
filters = [],
calculation,
}: {
field: string
tableId: string
groupBy?: string
filters?: ViewFilter[]
calculation?: string
},
groupByMulti?: boolean groupByMulti?: boolean
): DBView { ): DBView {
// first filter can't have a conjunction // first filter can't have a conjunction
@ -168,7 +180,7 @@ export default function (
const parsedFilters = parseFilterExpression(filters) const parsedFilters = parseFilterExpression(filters)
const filterExpression = parsedFilters ? `&& (${parsedFilters})` : "" const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
const emitExpression = parseEmitExpression(field, groupBy) const emitExpression = parseEmitExpression(field, groupBy || "_id")
const tableExpression = `doc.tableId === "${tableId}"` const tableExpression = `doc.tableId === "${tableId}"`
const coreExpression = statFilter const coreExpression = statFilter
? `(${tableExpression} && ${statFilter})` ? `(${tableExpression} && ${statFilter})`

6
packages/server/src/api/controllers/view/viewsV2.ts
View File

@ -123,9 +123,11 @@ async function parseSchema(view: CreateViewRequest) {
} }
export async function get(ctx: Ctx<void, ViewResponseEnriched>) { export async function get(ctx: Ctx<void, ViewResponseEnriched>) {
ctx.body = { const view = await sdk.views.getEnriched(ctx.params.viewId)
data: await sdk.views.getEnriched(ctx.params.viewId), if (!view) {
ctx.throw(404)
} }
ctx.body = { data: view }
} }
export async function fetch(ctx: Ctx<void, ViewFetchResponseEnriched>) { export async function fetch(ctx: Ctx<void, ViewFetchResponseEnriched>) {

10
packages/server/src/api/routes/migrations.ts
View File

@ -1,16 +1,8 @@
import Router from "@koa/router" import Router from "@koa/router"
import * as migrationsController from "../controllers/migrations" import * as migrationsController from "../controllers/migrations"
import { auth } from "@budibase/backend-core"
const router: Router = new Router() const router: Router = new Router()
router router.get("/api/migrations/status", migrationsController.getMigrationStatus)
.post("/api/migrations/run", auth.internalApi, migrationsController.migrate)
.get(
"/api/migrations/definitions",
auth.internalApi,
migrationsController.fetchDefinitions
)
.get("/api/migrations/status", migrationsController.getMigrationStatus)
export default router export default router

4
packages/server/src/api/routes/tests/automation.spec.ts
View File

@ -19,9 +19,11 @@ import {
Table, Table,
} from "@budibase/types" } from "@budibase/types"
import { mocks } from "@budibase/backend-core/tests" import { mocks } from "@budibase/backend-core/tests"
import { FilterConditions } from "../../../automations/steps/filter"
import { removeDeprecated } from "../../../automations/utils" import { removeDeprecated } from "../../../automations/utils"
import { createAutomationBuilder } from "../../../automations/tests/utilities/AutomationTestBuilder" import { createAutomationBuilder } from "../../../automations/tests/utilities/AutomationTestBuilder"
import { automations } from "@budibase/shared-core"
const FilterConditions = automations.steps.filter.FilterConditions
const MAX_RETRIES = 4 const MAX_RETRIES = 4
let { let {

55
packages/server/src/automations/actions.ts
View File

@ -1,3 +1,4 @@
import { automations } from "@budibase/shared-core"
import * as sendSmtpEmail from "./steps/sendSmtpEmail" import * as sendSmtpEmail from "./steps/sendSmtpEmail"
import * as createRow from "./steps/createRow" import * as createRow from "./steps/createRow"
import * as updateRow from "./steps/updateRow" import * as updateRow from "./steps/updateRow"
@ -14,11 +15,10 @@ import * as make from "./steps/make"
import * as filter from "./steps/filter" import * as filter from "./steps/filter"
import * as delay from "./steps/delay" import * as delay from "./steps/delay"
import * as queryRow from "./steps/queryRows" import * as queryRow from "./steps/queryRows"
import * as loop from "./steps/loop"
import * as collect from "./steps/collect" import * as collect from "./steps/collect"
import * as branch from "./steps/branch"
import * as triggerAutomationRun from "./steps/triggerAutomationRun" import * as triggerAutomationRun from "./steps/triggerAutomationRun"
import * as openai from "./steps/openai" import * as openai from "./steps/openai"
import * as bash from "./steps/bash"
import env from "../environment" import env from "../environment"
import { import {
PluginType, PluginType,
@ -62,42 +62,39 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<
string, string,
AutomationStepDefinition AutomationStepDefinition
> = { > = {
SEND_EMAIL_SMTP: sendSmtpEmail.definition, SEND_EMAIL_SMTP: automations.steps.sendSmtpEmail.definition,
CREATE_ROW: createRow.definition, CREATE_ROW: automations.steps.createRow.definition,
UPDATE_ROW: updateRow.definition, UPDATE_ROW: automations.steps.updateRow.definition,
DELETE_ROW: deleteRow.definition, DELETE_ROW: automations.steps.deleteRow.definition,
OUTGOING_WEBHOOK: outgoingWebhook.definition, OUTGOING_WEBHOOK: automations.steps.outgoingWebhook.definition,
EXECUTE_SCRIPT: executeScript.definition, EXECUTE_SCRIPT: automations.steps.executeScript.definition,
EXECUTE_QUERY: executeQuery.definition, EXECUTE_QUERY: automations.steps.executeQuery.definition,
SERVER_LOG: serverLog.definition, SERVER_LOG: automations.steps.serverLog.definition,
DELAY: delay.definition, DELAY: automations.steps.delay.definition,
FILTER: filter.definition, FILTER: automations.steps.filter.definition,
QUERY_ROWS: queryRow.definition, QUERY_ROWS: automations.steps.queryRows.definition,
LOOP: loop.definition, LOOP: automations.steps.loop.definition,
COLLECT: collect.definition, COLLECT: automations.steps.collect.definition,
TRIGGER_AUTOMATION_RUN: triggerAutomationRun.definition, TRIGGER_AUTOMATION_RUN: automations.steps.triggerAutomationRun.definition,
BRANCH: branch.definition, BRANCH: automations.steps.branch.definition,
// these used to be lowercase step IDs, maintain for backwards compat // these used to be lowercase step IDs, maintain for backwards compat
discord: discord.definition, discord: automations.steps.discord.definition,
slack: slack.definition, slack: automations.steps.slack.definition,
zapier: zapier.definition, zapier: automations.steps.zapier.definition,
integromat: make.definition, integromat: automations.steps.make.definition,
n8n: n8n.definition, n8n: automations.steps.n8n.definition,
} }
// don't add the bash script/definitions unless in self host // don't add the bash script/definitions unless in self host
// the fact this isn't included in any definitions means it cannot be // the fact this isn't included in any definitions means it cannot be
// ran at all // ran at all
if (env.SELF_HOSTED) { if (env.SELF_HOSTED) {
const bash = require("./steps/bash") // @ts-expect-error
// @ts-ignore
ACTION_IMPLS["EXECUTE_BASH"] = bash.run ACTION_IMPLS["EXECUTE_BASH"] = bash.run
// @ts-ignore BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = automations.steps.bash.definition
BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
if (env.isTest()) { if (env.isTest()) {
BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition
} }
} }
@ -105,7 +102,7 @@ export async function getActionDefinitions(): Promise<
Record<keyof typeof AutomationActionStepId, AutomationStepDefinition> Record<keyof typeof AutomationActionStepId, AutomationStepDefinition>
> { > {
if (env.SELF_HOSTED) { if (env.SELF_HOSTED) {
BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition
} }
const actionDefinitions = BUILTIN_ACTION_DEFINITIONS const actionDefinitions = BUILTIN_ACTION_DEFINITIONS

50
packages/server/src/automations/steps/bash.ts
View File

@ -2,55 +2,7 @@ import { execSync } from "child_process"
import { processStringSync } from "@budibase/string-templates" import { processStringSync } from "@budibase/string-templates"
import * as automationUtils from "../automationUtils" import * as automationUtils from "../automationUtils"
import environment from "../../environment" import environment from "../../environment"
import { import { BashStepInputs, BashStepOutputs } from "@budibase/types"
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
BashStepInputs,
BashStepOutputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Bash Scripting",
tagline: "Execute a bash command",
icon: "JourneyEvent",
description: "Run a bash script",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.EXECUTE_BASH,
inputs: {},
schema: {
inputs: {
properties: {
code: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.CODE,
title: "Code",
},
},
required: ["code"],
},
outputs: {
properties: {
stdout: {
type: AutomationIOType.STRING,
description: "Standard output of your bash command or script",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the command was successful",
},
},
required: ["stdout"],
},
},
}
export async function run({ export async function run({
inputs, inputs,

46
packages/server/src/automations/steps/collect.ts
View File

@ -1,48 +1,4 @@
import { import { CollectStepInputs, CollectStepOutputs } from "@budibase/types"
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
CollectStepInputs,
CollectStepOutputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Collect Data",
tagline: "Collect data to be sent to design",
icon: "Collection",
description:
"Collects specified data so it can be provided to the design section",
type: AutomationStepType.ACTION,
internal: true,
features: {},
stepId: AutomationActionStepId.COLLECT,
inputs: {},
schema: {
inputs: {
properties: {
collection: {
type: AutomationIOType.STRING,
title: "What to Collect",
},
},
required: ["collection"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
value: {
type: AutomationIOType.STRING,
description: "Collected data",
},
},
required: ["success", "value"],
},
},
}
export async function run({ export async function run({
inputs, inputs,

70
packages/server/src/automations/steps/createRow.ts
View File

@ -5,77 +5,9 @@ import {
sendAutomationAttachmentsToStorage, sendAutomationAttachmentsToStorage,
} from "../automationUtils" } from "../automationUtils"
import { buildCtx } from "./utils" import { buildCtx } from "./utils"
import { import { CreateRowStepInputs, CreateRowStepOutputs } from "@budibase/types"
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
CreateRowStepInputs,
CreateRowStepOutputs,
} from "@budibase/types"
import { EventEmitter } from "events" import { EventEmitter } from "events"
export const definition: AutomationStepDefinition = {
name: "Create Row",
tagline: "Create a {{inputs.enriched.table.name}} row",
icon: "TableRowAddBottom",
description: "Add a row to your database",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.CREATE_ROW,
inputs: {},
schema: {
inputs: {
properties: {
row: {
type: AutomationIOType.OBJECT,
properties: {
tableId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.TABLE,
},
},
customType: AutomationCustomIOType.ROW,
title: "Table",
required: ["tableId"],
},
},
required: ["row"],
},
outputs: {
properties: {
row: {
type: AutomationIOType.OBJECT,
customType: AutomationCustomIOType.ROW,
description: "The new row",
},
response: {
type: AutomationIOType.OBJECT,
description: "The response from the table",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the row creation was successful",
},
id: {
type: AutomationIOType.STRING,
description: "The identifier of the new row",
},
revision: {
type: AutomationIOType.STRING,
description: "The revision of the new row",
},
},
required: ["success", "id", "revision"],
},
},
}
export async function run({ export async function run({
inputs, inputs,
appId, appId,

41
packages/server/src/automations/steps/delay.ts
View File

@ -1,44 +1,5 @@
import { wait } from "../../utilities" import { wait } from "../../utilities"
import { import { DelayStepInputs, DelayStepOutputs } from "@budibase/types"
AutomationActionStepId,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
DelayStepInputs,
DelayStepOutputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Delay",
icon: "Clock",
tagline: "Delay for {{inputs.time}} milliseconds",
description: "Delay the automation until an amount of time has passed",
stepId: AutomationActionStepId.DELAY,
internal: true,
features: {},
inputs: {},
schema: {
inputs: {
properties: {
time: {
type: AutomationIOType.NUMBER,
title: "Delay in milliseconds",
},
},
required: ["time"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the delay was successful",
},
},
required: ["success"],
},
},
type: AutomationStepType.LOGIC,
}
export async function run({ export async function run({
inputs, inputs,

59
packages/server/src/automations/steps/deleteRow.ts
View File

@ -2,64 +2,7 @@ import { EventEmitter } from "events"
import { destroy } from "../../api/controllers/row" import { destroy } from "../../api/controllers/row"
import { buildCtx } from "./utils" import { buildCtx } from "./utils"
import { getError } from "../automationUtils" import { getError } from "../automationUtils"
import { import { DeleteRowStepInputs, DeleteRowStepOutputs } from "@budibase/types"
AutomationActionStepId,
AutomationStepType,
AutomationIOType,
AutomationCustomIOType,
AutomationFeature,
DeleteRowStepInputs,
DeleteRowStepOutputs,
AutomationStepDefinition,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
description: "Delete a row from your database",
icon: "TableRowRemoveCenter",
name: "Delete Row",
tagline: "Delete a {{inputs.enriched.table.name}} row",
type: AutomationStepType.ACTION,
stepId: AutomationActionStepId.DELETE_ROW,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
tableId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.TABLE,
title: "Table",
},
id: {
type: AutomationIOType.STRING,
title: "Row ID",
},
},
required: ["tableId", "id"],
},
outputs: {
properties: {
row: {
type: AutomationIOType.OBJECT,
customType: AutomationCustomIOType.ROW,
description: "The deleted row",
},
response: {
type: AutomationIOType.OBJECT,
description: "The response from the table",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the deletion was successful",
},
},
required: ["row", "success"],
},
},
}
export async function run({ export async function run({
inputs, inputs,

63
packages/server/src/automations/steps/discord.ts
View File

@ -1,71 +1,10 @@
import fetch from "node-fetch" import fetch from "node-fetch"
import { getFetchResponse } from "./utils" import { getFetchResponse } from "./utils"
import { import { ExternalAppStepOutputs, DiscordStepInputs } from "@budibase/types"
AutomationActionStepId,
AutomationStepType,
AutomationIOType,
AutomationFeature,
ExternalAppStepOutputs,
DiscordStepInputs,
AutomationStepDefinition,
} from "@budibase/types"
const DEFAULT_USERNAME = "Budibase Automate" const DEFAULT_USERNAME = "Budibase Automate"
const DEFAULT_AVATAR_URL = "https://i.imgur.com/a1cmTKM.png" const DEFAULT_AVATAR_URL = "https://i.imgur.com/a1cmTKM.png"
export const definition: AutomationStepDefinition = {
name: "Discord Message",
tagline: "Send a message to a Discord server",
description: "Send a message to a Discord server",
icon: "ri-discord-line",
stepId: AutomationActionStepId.discord,
type: AutomationStepType.ACTION,
internal: false,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
url: {
type: AutomationIOType.STRING,
title: "Discord Webhook URL",
},
username: {
type: AutomationIOType.STRING,
title: "Bot Name",
},
avatar_url: {
type: AutomationIOType.STRING,
title: "Bot Avatar URL",
},
content: {
type: AutomationIOType.STRING,
title: "Message",
},
},
required: ["url", "content"],
},
outputs: {
properties: {
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code of the request",
},
response: {
type: AutomationIOType.STRING,
description: "The response from the Discord Webhook",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the message sent successfully",
},
},
},
},
}
export async function run({ export async function run({
inputs, inputs,
}: { }: {

57
packages/server/src/automations/steps/executeQuery.ts
View File

@ -3,67 +3,10 @@ import * as queryController from "../../api/controllers/query"
import { buildCtx } from "./utils" import { buildCtx } from "./utils"
import * as automationUtils from "../automationUtils" import * as automationUtils from "../automationUtils"
import { import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
ExecuteQueryStepInputs, ExecuteQueryStepInputs,
ExecuteQueryStepOutputs, ExecuteQueryStepOutputs,
} from "@budibase/types" } from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "External Data Connector",
tagline: "Execute Data Connector",
icon: "Data",
description: "Execute a query in an external data connector",
type: AutomationStepType.ACTION,
stepId: AutomationActionStepId.EXECUTE_QUERY,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
query: {
type: AutomationIOType.OBJECT,
properties: {
queryId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.QUERY,
},
},
customType: AutomationCustomIOType.QUERY_PARAMS,
title: "Parameters",
required: ["queryId"],
},
},
required: ["query"],
},
outputs: {
properties: {
response: {
type: AutomationIOType.OBJECT,
description: "The response from the datasource execution",
},
info: {
type: AutomationIOType.OBJECT,
description:
"Some query types may return extra data, like headers from a REST query",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
},
required: ["response", "success"],
},
},
}
export async function run({ export async function run({
inputs, inputs,
appId, appId,

45
packages/server/src/automations/steps/executeScript.ts
View File

@ -2,56 +2,11 @@ import * as scriptController from "../../api/controllers/script"
import { buildCtx } from "./utils" import { buildCtx } from "./utils"
import * as automationUtils from "../automationUtils" import * as automationUtils from "../automationUtils"
import { import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
ExecuteScriptStepInputs, ExecuteScriptStepInputs,
ExecuteScriptStepOutputs, ExecuteScriptStepOutputs,
} from "@budibase/types" } from "@budibase/types"
import { EventEmitter } from "events" import { EventEmitter } from "events"
export const definition: AutomationStepDefinition = {
name: "JS Scripting",
tagline: "Execute JavaScript Code",
icon: "Code",
description: "Run a piece of JavaScript code in your automation",
type: AutomationStepType.ACTION,
internal: true,
stepId: AutomationActionStepId.EXECUTE_SCRIPT,
inputs: {},
features: {
[AutomationFeature.LOOPING]: true,
},
schema: {
inputs: {
properties: {
code: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.CODE,
title: "Code",
},
},
required: ["code"],
},
outputs: {
properties: {
value: {
type: AutomationIOType.STRING,
description: "The result of the return statement",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
},
required: ["success"],
},
},
}
export async function run({ export async function run({
inputs, inputs,
appId, appId,

73
packages/server/src/automations/steps/filter.ts
View File

@ -1,74 +1,7 @@
import { import { FilterStepInputs, FilterStepOutputs } from "@budibase/types"
AutomationActionStepId, import { automations } from "@budibase/shared-core"
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
FilterStepInputs,
FilterStepOutputs,
} from "@budibase/types"
export const FilterConditions = { const FilterConditions = automations.steps.filter.FilterConditions
EQUAL: "EQUAL",
NOT_EQUAL: "NOT_EQUAL",
GREATER_THAN: "GREATER_THAN",
LESS_THAN: "LESS_THAN",
}
export const PrettyFilterConditions = {
[FilterConditions.EQUAL]: "Equals",
[FilterConditions.NOT_EQUAL]: "Not equals",
[FilterConditions.GREATER_THAN]: "Greater than",
[FilterConditions.LESS_THAN]: "Less than",
}
export const definition: AutomationStepDefinition = {
name: "Condition",
tagline: "{{inputs.field}} {{inputs.condition}} {{inputs.value}}",
icon: "Branch2",
description:
"Conditionally halt automations which do not meet certain conditions",
type: AutomationStepType.LOGIC,
internal: true,
features: {},
stepId: AutomationActionStepId.FILTER,
inputs: {
condition: FilterConditions.EQUAL,
},
schema: {
inputs: {
properties: {
field: {
type: AutomationIOType.STRING,
title: "Reference Value",
},
condition: {
type: AutomationIOType.STRING,
title: "Condition",
enum: Object.values(FilterConditions),
pretty: Object.values(PrettyFilterConditions),
},
value: {
type: AutomationIOType.STRING,
title: "Comparison Value",
},
},
required: ["field", "condition", "value"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
result: {
type: AutomationIOType.BOOLEAN,
description: "Whether the logic block passed",
},
},
required: ["success", "result"],
},
},
}
export async function run({ export async function run({
inputs, inputs,

58
packages/server/src/automations/steps/make.ts
View File

@ -1,62 +1,6 @@
import fetch from "node-fetch" import fetch from "node-fetch"
import { getFetchResponse } from "./utils" import { getFetchResponse } from "./utils"
import { import { ExternalAppStepOutputs, MakeIntegrationInputs } from "@budibase/types"
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
AutomationFeature,
ExternalAppStepOutputs,
MakeIntegrationInputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Make Integration",
stepTitle: "Make",
tagline: "Trigger a Make scenario",
description:
"Performs a webhook call to Make and gets the response (if configured)",
icon: "ri-shut-down-line",
stepId: AutomationActionStepId.integromat,
type: AutomationStepType.ACTION,
internal: false,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
url: {
type: AutomationIOType.STRING,
title: "Webhook URL",
},
body: {
type: AutomationIOType.JSON,
title: "Payload",
},
},
required: ["url", "body"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether call was successful",
},
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code returned",
},
response: {
type: AutomationIOType.OBJECT,
description: "The webhook response - this can have properties",
},
},
required: ["success", "response"],
},
},
}
export async function run({ export async function run({
inputs, inputs,

62
packages/server/src/automations/steps/n8n.ts
View File

@ -1,73 +1,11 @@
import fetch, { HeadersInit } from "node-fetch" import fetch, { HeadersInit } from "node-fetch"
import { getFetchResponse } from "./utils" import { getFetchResponse } from "./utils"
import { import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
AutomationFeature,
HttpMethod, HttpMethod,
ExternalAppStepOutputs, ExternalAppStepOutputs,
n8nStepInputs, n8nStepInputs,
} from "@budibase/types" } from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "n8n Integration",
stepTitle: "n8n",
tagline: "Trigger an n8n workflow",
description:
"Performs a webhook call to n8n and gets the response (if configured)",
icon: "ri-shut-down-line",
stepId: AutomationActionStepId.n8n,
type: AutomationStepType.ACTION,
internal: false,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
url: {
type: AutomationIOType.STRING,
title: "Webhook URL",
},
method: {
type: AutomationIOType.STRING,
title: "Method",
enum: Object.values(HttpMethod),
},
authorization: {
type: AutomationIOType.STRING,
title: "Authorization",
},
body: {
type: AutomationIOType.JSON,
title: "Payload",
},
},
required: ["url", "method"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether call was successful",
},
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code returned",
},
response: {
type: AutomationIOType.OBJECT,
description: "The webhook response - this can have properties",
},
},
required: ["success", "response"],
},
},
}
export async function run({ export async function run({
inputs, inputs,
}: { }: {

59
packages/server/src/automations/steps/openai.ts
View File

@ -1,67 +1,10 @@
import { OpenAI } from "openai" import { OpenAI } from "openai"
import { import { OpenAIStepInputs, OpenAIStepOutputs } from "@budibase/types"
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
OpenAIStepInputs,
OpenAIStepOutputs,
} from "@budibase/types"
import { env } from "@budibase/backend-core" import { env } from "@budibase/backend-core"
import * as automationUtils from "../automationUtils" import * as automationUtils from "../automationUtils"
import * as pro from "@budibase/pro" import * as pro from "@budibase/pro"
enum Model {
GPT_4O_MINI = "gpt-4o-mini",
GPT_4O = "gpt-4o",
GPT_4 = "gpt-4",
GPT_35_TURBO = "gpt-3.5-turbo",
}
export const definition: AutomationStepDefinition = {
name: "OpenAI",
tagline: "Send prompts to ChatGPT",
icon: "Algorithm",
description: "Interact with the OpenAI ChatGPT API.",
type: AutomationStepType.ACTION,
internal: true,
features: {},
stepId: AutomationActionStepId.OPENAI,
inputs: {
prompt: "",
},
schema: {
inputs: {
properties: {
prompt: {
type: AutomationIOType.STRING,
title: "Prompt",
},
model: {
type: AutomationIOType.STRING,
title: "Model",
enum: Object.values(Model),
},
},
required: ["prompt", "model"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
response: {
type: AutomationIOType.STRING,
description: "What was output",
},
},
required: ["success", "response"],
},
},
}
/** /**
* Maintains backward compatibility with automation steps created before the introduction * Maintains backward compatibility with automation steps created before the introduction
* of custom configurations and Budibase AI * of custom configurations and Budibase AI

69
packages/server/src/automations/steps/outgoingWebhook.ts
View File

@ -2,12 +2,6 @@ import fetch from "node-fetch"
import { getFetchResponse } from "./utils" import { getFetchResponse } from "./utils"
import * as automationUtils from "../automationUtils" import * as automationUtils from "../automationUtils"
import { import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
ExternalAppStepOutputs, ExternalAppStepOutputs,
OutgoingWebhookStepInputs, OutgoingWebhookStepInputs,
} from "@budibase/types" } from "@budibase/types"
@ -26,69 +20,6 @@ const BODY_REQUESTS = [RequestType.POST, RequestType.PUT, RequestType.PATCH]
* NOTE: this functionality is deprecated - it no longer should be used. * NOTE: this functionality is deprecated - it no longer should be used.
*/ */
export const definition: AutomationStepDefinition = {
deprecated: true,
name: "Outgoing webhook",
tagline: "Send a {{inputs.requestMethod}} request",
icon: "Send",
description: "Send a request of specified method to a URL",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.OUTGOING_WEBHOOK,
inputs: {
requestMethod: "POST",
url: "http://",
requestBody: "{}",
headers: "{}",
},
schema: {
inputs: {
properties: {
requestMethod: {
type: AutomationIOType.STRING,
enum: Object.values(RequestType),
title: "Request method",
},
url: {
type: AutomationIOType.STRING,
title: "URL",
},
requestBody: {
type: AutomationIOType.STRING,
title: "JSON Body",
customType: AutomationCustomIOType.WIDE,
},
headers: {
type: AutomationIOType.STRING,
title: "Headers",
customType: AutomationCustomIOType.WIDE,
},
},
required: ["requestMethod", "url"],
},
outputs: {
properties: {
response: {
type: AutomationIOType.OBJECT,
description: "The response from the webhook",
},
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code returned",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
},
required: ["response", "success"],
},
},
}
export async function run({ export async function run({
inputs, inputs,
}: { }: {

72
packages/server/src/automations/steps/queryRows.ts
View File

@ -4,84 +4,12 @@ import { buildCtx } from "./utils"
import * as automationUtils from "../automationUtils" import * as automationUtils from "../automationUtils"
import { import {
FieldType, FieldType,
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
EmptyFilterOption, EmptyFilterOption,
SortOrder, SortOrder,
QueryRowsStepInputs, QueryRowsStepInputs,
QueryRowsStepOutputs, QueryRowsStepOutputs,
} from "@budibase/types" } from "@budibase/types"
const SortOrderPretty = {
[SortOrder.ASCENDING]: "Ascending",
[SortOrder.DESCENDING]: "Descending",
}
export const definition: AutomationStepDefinition = {
description: "Query rows from the database",
icon: "Search",
name: "Query rows",
tagline: "Query rows from {{inputs.enriched.table.name}} table",
type: AutomationStepType.ACTION,
stepId: AutomationActionStepId.QUERY_ROWS,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
tableId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.TABLE,
title: "Table",
},
filters: {
type: AutomationIOType.OBJECT,
customType: AutomationCustomIOType.FILTERS,
title: "Filtering",
},
sortColumn: {
type: AutomationIOType.STRING,
title: "Sort Column",
customType: AutomationCustomIOType.COLUMN,
},
sortOrder: {
type: AutomationIOType.STRING,
title: "Sort Order",
enum: Object.values(SortOrder),
pretty: Object.values(SortOrderPretty),
},
limit: {
type: AutomationIOType.NUMBER,
title: "Limit",
customType: AutomationCustomIOType.QUERY_LIMIT,
},
},
required: ["tableId"],
},
outputs: {
properties: {
rows: {
type: AutomationIOType.ARRAY,
customType: AutomationCustomIOType.ROWS,
description: "The rows that were found",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the query was successful",
},
},
required: ["rows", "success"],
},
},
}
async function getTable(appId: string, tableId: string) { async function getTable(appId: string, tableId: string) {
const ctx: any = buildCtx(appId, null, { const ctx: any = buildCtx(appId, null, {
params: { params: {

98
packages/server/src/automations/steps/sendSmtpEmail.ts
View File

@ -1,102 +1,6 @@
import { sendSmtpEmail } from "../../utilities/workerRequests" import { sendSmtpEmail } from "../../utilities/workerRequests"
import * as automationUtils from "../automationUtils" import * as automationUtils from "../automationUtils"
import { import { SmtpEmailStepInputs, BaseAutomationOutputs } from "@budibase/types"
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
AutomationFeature,
AutomationCustomIOType,
SmtpEmailStepInputs,
BaseAutomationOutputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
description: "Send an email using SMTP",
tagline: "Send SMTP email to {{inputs.to}}",
icon: "Email",
name: "Send Email (SMTP)",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.SEND_EMAIL_SMTP,
inputs: {},
schema: {
inputs: {
properties: {
to: {
type: AutomationIOType.STRING,
title: "Send To",
},
from: {
type: AutomationIOType.STRING,
title: "Send From",
},
cc: {
type: AutomationIOType.STRING,
title: "CC",
},
bcc: {
type: AutomationIOType.STRING,
title: "BCC",
},
subject: {
type: AutomationIOType.STRING,
title: "Email Subject",
},
contents: {
type: AutomationIOType.STRING,
title: "HTML Contents",
},
addInvite: {
type: AutomationIOType.BOOLEAN,
title: "Add calendar invite",
},
startTime: {
type: AutomationIOType.DATE,
title: "Start Time",
dependsOn: "addInvite",
},
endTime: {
type: AutomationIOType.DATE,
title: "End Time",
dependsOn: "addInvite",
},
summary: {
type: AutomationIOType.STRING,
title: "Meeting Summary",
dependsOn: "addInvite",
},
location: {
type: AutomationIOType.STRING,
title: "Location",
dependsOn: "addInvite",
},
attachments: {
type: AutomationIOType.ATTACHMENT,
customType: AutomationCustomIOType.MULTI_ATTACHMENTS,
title: "Attachments",
},
},
required: ["to", "from", "subject", "contents"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the email was sent",
},
response: {
type: AutomationIOType.OBJECT,
description: "A response from the email client, this may be an error",
},
},
required: ["success"],
},
},
}
export async function run({ export async function run({
inputs, inputs,

56
packages/server/src/automations/steps/serverLog.ts
View File

@ -1,58 +1,4 @@
import { import { ServerLogStepInputs, ServerLogStepOutputs } from "@budibase/types"
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
AutomationFeature,
ServerLogStepInputs,
ServerLogStepOutputs,
} from "@budibase/types"
/**
* Note, there is some functionality in this that is not currently exposed as it
* is complex and maybe better to be opinionated here.
* GET/DELETE requests cannot handle body elements so they will not be sent if configured.
*/
export const definition: AutomationStepDefinition = {
name: "Backend log",
tagline: "Console log a value in the backend",
icon: "Monitoring",
description: "Logs the given text to the server (using console.log)",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.SERVER_LOG,
inputs: {
text: "",
},
schema: {
inputs: {
properties: {
text: {
type: AutomationIOType.STRING,
title: "Log",
},
},
required: ["text"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
message: {
type: AutomationIOType.STRING,
description: "What was output",
},
},
required: ["success", "message"],
},
},
}
export async function run({ export async function run({
inputs, inputs,

55
packages/server/src/automations/steps/slack.ts
View File

@ -1,59 +1,6 @@
import fetch from "node-fetch" import fetch from "node-fetch"
import { getFetchResponse } from "./utils" import { getFetchResponse } from "./utils"
import { import { ExternalAppStepOutputs, SlackStepInputs } from "@budibase/types"
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
AutomationFeature,
ExternalAppStepOutputs,
SlackStepInputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Slack Message",
tagline: "Send a message to Slack",
description: "Send a message to Slack",
icon: "ri-slack-line",
stepId: AutomationActionStepId.slack,
type: AutomationStepType.ACTION,
internal: false,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
url: {
type: AutomationIOType.STRING,
title: "Incoming Webhook URL",
},
text: {
type: AutomationIOType.STRING,
title: "Message",
},
},
required: ["url", "text"],
},
outputs: {
properties: {
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code of the request",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the message sent successfully",
},
response: {
type: AutomationIOType.STRING,
description: "The response from the Slack Webhook",
},
},
},
},
}
export async function run({ export async function run({
inputs, inputs,

53
packages/server/src/automations/steps/triggerAutomationRun.ts
View File

@ -1,10 +1,5 @@
import { import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
Automation, Automation,
AutomationCustomIOType,
TriggerAutomationStepInputs, TriggerAutomationStepInputs,
TriggerAutomationStepOutputs, TriggerAutomationStepOutputs,
} from "@budibase/types" } from "@budibase/types"
@ -13,54 +8,6 @@ import { context } from "@budibase/backend-core"
import { features } from "@budibase/pro" import { features } from "@budibase/pro"
import env from "../../environment" import env from "../../environment"
export const definition: AutomationStepDefinition = {
name: "Trigger an automation",
tagline: "Triggers an automation synchronously",
icon: "Sync",
description: "Triggers an automation synchronously",
type: AutomationStepType.ACTION,
internal: true,
features: {},
stepId: AutomationActionStepId.TRIGGER_AUTOMATION_RUN,
inputs: {},
schema: {
inputs: {
properties: {
automation: {
type: AutomationIOType.OBJECT,
properties: {
automationId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.AUTOMATION,
},
},
customType: AutomationCustomIOType.AUTOMATION_FIELDS,
title: "automatioFields",
required: ["automationId"],
},
timeout: {
type: AutomationIOType.NUMBER,
title: "Timeout (ms)",
},
},
required: ["automationId"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the automation was successful",
},
value: {
type: AutomationIOType.OBJECT,
description: "Automation Result",
},
},
required: ["success", "value"],
},
},
}
export async function run({ export async function run({
inputs, inputs,
}: { }: {

70
packages/server/src/automations/steps/updateRow.ts
View File

@ -2,76 +2,8 @@ import { EventEmitter } from "events"
import * as rowController from "../../api/controllers/row" import * as rowController from "../../api/controllers/row"
import * as automationUtils from "../automationUtils" import * as automationUtils from "../automationUtils"
import { buildCtx } from "./utils" import { buildCtx } from "./utils"
import { import { UpdateRowStepInputs, UpdateRowStepOutputs } from "@budibase/types"
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
UpdateRowStepInputs,
UpdateRowStepOutputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Update Row",
tagline: "Update a {{inputs.enriched.table.name}} row",
icon: "Refresh",
description: "Update a row in your database",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.UPDATE_ROW,
inputs: {},
schema: {
inputs: {
properties: {
meta: {
type: AutomationIOType.OBJECT,
title: "Field settings",
},
row: {
type: AutomationIOType.OBJECT,
customType: AutomationCustomIOType.ROW,
title: "Table",
},
rowId: {
type: AutomationIOType.STRING,
title: "Row ID",
},
},
required: ["row", "rowId"],
},
outputs: {
properties: {
row: {
type: AutomationIOType.OBJECT,
customType: AutomationCustomIOType.ROW,
description: "The updated row",
},
response: {
type: AutomationIOType.OBJECT,
description: "The response from the table",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
id: {
type: AutomationIOType.STRING,
description: "The identifier of the updated row",
},
revision: {
type: AutomationIOType.STRING,
description: "The revision of the updated row",
},
},
required: ["success", "id", "revision"],
},
},
}
export async function run({ export async function run({
inputs, inputs,
appId, appId,

51
packages/server/src/automations/steps/zapier.ts
View File

@ -1,55 +1,6 @@
import fetch from "node-fetch" import fetch from "node-fetch"
import { getFetchResponse } from "./utils" import { getFetchResponse } from "./utils"
import { import { ZapierStepInputs, ZapierStepOutputs } from "@budibase/types"
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
AutomationFeature,
ZapierStepInputs,
ZapierStepOutputs,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Zapier Webhook",
stepId: AutomationActionStepId.zapier,
type: AutomationStepType.ACTION,
internal: false,
features: {
[AutomationFeature.LOOPING]: true,
},
description: "Trigger a Zapier Zap via webhooks",
tagline: "Trigger a Zapier Zap",
icon: "ri-flashlight-line",
inputs: {},
schema: {
inputs: {
properties: {
url: {
type: AutomationIOType.STRING,
title: "Webhook URL",
},
body: {
type: AutomationIOType.JSON,
title: "Payload",
},
},
required: ["url"],
},
outputs: {
properties: {
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code of the request",
},
response: {
type: AutomationIOType.STRING,
description: "The response from Zapier",
},
},
},
},
}
export async function run({ export async function run({
inputs, inputs,

4
packages/server/src/automations/tests/filter.spec.ts
View File

@ -1,5 +1,7 @@
import * as setup from "./utilities" import * as setup from "./utilities"
import { FilterConditions } from "../steps/filter" import { automations } from "@budibase/shared-core"
const FilterConditions = automations.steps.filter.FilterConditions
describe("test the filter logic", () => { describe("test the filter logic", () => {
const config = setup.getConfig() const config = setup.getConfig()

4
packages/server/src/automations/tests/scenarios/scenarios.spec.ts
View File

@ -6,9 +6,11 @@ import {
DatabaseName, DatabaseName,
datasourceDescribe, datasourceDescribe,
} from "../../../integrations/tests/utils" } from "../../../integrations/tests/utils"
import { FilterConditions } from "../../../automations/steps/filter"
import { Knex } from "knex" import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests" import { generator } from "@budibase/backend-core/tests"
import { automations } from "@budibase/shared-core"
const FilterConditions = automations.steps.filter.FilterConditions
describe("Automation Scenarios", () => { describe("Automation Scenarios", () => {
let config = setup.getConfig() let config = setup.getConfig()

4
packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts
View File

@ -42,7 +42,7 @@ import {
} from "@budibase/types" } from "@budibase/types"
import TestConfiguration from "../../../tests/utilities/TestConfiguration" import TestConfiguration from "../../../tests/utilities/TestConfiguration"
import * as setup from "../utilities" import * as setup from "../utilities"
import { definition } from "../../../automations/steps/branch" import { automations } from "@budibase/shared-core"
type TriggerOutputs = type TriggerOutputs =
| RowCreatedTriggerOutputs | RowCreatedTriggerOutputs
@ -103,7 +103,7 @@ class BaseStepBuilder {
branchStepInputs.children![branchId] = stepBuilder.build() branchStepInputs.children![branchId] = stepBuilder.build()
}) })
const branchStep: AutomationStep = { const branchStep: AutomationStep = {
...definition, ...automations.steps.branch.definition,
id: uuidv4(), id: uuidv4(),
stepId: AutomationActionStepId.BRANCH, stepId: AutomationActionStepId.BRANCH,
inputs: branchStepInputs, inputs: branchStepInputs,

1
packages/server/src/environment.ts
View File

@ -54,7 +54,6 @@ const environment = {
REDIS_URL: process.env.REDIS_URL, REDIS_URL: process.env.REDIS_URL,
REDIS_PASSWORD: process.env.REDIS_PASSWORD, REDIS_PASSWORD: process.env.REDIS_PASSWORD,
REDIS_CLUSTERED: process.env.REDIS_CLUSTERED, REDIS_CLUSTERED: process.env.REDIS_CLUSTERED,
HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
CLUSTER_MODE: process.env.CLUSTER_MODE, CLUSTER_MODE: process.env.CLUSTER_MODE,
API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC, API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

27
packages/server/src/migrations/functions/appUrls.ts
View File

@ -1,27 +0,0 @@
import { db as dbCore } from "@budibase/backend-core"
import sdk from "../../sdk"
/**
* Date:
* January 2022
*
* Description:
* Add the url to the app metadata if it doesn't exist
*/
export const run = async (appDb: any) => {
let metadata
try {
metadata = await appDb.get(dbCore.DocumentType.APP_METADATA)
} catch (e) {
// sometimes the metadata document doesn't exist
// exit early instead of failing the migration
console.error("Error retrieving app metadata. Skipping", e)
return
}
if (!metadata.url) {
metadata.url = sdk.applications.getAppUrl({ name: metadata.name })
console.log(`Adding url to app: ${metadata.url}`)
await appDb.put(metadata)
}
}

149
packages/server/src/migrations/functions/backfill/app.ts
View File

@ -1,149 +0,0 @@
import * as automations from "./app/automations"
import * as datasources from "./app/datasources"
import * as layouts from "./app/layouts"
import * as queries from "./app/queries"
import * as roles from "./app/roles"
import * as tables from "./app/tables"
import * as screens from "./app/screens"
import * as global from "./global"
import { App, AppBackfillSucceededEvent, Event } from "@budibase/types"
import { db as dbUtils, events } from "@budibase/backend-core"
import env from "../../../environment"
import { DEFAULT_TIMESTAMP } from "."
const failGraceful = env.SELF_HOSTED && !env.isDev()
const handleError = (e: any, errors?: any) => {
if (failGraceful) {
if (errors) {
errors.push(e)
}
return
}
console.trace(e)
throw e
}
const EVENTS = [
Event.AUTOMATION_CREATED,
Event.AUTOMATION_STEP_CREATED,
Event.DATASOURCE_CREATED,
Event.LAYOUT_CREATED,
Event.QUERY_CREATED,
Event.ROLE_CREATED,
Event.SCREEN_CREATED,
Event.TABLE_CREATED,
Event.VIEW_CREATED,
Event.VIEW_CALCULATION_CREATED,
Event.VIEW_FILTER_CREATED,
Event.APP_PUBLISHED,
Event.APP_CREATED,
]
/**
* Date:
* May 2022
*
* Description:
* Backfill app events.
*/
export const run = async (appDb: any) => {
try {
if (await global.isComplete()) {
// make sure new apps aren't backfilled
// return if the global migration for this tenant is complete
// which runs after the app migrations
return
}
// tell the event pipeline to start caching
// events for this tenant
await events.backfillCache.start(EVENTS)
let timestamp: string | number = DEFAULT_TIMESTAMP
const app: App = await appDb.get(dbUtils.DocumentType.APP_METADATA)
if (app.createdAt) {
timestamp = app.createdAt as string
}
if (dbUtils.isProdAppID(app.appId)) {
await events.app.published(app, timestamp)
}
const totals: any = {}
const errors: any = []
if (dbUtils.isDevAppID(app.appId)) {
await events.app.created(app, timestamp)
try {
totals.automations = await automations.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.datasources = await datasources.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.layouts = await layouts.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.queries = await queries.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.roles = await roles.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.screens = await screens.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.tables = await tables.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
}
const properties: AppBackfillSucceededEvent = {
appId: app.appId,
automations: totals.automations,
datasources: totals.datasources,
layouts: totals.layouts,
queries: totals.queries,
roles: totals.roles,
tables: totals.tables,
screens: totals.screens,
}
if (errors.length) {
properties.errors = errors.map((e: any) =>
JSON.stringify(e, Object.getOwnPropertyNames(e))
)
properties.errorCount = errors.length
} else {
properties.errorCount = 0
}
await events.backfill.appSucceeded(properties)
// tell the event pipeline to stop caching events for this tenant
await events.backfillCache.end()
} catch (e) {
handleError(e)
await events.backfill.appFailed(e)
}
}

26
packages/server/src/migrations/functions/backfill/app/automations.ts
View File

@ -1,26 +0,0 @@
import { events } from "@budibase/backend-core"
import { getAutomationParams } from "../../../../db/utils"
import { Automation } from "@budibase/types"
const getAutomations = async (appDb: any): Promise<Automation[]> => {
const response = await appDb.allDocs(
getAutomationParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const automations = await getAutomations(appDb)
for (const automation of automations) {
await events.automation.created(automation, timestamp)
for (const step of automation.definition.steps) {
await events.automation.stepCreated(automation, step, timestamp)
}
}
return automations.length
}

22
packages/server/src/migrations/functions/backfill/app/datasources.ts
View File

@ -1,22 +0,0 @@
import { events } from "@budibase/backend-core"
import { getDatasourceParams } from "../../../../db/utils"
import { Datasource } from "@budibase/types"
const getDatasources = async (appDb: any): Promise<Datasource[]> => {
const response = await appDb.allDocs(
getDatasourceParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const datasources: Datasource[] = await getDatasources(appDb)
for (const datasource of datasources) {
await events.datasource.created(datasource, timestamp)
}
return datasources.length
}

29
packages/server/src/migrations/functions/backfill/app/layouts.ts
View File

@ -1,29 +0,0 @@
import { events } from "@budibase/backend-core"
import { getLayoutParams } from "../../../../db/utils"
import { Layout } from "@budibase/types"
const getLayouts = async (appDb: any): Promise<Layout[]> => {
const response = await appDb.allDocs(
getLayoutParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const layouts: Layout[] = await getLayouts(appDb)
for (const layout of layouts) {
// exclude default layouts
if (
layout._id === "layout_private_master" ||
layout._id === "layout_public_master"
) {
continue
}
await events.layout.created(layout, timestamp)
}
return layouts.length
}

47
packages/server/src/migrations/functions/backfill/app/queries.ts
View File

@ -1,47 +0,0 @@
import { events } from "@budibase/backend-core"
import { getQueryParams } from "../../../../db/utils"
import { Query, Datasource, SourceName } from "@budibase/types"
const getQueries = async (appDb: any): Promise<Query[]> => {
const response = await appDb.allDocs(
getQueryParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
const getDatasource = async (
appDb: any,
datasourceId: string
): Promise<Datasource> => {
return appDb.get(datasourceId)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const queries: Query[] = await getQueries(appDb)
for (const query of queries) {
let datasource: Datasource
try {
datasource = await getDatasource(appDb, query.datasourceId)
} catch (e: any) {
// handle known bug where a datasource has been deleted
// and the query has not
if (e.status === 404) {
datasource = {
type: "unknown",
_id: query.datasourceId,
source: "unknown" as SourceName,
}
} else {
throw e
}
}
await events.query.created(datasource, query, timestamp)
}
return queries.length
}

22
packages/server/src/migrations/functions/backfill/app/roles.ts
View File

@ -1,22 +0,0 @@
import { events } from "@budibase/backend-core"
import { getRoleParams } from "../../../../db/utils"
import { Role } from "@budibase/types"
const getRoles = async (appDb: any): Promise<Role[]> => {
const response = await appDb.allDocs(
getRoleParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const roles = await getRoles(appDb)
for (const role of roles) {
await events.role.created(role, timestamp)
}
return roles.length
}

22
packages/server/src/migrations/functions/backfill/app/screens.ts
View File

@ -1,22 +0,0 @@
import { events } from "@budibase/backend-core"
import { getScreenParams } from "../../../../db/utils"
import { Screen } from "@budibase/types"
const getScreens = async (appDb: any): Promise<Screen[]> => {
const response = await appDb.allDocs(
getScreenParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const screens = await getScreens(appDb)
for (const screen of screens) {
await events.screen.created(screen, timestamp)
}
return screens.length
}

13
packages/server/src/migrations/functions/backfill/app/tables.ts
View File

@ -1,13 +0,0 @@
import { events } from "@budibase/backend-core"
import { Database } from "@budibase/types"
import sdk from "../../../../sdk"
export const backfill = async (appDb: Database, timestamp: string | number) => {
const tables = await sdk.tables.getAllInternalTables(appDb)
for (const table of tables) {
await events.table.created(table, timestamp)
}
return tables.length
}

214
packages/server/src/migrations/functions/backfill/global.ts
View File

@ -1,214 +0,0 @@
import * as users from "./global/users"
import * as configs from "./global/configs"
import * as quotas from "./global/quotas"
import {
tenancy,
events,
migrations,
accounts,
db as dbUtils,
} from "@budibase/backend-core"
import {
App,
CloudAccount,
Event,
Hosting,
QuotaUsage,
TenantBackfillSucceededEvent,
User,
} from "@budibase/types"
import env from "../../../environment"
import { DEFAULT_TIMESTAMP } from "."
const failGraceful = env.SELF_HOSTED && !env.isDev()
const handleError = (e: any, errors?: any) => {
if (failGraceful) {
if (errors) {
errors.push(e)
}
return
}
throw e
}
const formatUsage = (usage: QuotaUsage) => {
let maxAutomations = 0
let maxQueries = 0
let rows = 0
if (usage) {
if (usage.usageQuota) {
rows = usage.usageQuota.rows
}
if (usage.monthly) {
for (const value of Object.values(usage.monthly)) {
if (value.automations > maxAutomations) {
maxAutomations = value.automations
}
if (value.queries > maxQueries) {
maxQueries = value.queries
}
}
}
}
return {
maxAutomations,
maxQueries,
rows,
}
}
const EVENTS = [
Event.EMAIL_SMTP_CREATED,
Event.AUTH_SSO_CREATED,
Event.AUTH_SSO_ACTIVATED,
Event.ORG_NAME_UPDATED,
Event.ORG_LOGO_UPDATED,
Event.ORG_PLATFORM_URL_UPDATED,
Event.USER_CREATED,
Event.USER_PERMISSION_ADMIN_ASSIGNED,
Event.USER_PERMISSION_BUILDER_ASSIGNED,
Event.ROLE_ASSIGNED,
Event.ROWS_CREATED,
Event.QUERIES_RUN,
Event.AUTOMATIONS_RUN,
]
/**
* Date:
* May 2022
*
* Description:
* Backfill global events.
*/
export const run = async (db: any) => {
try {
const tenantId = tenancy.getTenantId()
let timestamp: string | number = DEFAULT_TIMESTAMP
const totals: any = {}
const errors: any = []
let allUsers: User[] = []
try {
allUsers = await users.getUsers(db)
} catch (e: any) {
handleError(e, errors)
}
if (!allUsers || allUsers.length === 0) {
// first time startup - we don't need to backfill anything
// tenant will be identified when admin user is created
if (env.SELF_HOSTED) {
await events.installation.firstStartup()
}
return
}
try {
const installTimestamp = await getInstallTimestamp(db, allUsers)
if (installTimestamp) {
timestamp = installTimestamp
}
} catch (e) {
handleError(e, errors)
}
let account: CloudAccount | undefined
if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
account = await accounts.getAccountByTenantId(tenantId)
}
try {
await events.identification.identifyTenantGroup(
tenantId,
env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD,
timestamp
)
} catch (e) {
handleError(e, errors)
}
// tell the event pipeline to start caching
// events for this tenant
await events.backfillCache.start(EVENTS)
try {
await configs.backfill(db, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.users = await users.backfill(db, account)
} catch (e) {
handleError(e, errors)
}
try {
const allApps = (await dbUtils.getAllApps({ dev: true })) as App[]
totals.apps = allApps.length
totals.usage = await quotas.backfill(allApps)
} catch (e) {
handleError(e, errors)
}
const properties: TenantBackfillSucceededEvent = {
apps: totals.apps,
users: totals.users,
...formatUsage(totals.usage),
usage: totals.usage,
}
if (errors.length) {
properties.errors = errors.map((e: any) =>
JSON.stringify(e, Object.getOwnPropertyNames(e))
)
properties.errorCount = errors.length
} else {
properties.errorCount = 0
}
await events.backfill.tenantSucceeded(properties)
// tell the event pipeline to stop caching events for this tenant
await events.backfillCache.end()
} catch (e) {
handleError(e)
await events.backfill.tenantFailed(e)
}
}
export const isComplete = async (): Promise<boolean> => {
const globalDb = tenancy.getGlobalDB()
const migrationsDoc = await migrations.getMigrationsDoc(globalDb)
return !!migrationsDoc.event_global_backfill
}
export const getInstallTimestamp = async (
globalDb: any,
allUsers?: User[]
): Promise<number | undefined> => {
if (!allUsers) {
allUsers = await users.getUsers(globalDb)
}
// get the oldest user timestamp
if (allUsers) {
const timestamps = allUsers
.map(user => user.createdAt)
.filter(timestamp => !!timestamp)
.sort(
(a, b) =>
new Date(a as number).getTime() - new Date(b as number).getTime()
)
if (timestamps.length) {
return timestamps[0]
}
}
}

74
packages/server/src/migrations/functions/backfill/global/configs.ts
View File

@ -1,74 +0,0 @@
import {
events,
DocumentType,
SEPARATOR,
UNICODE_MAX,
} from "@budibase/backend-core"
import {
Config,
isSMTPConfig,
isGoogleConfig,
isOIDCConfig,
isSettingsConfig,
ConfigType,
DatabaseQueryOpts,
} from "@budibase/types"
import env from "./../../../../environment"
export function getConfigParams(): DatabaseQueryOpts {
return {
include_docs: true,
startkey: `${DocumentType.CONFIG}${SEPARATOR}`,
endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`,
}
}
const getConfigs = async (globalDb: any): Promise<Config[]> => {
const response = await globalDb.allDocs(getConfigParams())
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (
globalDb: any,
timestamp: string | number | undefined
) => {
const configs = await getConfigs(globalDb)
for (const config of configs) {
if (isSMTPConfig(config)) {
await events.email.SMTPCreated(timestamp)
}
if (isGoogleConfig(config)) {
await events.auth.SSOCreated(ConfigType.GOOGLE, timestamp)
if (config.config.activated) {
await events.auth.SSOActivated(ConfigType.GOOGLE, timestamp)
}
}
if (isOIDCConfig(config)) {
await events.auth.SSOCreated(ConfigType.OIDC, timestamp)
if (config.config.configs[0].activated) {
await events.auth.SSOActivated(ConfigType.OIDC, timestamp)
}
}
if (isSettingsConfig(config)) {
const company = config.config.company
if (company && company !== "Budibase") {
await events.org.nameUpdated(timestamp)
}
const logoUrl = config.config.logoUrl
if (logoUrl) {
await events.org.logoUpdated(timestamp)
}
const platformUrl = config.config.platformUrl
if (
platformUrl &&
platformUrl !== "http://localhost:10000" &&
env.SELF_HOSTED
) {
await events.org.platformURLUpdated(timestamp)
}
}
}
}

60
packages/server/src/migrations/functions/backfill/global/quotas.ts
View File

@ -1,60 +0,0 @@
import { DEFAULT_TIMESTAMP } from "./../index"
import { events } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { App } from "@budibase/types"
const getOldestCreatedAt = (allApps: App[]): string | undefined => {
const timestamps = allApps
.filter(app => !!app.createdAt)
.map(app => app.createdAt as string)
.sort((a, b) => new Date(a).getTime() - new Date(b).getTime())
if (timestamps.length) {
return timestamps[0]
}
}
const getMonthTimestamp = (monthString: string): number => {
const parts = monthString.split("-")
const month = parseInt(parts[0]) - 1 // we already do +1 in month string calculation
const year = parseInt(parts[1])
// using 0 as the day in next month gives us last day in previous month
const date = new Date(year, month + 1, 0).getTime()
const now = new Date().getTime()
if (date > now) {
return now
} else {
return date
}
}
export const backfill = async (allApps: App[]) => {
const usage = await quotas.getQuotaUsage()
const rows = usage.usageQuota.rows
let timestamp: string | number = DEFAULT_TIMESTAMP
const oldestAppTimestamp = getOldestCreatedAt(allApps)
if (oldestAppTimestamp) {
timestamp = oldestAppTimestamp
}
await events.rows.created(rows, timestamp)
for (const [monthString, quotas] of Object.entries(usage.monthly)) {
if (monthString === "current") {
continue
}
const monthTimestamp = getMonthTimestamp(monthString)
const queries = quotas.queries
await events.query.run(queries, monthTimestamp)
const automations = quotas.automations
await events.automation.run(automations, monthTimestamp)
}
return usage
}

53
packages/server/src/migrations/functions/backfill/global/users.ts
View File

@ -1,53 +0,0 @@
import {
events,
db as dbUtils,
users as usersCore,
} from "@budibase/backend-core"
import { User, CloudAccount } from "@budibase/types"
import { DEFAULT_TIMESTAMP } from ".."
// manually define user doc params - normally server doesn't read users from the db
const getUserParams = (props: any) => {
return dbUtils.getDocParams(dbUtils.DocumentType.USER, null, props)
}
export const getUsers = async (globalDb: any): Promise<User[]> => {
const response = await globalDb.allDocs(
getUserParams({
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (
globalDb: any,
account: CloudAccount | undefined
) => {
const users = await getUsers(globalDb)
for (const user of users) {
let timestamp: string | number = DEFAULT_TIMESTAMP
if (user.createdAt) {
timestamp = user.createdAt
}
await events.identification.identifyUser(user, account, timestamp)
await events.user.created(user, timestamp)
if (usersCore.hasAdminPermissions(user)) {
await events.user.permissionAdminAssigned(user, timestamp)
}
if (usersCore.hasBuilderPermissions(user)) {
await events.user.permissionBuilderAssigned(user, timestamp)
}
if (user.roles) {
for (const [, role] of Object.entries(user.roles)) {
await events.role.assigned(user, role, timestamp)
}
}
}
return users.length
}

7
packages/server/src/migrations/functions/backfill/index.ts
View File

@ -1,7 +0,0 @@
export * as app from "./app"
export * as global from "./global"
export * as installation from "./installation"
// historical events are free in posthog - make sure we default to a
// historical time if no other can be found
export const DEFAULT_TIMESTAMP = new Date(2022, 0, 1).getTime()

50
packages/server/src/migrations/functions/backfill/installation.ts
View File

@ -1,50 +0,0 @@
import { DEFAULT_TIMESTAMP } from "./index"
import { events, tenancy, installation } from "@budibase/backend-core"
import { Installation } from "@budibase/types"
import * as global from "./global"
import env from "../../../environment"
const failGraceful = env.SELF_HOSTED
const handleError = (e: any, errors?: any) => {
if (failGraceful) {
if (errors) {
errors.push(e)
}
return
}
throw e
}
/**
* Date:
* May 2022
*
* Description:
* Backfill installation events.
*/
export const run = async () => {
try {
// need to use the default tenant to try to get the installation time
await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => {
const db = tenancy.getGlobalDB()
let timestamp: string | number = DEFAULT_TIMESTAMP
const installTimestamp = await global.getInstallTimestamp(db)
if (installTimestamp) {
timestamp = installTimestamp
}
const install: Installation = await installation.getInstall()
await events.identification.identifyInstallationGroup(
install.installId,
timestamp
)
})
await events.backfill.installationSucceeded()
} catch (e) {
handleError(e)
await events.backfill.installationFailed(e)
}
}

19
packages/server/src/migrations/functions/syncQuotas.ts
View File

@ -1,19 +0,0 @@
import { runQuotaMigration } from "./usageQuotas"
import * as syncApps from "./usageQuotas/syncApps"
import * as syncRows from "./usageQuotas/syncRows"
import * as syncPlugins from "./usageQuotas/syncPlugins"
import * as syncUsers from "./usageQuotas/syncUsers"
import * as syncCreators from "./usageQuotas/syncCreators"
/**
* Synchronise quotas to the state of the db.
*/
export const run = async () => {
await runQuotaMigration(async () => {
await syncApps.run()
await syncRows.run()
await syncPlugins.run()
await syncUsers.run()
await syncCreators.run()
})
}

145
packages/server/src/migrations/functions/tableSettings.ts
View File

@ -1,145 +0,0 @@
import { getScreenParams } from "../../db/utils"
import { Screen } from "@budibase/types"
import { makePropSafe as safe } from "@budibase/string-templates"
/**
* Date:
* November 2022
*
* Description:
* Update table settings to use actions instead of links. We do not remove the
* legacy values here as we cannot guarantee that their apps are up-t-date.
* It is safe to simply save both the new and old structure in the definition.
*
* Migration 1:
* Legacy "linkRows", "linkURL", "linkPeek" and "linkColumn" settings on tables
* and table blocks are migrated into a "Navigate To" action under the new
* "onClick" setting.
*
* Migration 2:
* Legacy "titleButtonURL" and "titleButtonPeek" settings on table blocks are
* migrated into a "Navigate To" action under the new "onClickTitleButton"
* setting.
*/
export const run = async (appDb: any) => {
// Get all app screens
let screens: Screen[]
try {
screens = (
await appDb.allDocs(
getScreenParams(null, {
include_docs: true,
})
)
).rows.map((row: any) => row.doc)
} catch (e) {
// sometimes the metadata document doesn't exist
// exit early instead of failing the migration
console.error("Error retrieving app metadata. Skipping", e)
return
}
// Recursively update any relevant components and mutate the screen docs
for (let screen of screens) {
const changed = migrateTableSettings(screen.props)
// Save screen if we updated it
if (changed) {
await appDb.put(screen)
console.log(
`Screen ${screen.routing?.route} contained table settings which were migrated`
)
}
}
}
// Recursively searches and mutates a screen doc to migrate table component
// and table block settings
const migrateTableSettings = (component: any) => {
let changed = false
if (!component) {
return changed
}
// Migration 1: migrate table row click settings
if (
component._component.endsWith("/table") ||
component._component.endsWith("/tableblock")
) {
const { linkRows, linkURL, linkPeek, linkColumn, onClick } = component
if (linkRows && !onClick) {
const column = linkColumn || "_id"
const action = convertLinkSettingToAction(linkURL, !!linkPeek, column)
if (action) {
changed = true
component.onClick = action
if (component._component.endsWith("/tableblock")) {
component.clickBehaviour = "actions"
}
}
}
}
// Migration 2: migrate table block title button settings
if (component._component.endsWith("/tableblock")) {
const {
showTitleButton,
titleButtonURL,
titleButtonPeek,
onClickTitleButton,
} = component
if (showTitleButton && !onClickTitleButton) {
const action = convertLinkSettingToAction(
titleButtonURL,
!!titleButtonPeek
)
if (action) {
changed = true
component.onClickTitleButton = action
component.titleButtonClickBehaviour = "actions"
}
}
}
// Recurse down the tree as needed
component._children?.forEach((child: any) => {
const childChanged = migrateTableSettings(child)
changed = changed || childChanged
})
return changed
}
// Util ti convert the legacy settings into a navigation action structure
const convertLinkSettingToAction = (
linkURL: string,
linkPeek: boolean,
linkColumn?: string
) => {
// Sanity check we have a URL
if (!linkURL) {
return null
}
// Default URL to the old URL setting
let url = linkURL
// If we enriched the old URL with a column, update the url
if (linkColumn && linkURL.includes("/:")) {
// Convert old link URL setting, which is a screen URL, into a valid
// binding using the new clicked row binding
const split = linkURL.split("/:")
const col = linkColumn || "_id"
const binding = `{{ ${safe("eventContext")}.${safe("row")}.${safe(col)} }}`
url = `${split[0]}/${binding}`
}
// Create action structure
return [
{
"##eventHandlerType": "Navigate To",
parameters: {
url,
peek: linkPeek,
},
},
]
}

26
packages/server/src/migrations/functions/tests/appUrls.spec.js
View File

@ -1,26 +0,0 @@
const { db: dbCore } = require("@budibase/backend-core")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
const migration = require("../appUrls")
describe("run", () => {
let config = new TestConfig(false)
beforeAll(async () => {
await config.init()
})
afterAll(config.end)
it("runs successfully", async () => {
const app = await config.createApp("testApp")
const metadata = await dbCore.doWithDB(app.appId, async db => {
const metadataDoc = await db.get(dbCore.DocumentType.APP_METADATA)
delete metadataDoc.url
await db.put(metadataDoc)
await migration.run(db)
return await db.get(dbCore.DocumentType.APP_METADATA)
})
expect(metadata.url).toEqual("/testapp")
})
})

144
packages/server/src/migrations/functions/tests/tableSettings.spec.ts
View File

@ -1,144 +0,0 @@
import { App, Screen } from "@budibase/types"
import { db as dbCore } from "@budibase/backend-core"
import TestConfig from "../../../tests/utilities/TestConfiguration"
import { run as runMigration } from "../tableSettings"
describe("run", () => {
const config = new TestConfig(false)
let app: App
let screen: Screen
beforeAll(async () => {
await config.init()
app = await config.createApp("testApp")
screen = await config.createScreen()
})
afterAll(config.end)
it("migrates table block row on click settings", async () => {
// Add legacy table block as first child
screen.props._children = [
{
_instanceName: "Table Block",
_styles: {},
_component: "@budibase/standard-components/tableblock",
_id: "foo",
linkRows: true,
linkURL: "/rows/:id",
linkPeek: true,
linkColumn: "name",
},
]
await config.createScreen(screen)
// Run migration
screen = await dbCore.doWithDB(app.appId, async (db: any) => {
await runMigration(db)
return await db.get(screen._id)
})
// Verify new "onClick" setting
const onClick = screen.props._children?.[0].onClick
expect(onClick).toBeDefined()
expect(onClick.length).toBe(1)
expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
expect(onClick[0].parameters.url).toBe(
`/rows/{{ [eventContext].[row].[name] }}`
)
expect(onClick[0].parameters.peek).toBeTruthy()
})
it("migrates table row on click settings", async () => {
// Add legacy table block as first child
screen.props._children = [
{
_instanceName: "Table",
_styles: {},
_component: "@budibase/standard-components/table",
_id: "foo",
linkRows: true,
linkURL: "/rows/:id",
linkPeek: true,
linkColumn: "name",
},
]
await config.createScreen(screen)
// Run migration
screen = await dbCore.doWithDB(app.appId, async (db: any) => {
await runMigration(db)
return await db.get(screen._id)
})
// Verify new "onClick" setting
const onClick = screen.props._children?.[0].onClick
expect(onClick).toBeDefined()
expect(onClick.length).toBe(1)
expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
expect(onClick[0].parameters.url).toBe(
`/rows/{{ [eventContext].[row].[name] }}`
)
expect(onClick[0].parameters.peek).toBeTruthy()
})
it("migrates table block title button settings", async () => {
// Add legacy table block as first child
screen.props._children = [
{
_instanceName: "Table Block",
_styles: {},
_component: "@budibase/standard-components/tableblock",
_id: "foo",
showTitleButton: true,
titleButtonURL: "/url",
titleButtonPeek: true,
},
]
await config.createScreen(screen)
// Run migration
screen = await dbCore.doWithDB(app.appId, async (db: any) => {
await runMigration(db)
return await db.get(screen._id)
})
// Verify new "onClickTitleButton" setting
const onClick = screen.props._children?.[0].onClickTitleButton
expect(onClick).toBeDefined()
expect(onClick.length).toBe(1)
expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
expect(onClick[0].parameters.url).toBe("/url")
expect(onClick[0].parameters.peek).toBeTruthy()
})
it("ignores components that have already been migrated", async () => {
// Add legacy table block as first child
screen.props._children = [
{
_instanceName: "Table Block",
_styles: {},
_component: "@budibase/standard-components/tableblock",
_id: "foo",
linkRows: true,
linkURL: "/rows/:id",
linkPeek: true,
linkColumn: "name",
onClick: "foo",
},
]
const initialDefinition = JSON.stringify(screen.props._children?.[0])
await config.createScreen(screen)
// Run migration
screen = await dbCore.doWithDB(app.appId, async (db: any) => {
await runMigration(db)
return await db.get(screen._id)
})
// Verify new "onClick" setting
const newDefinition = JSON.stringify(screen.props._children?.[0])
expect(initialDefinition).toEqual(newDefinition)
})
})

34
packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js
View File

@ -1,34 +0,0 @@
jest.mock("@budibase/backend-core", () => {
const core = jest.requireActual("@budibase/backend-core")
return {
...core,
db: {
...core.db,
createNewUserEmailView: jest.fn(),
},
}
})
const { context, db: dbCore } = require("@budibase/backend-core")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
// mock email view creation
const migration = require("../userEmailViewCasing")
describe("run", () => {
let config = new TestConfig(false)
beforeAll(async () => {
await config.init()
})
afterAll(config.end)
it("runs successfully", async () => {
await config.doInTenant(async () => {
const globalDb = context.getGlobalDB()
await migration.run(globalDb)
expect(dbCore.createNewUserEmailView).toHaveBeenCalledTimes(1)
})
})
})

3
packages/server/src/migrations/functions/usageQuotas/index.ts
View File

@ -1,3 +0,0 @@
export const runQuotaMigration = async (migration: () => Promise<void>) => {
await migration()
}

13
packages/server/src/migrations/functions/usageQuotas/syncApps.ts
View File

@ -1,13 +0,0 @@
import { db as dbCore } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
export const run = async () => {
// get app count
const devApps = await dbCore.getAllApps({ dev: true })
const appCount = devApps ? devApps.length : 0
// sync app count
console.log(`Syncing app count: ${appCount}`)
await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
}

13
packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
View File

@ -1,13 +0,0 @@
import { users } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
export const run = async () => {
const creatorCount = await users.getCreatorCount()
console.log(`Syncing creator count: ${creatorCount}`)
await quotas.setUsage(
creatorCount,
StaticQuotaName.CREATORS,
QuotaUsageType.STATIC
)
}

10
packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts
View File

@ -1,10 +0,0 @@
import { logging } from "@budibase/backend-core"
import { plugins } from "@budibase/pro"
export const run = async () => {
try {
await plugins.checkPluginQuotas()
} catch (err) {
logging.logAlert("Failed to update plugin quotas", err)
}
}

27
packages/server/src/migrations/functions/usageQuotas/syncRows.ts
View File

@ -1,27 +0,0 @@
import { db as dbCore } from "@budibase/backend-core"
import { getUniqueRows } from "../../../utilities/usageQuota/rows"
import { quotas } from "@budibase/pro"
import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types"
export const run = async () => {
// get all rows in all apps
const allApps = (await dbCore.getAllApps({ all: true })) as App[]
const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
const { appRows } = await getUniqueRows(appIds)
// get the counts per app
const counts: { [key: string]: number } = {}
let rowCount = 0
Object.entries(appRows).forEach(([appId, rows]) => {
counts[appId] = rows.length
rowCount += rows.length
})
// sync row count
console.log(`Syncing row count: ${rowCount}`)
await quotas.setUsagePerApp(
counts,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
}

9
packages/server/src/migrations/functions/usageQuotas/syncUsers.ts
View File

@ -1,9 +0,0 @@
import { users } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
export const run = async () => {
const userCount = await users.getUserCount()
console.log(`Syncing user count: ${userCount}`)
await quotas.setUsage(userCount, StaticQuotaName.USERS, QuotaUsageType.STATIC)
}

35
packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts
View File

@ -1,35 +0,0 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import * as syncApps from "../syncApps"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
describe("syncApps", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
})
afterAll(config.end)
it("runs successfully", async () => {
return config.doInContext(undefined, async () => {
// create the usage quota doc and mock usages
await quotas.getQuotaUsage()
await quotas.setUsage(3, StaticQuotaName.APPS, QuotaUsageType.STATIC)
let usageDoc = await quotas.getQuotaUsage()
expect(usageDoc.usageQuota.apps).toEqual(3)
// create an extra app to test the migration
await config.createApp("quota-test")
// migrate
await syncApps.run()
// assert the migration worked
usageDoc = await quotas.getQuotaUsage()
expect(usageDoc.usageQuota.apps).toEqual(2)
})
})
})

26
packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts
View File

@ -1,26 +0,0 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import * as syncCreators from "../syncCreators"
import { quotas } from "@budibase/pro"
describe("syncCreators", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
})
afterAll(config.end)
it("syncs creators", async () => {
return config.doInContext(undefined, async () => {
await config.createUser({ admin: { global: true } })
await syncCreators.run()
const usageDoc = await quotas.getQuotaUsage()
// default + additional creator
const creatorsCount = 2
expect(usageDoc.usageQuota.creators).toBe(creatorsCount)
})
})
})

53
packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
View File

@ -1,53 +0,0 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import * as syncRows from "../syncRows"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
import { db as dbCore, context } from "@budibase/backend-core"
describe("syncRows", () => {
const config = new TestConfig()
beforeEach(async () => {
await config.init()
})
afterAll(config.end)
it("runs successfully", async () => {
return config.doInContext(undefined, async () => {
// create the usage quota doc and mock usages
await quotas.getQuotaUsage()
await quotas.setUsage(300, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
let usageDoc = await quotas.getQuotaUsage()
expect(usageDoc.usageQuota.rows).toEqual(300)
// app 1
const app1 = config.app
await context.doInAppContext(app1!.appId, async () => {
await config.createTable()
await config.createRow()
})
// app 2
const app2 = await config.createApp("second-app")
await context.doInAppContext(app2.appId, async () => {
await config.createTable()
await config.createRow()
await config.createRow()
})
// migrate
await syncRows.run()
// assert the migration worked
usageDoc = await quotas.getQuotaUsage()
expect(usageDoc.usageQuota.rows).toEqual(3)
expect(
usageDoc.apps?.[dbCore.getProdAppID(app1!.appId)].usageQuota.rows
).toEqual(1)
expect(
usageDoc.apps?.[dbCore.getProdAppID(app2.appId)].usageQuota.rows
).toEqual(2)
})
})
})

26
packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts
View File

@ -1,26 +0,0 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import * as syncUsers from "../syncUsers"
import { quotas } from "@budibase/pro"
describe("syncUsers", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
})
afterAll(config.end)
it("syncs users", async () => {
return config.doInContext(undefined, async () => {
await config.createUser()
await syncUsers.run()
const usageDoc = await quotas.getQuotaUsage()
// default + additional user
const userCount = 2
expect(usageDoc.usageQuota.users).toBe(userCount)
})
})
})

13
packages/server/src/migrations/functions/userEmailViewCasing.ts
View File

@ -1,13 +0,0 @@
import { db as dbCore } from "@budibase/backend-core"
/**
* Date:
* October 2021
*
* Description:
* Recreate the user email view to include latest changes i.e. lower casing the email address
*/
export const run = async () => {
await dbCore.createNewUserEmailView()
}

115
packages/server/src/migrations/index.ts
View File

@ -1,115 +0,0 @@
import { locks, migrations } from "@budibase/backend-core"
import {
Migration,
MigrationOptions,
MigrationName,
LockType,
LockName,
} from "@budibase/types"
import env from "../environment"
// migration functions
import * as userEmailViewCasing from "./functions/userEmailViewCasing"
import * as syncQuotas from "./functions/syncQuotas"
import * as appUrls from "./functions/appUrls"
import * as tableSettings from "./functions/tableSettings"
import * as backfill from "./functions/backfill"
/**
* Populate the migration function and additional configuration from
* the static migration definitions.
*/
export const buildMigrations = () => {
const definitions = migrations.DEFINITIONS
const serverMigrations: Migration[] = []
for (const definition of definitions) {
switch (definition.name) {
case MigrationName.USER_EMAIL_VIEW_CASING: {
serverMigrations.push({
...definition,
fn: userEmailViewCasing.run,
})
break
}
case MigrationName.SYNC_QUOTAS: {
serverMigrations.push({
...definition,
fn: syncQuotas.run,
})
break
}
case MigrationName.APP_URLS: {
serverMigrations.push({
...definition,
appOpts: { all: true },
fn: appUrls.run,
})
break
}
case MigrationName.EVENT_APP_BACKFILL: {
serverMigrations.push({
...definition,
appOpts: { all: true },
fn: backfill.app.run,
silent: !!env.SELF_HOSTED, // reduce noisy logging
preventRetry: !!env.SELF_HOSTED, // only ever run once
})
break
}
case MigrationName.EVENT_GLOBAL_BACKFILL: {
serverMigrations.push({
...definition,
fn: backfill.global.run,
silent: !!env.SELF_HOSTED, // reduce noisy logging
preventRetry: !!env.SELF_HOSTED, // only ever run once
})
break
}
case MigrationName.EVENT_INSTALLATION_BACKFILL: {
serverMigrations.push({
...definition,
fn: backfill.installation.run,
silent: !!env.SELF_HOSTED, // reduce noisy logging
preventRetry: !!env.SELF_HOSTED, // only ever run once
})
break
}
case MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS: {
serverMigrations.push({
...definition,
appOpts: { dev: true },
fn: tableSettings.run,
})
break
}
}
}
return serverMigrations
}
export const MIGRATIONS = buildMigrations()
export const migrate = async (options?: MigrationOptions) => {
if (env.SELF_HOSTED) {
// self host runs migrations on startup
// make sure only a single instance runs them
await migrateWithLock(options)
} else {
await migrations.runMigrations(MIGRATIONS, options)
}
}
const migrateWithLock = async (options?: MigrationOptions) => {
await locks.doWithLock(
{
type: LockType.TRY_ONCE,
name: LockName.MIGRATIONS,
ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
systemLock: true,
},
async () => {
await migrations.runMigrations(MIGRATIONS, options)
}
)
}

40
packages/server/src/migrations/tests/helpers.ts
View File

@ -1,40 +0,0 @@
// Mimic configs test configuration from worker, creation configs directly in database
import * as structures from "./structures"
import { configs } from "@budibase/backend-core"
import { Config } from "@budibase/types"
export const saveSettingsConfig = async (globalDb: any) => {
const config = structures.settings()
await saveConfig(config, globalDb)
}
export const saveGoogleConfig = async (globalDb: any) => {
const config = structures.google()
await saveConfig(config, globalDb)
}
export const saveOIDCConfig = async (globalDb: any) => {
const config = structures.oidc()
await saveConfig(config, globalDb)
}
export const saveSmtpConfig = async (globalDb: any) => {
const config = structures.smtp()
await saveConfig(config, globalDb)
}
const saveConfig = async (config: Config, globalDb: any) => {
config._id = configs.generateConfigID(config.type)
let response
try {
response = await globalDb.get(config._id)
config._rev = response._rev
await globalDb.put(config)
} catch (e: any) {
if (e.status === 404) {
await globalDb.put(config)
}
}
}

137
packages/server/src/migrations/tests/index.spec.ts
View File

@ -1,137 +0,0 @@
import {
events,
migrations,
tenancy,
DocumentType,
context,
} from "@budibase/backend-core"
import TestConfig from "../../tests/utilities/TestConfiguration"
import * as structures from "../../tests/utilities/structures"
import { MIGRATIONS } from "../"
import * as helpers from "./helpers"
import tk from "timekeeper"
import { View } from "@budibase/types"
const timestamp = new Date().toISOString()
tk.freeze(timestamp)
const clearMigrations = async () => {
const dbs = [context.getDevAppDB(), context.getProdAppDB()]
for (const db of dbs) {
const doc = await db.get<any>(DocumentType.MIGRATIONS)
const newDoc = { _id: doc._id, _rev: doc._rev }
await db.put(newDoc)
}
}
describe("migrations", () => {
const config = new TestConfig()
beforeAll(async () => {
await config.init()
})
afterAll(() => {
config.end()
})
describe("backfill", () => {
it("runs app db migration", async () => {
await config.doInContext(undefined, async () => {
await clearMigrations()
await config.createAutomation()
await config.createAutomation(structures.newAutomation())
await config.createDatasource()
await config.createDatasource()
await config.createLayout()
await config.createQuery()
await config.createQuery()
await config.createRole()
await config.createRole()
await config.createTable()
await config.createLegacyView()
await config.createTable()
await config.createLegacyView(
structures.view(config.table!._id!) as View
)
await config.createScreen()
await config.createScreen()
jest.clearAllMocks()
const migration = MIGRATIONS.filter(
m => m.name === "event_app_backfill"
)[0]
await migrations.runMigration(migration)
expect(events.app.created).toHaveBeenCalledTimes(1)
expect(events.app.published).toHaveBeenCalledTimes(1)
expect(events.automation.created).toHaveBeenCalledTimes(2)
expect(events.automation.stepCreated).toHaveBeenCalledTimes(1)
expect(events.datasource.created).toHaveBeenCalledTimes(2)
expect(events.layout.created).toHaveBeenCalledTimes(1)
expect(events.query.created).toHaveBeenCalledTimes(2)
expect(events.role.created).toHaveBeenCalledTimes(3) // created roles + admin (created on table creation)
expect(events.table.created).toHaveBeenCalledTimes(3)
expect(events.backfill.appSucceeded).toHaveBeenCalledTimes(2)
// to make sure caching is working as expected
expect(
events.processors.analyticsProcessor.processEvent
).toHaveBeenCalledTimes(20) // Addition of of the events above
})
})
})
it("runs global db migration", async () => {
await config.doInContext(undefined, async () => {
await clearMigrations()
const appId = config.getProdAppId()
const roles = { [appId]: "role_12345" }
await config.createUser({
builder: { global: false },
admin: { global: true },
roles,
}) // admin only
await config.createUser({
builder: { global: false },
admin: { global: false },
roles,
}) // non admin non builder
await config.createTable()
await config.createRow()
await config.createRow()
const db = tenancy.getGlobalDB()
await helpers.saveGoogleConfig(db)
await helpers.saveOIDCConfig(db)
await helpers.saveSettingsConfig(db)
await helpers.saveSmtpConfig(db)
jest.clearAllMocks()
const migration = MIGRATIONS.filter(
m => m.name === "event_global_backfill"
)[0]
await migrations.runMigration(migration)
expect(events.user.created).toHaveBeenCalledTimes(3)
expect(events.role.assigned).toHaveBeenCalledTimes(2)
expect(events.user.permissionBuilderAssigned).toHaveBeenCalledTimes(1) // default test user
expect(events.user.permissionAdminAssigned).toHaveBeenCalledTimes(1) // admin from above
expect(events.rows.created).toHaveBeenCalledTimes(1)
expect(events.rows.created).toHaveBeenCalledWith(2, timestamp)
expect(events.email.SMTPCreated).toHaveBeenCalledTimes(1)
expect(events.auth.SSOCreated).toHaveBeenCalledTimes(2)
expect(events.auth.SSOActivated).toHaveBeenCalledTimes(2)
expect(events.org.logoUpdated).toHaveBeenCalledTimes(1)
expect(events.org.nameUpdated).toHaveBeenCalledTimes(1)
expect(events.org.platformURLUpdated).toHaveBeenCalledTimes(1)
expect(events.backfill.tenantSucceeded).toHaveBeenCalledTimes(1)
// to make sure caching is working as expected
expect(
events.processors.analyticsProcessor.processEvent
).toHaveBeenCalledTimes(19)
})
})
})

67
packages/server/src/migrations/tests/structures.ts
View File

@ -1,67 +0,0 @@
import { utils } from "@budibase/backend-core"
import {
SMTPConfig,
OIDCConfig,
GoogleConfig,
SettingsConfig,
ConfigType,
} from "@budibase/types"
export const oidc = (conf?: OIDCConfig): OIDCConfig => {
return {
type: ConfigType.OIDC,
config: {
configs: [
{
configUrl: "http://someconfigurl",
clientID: "clientId",
clientSecret: "clientSecret",
logo: "Microsoft",
name: "Active Directory",
uuid: utils.newid(),
activated: true,
scopes: [],
...conf,
},
],
},
}
}
export const google = (conf?: GoogleConfig): GoogleConfig => {
return {
type: ConfigType.GOOGLE,
config: {
clientID: "clientId",
clientSecret: "clientSecret",
activated: true,
...conf,
},
}
}
export const smtp = (conf?: SMTPConfig): SMTPConfig => {
return {
type: ConfigType.SMTP,
config: {
port: 12345,
host: "smtptesthost.com",
from: "testfrom@example.com",
subject: "Hello!",
secure: false,
...conf,
},
}
}
export const settings = (conf?: SettingsConfig): SettingsConfig => {
return {
type: ConfigType.SETTINGS,
config: {
platformUrl: "http://mycustomdomain.com",
logoUrl: "http://mylogourl,com",
company: "mycompany",
...conf,
},
}
}

6
packages/server/src/sdk/app/views/external.ts
View File

@ -22,7 +22,9 @@ export async function get(viewId: string): Promise<ViewV2> {
return ensureQueryUISet(found) return ensureQueryUISet(found)
} }
export async function getEnriched(viewId: string): Promise<ViewV2Enriched> { export async function getEnriched(
viewId: string
): Promise<ViewV2Enriched | undefined> {
const { tableId } = utils.extractViewInfoFromID(viewId) const { tableId } = utils.extractViewInfoFromID(viewId)
const { datasourceId, tableName } = breakExternalTableId(tableId) const { datasourceId, tableName } = breakExternalTableId(tableId)
@ -32,7 +34,7 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
const views = Object.values(table.views!).filter(isV2) const views = Object.values(table.views!).filter(isV2)
const found = views.find(v => v.id === viewId) const found = views.find(v => v.id === viewId)
if (!found) { if (!found) {
throw new Error("No view found") return
} }
return await enrichSchema(ensureQueryUISet(found), table.schema) return await enrichSchema(ensureQueryUISet(found), table.schema)
} }

4
packages/server/src/sdk/app/views/index.ts
View File

@ -40,7 +40,9 @@ export async function get(viewId: string): Promise<ViewV2> {
return pickApi(tableId).get(viewId) return pickApi(tableId).get(viewId)
} }
export async function getEnriched(viewId: string): Promise<ViewV2Enriched> { export async function getEnriched(
viewId: string
): Promise<ViewV2Enriched | undefined> {
const { tableId } = utils.extractViewInfoFromID(viewId) const { tableId } = utils.extractViewInfoFromID(viewId)
return pickApi(tableId).getEnriched(viewId) return pickApi(tableId).getEnriched(viewId)
} }

6
packages/server/src/sdk/app/views/internal.ts
View File

@ -17,13 +17,15 @@ export async function get(viewId: string): Promise<ViewV2> {
return ensureQueryUISet(found) return ensureQueryUISet(found)
} }
export async function getEnriched(viewId: string): Promise<ViewV2Enriched> { export async function getEnriched(
viewId: string
): Promise<ViewV2Enriched | undefined> {
const { tableId } = utils.extractViewInfoFromID(viewId) const { tableId } = utils.extractViewInfoFromID(viewId)
const table = await sdk.tables.getTable(tableId) const table = await sdk.tables.getTable(tableId)
const views = Object.values(table.views!).filter(isV2) const views = Object.values(table.views!).filter(isV2)
const found = views.find(v => v.id === viewId) const found = views.find(v => v.id === viewId)
if (!found) { if (!found) {
throw new Error("No view found") return
} }
return await enrichSchema(ensureQueryUISet(found), table.schema) return await enrichSchema(ensureQueryUISet(found), table.schema)
} }

13
packages/server/src/startup/index.ts
View File

@ -15,7 +15,6 @@ import { watch } from "../watch"
import * as automations from "../automations" import * as automations from "../automations"
import * as fileSystem from "../utilities/fileSystem" import * as fileSystem from "../utilities/fileSystem"
import { default as eventEmitter, init as eventInit } from "../events" import { default as eventEmitter, init as eventInit } from "../events"
import * as migrations from "../migrations"
import * as bullboard from "../automations/bullboard" import * as bullboard from "../automations/bullboard"
import * as appMigrations from "../appMigrations/queue" import * as appMigrations from "../appMigrations/queue"
import * as pro from "@budibase/pro" import * as pro from "@budibase/pro"
@ -106,18 +105,6 @@ export async function startup(
initialiseWebsockets(app, server) initialiseWebsockets(app, server)
} }
// run migrations on startup if not done via http
// not recommended in a clustered environment
if (!env.HTTP_MIGRATIONS && !env.isTest()) {
console.log("Running migrations")
try {
await migrations.migrate()
} catch (e) {
logging.logAlert("Error performing migrations. Exiting.", e)
shutdown(server)
}
}
// monitor plugin directory if required // monitor plugin directory if required
if ( if (
env.SELF_HOSTED && env.SELF_HOSTED &&

1
packages/shared-core/src/automations/index.ts Normal file
View File

@ -0,0 +1 @@
export * as steps from "./steps/index"

47
packages/shared-core/src/automations/steps/bash.ts Normal file
View File

@ -0,0 +1,47 @@
import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Bash Scripting",
tagline: "Execute a bash command",
icon: "JourneyEvent",
description: "Run a bash script",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.EXECUTE_BASH,
inputs: {},
schema: {
inputs: {
properties: {
code: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.CODE,
title: "Code",
},
},
required: ["code"],
},
outputs: {
properties: {
stdout: {
type: AutomationIOType.STRING,
description: "Standard output of your bash command or script",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the command was successful",
},
},
required: ["stdout"],
},
},
}

0
packages/server/src/automations/steps/branch.ts → packages/shared-core/src/automations/steps/branch.ts
View File

43
packages/shared-core/src/automations/steps/collect.ts Normal file
View File

@ -0,0 +1,43 @@
import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Collect Data",
tagline: "Collect data to be sent to design",
icon: "Collection",
description:
"Collects specified data so it can be provided to the design section",
type: AutomationStepType.ACTION,
internal: true,
features: {},
stepId: AutomationActionStepId.COLLECT,
inputs: {},
schema: {
inputs: {
properties: {
collection: {
type: AutomationIOType.STRING,
title: "What to Collect",
},
},
required: ["collection"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
value: {
type: AutomationIOType.STRING,
description: "Collected data",
},
},
required: ["success", "value"],
},
},
}

67
packages/shared-core/src/automations/steps/createRow.ts Normal file
View File

@ -0,0 +1,67 @@
import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Create Row",
tagline: "Create a {{inputs.enriched.table.name}} row",
icon: "TableRowAddBottom",
description: "Add a row to your database",
type: AutomationStepType.ACTION,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
stepId: AutomationActionStepId.CREATE_ROW,
inputs: {},
schema: {
inputs: {
properties: {
row: {
type: AutomationIOType.OBJECT,
properties: {
tableId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.TABLE,
},
},
customType: AutomationCustomIOType.ROW,
title: "Table",
required: ["tableId"],
},
},
required: ["row"],
},
outputs: {
properties: {
row: {
type: AutomationIOType.OBJECT,
customType: AutomationCustomIOType.ROW,
description: "The new row",
},
response: {
type: AutomationIOType.OBJECT,
description: "The response from the table",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the row creation was successful",
},
id: {
type: AutomationIOType.STRING,
description: "The identifier of the new row",
},
revision: {
type: AutomationIOType.STRING,
description: "The revision of the new row",
},
},
required: ["success", "id", "revision"],
},
},
}

38
packages/shared-core/src/automations/steps/delay.ts Normal file
View File

@ -0,0 +1,38 @@
import {
AutomationActionStepId,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Delay",
icon: "Clock",
tagline: "Delay for {{inputs.time}} milliseconds",
description: "Delay the automation until an amount of time has passed",
stepId: AutomationActionStepId.DELAY,
internal: true,
features: {},
inputs: {},
schema: {
inputs: {
properties: {
time: {
type: AutomationIOType.NUMBER,
title: "Delay in milliseconds",
},
},
required: ["time"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the delay was successful",
},
},
required: ["success"],
},
},
type: AutomationStepType.LOGIC,
}

56
packages/shared-core/src/automations/steps/deleteRow.ts Normal file
View File

@ -0,0 +1,56 @@
import {
AutomationActionStepId,
AutomationStepType,
AutomationIOType,
AutomationCustomIOType,
AutomationFeature,
AutomationStepDefinition,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
description: "Delete a row from your database",
icon: "TableRowRemoveCenter",
name: "Delete Row",
tagline: "Delete a {{inputs.enriched.table.name}} row",
type: AutomationStepType.ACTION,
stepId: AutomationActionStepId.DELETE_ROW,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
tableId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.TABLE,
title: "Table",
},
id: {
type: AutomationIOType.STRING,
title: "Row ID",
},
},
required: ["tableId", "id"],
},
outputs: {
properties: {
row: {
type: AutomationIOType.OBJECT,
customType: AutomationCustomIOType.ROW,
description: "The deleted row",
},
response: {
type: AutomationIOType.OBJECT,
description: "The response from the table",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the deletion was successful",
},
},
required: ["row", "success"],
},
},
}

60
packages/shared-core/src/automations/steps/discord.ts Normal file
View File

@ -0,0 +1,60 @@
import {
AutomationActionStepId,
AutomationStepType,
AutomationIOType,
AutomationFeature,
AutomationStepDefinition,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "Discord Message",
tagline: "Send a message to a Discord server",
description: "Send a message to a Discord server",
icon: "ri-discord-line",
stepId: AutomationActionStepId.discord,
type: AutomationStepType.ACTION,
internal: false,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
url: {
type: AutomationIOType.STRING,
title: "Discord Webhook URL",
},
username: {
type: AutomationIOType.STRING,
title: "Bot Name",
},
avatar_url: {
type: AutomationIOType.STRING,
title: "Bot Avatar URL",
},
content: {
type: AutomationIOType.STRING,
title: "Message",
},
},
required: ["url", "content"],
},
outputs: {
properties: {
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code of the request",
},
response: {
type: AutomationIOType.STRING,
description: "The response from the Discord Webhook",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the message sent successfully",
},
},
},
},
}

59
packages/shared-core/src/automations/steps/executeQuery.ts Normal file
View File

@ -0,0 +1,59 @@
import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "External Data Connector",
tagline: "Execute Data Connector",
icon: "Data",
description: "Execute a query in an external data connector",
type: AutomationStepType.ACTION,
stepId: AutomationActionStepId.EXECUTE_QUERY,
internal: true,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
query: {
type: AutomationIOType.OBJECT,
properties: {
queryId: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.QUERY,
},
},
customType: AutomationCustomIOType.QUERY_PARAMS,
title: "Parameters",
required: ["queryId"],
},
},
required: ["query"],
},
outputs: {
properties: {
response: {
type: AutomationIOType.OBJECT,
description: "The response from the datasource execution",
},
info: {
type: AutomationIOType.OBJECT,
description:
"Some query types may return extra data, like headers from a REST query",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
},
required: ["response", "success"],
},
},
}

47
packages/shared-core/src/automations/steps/executeScript.ts Normal file
View File

@ -0,0 +1,47 @@
import {
AutomationActionStepId,
AutomationCustomIOType,
AutomationFeature,
AutomationIOType,
AutomationStepDefinition,
AutomationStepType,
} from "@budibase/types"
export const definition: AutomationStepDefinition = {
name: "JS Scripting",
tagline: "Execute JavaScript Code",
icon: "Code",
description: "Run a piece of JavaScript code in your automation",
type: AutomationStepType.ACTION,
internal: true,
stepId: AutomationActionStepId.EXECUTE_SCRIPT,
inputs: {},
features: {
[AutomationFeature.LOOPING]: true,
},
schema: {
inputs: {
properties: {
code: {
type: AutomationIOType.STRING,
customType: AutomationCustomIOType.CODE,
title: "Code",
},
},
required: ["code"],
},
outputs: {
properties: {
value: {
type: AutomationIOType.STRING,
description: "The result of the return statement",
},
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
},
required: ["success"],
},
},
}

69
packages/shared-core/src/automations/steps/filter.ts Normal file
View File

@ -0,0 +1,69 @@
import {
AutomationActionStepId,
AutomationStepDefinition,
AutomationStepType,
AutomationIOType,
} from "@budibase/types"
export const FilterConditions = {
EQUAL: "EQUAL",
NOT_EQUAL: "NOT_EQUAL",
GREATER_THAN: "GREATER_THAN",
LESS_THAN: "LESS_THAN",
}
export const PrettyFilterConditions = {
[FilterConditions.EQUAL]: "Equals",
[FilterConditions.NOT_EQUAL]: "Not equals",
[FilterConditions.GREATER_THAN]: "Greater than",
[FilterConditions.LESS_THAN]: "Less than",
}
export const definition: AutomationStepDefinition = {
name: "Condition",
tagline: "{{inputs.field}} {{inputs.condition}} {{inputs.value}}",
icon: "Branch2",
description:
"Conditionally halt automations which do not meet certain conditions",
type: AutomationStepType.LOGIC,
internal: true,
features: {},
stepId: AutomationActionStepId.FILTER,
inputs: {
condition: FilterConditions.EQUAL,
},
schema: {
inputs: {
properties: {
field: {
type: AutomationIOType.STRING,
title: "Reference Value",
},
condition: {
type: AutomationIOType.STRING,
title: "Condition",
enum: Object.values(FilterConditions),
pretty: Object.values(PrettyFilterConditions),
},
value: {
type: AutomationIOType.STRING,
title: "Comparison Value",
},
},
required: ["field", "condition", "value"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether the action was successful",
},
result: {
type: AutomationIOType.BOOLEAN,
description: "Whether the logic block passed",
},
},
required: ["success", "result"],
},
},
}

Some files were not shown because too many files have changed in this diff Show More