From 80fbaa1599a277d1d7c8749cb9934d056e14248a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 15 Jan 2025 17:37:37 +0000 Subject: [PATCH 01/22] Remove old migration system. --- packages/backend-core/src/index.ts | 1 - .../src/migrations/definitions.ts | 40 ---- packages/backend-core/src/migrations/index.ts | 2 - .../backend-core/src/migrations/migrations.ts | 186 --------------- .../__snapshots__/migrations.spec.ts.snap | 11 - .../src/migrations/tests/migrations.spec.ts | 64 ------ .../users/{users.spec.js => users.spec.ts} | 21 +- packages/frontend-core/src/api/migrations.ts | 4 +- packages/server/scripts/dev/manage.js | 1 - .../server/src/api/controllers/application.ts | 9 - .../server/src/api/controllers/migrations.ts | 28 +-- packages/server/src/api/routes/migrations.ts | 10 +- packages/server/src/environment.ts | 1 - .../src/migrations/functions/appUrls.ts | 27 --- .../src/migrations/functions/backfill/app.ts | 149 ------------ .../functions/backfill/app/automations.ts | 26 --- .../functions/backfill/app/datasources.ts | 22 -- .../functions/backfill/app/layouts.ts | 29 --- .../functions/backfill/app/queries.ts | 47 ---- .../functions/backfill/app/roles.ts | 22 -- .../functions/backfill/app/screens.ts | 22 -- .../functions/backfill/app/tables.ts | 13 -- .../migrations/functions/backfill/global.ts | 214 ------------------ .../functions/backfill/global/configs.ts | 74 ------ .../functions/backfill/global/quotas.ts | 60 ----- .../functions/backfill/global/users.ts | 53 ----- .../migrations/functions/backfill/index.ts | 7 - .../functions/backfill/installation.ts | 50 ---- .../src/migrations/functions/syncQuotas.ts | 19 -- .../src/migrations/functions/tableSettings.ts | 145 ------------ .../functions/tests/appUrls.spec.js | 26 --- .../functions/tests/tableSettings.spec.ts | 144 ------------ .../tests/userEmailViewCasing.spec.js | 34 --- .../migrations/functions/usageQuotas/index.ts | 3 - .../functions/usageQuotas/syncApps.ts | 13 -- .../functions/usageQuotas/syncCreators.ts | 13 -- .../functions/usageQuotas/syncPlugins.ts | 10 - .../functions/usageQuotas/syncRows.ts | 27 --- .../functions/usageQuotas/syncUsers.ts | 9 - .../usageQuotas/tests/syncApps.spec.ts | 35 --- .../usageQuotas/tests/syncCreators.spec.ts | 26 --- .../usageQuotas/tests/syncRows.spec.ts | 53 ----- .../usageQuotas/tests/syncUsers.spec.ts | 26 --- .../functions/userEmailViewCasing.ts | 13 -- packages/server/src/migrations/index.ts | 115 ---------- .../server/src/migrations/tests/helpers.ts | 40 ---- .../server/src/migrations/tests/index.spec.ts | 137 ----------- .../server/src/migrations/tests/structures.ts | 67 ------ packages/server/src/startup/index.ts | 13 -- .../types/src/api/web/global/oldMigration.ts | 11 +- packages/types/src/api/web/system/index.ts | 1 - .../types/src/api/web/system/migration.ts | 8 - packages/types/src/sdk/index.ts | 1 - packages/types/src/sdk/migrations.ts | 57 ----- .../src/api/controllers/global/users.ts | 6 - .../src/api/controllers/system/migrations.ts | 23 -- packages/worker/src/api/routes/index.ts | 2 - .../src/api/routes/system/migrations.ts | 19 -- packages/worker/src/migrations/index.ts | 62 ----- 59 files changed, 16 insertions(+), 2335 deletions(-) delete mode 100644 packages/backend-core/src/migrations/definitions.ts delete mode 100644 packages/backend-core/src/migrations/index.ts delete mode 100644 packages/backend-core/src/migrations/migrations.ts delete mode 100644 packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap delete mode 100644 packages/backend-core/src/migrations/tests/migrations.spec.ts rename packages/backend-core/tests/core/users/{users.spec.js => users.spec.ts} (67%) delete mode 100644 packages/server/src/migrations/functions/appUrls.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/automations.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/datasources.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/layouts.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/queries.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/roles.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/screens.ts delete mode 100644 packages/server/src/migrations/functions/backfill/app/tables.ts delete mode 100644 packages/server/src/migrations/functions/backfill/global.ts delete mode 100644 packages/server/src/migrations/functions/backfill/global/configs.ts delete mode 100644 packages/server/src/migrations/functions/backfill/global/quotas.ts delete mode 100644 packages/server/src/migrations/functions/backfill/global/users.ts delete mode 100644 packages/server/src/migrations/functions/backfill/index.ts delete mode 100644 packages/server/src/migrations/functions/backfill/installation.ts delete mode 100644 packages/server/src/migrations/functions/syncQuotas.ts delete mode 100644 packages/server/src/migrations/functions/tableSettings.ts delete mode 100644 packages/server/src/migrations/functions/tests/appUrls.spec.js delete mode 100644 packages/server/src/migrations/functions/tests/tableSettings.spec.ts delete mode 100644 packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js delete mode 100644 packages/server/src/migrations/functions/usageQuotas/index.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/syncApps.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/syncCreators.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/syncRows.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/syncUsers.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts delete mode 100644 packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts delete mode 100644 packages/server/src/migrations/functions/userEmailViewCasing.ts delete mode 100644 packages/server/src/migrations/index.ts delete mode 100644 packages/server/src/migrations/tests/helpers.ts delete mode 100644 packages/server/src/migrations/tests/index.spec.ts delete mode 100644 packages/server/src/migrations/tests/structures.ts delete mode 100644 packages/types/src/api/web/system/migration.ts delete mode 100644 packages/types/src/sdk/migrations.ts delete mode 100644 packages/worker/src/api/controllers/system/migrations.ts delete mode 100644 packages/worker/src/api/routes/system/migrations.ts delete mode 100644 packages/worker/src/migrations/index.ts diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index dbdce51c50..d4e6e9a1ec 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -1,6 +1,5 @@ export * as configs from "./configs" export * as events from "./events" -export * as migrations from "./migrations" export * as users from "./users" export * as userUtils from "./users/utils" export * as roles from "./security/roles" diff --git a/packages/backend-core/src/migrations/definitions.ts b/packages/backend-core/src/migrations/definitions.ts deleted file mode 100644 index 0dd57fe639..0000000000 --- a/packages/backend-core/src/migrations/definitions.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { - MigrationType, - MigrationName, - MigrationDefinition, -} from "@budibase/types" - -export const DEFINITIONS: MigrationDefinition[] = [ - { - type: MigrationType.GLOBAL, - name: MigrationName.USER_EMAIL_VIEW_CASING, - }, - { - type: MigrationType.GLOBAL, - name: MigrationName.SYNC_QUOTAS, - }, - { - type: MigrationType.APP, - name: MigrationName.APP_URLS, - }, - { - type: MigrationType.APP, - name: MigrationName.EVENT_APP_BACKFILL, - }, - { - type: MigrationType.APP, - name: MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS, - }, - { - type: MigrationType.GLOBAL, - name: MigrationName.EVENT_GLOBAL_BACKFILL, - }, - { - type: MigrationType.INSTALLATION, - name: MigrationName.EVENT_INSTALLATION_BACKFILL, - }, - { - type: MigrationType.GLOBAL, - name: MigrationName.GLOBAL_INFO_SYNC_USERS, - }, -] diff --git a/packages/backend-core/src/migrations/index.ts b/packages/backend-core/src/migrations/index.ts deleted file mode 100644 index bce0cfc75c..0000000000 --- a/packages/backend-core/src/migrations/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./migrations" -export * from "./definitions" diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts deleted file mode 100644 index c8320b5724..0000000000 --- a/packages/backend-core/src/migrations/migrations.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { DEFAULT_TENANT_ID } from "../constants" -import { - DocumentType, - StaticDatabases, - getAllApps, - getGlobalDBName, - getDB, -} from "../db" -import environment from "../environment" -import * as platform from "../platform" -import * as context from "../context" -import { DEFINITIONS } from "." -import { - Migration, - MigrationOptions, - MigrationType, - MigrationNoOpOptions, - App, -} from "@budibase/types" - -export const getMigrationsDoc = async (db: any) => { - // get the migrations doc - try { - return await db.get(DocumentType.MIGRATIONS) - } catch (err: any) { - if (err.status && err.status === 404) { - return { _id: DocumentType.MIGRATIONS } - } else { - throw err - } - } -} - -export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => { - // filter migrations to the type and populate a no-op migration - const migrations: Migration[] = DEFINITIONS.filter( - def => def.type === opts.type - ).map(d => ({ ...d, fn: async () => {} })) - await runMigrations(migrations, { noOp: opts }) -} - -export const runMigration = async ( - migration: Migration, - options: MigrationOptions = {} -) => { - const migrationType = migration.type - const migrationName = migration.name - const silent = migration.silent - - const log = (message: string) => { - if (!silent) { - console.log(message) - } - } - - // get the db to store the migration in - let dbNames: string[] - if (migrationType === MigrationType.GLOBAL) { - dbNames = [getGlobalDBName()] - } else if (migrationType === MigrationType.APP) { - if (options.noOp) { - if (!options.noOp.appId) { - throw new Error("appId is required for noOp app migration") - } - dbNames = [options.noOp.appId] - } else { - const apps = (await getAllApps(migration.appOpts)) as App[] - dbNames = apps.map(app => app.appId) - } - } else if (migrationType === MigrationType.INSTALLATION) { - dbNames = [StaticDatabases.PLATFORM_INFO.name] - } else { - throw new Error(`Unrecognised migration type [${migrationType}]`) - } - - const length = dbNames.length - let count = 0 - - // run the migration against each db - for (const dbName of dbNames) { - count++ - const lengthStatement = length > 1 ? `[${count}/${length}]` : "" - - const db = getDB(dbName) - - try { - const doc = await getMigrationsDoc(db) - - // the migration has already been run - if (doc[migrationName]) { - // check for force - if ( - options.force && - options.force[migrationType] && - options.force[migrationType].includes(migrationName) - ) { - log(`[Migration: ${migrationName}] [DB: ${dbName}] Forcing`) - } else { - // no force, exit - return - } - } - - // check if the migration is not a no-op - if (!options.noOp) { - log( - `[Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` - ) - - if (migration.preventRetry) { - // eagerly set the completion date - // so that we never run this migration twice even upon failure - doc[migrationName] = Date.now() - const response = await db.put(doc) - doc._rev = response.rev - } - - // run the migration - if (migrationType === MigrationType.APP) { - await context.doInAppContext(db.name, async () => { - await migration.fn(db) - }) - } else { - await migration.fn(db) - } - - log(`[Migration: ${migrationName}] [DB: ${dbName}] Complete`) - } - - // mark as complete - doc[migrationName] = Date.now() - await db.put(doc) - } catch (err) { - console.error( - `[Migration: ${migrationName}] [DB: ${dbName}] Error: `, - err - ) - throw err - } - } -} - -export const runMigrations = async ( - migrations: Migration[], - options: MigrationOptions = {} -) => { - let tenantIds - - if (environment.MULTI_TENANCY) { - if (options.noOp) { - tenantIds = [options.noOp.tenantId] - } else if (!options.tenantIds || !options.tenantIds.length) { - // run for all tenants - tenantIds = await platform.tenants.getTenantIds() - } else { - tenantIds = options.tenantIds - } - } else { - // single tenancy - tenantIds = [DEFAULT_TENANT_ID] - } - - if (tenantIds.length > 1) { - console.log(`Checking migrations for ${tenantIds.length} tenants`) - } else { - console.log("Checking migrations") - } - - let count = 0 - // for all tenants - for (const tenantId of tenantIds) { - count++ - if (tenantIds.length > 1) { - console.log(`Progress [${count}/${tenantIds.length}]`) - } - // for all migrations - for (const migration of migrations) { - // run the migration - await context.doInTenant( - tenantId, - async () => await runMigration(migration, options) - ) - } - } - console.log("Migrations complete") -} diff --git a/packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap b/packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap deleted file mode 100644 index 377900b5d5..0000000000 --- a/packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap +++ /dev/null @@ -1,11 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`migrations should match snapshot 1`] = ` -{ - "_id": "migrations", - "_rev": "1-2f64479842a0513aa8b97f356b0b9127", - "createdAt": "2020-01-01T00:00:00.000Z", - "test": 1577836800000, - "updatedAt": "2020-01-01T00:00:00.000Z", -} -`; diff --git a/packages/backend-core/src/migrations/tests/migrations.spec.ts b/packages/backend-core/src/migrations/tests/migrations.spec.ts deleted file mode 100644 index af2eb33cf5..0000000000 --- a/packages/backend-core/src/migrations/tests/migrations.spec.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { testEnv, DBTestConfiguration } from "../../../tests/extra" -import * as migrations from "../index" -import * as context from "../../context" -import { MigrationType } from "@budibase/types" - -testEnv.multiTenant() - -describe("migrations", () => { - const config = new DBTestConfiguration() - - const migrationFunction = jest.fn() - - const MIGRATIONS = [ - { - type: MigrationType.GLOBAL, - name: "test" as any, - fn: migrationFunction, - }, - ] - - beforeEach(() => { - config.newTenant() - }) - - afterEach(async () => { - jest.clearAllMocks() - }) - - const migrate = () => { - return migrations.runMigrations(MIGRATIONS, { - tenantIds: [config.tenantId], - }) - } - - it("should run a new migration", async () => { - await config.doInTenant(async () => { - await migrate() - expect(migrationFunction).toHaveBeenCalled() - const db = context.getGlobalDB() - const doc = await migrations.getMigrationsDoc(db) - expect(doc.test).toBeDefined() - }) - }) - - it("should match snapshot", async () => { - await config.doInTenant(async () => { - await migrate() - const doc = await migrations.getMigrationsDoc(context.getGlobalDB()) - expect(doc).toMatchSnapshot() - }) - }) - - it("should skip a previously run migration", async () => { - await config.doInTenant(async () => { - const db = context.getGlobalDB() - await migrate() - const previousDoc = await migrations.getMigrationsDoc(db) - await migrate() - const currentDoc = await migrations.getMigrationsDoc(db) - expect(migrationFunction).toHaveBeenCalledTimes(1) - expect(currentDoc.test).toBe(previousDoc.test) - }) - }) -}) diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.ts similarity index 67% rename from packages/backend-core/tests/core/users/users.spec.js rename to packages/backend-core/tests/core/users/users.spec.ts index dde0d87fb7..b14f553266 100644 --- a/packages/backend-core/tests/core/users/users.spec.js +++ b/packages/backend-core/tests/core/users/users.spec.ts @@ -1,17 +1,17 @@ -const _ = require("lodash/fp") -const { structures } = require("../../../tests") +import { range } from "lodash/fp" +import { structures } from "../.." jest.mock("../../../src/context") jest.mock("../../../src/db") -const context = require("../../../src/context") -const db = require("../../../src/db") +import * as context from "../../../src/context" +import * as db from "../../../src/db" -const { getCreatorCount } = require("../../../src/users/users") +import { getCreatorCount } from "../../../src/users/users" describe("Users", () => { - let getGlobalDBMock - let paginationMock + let getGlobalDBMock: jest.SpyInstance + let paginationMock: jest.SpyInstance beforeEach(() => { jest.resetAllMocks() @@ -22,11 +22,10 @@ describe("Users", () => { jest.spyOn(db, "getGlobalUserParams") }) - it("Retrieves the number of creators", async () => { - const getUsers = (offset, limit, creators = false) => { - const range = _.range(offset, limit) + it("retrieves the number of creators", async () => { + const getUsers = (offset: number, limit: number, creators = false) => { const opts = creators ? { builder: { global: true } } : undefined - return range.map(() => structures.users.user(opts)) + return range(offset, limit).map(() => structures.users.user(opts)) } const page1Data = getUsers(0, 8) const page2Data = getUsers(8, 12, true) diff --git a/packages/frontend-core/src/api/migrations.ts b/packages/frontend-core/src/api/migrations.ts index 8213691205..35d2f95dbc 100644 --- a/packages/frontend-core/src/api/migrations.ts +++ b/packages/frontend-core/src/api/migrations.ts @@ -1,8 +1,8 @@ -import { GetOldMigrationStatus } from "@budibase/types" +import { GetMigrationStatus } from "@budibase/types" import { BaseAPIClient } from "./types" export interface MigrationEndpoints { - getMigrationStatus: () => Promise + getMigrationStatus: () => Promise } export const buildMigrationEndpoints = ( diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index 0794c53506..84e1abea69 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -43,7 +43,6 @@ async function init() { BB_ADMIN_USER_EMAIL: "", BB_ADMIN_USER_PASSWORD: "", PLUGINS_DIR: "", - HTTP_MIGRATIONS: "0", HTTP_LOGGING: "0", VERSION: "0.0.0+local", PASSWORD_MIN_LENGTH: "1", diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 4169087a63..b62d022cbd 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -27,7 +27,6 @@ import { env as envCore, ErrorCode, events, - migrations, objectStore, roles, tenancy, @@ -43,7 +42,6 @@ import { groups, licensing, quotas } from "@budibase/pro" import { App, Layout, - MigrationType, PlanType, Screen, UserCtx, @@ -488,13 +486,6 @@ async function creationEvents(request: BBRequest, app: App) { } async function appPostCreate(ctx: UserCtx, app: App) { - const tenantId = tenancy.getTenantId() - await migrations.backPopulateMigrations({ - type: MigrationType.APP, - tenantId, - appId: app.appId, - }) - await creationEvents(ctx.request, app) // app import, template creation and duplication diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts index edf4ec6f51..fc3a1a1548 100644 --- a/packages/server/src/api/controllers/migrations.ts +++ b/packages/server/src/api/controllers/migrations.ts @@ -1,35 +1,11 @@ import { context } from "@budibase/backend-core" -import { migrate as migrationImpl, MIGRATIONS } from "../../migrations" -import { - Ctx, - FetchOldMigrationResponse, - GetOldMigrationStatus, - RuneOldMigrationResponse, - RunOldMigrationRequest, -} from "@budibase/types" +import { Ctx, GetMigrationStatus } from "@budibase/types" import { getAppMigrationVersion, getLatestEnabledMigrationId, } from "../../appMigrations" -export async function migrate( - ctx: Ctx -) { - const options = ctx.request.body - // don't await as can take a while, just return - migrationImpl(options) - ctx.body = { message: "Migration started." } -} - -export async function fetchDefinitions( - ctx: Ctx -) { - ctx.body = MIGRATIONS -} - -export async function getMigrationStatus( - ctx: Ctx -) { +export async function getMigrationStatus(ctx: Ctx) { const appId = context.getAppId() if (!appId) { diff --git a/packages/server/src/api/routes/migrations.ts b/packages/server/src/api/routes/migrations.ts index 918b197de2..0ffc334551 100644 --- a/packages/server/src/api/routes/migrations.ts +++ b/packages/server/src/api/routes/migrations.ts @@ -1,16 +1,8 @@ import Router from "@koa/router" import * as migrationsController from "../controllers/migrations" -import { auth } from "@budibase/backend-core" const router: Router = new Router() -router - .post("/api/migrations/run", auth.internalApi, migrationsController.migrate) - .get( - "/api/migrations/definitions", - auth.internalApi, - migrationsController.fetchDefinitions - ) - .get("/api/migrations/status", migrationsController.getMigrationStatus) +router.get("/api/migrations/status", migrationsController.getMigrationStatus) export default router diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 45d675ec3f..a9867b1231 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -54,7 +54,6 @@ const environment = { REDIS_URL: process.env.REDIS_URL, REDIS_PASSWORD: process.env.REDIS_PASSWORD, REDIS_CLUSTERED: process.env.REDIS_CLUSTERED, - HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS, CLUSTER_MODE: process.env.CLUSTER_MODE, API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/server/src/migrations/functions/appUrls.ts b/packages/server/src/migrations/functions/appUrls.ts deleted file mode 100644 index be03d3c81e..0000000000 --- a/packages/server/src/migrations/functions/appUrls.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { db as dbCore } from "@budibase/backend-core" -import sdk from "../../sdk" - -/** - * Date: - * January 2022 - * - * Description: - * Add the url to the app metadata if it doesn't exist - */ -export const run = async (appDb: any) => { - let metadata - try { - metadata = await appDb.get(dbCore.DocumentType.APP_METADATA) - } catch (e) { - // sometimes the metadata document doesn't exist - // exit early instead of failing the migration - console.error("Error retrieving app metadata. Skipping", e) - return - } - - if (!metadata.url) { - metadata.url = sdk.applications.getAppUrl({ name: metadata.name }) - console.log(`Adding url to app: ${metadata.url}`) - await appDb.put(metadata) - } -} diff --git a/packages/server/src/migrations/functions/backfill/app.ts b/packages/server/src/migrations/functions/backfill/app.ts deleted file mode 100644 index 51a37108b0..0000000000 --- a/packages/server/src/migrations/functions/backfill/app.ts +++ /dev/null @@ -1,149 +0,0 @@ -import * as automations from "./app/automations" -import * as datasources from "./app/datasources" -import * as layouts from "./app/layouts" -import * as queries from "./app/queries" -import * as roles from "./app/roles" -import * as tables from "./app/tables" -import * as screens from "./app/screens" -import * as global from "./global" -import { App, AppBackfillSucceededEvent, Event } from "@budibase/types" -import { db as dbUtils, events } from "@budibase/backend-core" -import env from "../../../environment" -import { DEFAULT_TIMESTAMP } from "." - -const failGraceful = env.SELF_HOSTED && !env.isDev() - -const handleError = (e: any, errors?: any) => { - if (failGraceful) { - if (errors) { - errors.push(e) - } - return - } - console.trace(e) - throw e -} - -const EVENTS = [ - Event.AUTOMATION_CREATED, - Event.AUTOMATION_STEP_CREATED, - Event.DATASOURCE_CREATED, - Event.LAYOUT_CREATED, - Event.QUERY_CREATED, - Event.ROLE_CREATED, - Event.SCREEN_CREATED, - Event.TABLE_CREATED, - Event.VIEW_CREATED, - Event.VIEW_CALCULATION_CREATED, - Event.VIEW_FILTER_CREATED, - Event.APP_PUBLISHED, - Event.APP_CREATED, -] - -/** - * Date: - * May 2022 - * - * Description: - * Backfill app events. - */ - -export const run = async (appDb: any) => { - try { - if (await global.isComplete()) { - // make sure new apps aren't backfilled - // return if the global migration for this tenant is complete - // which runs after the app migrations - return - } - - // tell the event pipeline to start caching - // events for this tenant - await events.backfillCache.start(EVENTS) - - let timestamp: string | number = DEFAULT_TIMESTAMP - const app: App = await appDb.get(dbUtils.DocumentType.APP_METADATA) - if (app.createdAt) { - timestamp = app.createdAt as string - } - - if (dbUtils.isProdAppID(app.appId)) { - await events.app.published(app, timestamp) - } - - const totals: any = {} - const errors: any = [] - - if (dbUtils.isDevAppID(app.appId)) { - await events.app.created(app, timestamp) - try { - totals.automations = await automations.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.datasources = await datasources.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.layouts = await layouts.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.queries = await queries.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.roles = await roles.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.screens = await screens.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.tables = await tables.backfill(appDb, timestamp) - } catch (e) { - handleError(e, errors) - } - } - - const properties: AppBackfillSucceededEvent = { - appId: app.appId, - automations: totals.automations, - datasources: totals.datasources, - layouts: totals.layouts, - queries: totals.queries, - roles: totals.roles, - tables: totals.tables, - screens: totals.screens, - } - - if (errors.length) { - properties.errors = errors.map((e: any) => - JSON.stringify(e, Object.getOwnPropertyNames(e)) - ) - properties.errorCount = errors.length - } else { - properties.errorCount = 0 - } - - await events.backfill.appSucceeded(properties) - // tell the event pipeline to stop caching events for this tenant - await events.backfillCache.end() - } catch (e) { - handleError(e) - await events.backfill.appFailed(e) - } -} diff --git a/packages/server/src/migrations/functions/backfill/app/automations.ts b/packages/server/src/migrations/functions/backfill/app/automations.ts deleted file mode 100644 index 20da8fd3c0..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/automations.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getAutomationParams } from "../../../../db/utils" -import { Automation } from "@budibase/types" - -const getAutomations = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getAutomationParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const automations = await getAutomations(appDb) - - for (const automation of automations) { - await events.automation.created(automation, timestamp) - - for (const step of automation.definition.steps) { - await events.automation.stepCreated(automation, step, timestamp) - } - } - - return automations.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/datasources.ts b/packages/server/src/migrations/functions/backfill/app/datasources.ts deleted file mode 100644 index 5d7e1ad866..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/datasources.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getDatasourceParams } from "../../../../db/utils" -import { Datasource } from "@budibase/types" - -const getDatasources = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getDatasourceParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const datasources: Datasource[] = await getDatasources(appDb) - - for (const datasource of datasources) { - await events.datasource.created(datasource, timestamp) - } - - return datasources.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/layouts.ts b/packages/server/src/migrations/functions/backfill/app/layouts.ts deleted file mode 100644 index ee5806459b..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/layouts.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getLayoutParams } from "../../../../db/utils" -import { Layout } from "@budibase/types" - -const getLayouts = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getLayoutParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const layouts: Layout[] = await getLayouts(appDb) - - for (const layout of layouts) { - // exclude default layouts - if ( - layout._id === "layout_private_master" || - layout._id === "layout_public_master" - ) { - continue - } - await events.layout.created(layout, timestamp) - } - - return layouts.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/queries.ts b/packages/server/src/migrations/functions/backfill/app/queries.ts deleted file mode 100644 index e028721bce..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/queries.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getQueryParams } from "../../../../db/utils" -import { Query, Datasource, SourceName } from "@budibase/types" - -const getQueries = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getQueryParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -const getDatasource = async ( - appDb: any, - datasourceId: string -): Promise => { - return appDb.get(datasourceId) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const queries: Query[] = await getQueries(appDb) - - for (const query of queries) { - let datasource: Datasource - - try { - datasource = await getDatasource(appDb, query.datasourceId) - } catch (e: any) { - // handle known bug where a datasource has been deleted - // and the query has not - if (e.status === 404) { - datasource = { - type: "unknown", - _id: query.datasourceId, - source: "unknown" as SourceName, - } - } else { - throw e - } - } - - await events.query.created(datasource, query, timestamp) - } - - return queries.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/roles.ts b/packages/server/src/migrations/functions/backfill/app/roles.ts deleted file mode 100644 index 494b6f6923..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/roles.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getRoleParams } from "../../../../db/utils" -import { Role } from "@budibase/types" - -const getRoles = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getRoleParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const roles = await getRoles(appDb) - - for (const role of roles) { - await events.role.created(role, timestamp) - } - - return roles.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/screens.ts b/packages/server/src/migrations/functions/backfill/app/screens.ts deleted file mode 100644 index ab3b4b9d3c..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/screens.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { events } from "@budibase/backend-core" -import { getScreenParams } from "../../../../db/utils" -import { Screen } from "@budibase/types" - -const getScreens = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getScreenParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const screens = await getScreens(appDb) - - for (const screen of screens) { - await events.screen.created(screen, timestamp) - } - - return screens.length -} diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts deleted file mode 100644 index e8437bd529..0000000000 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { events } from "@budibase/backend-core" -import { Database } from "@budibase/types" -import sdk from "../../../../sdk" - -export const backfill = async (appDb: Database, timestamp: string | number) => { - const tables = await sdk.tables.getAllInternalTables(appDb) - - for (const table of tables) { - await events.table.created(table, timestamp) - } - - return tables.length -} diff --git a/packages/server/src/migrations/functions/backfill/global.ts b/packages/server/src/migrations/functions/backfill/global.ts deleted file mode 100644 index 7f718cee2f..0000000000 --- a/packages/server/src/migrations/functions/backfill/global.ts +++ /dev/null @@ -1,214 +0,0 @@ -import * as users from "./global/users" -import * as configs from "./global/configs" -import * as quotas from "./global/quotas" -import { - tenancy, - events, - migrations, - accounts, - db as dbUtils, -} from "@budibase/backend-core" -import { - App, - CloudAccount, - Event, - Hosting, - QuotaUsage, - TenantBackfillSucceededEvent, - User, -} from "@budibase/types" -import env from "../../../environment" -import { DEFAULT_TIMESTAMP } from "." - -const failGraceful = env.SELF_HOSTED && !env.isDev() - -const handleError = (e: any, errors?: any) => { - if (failGraceful) { - if (errors) { - errors.push(e) - } - return - } - throw e -} - -const formatUsage = (usage: QuotaUsage) => { - let maxAutomations = 0 - let maxQueries = 0 - let rows = 0 - - if (usage) { - if (usage.usageQuota) { - rows = usage.usageQuota.rows - } - - if (usage.monthly) { - for (const value of Object.values(usage.monthly)) { - if (value.automations > maxAutomations) { - maxAutomations = value.automations - } - if (value.queries > maxQueries) { - maxQueries = value.queries - } - } - } - } - - return { - maxAutomations, - maxQueries, - rows, - } -} - -const EVENTS = [ - Event.EMAIL_SMTP_CREATED, - Event.AUTH_SSO_CREATED, - Event.AUTH_SSO_ACTIVATED, - Event.ORG_NAME_UPDATED, - Event.ORG_LOGO_UPDATED, - Event.ORG_PLATFORM_URL_UPDATED, - Event.USER_CREATED, - Event.USER_PERMISSION_ADMIN_ASSIGNED, - Event.USER_PERMISSION_BUILDER_ASSIGNED, - Event.ROLE_ASSIGNED, - Event.ROWS_CREATED, - Event.QUERIES_RUN, - Event.AUTOMATIONS_RUN, -] - -/** - * Date: - * May 2022 - * - * Description: - * Backfill global events. - */ - -export const run = async (db: any) => { - try { - const tenantId = tenancy.getTenantId() - let timestamp: string | number = DEFAULT_TIMESTAMP - - const totals: any = {} - const errors: any = [] - - let allUsers: User[] = [] - try { - allUsers = await users.getUsers(db) - } catch (e: any) { - handleError(e, errors) - } - - if (!allUsers || allUsers.length === 0) { - // first time startup - we don't need to backfill anything - // tenant will be identified when admin user is created - if (env.SELF_HOSTED) { - await events.installation.firstStartup() - } - return - } - - try { - const installTimestamp = await getInstallTimestamp(db, allUsers) - if (installTimestamp) { - timestamp = installTimestamp - } - } catch (e) { - handleError(e, errors) - } - - let account: CloudAccount | undefined - if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { - account = await accounts.getAccountByTenantId(tenantId) - } - - try { - await events.identification.identifyTenantGroup( - tenantId, - env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD, - timestamp - ) - } catch (e) { - handleError(e, errors) - } - - // tell the event pipeline to start caching - // events for this tenant - await events.backfillCache.start(EVENTS) - - try { - await configs.backfill(db, timestamp) - } catch (e) { - handleError(e, errors) - } - - try { - totals.users = await users.backfill(db, account) - } catch (e) { - handleError(e, errors) - } - - try { - const allApps = (await dbUtils.getAllApps({ dev: true })) as App[] - totals.apps = allApps.length - - totals.usage = await quotas.backfill(allApps) - } catch (e) { - handleError(e, errors) - } - - const properties: TenantBackfillSucceededEvent = { - apps: totals.apps, - users: totals.users, - ...formatUsage(totals.usage), - usage: totals.usage, - } - - if (errors.length) { - properties.errors = errors.map((e: any) => - JSON.stringify(e, Object.getOwnPropertyNames(e)) - ) - properties.errorCount = errors.length - } else { - properties.errorCount = 0 - } - - await events.backfill.tenantSucceeded(properties) - // tell the event pipeline to stop caching events for this tenant - await events.backfillCache.end() - } catch (e) { - handleError(e) - await events.backfill.tenantFailed(e) - } -} - -export const isComplete = async (): Promise => { - const globalDb = tenancy.getGlobalDB() - const migrationsDoc = await migrations.getMigrationsDoc(globalDb) - return !!migrationsDoc.event_global_backfill -} - -export const getInstallTimestamp = async ( - globalDb: any, - allUsers?: User[] -): Promise => { - if (!allUsers) { - allUsers = await users.getUsers(globalDb) - } - - // get the oldest user timestamp - if (allUsers) { - const timestamps = allUsers - .map(user => user.createdAt) - .filter(timestamp => !!timestamp) - .sort( - (a, b) => - new Date(a as number).getTime() - new Date(b as number).getTime() - ) - - if (timestamps.length) { - return timestamps[0] - } - } -} diff --git a/packages/server/src/migrations/functions/backfill/global/configs.ts b/packages/server/src/migrations/functions/backfill/global/configs.ts deleted file mode 100644 index 04eb9caff2..0000000000 --- a/packages/server/src/migrations/functions/backfill/global/configs.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { - events, - DocumentType, - SEPARATOR, - UNICODE_MAX, -} from "@budibase/backend-core" -import { - Config, - isSMTPConfig, - isGoogleConfig, - isOIDCConfig, - isSettingsConfig, - ConfigType, - DatabaseQueryOpts, -} from "@budibase/types" -import env from "./../../../../environment" - -export function getConfigParams(): DatabaseQueryOpts { - return { - include_docs: true, - startkey: `${DocumentType.CONFIG}${SEPARATOR}`, - endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`, - } -} - -const getConfigs = async (globalDb: any): Promise => { - const response = await globalDb.allDocs(getConfigParams()) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async ( - globalDb: any, - timestamp: string | number | undefined -) => { - const configs = await getConfigs(globalDb) - - for (const config of configs) { - if (isSMTPConfig(config)) { - await events.email.SMTPCreated(timestamp) - } - if (isGoogleConfig(config)) { - await events.auth.SSOCreated(ConfigType.GOOGLE, timestamp) - if (config.config.activated) { - await events.auth.SSOActivated(ConfigType.GOOGLE, timestamp) - } - } - if (isOIDCConfig(config)) { - await events.auth.SSOCreated(ConfigType.OIDC, timestamp) - if (config.config.configs[0].activated) { - await events.auth.SSOActivated(ConfigType.OIDC, timestamp) - } - } - if (isSettingsConfig(config)) { - const company = config.config.company - if (company && company !== "Budibase") { - await events.org.nameUpdated(timestamp) - } - - const logoUrl = config.config.logoUrl - if (logoUrl) { - await events.org.logoUpdated(timestamp) - } - - const platformUrl = config.config.platformUrl - if ( - platformUrl && - platformUrl !== "http://localhost:10000" && - env.SELF_HOSTED - ) { - await events.org.platformURLUpdated(timestamp) - } - } - } -} diff --git a/packages/server/src/migrations/functions/backfill/global/quotas.ts b/packages/server/src/migrations/functions/backfill/global/quotas.ts deleted file mode 100644 index 505274a318..0000000000 --- a/packages/server/src/migrations/functions/backfill/global/quotas.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { DEFAULT_TIMESTAMP } from "./../index" -import { events } from "@budibase/backend-core" -import { quotas } from "@budibase/pro" -import { App } from "@budibase/types" - -const getOldestCreatedAt = (allApps: App[]): string | undefined => { - const timestamps = allApps - .filter(app => !!app.createdAt) - .map(app => app.createdAt as string) - .sort((a, b) => new Date(a).getTime() - new Date(b).getTime()) - - if (timestamps.length) { - return timestamps[0] - } -} - -const getMonthTimestamp = (monthString: string): number => { - const parts = monthString.split("-") - const month = parseInt(parts[0]) - 1 // we already do +1 in month string calculation - const year = parseInt(parts[1]) - - // using 0 as the day in next month gives us last day in previous month - const date = new Date(year, month + 1, 0).getTime() - const now = new Date().getTime() - - if (date > now) { - return now - } else { - return date - } -} - -export const backfill = async (allApps: App[]) => { - const usage = await quotas.getQuotaUsage() - - const rows = usage.usageQuota.rows - let timestamp: string | number = DEFAULT_TIMESTAMP - - const oldestAppTimestamp = getOldestCreatedAt(allApps) - if (oldestAppTimestamp) { - timestamp = oldestAppTimestamp - } - - await events.rows.created(rows, timestamp) - - for (const [monthString, quotas] of Object.entries(usage.monthly)) { - if (monthString === "current") { - continue - } - const monthTimestamp = getMonthTimestamp(monthString) - - const queries = quotas.queries - await events.query.run(queries, monthTimestamp) - - const automations = quotas.automations - await events.automation.run(automations, monthTimestamp) - } - - return usage -} diff --git a/packages/server/src/migrations/functions/backfill/global/users.ts b/packages/server/src/migrations/functions/backfill/global/users.ts deleted file mode 100644 index b3dae822d7..0000000000 --- a/packages/server/src/migrations/functions/backfill/global/users.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { - events, - db as dbUtils, - users as usersCore, -} from "@budibase/backend-core" -import { User, CloudAccount } from "@budibase/types" -import { DEFAULT_TIMESTAMP } from ".." - -// manually define user doc params - normally server doesn't read users from the db -const getUserParams = (props: any) => { - return dbUtils.getDocParams(dbUtils.DocumentType.USER, null, props) -} - -export const getUsers = async (globalDb: any): Promise => { - const response = await globalDb.allDocs( - getUserParams({ - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async ( - globalDb: any, - account: CloudAccount | undefined -) => { - const users = await getUsers(globalDb) - - for (const user of users) { - let timestamp: string | number = DEFAULT_TIMESTAMP - if (user.createdAt) { - timestamp = user.createdAt - } - await events.identification.identifyUser(user, account, timestamp) - await events.user.created(user, timestamp) - - if (usersCore.hasAdminPermissions(user)) { - await events.user.permissionAdminAssigned(user, timestamp) - } - - if (usersCore.hasBuilderPermissions(user)) { - await events.user.permissionBuilderAssigned(user, timestamp) - } - - if (user.roles) { - for (const [, role] of Object.entries(user.roles)) { - await events.role.assigned(user, role, timestamp) - } - } - } - - return users.length -} diff --git a/packages/server/src/migrations/functions/backfill/index.ts b/packages/server/src/migrations/functions/backfill/index.ts deleted file mode 100644 index 00c04722b4..0000000000 --- a/packages/server/src/migrations/functions/backfill/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * as app from "./app" -export * as global from "./global" -export * as installation from "./installation" - -// historical events are free in posthog - make sure we default to a -// historical time if no other can be found -export const DEFAULT_TIMESTAMP = new Date(2022, 0, 1).getTime() diff --git a/packages/server/src/migrations/functions/backfill/installation.ts b/packages/server/src/migrations/functions/backfill/installation.ts deleted file mode 100644 index 3c2b8bd3fc..0000000000 --- a/packages/server/src/migrations/functions/backfill/installation.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { DEFAULT_TIMESTAMP } from "./index" -import { events, tenancy, installation } from "@budibase/backend-core" -import { Installation } from "@budibase/types" -import * as global from "./global" -import env from "../../../environment" - -const failGraceful = env.SELF_HOSTED - -const handleError = (e: any, errors?: any) => { - if (failGraceful) { - if (errors) { - errors.push(e) - } - return - } - throw e -} - -/** - * Date: - * May 2022 - * - * Description: - * Backfill installation events. - */ - -export const run = async () => { - try { - // need to use the default tenant to try to get the installation time - await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => { - const db = tenancy.getGlobalDB() - let timestamp: string | number = DEFAULT_TIMESTAMP - - const installTimestamp = await global.getInstallTimestamp(db) - if (installTimestamp) { - timestamp = installTimestamp - } - - const install: Installation = await installation.getInstall() - await events.identification.identifyInstallationGroup( - install.installId, - timestamp - ) - }) - await events.backfill.installationSucceeded() - } catch (e) { - handleError(e) - await events.backfill.installationFailed(e) - } -} diff --git a/packages/server/src/migrations/functions/syncQuotas.ts b/packages/server/src/migrations/functions/syncQuotas.ts deleted file mode 100644 index 83a7670e78..0000000000 --- a/packages/server/src/migrations/functions/syncQuotas.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { runQuotaMigration } from "./usageQuotas" -import * as syncApps from "./usageQuotas/syncApps" -import * as syncRows from "./usageQuotas/syncRows" -import * as syncPlugins from "./usageQuotas/syncPlugins" -import * as syncUsers from "./usageQuotas/syncUsers" -import * as syncCreators from "./usageQuotas/syncCreators" - -/** - * Synchronise quotas to the state of the db. - */ -export const run = async () => { - await runQuotaMigration(async () => { - await syncApps.run() - await syncRows.run() - await syncPlugins.run() - await syncUsers.run() - await syncCreators.run() - }) -} diff --git a/packages/server/src/migrations/functions/tableSettings.ts b/packages/server/src/migrations/functions/tableSettings.ts deleted file mode 100644 index 2db3df0d0f..0000000000 --- a/packages/server/src/migrations/functions/tableSettings.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { getScreenParams } from "../../db/utils" -import { Screen } from "@budibase/types" -import { makePropSafe as safe } from "@budibase/string-templates" -/** - * Date: - * November 2022 - * - * Description: - * Update table settings to use actions instead of links. We do not remove the - * legacy values here as we cannot guarantee that their apps are up-t-date. - * It is safe to simply save both the new and old structure in the definition. - * - * Migration 1: - * Legacy "linkRows", "linkURL", "linkPeek" and "linkColumn" settings on tables - * and table blocks are migrated into a "Navigate To" action under the new - * "onClick" setting. - * - * Migration 2: - * Legacy "titleButtonURL" and "titleButtonPeek" settings on table blocks are - * migrated into a "Navigate To" action under the new "onClickTitleButton" - * setting. - */ -export const run = async (appDb: any) => { - // Get all app screens - let screens: Screen[] - try { - screens = ( - await appDb.allDocs( - getScreenParams(null, { - include_docs: true, - }) - ) - ).rows.map((row: any) => row.doc) - } catch (e) { - // sometimes the metadata document doesn't exist - // exit early instead of failing the migration - console.error("Error retrieving app metadata. Skipping", e) - return - } - - // Recursively update any relevant components and mutate the screen docs - for (let screen of screens) { - const changed = migrateTableSettings(screen.props) - - // Save screen if we updated it - if (changed) { - await appDb.put(screen) - console.log( - `Screen ${screen.routing?.route} contained table settings which were migrated` - ) - } - } -} - -// Recursively searches and mutates a screen doc to migrate table component -// and table block settings -const migrateTableSettings = (component: any) => { - let changed = false - if (!component) { - return changed - } - - // Migration 1: migrate table row click settings - if ( - component._component.endsWith("/table") || - component._component.endsWith("/tableblock") - ) { - const { linkRows, linkURL, linkPeek, linkColumn, onClick } = component - if (linkRows && !onClick) { - const column = linkColumn || "_id" - const action = convertLinkSettingToAction(linkURL, !!linkPeek, column) - if (action) { - changed = true - component.onClick = action - if (component._component.endsWith("/tableblock")) { - component.clickBehaviour = "actions" - } - } - } - } - - // Migration 2: migrate table block title button settings - if (component._component.endsWith("/tableblock")) { - const { - showTitleButton, - titleButtonURL, - titleButtonPeek, - onClickTitleButton, - } = component - if (showTitleButton && !onClickTitleButton) { - const action = convertLinkSettingToAction( - titleButtonURL, - !!titleButtonPeek - ) - if (action) { - changed = true - component.onClickTitleButton = action - component.titleButtonClickBehaviour = "actions" - } - } - } - - // Recurse down the tree as needed - component._children?.forEach((child: any) => { - const childChanged = migrateTableSettings(child) - changed = changed || childChanged - }) - return changed -} - -// Util ti convert the legacy settings into a navigation action structure -const convertLinkSettingToAction = ( - linkURL: string, - linkPeek: boolean, - linkColumn?: string -) => { - // Sanity check we have a URL - if (!linkURL) { - return null - } - - // Default URL to the old URL setting - let url = linkURL - - // If we enriched the old URL with a column, update the url - if (linkColumn && linkURL.includes("/:")) { - // Convert old link URL setting, which is a screen URL, into a valid - // binding using the new clicked row binding - const split = linkURL.split("/:") - const col = linkColumn || "_id" - const binding = `{{ ${safe("eventContext")}.${safe("row")}.${safe(col)} }}` - url = `${split[0]}/${binding}` - } - - // Create action structure - return [ - { - "##eventHandlerType": "Navigate To", - parameters: { - url, - peek: linkPeek, - }, - }, - ] -} diff --git a/packages/server/src/migrations/functions/tests/appUrls.spec.js b/packages/server/src/migrations/functions/tests/appUrls.spec.js deleted file mode 100644 index 7a2f324552..0000000000 --- a/packages/server/src/migrations/functions/tests/appUrls.spec.js +++ /dev/null @@ -1,26 +0,0 @@ -const { db: dbCore } = require("@budibase/backend-core") -const TestConfig = require("../../../tests/utilities/TestConfiguration") - -const migration = require("../appUrls") - -describe("run", () => { - let config = new TestConfig(false) - - beforeAll(async () => { - await config.init() - }) - - afterAll(config.end) - - it("runs successfully", async () => { - const app = await config.createApp("testApp") - const metadata = await dbCore.doWithDB(app.appId, async db => { - const metadataDoc = await db.get(dbCore.DocumentType.APP_METADATA) - delete metadataDoc.url - await db.put(metadataDoc) - await migration.run(db) - return await db.get(dbCore.DocumentType.APP_METADATA) - }) - expect(metadata.url).toEqual("/testapp") - }) -}) diff --git a/packages/server/src/migrations/functions/tests/tableSettings.spec.ts b/packages/server/src/migrations/functions/tests/tableSettings.spec.ts deleted file mode 100644 index 8d28a43322..0000000000 --- a/packages/server/src/migrations/functions/tests/tableSettings.spec.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { App, Screen } from "@budibase/types" - -import { db as dbCore } from "@budibase/backend-core" -import TestConfig from "../../../tests/utilities/TestConfiguration" -import { run as runMigration } from "../tableSettings" - -describe("run", () => { - const config = new TestConfig(false) - let app: App - let screen: Screen - - beforeAll(async () => { - await config.init() - app = await config.createApp("testApp") - screen = await config.createScreen() - }) - - afterAll(config.end) - - it("migrates table block row on click settings", async () => { - // Add legacy table block as first child - screen.props._children = [ - { - _instanceName: "Table Block", - _styles: {}, - _component: "@budibase/standard-components/tableblock", - _id: "foo", - linkRows: true, - linkURL: "/rows/:id", - linkPeek: true, - linkColumn: "name", - }, - ] - await config.createScreen(screen) - - // Run migration - screen = await dbCore.doWithDB(app.appId, async (db: any) => { - await runMigration(db) - return await db.get(screen._id) - }) - - // Verify new "onClick" setting - const onClick = screen.props._children?.[0].onClick - expect(onClick).toBeDefined() - expect(onClick.length).toBe(1) - expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To") - expect(onClick[0].parameters.url).toBe( - `/rows/{{ [eventContext].[row].[name] }}` - ) - expect(onClick[0].parameters.peek).toBeTruthy() - }) - - it("migrates table row on click settings", async () => { - // Add legacy table block as first child - screen.props._children = [ - { - _instanceName: "Table", - _styles: {}, - _component: "@budibase/standard-components/table", - _id: "foo", - linkRows: true, - linkURL: "/rows/:id", - linkPeek: true, - linkColumn: "name", - }, - ] - await config.createScreen(screen) - - // Run migration - screen = await dbCore.doWithDB(app.appId, async (db: any) => { - await runMigration(db) - return await db.get(screen._id) - }) - - // Verify new "onClick" setting - const onClick = screen.props._children?.[0].onClick - expect(onClick).toBeDefined() - expect(onClick.length).toBe(1) - expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To") - expect(onClick[0].parameters.url).toBe( - `/rows/{{ [eventContext].[row].[name] }}` - ) - expect(onClick[0].parameters.peek).toBeTruthy() - }) - - it("migrates table block title button settings", async () => { - // Add legacy table block as first child - screen.props._children = [ - { - _instanceName: "Table Block", - _styles: {}, - _component: "@budibase/standard-components/tableblock", - _id: "foo", - showTitleButton: true, - titleButtonURL: "/url", - titleButtonPeek: true, - }, - ] - await config.createScreen(screen) - - // Run migration - screen = await dbCore.doWithDB(app.appId, async (db: any) => { - await runMigration(db) - return await db.get(screen._id) - }) - - // Verify new "onClickTitleButton" setting - const onClick = screen.props._children?.[0].onClickTitleButton - expect(onClick).toBeDefined() - expect(onClick.length).toBe(1) - expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To") - expect(onClick[0].parameters.url).toBe("/url") - expect(onClick[0].parameters.peek).toBeTruthy() - }) - - it("ignores components that have already been migrated", async () => { - // Add legacy table block as first child - screen.props._children = [ - { - _instanceName: "Table Block", - _styles: {}, - _component: "@budibase/standard-components/tableblock", - _id: "foo", - linkRows: true, - linkURL: "/rows/:id", - linkPeek: true, - linkColumn: "name", - onClick: "foo", - }, - ] - const initialDefinition = JSON.stringify(screen.props._children?.[0]) - await config.createScreen(screen) - - // Run migration - screen = await dbCore.doWithDB(app.appId, async (db: any) => { - await runMigration(db) - return await db.get(screen._id) - }) - - // Verify new "onClick" setting - const newDefinition = JSON.stringify(screen.props._children?.[0]) - expect(initialDefinition).toEqual(newDefinition) - }) -}) diff --git a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js deleted file mode 100644 index f0da25893c..0000000000 --- a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js +++ /dev/null @@ -1,34 +0,0 @@ -jest.mock("@budibase/backend-core", () => { - const core = jest.requireActual("@budibase/backend-core") - return { - ...core, - db: { - ...core.db, - createNewUserEmailView: jest.fn(), - }, - } -}) -const { context, db: dbCore } = require("@budibase/backend-core") -const TestConfig = require("../../../tests/utilities/TestConfiguration") - -// mock email view creation - -const migration = require("../userEmailViewCasing") - -describe("run", () => { - let config = new TestConfig(false) - - beforeAll(async () => { - await config.init() - }) - - afterAll(config.end) - - it("runs successfully", async () => { - await config.doInTenant(async () => { - const globalDb = context.getGlobalDB() - await migration.run(globalDb) - expect(dbCore.createNewUserEmailView).toHaveBeenCalledTimes(1) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/usageQuotas/index.ts b/packages/server/src/migrations/functions/usageQuotas/index.ts deleted file mode 100644 index e94e993b21..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const runQuotaMigration = async (migration: () => Promise) => { - await migration() -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncApps.ts b/packages/server/src/migrations/functions/usageQuotas/syncApps.ts deleted file mode 100644 index 80ed1953d3..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncApps.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { db as dbCore } from "@budibase/backend-core" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" - -export const run = async () => { - // get app count - const devApps = await dbCore.getAllApps({ dev: true }) - const appCount = devApps ? devApps.length : 0 - - // sync app count - console.log(`Syncing app count: ${appCount}`) - await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC) -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts b/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts deleted file mode 100644 index ce53be925a..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { users } from "@budibase/backend-core" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" - -export const run = async () => { - const creatorCount = await users.getCreatorCount() - console.log(`Syncing creator count: ${creatorCount}`) - await quotas.setUsage( - creatorCount, - StaticQuotaName.CREATORS, - QuotaUsageType.STATIC - ) -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts b/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts deleted file mode 100644 index b00970aea2..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { logging } from "@budibase/backend-core" -import { plugins } from "@budibase/pro" - -export const run = async () => { - try { - await plugins.checkPluginQuotas() - } catch (err) { - logging.logAlert("Failed to update plugin quotas", err) - } -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncRows.ts b/packages/server/src/migrations/functions/usageQuotas/syncRows.ts deleted file mode 100644 index 506218a41f..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncRows.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { db as dbCore } from "@budibase/backend-core" -import { getUniqueRows } from "../../../utilities/usageQuota/rows" -import { quotas } from "@budibase/pro" -import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types" - -export const run = async () => { - // get all rows in all apps - const allApps = (await dbCore.getAllApps({ all: true })) as App[] - const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : [] - const { appRows } = await getUniqueRows(appIds) - - // get the counts per app - const counts: { [key: string]: number } = {} - let rowCount = 0 - Object.entries(appRows).forEach(([appId, rows]) => { - counts[appId] = rows.length - rowCount += rows.length - }) - - // sync row count - console.log(`Syncing row count: ${rowCount}`) - await quotas.setUsagePerApp( - counts, - StaticQuotaName.ROWS, - QuotaUsageType.STATIC - ) -} diff --git a/packages/server/src/migrations/functions/usageQuotas/syncUsers.ts b/packages/server/src/migrations/functions/usageQuotas/syncUsers.ts deleted file mode 100644 index c9913dced8..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/syncUsers.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { users } from "@budibase/backend-core" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" - -export const run = async () => { - const userCount = await users.getUserCount() - console.log(`Syncing user count: ${userCount}`) - await quotas.setUsage(userCount, StaticQuotaName.USERS, QuotaUsageType.STATIC) -} diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts deleted file mode 100644 index 1d4d4d0f71..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts +++ /dev/null @@ -1,35 +0,0 @@ -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import * as syncApps from "../syncApps" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" - -describe("syncApps", () => { - let config = new TestConfig(false) - - beforeEach(async () => { - await config.init() - }) - - afterAll(config.end) - - it("runs successfully", async () => { - return config.doInContext(undefined, async () => { - // create the usage quota doc and mock usages - await quotas.getQuotaUsage() - await quotas.setUsage(3, StaticQuotaName.APPS, QuotaUsageType.STATIC) - - let usageDoc = await quotas.getQuotaUsage() - expect(usageDoc.usageQuota.apps).toEqual(3) - - // create an extra app to test the migration - await config.createApp("quota-test") - - // migrate - await syncApps.run() - - // assert the migration worked - usageDoc = await quotas.getQuotaUsage() - expect(usageDoc.usageQuota.apps).toEqual(2) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts deleted file mode 100644 index 93b7d4949b..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts +++ /dev/null @@ -1,26 +0,0 @@ -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import * as syncCreators from "../syncCreators" -import { quotas } from "@budibase/pro" - -describe("syncCreators", () => { - let config = new TestConfig(false) - - beforeEach(async () => { - await config.init() - }) - - afterAll(config.end) - - it("syncs creators", async () => { - return config.doInContext(undefined, async () => { - await config.createUser({ admin: { global: true } }) - - await syncCreators.run() - - const usageDoc = await quotas.getQuotaUsage() - // default + additional creator - const creatorsCount = 2 - expect(usageDoc.usageQuota.creators).toBe(creatorsCount) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts deleted file mode 100644 index 730278683c..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts +++ /dev/null @@ -1,53 +0,0 @@ -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import * as syncRows from "../syncRows" -import { quotas } from "@budibase/pro" -import { QuotaUsageType, StaticQuotaName } from "@budibase/types" -import { db as dbCore, context } from "@budibase/backend-core" - -describe("syncRows", () => { - const config = new TestConfig() - - beforeEach(async () => { - await config.init() - }) - - afterAll(config.end) - - it("runs successfully", async () => { - return config.doInContext(undefined, async () => { - // create the usage quota doc and mock usages - await quotas.getQuotaUsage() - await quotas.setUsage(300, StaticQuotaName.ROWS, QuotaUsageType.STATIC) - - let usageDoc = await quotas.getQuotaUsage() - expect(usageDoc.usageQuota.rows).toEqual(300) - - // app 1 - const app1 = config.app - await context.doInAppContext(app1!.appId, async () => { - await config.createTable() - await config.createRow() - }) - // app 2 - const app2 = await config.createApp("second-app") - await context.doInAppContext(app2.appId, async () => { - await config.createTable() - await config.createRow() - await config.createRow() - }) - - // migrate - await syncRows.run() - - // assert the migration worked - usageDoc = await quotas.getQuotaUsage() - expect(usageDoc.usageQuota.rows).toEqual(3) - expect( - usageDoc.apps?.[dbCore.getProdAppID(app1!.appId)].usageQuota.rows - ).toEqual(1) - expect( - usageDoc.apps?.[dbCore.getProdAppID(app2.appId)].usageQuota.rows - ).toEqual(2) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts deleted file mode 100644 index 2731cc041d..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts +++ /dev/null @@ -1,26 +0,0 @@ -import TestConfig from "../../../../tests/utilities/TestConfiguration" -import * as syncUsers from "../syncUsers" -import { quotas } from "@budibase/pro" - -describe("syncUsers", () => { - let config = new TestConfig(false) - - beforeEach(async () => { - await config.init() - }) - - afterAll(config.end) - - it("syncs users", async () => { - return config.doInContext(undefined, async () => { - await config.createUser() - - await syncUsers.run() - - const usageDoc = await quotas.getQuotaUsage() - // default + additional user - const userCount = 2 - expect(usageDoc.usageQuota.users).toBe(userCount) - }) - }) -}) diff --git a/packages/server/src/migrations/functions/userEmailViewCasing.ts b/packages/server/src/migrations/functions/userEmailViewCasing.ts deleted file mode 100644 index 078289cddf..0000000000 --- a/packages/server/src/migrations/functions/userEmailViewCasing.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { db as dbCore } from "@budibase/backend-core" - -/** - * Date: - * October 2021 - * - * Description: - * Recreate the user email view to include latest changes i.e. lower casing the email address - */ - -export const run = async () => { - await dbCore.createNewUserEmailView() -} diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts deleted file mode 100644 index a66d793142..0000000000 --- a/packages/server/src/migrations/index.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { locks, migrations } from "@budibase/backend-core" -import { - Migration, - MigrationOptions, - MigrationName, - LockType, - LockName, -} from "@budibase/types" -import env from "../environment" - -// migration functions -import * as userEmailViewCasing from "./functions/userEmailViewCasing" -import * as syncQuotas from "./functions/syncQuotas" -import * as appUrls from "./functions/appUrls" -import * as tableSettings from "./functions/tableSettings" -import * as backfill from "./functions/backfill" -/** - * Populate the migration function and additional configuration from - * the static migration definitions. - */ -export const buildMigrations = () => { - const definitions = migrations.DEFINITIONS - const serverMigrations: Migration[] = [] - - for (const definition of definitions) { - switch (definition.name) { - case MigrationName.USER_EMAIL_VIEW_CASING: { - serverMigrations.push({ - ...definition, - fn: userEmailViewCasing.run, - }) - break - } - case MigrationName.SYNC_QUOTAS: { - serverMigrations.push({ - ...definition, - fn: syncQuotas.run, - }) - break - } - case MigrationName.APP_URLS: { - serverMigrations.push({ - ...definition, - appOpts: { all: true }, - fn: appUrls.run, - }) - break - } - case MigrationName.EVENT_APP_BACKFILL: { - serverMigrations.push({ - ...definition, - appOpts: { all: true }, - fn: backfill.app.run, - silent: !!env.SELF_HOSTED, // reduce noisy logging - preventRetry: !!env.SELF_HOSTED, // only ever run once - }) - break - } - case MigrationName.EVENT_GLOBAL_BACKFILL: { - serverMigrations.push({ - ...definition, - fn: backfill.global.run, - silent: !!env.SELF_HOSTED, // reduce noisy logging - preventRetry: !!env.SELF_HOSTED, // only ever run once - }) - break - } - case MigrationName.EVENT_INSTALLATION_BACKFILL: { - serverMigrations.push({ - ...definition, - fn: backfill.installation.run, - silent: !!env.SELF_HOSTED, // reduce noisy logging - preventRetry: !!env.SELF_HOSTED, // only ever run once - }) - break - } - case MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS: { - serverMigrations.push({ - ...definition, - appOpts: { dev: true }, - fn: tableSettings.run, - }) - break - } - } - } - - return serverMigrations -} - -export const MIGRATIONS = buildMigrations() - -export const migrate = async (options?: MigrationOptions) => { - if (env.SELF_HOSTED) { - // self host runs migrations on startup - // make sure only a single instance runs them - await migrateWithLock(options) - } else { - await migrations.runMigrations(MIGRATIONS, options) - } -} - -const migrateWithLock = async (options?: MigrationOptions) => { - await locks.doWithLock( - { - type: LockType.TRY_ONCE, - name: LockName.MIGRATIONS, - ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes - systemLock: true, - }, - async () => { - await migrations.runMigrations(MIGRATIONS, options) - } - ) -} diff --git a/packages/server/src/migrations/tests/helpers.ts b/packages/server/src/migrations/tests/helpers.ts deleted file mode 100644 index 35831a2fd0..0000000000 --- a/packages/server/src/migrations/tests/helpers.ts +++ /dev/null @@ -1,40 +0,0 @@ -// Mimic configs test configuration from worker, creation configs directly in database - -import * as structures from "./structures" -import { configs } from "@budibase/backend-core" -import { Config } from "@budibase/types" - -export const saveSettingsConfig = async (globalDb: any) => { - const config = structures.settings() - await saveConfig(config, globalDb) -} - -export const saveGoogleConfig = async (globalDb: any) => { - const config = structures.google() - await saveConfig(config, globalDb) -} - -export const saveOIDCConfig = async (globalDb: any) => { - const config = structures.oidc() - await saveConfig(config, globalDb) -} - -export const saveSmtpConfig = async (globalDb: any) => { - const config = structures.smtp() - await saveConfig(config, globalDb) -} - -const saveConfig = async (config: Config, globalDb: any) => { - config._id = configs.generateConfigID(config.type) - - let response - try { - response = await globalDb.get(config._id) - config._rev = response._rev - await globalDb.put(config) - } catch (e: any) { - if (e.status === 404) { - await globalDb.put(config) - } - } -} diff --git a/packages/server/src/migrations/tests/index.spec.ts b/packages/server/src/migrations/tests/index.spec.ts deleted file mode 100644 index 3a23d8f011..0000000000 --- a/packages/server/src/migrations/tests/index.spec.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { - events, - migrations, - tenancy, - DocumentType, - context, -} from "@budibase/backend-core" -import TestConfig from "../../tests/utilities/TestConfiguration" -import * as structures from "../../tests/utilities/structures" -import { MIGRATIONS } from "../" -import * as helpers from "./helpers" - -import tk from "timekeeper" -import { View } from "@budibase/types" - -const timestamp = new Date().toISOString() -tk.freeze(timestamp) - -const clearMigrations = async () => { - const dbs = [context.getDevAppDB(), context.getProdAppDB()] - for (const db of dbs) { - const doc = await db.get(DocumentType.MIGRATIONS) - const newDoc = { _id: doc._id, _rev: doc._rev } - await db.put(newDoc) - } -} - -describe("migrations", () => { - const config = new TestConfig() - - beforeAll(async () => { - await config.init() - }) - - afterAll(() => { - config.end() - }) - - describe("backfill", () => { - it("runs app db migration", async () => { - await config.doInContext(undefined, async () => { - await clearMigrations() - await config.createAutomation() - await config.createAutomation(structures.newAutomation()) - await config.createDatasource() - await config.createDatasource() - await config.createLayout() - await config.createQuery() - await config.createQuery() - await config.createRole() - await config.createRole() - await config.createTable() - await config.createLegacyView() - await config.createTable() - await config.createLegacyView( - structures.view(config.table!._id!) as View - ) - await config.createScreen() - await config.createScreen() - - jest.clearAllMocks() - const migration = MIGRATIONS.filter( - m => m.name === "event_app_backfill" - )[0] - await migrations.runMigration(migration) - - expect(events.app.created).toHaveBeenCalledTimes(1) - expect(events.app.published).toHaveBeenCalledTimes(1) - expect(events.automation.created).toHaveBeenCalledTimes(2) - expect(events.automation.stepCreated).toHaveBeenCalledTimes(1) - expect(events.datasource.created).toHaveBeenCalledTimes(2) - expect(events.layout.created).toHaveBeenCalledTimes(1) - expect(events.query.created).toHaveBeenCalledTimes(2) - expect(events.role.created).toHaveBeenCalledTimes(3) // created roles + admin (created on table creation) - expect(events.table.created).toHaveBeenCalledTimes(3) - expect(events.backfill.appSucceeded).toHaveBeenCalledTimes(2) - - // to make sure caching is working as expected - expect( - events.processors.analyticsProcessor.processEvent - ).toHaveBeenCalledTimes(20) // Addition of of the events above - }) - }) - }) - - it("runs global db migration", async () => { - await config.doInContext(undefined, async () => { - await clearMigrations() - const appId = config.getProdAppId() - const roles = { [appId]: "role_12345" } - await config.createUser({ - builder: { global: false }, - admin: { global: true }, - roles, - }) // admin only - await config.createUser({ - builder: { global: false }, - admin: { global: false }, - roles, - }) // non admin non builder - await config.createTable() - await config.createRow() - await config.createRow() - - const db = tenancy.getGlobalDB() - await helpers.saveGoogleConfig(db) - await helpers.saveOIDCConfig(db) - await helpers.saveSettingsConfig(db) - await helpers.saveSmtpConfig(db) - - jest.clearAllMocks() - const migration = MIGRATIONS.filter( - m => m.name === "event_global_backfill" - )[0] - await migrations.runMigration(migration) - - expect(events.user.created).toHaveBeenCalledTimes(3) - expect(events.role.assigned).toHaveBeenCalledTimes(2) - expect(events.user.permissionBuilderAssigned).toHaveBeenCalledTimes(1) // default test user - expect(events.user.permissionAdminAssigned).toHaveBeenCalledTimes(1) // admin from above - expect(events.rows.created).toHaveBeenCalledTimes(1) - expect(events.rows.created).toHaveBeenCalledWith(2, timestamp) - expect(events.email.SMTPCreated).toHaveBeenCalledTimes(1) - expect(events.auth.SSOCreated).toHaveBeenCalledTimes(2) - expect(events.auth.SSOActivated).toHaveBeenCalledTimes(2) - expect(events.org.logoUpdated).toHaveBeenCalledTimes(1) - expect(events.org.nameUpdated).toHaveBeenCalledTimes(1) - expect(events.org.platformURLUpdated).toHaveBeenCalledTimes(1) - expect(events.backfill.tenantSucceeded).toHaveBeenCalledTimes(1) - - // to make sure caching is working as expected - expect( - events.processors.analyticsProcessor.processEvent - ).toHaveBeenCalledTimes(19) - }) - }) -}) diff --git a/packages/server/src/migrations/tests/structures.ts b/packages/server/src/migrations/tests/structures.ts deleted file mode 100644 index e2113e6a7c..0000000000 --- a/packages/server/src/migrations/tests/structures.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { utils } from "@budibase/backend-core" -import { - SMTPConfig, - OIDCConfig, - GoogleConfig, - SettingsConfig, - ConfigType, -} from "@budibase/types" - -export const oidc = (conf?: OIDCConfig): OIDCConfig => { - return { - type: ConfigType.OIDC, - config: { - configs: [ - { - configUrl: "http://someconfigurl", - clientID: "clientId", - clientSecret: "clientSecret", - logo: "Microsoft", - name: "Active Directory", - uuid: utils.newid(), - activated: true, - scopes: [], - ...conf, - }, - ], - }, - } -} - -export const google = (conf?: GoogleConfig): GoogleConfig => { - return { - type: ConfigType.GOOGLE, - config: { - clientID: "clientId", - clientSecret: "clientSecret", - activated: true, - ...conf, - }, - } -} - -export const smtp = (conf?: SMTPConfig): SMTPConfig => { - return { - type: ConfigType.SMTP, - config: { - port: 12345, - host: "smtptesthost.com", - from: "testfrom@example.com", - subject: "Hello!", - secure: false, - ...conf, - }, - } -} - -export const settings = (conf?: SettingsConfig): SettingsConfig => { - return { - type: ConfigType.SETTINGS, - config: { - platformUrl: "http://mycustomdomain.com", - logoUrl: "http://mylogourl,com", - company: "mycompany", - ...conf, - }, - } -} diff --git a/packages/server/src/startup/index.ts b/packages/server/src/startup/index.ts index edca64db7d..54e982ae56 100644 --- a/packages/server/src/startup/index.ts +++ b/packages/server/src/startup/index.ts @@ -15,7 +15,6 @@ import { watch } from "../watch" import * as automations from "../automations" import * as fileSystem from "../utilities/fileSystem" import { default as eventEmitter, init as eventInit } from "../events" -import * as migrations from "../migrations" import * as bullboard from "../automations/bullboard" import * as appMigrations from "../appMigrations/queue" import * as pro from "@budibase/pro" @@ -106,18 +105,6 @@ export async function startup( initialiseWebsockets(app, server) } - // run migrations on startup if not done via http - // not recommended in a clustered environment - if (!env.HTTP_MIGRATIONS && !env.isTest()) { - console.log("Running migrations") - try { - await migrations.migrate() - } catch (e) { - logging.logAlert("Error performing migrations. Exiting.", e) - shutdown(server) - } - } - // monitor plugin directory if required if ( env.SELF_HOSTED && diff --git a/packages/types/src/api/web/global/oldMigration.ts b/packages/types/src/api/web/global/oldMigration.ts index 812ee1e593..f0da6c52f3 100644 --- a/packages/types/src/api/web/global/oldMigration.ts +++ b/packages/types/src/api/web/global/oldMigration.ts @@ -1,12 +1,3 @@ -import { Migration, MigrationOptions } from "../../../sdk" - -export interface RunOldMigrationRequest extends MigrationOptions {} -export interface RuneOldMigrationResponse { - message: string -} - -export type FetchOldMigrationResponse = Migration[] - -export interface GetOldMigrationStatus { +export interface GetMigrationStatus { migrated: boolean } diff --git a/packages/types/src/api/web/system/index.ts b/packages/types/src/api/web/system/index.ts index 9b03ddd438..18ed533e9d 100644 --- a/packages/types/src/api/web/system/index.ts +++ b/packages/types/src/api/web/system/index.ts @@ -3,6 +3,5 @@ export * from "./status" export * from "./ops" export * from "./account" export * from "./log" -export * from "./migration" export * from "./restore" export * from "./tenant" diff --git a/packages/types/src/api/web/system/migration.ts b/packages/types/src/api/web/system/migration.ts deleted file mode 100644 index a18112744c..0000000000 --- a/packages/types/src/api/web/system/migration.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { MigrationDefinition, MigrationOptions } from "../../../sdk" - -export interface RunGlobalMigrationRequest extends MigrationOptions {} -export interface RunGlobalMigrationResponse { - message: string -} - -export type FetchMigrationDefinitionsResponse = MigrationDefinition[] diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index 86eb5b1a24..eb9e23b3d1 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -4,7 +4,6 @@ export * from "./hosting" export * from "./context" export * from "./events" export * from "./licensing" -export * from "./migrations" export * from "./datasources" export * from "./search" export * from "./koa" diff --git a/packages/types/src/sdk/migrations.ts b/packages/types/src/sdk/migrations.ts deleted file mode 100644 index 6db0c85879..0000000000 --- a/packages/types/src/sdk/migrations.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { Database } from "./db" - -export interface Migration extends MigrationDefinition { - appOpts?: object - fn: (db: Database) => Promise - silent?: boolean - preventRetry?: boolean -} - -export enum MigrationType { - // run once per tenant, recorded in global db, global db is provided as an argument - GLOBAL = "global", - // run per app, recorded in each app db, app db is provided as an argument - APP = "app", - // run once, recorded in global info db, global info db is provided as an argument - INSTALLATION = "installation", -} - -export interface MigrationNoOpOptions { - type: MigrationType - tenantId: string - appId?: string -} - -/** - * e.g. - * { - * tenantIds: ['bb'], - * force: { - * global: ['quota_1'] - * } - * } - */ -export interface MigrationOptions { - tenantIds?: string[] - force?: { - [type: string]: string[] - } - noOp?: MigrationNoOpOptions -} - -export enum MigrationName { - USER_EMAIL_VIEW_CASING = "user_email_view_casing", - APP_URLS = "app_urls", - EVENT_APP_BACKFILL = "event_app_backfill", - EVENT_GLOBAL_BACKFILL = "event_global_backfill", - EVENT_INSTALLATION_BACKFILL = "event_installation_backfill", - GLOBAL_INFO_SYNC_USERS = "global_info_sync_users", - TABLE_SETTINGS_LINKS_TO_ACTIONS = "table_settings_links_to_actions", - // increment this number to re-activate this migration - SYNC_QUOTAS = "sync_quotas_2", -} - -export interface MigrationDefinition { - type: MigrationType - name: MigrationName -} diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 0bcdadfefc..e36c45a3ba 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -28,7 +28,6 @@ import { LockType, LookupAccountHolderResponse, LookupTenantUserResponse, - MigrationType, PlatformUserByEmail, SaveUserResponse, SearchUsersRequest, @@ -45,7 +44,6 @@ import { cache, ErrorCode, events, - migrations, platform, tenancy, db, @@ -187,10 +185,6 @@ export const adminUser = async ( if (env.MULTI_TENANCY) { // store the new tenant record in the platform db await platform.tenants.addTenant(tenantId) - await migrations.backPopulateMigrations({ - type: MigrationType.GLOBAL, - tenantId, - }) } await tenancy.doInTenant(tenantId, async () => { diff --git a/packages/worker/src/api/controllers/system/migrations.ts b/packages/worker/src/api/controllers/system/migrations.ts deleted file mode 100644 index fc253d839d..0000000000 --- a/packages/worker/src/api/controllers/system/migrations.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { - FetchMigrationDefinitionsResponse, - RunGlobalMigrationRequest, - RunGlobalMigrationResponse, - UserCtx, -} from "@budibase/types" - -const { migrate, MIGRATIONS } = require("../../../migrations") - -export const runMigrations = async ( - ctx: UserCtx -) => { - const options = ctx.request.body - // don't await as can take a while, just return - migrate(options) - ctx.body = { message: "Migration started." } -} - -export const fetchDefinitions = async ( - ctx: UserCtx -) => { - ctx.body = MIGRATIONS -} diff --git a/packages/worker/src/api/routes/index.ts b/packages/worker/src/api/routes/index.ts index 741026543c..d4ddb41522 100644 --- a/packages/worker/src/api/routes/index.ts +++ b/packages/worker/src/api/routes/index.ts @@ -12,7 +12,6 @@ import tenantsRoutes from "./system/tenants" import statusRoutes from "./system/status" import selfRoutes from "./global/self" import licenseRoutes from "./global/license" -import migrationRoutes from "./system/migrations" import accountRoutes from "./system/accounts" import restoreRoutes from "./system/restore" import systemLogRoutes from "./system/logs" @@ -34,7 +33,6 @@ export const routes: Router[] = [ licenseRoutes, pro.groups, pro.auditLogs, - migrationRoutes, accountRoutes, restoreRoutes, eventRoutes, diff --git a/packages/worker/src/api/routes/system/migrations.ts b/packages/worker/src/api/routes/system/migrations.ts deleted file mode 100644 index a8189b5a91..0000000000 --- a/packages/worker/src/api/routes/system/migrations.ts +++ /dev/null @@ -1,19 +0,0 @@ -import Router from "@koa/router" -import * as migrationsController from "../../controllers/system/migrations" -import { auth } from "@budibase/backend-core" - -const router: Router = new Router() - -router - .post( - "/api/system/migrations/run", - auth.internalApi, - migrationsController.runMigrations - ) - .get( - "/api/system/migrations/definitions", - auth.internalApi, - migrationsController.fetchDefinitions - ) - -export default router diff --git a/packages/worker/src/migrations/index.ts b/packages/worker/src/migrations/index.ts deleted file mode 100644 index 642fbeb54e..0000000000 --- a/packages/worker/src/migrations/index.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { migrations, locks } from "@budibase/backend-core" -import { - Migration, - MigrationOptions, - MigrationName, - LockType, - LockName, -} from "@budibase/types" -import env from "../environment" - -// migration functions -import * as syncUserInfo from "./functions/globalInfoSyncUsers" - -/** - * Populate the migration function and additional configuration from - * the static migration definitions. - */ -export const buildMigrations = () => { - const definitions = migrations.DEFINITIONS - const workerMigrations: Migration[] = [] - - for (const definition of definitions) { - switch (definition.name) { - case MigrationName.GLOBAL_INFO_SYNC_USERS: { - // only needed in cloud - if (!env.SELF_HOSTED) { - workerMigrations.push({ - ...definition, - fn: syncUserInfo.run, - }) - } - break - } - } - } - - return workerMigrations -} - -export const MIGRATIONS = buildMigrations() - -export const migrate = async (options?: MigrationOptions) => { - if (env.SELF_HOSTED) { - await migrateWithLock(options) - } else { - await migrations.runMigrations(MIGRATIONS, options) - } -} - -const migrateWithLock = async (options?: MigrationOptions) => { - await locks.doWithLock( - { - type: LockType.TRY_ONCE, - name: LockName.MIGRATIONS, - ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes - systemLock: true, - }, - async () => { - await migrations.runMigrations(MIGRATIONS, options) - } - ) -} From ff4f8f1d666f7a68c2df54987c5f2454192098c0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 15 Jan 2025 17:45:28 +0000 Subject: [PATCH 02/22] Remove unused test. --- .../routes/system/tests/migrations.spec.ts | 63 ------------------- packages/worker/src/tests/api/index.ts | 3 - packages/worker/src/tests/api/migrations.ts | 17 ----- 3 files changed, 83 deletions(-) delete mode 100644 packages/worker/src/api/routes/system/tests/migrations.spec.ts delete mode 100644 packages/worker/src/tests/api/migrations.ts diff --git a/packages/worker/src/api/routes/system/tests/migrations.spec.ts b/packages/worker/src/api/routes/system/tests/migrations.spec.ts deleted file mode 100644 index fe91a1070c..0000000000 --- a/packages/worker/src/api/routes/system/tests/migrations.spec.ts +++ /dev/null @@ -1,63 +0,0 @@ -const migrateFn = jest.fn() - -import { TestConfiguration } from "../../../../tests" - -jest.mock("../../../../migrations", () => { - return { - ...jest.requireActual("../../../../migrations"), - migrate: migrateFn, - } -}) - -describe("/api/system/migrations", () => { - const config = new TestConfiguration() - - beforeAll(async () => { - await config.beforeAll() - }) - - afterAll(async () => { - await config.afterAll() - }) - - afterEach(() => { - jest.clearAllMocks() - }) - - describe("POST /api/system/migrations/run", () => { - it("fails with no internal api key", async () => { - const res = await config.api.migrations.runMigrations({ - headers: {}, - status: 403, - }) - expect(res.body).toEqual({ message: "Unauthorized", status: 403 }) - expect(migrateFn).toHaveBeenCalledTimes(0) - }) - - it("runs migrations", async () => { - const res = await config.api.migrations.runMigrations() - expect(res.body.message).toBeDefined() - expect(migrateFn).toHaveBeenCalledTimes(1) - }) - }) - - describe("DELETE /api/system/migrations/definitions", () => { - it("fails with no internal api key", async () => { - const res = await config.api.migrations.getMigrationDefinitions({ - headers: {}, - status: 403, - }) - expect(res.body).toEqual({ message: "Unauthorized", status: 403 }) - }) - - it("returns definitions", async () => { - const res = await config.api.migrations.getMigrationDefinitions() - expect(res.body).toEqual([ - { - name: "global_info_sync_users", - type: "global", - }, - ]) - }) - }) -}) diff --git a/packages/worker/src/tests/api/index.ts b/packages/worker/src/tests/api/index.ts index fa4e54184c..0f0334c8da 100644 --- a/packages/worker/src/tests/api/index.ts +++ b/packages/worker/src/tests/api/index.ts @@ -6,7 +6,6 @@ import { EmailAPI } from "./email" import { SelfAPI } from "./self" import { UserAPI } from "./users" import { EnvironmentAPI } from "./environment" -import { MigrationAPI } from "./migrations" import { StatusAPI } from "./status" import { RestoreAPI } from "./restore" import { TenantAPI } from "./tenants" @@ -26,7 +25,6 @@ export default class API { self: SelfAPI users: UserAPI environment: EnvironmentAPI - migrations: MigrationAPI status: StatusAPI restore: RestoreAPI tenants: TenantAPI @@ -46,7 +44,6 @@ export default class API { this.self = new SelfAPI(config) this.users = new UserAPI(config) this.environment = new EnvironmentAPI(config) - this.migrations = new MigrationAPI(config) this.status = new StatusAPI(config) this.restore = new RestoreAPI(config) this.tenants = new TenantAPI(config) diff --git a/packages/worker/src/tests/api/migrations.ts b/packages/worker/src/tests/api/migrations.ts deleted file mode 100644 index 6d62fe994a..0000000000 --- a/packages/worker/src/tests/api/migrations.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { TestAPI, TestAPIOpts } from "./base" - -export class MigrationAPI extends TestAPI { - runMigrations = (opts?: TestAPIOpts) => { - return this.request - .post(`/api/system/migrations/run`) - .set(opts?.headers ? opts.headers : this.config.internalAPIHeaders()) - .expect(opts?.status ? opts.status : 200) - } - - getMigrationDefinitions = (opts?: TestAPIOpts) => { - return this.request - .get(`/api/system/migrations/definitions`) - .set(opts?.headers ? opts.headers : this.config.internalAPIHeaders()) - .expect(opts?.status ? opts.status : 200) - } -} From f74e2c6693a757c6eab28b2323a9d50eaa5018e6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 16 Jan 2025 16:40:51 +0100 Subject: [PATCH 03/22] Convert form block --- .../app/blocks/form/FormBlock.svelte | 31 ++++++++++--------- packages/client/src/index.ts | 4 +++ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index 656aa5933b..69f51b84e8 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -1,12 +1,14 @@ - + +export type Context = Readable From a466a2d6705ef934f0ce03b839a87fe07e06557b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 16 Jan 2025 16:44:41 +0100 Subject: [PATCH 04/22] Type params --- .../app/blocks/form/FormBlock.svelte | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index 69f51b84e8..2a0c622750 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -7,21 +7,24 @@ import { Component, Context, SDK } from "../../../../index" import { TableSchema, UIDatasource } from "@budibase/types" - export let actionType + export let actionType: string export let dataSource: UIDatasource - export let size - export let disabled + export let size: string + export let disabled: boolean export let fields - export let buttons - export let buttonPosition - export let title - export let description - export let rowId - export let actionUrl - export let noRowsMessage - export let notificationOverride - export let buttonsCollapsed - export let buttonsCollapsedText + export let buttons: { + "##eventHandlerType": string + parameters: Record + }[] + export let buttonPosition: "top" | "bottom" + export let title: string + export let description: string + export let rowId: string + export let actionUrl: string + export let noRowsMessage: string + export let notificationOverride: boolean + export let buttonsCollapsed: boolean + export let buttonsCollapsedText: string // Legacy export let showDeleteButton From 029b7754ca374719f248da4a77a8855a1fdeec5d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 16 Jan 2025 16:44:55 +0100 Subject: [PATCH 05/22] Type other params --- .../src/components/app/blocks/form/FormBlock.svelte | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index 2a0c622750..22f21b5790 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -27,10 +27,10 @@ export let buttonsCollapsedText: string // Legacy - export let showDeleteButton - export let showSaveButton - export let saveButtonLabel - export let deleteButtonLabel + export let showDeleteButton: boolean + export let showSaveButton: boolean + export let saveButtonLabel: boolean + export let deleteButtonLabel: boolean const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk") const component = getContext("component") From 325fea0c10644732b70a39c9885955aac4187838 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 16 Jan 2025 16:55:24 +0100 Subject: [PATCH 06/22] ViewV2, return 404 if not found --- packages/server/src/api/controllers/view/viewsV2.ts | 6 ++++-- packages/server/src/sdk/app/views/index.ts | 4 +++- packages/server/src/sdk/app/views/internal.ts | 6 ++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 5376ce49c3..ff8daf3bc6 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -123,9 +123,11 @@ async function parseSchema(view: CreateViewRequest) { } export async function get(ctx: Ctx) { - ctx.body = { - data: await sdk.views.getEnriched(ctx.params.viewId), + const view = await sdk.views.getEnriched(ctx.params.viewId) + if (!view) { + ctx.throw(404) } + ctx.body = { data: view } } export async function fetch(ctx: Ctx) { diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 4f978253d6..157e741acd 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -40,7 +40,9 @@ export async function get(viewId: string): Promise { return pickApi(tableId).get(viewId) } -export async function getEnriched(viewId: string): Promise { +export async function getEnriched( + viewId: string +): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) return pickApi(tableId).getEnriched(viewId) } diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts index 4f7abad357..9eeebc4cc4 100644 --- a/packages/server/src/sdk/app/views/internal.ts +++ b/packages/server/src/sdk/app/views/internal.ts @@ -17,13 +17,15 @@ export async function get(viewId: string): Promise { return ensureQueryUISet(found) } -export async function getEnriched(viewId: string): Promise { +export async function getEnriched( + viewId: string +): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) const table = await sdk.tables.getTable(tableId) const views = Object.values(table.views!).filter(isV2) const found = views.find(v => v.id === viewId) if (!found) { - throw new Error("No view found") + return } return await enrichSchema(ensureQueryUISet(found), table.schema) } From 566da4b2a3ac279407a43c9018448bd44555ef5a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 16 Jan 2025 16:58:57 +0100 Subject: [PATCH 07/22] ViewV2, return 404 if not found --- packages/server/src/sdk/app/views/external.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts index 65e0ff410d..fabee903fe 100644 --- a/packages/server/src/sdk/app/views/external.ts +++ b/packages/server/src/sdk/app/views/external.ts @@ -22,7 +22,9 @@ export async function get(viewId: string): Promise { return ensureQueryUISet(found) } -export async function getEnriched(viewId: string): Promise { +export async function getEnriched( + viewId: string +): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) const { datasourceId, tableName } = breakExternalTableId(tableId) @@ -32,7 +34,7 @@ export async function getEnriched(viewId: string): Promise { const views = Object.values(table.views!).filter(isV2) const found = views.find(v => v.id === viewId) if (!found) { - throw new Error("No view found") + return } return await enrichSchema(ensureQueryUISet(found), table.schema) } From da1f1177db3771ee410b3b0cb142cd033b7287c1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 16 Jan 2025 17:08:44 +0100 Subject: [PATCH 08/22] Type ComponentErrorState --- .../error-states/ComponentErrorState.svelte | 13 ++++++++----- packages/client/src/index.ts | 8 ++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/client/src/components/error-states/ComponentErrorState.svelte b/packages/client/src/components/error-states/ComponentErrorState.svelte index ec7c5169b0..b846eaa230 100644 --- a/packages/client/src/components/error-states/ComponentErrorState.svelte +++ b/packages/client/src/components/error-states/ComponentErrorState.svelte @@ -1,14 +1,17 @@ - From 177005ed5e450f2088ba841a14051e77723a4ace Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 16 Jan 2025 16:53:21 +0000 Subject: [PATCH 10/22] Convert viewBuilder.spec.js to ts --- .../__snapshots__/viewBuilder.spec.js.snap | 147 --------------- .../view/tests/viewBuilder.spec.js | 75 -------- .../view/tests/viewBuilder.spec.ts | 174 ++++++++++++++++++ .../src/api/controllers/view/viewBuilder.ts | 20 +- packages/types/src/documents/app/view.ts | 4 +- 5 files changed, 192 insertions(+), 228 deletions(-) delete mode 100644 packages/server/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap delete mode 100644 packages/server/src/api/controllers/view/tests/viewBuilder.spec.js create mode 100644 packages/server/src/api/controllers/view/tests/viewBuilder.spec.ts diff --git a/packages/server/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap b/packages/server/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap deleted file mode 100644 index 80f2188e69..0000000000 --- a/packages/server/src/api/controllers/view/tests/__snapshots__/viewBuilder.spec.js.snap +++ /dev/null @@ -1,147 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`viewBuilder Calculate and filter creates a view with the calculation statistics and filter schema 1`] = ` -{ - "map": "function (doc) { - if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !( - doc["myField"] === undefined || - doc["myField"] === null || - doc["myField"] === "" || - (Array.isArray(doc["myField"]) && doc["myField"].length === 0) - )) && (doc["age"] > 17)) { - emit(doc["_id"], doc["myField"]); - } - }", - "meta": { - "calculation": "stats", - "field": "myField", - "filters": [ - { - "condition": "MT", - "key": "age", - "value": 17, - }, - ], - "groupBy": undefined, - "schema": { - "avg": { - "type": "number", - }, - "count": { - "type": "number", - }, - "field": { - "type": "string", - }, - "max": { - "type": "number", - }, - "min": { - "type": "number", - }, - "sum": { - "type": "number", - }, - "sumsqr": { - "type": "number", - }, - }, - "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0", - }, - "reduce": "_stats", -} -`; - -exports[`viewBuilder Calculate creates a view with the calculation statistics schema 1`] = ` -{ - "map": "function (doc) { - if ((doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && !( - doc["myField"] === undefined || - doc["myField"] === null || - doc["myField"] === "" || - (Array.isArray(doc["myField"]) && doc["myField"].length === 0) - )) ) { - emit(doc["_id"], doc["myField"]); - } - }", - "meta": { - "calculation": "stats", - "field": "myField", - "filters": [], - "groupBy": undefined, - "schema": { - "avg": { - "type": "number", - }, - "count": { - "type": "number", - }, - "field": { - "type": "string", - }, - "max": { - "type": "number", - }, - "min": { - "type": "number", - }, - "sum": { - "type": "number", - }, - "sumsqr": { - "type": "number", - }, - }, - "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0", - }, - "reduce": "_stats", -} -`; - -exports[`viewBuilder Filter creates a view with multiple filters and conjunctions 1`] = ` -{ - "map": "function (doc) { - if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" && (doc["Name"] === "Test" || doc["Yes"] > "Value")) { - emit(doc["_id"], doc["undefined"]); - } - }", - "meta": { - "calculation": undefined, - "field": undefined, - "filters": [ - { - "condition": "EQUALS", - "key": "Name", - "value": "Test", - }, - { - "condition": "MT", - "conjunction": "OR", - "key": "Yes", - "value": "Value", - }, - ], - "groupBy": undefined, - "schema": null, - "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0", - }, -} -`; - -exports[`viewBuilder Group By creates a view emitting the group by field 1`] = ` -{ - "map": "function (doc) { - if (doc.tableId === "14f1c4e94d6a47b682ce89d35d4c78b0" ) { - emit(doc["age"], doc["score"]); - } - }", - "meta": { - "calculation": undefined, - "field": "score", - "filters": [], - "groupBy": "age", - "schema": null, - "tableId": "14f1c4e94d6a47b682ce89d35d4c78b0", - }, -} -`; diff --git a/packages/server/src/api/controllers/view/tests/viewBuilder.spec.js b/packages/server/src/api/controllers/view/tests/viewBuilder.spec.js deleted file mode 100644 index 7f4298b2ff..0000000000 --- a/packages/server/src/api/controllers/view/tests/viewBuilder.spec.js +++ /dev/null @@ -1,75 +0,0 @@ -const viewTemplate = require("../viewBuilder").default - -describe("viewBuilder", () => { - describe("Filter", () => { - it("creates a view with multiple filters and conjunctions", () => { - expect( - viewTemplate({ - name: "Test View", - tableId: "14f1c4e94d6a47b682ce89d35d4c78b0", - filters: [ - { - value: "Test", - condition: "EQUALS", - key: "Name", - }, - { - value: "Value", - condition: "MT", - key: "Yes", - conjunction: "OR", - }, - ], - }) - ).toMatchSnapshot() - }) - }) - - describe("Calculate", () => { - it("creates a view with the calculation statistics schema", () => { - expect( - viewTemplate({ - name: "Calculate View", - field: "myField", - calculation: "stats", - tableId: "14f1c4e94d6a47b682ce89d35d4c78b0", - filters: [], - }) - ).toMatchSnapshot() - }) - }) - - describe("Group By", () => { - it("creates a view emitting the group by field", () => { - expect( - viewTemplate({ - name: "Test Scores Grouped By Age", - tableId: "14f1c4e94d6a47b682ce89d35d4c78b0", - groupBy: "age", - field: "score", - filters: [], - }) - ).toMatchSnapshot() - }) - }) - - describe("Calculate and filter", () => { - it("creates a view with the calculation statistics and filter schema", () => { - expect( - viewTemplate({ - name: "Calculate View", - field: "myField", - calculation: "stats", - tableId: "14f1c4e94d6a47b682ce89d35d4c78b0", - filters: [ - { - value: 17, - condition: "MT", - key: "age", - }, - ], - }) - ).toMatchSnapshot() - }) - }) -}) diff --git a/packages/server/src/api/controllers/view/tests/viewBuilder.spec.ts b/packages/server/src/api/controllers/view/tests/viewBuilder.spec.ts new file mode 100644 index 0000000000..34526345f3 --- /dev/null +++ b/packages/server/src/api/controllers/view/tests/viewBuilder.spec.ts @@ -0,0 +1,174 @@ +import viewTemplate from "../viewBuilder" + +describe("viewBuilder", () => { + describe("Filter", () => { + it("creates a view with multiple filters and conjunctions", () => { + expect( + viewTemplate({ + field: "myField", + tableId: "tableId", + filters: [ + { + value: "Test", + condition: "EQUALS", + key: "Name", + }, + { + value: "Value", + condition: "MT", + key: "Yes", + conjunction: "OR", + }, + ], + }) + ).toEqual({ + map: `function (doc) { + if (doc.tableId === "tableId" && (doc["Name"] === "Test" || doc["Yes"] > "Value")) { + emit(doc["_id"], doc["myField"]); + } + }`, + meta: { + calculation: undefined, + field: "myField", + filters: [ + { + condition: "EQUALS", + key: "Name", + value: "Test", + }, + { + condition: "MT", + conjunction: "OR", + key: "Yes", + value: "Value", + }, + ], + groupBy: undefined, + schema: null, + tableId: "tableId", + }, + }) + }) + }) + + describe("Calculate", () => { + it("creates a view with the calculation statistics schema", () => { + expect( + viewTemplate({ + field: "myField", + calculation: "stats", + tableId: "tableId", + filters: [], + }) + ).toEqual({ + map: `function (doc) { + if ((doc.tableId === "tableId" && !( + doc["myField"] === undefined || + doc["myField"] === null || + doc["myField"] === "" || + (Array.isArray(doc["myField"]) && doc["myField"].length === 0) + )) ) { + emit(doc["_id"], doc["myField"]); + } + }`, + meta: { + calculation: "stats", + field: "myField", + filters: [], + groupBy: undefined, + schema: { + min: { type: "number" }, + max: { type: "number" }, + avg: { type: "number" }, + count: { type: "number" }, + sumsqr: { type: "number" }, + sum: { type: "number" }, + field: { type: "string" }, + }, + tableId: "tableId", + }, + reduce: "_stats", + }) + }) + }) + + describe("Group By", () => { + it("creates a view emitting the group by field", () => { + expect( + viewTemplate({ + tableId: "tableId", + groupBy: "age", + field: "score", + filters: [], + }) + ).toEqual({ + map: `function (doc) { + if (doc.tableId === "tableId" ) { + emit(doc["age"], doc["score"]); + } + }`, + meta: { + calculation: undefined, + field: "score", + filters: [], + groupBy: "age", + schema: null, + tableId: "tableId", + }, + }) + }) + }) + + describe("Calculate and filter", () => { + it("creates a view with the calculation statistics and filter schema", () => { + expect( + viewTemplate({ + field: "myField", + calculation: "stats", + tableId: "tableId", + filters: [ + { + value: 17, + condition: "MT", + key: "age", + }, + ], + }) + ).toEqual({ + map: `function (doc) { + if ((doc.tableId === "tableId" && !( + doc["myField"] === undefined || + doc["myField"] === null || + doc["myField"] === "" || + (Array.isArray(doc["myField"]) && doc["myField"].length === 0) + )) && (doc["age"] > 17)) { + emit(doc["_id"], doc["myField"]); + } + }`, + meta: { + calculation: "stats", + field: "myField", + filters: [ + { + condition: "MT", + key: "age", + value: 17, + }, + ], + groupBy: undefined, + schema: { + min: { type: "number" }, + max: { type: "number" }, + avg: { type: "number" }, + count: { type: "number" }, + sumsqr: { type: "number" }, + sum: { type: "number" }, + field: { type: "string" }, + }, + tableId: "tableId", + }, + reduce: "_stats", + }) + }) + }) +}) diff --git a/packages/server/src/api/controllers/view/viewBuilder.ts b/packages/server/src/api/controllers/view/viewBuilder.ts index 3df9df6657..2cedd1fc40 100644 --- a/packages/server/src/api/controllers/view/viewBuilder.ts +++ b/packages/server/src/api/controllers/view/viewBuilder.ts @@ -1,4 +1,4 @@ -import { ViewFilter, ViewTemplateOpts, DBView } from "@budibase/types" +import { ViewFilter, DBView } from "@budibase/types" const TOKEN_MAP: Record = { EQUALS: "===", @@ -120,7 +120,7 @@ function parseFilterExpression(filters: ViewFilter[]) { * @param groupBy - field to group calculation results on, if any */ function parseEmitExpression(field: string, groupBy: string) { - return `emit(doc["${groupBy || "_id"}"], doc["${field}"]);` + return `emit(doc["${groupBy}"], doc["${field}"]);` } /** @@ -135,7 +135,19 @@ function parseEmitExpression(field: string, groupBy: string) { * calculation: an optional calculation to be performed over the view data. */ export default function ( - { field, tableId, groupBy, filters = [], calculation }: ViewTemplateOpts, + { + field, + tableId, + groupBy, + filters = [], + calculation, + }: { + field: string + tableId: string + groupBy?: string + filters?: ViewFilter[] + calculation?: string + }, groupByMulti?: boolean ): DBView { // first filter can't have a conjunction @@ -168,7 +180,7 @@ export default function ( const parsedFilters = parseFilterExpression(filters) const filterExpression = parsedFilters ? `&& (${parsedFilters})` : "" - const emitExpression = parseEmitExpression(field, groupBy) + const emitExpression = parseEmitExpression(field, groupBy || "_id") const tableExpression = `doc.tableId === "${tableId}"` const coreExpression = statFilter ? `(${tableExpression} && ${statFilter})` diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index fd97cee409..a7fbf06377 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -6,10 +6,10 @@ import { DBView, SearchFilters } from "../../sdk" export type ViewTemplateOpts = { field: string tableId: string - groupBy: string + groupBy?: string filters: ViewFilter[] schema: any - calculation: string + calculation?: string groupByMulti?: boolean } From cd6c3e39459c6b1261c65ac5456a3a440db079e6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 16 Jan 2025 17:25:13 +0000 Subject: [PATCH 11/22] Extract automation step definitions out into shared-core. --- .../src/api/routes/tests/automation.spec.ts | 4 +- packages/server/src/automations/actions.ts | 47 +++++---- packages/server/src/automations/steps/bash.ts | 50 +--------- .../server/src/automations/steps/collect.ts | 46 +-------- .../server/src/automations/steps/createRow.ts | 70 +------------ .../server/src/automations/steps/delay.ts | 41 +------- .../server/src/automations/steps/deleteRow.ts | 59 +---------- .../server/src/automations/steps/discord.ts | 63 +----------- .../src/automations/steps/executeQuery.ts | 57 ----------- .../src/automations/steps/executeScript.ts | 45 --------- .../server/src/automations/steps/filter.ts | 73 +------------- packages/server/src/automations/steps/make.ts | 58 +---------- packages/server/src/automations/steps/n8n.ts | 62 ------------ .../server/src/automations/steps/openai.ts | 59 +---------- .../src/automations/steps/outgoingWebhook.ts | 69 ------------- .../server/src/automations/steps/queryRows.ts | 72 -------------- .../src/automations/steps/sendSmtpEmail.ts | 98 +------------------ .../server/src/automations/steps/serverLog.ts | 56 +---------- .../server/src/automations/steps/slack.ts | 55 +---------- .../automations/steps/triggerAutomationRun.ts | 53 ---------- .../server/src/automations/steps/updateRow.ts | 70 +------------ .../server/src/automations/steps/zapier.ts | 51 +--------- .../src/automations/tests/filter.spec.ts | 4 +- .../tests/scenarios/scenarios.spec.ts | 4 +- .../tests/utilities/AutomationTestBuilder.ts | 4 +- packages/shared-core/src/automations/index.ts | 1 + .../shared-core/src/automations/steps/bash.ts | 47 +++++++++ .../src/automations/steps/branch.ts | 0 .../src/automations/steps/collect.ts | 43 ++++++++ .../src/automations/steps/createRow.ts | 67 +++++++++++++ .../src/automations/steps/delay.ts | 38 +++++++ .../src/automations/steps/deleteRow.ts | 56 +++++++++++ .../src/automations/steps/discord.ts | 60 ++++++++++++ .../src/automations/steps/executeQuery.ts | 59 +++++++++++ .../src/automations/steps/executeScript.ts | 47 +++++++++ .../src/automations/steps/filter.ts | 69 +++++++++++++ .../src/automations/steps/index.ts | 22 +++++ .../src/automations/steps/loop.ts | 0 .../shared-core/src/automations/steps/make.ts | 55 +++++++++++ .../shared-core/src/automations/steps/n8n.ts | 65 ++++++++++++ .../src/automations/steps/openai.ts | 56 +++++++++++ .../src/automations/steps/outgoingWebhook.ts | 79 +++++++++++++++ .../src/automations/steps/queryRows.ts | 75 ++++++++++++++ .../src/automations/steps/sendSmtpEmail.ts | 95 ++++++++++++++++++ .../src/automations/steps/serverLog.ts | 47 +++++++++ .../src/automations/steps/slack.ts | 52 ++++++++++ .../automations/steps/triggerAutomationRun.ts | 55 +++++++++++ .../src/automations/steps/updateRow.ts | 68 +++++++++++++ .../src/automations/steps/zapier.ts | 48 +++++++++ packages/shared-core/src/index.ts | 1 + 50 files changed, 1255 insertions(+), 1220 deletions(-) create mode 100644 packages/shared-core/src/automations/index.ts create mode 100644 packages/shared-core/src/automations/steps/bash.ts rename packages/{server => shared-core}/src/automations/steps/branch.ts (100%) create mode 100644 packages/shared-core/src/automations/steps/collect.ts create mode 100644 packages/shared-core/src/automations/steps/createRow.ts create mode 100644 packages/shared-core/src/automations/steps/delay.ts create mode 100644 packages/shared-core/src/automations/steps/deleteRow.ts create mode 100644 packages/shared-core/src/automations/steps/discord.ts create mode 100644 packages/shared-core/src/automations/steps/executeQuery.ts create mode 100644 packages/shared-core/src/automations/steps/executeScript.ts create mode 100644 packages/shared-core/src/automations/steps/filter.ts create mode 100644 packages/shared-core/src/automations/steps/index.ts rename packages/{server => shared-core}/src/automations/steps/loop.ts (100%) create mode 100644 packages/shared-core/src/automations/steps/make.ts create mode 100644 packages/shared-core/src/automations/steps/n8n.ts create mode 100644 packages/shared-core/src/automations/steps/openai.ts create mode 100644 packages/shared-core/src/automations/steps/outgoingWebhook.ts create mode 100644 packages/shared-core/src/automations/steps/queryRows.ts create mode 100644 packages/shared-core/src/automations/steps/sendSmtpEmail.ts create mode 100644 packages/shared-core/src/automations/steps/serverLog.ts create mode 100644 packages/shared-core/src/automations/steps/slack.ts create mode 100644 packages/shared-core/src/automations/steps/triggerAutomationRun.ts create mode 100644 packages/shared-core/src/automations/steps/updateRow.ts create mode 100644 packages/shared-core/src/automations/steps/zapier.ts diff --git a/packages/server/src/api/routes/tests/automation.spec.ts b/packages/server/src/api/routes/tests/automation.spec.ts index a517346c82..5c0b86d9a0 100644 --- a/packages/server/src/api/routes/tests/automation.spec.ts +++ b/packages/server/src/api/routes/tests/automation.spec.ts @@ -19,9 +19,11 @@ import { Table, } from "@budibase/types" import { mocks } from "@budibase/backend-core/tests" -import { FilterConditions } from "../../../automations/steps/filter" import { removeDeprecated } from "../../../automations/utils" import { createAutomationBuilder } from "../../../automations/tests/utilities/AutomationTestBuilder" +import { automations } from "@budibase/shared-core" + +const FilterConditions = automations.steps.filter.FilterConditions const MAX_RETRIES = 4 let { diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index 537b6befc3..f35f96babe 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -1,3 +1,4 @@ +import { automations } from "@budibase/shared-core" import * as sendSmtpEmail from "./steps/sendSmtpEmail" import * as createRow from "./steps/createRow" import * as updateRow from "./steps/updateRow" @@ -14,9 +15,7 @@ import * as make from "./steps/make" import * as filter from "./steps/filter" import * as delay from "./steps/delay" import * as queryRow from "./steps/queryRows" -import * as loop from "./steps/loop" import * as collect from "./steps/collect" -import * as branch from "./steps/branch" import * as triggerAutomationRun from "./steps/triggerAutomationRun" import * as openai from "./steps/openai" import env from "../environment" @@ -62,27 +61,27 @@ export const BUILTIN_ACTION_DEFINITIONS: Record< string, AutomationStepDefinition > = { - SEND_EMAIL_SMTP: sendSmtpEmail.definition, - CREATE_ROW: createRow.definition, - UPDATE_ROW: updateRow.definition, - DELETE_ROW: deleteRow.definition, - OUTGOING_WEBHOOK: outgoingWebhook.definition, - EXECUTE_SCRIPT: executeScript.definition, - EXECUTE_QUERY: executeQuery.definition, - SERVER_LOG: serverLog.definition, - DELAY: delay.definition, - FILTER: filter.definition, - QUERY_ROWS: queryRow.definition, - LOOP: loop.definition, - COLLECT: collect.definition, - TRIGGER_AUTOMATION_RUN: triggerAutomationRun.definition, - BRANCH: branch.definition, + SEND_EMAIL_SMTP: automations.steps.sendSmtpEmail.definition, + CREATE_ROW: automations.steps.createRow.definition, + UPDATE_ROW: automations.steps.updateRow.definition, + DELETE_ROW: automations.steps.deleteRow.definition, + OUTGOING_WEBHOOK: automations.steps.outgoingWebhook.definition, + EXECUTE_SCRIPT: automations.steps.executeScript.definition, + EXECUTE_QUERY: automations.steps.executeQuery.definition, + SERVER_LOG: automations.steps.serverLog.definition, + DELAY: automations.steps.delay.definition, + FILTER: automations.steps.filter.definition, + QUERY_ROWS: automations.steps.queryRows.definition, + LOOP: automations.steps.loop.definition, + COLLECT: automations.steps.collect.definition, + TRIGGER_AUTOMATION_RUN: automations.steps.triggerAutomationRun.definition, + BRANCH: automations.steps.branch.definition, // these used to be lowercase step IDs, maintain for backwards compat - discord: discord.definition, - slack: slack.definition, - zapier: zapier.definition, - integromat: make.definition, - n8n: n8n.definition, + discord: automations.steps.discord.definition, + slack: automations.steps.slack.definition, + zapier: automations.steps.zapier.definition, + integromat: automations.steps.make.definition, + n8n: automations.steps.n8n.definition, } // don't add the bash script/definitions unless in self host @@ -97,7 +96,7 @@ if (env.SELF_HOSTED) { BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition if (env.isTest()) { - BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition + BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition } } @@ -105,7 +104,7 @@ export async function getActionDefinitions(): Promise< Record > { if (env.SELF_HOSTED) { - BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition + BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition } const actionDefinitions = BUILTIN_ACTION_DEFINITIONS diff --git a/packages/server/src/automations/steps/bash.ts b/packages/server/src/automations/steps/bash.ts index 636e659ffe..f6d2feecf6 100644 --- a/packages/server/src/automations/steps/bash.ts +++ b/packages/server/src/automations/steps/bash.ts @@ -2,55 +2,7 @@ import { execSync } from "child_process" import { processStringSync } from "@budibase/string-templates" import * as automationUtils from "../automationUtils" import environment from "../../environment" -import { - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, - BashStepInputs, - BashStepOutputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - name: "Bash Scripting", - tagline: "Execute a bash command", - icon: "JourneyEvent", - description: "Run a bash script", - type: AutomationStepType.ACTION, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - stepId: AutomationActionStepId.EXECUTE_BASH, - inputs: {}, - schema: { - inputs: { - properties: { - code: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.CODE, - title: "Code", - }, - }, - required: ["code"], - }, - outputs: { - properties: { - stdout: { - type: AutomationIOType.STRING, - description: "Standard output of your bash command or script", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the command was successful", - }, - }, - required: ["stdout"], - }, - }, -} +import { BashStepInputs, BashStepOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/collect.ts b/packages/server/src/automations/steps/collect.ts index 3b2ca7d21f..689c3d347f 100644 --- a/packages/server/src/automations/steps/collect.ts +++ b/packages/server/src/automations/steps/collect.ts @@ -1,48 +1,4 @@ -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - CollectStepInputs, - CollectStepOutputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - name: "Collect Data", - tagline: "Collect data to be sent to design", - icon: "Collection", - description: - "Collects specified data so it can be provided to the design section", - type: AutomationStepType.ACTION, - internal: true, - features: {}, - stepId: AutomationActionStepId.COLLECT, - inputs: {}, - schema: { - inputs: { - properties: { - collection: { - type: AutomationIOType.STRING, - title: "What to Collect", - }, - }, - required: ["collection"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - value: { - type: AutomationIOType.STRING, - description: "Collected data", - }, - }, - required: ["success", "value"], - }, - }, -} +import { CollectStepInputs, CollectStepOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/createRow.ts b/packages/server/src/automations/steps/createRow.ts index cd16be2cf3..24dada422d 100644 --- a/packages/server/src/automations/steps/createRow.ts +++ b/packages/server/src/automations/steps/createRow.ts @@ -5,77 +5,9 @@ import { sendAutomationAttachmentsToStorage, } from "../automationUtils" import { buildCtx } from "./utils" -import { - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, - CreateRowStepInputs, - CreateRowStepOutputs, -} from "@budibase/types" +import { CreateRowStepInputs, CreateRowStepOutputs } from "@budibase/types" import { EventEmitter } from "events" -export const definition: AutomationStepDefinition = { - name: "Create Row", - tagline: "Create a {{inputs.enriched.table.name}} row", - icon: "TableRowAddBottom", - description: "Add a row to your database", - type: AutomationStepType.ACTION, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - stepId: AutomationActionStepId.CREATE_ROW, - inputs: {}, - schema: { - inputs: { - properties: { - row: { - type: AutomationIOType.OBJECT, - properties: { - tableId: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.TABLE, - }, - }, - customType: AutomationCustomIOType.ROW, - title: "Table", - required: ["tableId"], - }, - }, - required: ["row"], - }, - outputs: { - properties: { - row: { - type: AutomationIOType.OBJECT, - customType: AutomationCustomIOType.ROW, - description: "The new row", - }, - response: { - type: AutomationIOType.OBJECT, - description: "The response from the table", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the row creation was successful", - }, - id: { - type: AutomationIOType.STRING, - description: "The identifier of the new row", - }, - revision: { - type: AutomationIOType.STRING, - description: "The revision of the new row", - }, - }, - required: ["success", "id", "revision"], - }, - }, -} - export async function run({ inputs, appId, diff --git a/packages/server/src/automations/steps/delay.ts b/packages/server/src/automations/steps/delay.ts index c623693488..4ca5a16942 100644 --- a/packages/server/src/automations/steps/delay.ts +++ b/packages/server/src/automations/steps/delay.ts @@ -1,44 +1,5 @@ import { wait } from "../../utilities" -import { - AutomationActionStepId, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, - DelayStepInputs, - DelayStepOutputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - name: "Delay", - icon: "Clock", - tagline: "Delay for {{inputs.time}} milliseconds", - description: "Delay the automation until an amount of time has passed", - stepId: AutomationActionStepId.DELAY, - internal: true, - features: {}, - inputs: {}, - schema: { - inputs: { - properties: { - time: { - type: AutomationIOType.NUMBER, - title: "Delay in milliseconds", - }, - }, - required: ["time"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the delay was successful", - }, - }, - required: ["success"], - }, - }, - type: AutomationStepType.LOGIC, -} +import { DelayStepInputs, DelayStepOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/deleteRow.ts b/packages/server/src/automations/steps/deleteRow.ts index 6b8a68eeb8..7c50fe4dcb 100644 --- a/packages/server/src/automations/steps/deleteRow.ts +++ b/packages/server/src/automations/steps/deleteRow.ts @@ -2,64 +2,7 @@ import { EventEmitter } from "events" import { destroy } from "../../api/controllers/row" import { buildCtx } from "./utils" import { getError } from "../automationUtils" -import { - AutomationActionStepId, - AutomationStepType, - AutomationIOType, - AutomationCustomIOType, - AutomationFeature, - DeleteRowStepInputs, - DeleteRowStepOutputs, - AutomationStepDefinition, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - description: "Delete a row from your database", - icon: "TableRowRemoveCenter", - name: "Delete Row", - tagline: "Delete a {{inputs.enriched.table.name}} row", - type: AutomationStepType.ACTION, - stepId: AutomationActionStepId.DELETE_ROW, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - tableId: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.TABLE, - title: "Table", - }, - id: { - type: AutomationIOType.STRING, - title: "Row ID", - }, - }, - required: ["tableId", "id"], - }, - outputs: { - properties: { - row: { - type: AutomationIOType.OBJECT, - customType: AutomationCustomIOType.ROW, - description: "The deleted row", - }, - response: { - type: AutomationIOType.OBJECT, - description: "The response from the table", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the deletion was successful", - }, - }, - required: ["row", "success"], - }, - }, -} +import { DeleteRowStepInputs, DeleteRowStepOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/discord.ts b/packages/server/src/automations/steps/discord.ts index 1ce21fc89e..30d3e8137b 100644 --- a/packages/server/src/automations/steps/discord.ts +++ b/packages/server/src/automations/steps/discord.ts @@ -1,71 +1,10 @@ import fetch from "node-fetch" import { getFetchResponse } from "./utils" -import { - AutomationActionStepId, - AutomationStepType, - AutomationIOType, - AutomationFeature, - ExternalAppStepOutputs, - DiscordStepInputs, - AutomationStepDefinition, -} from "@budibase/types" +import { ExternalAppStepOutputs, DiscordStepInputs } from "@budibase/types" const DEFAULT_USERNAME = "Budibase Automate" const DEFAULT_AVATAR_URL = "https://i.imgur.com/a1cmTKM.png" -export const definition: AutomationStepDefinition = { - name: "Discord Message", - tagline: "Send a message to a Discord server", - description: "Send a message to a Discord server", - icon: "ri-discord-line", - stepId: AutomationActionStepId.discord, - type: AutomationStepType.ACTION, - internal: false, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - url: { - type: AutomationIOType.STRING, - title: "Discord Webhook URL", - }, - username: { - type: AutomationIOType.STRING, - title: "Bot Name", - }, - avatar_url: { - type: AutomationIOType.STRING, - title: "Bot Avatar URL", - }, - content: { - type: AutomationIOType.STRING, - title: "Message", - }, - }, - required: ["url", "content"], - }, - outputs: { - properties: { - httpStatus: { - type: AutomationIOType.NUMBER, - description: "The HTTP status code of the request", - }, - response: { - type: AutomationIOType.STRING, - description: "The response from the Discord Webhook", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the message sent successfully", - }, - }, - }, - }, -} - export async function run({ inputs, }: { diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts index 7af540093f..9816e31b1e 100644 --- a/packages/server/src/automations/steps/executeQuery.ts +++ b/packages/server/src/automations/steps/executeQuery.ts @@ -3,67 +3,10 @@ import * as queryController from "../../api/controllers/query" import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" import { - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, ExecuteQueryStepInputs, ExecuteQueryStepOutputs, } from "@budibase/types" -export const definition: AutomationStepDefinition = { - name: "External Data Connector", - tagline: "Execute Data Connector", - icon: "Data", - description: "Execute a query in an external data connector", - type: AutomationStepType.ACTION, - stepId: AutomationActionStepId.EXECUTE_QUERY, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - query: { - type: AutomationIOType.OBJECT, - properties: { - queryId: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.QUERY, - }, - }, - customType: AutomationCustomIOType.QUERY_PARAMS, - title: "Parameters", - required: ["queryId"], - }, - }, - required: ["query"], - }, - outputs: { - properties: { - response: { - type: AutomationIOType.OBJECT, - description: "The response from the datasource execution", - }, - info: { - type: AutomationIOType.OBJECT, - description: - "Some query types may return extra data, like headers from a REST query", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - }, - required: ["response", "success"], - }, - }, -} - export async function run({ inputs, appId, diff --git a/packages/server/src/automations/steps/executeScript.ts b/packages/server/src/automations/steps/executeScript.ts index 97145f3e14..105543d34c 100644 --- a/packages/server/src/automations/steps/executeScript.ts +++ b/packages/server/src/automations/steps/executeScript.ts @@ -2,56 +2,11 @@ import * as scriptController from "../../api/controllers/script" import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" import { - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, ExecuteScriptStepInputs, ExecuteScriptStepOutputs, } from "@budibase/types" import { EventEmitter } from "events" -export const definition: AutomationStepDefinition = { - name: "JS Scripting", - tagline: "Execute JavaScript Code", - icon: "Code", - description: "Run a piece of JavaScript code in your automation", - type: AutomationStepType.ACTION, - internal: true, - stepId: AutomationActionStepId.EXECUTE_SCRIPT, - inputs: {}, - features: { - [AutomationFeature.LOOPING]: true, - }, - schema: { - inputs: { - properties: { - code: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.CODE, - title: "Code", - }, - }, - required: ["code"], - }, - outputs: { - properties: { - value: { - type: AutomationIOType.STRING, - description: "The result of the return statement", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - }, - required: ["success"], - }, - }, -} - export async function run({ inputs, appId, diff --git a/packages/server/src/automations/steps/filter.ts b/packages/server/src/automations/steps/filter.ts index 84cea00b93..9b7e347034 100644 --- a/packages/server/src/automations/steps/filter.ts +++ b/packages/server/src/automations/steps/filter.ts @@ -1,74 +1,7 @@ -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - FilterStepInputs, - FilterStepOutputs, -} from "@budibase/types" +import { FilterStepInputs, FilterStepOutputs } from "@budibase/types" +import { automations } from "@budibase/shared-core" -export const FilterConditions = { - EQUAL: "EQUAL", - NOT_EQUAL: "NOT_EQUAL", - GREATER_THAN: "GREATER_THAN", - LESS_THAN: "LESS_THAN", -} - -export const PrettyFilterConditions = { - [FilterConditions.EQUAL]: "Equals", - [FilterConditions.NOT_EQUAL]: "Not equals", - [FilterConditions.GREATER_THAN]: "Greater than", - [FilterConditions.LESS_THAN]: "Less than", -} - -export const definition: AutomationStepDefinition = { - name: "Condition", - tagline: "{{inputs.field}} {{inputs.condition}} {{inputs.value}}", - icon: "Branch2", - description: - "Conditionally halt automations which do not meet certain conditions", - type: AutomationStepType.LOGIC, - internal: true, - features: {}, - stepId: AutomationActionStepId.FILTER, - inputs: { - condition: FilterConditions.EQUAL, - }, - schema: { - inputs: { - properties: { - field: { - type: AutomationIOType.STRING, - title: "Reference Value", - }, - condition: { - type: AutomationIOType.STRING, - title: "Condition", - enum: Object.values(FilterConditions), - pretty: Object.values(PrettyFilterConditions), - }, - value: { - type: AutomationIOType.STRING, - title: "Comparison Value", - }, - }, - required: ["field", "condition", "value"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - result: { - type: AutomationIOType.BOOLEAN, - description: "Whether the logic block passed", - }, - }, - required: ["success", "result"], - }, - }, -} +const FilterConditions = automations.steps.filter.FilterConditions export async function run({ inputs, diff --git a/packages/server/src/automations/steps/make.ts b/packages/server/src/automations/steps/make.ts index 62cfd66302..241cccce9e 100644 --- a/packages/server/src/automations/steps/make.ts +++ b/packages/server/src/automations/steps/make.ts @@ -1,62 +1,6 @@ import fetch from "node-fetch" import { getFetchResponse } from "./utils" -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - AutomationFeature, - ExternalAppStepOutputs, - MakeIntegrationInputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - name: "Make Integration", - stepTitle: "Make", - tagline: "Trigger a Make scenario", - description: - "Performs a webhook call to Make and gets the response (if configured)", - icon: "ri-shut-down-line", - stepId: AutomationActionStepId.integromat, - type: AutomationStepType.ACTION, - internal: false, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - url: { - type: AutomationIOType.STRING, - title: "Webhook URL", - }, - body: { - type: AutomationIOType.JSON, - title: "Payload", - }, - }, - required: ["url", "body"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether call was successful", - }, - httpStatus: { - type: AutomationIOType.NUMBER, - description: "The HTTP status code returned", - }, - response: { - type: AutomationIOType.OBJECT, - description: "The webhook response - this can have properties", - }, - }, - required: ["success", "response"], - }, - }, -} +import { ExternalAppStepOutputs, MakeIntegrationInputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/n8n.ts b/packages/server/src/automations/steps/n8n.ts index 8e55d2535b..29883dd327 100644 --- a/packages/server/src/automations/steps/n8n.ts +++ b/packages/server/src/automations/steps/n8n.ts @@ -1,73 +1,11 @@ import fetch, { HeadersInit } from "node-fetch" import { getFetchResponse } from "./utils" import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - AutomationFeature, HttpMethod, ExternalAppStepOutputs, n8nStepInputs, } from "@budibase/types" -export const definition: AutomationStepDefinition = { - name: "n8n Integration", - stepTitle: "n8n", - tagline: "Trigger an n8n workflow", - description: - "Performs a webhook call to n8n and gets the response (if configured)", - icon: "ri-shut-down-line", - stepId: AutomationActionStepId.n8n, - type: AutomationStepType.ACTION, - internal: false, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - url: { - type: AutomationIOType.STRING, - title: "Webhook URL", - }, - method: { - type: AutomationIOType.STRING, - title: "Method", - enum: Object.values(HttpMethod), - }, - authorization: { - type: AutomationIOType.STRING, - title: "Authorization", - }, - body: { - type: AutomationIOType.JSON, - title: "Payload", - }, - }, - required: ["url", "method"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether call was successful", - }, - httpStatus: { - type: AutomationIOType.NUMBER, - description: "The HTTP status code returned", - }, - response: { - type: AutomationIOType.OBJECT, - description: "The webhook response - this can have properties", - }, - }, - required: ["success", "response"], - }, - }, -} - export async function run({ inputs, }: { diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 53e41ceb09..55f57aff7c 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -1,67 +1,10 @@ import { OpenAI } from "openai" -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - OpenAIStepInputs, - OpenAIStepOutputs, -} from "@budibase/types" +import { OpenAIStepInputs, OpenAIStepOutputs } from "@budibase/types" import { env } from "@budibase/backend-core" import * as automationUtils from "../automationUtils" import * as pro from "@budibase/pro" -enum Model { - GPT_4O_MINI = "gpt-4o-mini", - GPT_4O = "gpt-4o", - GPT_4 = "gpt-4", - GPT_35_TURBO = "gpt-3.5-turbo", -} - -export const definition: AutomationStepDefinition = { - name: "OpenAI", - tagline: "Send prompts to ChatGPT", - icon: "Algorithm", - description: "Interact with the OpenAI ChatGPT API.", - type: AutomationStepType.ACTION, - internal: true, - features: {}, - stepId: AutomationActionStepId.OPENAI, - inputs: { - prompt: "", - }, - schema: { - inputs: { - properties: { - prompt: { - type: AutomationIOType.STRING, - title: "Prompt", - }, - model: { - type: AutomationIOType.STRING, - title: "Model", - enum: Object.values(Model), - }, - }, - required: ["prompt", "model"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - response: { - type: AutomationIOType.STRING, - description: "What was output", - }, - }, - required: ["success", "response"], - }, - }, -} - /** * Maintains backward compatibility with automation steps created before the introduction * of custom configurations and Budibase AI diff --git a/packages/server/src/automations/steps/outgoingWebhook.ts b/packages/server/src/automations/steps/outgoingWebhook.ts index 70b8b1f476..a1211ea281 100644 --- a/packages/server/src/automations/steps/outgoingWebhook.ts +++ b/packages/server/src/automations/steps/outgoingWebhook.ts @@ -2,12 +2,6 @@ import fetch from "node-fetch" import { getFetchResponse } from "./utils" import * as automationUtils from "../automationUtils" import { - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, ExternalAppStepOutputs, OutgoingWebhookStepInputs, } from "@budibase/types" @@ -26,69 +20,6 @@ const BODY_REQUESTS = [RequestType.POST, RequestType.PUT, RequestType.PATCH] * NOTE: this functionality is deprecated - it no longer should be used. */ -export const definition: AutomationStepDefinition = { - deprecated: true, - name: "Outgoing webhook", - tagline: "Send a {{inputs.requestMethod}} request", - icon: "Send", - description: "Send a request of specified method to a URL", - type: AutomationStepType.ACTION, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - stepId: AutomationActionStepId.OUTGOING_WEBHOOK, - inputs: { - requestMethod: "POST", - url: "http://", - requestBody: "{}", - headers: "{}", - }, - schema: { - inputs: { - properties: { - requestMethod: { - type: AutomationIOType.STRING, - enum: Object.values(RequestType), - title: "Request method", - }, - url: { - type: AutomationIOType.STRING, - title: "URL", - }, - requestBody: { - type: AutomationIOType.STRING, - title: "JSON Body", - customType: AutomationCustomIOType.WIDE, - }, - headers: { - type: AutomationIOType.STRING, - title: "Headers", - customType: AutomationCustomIOType.WIDE, - }, - }, - required: ["requestMethod", "url"], - }, - outputs: { - properties: { - response: { - type: AutomationIOType.OBJECT, - description: "The response from the webhook", - }, - httpStatus: { - type: AutomationIOType.NUMBER, - description: "The HTTP status code returned", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - }, - required: ["response", "success"], - }, - }, -} - export async function run({ inputs, }: { diff --git a/packages/server/src/automations/steps/queryRows.ts b/packages/server/src/automations/steps/queryRows.ts index 4a02e80686..9c60a63dac 100644 --- a/packages/server/src/automations/steps/queryRows.ts +++ b/packages/server/src/automations/steps/queryRows.ts @@ -4,84 +4,12 @@ import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" import { FieldType, - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, EmptyFilterOption, SortOrder, QueryRowsStepInputs, QueryRowsStepOutputs, } from "@budibase/types" -const SortOrderPretty = { - [SortOrder.ASCENDING]: "Ascending", - [SortOrder.DESCENDING]: "Descending", -} - -export const definition: AutomationStepDefinition = { - description: "Query rows from the database", - icon: "Search", - name: "Query rows", - tagline: "Query rows from {{inputs.enriched.table.name}} table", - type: AutomationStepType.ACTION, - stepId: AutomationActionStepId.QUERY_ROWS, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - tableId: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.TABLE, - title: "Table", - }, - filters: { - type: AutomationIOType.OBJECT, - customType: AutomationCustomIOType.FILTERS, - title: "Filtering", - }, - sortColumn: { - type: AutomationIOType.STRING, - title: "Sort Column", - customType: AutomationCustomIOType.COLUMN, - }, - sortOrder: { - type: AutomationIOType.STRING, - title: "Sort Order", - enum: Object.values(SortOrder), - pretty: Object.values(SortOrderPretty), - }, - limit: { - type: AutomationIOType.NUMBER, - title: "Limit", - customType: AutomationCustomIOType.QUERY_LIMIT, - }, - }, - required: ["tableId"], - }, - outputs: { - properties: { - rows: { - type: AutomationIOType.ARRAY, - customType: AutomationCustomIOType.ROWS, - description: "The rows that were found", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the query was successful", - }, - }, - required: ["rows", "success"], - }, - }, -} - async function getTable(appId: string, tableId: string) { const ctx: any = buildCtx(appId, null, { params: { diff --git a/packages/server/src/automations/steps/sendSmtpEmail.ts b/packages/server/src/automations/steps/sendSmtpEmail.ts index 72ca410528..bfa2fa31ad 100644 --- a/packages/server/src/automations/steps/sendSmtpEmail.ts +++ b/packages/server/src/automations/steps/sendSmtpEmail.ts @@ -1,102 +1,6 @@ import { sendSmtpEmail } from "../../utilities/workerRequests" import * as automationUtils from "../automationUtils" -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - AutomationFeature, - AutomationCustomIOType, - SmtpEmailStepInputs, - BaseAutomationOutputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - description: "Send an email using SMTP", - tagline: "Send SMTP email to {{inputs.to}}", - icon: "Email", - name: "Send Email (SMTP)", - type: AutomationStepType.ACTION, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - stepId: AutomationActionStepId.SEND_EMAIL_SMTP, - inputs: {}, - schema: { - inputs: { - properties: { - to: { - type: AutomationIOType.STRING, - title: "Send To", - }, - from: { - type: AutomationIOType.STRING, - title: "Send From", - }, - cc: { - type: AutomationIOType.STRING, - title: "CC", - }, - bcc: { - type: AutomationIOType.STRING, - title: "BCC", - }, - subject: { - type: AutomationIOType.STRING, - title: "Email Subject", - }, - contents: { - type: AutomationIOType.STRING, - title: "HTML Contents", - }, - addInvite: { - type: AutomationIOType.BOOLEAN, - title: "Add calendar invite", - }, - startTime: { - type: AutomationIOType.DATE, - title: "Start Time", - dependsOn: "addInvite", - }, - endTime: { - type: AutomationIOType.DATE, - title: "End Time", - dependsOn: "addInvite", - }, - summary: { - type: AutomationIOType.STRING, - title: "Meeting Summary", - dependsOn: "addInvite", - }, - location: { - type: AutomationIOType.STRING, - title: "Location", - dependsOn: "addInvite", - }, - attachments: { - type: AutomationIOType.ATTACHMENT, - customType: AutomationCustomIOType.MULTI_ATTACHMENTS, - title: "Attachments", - }, - }, - required: ["to", "from", "subject", "contents"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the email was sent", - }, - response: { - type: AutomationIOType.OBJECT, - description: "A response from the email client, this may be an error", - }, - }, - required: ["success"], - }, - }, -} +import { SmtpEmailStepInputs, BaseAutomationOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/serverLog.ts b/packages/server/src/automations/steps/serverLog.ts index d0702ab3eb..b3588b7378 100644 --- a/packages/server/src/automations/steps/serverLog.ts +++ b/packages/server/src/automations/steps/serverLog.ts @@ -1,58 +1,4 @@ -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - AutomationFeature, - ServerLogStepInputs, - ServerLogStepOutputs, -} from "@budibase/types" - -/** - * Note, there is some functionality in this that is not currently exposed as it - * is complex and maybe better to be opinionated here. - * GET/DELETE requests cannot handle body elements so they will not be sent if configured. - */ - -export const definition: AutomationStepDefinition = { - name: "Backend log", - tagline: "Console log a value in the backend", - icon: "Monitoring", - description: "Logs the given text to the server (using console.log)", - type: AutomationStepType.ACTION, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - stepId: AutomationActionStepId.SERVER_LOG, - inputs: { - text: "", - }, - schema: { - inputs: { - properties: { - text: { - type: AutomationIOType.STRING, - title: "Log", - }, - }, - required: ["text"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - message: { - type: AutomationIOType.STRING, - description: "What was output", - }, - }, - required: ["success", "message"], - }, - }, -} +import { ServerLogStepInputs, ServerLogStepOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/slack.ts b/packages/server/src/automations/steps/slack.ts index 353611260c..12dcea74c7 100644 --- a/packages/server/src/automations/steps/slack.ts +++ b/packages/server/src/automations/steps/slack.ts @@ -1,59 +1,6 @@ import fetch from "node-fetch" import { getFetchResponse } from "./utils" -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - AutomationFeature, - ExternalAppStepOutputs, - SlackStepInputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - name: "Slack Message", - tagline: "Send a message to Slack", - description: "Send a message to Slack", - icon: "ri-slack-line", - stepId: AutomationActionStepId.slack, - type: AutomationStepType.ACTION, - internal: false, - features: { - [AutomationFeature.LOOPING]: true, - }, - inputs: {}, - schema: { - inputs: { - properties: { - url: { - type: AutomationIOType.STRING, - title: "Incoming Webhook URL", - }, - text: { - type: AutomationIOType.STRING, - title: "Message", - }, - }, - required: ["url", "text"], - }, - outputs: { - properties: { - httpStatus: { - type: AutomationIOType.NUMBER, - description: "The HTTP status code of the request", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the message sent successfully", - }, - response: { - type: AutomationIOType.STRING, - description: "The response from the Slack Webhook", - }, - }, - }, - }, -} +import { ExternalAppStepOutputs, SlackStepInputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/steps/triggerAutomationRun.ts b/packages/server/src/automations/steps/triggerAutomationRun.ts index f1cf65b182..5c7959648d 100644 --- a/packages/server/src/automations/steps/triggerAutomationRun.ts +++ b/packages/server/src/automations/steps/triggerAutomationRun.ts @@ -1,10 +1,5 @@ import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, Automation, - AutomationCustomIOType, TriggerAutomationStepInputs, TriggerAutomationStepOutputs, } from "@budibase/types" @@ -13,54 +8,6 @@ import { context } from "@budibase/backend-core" import { features } from "@budibase/pro" import env from "../../environment" -export const definition: AutomationStepDefinition = { - name: "Trigger an automation", - tagline: "Triggers an automation synchronously", - icon: "Sync", - description: "Triggers an automation synchronously", - type: AutomationStepType.ACTION, - internal: true, - features: {}, - stepId: AutomationActionStepId.TRIGGER_AUTOMATION_RUN, - inputs: {}, - schema: { - inputs: { - properties: { - automation: { - type: AutomationIOType.OBJECT, - properties: { - automationId: { - type: AutomationIOType.STRING, - customType: AutomationCustomIOType.AUTOMATION, - }, - }, - customType: AutomationCustomIOType.AUTOMATION_FIELDS, - title: "automatioFields", - required: ["automationId"], - }, - timeout: { - type: AutomationIOType.NUMBER, - title: "Timeout (ms)", - }, - }, - required: ["automationId"], - }, - outputs: { - properties: { - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the automation was successful", - }, - value: { - type: AutomationIOType.OBJECT, - description: "Automation Result", - }, - }, - required: ["success", "value"], - }, - }, -} - export async function run({ inputs, }: { diff --git a/packages/server/src/automations/steps/updateRow.ts b/packages/server/src/automations/steps/updateRow.ts index 552d846f18..46ae2a5c74 100644 --- a/packages/server/src/automations/steps/updateRow.ts +++ b/packages/server/src/automations/steps/updateRow.ts @@ -2,76 +2,8 @@ import { EventEmitter } from "events" import * as rowController from "../../api/controllers/row" import * as automationUtils from "../automationUtils" import { buildCtx } from "./utils" -import { - AutomationActionStepId, - AutomationCustomIOType, - AutomationFeature, - AutomationIOType, - AutomationStepDefinition, - AutomationStepType, - UpdateRowStepInputs, - UpdateRowStepOutputs, -} from "@budibase/types" +import { UpdateRowStepInputs, UpdateRowStepOutputs } from "@budibase/types" -export const definition: AutomationStepDefinition = { - name: "Update Row", - tagline: "Update a {{inputs.enriched.table.name}} row", - icon: "Refresh", - description: "Update a row in your database", - type: AutomationStepType.ACTION, - internal: true, - features: { - [AutomationFeature.LOOPING]: true, - }, - stepId: AutomationActionStepId.UPDATE_ROW, - inputs: {}, - schema: { - inputs: { - properties: { - meta: { - type: AutomationIOType.OBJECT, - title: "Field settings", - }, - row: { - type: AutomationIOType.OBJECT, - customType: AutomationCustomIOType.ROW, - title: "Table", - }, - rowId: { - type: AutomationIOType.STRING, - title: "Row ID", - }, - }, - required: ["row", "rowId"], - }, - outputs: { - properties: { - row: { - type: AutomationIOType.OBJECT, - customType: AutomationCustomIOType.ROW, - description: "The updated row", - }, - response: { - type: AutomationIOType.OBJECT, - description: "The response from the table", - }, - success: { - type: AutomationIOType.BOOLEAN, - description: "Whether the action was successful", - }, - id: { - type: AutomationIOType.STRING, - description: "The identifier of the updated row", - }, - revision: { - type: AutomationIOType.STRING, - description: "The revision of the updated row", - }, - }, - required: ["success", "id", "revision"], - }, - }, -} export async function run({ inputs, appId, diff --git a/packages/server/src/automations/steps/zapier.ts b/packages/server/src/automations/steps/zapier.ts index 888dae1e68..d17323cf0d 100644 --- a/packages/server/src/automations/steps/zapier.ts +++ b/packages/server/src/automations/steps/zapier.ts @@ -1,55 +1,6 @@ import fetch from "node-fetch" import { getFetchResponse } from "./utils" -import { - AutomationActionStepId, - AutomationStepDefinition, - AutomationStepType, - AutomationIOType, - AutomationFeature, - ZapierStepInputs, - ZapierStepOutputs, -} from "@budibase/types" - -export const definition: AutomationStepDefinition = { - name: "Zapier Webhook", - stepId: AutomationActionStepId.zapier, - type: AutomationStepType.ACTION, - internal: false, - features: { - [AutomationFeature.LOOPING]: true, - }, - description: "Trigger a Zapier Zap via webhooks", - tagline: "Trigger a Zapier Zap", - icon: "ri-flashlight-line", - inputs: {}, - schema: { - inputs: { - properties: { - url: { - type: AutomationIOType.STRING, - title: "Webhook URL", - }, - body: { - type: AutomationIOType.JSON, - title: "Payload", - }, - }, - required: ["url"], - }, - outputs: { - properties: { - httpStatus: { - type: AutomationIOType.NUMBER, - description: "The HTTP status code of the request", - }, - response: { - type: AutomationIOType.STRING, - description: "The response from Zapier", - }, - }, - }, - }, -} +import { ZapierStepInputs, ZapierStepOutputs } from "@budibase/types" export async function run({ inputs, diff --git a/packages/server/src/automations/tests/filter.spec.ts b/packages/server/src/automations/tests/filter.spec.ts index a1ed72dd74..674516517a 100644 --- a/packages/server/src/automations/tests/filter.spec.ts +++ b/packages/server/src/automations/tests/filter.spec.ts @@ -1,5 +1,7 @@ import * as setup from "./utilities" -import { FilterConditions } from "../steps/filter" +import { automations } from "@budibase/shared-core" + +const FilterConditions = automations.steps.filter.FilterConditions describe("test the filter logic", () => { const config = setup.getConfig() diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts index 45b251f4c1..3e203b7959 100644 --- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -6,9 +6,11 @@ import { DatabaseName, datasourceDescribe, } from "../../../integrations/tests/utils" -import { FilterConditions } from "../../../automations/steps/filter" import { Knex } from "knex" import { generator } from "@budibase/backend-core/tests" +import { automations } from "@budibase/shared-core" + +const FilterConditions = automations.steps.filter.FilterConditions describe("Automation Scenarios", () => { let config = setup.getConfig() diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index acc35a0eeb..830d2ee5ca 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -42,7 +42,7 @@ import { } from "@budibase/types" import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as setup from "../utilities" -import { definition } from "../../../automations/steps/branch" +import { automations } from "@budibase/shared-core" type TriggerOutputs = | RowCreatedTriggerOutputs @@ -103,7 +103,7 @@ class BaseStepBuilder { branchStepInputs.children![branchId] = stepBuilder.build() }) const branchStep: AutomationStep = { - ...definition, + ...automations.steps.branch.definition, id: uuidv4(), stepId: AutomationActionStepId.BRANCH, inputs: branchStepInputs, diff --git a/packages/shared-core/src/automations/index.ts b/packages/shared-core/src/automations/index.ts new file mode 100644 index 0000000000..064796845e --- /dev/null +++ b/packages/shared-core/src/automations/index.ts @@ -0,0 +1 @@ +export * as steps from "./steps/index" diff --git a/packages/shared-core/src/automations/steps/bash.ts b/packages/shared-core/src/automations/steps/bash.ts new file mode 100644 index 0000000000..2980e18f0e --- /dev/null +++ b/packages/shared-core/src/automations/steps/bash.ts @@ -0,0 +1,47 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Bash Scripting", + tagline: "Execute a bash command", + icon: "JourneyEvent", + description: "Run a bash script", + type: AutomationStepType.ACTION, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + stepId: AutomationActionStepId.EXECUTE_BASH, + inputs: {}, + schema: { + inputs: { + properties: { + code: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.CODE, + title: "Code", + }, + }, + required: ["code"], + }, + outputs: { + properties: { + stdout: { + type: AutomationIOType.STRING, + description: "Standard output of your bash command or script", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the command was successful", + }, + }, + required: ["stdout"], + }, + }, +} diff --git a/packages/server/src/automations/steps/branch.ts b/packages/shared-core/src/automations/steps/branch.ts similarity index 100% rename from packages/server/src/automations/steps/branch.ts rename to packages/shared-core/src/automations/steps/branch.ts diff --git a/packages/shared-core/src/automations/steps/collect.ts b/packages/shared-core/src/automations/steps/collect.ts new file mode 100644 index 0000000000..d4a6f6ec98 --- /dev/null +++ b/packages/shared-core/src/automations/steps/collect.ts @@ -0,0 +1,43 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Collect Data", + tagline: "Collect data to be sent to design", + icon: "Collection", + description: + "Collects specified data so it can be provided to the design section", + type: AutomationStepType.ACTION, + internal: true, + features: {}, + stepId: AutomationActionStepId.COLLECT, + inputs: {}, + schema: { + inputs: { + properties: { + collection: { + type: AutomationIOType.STRING, + title: "What to Collect", + }, + }, + required: ["collection"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + value: { + type: AutomationIOType.STRING, + description: "Collected data", + }, + }, + required: ["success", "value"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/createRow.ts b/packages/shared-core/src/automations/steps/createRow.ts new file mode 100644 index 0000000000..f51fdabb67 --- /dev/null +++ b/packages/shared-core/src/automations/steps/createRow.ts @@ -0,0 +1,67 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Create Row", + tagline: "Create a {{inputs.enriched.table.name}} row", + icon: "TableRowAddBottom", + description: "Add a row to your database", + type: AutomationStepType.ACTION, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + stepId: AutomationActionStepId.CREATE_ROW, + inputs: {}, + schema: { + inputs: { + properties: { + row: { + type: AutomationIOType.OBJECT, + properties: { + tableId: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.TABLE, + }, + }, + customType: AutomationCustomIOType.ROW, + title: "Table", + required: ["tableId"], + }, + }, + required: ["row"], + }, + outputs: { + properties: { + row: { + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, + description: "The new row", + }, + response: { + type: AutomationIOType.OBJECT, + description: "The response from the table", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the row creation was successful", + }, + id: { + type: AutomationIOType.STRING, + description: "The identifier of the new row", + }, + revision: { + type: AutomationIOType.STRING, + description: "The revision of the new row", + }, + }, + required: ["success", "id", "revision"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/delay.ts b/packages/shared-core/src/automations/steps/delay.ts new file mode 100644 index 0000000000..627bc9ed2f --- /dev/null +++ b/packages/shared-core/src/automations/steps/delay.ts @@ -0,0 +1,38 @@ +import { + AutomationActionStepId, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Delay", + icon: "Clock", + tagline: "Delay for {{inputs.time}} milliseconds", + description: "Delay the automation until an amount of time has passed", + stepId: AutomationActionStepId.DELAY, + internal: true, + features: {}, + inputs: {}, + schema: { + inputs: { + properties: { + time: { + type: AutomationIOType.NUMBER, + title: "Delay in milliseconds", + }, + }, + required: ["time"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the delay was successful", + }, + }, + required: ["success"], + }, + }, + type: AutomationStepType.LOGIC, +} diff --git a/packages/shared-core/src/automations/steps/deleteRow.ts b/packages/shared-core/src/automations/steps/deleteRow.ts new file mode 100644 index 0000000000..5a9de74392 --- /dev/null +++ b/packages/shared-core/src/automations/steps/deleteRow.ts @@ -0,0 +1,56 @@ +import { + AutomationActionStepId, + AutomationStepType, + AutomationIOType, + AutomationCustomIOType, + AutomationFeature, + AutomationStepDefinition, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + description: "Delete a row from your database", + icon: "TableRowRemoveCenter", + name: "Delete Row", + tagline: "Delete a {{inputs.enriched.table.name}} row", + type: AutomationStepType.ACTION, + stepId: AutomationActionStepId.DELETE_ROW, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + tableId: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.TABLE, + title: "Table", + }, + id: { + type: AutomationIOType.STRING, + title: "Row ID", + }, + }, + required: ["tableId", "id"], + }, + outputs: { + properties: { + row: { + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, + description: "The deleted row", + }, + response: { + type: AutomationIOType.OBJECT, + description: "The response from the table", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the deletion was successful", + }, + }, + required: ["row", "success"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/discord.ts b/packages/shared-core/src/automations/steps/discord.ts new file mode 100644 index 0000000000..906b970e1d --- /dev/null +++ b/packages/shared-core/src/automations/steps/discord.ts @@ -0,0 +1,60 @@ +import { + AutomationActionStepId, + AutomationStepType, + AutomationIOType, + AutomationFeature, + AutomationStepDefinition, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Discord Message", + tagline: "Send a message to a Discord server", + description: "Send a message to a Discord server", + icon: "ri-discord-line", + stepId: AutomationActionStepId.discord, + type: AutomationStepType.ACTION, + internal: false, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + url: { + type: AutomationIOType.STRING, + title: "Discord Webhook URL", + }, + username: { + type: AutomationIOType.STRING, + title: "Bot Name", + }, + avatar_url: { + type: AutomationIOType.STRING, + title: "Bot Avatar URL", + }, + content: { + type: AutomationIOType.STRING, + title: "Message", + }, + }, + required: ["url", "content"], + }, + outputs: { + properties: { + httpStatus: { + type: AutomationIOType.NUMBER, + description: "The HTTP status code of the request", + }, + response: { + type: AutomationIOType.STRING, + description: "The response from the Discord Webhook", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the message sent successfully", + }, + }, + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/executeQuery.ts b/packages/shared-core/src/automations/steps/executeQuery.ts new file mode 100644 index 0000000000..bbb8b93d6b --- /dev/null +++ b/packages/shared-core/src/automations/steps/executeQuery.ts @@ -0,0 +1,59 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "External Data Connector", + tagline: "Execute Data Connector", + icon: "Data", + description: "Execute a query in an external data connector", + type: AutomationStepType.ACTION, + stepId: AutomationActionStepId.EXECUTE_QUERY, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + query: { + type: AutomationIOType.OBJECT, + properties: { + queryId: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.QUERY, + }, + }, + customType: AutomationCustomIOType.QUERY_PARAMS, + title: "Parameters", + required: ["queryId"], + }, + }, + required: ["query"], + }, + outputs: { + properties: { + response: { + type: AutomationIOType.OBJECT, + description: "The response from the datasource execution", + }, + info: { + type: AutomationIOType.OBJECT, + description: + "Some query types may return extra data, like headers from a REST query", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + }, + required: ["response", "success"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/executeScript.ts b/packages/shared-core/src/automations/steps/executeScript.ts new file mode 100644 index 0000000000..dcc52f2905 --- /dev/null +++ b/packages/shared-core/src/automations/steps/executeScript.ts @@ -0,0 +1,47 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "JS Scripting", + tagline: "Execute JavaScript Code", + icon: "Code", + description: "Run a piece of JavaScript code in your automation", + type: AutomationStepType.ACTION, + internal: true, + stepId: AutomationActionStepId.EXECUTE_SCRIPT, + inputs: {}, + features: { + [AutomationFeature.LOOPING]: true, + }, + schema: { + inputs: { + properties: { + code: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.CODE, + title: "Code", + }, + }, + required: ["code"], + }, + outputs: { + properties: { + value: { + type: AutomationIOType.STRING, + description: "The result of the return statement", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + }, + required: ["success"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/filter.ts b/packages/shared-core/src/automations/steps/filter.ts new file mode 100644 index 0000000000..70dcb6f66e --- /dev/null +++ b/packages/shared-core/src/automations/steps/filter.ts @@ -0,0 +1,69 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, +} from "@budibase/types" + +export const FilterConditions = { + EQUAL: "EQUAL", + NOT_EQUAL: "NOT_EQUAL", + GREATER_THAN: "GREATER_THAN", + LESS_THAN: "LESS_THAN", +} + +export const PrettyFilterConditions = { + [FilterConditions.EQUAL]: "Equals", + [FilterConditions.NOT_EQUAL]: "Not equals", + [FilterConditions.GREATER_THAN]: "Greater than", + [FilterConditions.LESS_THAN]: "Less than", +} + +export const definition: AutomationStepDefinition = { + name: "Condition", + tagline: "{{inputs.field}} {{inputs.condition}} {{inputs.value}}", + icon: "Branch2", + description: + "Conditionally halt automations which do not meet certain conditions", + type: AutomationStepType.LOGIC, + internal: true, + features: {}, + stepId: AutomationActionStepId.FILTER, + inputs: { + condition: FilterConditions.EQUAL, + }, + schema: { + inputs: { + properties: { + field: { + type: AutomationIOType.STRING, + title: "Reference Value", + }, + condition: { + type: AutomationIOType.STRING, + title: "Condition", + enum: Object.values(FilterConditions), + pretty: Object.values(PrettyFilterConditions), + }, + value: { + type: AutomationIOType.STRING, + title: "Comparison Value", + }, + }, + required: ["field", "condition", "value"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + result: { + type: AutomationIOType.BOOLEAN, + description: "Whether the logic block passed", + }, + }, + required: ["success", "result"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/index.ts b/packages/shared-core/src/automations/steps/index.ts new file mode 100644 index 0000000000..d23c5622b8 --- /dev/null +++ b/packages/shared-core/src/automations/steps/index.ts @@ -0,0 +1,22 @@ +export * as bash from "./bash" +export * as branch from "./branch" +export * as collect from "./collect" +export * as createRow from "./createRow" +export * as delay from "./delay" +export * as deleteRow from "./deleteRow" +export * as discord from "./discord" +export * as executeQuery from "./executeQuery" +export * as executeScript from "./executeScript" +export * as filter from "./filter" +export * as loop from "./loop" +export * as make from "./make" +export * as n8n from "./n8n" +export * as openai from "./openai" +export * as outgoingWebhook from "./outgoingWebhook" +export * as queryRows from "./queryRows" +export * as sendSmtpEmail from "./sendSmtpEmail" +export * as serverLog from "./serverLog" +export * as slack from "./slack" +export * as triggerAutomationRun from "./triggerAutomationRun" +export * as updateRow from "./updateRow" +export * as zapier from "./zapier" diff --git a/packages/server/src/automations/steps/loop.ts b/packages/shared-core/src/automations/steps/loop.ts similarity index 100% rename from packages/server/src/automations/steps/loop.ts rename to packages/shared-core/src/automations/steps/loop.ts diff --git a/packages/shared-core/src/automations/steps/make.ts b/packages/shared-core/src/automations/steps/make.ts new file mode 100644 index 0000000000..4f9e6c40ad --- /dev/null +++ b/packages/shared-core/src/automations/steps/make.ts @@ -0,0 +1,55 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationFeature, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Make Integration", + stepTitle: "Make", + tagline: "Trigger a Make scenario", + description: + "Performs a webhook call to Make and gets the response (if configured)", + icon: "ri-shut-down-line", + stepId: AutomationActionStepId.integromat, + type: AutomationStepType.ACTION, + internal: false, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + url: { + type: AutomationIOType.STRING, + title: "Webhook URL", + }, + body: { + type: AutomationIOType.JSON, + title: "Payload", + }, + }, + required: ["url", "body"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether call was successful", + }, + httpStatus: { + type: AutomationIOType.NUMBER, + description: "The HTTP status code returned", + }, + response: { + type: AutomationIOType.OBJECT, + description: "The webhook response - this can have properties", + }, + }, + required: ["success", "response"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/n8n.ts b/packages/shared-core/src/automations/steps/n8n.ts new file mode 100644 index 0000000000..fcb407bcbf --- /dev/null +++ b/packages/shared-core/src/automations/steps/n8n.ts @@ -0,0 +1,65 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationFeature, + HttpMethod, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "n8n Integration", + stepTitle: "n8n", + tagline: "Trigger an n8n workflow", + description: + "Performs a webhook call to n8n and gets the response (if configured)", + icon: "ri-shut-down-line", + stepId: AutomationActionStepId.n8n, + type: AutomationStepType.ACTION, + internal: false, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + url: { + type: AutomationIOType.STRING, + title: "Webhook URL", + }, + method: { + type: AutomationIOType.STRING, + title: "Method", + enum: Object.values(HttpMethod), + }, + authorization: { + type: AutomationIOType.STRING, + title: "Authorization", + }, + body: { + type: AutomationIOType.JSON, + title: "Payload", + }, + }, + required: ["url", "method"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether call was successful", + }, + httpStatus: { + type: AutomationIOType.NUMBER, + description: "The HTTP status code returned", + }, + response: { + type: AutomationIOType.OBJECT, + description: "The webhook response - this can have properties", + }, + }, + required: ["success", "response"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/openai.ts b/packages/shared-core/src/automations/steps/openai.ts new file mode 100644 index 0000000000..dc25e62f96 --- /dev/null +++ b/packages/shared-core/src/automations/steps/openai.ts @@ -0,0 +1,56 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, +} from "@budibase/types" + +enum Model { + GPT_4O_MINI = "gpt-4o-mini", + GPT_4O = "gpt-4o", + GPT_4 = "gpt-4", + GPT_35_TURBO = "gpt-3.5-turbo", +} + +export const definition: AutomationStepDefinition = { + name: "OpenAI", + tagline: "Send prompts to ChatGPT", + icon: "Algorithm", + description: "Interact with the OpenAI ChatGPT API.", + type: AutomationStepType.ACTION, + internal: true, + features: {}, + stepId: AutomationActionStepId.OPENAI, + inputs: { + prompt: "", + }, + schema: { + inputs: { + properties: { + prompt: { + type: AutomationIOType.STRING, + title: "Prompt", + }, + model: { + type: AutomationIOType.STRING, + title: "Model", + enum: Object.values(Model), + }, + }, + required: ["prompt", "model"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + response: { + type: AutomationIOType.STRING, + description: "What was output", + }, + }, + required: ["success", "response"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/outgoingWebhook.ts b/packages/shared-core/src/automations/steps/outgoingWebhook.ts new file mode 100644 index 0000000000..9e0b1b375b --- /dev/null +++ b/packages/shared-core/src/automations/steps/outgoingWebhook.ts @@ -0,0 +1,79 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +enum RequestType { + POST = "POST", + GET = "GET", + PUT = "PUT", + DELETE = "DELETE", + PATCH = "PATCH", +} + +export const definition: AutomationStepDefinition = { + deprecated: true, + name: "Outgoing webhook", + tagline: "Send a {{inputs.requestMethod}} request", + icon: "Send", + description: "Send a request of specified method to a URL", + type: AutomationStepType.ACTION, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + stepId: AutomationActionStepId.OUTGOING_WEBHOOK, + inputs: { + requestMethod: "POST", + url: "http://", + requestBody: "{}", + headers: "{}", + }, + schema: { + inputs: { + properties: { + requestMethod: { + type: AutomationIOType.STRING, + enum: Object.values(RequestType), + title: "Request method", + }, + url: { + type: AutomationIOType.STRING, + title: "URL", + }, + requestBody: { + type: AutomationIOType.STRING, + title: "JSON Body", + customType: AutomationCustomIOType.WIDE, + }, + headers: { + type: AutomationIOType.STRING, + title: "Headers", + customType: AutomationCustomIOType.WIDE, + }, + }, + required: ["requestMethod", "url"], + }, + outputs: { + properties: { + response: { + type: AutomationIOType.OBJECT, + description: "The response from the webhook", + }, + httpStatus: { + type: AutomationIOType.NUMBER, + description: "The HTTP status code returned", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + }, + required: ["response", "success"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/queryRows.ts b/packages/shared-core/src/automations/steps/queryRows.ts new file mode 100644 index 0000000000..31fec1e112 --- /dev/null +++ b/packages/shared-core/src/automations/steps/queryRows.ts @@ -0,0 +1,75 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, + SortOrder, +} from "@budibase/types" + +const SortOrderPretty = { + [SortOrder.ASCENDING]: "Ascending", + [SortOrder.DESCENDING]: "Descending", +} + +export const definition: AutomationStepDefinition = { + description: "Query rows from the database", + icon: "Search", + name: "Query rows", + tagline: "Query rows from {{inputs.enriched.table.name}} table", + type: AutomationStepType.ACTION, + stepId: AutomationActionStepId.QUERY_ROWS, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + tableId: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.TABLE, + title: "Table", + }, + filters: { + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.FILTERS, + title: "Filtering", + }, + sortColumn: { + type: AutomationIOType.STRING, + title: "Sort Column", + customType: AutomationCustomIOType.COLUMN, + }, + sortOrder: { + type: AutomationIOType.STRING, + title: "Sort Order", + enum: Object.values(SortOrder), + pretty: Object.values(SortOrderPretty), + }, + limit: { + type: AutomationIOType.NUMBER, + title: "Limit", + customType: AutomationCustomIOType.QUERY_LIMIT, + }, + }, + required: ["tableId"], + }, + outputs: { + properties: { + rows: { + type: AutomationIOType.ARRAY, + customType: AutomationCustomIOType.ROWS, + description: "The rows that were found", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the query was successful", + }, + }, + required: ["rows", "success"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/sendSmtpEmail.ts b/packages/shared-core/src/automations/steps/sendSmtpEmail.ts new file mode 100644 index 0000000000..99ded7fdb3 --- /dev/null +++ b/packages/shared-core/src/automations/steps/sendSmtpEmail.ts @@ -0,0 +1,95 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationFeature, + AutomationCustomIOType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + description: "Send an email using SMTP", + tagline: "Send SMTP email to {{inputs.to}}", + icon: "Email", + name: "Send Email (SMTP)", + type: AutomationStepType.ACTION, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + stepId: AutomationActionStepId.SEND_EMAIL_SMTP, + inputs: {}, + schema: { + inputs: { + properties: { + to: { + type: AutomationIOType.STRING, + title: "Send To", + }, + from: { + type: AutomationIOType.STRING, + title: "Send From", + }, + cc: { + type: AutomationIOType.STRING, + title: "CC", + }, + bcc: { + type: AutomationIOType.STRING, + title: "BCC", + }, + subject: { + type: AutomationIOType.STRING, + title: "Email Subject", + }, + contents: { + type: AutomationIOType.STRING, + title: "HTML Contents", + }, + addInvite: { + type: AutomationIOType.BOOLEAN, + title: "Add calendar invite", + }, + startTime: { + type: AutomationIOType.DATE, + title: "Start Time", + dependsOn: "addInvite", + }, + endTime: { + type: AutomationIOType.DATE, + title: "End Time", + dependsOn: "addInvite", + }, + summary: { + type: AutomationIOType.STRING, + title: "Meeting Summary", + dependsOn: "addInvite", + }, + location: { + type: AutomationIOType.STRING, + title: "Location", + dependsOn: "addInvite", + }, + attachments: { + type: AutomationIOType.ATTACHMENT, + customType: AutomationCustomIOType.MULTI_ATTACHMENTS, + title: "Attachments", + }, + }, + required: ["to", "from", "subject", "contents"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the email was sent", + }, + response: { + type: AutomationIOType.OBJECT, + description: "A response from the email client, this may be an error", + }, + }, + required: ["success"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/serverLog.ts b/packages/shared-core/src/automations/steps/serverLog.ts new file mode 100644 index 0000000000..44ffb84049 --- /dev/null +++ b/packages/shared-core/src/automations/steps/serverLog.ts @@ -0,0 +1,47 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationFeature, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Backend log", + tagline: "Console log a value in the backend", + icon: "Monitoring", + description: "Logs the given text to the server (using console.log)", + type: AutomationStepType.ACTION, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + stepId: AutomationActionStepId.SERVER_LOG, + inputs: { + text: "", + }, + schema: { + inputs: { + properties: { + text: { + type: AutomationIOType.STRING, + title: "Log", + }, + }, + required: ["text"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + message: { + type: AutomationIOType.STRING, + description: "What was output", + }, + }, + required: ["success", "message"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/slack.ts b/packages/shared-core/src/automations/steps/slack.ts new file mode 100644 index 0000000000..185e00a8ed --- /dev/null +++ b/packages/shared-core/src/automations/steps/slack.ts @@ -0,0 +1,52 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationFeature, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Slack Message", + tagline: "Send a message to Slack", + description: "Send a message to Slack", + icon: "ri-slack-line", + stepId: AutomationActionStepId.slack, + type: AutomationStepType.ACTION, + internal: false, + features: { + [AutomationFeature.LOOPING]: true, + }, + inputs: {}, + schema: { + inputs: { + properties: { + url: { + type: AutomationIOType.STRING, + title: "Incoming Webhook URL", + }, + text: { + type: AutomationIOType.STRING, + title: "Message", + }, + }, + required: ["url", "text"], + }, + outputs: { + properties: { + httpStatus: { + type: AutomationIOType.NUMBER, + description: "The HTTP status code of the request", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the message sent successfully", + }, + response: { + type: AutomationIOType.STRING, + description: "The response from the Slack Webhook", + }, + }, + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/triggerAutomationRun.ts b/packages/shared-core/src/automations/steps/triggerAutomationRun.ts new file mode 100644 index 0000000000..3aa78b6015 --- /dev/null +++ b/packages/shared-core/src/automations/steps/triggerAutomationRun.ts @@ -0,0 +1,55 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationCustomIOType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Trigger an automation", + tagline: "Triggers an automation synchronously", + icon: "Sync", + description: "Triggers an automation synchronously", + type: AutomationStepType.ACTION, + internal: true, + features: {}, + stepId: AutomationActionStepId.TRIGGER_AUTOMATION_RUN, + inputs: {}, + schema: { + inputs: { + properties: { + automation: { + type: AutomationIOType.OBJECT, + properties: { + automationId: { + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.AUTOMATION, + }, + }, + customType: AutomationCustomIOType.AUTOMATION_FIELDS, + title: "automatioFields", + required: ["automationId"], + }, + timeout: { + type: AutomationIOType.NUMBER, + title: "Timeout (ms)", + }, + }, + required: ["automationId"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the automation was successful", + }, + value: { + type: AutomationIOType.OBJECT, + description: "Automation Result", + }, + }, + required: ["success", "value"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/updateRow.ts b/packages/shared-core/src/automations/steps/updateRow.ts new file mode 100644 index 0000000000..30f86214bb --- /dev/null +++ b/packages/shared-core/src/automations/steps/updateRow.ts @@ -0,0 +1,68 @@ +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationFeature, + AutomationIOType, + AutomationStepDefinition, + AutomationStepType, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Update Row", + tagline: "Update a {{inputs.enriched.table.name}} row", + icon: "Refresh", + description: "Update a row in your database", + type: AutomationStepType.ACTION, + internal: true, + features: { + [AutomationFeature.LOOPING]: true, + }, + stepId: AutomationActionStepId.UPDATE_ROW, + inputs: {}, + schema: { + inputs: { + properties: { + meta: { + type: AutomationIOType.OBJECT, + title: "Field settings", + }, + row: { + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, + title: "Table", + }, + rowId: { + type: AutomationIOType.STRING, + title: "Row ID", + }, + }, + required: ["row", "rowId"], + }, + outputs: { + properties: { + row: { + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, + description: "The updated row", + }, + response: { + type: AutomationIOType.OBJECT, + description: "The response from the table", + }, + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + id: { + type: AutomationIOType.STRING, + description: "The identifier of the updated row", + }, + revision: { + type: AutomationIOType.STRING, + description: "The revision of the updated row", + }, + }, + required: ["success", "id", "revision"], + }, + }, +} diff --git a/packages/shared-core/src/automations/steps/zapier.ts b/packages/shared-core/src/automations/steps/zapier.ts new file mode 100644 index 0000000000..b1afd31def --- /dev/null +++ b/packages/shared-core/src/automations/steps/zapier.ts @@ -0,0 +1,48 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationStepType, + AutomationIOType, + AutomationFeature, +} from "@budibase/types" + +export const definition: AutomationStepDefinition = { + name: "Zapier Webhook", + stepId: AutomationActionStepId.zapier, + type: AutomationStepType.ACTION, + internal: false, + features: { + [AutomationFeature.LOOPING]: true, + }, + description: "Trigger a Zapier Zap via webhooks", + tagline: "Trigger a Zapier Zap", + icon: "ri-flashlight-line", + inputs: {}, + schema: { + inputs: { + properties: { + url: { + type: AutomationIOType.STRING, + title: "Webhook URL", + }, + body: { + type: AutomationIOType.JSON, + title: "Payload", + }, + }, + required: ["url"], + }, + outputs: { + properties: { + httpStatus: { + type: AutomationIOType.NUMBER, + description: "The HTTP status code of the request", + }, + response: { + type: AutomationIOType.STRING, + description: "The response from Zapier", + }, + }, + }, + }, +} diff --git a/packages/shared-core/src/index.ts b/packages/shared-core/src/index.ts index 0833f999ee..a4f4208f4a 100644 --- a/packages/shared-core/src/index.ts +++ b/packages/shared-core/src/index.ts @@ -5,3 +5,4 @@ export * as utils from "./utils" export * as sdk from "./sdk" export * from "./table" export * from "./themes" +export * as automations from "./automations" From 8cf7375202a3bebb97baee2ac33a75950cdc13e3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 17 Jan 2025 10:33:01 +0100 Subject: [PATCH 12/22] Type schema object --- packages/client/src/utils/schema.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/client/src/utils/schema.ts b/packages/client/src/utils/schema.ts index 5400d62087..9ea868751d 100644 --- a/packages/client/src/utils/schema.ts +++ b/packages/client/src/utils/schema.ts @@ -1,5 +1,6 @@ import { API } from "api" import { DataFetchMap, DataFetchType } from "@budibase/frontend-core" +import { FieldType, TableSchema } from "@budibase/types" /** * Constructs a fetch instance for a given datasource. @@ -42,14 +43,14 @@ export const fetchDatasourceSchema = async < } // Get the normal schema as long as we aren't wanting a form schema - let schema: any + let schema: TableSchema | undefined if (datasource?.type !== "query" || !options?.formSchema) { - schema = instance.getSchema(definition as any) + schema = instance.getSchema(definition as any) as TableSchema } else if ("parameters" in definition && definition.parameters?.length) { schema = {} - definition.parameters.forEach(param => { - schema[param.name] = { ...param, type: "string" } - }) + for (const param of definition.parameters) { + schema[param.name] = { ...param, type: FieldType.STRING } + } } if (!schema) { return null @@ -57,11 +58,11 @@ export const fetchDatasourceSchema = async < // Strip hidden fields from views if (datasource.type === "viewV2") { - Object.keys(schema).forEach(field => { + for (const field of Object.keys(schema)) { if (!schema[field].visible) { delete schema[field] } - }) + } } // Enrich schema with relationships if required From 84dbcb0f6914979285356c7576cf1162974ef410 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 17 Jan 2025 11:51:48 +0100 Subject: [PATCH 13/22] Change errorState to boolean --- packages/client/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 97c0525f55..80ac725df3 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -17,7 +17,7 @@ export interface SDK { export type Component = Writable<{ id: string styles: any - errorState: string + errorState: boolean }> export type Context = Readable From ed35acc01affb459b38a0ea3a0f7a454ecacf5c8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 17 Jan 2025 11:57:38 +0100 Subject: [PATCH 14/22] Revert Component store to readable --- packages/client/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 80ac725df3..ada30d53ec 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -1,6 +1,6 @@ import { APIClient } from "@budibase/frontend-core" import type { ActionTypes } from "./constants" -import { Readable, Writable } from "svelte/store" +import { Readable } from "svelte/store" export interface SDK { API: APIClient @@ -14,7 +14,7 @@ export interface SDK { }> } -export type Component = Writable<{ +export type Component = Readable<{ id: string styles: any errorState: boolean From 14bc491d2c83583c00b62e83cccc62c42112c3be Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 17 Jan 2025 12:44:39 +0100 Subject: [PATCH 15/22] More typings --- packages/bbui/src/Icon/Icon.svelte | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/bbui/src/Icon/Icon.svelte b/packages/bbui/src/Icon/Icon.svelte index 1a851248b9..35f176035e 100644 --- a/packages/bbui/src/Icon/Icon.svelte +++ b/packages/bbui/src/Icon/Icon.svelte @@ -5,19 +5,19 @@ TooltipType, } from "../Tooltip/AbsTooltip.svelte" - export let name = "Add" - export let hidden = false + export let name: string = "Add" + export let hidden: boolean = false export let size = "M" - export let hoverable = false - export let disabled = false + export let hoverable: boolean = false + export let disabled: boolean = false export let color: string = undefined - export let hoverColor = undefined - export let tooltip = undefined + export let hoverColor: string = undefined + export let tooltip: string = undefined export let tooltipPosition = TooltipPosition.Bottom export let tooltipType = TooltipType.Default - export let tooltipColor = undefined - export let tooltipWrap = true - export let newStyles = false + export let tooltipColor: string = undefined + export let tooltipWrap: boolean = true + export let newStyles: boolean = false Date: Fri, 17 Jan 2025 21:17:39 +0100 Subject: [PATCH 16/22] Fix types after merge --- packages/bbui/src/Icon/Icon.svelte | 8 ++++---- packages/bbui/src/Tooltip/AbsTooltip.svelte | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/bbui/src/Icon/Icon.svelte b/packages/bbui/src/Icon/Icon.svelte index 35f176035e..7438fab5fd 100644 --- a/packages/bbui/src/Icon/Icon.svelte +++ b/packages/bbui/src/Icon/Icon.svelte @@ -10,12 +10,12 @@ export let size = "M" export let hoverable: boolean = false export let disabled: boolean = false - export let color: string = undefined - export let hoverColor: string = undefined - export let tooltip: string = undefined + export let color: string | undefined = undefined + export let hoverColor: string | undefined = undefined + export let tooltip: string | undefined = undefined export let tooltipPosition = TooltipPosition.Bottom export let tooltipType = TooltipType.Default - export let tooltipColor: string = undefined + export let tooltipColor: string | undefined = undefined export let tooltipWrap: boolean = true export let newStyles: boolean = false diff --git a/packages/bbui/src/Tooltip/AbsTooltip.svelte b/packages/bbui/src/Tooltip/AbsTooltip.svelte index b85f4e1c03..a887db4102 100644 --- a/packages/bbui/src/Tooltip/AbsTooltip.svelte +++ b/packages/bbui/src/Tooltip/AbsTooltip.svelte @@ -23,7 +23,7 @@ export let type = TooltipType.Default export let text = "" export let fixed = false - export let color = null + export let color = "" export let noWrap = false let wrapper From fa930de15e8653a8adb5337f27b2297c711531ea Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 09:45:22 +0100 Subject: [PATCH 17/22] Type fields --- .../components/app/blocks/form/FormBlock.svelte | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index 22f21b5790..ab3c498ca3 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -7,11 +7,13 @@ import { Component, Context, SDK } from "../../../../index" import { TableSchema, UIDatasource } from "@budibase/types" + type Field = { name: string; active: boolean } + export let actionType: string export let dataSource: UIDatasource export let size: string export let disabled: boolean - export let fields + export let fields: (Field | string)[] export let buttons: { "##eventHandlerType": string parameters: Record @@ -66,11 +68,11 @@ } } - const convertOldFieldFormat = (fields: any[]) => { + const convertOldFieldFormat = (fields: (Field | string)[]): Field[] => { if (!fields) { return [] } - return fields.map((field: any) => { + return fields.map(field => { if (typeof field === "string") { // existed but was a string return { @@ -87,14 +89,11 @@ }) } - const getDefaultFields = ( - fields: { name: string; active: boolean }[], - schema: TableSchema - ) => { + const getDefaultFields = (fields: Field[], schema: TableSchema) => { if (!schema) { return [] } - let defaultFields: { name: string; active: boolean }[] = [] + let defaultFields: Field[] = [] if (!fields || fields.length === 0) { Object.values(schema) From 0b73f53974c301667d4f2972d4b3392629e2d64b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 14:51:37 +0100 Subject: [PATCH 18/22] Automatically type getContext response --- packages/client/src/components/app/DataProvider.svelte | 5 ++--- .../client/src/components/app/blocks/form/FormBlock.svelte | 7 +++---- .../src/components/error-states/ComponentErrorState.svelte | 5 ++--- packages/client/src/context.d.ts | 7 +++++++ 4 files changed, 14 insertions(+), 10 deletions(-) create mode 100644 packages/client/src/context.d.ts diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte index 154f69d4de..a80b9a5f74 100644 --- a/packages/client/src/components/app/DataProvider.svelte +++ b/packages/client/src/components/app/DataProvider.svelte @@ -14,7 +14,6 @@ GroupUserDatasource, DataFetchOptions, } from "@budibase/types" - import { SDK, Component } from "../../index" type ProviderDatasource = Exclude< DataFetchDatasource, @@ -29,8 +28,8 @@ export let paginate: boolean export let autoRefresh: number - const { styleable, Provider, ActionTypes, API } = getContext("sdk") - const component = getContext("component") + const { styleable, Provider, ActionTypes, API } = getContext("sdk") + const component = getContext("component") let interval: ReturnType let queryExtensions: Record = {} diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index ab3c498ca3..3f44aee1d7 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -4,7 +4,6 @@ import { Utils } from "@budibase/frontend-core" import FormBlockWrapper from "./FormBlockWrapper.svelte" import { get } from "svelte/store" - import { Component, Context, SDK } from "../../../../index" import { TableSchema, UIDatasource } from "@budibase/types" type Field = { name: string; active: boolean } @@ -34,9 +33,9 @@ export let saveButtonLabel: boolean export let deleteButtonLabel: boolean - const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk") - const component = getContext("component") - const context = getContext("context") + const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk") + const component = getContext("component") + const context = getContext("context") let schema: TableSchema diff --git a/packages/client/src/components/error-states/ComponentErrorState.svelte b/packages/client/src/components/error-states/ComponentErrorState.svelte index b846eaa230..d30f4916da 100644 --- a/packages/client/src/components/error-states/ComponentErrorState.svelte +++ b/packages/client/src/components/error-states/ComponentErrorState.svelte @@ -3,15 +3,14 @@ import { Icon } from "@budibase/bbui" import MissingRequiredSetting from "./MissingRequiredSetting.svelte" import MissingRequiredAncestor from "./MissingRequiredAncestor.svelte" - import { Component, SDK } from "../../index" export let missingRequiredSettings: | { key: string; label: string }[] | undefined export let missingRequiredAncestors: string[] | undefined - const component = getContext("component") - const { styleable, builderStore } = getContext("sdk") + const component = getContext("component") + const { styleable, builderStore } = getContext("sdk") $: styles = { ...$component.styles, normal: {}, custom: null, empty: true } $: requiredSetting = missingRequiredSettings?.[0] diff --git a/packages/client/src/context.d.ts b/packages/client/src/context.d.ts new file mode 100644 index 0000000000..1bc0e3f7ad --- /dev/null +++ b/packages/client/src/context.d.ts @@ -0,0 +1,7 @@ +import { Component, Context, SDK } from "." + +declare module "svelte" { + function getContext(name: "sdk"): SDK + function getContext(name: "component"): Component + function getContext(name: "context"): Context +} From cbabaaf8f7c4b52d7fc5d8e21fa07ce5867b1134 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 15:09:02 +0100 Subject: [PATCH 19/22] Move types --- packages/client/src/context.d.ts | 7 ++++--- packages/client/src/index.ts | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/client/src/context.d.ts b/packages/client/src/context.d.ts index 1bc0e3f7ad..6ac29df547 100644 --- a/packages/client/src/context.d.ts +++ b/packages/client/src/context.d.ts @@ -1,7 +1,8 @@ +import { Readable } from "svelte/store" import { Component, Context, SDK } from "." declare module "svelte" { - function getContext(name: "sdk"): SDK - function getContext(name: "component"): Component - function getContext(name: "context"): Context + export function getContext(key: "sdk"): SDK + export function getContext(key: "component"): Readable + export function getContext(key: "context"): Readable } diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index ada30d53ec..dabe66dab2 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -14,10 +14,10 @@ export interface SDK { }> } -export type Component = Readable<{ +export type Component = { id: string styles: any errorState: boolean -}> +} -export type Context = Readable +export type Context = {} From 199aa80e25433fe5ec103ef449ce29346138289e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 20 Jan 2025 15:10:03 +0100 Subject: [PATCH 20/22] Move types --- packages/client/src/context.d.ts | 5 ++--- packages/client/src/index.ts | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/client/src/context.d.ts b/packages/client/src/context.d.ts index 6ac29df547..994f9ef172 100644 --- a/packages/client/src/context.d.ts +++ b/packages/client/src/context.d.ts @@ -1,8 +1,7 @@ -import { Readable } from "svelte/store" import { Component, Context, SDK } from "." declare module "svelte" { export function getContext(key: "sdk"): SDK - export function getContext(key: "component"): Readable - export function getContext(key: "context"): Readable + export function getContext(key: "component"): Component + export function getContext(key: "context"): Context } diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index dabe66dab2..2a435c2f8c 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -14,10 +14,10 @@ export interface SDK { }> } -export type Component = { +export type Component = Readable<{ id: string styles: any errorState: boolean -} +}> -export type Context = {} +export type Context = Readable<{}> From 0b1ff21da5f52b5e0a38836ec83df8b58dba814e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 14:44:06 +0000 Subject: [PATCH 21/22] Fix tests. --- packages/server/src/automations/actions.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index f35f96babe..bdbd9d509d 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -88,12 +88,10 @@ export const BUILTIN_ACTION_DEFINITIONS: Record< // the fact this isn't included in any definitions means it cannot be // ran at all if (env.SELF_HOSTED) { - const bash = require("./steps/bash") - // @ts-ignore - ACTION_IMPLS["EXECUTE_BASH"] = bash.run + ACTION_IMPLS["EXECUTE_BASH"] = automations.steps.bash.run // @ts-ignore - BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition + BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = automations.steps.bash.definition if (env.isTest()) { BUILTIN_ACTION_DEFINITIONS["OPENAI"] = automations.steps.openai.definition From 75dac8f3343540eb8d0e6e843e8fed909be931a1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 20 Jan 2025 15:08:50 +0000 Subject: [PATCH 22/22] Fix tests (again). --- packages/server/src/automations/actions.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index bdbd9d509d..65a57c2586 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -18,6 +18,7 @@ import * as queryRow from "./steps/queryRows" import * as collect from "./steps/collect" import * as triggerAutomationRun from "./steps/triggerAutomationRun" import * as openai from "./steps/openai" +import * as bash from "./steps/bash" import env from "../environment" import { PluginType, @@ -88,9 +89,8 @@ export const BUILTIN_ACTION_DEFINITIONS: Record< // the fact this isn't included in any definitions means it cannot be // ran at all if (env.SELF_HOSTED) { - // @ts-ignore - ACTION_IMPLS["EXECUTE_BASH"] = automations.steps.bash.run - // @ts-ignore + // @ts-expect-error + ACTION_IMPLS["EXECUTE_BASH"] = bash.run BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = automations.steps.bash.definition if (env.isTest()) {