diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index dbdce51c50..d4e6e9a1ec 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -1,6 +1,5 @@
export * as configs from "./configs"
export * as events from "./events"
-export * as migrations from "./migrations"
export * as users from "./users"
export * as userUtils from "./users/utils"
export * as roles from "./security/roles"
diff --git a/packages/backend-core/src/migrations/definitions.ts b/packages/backend-core/src/migrations/definitions.ts
deleted file mode 100644
index 0dd57fe639..0000000000
--- a/packages/backend-core/src/migrations/definitions.ts
+++ /dev/null
@@ -1,40 +0,0 @@
-import {
- MigrationType,
- MigrationName,
- MigrationDefinition,
-} from "@budibase/types"
-
-export const DEFINITIONS: MigrationDefinition[] = [
- {
- type: MigrationType.GLOBAL,
- name: MigrationName.USER_EMAIL_VIEW_CASING,
- },
- {
- type: MigrationType.GLOBAL,
- name: MigrationName.SYNC_QUOTAS,
- },
- {
- type: MigrationType.APP,
- name: MigrationName.APP_URLS,
- },
- {
- type: MigrationType.APP,
- name: MigrationName.EVENT_APP_BACKFILL,
- },
- {
- type: MigrationType.APP,
- name: MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS,
- },
- {
- type: MigrationType.GLOBAL,
- name: MigrationName.EVENT_GLOBAL_BACKFILL,
- },
- {
- type: MigrationType.INSTALLATION,
- name: MigrationName.EVENT_INSTALLATION_BACKFILL,
- },
- {
- type: MigrationType.GLOBAL,
- name: MigrationName.GLOBAL_INFO_SYNC_USERS,
- },
-]
diff --git a/packages/backend-core/src/migrations/index.ts b/packages/backend-core/src/migrations/index.ts
deleted file mode 100644
index bce0cfc75c..0000000000
--- a/packages/backend-core/src/migrations/index.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-export * from "./migrations"
-export * from "./definitions"
diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts
deleted file mode 100644
index c8320b5724..0000000000
--- a/packages/backend-core/src/migrations/migrations.ts
+++ /dev/null
@@ -1,186 +0,0 @@
-import { DEFAULT_TENANT_ID } from "../constants"
-import {
- DocumentType,
- StaticDatabases,
- getAllApps,
- getGlobalDBName,
- getDB,
-} from "../db"
-import environment from "../environment"
-import * as platform from "../platform"
-import * as context from "../context"
-import { DEFINITIONS } from "."
-import {
- Migration,
- MigrationOptions,
- MigrationType,
- MigrationNoOpOptions,
- App,
-} from "@budibase/types"
-
-export const getMigrationsDoc = async (db: any) => {
- // get the migrations doc
- try {
- return await db.get(DocumentType.MIGRATIONS)
- } catch (err: any) {
- if (err.status && err.status === 404) {
- return { _id: DocumentType.MIGRATIONS }
- } else {
- throw err
- }
- }
-}
-
-export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => {
- // filter migrations to the type and populate a no-op migration
- const migrations: Migration[] = DEFINITIONS.filter(
- def => def.type === opts.type
- ).map(d => ({ ...d, fn: async () => {} }))
- await runMigrations(migrations, { noOp: opts })
-}
-
-export const runMigration = async (
- migration: Migration,
- options: MigrationOptions = {}
-) => {
- const migrationType = migration.type
- const migrationName = migration.name
- const silent = migration.silent
-
- const log = (message: string) => {
- if (!silent) {
- console.log(message)
- }
- }
-
- // get the db to store the migration in
- let dbNames: string[]
- if (migrationType === MigrationType.GLOBAL) {
- dbNames = [getGlobalDBName()]
- } else if (migrationType === MigrationType.APP) {
- if (options.noOp) {
- if (!options.noOp.appId) {
- throw new Error("appId is required for noOp app migration")
- }
- dbNames = [options.noOp.appId]
- } else {
- const apps = (await getAllApps(migration.appOpts)) as App[]
- dbNames = apps.map(app => app.appId)
- }
- } else if (migrationType === MigrationType.INSTALLATION) {
- dbNames = [StaticDatabases.PLATFORM_INFO.name]
- } else {
- throw new Error(`Unrecognised migration type [${migrationType}]`)
- }
-
- const length = dbNames.length
- let count = 0
-
- // run the migration against each db
- for (const dbName of dbNames) {
- count++
- const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
-
- const db = getDB(dbName)
-
- try {
- const doc = await getMigrationsDoc(db)
-
- // the migration has already been run
- if (doc[migrationName]) {
- // check for force
- if (
- options.force &&
- options.force[migrationType] &&
- options.force[migrationType].includes(migrationName)
- ) {
- log(`[Migration: ${migrationName}] [DB: ${dbName}] Forcing`)
- } else {
- // no force, exit
- return
- }
- }
-
- // check if the migration is not a no-op
- if (!options.noOp) {
- log(
- `[Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
- )
-
- if (migration.preventRetry) {
- // eagerly set the completion date
- // so that we never run this migration twice even upon failure
- doc[migrationName] = Date.now()
- const response = await db.put(doc)
- doc._rev = response.rev
- }
-
- // run the migration
- if (migrationType === MigrationType.APP) {
- await context.doInAppContext(db.name, async () => {
- await migration.fn(db)
- })
- } else {
- await migration.fn(db)
- }
-
- log(`[Migration: ${migrationName}] [DB: ${dbName}] Complete`)
- }
-
- // mark as complete
- doc[migrationName] = Date.now()
- await db.put(doc)
- } catch (err) {
- console.error(
- `[Migration: ${migrationName}] [DB: ${dbName}] Error: `,
- err
- )
- throw err
- }
- }
-}
-
-export const runMigrations = async (
- migrations: Migration[],
- options: MigrationOptions = {}
-) => {
- let tenantIds
-
- if (environment.MULTI_TENANCY) {
- if (options.noOp) {
- tenantIds = [options.noOp.tenantId]
- } else if (!options.tenantIds || !options.tenantIds.length) {
- // run for all tenants
- tenantIds = await platform.tenants.getTenantIds()
- } else {
- tenantIds = options.tenantIds
- }
- } else {
- // single tenancy
- tenantIds = [DEFAULT_TENANT_ID]
- }
-
- if (tenantIds.length > 1) {
- console.log(`Checking migrations for ${tenantIds.length} tenants`)
- } else {
- console.log("Checking migrations")
- }
-
- let count = 0
- // for all tenants
- for (const tenantId of tenantIds) {
- count++
- if (tenantIds.length > 1) {
- console.log(`Progress [${count}/${tenantIds.length}]`)
- }
- // for all migrations
- for (const migration of migrations) {
- // run the migration
- await context.doInTenant(
- tenantId,
- async () => await runMigration(migration, options)
- )
- }
- }
- console.log("Migrations complete")
-}
diff --git a/packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap b/packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap
deleted file mode 100644
index 377900b5d5..0000000000
--- a/packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap
+++ /dev/null
@@ -1,11 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`migrations should match snapshot 1`] = `
-{
- "_id": "migrations",
- "_rev": "1-2f64479842a0513aa8b97f356b0b9127",
- "createdAt": "2020-01-01T00:00:00.000Z",
- "test": 1577836800000,
- "updatedAt": "2020-01-01T00:00:00.000Z",
-}
-`;
diff --git a/packages/backend-core/src/migrations/tests/migrations.spec.ts b/packages/backend-core/src/migrations/tests/migrations.spec.ts
deleted file mode 100644
index af2eb33cf5..0000000000
--- a/packages/backend-core/src/migrations/tests/migrations.spec.ts
+++ /dev/null
@@ -1,64 +0,0 @@
-import { testEnv, DBTestConfiguration } from "../../../tests/extra"
-import * as migrations from "../index"
-import * as context from "../../context"
-import { MigrationType } from "@budibase/types"
-
-testEnv.multiTenant()
-
-describe("migrations", () => {
- const config = new DBTestConfiguration()
-
- const migrationFunction = jest.fn()
-
- const MIGRATIONS = [
- {
- type: MigrationType.GLOBAL,
- name: "test" as any,
- fn: migrationFunction,
- },
- ]
-
- beforeEach(() => {
- config.newTenant()
- })
-
- afterEach(async () => {
- jest.clearAllMocks()
- })
-
- const migrate = () => {
- return migrations.runMigrations(MIGRATIONS, {
- tenantIds: [config.tenantId],
- })
- }
-
- it("should run a new migration", async () => {
- await config.doInTenant(async () => {
- await migrate()
- expect(migrationFunction).toHaveBeenCalled()
- const db = context.getGlobalDB()
- const doc = await migrations.getMigrationsDoc(db)
- expect(doc.test).toBeDefined()
- })
- })
-
- it("should match snapshot", async () => {
- await config.doInTenant(async () => {
- await migrate()
- const doc = await migrations.getMigrationsDoc(context.getGlobalDB())
- expect(doc).toMatchSnapshot()
- })
- })
-
- it("should skip a previously run migration", async () => {
- await config.doInTenant(async () => {
- const db = context.getGlobalDB()
- await migrate()
- const previousDoc = await migrations.getMigrationsDoc(db)
- await migrate()
- const currentDoc = await migrations.getMigrationsDoc(db)
- expect(migrationFunction).toHaveBeenCalledTimes(1)
- expect(currentDoc.test).toBe(previousDoc.test)
- })
- })
-})
diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.ts
similarity index 67%
rename from packages/backend-core/tests/core/users/users.spec.js
rename to packages/backend-core/tests/core/users/users.spec.ts
index dde0d87fb7..b14f553266 100644
--- a/packages/backend-core/tests/core/users/users.spec.js
+++ b/packages/backend-core/tests/core/users/users.spec.ts
@@ -1,17 +1,17 @@
-const _ = require("lodash/fp")
-const { structures } = require("../../../tests")
+import { range } from "lodash/fp"
+import { structures } from "../.."
jest.mock("../../../src/context")
jest.mock("../../../src/db")
-const context = require("../../../src/context")
-const db = require("../../../src/db")
+import * as context from "../../../src/context"
+import * as db from "../../../src/db"
-const { getCreatorCount } = require("../../../src/users/users")
+import { getCreatorCount } from "../../../src/users/users"
describe("Users", () => {
- let getGlobalDBMock
- let paginationMock
+ let getGlobalDBMock: jest.SpyInstance
+ let paginationMock: jest.SpyInstance
beforeEach(() => {
jest.resetAllMocks()
@@ -22,11 +22,10 @@ describe("Users", () => {
jest.spyOn(db, "getGlobalUserParams")
})
- it("Retrieves the number of creators", async () => {
- const getUsers = (offset, limit, creators = false) => {
- const range = _.range(offset, limit)
+ it("retrieves the number of creators", async () => {
+ const getUsers = (offset: number, limit: number, creators = false) => {
const opts = creators ? { builder: { global: true } } : undefined
- return range.map(() => structures.users.user(opts))
+ return range(offset, limit).map(() => structures.users.user(opts))
}
const page1Data = getUsers(0, 8)
const page2Data = getUsers(8, 12, true)
diff --git a/packages/frontend-core/src/api/migrations.ts b/packages/frontend-core/src/api/migrations.ts
index 8213691205..35d2f95dbc 100644
--- a/packages/frontend-core/src/api/migrations.ts
+++ b/packages/frontend-core/src/api/migrations.ts
@@ -1,8 +1,8 @@
-import { GetOldMigrationStatus } from "@budibase/types"
+import { GetMigrationStatus } from "@budibase/types"
import { BaseAPIClient } from "./types"
export interface MigrationEndpoints {
- getMigrationStatus: () => Promise<GetOldMigrationStatus>
+ getMigrationStatus: () => Promise<GetMigrationStatus>
}
export const buildMigrationEndpoints = (
diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js
index 0794c53506..84e1abea69 100644
--- a/packages/server/scripts/dev/manage.js
+++ b/packages/server/scripts/dev/manage.js
@@ -43,7 +43,6 @@ async function init() {
BB_ADMIN_USER_EMAIL: "",
BB_ADMIN_USER_PASSWORD: "",
PLUGINS_DIR: "",
- HTTP_MIGRATIONS: "0",
HTTP_LOGGING: "0",
VERSION: "0.0.0+local",
PASSWORD_MIN_LENGTH: "1",
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 4169087a63..b62d022cbd 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -27,7 +27,6 @@ import {
env as envCore,
ErrorCode,
events,
- migrations,
objectStore,
roles,
tenancy,
@@ -43,7 +42,6 @@ import { groups, licensing, quotas } from "@budibase/pro"
import {
App,
Layout,
- MigrationType,
PlanType,
Screen,
UserCtx,
@@ -488,13 +486,6 @@ async function creationEvents(request: BBRequest<CreateAppRequest>, app: App) {
}
async function appPostCreate(ctx: UserCtx<CreateAppRequest, App>, app: App) {
- const tenantId = tenancy.getTenantId()
- await migrations.backPopulateMigrations({
- type: MigrationType.APP,
- tenantId,
- appId: app.appId,
- })
-
await creationEvents(ctx.request, app)
// app import, template creation and duplication
diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts
index edf4ec6f51..fc3a1a1548 100644
--- a/packages/server/src/api/controllers/migrations.ts
+++ b/packages/server/src/api/controllers/migrations.ts
@@ -1,35 +1,11 @@
import { context } from "@budibase/backend-core"
-import { migrate as migrationImpl, MIGRATIONS } from "../../migrations"
-import {
- Ctx,
- FetchOldMigrationResponse,
- GetOldMigrationStatus,
- RuneOldMigrationResponse,
- RunOldMigrationRequest,
-} from "@budibase/types"
+import { Ctx, GetMigrationStatus } from "@budibase/types"
import {
getAppMigrationVersion,
getLatestEnabledMigrationId,
} from "../../appMigrations"
-export async function migrate(
- ctx: Ctx<RunOldMigrationRequest, RuneOldMigrationResponse>
-) {
- const options = ctx.request.body
- // don't await as can take a while, just return
- migrationImpl(options)
- ctx.body = { message: "Migration started." }
-}
-
-export async function fetchDefinitions(
- ctx: Ctx<void, FetchOldMigrationResponse>
-) {
- ctx.body = MIGRATIONS
-}
-
-export async function getMigrationStatus(
- ctx: Ctx<void, GetOldMigrationStatus>
-) {
+export async function getMigrationStatus(ctx: Ctx<void, GetMigrationStatus>) {
const appId = context.getAppId()
if (!appId) {
diff --git a/packages/server/src/api/routes/migrations.ts b/packages/server/src/api/routes/migrations.ts
index 918b197de2..0ffc334551 100644
--- a/packages/server/src/api/routes/migrations.ts
+++ b/packages/server/src/api/routes/migrations.ts
@@ -1,16 +1,8 @@
import Router from "@koa/router"
import * as migrationsController from "../controllers/migrations"
-import { auth } from "@budibase/backend-core"
const router: Router = new Router()
-router
- .post("/api/migrations/run", auth.internalApi, migrationsController.migrate)
- .get(
- "/api/migrations/definitions",
- auth.internalApi,
- migrationsController.fetchDefinitions
- )
- .get("/api/migrations/status", migrationsController.getMigrationStatus)
+router.get("/api/migrations/status", migrationsController.getMigrationStatus)
export default router
diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index 45d675ec3f..a9867b1231 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -54,7 +54,6 @@ const environment = {
REDIS_URL: process.env.REDIS_URL,
REDIS_PASSWORD: process.env.REDIS_PASSWORD,
REDIS_CLUSTERED: process.env.REDIS_CLUSTERED,
- HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
CLUSTER_MODE: process.env.CLUSTER_MODE,
API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
diff --git a/packages/server/src/migrations/functions/appUrls.ts b/packages/server/src/migrations/functions/appUrls.ts
deleted file mode 100644
index be03d3c81e..0000000000
--- a/packages/server/src/migrations/functions/appUrls.ts
+++ /dev/null
@@ -1,27 +0,0 @@
-import { db as dbCore } from "@budibase/backend-core"
-import sdk from "../../sdk"
-
-/**
- * Date:
- * January 2022
- *
- * Description:
- * Add the url to the app metadata if it doesn't exist
- */
-export const run = async (appDb: any) => {
- let metadata
- try {
- metadata = await appDb.get(dbCore.DocumentType.APP_METADATA)
- } catch (e) {
- // sometimes the metadata document doesn't exist
- // exit early instead of failing the migration
- console.error("Error retrieving app metadata. Skipping", e)
- return
- }
-
- if (!metadata.url) {
- metadata.url = sdk.applications.getAppUrl({ name: metadata.name })
- console.log(`Adding url to app: ${metadata.url}`)
- await appDb.put(metadata)
- }
-}
diff --git a/packages/server/src/migrations/functions/backfill/app.ts b/packages/server/src/migrations/functions/backfill/app.ts
deleted file mode 100644
index 51a37108b0..0000000000
--- a/packages/server/src/migrations/functions/backfill/app.ts
+++ /dev/null
@@ -1,149 +0,0 @@
-import * as automations from "./app/automations"
-import * as datasources from "./app/datasources"
-import * as layouts from "./app/layouts"
-import * as queries from "./app/queries"
-import * as roles from "./app/roles"
-import * as tables from "./app/tables"
-import * as screens from "./app/screens"
-import * as global from "./global"
-import { App, AppBackfillSucceededEvent, Event } from "@budibase/types"
-import { db as dbUtils, events } from "@budibase/backend-core"
-import env from "../../../environment"
-import { DEFAULT_TIMESTAMP } from "."
-
-const failGraceful = env.SELF_HOSTED && !env.isDev()
-
-const handleError = (e: any, errors?: any) => {
- if (failGraceful) {
- if (errors) {
- errors.push(e)
- }
- return
- }
- console.trace(e)
- throw e
-}
-
-const EVENTS = [
- Event.AUTOMATION_CREATED,
- Event.AUTOMATION_STEP_CREATED,
- Event.DATASOURCE_CREATED,
- Event.LAYOUT_CREATED,
- Event.QUERY_CREATED,
- Event.ROLE_CREATED,
- Event.SCREEN_CREATED,
- Event.TABLE_CREATED,
- Event.VIEW_CREATED,
- Event.VIEW_CALCULATION_CREATED,
- Event.VIEW_FILTER_CREATED,
- Event.APP_PUBLISHED,
- Event.APP_CREATED,
-]
-
-/**
- * Date:
- * May 2022
- *
- * Description:
- * Backfill app events.
- */
-
-export const run = async (appDb: any) => {
- try {
- if (await global.isComplete()) {
- // make sure new apps aren't backfilled
- // return if the global migration for this tenant is complete
- // which runs after the app migrations
- return
- }
-
- // tell the event pipeline to start caching
- // events for this tenant
- await events.backfillCache.start(EVENTS)
-
- let timestamp: string | number = DEFAULT_TIMESTAMP
- const app: App = await appDb.get(dbUtils.DocumentType.APP_METADATA)
- if (app.createdAt) {
- timestamp = app.createdAt as string
- }
-
- if (dbUtils.isProdAppID(app.appId)) {
- await events.app.published(app, timestamp)
- }
-
- const totals: any = {}
- const errors: any = []
-
- if (dbUtils.isDevAppID(app.appId)) {
- await events.app.created(app, timestamp)
- try {
- totals.automations = await automations.backfill(appDb, timestamp)
- } catch (e) {
- handleError(e, errors)
- }
-
- try {
- totals.datasources = await datasources.backfill(appDb, timestamp)
- } catch (e) {
- handleError(e, errors)
- }
-
- try {
- totals.layouts = await layouts.backfill(appDb, timestamp)
- } catch (e) {
- handleError(e, errors)
- }
-
- try {
- totals.queries = await queries.backfill(appDb, timestamp)
- } catch (e) {
- handleError(e, errors)
- }
-
- try {
- totals.roles = await roles.backfill(appDb, timestamp)
- } catch (e) {
- handleError(e, errors)
- }
-
- try {
- totals.screens = await screens.backfill(appDb, timestamp)
- } catch (e) {
- handleError(e, errors)
- }
-
- try {
- totals.tables = await tables.backfill(appDb, timestamp)
- } catch (e) {
- handleError(e, errors)
- }
- }
-
- const properties: AppBackfillSucceededEvent = {
- appId: app.appId,
- automations: totals.automations,
- datasources: totals.datasources,
- layouts: totals.layouts,
- queries: totals.queries,
- roles: totals.roles,
- tables: totals.tables,
- screens: totals.screens,
- }
-
- if (errors.length) {
- properties.errors = errors.map((e: any) =>
- JSON.stringify(e, Object.getOwnPropertyNames(e))
- )
- properties.errorCount = errors.length
- } else {
- properties.errorCount = 0
- }
-
- await events.backfill.appSucceeded(properties)
- // tell the event pipeline to stop caching events for this tenant
- await events.backfillCache.end()
- } catch (e) {
- handleError(e)
- await events.backfill.appFailed(e)
- }
-}
diff --git a/packages/server/src/migrations/functions/backfill/app/automations.ts b/packages/server/src/migrations/functions/backfill/app/automations.ts
deleted file mode 100644
index 20da8fd3c0..0000000000
--- a/packages/server/src/migrations/functions/backfill/app/automations.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getAutomationParams } from "../../../../db/utils"
-import { Automation } from "@budibase/types"
-
-const getAutomations = async (appDb: any): Promise<Automation[]> => {
- const response = await appDb.allDocs(
- getAutomationParams(null, {
- include_docs: true,
- })
- )
- return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
- const automations = await getAutomations(appDb)
-
- for (const automation of automations) {
- await events.automation.created(automation, timestamp)
-
- for (const step of automation.definition.steps) {
- await events.automation.stepCreated(automation, step, timestamp)
- }
- }
-
- return automations.length
-}
diff --git a/packages/server/src/migrations/functions/backfill/app/datasources.ts b/packages/server/src/migrations/functions/backfill/app/datasources.ts
deleted file mode 100644
index 5d7e1ad866..0000000000
--- a/packages/server/src/migrations/functions/backfill/app/datasources.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getDatasourceParams } from "../../../../db/utils"
-import { Datasource } from "@budibase/types"
-
-const getDatasources = async (appDb: any): Promise<Datasource[]> => {
- const response = await appDb.allDocs(
- getDatasourceParams(null, {
- include_docs: true,
- })
- )
- return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
- const datasources: Datasource[] = await getDatasources(appDb)
-
- for (const datasource of datasources) {
- await events.datasource.created(datasource, timestamp)
- }
-
- return datasources.length
-}
diff --git a/packages/server/src/migrations/functions/backfill/app/layouts.ts b/packages/server/src/migrations/functions/backfill/app/layouts.ts
deleted file mode 100644
index ee5806459b..0000000000
--- a/packages/server/src/migrations/functions/backfill/app/layouts.ts
+++ /dev/null
@@ -1,29 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getLayoutParams } from "../../../../db/utils"
-import { Layout } from "@budibase/types"
-
-const getLayouts = async (appDb: any): Promise<Layout[]> => {
- const response = await appDb.allDocs(
- getLayoutParams(null, {
- include_docs: true,
- })
- )
- return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
- const layouts: Layout[] = await getLayouts(appDb)
-
- for (const layout of layouts) {
- // exclude default layouts
- if (
- layout._id === "layout_private_master" ||
- layout._id === "layout_public_master"
- ) {
- continue
- }
- await events.layout.created(layout, timestamp)
- }
-
- return layouts.length
-}
diff --git a/packages/server/src/migrations/functions/backfill/app/queries.ts b/packages/server/src/migrations/functions/backfill/app/queries.ts
deleted file mode 100644
index e028721bce..0000000000
--- a/packages/server/src/migrations/functions/backfill/app/queries.ts
+++ /dev/null
@@ -1,47 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getQueryParams } from "../../../../db/utils"
-import { Query, Datasource, SourceName } from "@budibase/types"
-
-const getQueries = async (appDb: any): Promise<Query[]> => {
- const response = await appDb.allDocs(
- getQueryParams(null, {
- include_docs: true,
- })
- )
- return response.rows.map((row: any) => row.doc)
-}
-
-const getDatasource = async (
- appDb: any,
- datasourceId: string
-): Promise<Datasource> => {
- return appDb.get(datasourceId)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
- const queries: Query[] = await getQueries(appDb)
-
- for (const query of queries) {
- let datasource: Datasource
-
- try {
- datasource = await getDatasource(appDb, query.datasourceId)
- } catch (e: any) {
- // handle known bug where a datasource has been deleted
- // and the query has not
- if (e.status === 404) {
- datasource = {
- type: "unknown",
- _id: query.datasourceId,
- source: "unknown" as SourceName,
- }
- } else {
- throw e
- }
- }
-
- await events.query.created(datasource, query, timestamp)
- }
-
- return queries.length
-}
diff --git a/packages/server/src/migrations/functions/backfill/app/roles.ts b/packages/server/src/migrations/functions/backfill/app/roles.ts
deleted file mode 100644
index 494b6f6923..0000000000
--- a/packages/server/src/migrations/functions/backfill/app/roles.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getRoleParams } from "../../../../db/utils"
-import { Role } from "@budibase/types"
-
-const getRoles = async (appDb: any): Promise<Role[]> => {
- const response = await appDb.allDocs(
- getRoleParams(null, {
- include_docs: true,
- })
- )
- return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
- const roles = await getRoles(appDb)
-
- for (const role of roles) {
- await events.role.created(role, timestamp)
- }
-
- return roles.length
-}
diff --git a/packages/server/src/migrations/functions/backfill/app/screens.ts b/packages/server/src/migrations/functions/backfill/app/screens.ts
deleted file mode 100644
index ab3b4b9d3c..0000000000
--- a/packages/server/src/migrations/functions/backfill/app/screens.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { getScreenParams } from "../../../../db/utils"
-import { Screen } from "@budibase/types"
-
-const getScreens = async (appDb: any): Promise<Screen[]> => {
- const response = await appDb.allDocs(
- getScreenParams(null, {
- include_docs: true,
- })
- )
- return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
- const screens = await getScreens(appDb)
-
- for (const screen of screens) {
- await events.screen.created(screen, timestamp)
- }
-
- return screens.length
-}
diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts
deleted file mode 100644
index e8437bd529..0000000000
--- a/packages/server/src/migrations/functions/backfill/app/tables.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import { events } from "@budibase/backend-core"
-import { Database } from "@budibase/types"
-import sdk from "../../../../sdk"
-
-export const backfill = async (appDb: Database, timestamp: string | number) => {
- const tables = await sdk.tables.getAllInternalTables(appDb)
-
- for (const table of tables) {
- await events.table.created(table, timestamp)
- }
-
- return tables.length
-}
diff --git a/packages/server/src/migrations/functions/backfill/global.ts b/packages/server/src/migrations/functions/backfill/global.ts
deleted file mode 100644
index 7f718cee2f..0000000000
--- a/packages/server/src/migrations/functions/backfill/global.ts
+++ /dev/null
@@ -1,214 +0,0 @@
-import * as users from "./global/users"
-import * as configs from "./global/configs"
-import * as quotas from "./global/quotas"
-import {
- tenancy,
- events,
- migrations,
- accounts,
- db as dbUtils,
-} from "@budibase/backend-core"
-import {
- App,
- CloudAccount,
- Event,
- Hosting,
- QuotaUsage,
- TenantBackfillSucceededEvent,
- User,
-} from "@budibase/types"
-import env from "../../../environment"
-import { DEFAULT_TIMESTAMP } from "."
-
-const failGraceful = env.SELF_HOSTED && !env.isDev()
-
-const handleError = (e: any, errors?: any) => {
- if (failGraceful) {
- if (errors) {
- errors.push(e)
- }
- return
- }
- throw e
-}
-
-const formatUsage = (usage: QuotaUsage) => {
- let maxAutomations = 0
- let maxQueries = 0
- let rows = 0
-
- if (usage) {
- if (usage.usageQuota) {
- rows = usage.usageQuota.rows
- }
-
- if (usage.monthly) {
- for (const value of Object.values(usage.monthly)) {
- if (value.automations > maxAutomations) {
- maxAutomations = value.automations
- }
- if (value.queries > maxQueries) {
- maxQueries = value.queries
- }
- }
- }
- }
-
- return {
- maxAutomations,
- maxQueries,
- rows,
- }
-}
-
-const EVENTS = [
- Event.EMAIL_SMTP_CREATED,
- Event.AUTH_SSO_CREATED,
- Event.AUTH_SSO_ACTIVATED,
- Event.ORG_NAME_UPDATED,
- Event.ORG_LOGO_UPDATED,
- Event.ORG_PLATFORM_URL_UPDATED,
- Event.USER_CREATED,
- Event.USER_PERMISSION_ADMIN_ASSIGNED,
- Event.USER_PERMISSION_BUILDER_ASSIGNED,
- Event.ROLE_ASSIGNED,
- Event.ROWS_CREATED,
- Event.QUERIES_RUN,
- Event.AUTOMATIONS_RUN,
-]
-
-/**
- * Date:
- * May 2022
- *
- * Description:
- * Backfill global events.
- */
-
-export const run = async (db: any) => {
- try {
- const tenantId = tenancy.getTenantId()
- let timestamp: string | number = DEFAULT_TIMESTAMP
-
- const totals: any = {}
- const errors: any = []
-
- let allUsers: User[] = []
- try {
- allUsers = await users.getUsers(db)
- } catch (e: any) {
- handleError(e, errors)
- }
-
- if (!allUsers || allUsers.length === 0) {
- // first time startup - we don't need to backfill anything
- // tenant will be identified when admin user is created
- if (env.SELF_HOSTED) {
- await events.installation.firstStartup()
- }
- return
- }
-
- try {
- const installTimestamp = await getInstallTimestamp(db, allUsers)
- if (installTimestamp) {
- timestamp = installTimestamp
- }
- } catch (e) {
- handleError(e, errors)
- }
-
- let account: CloudAccount | undefined
- if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
- account = await accounts.getAccountByTenantId(tenantId)
- }
-
- try {
- await events.identification.identifyTenantGroup(
- tenantId,
- env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD,
- timestamp
- )
- } catch (e) {
- handleError(e, errors)
- }
-
- // tell the event pipeline to start caching
- // events for this tenant
- await events.backfillCache.start(EVENTS)
-
- try {
- await configs.backfill(db, timestamp)
- } catch (e) {
- handleError(e, errors)
- }
-
- try {
- totals.users = await users.backfill(db, account)
- } catch (e) {
- handleError(e, errors)
- }
-
- try {
- const allApps = (await dbUtils.getAllApps({ dev: true })) as App[]
- totals.apps = allApps.length
-
- totals.usage = await quotas.backfill(allApps)
- } catch (e) {
- handleError(e, errors)
- }
-
- const properties: TenantBackfillSucceededEvent = {
- apps: totals.apps,
- users: totals.users,
- ...formatUsage(totals.usage),
- usage: totals.usage,
- }
-
- if (errors.length) {
- properties.errors = errors.map((e: any) =>
- JSON.stringify(e, Object.getOwnPropertyNames(e))
- )
- properties.errorCount = errors.length
- } else {
- properties.errorCount = 0
- }
-
- await events.backfill.tenantSucceeded(properties)
- // tell the event pipeline to stop caching events for this tenant
- await events.backfillCache.end()
- } catch (e) {
- handleError(e)
- await events.backfill.tenantFailed(e)
- }
-}
-
-export const isComplete = async (): Promise<boolean> => {
- const globalDb = tenancy.getGlobalDB()
- const migrationsDoc = await migrations.getMigrationsDoc(globalDb)
- return !!migrationsDoc.event_global_backfill
-}
-
-export const getInstallTimestamp = async (
- globalDb: any,
- allUsers?: User[]
-): Promise<number | undefined> => {
- if (!allUsers) {
- allUsers = await users.getUsers(globalDb)
- }
-
- // get the oldest user timestamp
- if (allUsers) {
- const timestamps = allUsers
- .map(user => user.createdAt)
- .filter(timestamp => !!timestamp)
- .sort(
- (a, b) =>
- new Date(a as number).getTime() - new Date(b as number).getTime()
- )
-
- if (timestamps.length) {
- return timestamps[0]
- }
- }
-}
diff --git a/packages/server/src/migrations/functions/backfill/global/configs.ts b/packages/server/src/migrations/functions/backfill/global/configs.ts
deleted file mode 100644
index 04eb9caff2..0000000000
--- a/packages/server/src/migrations/functions/backfill/global/configs.ts
+++ /dev/null
@@ -1,74 +0,0 @@
-import {
- events,
- DocumentType,
- SEPARATOR,
- UNICODE_MAX,
-} from "@budibase/backend-core"
-import {
- Config,
- isSMTPConfig,
- isGoogleConfig,
- isOIDCConfig,
- isSettingsConfig,
- ConfigType,
- DatabaseQueryOpts,
-} from "@budibase/types"
-import env from "./../../../../environment"
-
-export function getConfigParams(): DatabaseQueryOpts {
- return {
- include_docs: true,
- startkey: `${DocumentType.CONFIG}${SEPARATOR}`,
- endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`,
- }
-}
-
-const getConfigs = async (globalDb: any): Promise<Config[]> => {
- const response = await globalDb.allDocs(getConfigParams())
- return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (
- globalDb: any,
- timestamp: string | number | undefined
-) => {
- const configs = await getConfigs(globalDb)
-
- for (const config of configs) {
- if (isSMTPConfig(config)) {
- await events.email.SMTPCreated(timestamp)
- }
- if (isGoogleConfig(config)) {
- await events.auth.SSOCreated(ConfigType.GOOGLE, timestamp)
- if (config.config.activated) {
- await events.auth.SSOActivated(ConfigType.GOOGLE, timestamp)
- }
- }
- if (isOIDCConfig(config)) {
- await events.auth.SSOCreated(ConfigType.OIDC, timestamp)
- if (config.config.configs[0].activated) {
- await events.auth.SSOActivated(ConfigType.OIDC, timestamp)
- }
- }
- if (isSettingsConfig(config)) {
- const company = config.config.company
- if (company && company !== "Budibase") {
- await events.org.nameUpdated(timestamp)
- }
-
- const logoUrl = config.config.logoUrl
- if (logoUrl) {
- await events.org.logoUpdated(timestamp)
- }
-
- const platformUrl = config.config.platformUrl
- if (
- platformUrl &&
- platformUrl !== "http://localhost:10000" &&
- env.SELF_HOSTED
- ) {
- await events.org.platformURLUpdated(timestamp)
- }
- }
- }
-}
diff --git a/packages/server/src/migrations/functions/backfill/global/quotas.ts b/packages/server/src/migrations/functions/backfill/global/quotas.ts
deleted file mode 100644
index 505274a318..0000000000
--- a/packages/server/src/migrations/functions/backfill/global/quotas.ts
+++ /dev/null
@@ -1,60 +0,0 @@
-import { DEFAULT_TIMESTAMP } from "./../index"
-import { events } from "@budibase/backend-core"
-import { quotas } from "@budibase/pro"
-import { App } from "@budibase/types"
-
-const getOldestCreatedAt = (allApps: App[]): string | undefined => {
- const timestamps = allApps
- .filter(app => !!app.createdAt)
- .map(app => app.createdAt as string)
- .sort((a, b) => new Date(a).getTime() - new Date(b).getTime())
-
- if (timestamps.length) {
- return timestamps[0]
- }
-}
-
-const getMonthTimestamp = (monthString: string): number => {
- const parts = monthString.split("-")
- const month = parseInt(parts[0]) - 1 // we already do +1 in month string calculation
- const year = parseInt(parts[1])
-
- // using 0 as the day in next month gives us last day in previous month
- const date = new Date(year, month + 1, 0).getTime()
- const now = new Date().getTime()
-
- if (date > now) {
- return now
- } else {
- return date
- }
-}
-
-export const backfill = async (allApps: App[]) => {
- const usage = await quotas.getQuotaUsage()
-
- const rows = usage.usageQuota.rows
- let timestamp: string | number = DEFAULT_TIMESTAMP
-
- const oldestAppTimestamp = getOldestCreatedAt(allApps)
- if (oldestAppTimestamp) {
- timestamp = oldestAppTimestamp
- }
-
- await events.rows.created(rows, timestamp)
-
- for (const [monthString, quotas] of Object.entries(usage.monthly)) {
- if (monthString === "current") {
- continue
- }
- const monthTimestamp = getMonthTimestamp(monthString)
-
- const queries = quotas.queries
- await events.query.run(queries, monthTimestamp)
-
- const automations = quotas.automations
- await events.automation.run(automations, monthTimestamp)
- }
-
- return usage
-}
diff --git a/packages/server/src/migrations/functions/backfill/global/users.ts b/packages/server/src/migrations/functions/backfill/global/users.ts
deleted file mode 100644
index b3dae822d7..0000000000
--- a/packages/server/src/migrations/functions/backfill/global/users.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-import {
- events,
- db as dbUtils,
- users as usersCore,
-} from "@budibase/backend-core"
-import { User, CloudAccount } from "@budibase/types"
-import { DEFAULT_TIMESTAMP } from ".."
-
-// manually define user doc params - normally server doesn't read users from the db
-const getUserParams = (props: any) => {
- return dbUtils.getDocParams(dbUtils.DocumentType.USER, null, props)
-}
-
-export const getUsers = async (globalDb: any): Promise<User[]> => {
- const response = await globalDb.allDocs(
- getUserParams({
- include_docs: true,
- })
- )
- return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (
- globalDb: any,
- account: CloudAccount | undefined
-) => {
- const users = await getUsers(globalDb)
-
- for (const user of users) {
- let timestamp: string | number = DEFAULT_TIMESTAMP
- if (user.createdAt) {
- timestamp = user.createdAt
- }
- await events.identification.identifyUser(user, account, timestamp)
- await events.user.created(user, timestamp)
-
- if (usersCore.hasAdminPermissions(user)) {
- await events.user.permissionAdminAssigned(user, timestamp)
- }
-
- if (usersCore.hasBuilderPermissions(user)) {
- await events.user.permissionBuilderAssigned(user, timestamp)
- }
-
- if (user.roles) {
- for (const [, role] of Object.entries(user.roles)) {
- await events.role.assigned(user, role, timestamp)
- }
- }
- }
-
- return users.length
-}
diff --git a/packages/server/src/migrations/functions/backfill/index.ts b/packages/server/src/migrations/functions/backfill/index.ts
deleted file mode 100644
index 00c04722b4..0000000000
--- a/packages/server/src/migrations/functions/backfill/index.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-export * as app from "./app"
-export * as global from "./global"
-export * as installation from "./installation"
-
-// historical events are free in posthog - make sure we default to a
-// historical time if no other can be found
-export const DEFAULT_TIMESTAMP = new Date(2022, 0, 1).getTime()
diff --git a/packages/server/src/migrations/functions/backfill/installation.ts b/packages/server/src/migrations/functions/backfill/installation.ts
deleted file mode 100644
index 3c2b8bd3fc..0000000000
--- a/packages/server/src/migrations/functions/backfill/installation.ts
+++ /dev/null
@@ -1,50 +0,0 @@
-import { DEFAULT_TIMESTAMP } from "./index"
-import { events, tenancy, installation } from "@budibase/backend-core"
-import { Installation } from "@budibase/types"
-import * as global from "./global"
-import env from "../../../environment"
-
-const failGraceful = env.SELF_HOSTED
-
-const handleError = (e: any, errors?: any) => {
- if (failGraceful) {
- if (errors) {
- errors.push(e)
- }
- return
- }
- throw e
-}
-
-/**
- * Date:
- * May 2022
- *
- * Description:
- * Backfill installation events.
- */
-
-export const run = async () => {
- try {
- // need to use the default tenant to try to get the installation time
- await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => {
- const db = tenancy.getGlobalDB()
- let timestamp: string | number = DEFAULT_TIMESTAMP
-
- const installTimestamp = await global.getInstallTimestamp(db)
- if (installTimestamp) {
- timestamp = installTimestamp
- }
-
- const install: Installation = await installation.getInstall()
- await events.identification.identifyInstallationGroup(
- install.installId,
- timestamp
- )
- })
- await events.backfill.installationSucceeded()
- } catch (e) {
- handleError(e)
- await events.backfill.installationFailed(e)
- }
-}
diff --git a/packages/server/src/migrations/functions/syncQuotas.ts b/packages/server/src/migrations/functions/syncQuotas.ts
deleted file mode 100644
index 83a7670e78..0000000000
--- a/packages/server/src/migrations/functions/syncQuotas.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-import { runQuotaMigration } from "./usageQuotas"
-import * as syncApps from "./usageQuotas/syncApps"
-import * as syncRows from "./usageQuotas/syncRows"
-import * as syncPlugins from "./usageQuotas/syncPlugins"
-import * as syncUsers from "./usageQuotas/syncUsers"
-import * as syncCreators from "./usageQuotas/syncCreators"
-
-/**
- * Synchronise quotas to the state of the db.
- */
-export const run = async () => {
- await runQuotaMigration(async () => {
- await syncApps.run()
- await syncRows.run()
- await syncPlugins.run()
- await syncUsers.run()
- await syncCreators.run()
- })
-}
diff --git a/packages/server/src/migrations/functions/tableSettings.ts b/packages/server/src/migrations/functions/tableSettings.ts
deleted file mode 100644
index 2db3df0d0f..0000000000
--- a/packages/server/src/migrations/functions/tableSettings.ts
+++ /dev/null
@@ -1,145 +0,0 @@
-import { getScreenParams } from "../../db/utils"
-import { Screen } from "@budibase/types"
-import { makePropSafe as safe } from "@budibase/string-templates"
-/**
- * Date:
- * November 2022
- *
- * Description:
- * Update table settings to use actions instead of links. We do not remove the
- * legacy values here as we cannot guarantee that their apps are up-t-date.
- * It is safe to simply save both the new and old structure in the definition.
- *
- * Migration 1:
- * Legacy "linkRows", "linkURL", "linkPeek" and "linkColumn" settings on tables
- * and table blocks are migrated into a "Navigate To" action under the new
- * "onClick" setting.
- *
- * Migration 2:
- * Legacy "titleButtonURL" and "titleButtonPeek" settings on table blocks are
- * migrated into a "Navigate To" action under the new "onClickTitleButton"
- * setting.
- */
-export const run = async (appDb: any) => {
- // Get all app screens
- let screens: Screen[]
- try {
- screens = (
- await appDb.allDocs(
- getScreenParams(null, {
- include_docs: true,
- })
- )
- ).rows.map((row: any) => row.doc)
- } catch (e) {
- // sometimes the metadata document doesn't exist
- // exit early instead of failing the migration
- console.error("Error retrieving app metadata. Skipping", e)
- return
- }
-
- // Recursively update any relevant components and mutate the screen docs
- for (let screen of screens) {
- const changed = migrateTableSettings(screen.props)
-
- // Save screen if we updated it
- if (changed) {
- await appDb.put(screen)
- console.log(
- `Screen ${screen.routing?.route} contained table settings which were migrated`
- )
- }
- }
-}
-
-// Recursively searches and mutates a screen doc to migrate table component
-// and table block settings
-const migrateTableSettings = (component: any) => {
- let changed = false
- if (!component) {
- return changed
- }
-
- // Migration 1: migrate table row click settings
- if (
- component._component.endsWith("/table") ||
- component._component.endsWith("/tableblock")
- ) {
- const { linkRows, linkURL, linkPeek, linkColumn, onClick } = component
- if (linkRows && !onClick) {
- const column = linkColumn || "_id"
- const action = convertLinkSettingToAction(linkURL, !!linkPeek, column)
- if (action) {
- changed = true
- component.onClick = action
- if (component._component.endsWith("/tableblock")) {
- component.clickBehaviour = "actions"
- }
- }
- }
- }
-
- // Migration 2: migrate table block title button settings
- if (component._component.endsWith("/tableblock")) {
- const {
- showTitleButton,
- titleButtonURL,
- titleButtonPeek,
- onClickTitleButton,
- } = component
- if (showTitleButton && !onClickTitleButton) {
- const action = convertLinkSettingToAction(
- titleButtonURL,
- !!titleButtonPeek
- )
- if (action) {
- changed = true
- component.onClickTitleButton = action
- component.titleButtonClickBehaviour = "actions"
- }
- }
- }
-
- // Recurse down the tree as needed
- component._children?.forEach((child: any) => {
- const childChanged = migrateTableSettings(child)
- changed = changed || childChanged
- })
- return changed
-}
-
-// Util ti convert the legacy settings into a navigation action structure
-const convertLinkSettingToAction = (
- linkURL: string,
- linkPeek: boolean,
- linkColumn?: string
-) => {
- // Sanity check we have a URL
- if (!linkURL) {
- return null
- }
-
- // Default URL to the old URL setting
- let url = linkURL
-
- // If we enriched the old URL with a column, update the url
- if (linkColumn && linkURL.includes("/:")) {
- // Convert old link URL setting, which is a screen URL, into a valid
- // binding using the new clicked row binding
- const split = linkURL.split("/:")
- const col = linkColumn || "_id"
- const binding = `{{ ${safe("eventContext")}.${safe("row")}.${safe(col)} }}`
- url = `${split[0]}/${binding}`
- }
-
- // Create action structure
- return [
- {
- "##eventHandlerType": "Navigate To",
- parameters: {
- url,
- peek: linkPeek,
- },
- },
- ]
-}
diff --git a/packages/server/src/migrations/functions/tests/appUrls.spec.js b/packages/server/src/migrations/functions/tests/appUrls.spec.js
deleted file mode 100644
index 7a2f324552..0000000000
--- a/packages/server/src/migrations/functions/tests/appUrls.spec.js
+++ /dev/null
@@ -1,26 +0,0 @@
-const { db: dbCore } = require("@budibase/backend-core")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-
-const migration = require("../appUrls")
-
-describe("run", () => {
- let config = new TestConfig(false)
-
- beforeAll(async () => {
- await config.init()
- })
-
- afterAll(config.end)
-
- it("runs successfully", async () => {
- const app = await config.createApp("testApp")
- const metadata = await dbCore.doWithDB(app.appId, async db => {
- const metadataDoc = await db.get(dbCore.DocumentType.APP_METADATA)
- delete metadataDoc.url
- await db.put(metadataDoc)
- await migration.run(db)
- return await db.get(dbCore.DocumentType.APP_METADATA)
- })
- expect(metadata.url).toEqual("/testapp")
- })
-})
diff --git a/packages/server/src/migrations/functions/tests/tableSettings.spec.ts b/packages/server/src/migrations/functions/tests/tableSettings.spec.ts
deleted file mode 100644
index 8d28a43322..0000000000
--- a/packages/server/src/migrations/functions/tests/tableSettings.spec.ts
+++ /dev/null
@@ -1,144 +0,0 @@
-import { App, Screen } from "@budibase/types"
-
-import { db as dbCore } from "@budibase/backend-core"
-import TestConfig from "../../../tests/utilities/TestConfiguration"
-import { run as runMigration } from "../tableSettings"
-
-describe("run", () => {
- const config = new TestConfig(false)
- let app: App
- let screen: Screen
-
- beforeAll(async () => {
- await config.init()
- app = await config.createApp("testApp")
- screen = await config.createScreen()
- })
-
- afterAll(config.end)
-
- it("migrates table block row on click settings", async () => {
- // Add legacy table block as first child
- screen.props._children = [
- {
- _instanceName: "Table Block",
- _styles: {},
- _component: "@budibase/standard-components/tableblock",
- _id: "foo",
- linkRows: true,
- linkURL: "/rows/:id",
- linkPeek: true,
- linkColumn: "name",
- },
- ]
- await config.createScreen(screen)
-
- // Run migration
- screen = await dbCore.doWithDB(app.appId, async (db: any) => {
- await runMigration(db)
- return await db.get(screen._id)
- })
-
- // Verify new "onClick" setting
- const onClick = screen.props._children?.[0].onClick
- expect(onClick).toBeDefined()
- expect(onClick.length).toBe(1)
- expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
- expect(onClick[0].parameters.url).toBe(
- `/rows/{{ [eventContext].[row].[name] }}`
- )
- expect(onClick[0].parameters.peek).toBeTruthy()
- })
-
- it("migrates table row on click settings", async () => {
- // Add legacy table block as first child
- screen.props._children = [
- {
- _instanceName: "Table",
- _styles: {},
- _component: "@budibase/standard-components/table",
- _id: "foo",
- linkRows: true,
- linkURL: "/rows/:id",
- linkPeek: true,
- linkColumn: "name",
- },
- ]
- await config.createScreen(screen)
-
- // Run migration
- screen = await dbCore.doWithDB(app.appId, async (db: any) => {
- await runMigration(db)
- return await db.get(screen._id)
- })
-
- // Verify new "onClick" setting
- const onClick = screen.props._children?.[0].onClick
- expect(onClick).toBeDefined()
- expect(onClick.length).toBe(1)
- expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
- expect(onClick[0].parameters.url).toBe(
- `/rows/{{ [eventContext].[row].[name] }}`
- )
- expect(onClick[0].parameters.peek).toBeTruthy()
- })
-
- it("migrates table block title button settings", async () => {
- // Add legacy table block as first child
- screen.props._children = [
- {
- _instanceName: "Table Block",
- _styles: {},
- _component: "@budibase/standard-components/tableblock",
- _id: "foo",
- showTitleButton: true,
- titleButtonURL: "/url",
- titleButtonPeek: true,
- },
- ]
- await config.createScreen(screen)
-
- // Run migration
- screen = await dbCore.doWithDB(app.appId, async (db: any) => {
- await runMigration(db)
- return await db.get(screen._id)
- })
-
- // Verify new "onClickTitleButton" setting
- const onClick = screen.props._children?.[0].onClickTitleButton
- expect(onClick).toBeDefined()
- expect(onClick.length).toBe(1)
- expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
- expect(onClick[0].parameters.url).toBe("/url")
- expect(onClick[0].parameters.peek).toBeTruthy()
- })
-
- it("ignores components that have already been migrated", async () => {
- // Add legacy table block as first child
- screen.props._children = [
- {
- _instanceName: "Table Block",
- _styles: {},
- _component: "@budibase/standard-components/tableblock",
- _id: "foo",
- linkRows: true,
- linkURL: "/rows/:id",
- linkPeek: true,
- linkColumn: "name",
- onClick: "foo",
- },
- ]
- const initialDefinition = JSON.stringify(screen.props._children?.[0])
- await config.createScreen(screen)
-
- // Run migration
- screen = await dbCore.doWithDB(app.appId, async (db: any) => {
- await runMigration(db)
- return await db.get(screen._id)
- })
-
- // Verify new "onClick" setting
- const newDefinition = JSON.stringify(screen.props._children?.[0])
- expect(initialDefinition).toEqual(newDefinition)
- })
-})
diff --git a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js
deleted file mode 100644
index f0da25893c..0000000000
--- a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js
+++ /dev/null
@@ -1,34 +0,0 @@
-jest.mock("@budibase/backend-core", () => {
- const core = jest.requireActual("@budibase/backend-core")
- return {
- ...core,
- db: {
- ...core.db,
- createNewUserEmailView: jest.fn(),
- },
- }
-})
-const { context, db: dbCore } = require("@budibase/backend-core")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-
-// mock email view creation
-
-const migration = require("../userEmailViewCasing")
-
-describe("run", () => {
- let config = new TestConfig(false)
-
- beforeAll(async () => {
- await config.init()
- })
-
- afterAll(config.end)
-
- it("runs successfully", async () => {
- await config.doInTenant(async () => {
- const globalDb = context.getGlobalDB()
- await migration.run(globalDb)
- expect(dbCore.createNewUserEmailView).toHaveBeenCalledTimes(1)
- })
- })
-})
diff --git a/packages/server/src/migrations/functions/usageQuotas/index.ts b/packages/server/src/migrations/functions/usageQuotas/index.ts
deleted file mode 100644
index e94e993b21..0000000000
--- a/packages/server/src/migrations/functions/usageQuotas/index.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-export const runQuotaMigration = async (migration: () => Promise<void>) => {
- await migration()
-}
diff --git a/packages/server/src/migrations/functions/usageQuotas/syncApps.ts b/packages/server/src/migrations/functions/usageQuotas/syncApps.ts
deleted file mode 100644
index 80ed1953d3..0000000000
--- a/packages/server/src/migrations/functions/usageQuotas/syncApps.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import { db as dbCore } from "@budibase/backend-core"
-import { quotas } from "@budibase/pro"
-import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-
-export const run = async () => {
- // get app count
- const devApps = await dbCore.getAllApps({ dev: true })
- const appCount = devApps ? devApps.length : 0
-
- // sync app count
- console.log(`Syncing app count: ${appCount}`)
- await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
-}
diff --git a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts b/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
deleted file mode 100644
index ce53be925a..0000000000
--- a/packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import { users } from "@budibase/backend-core"
-import { quotas } from "@budibase/pro"
-import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-
-export const run = async () => {
- const creatorCount = await users.getCreatorCount()
- console.log(`Syncing creator count: ${creatorCount}`)
- await quotas.setUsage(
- creatorCount,
- StaticQuotaName.CREATORS,
- QuotaUsageType.STATIC
- )
-}
diff --git a/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts b/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts
deleted file mode 100644
index b00970aea2..0000000000
--- a/packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-import { logging } from "@budibase/backend-core"
-import { plugins } from "@budibase/pro"
-
-export const run = async () => {
- try {
- await plugins.checkPluginQuotas()
- } catch (err) {
- logging.logAlert("Failed to update plugin quotas", err)
- }
-}
diff --git a/packages/server/src/migrations/functions/usageQuotas/syncRows.ts b/packages/server/src/migrations/functions/usageQuotas/syncRows.ts
deleted file mode 100644
index 506218a41f..0000000000
--- a/packages/server/src/migrations/functions/usageQuotas/syncRows.ts
+++ /dev/null
@@ -1,27 +0,0 @@
-import { db as dbCore } from "@budibase/backend-core"
-import { getUniqueRows } from "../../../utilities/usageQuota/rows"
-import { quotas } from "@budibase/pro"
-import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types"
-
-export const run = async () => {
- // get all rows in all apps
- const allApps = (await dbCore.getAllApps({ all: true })) as App[]
- const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
- const { appRows } = await getUniqueRows(appIds)
-
- // get the counts per app
- const counts: { [key: string]: number } = {}
- let rowCount = 0
- Object.entries(appRows).forEach(([appId, rows]) => {
- counts[appId] = rows.length
- rowCount += rows.length
- })
-
- // sync row count
- console.log(`Syncing row count: ${rowCount}`)
- await quotas.setUsagePerApp(
- counts,
- StaticQuotaName.ROWS,
- QuotaUsageType.STATIC
- )
-}
diff --git a/packages/server/src/migrations/functions/usageQuotas/syncUsers.ts b/packages/server/src/migrations/functions/usageQuotas/syncUsers.ts
deleted file mode 100644
index c9913dced8..0000000000
--- a/packages/server/src/migrations/functions/usageQuotas/syncUsers.ts
+++ /dev/null
@@ -1,9 +0,0 @@
-import { users } from "@budibase/backend-core"
-import { quotas } from "@budibase/pro"
-import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-
-export const run = async () => {
- const userCount = await users.getUserCount()
- console.log(`Syncing user count: ${userCount}`)
- await quotas.setUsage(userCount, StaticQuotaName.USERS, QuotaUsageType.STATIC)
-}
diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts
deleted file mode 100644
index 1d4d4d0f71..0000000000
--- a/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-import TestConfig from "../../../../tests/utilities/TestConfiguration"
-import * as syncApps from "../syncApps"
-import { quotas } from "@budibase/pro"
-import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-
-describe("syncApps", () => {
- let config = new TestConfig(false)
-
- beforeEach(async () => {
- await config.init()
- })
-
- afterAll(config.end)
-
- it("runs successfully", async () => {
- return config.doInContext(undefined, async () => {
- // create the usage quota doc and mock usages
- await quotas.getQuotaUsage()
- await quotas.setUsage(3, StaticQuotaName.APPS, QuotaUsageType.STATIC)
-
- let usageDoc = await quotas.getQuotaUsage()
- expect(usageDoc.usageQuota.apps).toEqual(3)
-
- // create an extra app to test the migration
- await config.createApp("quota-test")
-
- // migrate
- await syncApps.run()
-
- // assert the migration worked
- usageDoc = await quotas.getQuotaUsage()
- expect(usageDoc.usageQuota.apps).toEqual(2)
- })
- })
-})
diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts
deleted file mode 100644
index 93b7d4949b..0000000000
--- a/packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import TestConfig from "../../../../tests/utilities/TestConfiguration"
-import * as syncCreators from "../syncCreators"
-import { quotas } from "@budibase/pro"
-
-describe("syncCreators", () => {
- let config = new TestConfig(false)
-
- beforeEach(async () => {
- await config.init()
- })
-
- afterAll(config.end)
-
- it("syncs creators", async () => {
- return config.doInContext(undefined, async () => {
- await config.createUser({ admin: { global: true } })
-
- await syncCreators.run()
-
- const usageDoc = await quotas.getQuotaUsage()
- // default + additional creator
- const creatorsCount = 2
- expect(usageDoc.usageQuota.creators).toBe(creatorsCount)
- })
- })
-})
diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
deleted file mode 100644
index 730278683c..0000000000
--- a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-import TestConfig from "../../../../tests/utilities/TestConfiguration"
-import * as syncRows from "../syncRows"
-import { quotas } from "@budibase/pro"
-import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
-import { db as dbCore, context } from "@budibase/backend-core"
-
-describe("syncRows", () => {
- const config = new TestConfig()
-
- beforeEach(async () => {
- await config.init()
- })
-
- afterAll(config.end)
-
- it("runs successfully", async () => {
- return config.doInContext(undefined, async () => {
- // create the usage quota doc and mock usages
- await quotas.getQuotaUsage()
- await quotas.setUsage(300, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
-
- let usageDoc = await quotas.getQuotaUsage()
- expect(usageDoc.usageQuota.rows).toEqual(300)
-
- // app 1
- const app1 = config.app
- await context.doInAppContext(app1!.appId, async () => {
- await config.createTable()
- await config.createRow()
- })
- // app 2
- const app2 = await config.createApp("second-app")
- await context.doInAppContext(app2.appId, async () => {
- await config.createTable()
- await config.createRow()
- await config.createRow()
- })
-
- // migrate
- await syncRows.run()
-
- // assert the migration worked
- usageDoc = await quotas.getQuotaUsage()
- expect(usageDoc.usageQuota.rows).toEqual(3)
- expect(
- usageDoc.apps?.[dbCore.getProdAppID(app1!.appId)].usageQuota.rows
- ).toEqual(1)
- expect(
- usageDoc.apps?.[dbCore.getProdAppID(app2.appId)].usageQuota.rows
- ).toEqual(2)
- })
- })
-})
diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts
deleted file mode 100644
index 2731cc041d..0000000000
--- a/packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import TestConfig from "../../../../tests/utilities/TestConfiguration"
-import * as syncUsers from "../syncUsers"
-import { quotas } from "@budibase/pro"
-
-describe("syncUsers", () => {
- let config = new TestConfig(false)
-
- beforeEach(async () => {
- await config.init()
- })
-
- afterAll(config.end)
-
- it("syncs users", async () => {
- return config.doInContext(undefined, async () => {
- await config.createUser()
-
- await syncUsers.run()
-
- const usageDoc = await quotas.getQuotaUsage()
- // default + additional user
- const userCount = 2
- expect(usageDoc.usageQuota.users).toBe(userCount)
- })
- })
-})
diff --git a/packages/server/src/migrations/functions/userEmailViewCasing.ts b/packages/server/src/migrations/functions/userEmailViewCasing.ts
deleted file mode 100644
index 078289cddf..0000000000
--- a/packages/server/src/migrations/functions/userEmailViewCasing.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import { db as dbCore } from "@budibase/backend-core"
-
-/**
- * Date:
- * October 2021
- *
- * Description:
- * Recreate the user email view to include latest changes i.e. lower casing the email address
- */
-
-export const run = async () => {
- await dbCore.createNewUserEmailView()
-}
diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts
deleted file mode 100644
index a66d793142..0000000000
--- a/packages/server/src/migrations/index.ts
+++ /dev/null
@@ -1,115 +0,0 @@
-import { locks, migrations } from "@budibase/backend-core"
-import {
- Migration,
- MigrationOptions,
- MigrationName,
- LockType,
- LockName,
-} from "@budibase/types"
-import env from "../environment"
-
-// migration functions
-import * as userEmailViewCasing from "./functions/userEmailViewCasing"
-import * as syncQuotas from "./functions/syncQuotas"
-import * as appUrls from "./functions/appUrls"
-import * as tableSettings from "./functions/tableSettings"
-import * as backfill from "./functions/backfill"
-/**
- * Populate the migration function and additional configuration from
- * the static migration definitions.
- */
-export const buildMigrations = () => {
- const definitions = migrations.DEFINITIONS
- const serverMigrations: Migration[] = []
-
- for (const definition of definitions) {
- switch (definition.name) {
- case MigrationName.USER_EMAIL_VIEW_CASING: {
- serverMigrations.push({
- ...definition,
- fn: userEmailViewCasing.run,
- })
- break
- }
- case MigrationName.SYNC_QUOTAS: {
- serverMigrations.push({
- ...definition,
- fn: syncQuotas.run,
- })
- break
- }
- case MigrationName.APP_URLS: {
- serverMigrations.push({
- ...definition,
- appOpts: { all: true },
- fn: appUrls.run,
- })
- break
- }
- case MigrationName.EVENT_APP_BACKFILL: {
- serverMigrations.push({
- ...definition,
- appOpts: { all: true },
- fn: backfill.app.run,
- silent: !!env.SELF_HOSTED, // reduce noisy logging
- preventRetry: !!env.SELF_HOSTED, // only ever run once
- })
- break
- }
- case MigrationName.EVENT_GLOBAL_BACKFILL: {
- serverMigrations.push({
- ...definition,
- fn: backfill.global.run,
- silent: !!env.SELF_HOSTED, // reduce noisy logging
- preventRetry: !!env.SELF_HOSTED, // only ever run once
- })
- break
- }
- case MigrationName.EVENT_INSTALLATION_BACKFILL: {
- serverMigrations.push({
- ...definition,
- fn: backfill.installation.run,
- silent: !!env.SELF_HOSTED, // reduce noisy logging
- preventRetry: !!env.SELF_HOSTED, // only ever run once
- })
- break
- }
- case MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS: {
- serverMigrations.push({
- ...definition,
- appOpts: { dev: true },
- fn: tableSettings.run,
- })
- break
- }
- }
- }
-
- return serverMigrations
-}
-
-export const MIGRATIONS = buildMigrations()
-
-export const migrate = async (options?: MigrationOptions) => {
- if (env.SELF_HOSTED) {
- // self host runs migrations on startup
- // make sure only a single instance runs them
- await migrateWithLock(options)
- } else {
- await migrations.runMigrations(MIGRATIONS, options)
- }
-}
-
-const migrateWithLock = async (options?: MigrationOptions) => {
- await locks.doWithLock(
- {
- type: LockType.TRY_ONCE,
- name: LockName.MIGRATIONS,
- ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
- systemLock: true,
- },
- async () => {
- await migrations.runMigrations(MIGRATIONS, options)
- }
- )
-}
diff --git a/packages/server/src/migrations/tests/helpers.ts b/packages/server/src/migrations/tests/helpers.ts
deleted file mode 100644
index 35831a2fd0..0000000000
--- a/packages/server/src/migrations/tests/helpers.ts
+++ /dev/null
@@ -1,40 +0,0 @@
-// Mimic configs test configuration from worker, creation configs directly in database
-
-import * as structures from "./structures"
-import { configs } from "@budibase/backend-core"
-import { Config } from "@budibase/types"
-
-export const saveSettingsConfig = async (globalDb: any) => {
- const config = structures.settings()
- await saveConfig(config, globalDb)
-}
-
-export const saveGoogleConfig = async (globalDb: any) => {
- const config = structures.google()
- await saveConfig(config, globalDb)
-}
-
-export const saveOIDCConfig = async (globalDb: any) => {
- const config = structures.oidc()
- await saveConfig(config, globalDb)
-}
-
-export const saveSmtpConfig = async (globalDb: any) => {
- const config = structures.smtp()
- await saveConfig(config, globalDb)
-}
-
-const saveConfig = async (config: Config, globalDb: any) => {
- config._id = configs.generateConfigID(config.type)
-
- let response
- try {
- response = await globalDb.get(config._id)
- config._rev = response._rev
- await globalDb.put(config)
- } catch (e: any) {
- if (e.status === 404) {
- await globalDb.put(config)
- }
- }
-}
diff --git a/packages/server/src/migrations/tests/index.spec.ts b/packages/server/src/migrations/tests/index.spec.ts
deleted file mode 100644
index 3a23d8f011..0000000000
--- a/packages/server/src/migrations/tests/index.spec.ts
+++ /dev/null
@@ -1,137 +0,0 @@
-import {
- events,
- migrations,
- tenancy,
- DocumentType,
- context,
-} from "@budibase/backend-core"
-import TestConfig from "../../tests/utilities/TestConfiguration"
-import * as structures from "../../tests/utilities/structures"
-import { MIGRATIONS } from "../"
-import * as helpers from "./helpers"
-
-import tk from "timekeeper"
-import { View } from "@budibase/types"
-
-const timestamp = new Date().toISOString()
-tk.freeze(timestamp)
-
-const clearMigrations = async () => {
- const dbs = [context.getDevAppDB(), context.getProdAppDB()]
- for (const db of dbs) {
- const doc = await db.get<any>(DocumentType.MIGRATIONS)
- const newDoc = { _id: doc._id, _rev: doc._rev }
- await db.put(newDoc)
- }
-}
-
-describe("migrations", () => {
- const config = new TestConfig()
-
- beforeAll(async () => {
- await config.init()
- })
-
- afterAll(() => {
- config.end()
- })
-
- describe("backfill", () => {
- it("runs app db migration", async () => {
- await config.doInContext(undefined, async () => {
- await clearMigrations()
- await config.createAutomation()
- await config.createAutomation(structures.newAutomation())
- await config.createDatasource()
- await config.createDatasource()
- await config.createLayout()
- await config.createQuery()
- await config.createQuery()
- await config.createRole()
- await config.createRole()
- await config.createTable()
- await config.createLegacyView()
- await config.createTable()
- await config.createLegacyView(
- structures.view(config.table!._id!) as View
- )
- await config.createScreen()
- await config.createScreen()
-
- jest.clearAllMocks()
- const migration = MIGRATIONS.filter(
- m => m.name === "event_app_backfill"
- )[0]
- await migrations.runMigration(migration)
-
- expect(events.app.created).toHaveBeenCalledTimes(1)
- expect(events.app.published).toHaveBeenCalledTimes(1)
- expect(events.automation.created).toHaveBeenCalledTimes(2)
- expect(events.automation.stepCreated).toHaveBeenCalledTimes(1)
- expect(events.datasource.created).toHaveBeenCalledTimes(2)
- expect(events.layout.created).toHaveBeenCalledTimes(1)
- expect(events.query.created).toHaveBeenCalledTimes(2)
- expect(events.role.created).toHaveBeenCalledTimes(3) // created roles + admin (created on table creation)
- expect(events.table.created).toHaveBeenCalledTimes(3)
- expect(events.backfill.appSucceeded).toHaveBeenCalledTimes(2)
-
- // to make sure caching is working as expected
- expect(
- events.processors.analyticsProcessor.processEvent
- ).toHaveBeenCalledTimes(20) // Addition of of the events above
- })
- })
- })
-
- it("runs global db migration", async () => {
- await config.doInContext(undefined, async () => {
- await clearMigrations()
- const appId = config.getProdAppId()
- const roles = { [appId]: "role_12345" }
- await config.createUser({
- builder: { global: false },
- admin: { global: true },
- roles,
- }) // admin only
- await config.createUser({
- builder: { global: false },
- admin: { global: false },
- roles,
- }) // non admin non builder
- await config.createTable()
- await config.createRow()
- await config.createRow()
-
- const db = tenancy.getGlobalDB()
- await helpers.saveGoogleConfig(db)
- await helpers.saveOIDCConfig(db)
- await helpers.saveSettingsConfig(db)
- await helpers.saveSmtpConfig(db)
-
- jest.clearAllMocks()
- const migration = MIGRATIONS.filter(
- m => m.name === "event_global_backfill"
- )[0]
- await migrations.runMigration(migration)
-
- expect(events.user.created).toHaveBeenCalledTimes(3)
- expect(events.role.assigned).toHaveBeenCalledTimes(2)
- expect(events.user.permissionBuilderAssigned).toHaveBeenCalledTimes(1) // default test user
- expect(events.user.permissionAdminAssigned).toHaveBeenCalledTimes(1) // admin from above
- expect(events.rows.created).toHaveBeenCalledTimes(1)
- expect(events.rows.created).toHaveBeenCalledWith(2, timestamp)
- expect(events.email.SMTPCreated).toHaveBeenCalledTimes(1)
- expect(events.auth.SSOCreated).toHaveBeenCalledTimes(2)
- expect(events.auth.SSOActivated).toHaveBeenCalledTimes(2)
- expect(events.org.logoUpdated).toHaveBeenCalledTimes(1)
- expect(events.org.nameUpdated).toHaveBeenCalledTimes(1)
- expect(events.org.platformURLUpdated).toHaveBeenCalledTimes(1)
- expect(events.backfill.tenantSucceeded).toHaveBeenCalledTimes(1)
-
- // to make sure caching is working as expected
- expect(
- events.processors.analyticsProcessor.processEvent
- ).toHaveBeenCalledTimes(19)
- })
- })
-})
diff --git a/packages/server/src/migrations/tests/structures.ts b/packages/server/src/migrations/tests/structures.ts
deleted file mode 100644
index e2113e6a7c..0000000000
--- a/packages/server/src/migrations/tests/structures.ts
+++ /dev/null
@@ -1,67 +0,0 @@
-import { utils } from "@budibase/backend-core"
-import {
- SMTPConfig,
- OIDCConfig,
- GoogleConfig,
- SettingsConfig,
- ConfigType,
-} from "@budibase/types"
-
-export const oidc = (conf?: OIDCConfig): OIDCConfig => {
- return {
- type: ConfigType.OIDC,
- config: {
- configs: [
- {
- configUrl: "http://someconfigurl",
- clientID: "clientId",
- clientSecret: "clientSecret",
- logo: "Microsoft",
- name: "Active Directory",
- uuid: utils.newid(),
- activated: true,
- scopes: [],
- ...conf,
- },
- ],
- },
- }
-}
-
-export const google = (conf?: GoogleConfig): GoogleConfig => {
- return {
- type: ConfigType.GOOGLE,
- config: {
- clientID: "clientId",
- clientSecret: "clientSecret",
- activated: true,
- ...conf,
- },
- }
-}
-
-export const smtp = (conf?: SMTPConfig): SMTPConfig => {
- return {
- type: ConfigType.SMTP,
- config: {
- port: 12345,
- host: "smtptesthost.com",
- from: "testfrom@example.com",
- subject: "Hello!",
- secure: false,
- ...conf,
- },
- }
-}
-
-export const settings = (conf?: SettingsConfig): SettingsConfig => {
- return {
- type: ConfigType.SETTINGS,
- config: {
- platformUrl: "http://mycustomdomain.com",
- logoUrl: "http://mylogourl,com",
- company: "mycompany",
- ...conf,
- },
- }
-}
diff --git a/packages/server/src/startup/index.ts b/packages/server/src/startup/index.ts
index edca64db7d..54e982ae56 100644
--- a/packages/server/src/startup/index.ts
+++ b/packages/server/src/startup/index.ts
@@ -15,7 +15,6 @@ import { watch } from "../watch"
import * as automations from "../automations"
import * as fileSystem from "../utilities/fileSystem"
import { default as eventEmitter, init as eventInit } from "../events"
-import * as migrations from "../migrations"
import * as bullboard from "../automations/bullboard"
import * as appMigrations from "../appMigrations/queue"
import * as pro from "@budibase/pro"
@@ -106,18 +105,6 @@ export async function startup(
initialiseWebsockets(app, server)
}
- // run migrations on startup if not done via http
- // not recommended in a clustered environment
- if (!env.HTTP_MIGRATIONS && !env.isTest()) {
- console.log("Running migrations")
- try {
- await migrations.migrate()
- } catch (e) {
- logging.logAlert("Error performing migrations. Exiting.", e)
- shutdown(server)
- }
- }
-
// monitor plugin directory if required
if (
env.SELF_HOSTED &&
diff --git a/packages/types/src/api/web/global/oldMigration.ts b/packages/types/src/api/web/global/oldMigration.ts
index 812ee1e593..f0da6c52f3 100644
--- a/packages/types/src/api/web/global/oldMigration.ts
+++ b/packages/types/src/api/web/global/oldMigration.ts
@@ -1,12 +1,3 @@
-import { Migration, MigrationOptions } from "../../../sdk"
-
-export interface RunOldMigrationRequest extends MigrationOptions {}
-export interface RuneOldMigrationResponse {
- message: string
-}
-
-export type FetchOldMigrationResponse = Migration[]
-
-export interface GetOldMigrationStatus {
+export interface GetMigrationStatus {
migrated: boolean
}
diff --git a/packages/types/src/api/web/system/index.ts b/packages/types/src/api/web/system/index.ts
index 9b03ddd438..18ed533e9d 100644
--- a/packages/types/src/api/web/system/index.ts
+++ b/packages/types/src/api/web/system/index.ts
@@ -3,6 +3,5 @@ export * from "./status"
export * from "./ops"
export * from "./account"
export * from "./log"
-export * from "./migration"
export * from "./restore"
export * from "./tenant"
diff --git a/packages/types/src/api/web/system/migration.ts b/packages/types/src/api/web/system/migration.ts
deleted file mode 100644
index a18112744c..0000000000
--- a/packages/types/src/api/web/system/migration.ts
+++ /dev/null
@@ -1,8 +0,0 @@
-import { MigrationDefinition, MigrationOptions } from "../../../sdk"
-
-export interface RunGlobalMigrationRequest extends MigrationOptions {}
-export interface RunGlobalMigrationResponse {
- message: string
-}
-
-export type FetchMigrationDefinitionsResponse = MigrationDefinition[]
diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts
index 86eb5b1a24..eb9e23b3d1 100644
--- a/packages/types/src/sdk/index.ts
+++ b/packages/types/src/sdk/index.ts
@@ -4,7 +4,6 @@ export * from "./hosting"
export * from "./context"
export * from "./events"
export * from "./licensing"
-export * from "./migrations"
export * from "./datasources"
export * from "./search"
export * from "./koa"
diff --git a/packages/types/src/sdk/migrations.ts b/packages/types/src/sdk/migrations.ts
deleted file mode 100644
index 6db0c85879..0000000000
--- a/packages/types/src/sdk/migrations.ts
+++ /dev/null
@@ -1,57 +0,0 @@
-import { Database } from "./db"
-
-export interface Migration extends MigrationDefinition {
- appOpts?: object
- fn: (db: Database) => Promise<void>
- silent?: boolean
- preventRetry?: boolean
-}
-
-export enum MigrationType {
- // run once per tenant, recorded in global db, global db is provided as an argument
- GLOBAL = "global",
- // run per app, recorded in each app db, app db is provided as an argument
- APP = "app",
- // run once, recorded in global info db, global info db is provided as an argument
- INSTALLATION = "installation",
-}
-
-export interface MigrationNoOpOptions {
- type: MigrationType
- tenantId: string
- appId?: string
-}
-
-/**
- * e.g.
- * {
- * tenantIds: ['bb'],
- * force: {
- * global: ['quota_1']
- * }
- * }
- */
-export interface MigrationOptions {
- tenantIds?: string[]
- force?: {
- [type: string]: string[]
- }
- noOp?: MigrationNoOpOptions
-}
-
-export enum MigrationName {
- USER_EMAIL_VIEW_CASING = "user_email_view_casing",
- APP_URLS = "app_urls",
- EVENT_APP_BACKFILL = "event_app_backfill",
- EVENT_GLOBAL_BACKFILL = "event_global_backfill",
- EVENT_INSTALLATION_BACKFILL = "event_installation_backfill",
- GLOBAL_INFO_SYNC_USERS = "global_info_sync_users",
- TABLE_SETTINGS_LINKS_TO_ACTIONS = "table_settings_links_to_actions",
- // increment this number to re-activate this migration
- SYNC_QUOTAS = "sync_quotas_2",
-}
-
-export interface MigrationDefinition {
- type: MigrationType
- name: MigrationName
-}
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index 0bcdadfefc..e36c45a3ba 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -28,7 +28,6 @@ import {
LockType,
LookupAccountHolderResponse,
LookupTenantUserResponse,
- MigrationType,
PlatformUserByEmail,
SaveUserResponse,
SearchUsersRequest,
@@ -45,7 +44,6 @@ import {
cache,
ErrorCode,
events,
- migrations,
platform,
tenancy,
db,
@@ -187,10 +185,6 @@ export const adminUser = async (
if (env.MULTI_TENANCY) {
// store the new tenant record in the platform db
await platform.tenants.addTenant(tenantId)
- await migrations.backPopulateMigrations({
- type: MigrationType.GLOBAL,
- tenantId,
- })
}
await tenancy.doInTenant(tenantId, async () => {
diff --git a/packages/worker/src/api/controllers/system/migrations.ts b/packages/worker/src/api/controllers/system/migrations.ts
deleted file mode 100644
index fc253d839d..0000000000
--- a/packages/worker/src/api/controllers/system/migrations.ts
+++ /dev/null
@@ -1,23 +0,0 @@
-import {
- FetchMigrationDefinitionsResponse,
- RunGlobalMigrationRequest,
- RunGlobalMigrationResponse,
- UserCtx,
-} from "@budibase/types"
-
-const { migrate, MIGRATIONS } = require("../../../migrations")
-
-export const runMigrations = async (
- ctx: UserCtx<RunGlobalMigrationRequest, RunGlobalMigrationResponse>
-) => {
- const options = ctx.request.body
- // don't await as can take a while, just return
- migrate(options)
- ctx.body = { message: "Migration started." }
-}
-
-export const fetchDefinitions = async (
- ctx: UserCtx<void, FetchMigrationDefinitionsResponse>
-) => {
- ctx.body = MIGRATIONS
-}
diff --git a/packages/worker/src/api/routes/index.ts b/packages/worker/src/api/routes/index.ts
index 741026543c..d4ddb41522 100644
--- a/packages/worker/src/api/routes/index.ts
+++ b/packages/worker/src/api/routes/index.ts
@@ -12,7 +12,6 @@ import tenantsRoutes from "./system/tenants"
import statusRoutes from "./system/status"
import selfRoutes from "./global/self"
import licenseRoutes from "./global/license"
-import migrationRoutes from "./system/migrations"
import accountRoutes from "./system/accounts"
import restoreRoutes from "./system/restore"
import systemLogRoutes from "./system/logs"
@@ -34,7 +33,6 @@ export const routes: Router[] = [
licenseRoutes,
pro.groups,
pro.auditLogs,
- migrationRoutes,
accountRoutes,
restoreRoutes,
eventRoutes,
diff --git a/packages/worker/src/api/routes/system/migrations.ts b/packages/worker/src/api/routes/system/migrations.ts
deleted file mode 100644
index a8189b5a91..0000000000
--- a/packages/worker/src/api/routes/system/migrations.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-import Router from "@koa/router"
-import * as migrationsController from "../../controllers/system/migrations"
-import { auth } from "@budibase/backend-core"
-
-const router: Router = new Router()
-
-router
- .post(
- "/api/system/migrations/run",
- auth.internalApi,
- migrationsController.runMigrations
- )
- .get(
- "/api/system/migrations/definitions",
- auth.internalApi,
- migrationsController.fetchDefinitions
- )
-
-export default router
diff --git a/packages/worker/src/migrations/index.ts b/packages/worker/src/migrations/index.ts
deleted file mode 100644
index 642fbeb54e..0000000000
--- a/packages/worker/src/migrations/index.ts
+++ /dev/null
@@ -1,62 +0,0 @@
-import { migrations, locks } from "@budibase/backend-core"
-import {
- Migration,
- MigrationOptions,
- MigrationName,
- LockType,
- LockName,
-} from "@budibase/types"
-import env from "../environment"
-
-// migration functions
-import * as syncUserInfo from "./functions/globalInfoSyncUsers"
-
-/**
- * Populate the migration function and additional configuration from
- * the static migration definitions.
- */
-export const buildMigrations = () => {
- const definitions = migrations.DEFINITIONS
- const workerMigrations: Migration[] = []
-
- for (const definition of definitions) {
- switch (definition.name) {
- case MigrationName.GLOBAL_INFO_SYNC_USERS: {
- // only needed in cloud
- if (!env.SELF_HOSTED) {
- workerMigrations.push({
- ...definition,
- fn: syncUserInfo.run,
- })
- }
- break
- }
- }
- }
-
- return workerMigrations
-}
-
-export const MIGRATIONS = buildMigrations()
-
-export const migrate = async (options?: MigrationOptions) => {
- if (env.SELF_HOSTED) {
- await migrateWithLock(options)
- } else {
- await migrations.runMigrations(MIGRATIONS, options)
- }
-}
-
-const migrateWithLock = async (options?: MigrationOptions) => {
- await locks.doWithLock(
- {
- type: LockType.TRY_ONCE,
- name: LockName.MIGRATIONS,
- ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
- systemLock: true,
- },
- async () => {
- await migrations.runMigrations(MIGRATIONS, options)
- }
- )
-}