MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15373 from Budibase/remove-dead-code 

Remove old migration system.
This commit is contained in:
Sam Rose 2025-01-20 14:48:05 +00:00 committed by GitHub
parent 7cededcd76 b8e5fce724
commit b28e4724b1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
61 changed files with 6 additions and 2407 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/backend-core/src/index.ts
View File

@ -1,6 +1,5 @@
export * as configs from "./configs"
export * as events from "./events"
export * as migrations from "./migrations"
export * as users from "./users"
export * as userUtils from "./users/utils"
export * as roles from "./security/roles"

40
packages/backend-core/src/migrations/definitions.ts
View File

@ -1,40 +0,0 @@
import {
MigrationType,
MigrationName,
MigrationDefinition,
} from "@budibase/types"
export const DEFINITIONS: MigrationDefinition[] = [
{
type: MigrationType.GLOBAL,
name: MigrationName.USER_EMAIL_VIEW_CASING,
},
{
type: MigrationType.GLOBAL,
name: MigrationName.SYNC_QUOTAS,
},
{
type: MigrationType.APP,
name: MigrationName.APP_URLS,
},
{
type: MigrationType.APP,
name: MigrationName.EVENT_APP_BACKFILL,
},
{
type: MigrationType.APP,
name: MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS,
},
{
type: MigrationType.GLOBAL,
name: MigrationName.EVENT_GLOBAL_BACKFILL,
},
{
type: MigrationType.INSTALLATION,
name: MigrationName.EVENT_INSTALLATION_BACKFILL,
},
{
type: MigrationType.GLOBAL,
name: MigrationName.GLOBAL_INFO_SYNC_USERS,
},
]

2
packages/backend-core/src/migrations/index.ts
View File

@ -1,2 +0,0 @@
export * from "./migrations"
export * from "./definitions"

186
packages/backend-core/src/migrations/migrations.ts
View File

@ -1,186 +0,0 @@
import { DEFAULT_TENANT_ID } from "../constants"
import {
DocumentType,
StaticDatabases,
getAllApps,
getGlobalDBName,
getDB,
} from "../db"
import environment from "../environment"
import * as platform from "../platform"
import * as context from "../context"
import { DEFINITIONS } from "."
import {
Migration,
MigrationOptions,
MigrationType,
MigrationNoOpOptions,
App,
} from "@budibase/types"
export const getMigrationsDoc = async (db: any) => {
// get the migrations doc
try {
return await db.get(DocumentType.MIGRATIONS)
} catch (err: any) {
if (err.status && err.status === 404) {
return { _id: DocumentType.MIGRATIONS }
} else {
throw err
}
}
}
export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => {
// filter migrations to the type and populate a no-op migration
const migrations: Migration[] = DEFINITIONS.filter(
def => def.type === opts.type
).map(d => ({ ...d, fn: async () => {} }))
await runMigrations(migrations, { noOp: opts })
}
export const runMigration = async (
migration: Migration,
options: MigrationOptions = {}
) => {
const migrationType = migration.type
const migrationName = migration.name
const silent = migration.silent
const log = (message: string) => {
if (!silent) {
console.log(message)
}
}
// get the db to store the migration in
let dbNames: string[]
if (migrationType === MigrationType.GLOBAL) {
dbNames = [getGlobalDBName()]
} else if (migrationType === MigrationType.APP) {
if (options.noOp) {
if (!options.noOp.appId) {
throw new Error("appId is required for noOp app migration")
}
dbNames = [options.noOp.appId]
} else {
const apps = (await getAllApps(migration.appOpts)) as App[]
dbNames = apps.map(app => app.appId)
}
} else if (migrationType === MigrationType.INSTALLATION) {
dbNames = [StaticDatabases.PLATFORM_INFO.name]
} else {
throw new Error(`Unrecognised migration type [${migrationType}]`)
}
const length = dbNames.length
let count = 0
// run the migration against each db
for (const dbName of dbNames) {
count++
const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
const db = getDB(dbName)
try {
const doc = await getMigrationsDoc(db)
// the migration has already been run
if (doc[migrationName]) {
// check for force
if (
options.force &&
options.force[migrationType] &&
options.force[migrationType].includes(migrationName)
) {
log(`[Migration: ${migrationName}] [DB: ${dbName}] Forcing`)
} else {
// no force, exit
return
}
}
// check if the migration is not a no-op
if (!options.noOp) {
log(
`[Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
)
if (migration.preventRetry) {
// eagerly set the completion date
// so that we never run this migration twice even upon failure
doc[migrationName] = Date.now()
const response = await db.put(doc)
doc._rev = response.rev
}
// run the migration
if (migrationType === MigrationType.APP) {
await context.doInAppContext(db.name, async () => {
await migration.fn(db)
})
} else {
await migration.fn(db)
}
log(`[Migration: ${migrationName}] [DB: ${dbName}] Complete`)
}
// mark as complete
doc[migrationName] = Date.now()
await db.put(doc)
} catch (err) {
console.error(
`[Migration: ${migrationName}] [DB: ${dbName}] Error: `,
err
)
throw err
}
}
}
export const runMigrations = async (
migrations: Migration[],
options: MigrationOptions = {}
) => {
let tenantIds
if (environment.MULTI_TENANCY) {
if (options.noOp) {
tenantIds = [options.noOp.tenantId]
} else if (!options.tenantIds || !options.tenantIds.length) {
// run for all tenants
tenantIds = await platform.tenants.getTenantIds()
} else {
tenantIds = options.tenantIds
}
} else {
// single tenancy
tenantIds = [DEFAULT_TENANT_ID]
}
if (tenantIds.length > 1) {
console.log(`Checking migrations for ${tenantIds.length} tenants`)
} else {
console.log("Checking migrations")
}
let count = 0
// for all tenants
for (const tenantId of tenantIds) {
count++
if (tenantIds.length > 1) {
console.log(`Progress [${count}/${tenantIds.length}]`)
}
// for all migrations
for (const migration of migrations) {
// run the migration
await context.doInTenant(
tenantId,
async () => await runMigration(migration, options)
)
}
}
console.log("Migrations complete")
}

11
packages/backend-core/src/migrations/tests/__snapshots__/migrations.spec.ts.snap
View File

@ -1,11 +0,0 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`migrations should match snapshot 1`] = `
{
"_id": "migrations",
"_rev": "1-2f64479842a0513aa8b97f356b0b9127",
"createdAt": "2020-01-01T00:00:00.000Z",
"test": 1577836800000,
"updatedAt": "2020-01-01T00:00:00.000Z",
}
`;

64
packages/backend-core/src/migrations/tests/migrations.spec.ts
View File

@ -1,64 +0,0 @@
import { testEnv, DBTestConfiguration } from "../../../tests/extra"
import * as migrations from "../index"
import * as context from "../../context"
import { MigrationType } from "@budibase/types"
testEnv.multiTenant()
describe("migrations", () => {
const config = new DBTestConfiguration()
const migrationFunction = jest.fn()
const MIGRATIONS = [
{
type: MigrationType.GLOBAL,
name: "test" as any,
fn: migrationFunction,
},
]
beforeEach(() => {
config.newTenant()
})
afterEach(async () => {
jest.clearAllMocks()
})
const migrate = () => {
return migrations.runMigrations(MIGRATIONS, {
tenantIds: [config.tenantId],
})
}
it("should run a new migration", async () => {
await config.doInTenant(async () => {
await migrate()
expect(migrationFunction).toHaveBeenCalled()
const db = context.getGlobalDB()
const doc = await migrations.getMigrationsDoc(db)
expect(doc.test).toBeDefined()
})
})
it("should match snapshot", async () => {
await config.doInTenant(async () => {
await migrate()
const doc = await migrations.getMigrationsDoc(context.getGlobalDB())
expect(doc).toMatchSnapshot()
})
})
it("should skip a previously run migration", async () => {
await config.doInTenant(async () => {
const db = context.getGlobalDB()
await migrate()
const previousDoc = await migrations.getMigrationsDoc(db)
await migrate()
const currentDoc = await migrations.getMigrationsDoc(db)
expect(migrationFunction).toHaveBeenCalledTimes(1)
expect(currentDoc.test).toBe(previousDoc.test)
})
})
})

4
packages/frontend-core/src/api/migrations.ts
View File

@ -1,8 +1,8 @@
import { GetOldMigrationStatus } from "@budibase/types"
import { GetMigrationStatus } from "@budibase/types"
import { BaseAPIClient } from "./types"
export interface MigrationEndpoints {
getMigrationStatus: () => Promise<GetOldMigrationStatus>
getMigrationStatus: () => Promise<GetMigrationStatus>
}
export const buildMigrationEndpoints = (

1
packages/server/scripts/dev/manage.js
View File

@ -43,7 +43,6 @@ async function init() {
BB_ADMIN_USER_EMAIL: "",
BB_ADMIN_USER_PASSWORD: "",
PLUGINS_DIR: "",
HTTP_MIGRATIONS: "0",
HTTP_LOGGING: "0",
VERSION: "0.0.0+local",
PASSWORD_MIN_LENGTH: "1",

9
packages/server/src/api/controllers/application.ts
View File

@ -27,7 +27,6 @@ import {
env as envCore,
ErrorCode,
events,
migrations,
objectStore,
roles,
tenancy,
@ -43,7 +42,6 @@ import { groups, licensing, quotas } from "@budibase/pro"
import {
App,
Layout,
MigrationType,
PlanType,
Screen,
UserCtx,
@ -488,13 +486,6 @@ async function creationEvents(request: BBRequest<CreateAppRequest>, app: App) {
}
async function appPostCreate(ctx: UserCtx<CreateAppRequest, App>, app: App) {
const tenantId = tenancy.getTenantId()
await migrations.backPopulateMigrations({
type: MigrationType.APP,
tenantId,
appId: app.appId,
})
await creationEvents(ctx.request, app)
// app import, template creation and duplication

28
packages/server/src/api/controllers/migrations.ts
View File

@ -1,35 +1,11 @@
import { context } from "@budibase/backend-core"
import { migrate as migrationImpl, MIGRATIONS } from "../../migrations"
import {
Ctx,
FetchOldMigrationResponse,
GetOldMigrationStatus,
RuneOldMigrationResponse,
RunOldMigrationRequest,
} from "@budibase/types"
import { Ctx, GetMigrationStatus } from "@budibase/types"
import {
getAppMigrationVersion,
getLatestEnabledMigrationId,
} from "../../appMigrations"
export async function migrate(
ctx: Ctx<RunOldMigrationRequest, RuneOldMigrationResponse>
) {
const options = ctx.request.body
// don't await as can take a while, just return
migrationImpl(options)
ctx.body = { message: "Migration started." }
}
export async function fetchDefinitions(
ctx: Ctx<void, FetchOldMigrationResponse>
) {
ctx.body = MIGRATIONS
}
export async function getMigrationStatus(
ctx: Ctx<void, GetOldMigrationStatus>
) {
export async function getMigrationStatus(ctx: Ctx<void, GetMigrationStatus>) {
const appId = context.getAppId()
if (!appId) {

10
packages/server/src/api/routes/migrations.ts
View File

@ -1,16 +1,8 @@
import Router from "@koa/router"
import * as migrationsController from "../controllers/migrations"
import { auth } from "@budibase/backend-core"
const router: Router = new Router()
router
.post("/api/migrations/run", auth.internalApi, migrationsController.migrate)
.get(
"/api/migrations/definitions",
auth.internalApi,
migrationsController.fetchDefinitions
)
.get("/api/migrations/status", migrationsController.getMigrationStatus)
router.get("/api/migrations/status", migrationsController.getMigrationStatus)
export default router

1
packages/server/src/environment.ts
View File

@ -54,7 +54,6 @@ const environment = {
REDIS_URL: process.env.REDIS_URL,
REDIS_PASSWORD: process.env.REDIS_PASSWORD,
REDIS_CLUSTERED: process.env.REDIS_CLUSTERED,
HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
CLUSTER_MODE: process.env.CLUSTER_MODE,
API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

27
packages/server/src/migrations/functions/appUrls.ts
View File

@ -1,27 +0,0 @@
import { db as dbCore } from "@budibase/backend-core"
import sdk from "../../sdk"
/**
* Date:
* January 2022
*
* Description:
* Add the url to the app metadata if it doesn't exist
*/
export const run = async (appDb: any) => {
let metadata
try {
metadata = await appDb.get(dbCore.DocumentType.APP_METADATA)
} catch (e) {
// sometimes the metadata document doesn't exist
// exit early instead of failing the migration
console.error("Error retrieving app metadata. Skipping", e)
return
}
if (!metadata.url) {
metadata.url = sdk.applications.getAppUrl({ name: metadata.name })
console.log(`Adding url to app: ${metadata.url}`)
await appDb.put(metadata)
}
}

149
packages/server/src/migrations/functions/backfill/app.ts
View File

@ -1,149 +0,0 @@
import * as automations from "./app/automations"
import * as datasources from "./app/datasources"
import * as layouts from "./app/layouts"
import * as queries from "./app/queries"
import * as roles from "./app/roles"
import * as tables from "./app/tables"
import * as screens from "./app/screens"
import * as global from "./global"
import { App, AppBackfillSucceededEvent, Event } from "@budibase/types"
import { db as dbUtils, events } from "@budibase/backend-core"
import env from "../../../environment"
import { DEFAULT_TIMESTAMP } from "."
const failGraceful = env.SELF_HOSTED && !env.isDev()
const handleError = (e: any, errors?: any) => {
if (failGraceful) {
if (errors) {
errors.push(e)
}
return
}
console.trace(e)
throw e
}
const EVENTS = [
Event.AUTOMATION_CREATED,
Event.AUTOMATION_STEP_CREATED,
Event.DATASOURCE_CREATED,
Event.LAYOUT_CREATED,
Event.QUERY_CREATED,
Event.ROLE_CREATED,
Event.SCREEN_CREATED,
Event.TABLE_CREATED,
Event.VIEW_CREATED,
Event.VIEW_CALCULATION_CREATED,
Event.VIEW_FILTER_CREATED,
Event.APP_PUBLISHED,
Event.APP_CREATED,
]
/**
* Date:
* May 2022
*
* Description:
* Backfill app events.
*/
export const run = async (appDb: any) => {
try {
if (await global.isComplete()) {
// make sure new apps aren't backfilled
// return if the global migration for this tenant is complete
// which runs after the app migrations
return
}
// tell the event pipeline to start caching
// events for this tenant
await events.backfillCache.start(EVENTS)
let timestamp: string | number = DEFAULT_TIMESTAMP
const app: App = await appDb.get(dbUtils.DocumentType.APP_METADATA)
if (app.createdAt) {
timestamp = app.createdAt as string
}
if (dbUtils.isProdAppID(app.appId)) {
await events.app.published(app, timestamp)
}
const totals: any = {}
const errors: any = []
if (dbUtils.isDevAppID(app.appId)) {
await events.app.created(app, timestamp)
try {
totals.automations = await automations.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.datasources = await datasources.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.layouts = await layouts.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.queries = await queries.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.roles = await roles.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.screens = await screens.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.tables = await tables.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
}
const properties: AppBackfillSucceededEvent = {
appId: app.appId,
automations: totals.automations,
datasources: totals.datasources,
layouts: totals.layouts,
queries: totals.queries,
roles: totals.roles,
tables: totals.tables,
screens: totals.screens,
}
if (errors.length) {
properties.errors = errors.map((e: any) =>
JSON.stringify(e, Object.getOwnPropertyNames(e))
)
properties.errorCount = errors.length
} else {
properties.errorCount = 0
}
await events.backfill.appSucceeded(properties)
// tell the event pipeline to stop caching events for this tenant
await events.backfillCache.end()
} catch (e) {
handleError(e)
await events.backfill.appFailed(e)
}
}

26
packages/server/src/migrations/functions/backfill/app/automations.ts
View File

@ -1,26 +0,0 @@
import { events } from "@budibase/backend-core"
import { getAutomationParams } from "../../../../db/utils"
import { Automation } from "@budibase/types"
const getAutomations = async (appDb: any): Promise<Automation[]> => {
const response = await appDb.allDocs(
getAutomationParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const automations = await getAutomations(appDb)
for (const automation of automations) {
await events.automation.created(automation, timestamp)
for (const step of automation.definition.steps) {
await events.automation.stepCreated(automation, step, timestamp)
}
}
return automations.length
}

22
packages/server/src/migrations/functions/backfill/app/datasources.ts
View File

@ -1,22 +0,0 @@
import { events } from "@budibase/backend-core"
import { getDatasourceParams } from "../../../../db/utils"
import { Datasource } from "@budibase/types"
const getDatasources = async (appDb: any): Promise<Datasource[]> => {
const response = await appDb.allDocs(
getDatasourceParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const datasources: Datasource[] = await getDatasources(appDb)
for (const datasource of datasources) {
await events.datasource.created(datasource, timestamp)
}
return datasources.length
}

29
packages/server/src/migrations/functions/backfill/app/layouts.ts
View File

@ -1,29 +0,0 @@
import { events } from "@budibase/backend-core"
import { getLayoutParams } from "../../../../db/utils"
import { Layout } from "@budibase/types"
const getLayouts = async (appDb: any): Promise<Layout[]> => {
const response = await appDb.allDocs(
getLayoutParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const layouts: Layout[] = await getLayouts(appDb)
for (const layout of layouts) {
// exclude default layouts
if (
layout._id === "layout_private_master" ||
layout._id === "layout_public_master"
) {
continue
}
await events.layout.created(layout, timestamp)
}
return layouts.length
}

47
packages/server/src/migrations/functions/backfill/app/queries.ts
View File

@ -1,47 +0,0 @@
import { events } from "@budibase/backend-core"
import { getQueryParams } from "../../../../db/utils"
import { Query, Datasource, SourceName } from "@budibase/types"
const getQueries = async (appDb: any): Promise<Query[]> => {
const response = await appDb.allDocs(
getQueryParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
const getDatasource = async (
appDb: any,
datasourceId: string
): Promise<Datasource> => {
return appDb.get(datasourceId)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const queries: Query[] = await getQueries(appDb)
for (const query of queries) {
let datasource: Datasource
try {
datasource = await getDatasource(appDb, query.datasourceId)
} catch (e: any) {
// handle known bug where a datasource has been deleted
// and the query has not
if (e.status === 404) {
datasource = {
type: "unknown",
_id: query.datasourceId,
source: "unknown" as SourceName,
}
} else {
throw e
}
}
await events.query.created(datasource, query, timestamp)
}
return queries.length
}

22
packages/server/src/migrations/functions/backfill/app/roles.ts
View File

@ -1,22 +0,0 @@
import { events } from "@budibase/backend-core"
import { getRoleParams } from "../../../../db/utils"
import { Role } from "@budibase/types"
const getRoles = async (appDb: any): Promise<Role[]> => {
const response = await appDb.allDocs(
getRoleParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const roles = await getRoles(appDb)
for (const role of roles) {
await events.role.created(role, timestamp)
}
return roles.length
}

22
packages/server/src/migrations/functions/backfill/app/screens.ts
View File

@ -1,22 +0,0 @@
import { events } from "@budibase/backend-core"
import { getScreenParams } from "../../../../db/utils"
import { Screen } from "@budibase/types"
const getScreens = async (appDb: any): Promise<Screen[]> => {
const response = await appDb.allDocs(
getScreenParams(null, {
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (appDb: any, timestamp: string | number) => {
const screens = await getScreens(appDb)
for (const screen of screens) {
await events.screen.created(screen, timestamp)
}
return screens.length
}

13
packages/server/src/migrations/functions/backfill/app/tables.ts
View File

@ -1,13 +0,0 @@
import { events } from "@budibase/backend-core"
import { Database } from "@budibase/types"
import sdk from "../../../../sdk"
export const backfill = async (appDb: Database, timestamp: string | number) => {
const tables = await sdk.tables.getAllInternalTables(appDb)
for (const table of tables) {
await events.table.created(table, timestamp)
}
return tables.length
}

214
packages/server/src/migrations/functions/backfill/global.ts
View File

@ -1,214 +0,0 @@
import * as users from "./global/users"
import * as configs from "./global/configs"
import * as quotas from "./global/quotas"
import {
tenancy,
events,
migrations,
accounts,
db as dbUtils,
} from "@budibase/backend-core"
import {
App,
CloudAccount,
Event,
Hosting,
QuotaUsage,
TenantBackfillSucceededEvent,
User,
} from "@budibase/types"
import env from "../../../environment"
import { DEFAULT_TIMESTAMP } from "."
const failGraceful = env.SELF_HOSTED && !env.isDev()
const handleError = (e: any, errors?: any) => {
if (failGraceful) {
if (errors) {
errors.push(e)
}
return
}
throw e
}
const formatUsage = (usage: QuotaUsage) => {
let maxAutomations = 0
let maxQueries = 0
let rows = 0
if (usage) {
if (usage.usageQuota) {
rows = usage.usageQuota.rows
}
if (usage.monthly) {
for (const value of Object.values(usage.monthly)) {
if (value.automations > maxAutomations) {
maxAutomations = value.automations
}
if (value.queries > maxQueries) {
maxQueries = value.queries
}
}
}
}
return {
maxAutomations,
maxQueries,
rows,
}
}
const EVENTS = [
Event.EMAIL_SMTP_CREATED,
Event.AUTH_SSO_CREATED,
Event.AUTH_SSO_ACTIVATED,
Event.ORG_NAME_UPDATED,
Event.ORG_LOGO_UPDATED,
Event.ORG_PLATFORM_URL_UPDATED,
Event.USER_CREATED,
Event.USER_PERMISSION_ADMIN_ASSIGNED,
Event.USER_PERMISSION_BUILDER_ASSIGNED,
Event.ROLE_ASSIGNED,
Event.ROWS_CREATED,
Event.QUERIES_RUN,
Event.AUTOMATIONS_RUN,
]
/**
* Date:
* May 2022
*
* Description:
* Backfill global events.
*/
export const run = async (db: any) => {
try {
const tenantId = tenancy.getTenantId()
let timestamp: string | number = DEFAULT_TIMESTAMP
const totals: any = {}
const errors: any = []
let allUsers: User[] = []
try {
allUsers = await users.getUsers(db)
} catch (e: any) {
handleError(e, errors)
}
if (!allUsers || allUsers.length === 0) {
// first time startup - we don't need to backfill anything
// tenant will be identified when admin user is created
if (env.SELF_HOSTED) {
await events.installation.firstStartup()
}
return
}
try {
const installTimestamp = await getInstallTimestamp(db, allUsers)
if (installTimestamp) {
timestamp = installTimestamp
}
} catch (e) {
handleError(e, errors)
}
let account: CloudAccount | undefined
if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
account = await accounts.getAccountByTenantId(tenantId)
}
try {
await events.identification.identifyTenantGroup(
tenantId,
env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD,
timestamp
)
} catch (e) {
handleError(e, errors)
}
// tell the event pipeline to start caching
// events for this tenant
await events.backfillCache.start(EVENTS)
try {
await configs.backfill(db, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.users = await users.backfill(db, account)
} catch (e) {
handleError(e, errors)
}
try {
const allApps = (await dbUtils.getAllApps({ dev: true })) as App[]
totals.apps = allApps.length
totals.usage = await quotas.backfill(allApps)
} catch (e) {
handleError(e, errors)
}
const properties: TenantBackfillSucceededEvent = {
apps: totals.apps,
users: totals.users,
...formatUsage(totals.usage),
usage: totals.usage,
}
if (errors.length) {
properties.errors = errors.map((e: any) =>
JSON.stringify(e, Object.getOwnPropertyNames(e))
)
properties.errorCount = errors.length
} else {
properties.errorCount = 0
}
await events.backfill.tenantSucceeded(properties)
// tell the event pipeline to stop caching events for this tenant
await events.backfillCache.end()
} catch (e) {
handleError(e)
await events.backfill.tenantFailed(e)
}
}
export const isComplete = async (): Promise<boolean> => {
const globalDb = tenancy.getGlobalDB()
const migrationsDoc = await migrations.getMigrationsDoc(globalDb)
return !!migrationsDoc.event_global_backfill
}
export const getInstallTimestamp = async (
globalDb: any,
allUsers?: User[]
): Promise<number | undefined> => {
if (!allUsers) {
allUsers = await users.getUsers(globalDb)
}
// get the oldest user timestamp
if (allUsers) {
const timestamps = allUsers
.map(user => user.createdAt)
.filter(timestamp => !!timestamp)
.sort(
(a, b) =>
new Date(a as number).getTime() - new Date(b as number).getTime()
)
if (timestamps.length) {
return timestamps[0]
}
}
}

74
packages/server/src/migrations/functions/backfill/global/configs.ts
View File

@ -1,74 +0,0 @@
import {
events,
DocumentType,
SEPARATOR,
UNICODE_MAX,
} from "@budibase/backend-core"
import {
Config,
isSMTPConfig,
isGoogleConfig,
isOIDCConfig,
isSettingsConfig,
ConfigType,
DatabaseQueryOpts,
} from "@budibase/types"
import env from "./../../../../environment"
export function getConfigParams(): DatabaseQueryOpts {
return {
include_docs: true,
startkey: `${DocumentType.CONFIG}${SEPARATOR}`,
endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`,
}
}
const getConfigs = async (globalDb: any): Promise<Config[]> => {
const response = await globalDb.allDocs(getConfigParams())
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (
globalDb: any,
timestamp: string | number | undefined
) => {
const configs = await getConfigs(globalDb)
for (const config of configs) {
if (isSMTPConfig(config)) {
await events.email.SMTPCreated(timestamp)
}
if (isGoogleConfig(config)) {
await events.auth.SSOCreated(ConfigType.GOOGLE, timestamp)
if (config.config.activated) {
await events.auth.SSOActivated(ConfigType.GOOGLE, timestamp)
}
}
if (isOIDCConfig(config)) {
await events.auth.SSOCreated(ConfigType.OIDC, timestamp)
if (config.config.configs[0].activated) {
await events.auth.SSOActivated(ConfigType.OIDC, timestamp)
}
}
if (isSettingsConfig(config)) {
const company = config.config.company
if (company && company !== "Budibase") {
await events.org.nameUpdated(timestamp)
}
const logoUrl = config.config.logoUrl
if (logoUrl) {
await events.org.logoUpdated(timestamp)
}
const platformUrl = config.config.platformUrl
if (
platformUrl &&
platformUrl !== "http://localhost:10000" &&
env.SELF_HOSTED
) {
await events.org.platformURLUpdated(timestamp)
}
}
}
}

60
packages/server/src/migrations/functions/backfill/global/quotas.ts
View File

@ -1,60 +0,0 @@
import { DEFAULT_TIMESTAMP } from "./../index"
import { events } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { App } from "@budibase/types"
const getOldestCreatedAt = (allApps: App[]): string | undefined => {
const timestamps = allApps
.filter(app => !!app.createdAt)
.map(app => app.createdAt as string)
.sort((a, b) => new Date(a).getTime() - new Date(b).getTime())
if (timestamps.length) {
return timestamps[0]
}
}
const getMonthTimestamp = (monthString: string): number => {
const parts = monthString.split("-")
const month = parseInt(parts[0]) - 1 // we already do +1 in month string calculation
const year = parseInt(parts[1])
// using 0 as the day in next month gives us last day in previous month
const date = new Date(year, month + 1, 0).getTime()
const now = new Date().getTime()
if (date > now) {
return now
} else {
return date
}
}
export const backfill = async (allApps: App[]) => {
const usage = await quotas.getQuotaUsage()
const rows = usage.usageQuota.rows
let timestamp: string | number = DEFAULT_TIMESTAMP
const oldestAppTimestamp = getOldestCreatedAt(allApps)
if (oldestAppTimestamp) {
timestamp = oldestAppTimestamp
}
await events.rows.created(rows, timestamp)
for (const [monthString, quotas] of Object.entries(usage.monthly)) {
if (monthString === "current") {
continue
}
const monthTimestamp = getMonthTimestamp(monthString)
const queries = quotas.queries
await events.query.run(queries, monthTimestamp)
const automations = quotas.automations
await events.automation.run(automations, monthTimestamp)
}
return usage
}

53
packages/server/src/migrations/functions/backfill/global/users.ts
View File

@ -1,53 +0,0 @@
import {
events,
db as dbUtils,
users as usersCore,
} from "@budibase/backend-core"
import { User, CloudAccount } from "@budibase/types"
import { DEFAULT_TIMESTAMP } from ".."
// manually define user doc params - normally server doesn't read users from the db
const getUserParams = (props: any) => {
return dbUtils.getDocParams(dbUtils.DocumentType.USER, null, props)
}
export const getUsers = async (globalDb: any): Promise<User[]> => {
const response = await globalDb.allDocs(
getUserParams({
include_docs: true,
})
)
return response.rows.map((row: any) => row.doc)
}
export const backfill = async (
globalDb: any,
account: CloudAccount | undefined
) => {
const users = await getUsers(globalDb)
for (const user of users) {
let timestamp: string | number = DEFAULT_TIMESTAMP
if (user.createdAt) {
timestamp = user.createdAt
}
await events.identification.identifyUser(user, account, timestamp)
await events.user.created(user, timestamp)
if (usersCore.hasAdminPermissions(user)) {
await events.user.permissionAdminAssigned(user, timestamp)
}
if (usersCore.hasBuilderPermissions(user)) {
await events.user.permissionBuilderAssigned(user, timestamp)
}
if (user.roles) {
for (const [, role] of Object.entries(user.roles)) {
await events.role.assigned(user, role, timestamp)
}
}
}
return users.length
}

7
packages/server/src/migrations/functions/backfill/index.ts
View File

@ -1,7 +0,0 @@
export * as app from "./app"
export * as global from "./global"
export * as installation from "./installation"
// historical events are free in posthog - make sure we default to a
// historical time if no other can be found
export const DEFAULT_TIMESTAMP = new Date(2022, 0, 1).getTime()

50
packages/server/src/migrations/functions/backfill/installation.ts
View File

@ -1,50 +0,0 @@
import { DEFAULT_TIMESTAMP } from "./index"
import { events, tenancy, installation } from "@budibase/backend-core"
import { Installation } from "@budibase/types"
import * as global from "./global"
import env from "../../../environment"
const failGraceful = env.SELF_HOSTED
const handleError = (e: any, errors?: any) => {
if (failGraceful) {
if (errors) {
errors.push(e)
}
return
}
throw e
}
/**
* Date:
* May 2022
*
* Description:
* Backfill installation events.
*/
export const run = async () => {
try {
// need to use the default tenant to try to get the installation time
await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => {
const db = tenancy.getGlobalDB()
let timestamp: string | number = DEFAULT_TIMESTAMP
const installTimestamp = await global.getInstallTimestamp(db)
if (installTimestamp) {
timestamp = installTimestamp
}
const install: Installation = await installation.getInstall()
await events.identification.identifyInstallationGroup(
install.installId,
timestamp
)
})
await events.backfill.installationSucceeded()
} catch (e) {
handleError(e)
await events.backfill.installationFailed(e)
}
}

19
packages/server/src/migrations/functions/syncQuotas.ts
View File

@ -1,19 +0,0 @@
import { runQuotaMigration } from "./usageQuotas"
import * as syncApps from "./usageQuotas/syncApps"
import * as syncRows from "./usageQuotas/syncRows"
import * as syncPlugins from "./usageQuotas/syncPlugins"
import * as syncUsers from "./usageQuotas/syncUsers"
import * as syncCreators from "./usageQuotas/syncCreators"
/**
* Synchronise quotas to the state of the db.
*/
export const run = async () => {
await runQuotaMigration(async () => {
await syncApps.run()
await syncRows.run()
await syncPlugins.run()
await syncUsers.run()
await syncCreators.run()
})
}

145
packages/server/src/migrations/functions/tableSettings.ts
View File

@ -1,145 +0,0 @@
import { getScreenParams } from "../../db/utils"
import { Screen } from "@budibase/types"
import { makePropSafe as safe } from "@budibase/string-templates"
/**
* Date:
* November 2022
*
* Description:
* Update table settings to use actions instead of links. We do not remove the
* legacy values here as we cannot guarantee that their apps are up-t-date.
* It is safe to simply save both the new and old structure in the definition.
*
* Migration 1:
* Legacy "linkRows", "linkURL", "linkPeek" and "linkColumn" settings on tables
* and table blocks are migrated into a "Navigate To" action under the new
* "onClick" setting.
*
* Migration 2:
* Legacy "titleButtonURL" and "titleButtonPeek" settings on table blocks are
* migrated into a "Navigate To" action under the new "onClickTitleButton"
* setting.
*/
export const run = async (appDb: any) => {
// Get all app screens
let screens: Screen[]
try {
screens = (
await appDb.allDocs(
getScreenParams(null, {
include_docs: true,
})
)
).rows.map((row: any) => row.doc)
} catch (e) {
// sometimes the metadata document doesn't exist
// exit early instead of failing the migration
console.error("Error retrieving app metadata. Skipping", e)
return
}
// Recursively update any relevant components and mutate the screen docs
for (let screen of screens) {
const changed = migrateTableSettings(screen.props)
// Save screen if we updated it
if (changed) {
await appDb.put(screen)
console.log(
`Screen ${screen.routing?.route} contained table settings which were migrated`
)
}
}
}
// Recursively searches and mutates a screen doc to migrate table component
// and table block settings
const migrateTableSettings = (component: any) => {
let changed = false
if (!component) {
return changed
}
// Migration 1: migrate table row click settings
if (
component._component.endsWith("/table") ||
component._component.endsWith("/tableblock")
) {
const { linkRows, linkURL, linkPeek, linkColumn, onClick } = component
if (linkRows && !onClick) {
const column = linkColumn || "_id"
const action = convertLinkSettingToAction(linkURL, !!linkPeek, column)
if (action) {
changed = true
component.onClick = action
if (component._component.endsWith("/tableblock")) {
component.clickBehaviour = "actions"
}
}
}
}
// Migration 2: migrate table block title button settings
if (component._component.endsWith("/tableblock")) {
const {
showTitleButton,
titleButtonURL,
titleButtonPeek,
onClickTitleButton,
} = component
if (showTitleButton && !onClickTitleButton) {
const action = convertLinkSettingToAction(
titleButtonURL,
!!titleButtonPeek
)
if (action) {
changed = true
component.onClickTitleButton = action
component.titleButtonClickBehaviour = "actions"
}
}
}
// Recurse down the tree as needed
component._children?.forEach((child: any) => {
const childChanged = migrateTableSettings(child)
changed = changed || childChanged
})
return changed
}
// Util ti convert the legacy settings into a navigation action structure
const convertLinkSettingToAction = (
linkURL: string,
linkPeek: boolean,
linkColumn?: string
) => {
// Sanity check we have a URL
if (!linkURL) {
return null
}
// Default URL to the old URL setting
let url = linkURL
// If we enriched the old URL with a column, update the url
if (linkColumn && linkURL.includes("/:")) {
// Convert old link URL setting, which is a screen URL, into a valid
// binding using the new clicked row binding
const split = linkURL.split("/:")
const col = linkColumn || "_id"
const binding = `{{ ${safe("eventContext")}.${safe("row")}.${safe(col)} }}`
url = `${split[0]}/${binding}`
}
// Create action structure
return [
{
"##eventHandlerType": "Navigate To",
parameters: {
url,
peek: linkPeek,
},
},
]
}

26
packages/server/src/migrations/functions/tests/appUrls.spec.js
View File

@ -1,26 +0,0 @@
const { db: dbCore } = require("@budibase/backend-core")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
const migration = require("../appUrls")
describe("run", () => {
let config = new TestConfig(false)
beforeAll(async () => {
await config.init()
})
afterAll(config.end)
it("runs successfully", async () => {
const app = await config.createApp("testApp")
const metadata = await dbCore.doWithDB(app.appId, async db => {
const metadataDoc = await db.get(dbCore.DocumentType.APP_METADATA)
delete metadataDoc.url
await db.put(metadataDoc)
await migration.run(db)
return await db.get(dbCore.DocumentType.APP_METADATA)
})
expect(metadata.url).toEqual("/testapp")
})
})

144
packages/server/src/migrations/functions/tests/tableSettings.spec.ts
View File

@ -1,144 +0,0 @@
import { App, Screen } from "@budibase/types"
import { db as dbCore } from "@budibase/backend-core"
import TestConfig from "../../../tests/utilities/TestConfiguration"
import { run as runMigration } from "../tableSettings"
describe("run", () => {
const config = new TestConfig(false)
let app: App
let screen: Screen
beforeAll(async () => {
await config.init()
app = await config.createApp("testApp")
screen = await config.createScreen()
})
afterAll(config.end)
it("migrates table block row on click settings", async () => {
// Add legacy table block as first child
screen.props._children = [
{
_instanceName: "Table Block",
_styles: {},
_component: "@budibase/standard-components/tableblock",
_id: "foo",
linkRows: true,
linkURL: "/rows/:id",
linkPeek: true,
linkColumn: "name",
},
]
await config.createScreen(screen)
// Run migration
screen = await dbCore.doWithDB(app.appId, async (db: any) => {
await runMigration(db)
return await db.get(screen._id)
})
// Verify new "onClick" setting
const onClick = screen.props._children?.[0].onClick
expect(onClick).toBeDefined()
expect(onClick.length).toBe(1)
expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
expect(onClick[0].parameters.url).toBe(
`/rows/{{ [eventContext].[row].[name] }}`
)
expect(onClick[0].parameters.peek).toBeTruthy()
})
it("migrates table row on click settings", async () => {
// Add legacy table block as first child
screen.props._children = [
{
_instanceName: "Table",
_styles: {},
_component: "@budibase/standard-components/table",
_id: "foo",
linkRows: true,
linkURL: "/rows/:id",
linkPeek: true,
linkColumn: "name",
},
]
await config.createScreen(screen)
// Run migration
screen = await dbCore.doWithDB(app.appId, async (db: any) => {
await runMigration(db)
return await db.get(screen._id)
})
// Verify new "onClick" setting
const onClick = screen.props._children?.[0].onClick
expect(onClick).toBeDefined()
expect(onClick.length).toBe(1)
expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
expect(onClick[0].parameters.url).toBe(
`/rows/{{ [eventContext].[row].[name] }}`
)
expect(onClick[0].parameters.peek).toBeTruthy()
})
it("migrates table block title button settings", async () => {
// Add legacy table block as first child
screen.props._children = [
{
_instanceName: "Table Block",
_styles: {},
_component: "@budibase/standard-components/tableblock",
_id: "foo",
showTitleButton: true,
titleButtonURL: "/url",
titleButtonPeek: true,
},
]
await config.createScreen(screen)
// Run migration
screen = await dbCore.doWithDB(app.appId, async (db: any) => {
await runMigration(db)
return await db.get(screen._id)
})
// Verify new "onClickTitleButton" setting
const onClick = screen.props._children?.[0].onClickTitleButton
expect(onClick).toBeDefined()
expect(onClick.length).toBe(1)
expect(onClick[0]["##eventHandlerType"]).toBe("Navigate To")
expect(onClick[0].parameters.url).toBe("/url")
expect(onClick[0].parameters.peek).toBeTruthy()
})
it("ignores components that have already been migrated", async () => {
// Add legacy table block as first child
screen.props._children = [
{
_instanceName: "Table Block",
_styles: {},
_component: "@budibase/standard-components/tableblock",
_id: "foo",
linkRows: true,
linkURL: "/rows/:id",
linkPeek: true,
linkColumn: "name",
onClick: "foo",
},
]
const initialDefinition = JSON.stringify(screen.props._children?.[0])
await config.createScreen(screen)
// Run migration
screen = await dbCore.doWithDB(app.appId, async (db: any) => {
await runMigration(db)
return await db.get(screen._id)
})
// Verify new "onClick" setting
const newDefinition = JSON.stringify(screen.props._children?.[0])
expect(initialDefinition).toEqual(newDefinition)
})
})

34
packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js
View File

@ -1,34 +0,0 @@
jest.mock("@budibase/backend-core", () => {
const core = jest.requireActual("@budibase/backend-core")
return {
...core,
db: {
...core.db,
createNewUserEmailView: jest.fn(),
},
}
})
const { context, db: dbCore } = require("@budibase/backend-core")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
// mock email view creation
const migration = require("../userEmailViewCasing")
describe("run", () => {
let config = new TestConfig(false)
beforeAll(async () => {
await config.init()
})
afterAll(config.end)
it("runs successfully", async () => {
await config.doInTenant(async () => {
const globalDb = context.getGlobalDB()
await migration.run(globalDb)
expect(dbCore.createNewUserEmailView).toHaveBeenCalledTimes(1)
})
})
})

3
packages/server/src/migrations/functions/usageQuotas/index.ts
View File

@ -1,3 +0,0 @@
export const runQuotaMigration = async (migration: () => Promise<void>) => {
await migration()
}

13
packages/server/src/migrations/functions/usageQuotas/syncApps.ts
View File

@ -1,13 +0,0 @@
import { db as dbCore } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
export const run = async () => {
// get app count
const devApps = await dbCore.getAllApps({ dev: true })
const appCount = devApps ? devApps.length : 0
// sync app count
console.log(`Syncing app count: ${appCount}`)
await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
}

13
packages/server/src/migrations/functions/usageQuotas/syncCreators.ts
View File

@ -1,13 +0,0 @@
import { users } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
export const run = async () => {
const creatorCount = await users.getCreatorCount()
console.log(`Syncing creator count: ${creatorCount}`)
await quotas.setUsage(
creatorCount,
StaticQuotaName.CREATORS,
QuotaUsageType.STATIC
)
}

10
packages/server/src/migrations/functions/usageQuotas/syncPlugins.ts
View File

@ -1,10 +0,0 @@
import { logging } from "@budibase/backend-core"
import { plugins } from "@budibase/pro"
export const run = async () => {
try {
await plugins.checkPluginQuotas()
} catch (err) {
logging.logAlert("Failed to update plugin quotas", err)
}
}

27
packages/server/src/migrations/functions/usageQuotas/syncRows.ts
View File

@ -1,27 +0,0 @@
import { db as dbCore } from "@budibase/backend-core"
import { getUniqueRows } from "../../../utilities/usageQuota/rows"
import { quotas } from "@budibase/pro"
import { StaticQuotaName, QuotaUsageType, App } from "@budibase/types"
export const run = async () => {
// get all rows in all apps
const allApps = (await dbCore.getAllApps({ all: true })) as App[]
const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
const { appRows } = await getUniqueRows(appIds)
// get the counts per app
const counts: { [key: string]: number } = {}
let rowCount = 0
Object.entries(appRows).forEach(([appId, rows]) => {
counts[appId] = rows.length
rowCount += rows.length
})
// sync row count
console.log(`Syncing row count: ${rowCount}`)
await quotas.setUsagePerApp(
counts,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
}

9
packages/server/src/migrations/functions/usageQuotas/syncUsers.ts
View File

@ -1,9 +0,0 @@
import { users } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
export const run = async () => {
const userCount = await users.getUserCount()
console.log(`Syncing user count: ${userCount}`)
await quotas.setUsage(userCount, StaticQuotaName.USERS, QuotaUsageType.STATIC)
}

35
packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts
View File

@ -1,35 +0,0 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import * as syncApps from "../syncApps"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
describe("syncApps", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
})
afterAll(config.end)
it("runs successfully", async () => {
return config.doInContext(undefined, async () => {
// create the usage quota doc and mock usages
await quotas.getQuotaUsage()
await quotas.setUsage(3, StaticQuotaName.APPS, QuotaUsageType.STATIC)
let usageDoc = await quotas.getQuotaUsage()
expect(usageDoc.usageQuota.apps).toEqual(3)
// create an extra app to test the migration
await config.createApp("quota-test")
// migrate
await syncApps.run()
// assert the migration worked
usageDoc = await quotas.getQuotaUsage()
expect(usageDoc.usageQuota.apps).toEqual(2)
})
})
})

26
packages/server/src/migrations/functions/usageQuotas/tests/syncCreators.spec.ts
View File

@ -1,26 +0,0 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import * as syncCreators from "../syncCreators"
import { quotas } from "@budibase/pro"
describe("syncCreators", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
})
afterAll(config.end)
it("syncs creators", async () => {
return config.doInContext(undefined, async () => {
await config.createUser({ admin: { global: true } })
await syncCreators.run()
const usageDoc = await quotas.getQuotaUsage()
// default + additional creator
const creatorsCount = 2
expect(usageDoc.usageQuota.creators).toBe(creatorsCount)
})
})
})

53
packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
View File

@ -1,53 +0,0 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import * as syncRows from "../syncRows"
import { quotas } from "@budibase/pro"
import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
import { db as dbCore, context } from "@budibase/backend-core"
describe("syncRows", () => {
const config = new TestConfig()
beforeEach(async () => {
await config.init()
})
afterAll(config.end)
it("runs successfully", async () => {
return config.doInContext(undefined, async () => {
// create the usage quota doc and mock usages
await quotas.getQuotaUsage()
await quotas.setUsage(300, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
let usageDoc = await quotas.getQuotaUsage()
expect(usageDoc.usageQuota.rows).toEqual(300)
// app 1
const app1 = config.app
await context.doInAppContext(app1!.appId, async () => {
await config.createTable()
await config.createRow()
})
// app 2
const app2 = await config.createApp("second-app")
await context.doInAppContext(app2.appId, async () => {
await config.createTable()
await config.createRow()
await config.createRow()
})
// migrate
await syncRows.run()
// assert the migration worked
usageDoc = await quotas.getQuotaUsage()
expect(usageDoc.usageQuota.rows).toEqual(3)
expect(
usageDoc.apps?.[dbCore.getProdAppID(app1!.appId)].usageQuota.rows
).toEqual(1)
expect(
usageDoc.apps?.[dbCore.getProdAppID(app2.appId)].usageQuota.rows
).toEqual(2)
})
})
})

26
packages/server/src/migrations/functions/usageQuotas/tests/syncUsers.spec.ts
View File

@ -1,26 +0,0 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import * as syncUsers from "../syncUsers"
import { quotas } from "@budibase/pro"
describe("syncUsers", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
})
afterAll(config.end)
it("syncs users", async () => {
return config.doInContext(undefined, async () => {
await config.createUser()
await syncUsers.run()
const usageDoc = await quotas.getQuotaUsage()
// default + additional user
const userCount = 2
expect(usageDoc.usageQuota.users).toBe(userCount)
})
})
})

13
packages/server/src/migrations/functions/userEmailViewCasing.ts
View File

@ -1,13 +0,0 @@
import { db as dbCore } from "@budibase/backend-core"
/**
* Date:
* October 2021
*
* Description:
* Recreate the user email view to include latest changes i.e. lower casing the email address
*/
export const run = async () => {
await dbCore.createNewUserEmailView()
}

115
packages/server/src/migrations/index.ts
View File

@ -1,115 +0,0 @@
import { locks, migrations } from "@budibase/backend-core"
import {
Migration,
MigrationOptions,
MigrationName,
LockType,
LockName,
} from "@budibase/types"
import env from "../environment"
// migration functions
import * as userEmailViewCasing from "./functions/userEmailViewCasing"
import * as syncQuotas from "./functions/syncQuotas"
import * as appUrls from "./functions/appUrls"
import * as tableSettings from "./functions/tableSettings"
import * as backfill from "./functions/backfill"
/**
* Populate the migration function and additional configuration from
* the static migration definitions.
*/
export const buildMigrations = () => {
const definitions = migrations.DEFINITIONS
const serverMigrations: Migration[] = []
for (const definition of definitions) {
switch (definition.name) {
case MigrationName.USER_EMAIL_VIEW_CASING: {
serverMigrations.push({
...definition,
fn: userEmailViewCasing.run,
})
break
}
case MigrationName.SYNC_QUOTAS: {
serverMigrations.push({
...definition,
fn: syncQuotas.run,
})
break
}
case MigrationName.APP_URLS: {
serverMigrations.push({
...definition,
appOpts: { all: true },
fn: appUrls.run,
})
break
}
case MigrationName.EVENT_APP_BACKFILL: {
serverMigrations.push({
...definition,
appOpts: { all: true },
fn: backfill.app.run,
silent: !!env.SELF_HOSTED, // reduce noisy logging
preventRetry: !!env.SELF_HOSTED, // only ever run once
})
break
}
case MigrationName.EVENT_GLOBAL_BACKFILL: {
serverMigrations.push({
...definition,
fn: backfill.global.run,
silent: !!env.SELF_HOSTED, // reduce noisy logging
preventRetry: !!env.SELF_HOSTED, // only ever run once
})
break
}
case MigrationName.EVENT_INSTALLATION_BACKFILL: {
serverMigrations.push({
...definition,
fn: backfill.installation.run,
silent: !!env.SELF_HOSTED, // reduce noisy logging
preventRetry: !!env.SELF_HOSTED, // only ever run once
})
break
}
case MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS: {
serverMigrations.push({
...definition,
appOpts: { dev: true },
fn: tableSettings.run,
})
break
}
}
}
return serverMigrations
}
export const MIGRATIONS = buildMigrations()
export const migrate = async (options?: MigrationOptions) => {
if (env.SELF_HOSTED) {
// self host runs migrations on startup
// make sure only a single instance runs them
await migrateWithLock(options)
} else {
await migrations.runMigrations(MIGRATIONS, options)
}
}
const migrateWithLock = async (options?: MigrationOptions) => {
await locks.doWithLock(
{
type: LockType.TRY_ONCE,
name: LockName.MIGRATIONS,
ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
systemLock: true,
},
async () => {
await migrations.runMigrations(MIGRATIONS, options)
}
)
}

40
packages/server/src/migrations/tests/helpers.ts
View File

@ -1,40 +0,0 @@
// Mimic configs test configuration from worker, creation configs directly in database
import * as structures from "./structures"
import { configs } from "@budibase/backend-core"
import { Config } from "@budibase/types"
export const saveSettingsConfig = async (globalDb: any) => {
const config = structures.settings()
await saveConfig(config, globalDb)
}
export const saveGoogleConfig = async (globalDb: any) => {
const config = structures.google()
await saveConfig(config, globalDb)
}
export const saveOIDCConfig = async (globalDb: any) => {
const config = structures.oidc()
await saveConfig(config, globalDb)
}
export const saveSmtpConfig = async (globalDb: any) => {
const config = structures.smtp()
await saveConfig(config, globalDb)
}
const saveConfig = async (config: Config, globalDb: any) => {
config._id = configs.generateConfigID(config.type)
let response
try {
response = await globalDb.get(config._id)
config._rev = response._rev
await globalDb.put(config)
} catch (e: any) {
if (e.status === 404) {
await globalDb.put(config)
}
}
}

137
packages/server/src/migrations/tests/index.spec.ts
View File

@ -1,137 +0,0 @@
import {
events,
migrations,
tenancy,
DocumentType,
context,
} from "@budibase/backend-core"
import TestConfig from "../../tests/utilities/TestConfiguration"
import * as structures from "../../tests/utilities/structures"
import { MIGRATIONS } from "../"
import * as helpers from "./helpers"
import tk from "timekeeper"
import { View } from "@budibase/types"
const timestamp = new Date().toISOString()
tk.freeze(timestamp)
const clearMigrations = async () => {
const dbs = [context.getDevAppDB(), context.getProdAppDB()]
for (const db of dbs) {
const doc = await db.get<any>(DocumentType.MIGRATIONS)
const newDoc = { _id: doc._id, _rev: doc._rev }
await db.put(newDoc)
}
}
describe("migrations", () => {
const config = new TestConfig()
beforeAll(async () => {
await config.init()
})
afterAll(() => {
config.end()
})
describe("backfill", () => {
it("runs app db migration", async () => {
await config.doInContext(undefined, async () => {
await clearMigrations()
await config.createAutomation()
await config.createAutomation(structures.newAutomation())
await config.createDatasource()
await config.createDatasource()
await config.createLayout()
await config.createQuery()
await config.createQuery()
await config.createRole()
await config.createRole()
await config.createTable()
await config.createLegacyView()
await config.createTable()
await config.createLegacyView(
structures.view(config.table!._id!) as View
)
await config.createScreen()
await config.createScreen()
jest.clearAllMocks()
const migration = MIGRATIONS.filter(
m => m.name === "event_app_backfill"
)[0]
await migrations.runMigration(migration)
expect(events.app.created).toHaveBeenCalledTimes(1)
expect(events.app.published).toHaveBeenCalledTimes(1)
expect(events.automation.created).toHaveBeenCalledTimes(2)
expect(events.automation.stepCreated).toHaveBeenCalledTimes(1)
expect(events.datasource.created).toHaveBeenCalledTimes(2)
expect(events.layout.created).toHaveBeenCalledTimes(1)
expect(events.query.created).toHaveBeenCalledTimes(2)
expect(events.role.created).toHaveBeenCalledTimes(3) // created roles + admin (created on table creation)
expect(events.table.created).toHaveBeenCalledTimes(3)
expect(events.backfill.appSucceeded).toHaveBeenCalledTimes(2)
// to make sure caching is working as expected
expect(
events.processors.analyticsProcessor.processEvent
).toHaveBeenCalledTimes(20) // Addition of of the events above
})
})
})
it("runs global db migration", async () => {
await config.doInContext(undefined, async () => {
await clearMigrations()
const appId = config.getProdAppId()
const roles = { [appId]: "role_12345" }
await config.createUser({
builder: { global: false },
admin: { global: true },
roles,
}) // admin only
await config.createUser({
builder: { global: false },
admin: { global: false },
roles,
}) // non admin non builder
await config.createTable()
await config.createRow()
await config.createRow()
const db = tenancy.getGlobalDB()
await helpers.saveGoogleConfig(db)
await helpers.saveOIDCConfig(db)
await helpers.saveSettingsConfig(db)
await helpers.saveSmtpConfig(db)
jest.clearAllMocks()
const migration = MIGRATIONS.filter(
m => m.name === "event_global_backfill"
)[0]
await migrations.runMigration(migration)
expect(events.user.created).toHaveBeenCalledTimes(3)
expect(events.role.assigned).toHaveBeenCalledTimes(2)
expect(events.user.permissionBuilderAssigned).toHaveBeenCalledTimes(1) // default test user
expect(events.user.permissionAdminAssigned).toHaveBeenCalledTimes(1) // admin from above
expect(events.rows.created).toHaveBeenCalledTimes(1)
expect(events.rows.created).toHaveBeenCalledWith(2, timestamp)
expect(events.email.SMTPCreated).toHaveBeenCalledTimes(1)
expect(events.auth.SSOCreated).toHaveBeenCalledTimes(2)
expect(events.auth.SSOActivated).toHaveBeenCalledTimes(2)
expect(events.org.logoUpdated).toHaveBeenCalledTimes(1)
expect(events.org.nameUpdated).toHaveBeenCalledTimes(1)
expect(events.org.platformURLUpdated).toHaveBeenCalledTimes(1)
expect(events.backfill.tenantSucceeded).toHaveBeenCalledTimes(1)
// to make sure caching is working as expected
expect(
events.processors.analyticsProcessor.processEvent
).toHaveBeenCalledTimes(19)
})
})
})

67
packages/server/src/migrations/tests/structures.ts
View File

@ -1,67 +0,0 @@
import { utils } from "@budibase/backend-core"
import {
SMTPConfig,
OIDCConfig,
GoogleConfig,
SettingsConfig,
ConfigType,
} from "@budibase/types"
export const oidc = (conf?: OIDCConfig): OIDCConfig => {
return {
type: ConfigType.OIDC,
config: {
configs: [
{
configUrl: "http://someconfigurl",
clientID: "clientId",
clientSecret: "clientSecret",
logo: "Microsoft",
name: "Active Directory",
uuid: utils.newid(),
activated: true,
scopes: [],
...conf,
},
],
},
}
}
export const google = (conf?: GoogleConfig): GoogleConfig => {
return {
type: ConfigType.GOOGLE,
config: {
clientID: "clientId",
clientSecret: "clientSecret",
activated: true,
...conf,
},
}
}
export const smtp = (conf?: SMTPConfig): SMTPConfig => {
return {
type: ConfigType.SMTP,
config: {
port: 12345,
host: "smtptesthost.com",
from: "testfrom@example.com",
subject: "Hello!",
secure: false,
...conf,
},
}
}
export const settings = (conf?: SettingsConfig): SettingsConfig => {
return {
type: ConfigType.SETTINGS,
config: {
platformUrl: "http://mycustomdomain.com",
logoUrl: "http://mylogourl,com",
company: "mycompany",
...conf,
},
}
}

13
packages/server/src/startup/index.ts
View File

@ -15,7 +15,6 @@ import { watch } from "../watch"
import * as automations from "../automations"
import * as fileSystem from "../utilities/fileSystem"
import { default as eventEmitter, init as eventInit } from "../events"
import * as migrations from "../migrations"
import * as bullboard from "../automations/bullboard"
import * as appMigrations from "../appMigrations/queue"
import * as pro from "@budibase/pro"
@ -106,18 +105,6 @@ export async function startup(
initialiseWebsockets(app, server)
}
// run migrations on startup if not done via http
// not recommended in a clustered environment
if (!env.HTTP_MIGRATIONS && !env.isTest()) {
console.log("Running migrations")
try {
await migrations.migrate()
} catch (e) {
logging.logAlert("Error performing migrations. Exiting.", e)
shutdown(server)
}
}
// monitor plugin directory if required
if (
env.SELF_HOSTED &&

11
packages/types/src/api/web/global/oldMigration.ts
View File

@ -1,12 +1,3 @@
import { Migration, MigrationOptions } from "../../../sdk"
export interface RunOldMigrationRequest extends MigrationOptions {}
export interface RuneOldMigrationResponse {
message: string
}
export type FetchOldMigrationResponse = Migration[]
export interface GetOldMigrationStatus {
export interface GetMigrationStatus {
migrated: boolean
}

1
packages/types/src/api/web/system/index.ts
View File

@ -3,6 +3,5 @@ export * from "./status"
export * from "./ops"
export * from "./account"
export * from "./log"
export * from "./migration"
export * from "./restore"
export * from "./tenant"

8
packages/types/src/api/web/system/migration.ts
View File

@ -1,8 +0,0 @@
import { MigrationDefinition, MigrationOptions } from "../../../sdk"
export interface RunGlobalMigrationRequest extends MigrationOptions {}
export interface RunGlobalMigrationResponse {
message: string
}
export type FetchMigrationDefinitionsResponse = MigrationDefinition[]

1
packages/types/src/sdk/index.ts
View File

@ -4,7 +4,6 @@ export * from "./hosting"
export * from "./context"
export * from "./events"
export * from "./licensing"
export * from "./migrations"
export * from "./datasources"
export * from "./search"
export * from "./koa"

57
packages/types/src/sdk/migrations.ts
View File

@ -1,57 +0,0 @@
import { Database } from "./db"
export interface Migration extends MigrationDefinition {
appOpts?: object
fn: (db: Database) => Promise<void>
silent?: boolean
preventRetry?: boolean
}
export enum MigrationType {
// run once per tenant, recorded in global db, global db is provided as an argument
GLOBAL = "global",
// run per app, recorded in each app db, app db is provided as an argument
APP = "app",
// run once, recorded in global info db, global info db is provided as an argument
INSTALLATION = "installation",
}
export interface MigrationNoOpOptions {
type: MigrationType
tenantId: string
appId?: string
}
/**
* e.g.
* {
* tenantIds: ['bb'],
* force: {
* global: ['quota_1']
* }
* }
*/
export interface MigrationOptions {
tenantIds?: string[]
force?: {
[type: string]: string[]
}
noOp?: MigrationNoOpOptions
}
export enum MigrationName {
USER_EMAIL_VIEW_CASING = "user_email_view_casing",
APP_URLS = "app_urls",
EVENT_APP_BACKFILL = "event_app_backfill",
EVENT_GLOBAL_BACKFILL = "event_global_backfill",
EVENT_INSTALLATION_BACKFILL = "event_installation_backfill",
GLOBAL_INFO_SYNC_USERS = "global_info_sync_users",
TABLE_SETTINGS_LINKS_TO_ACTIONS = "table_settings_links_to_actions",
// increment this number to re-activate this migration
SYNC_QUOTAS = "sync_quotas_2",
}
export interface MigrationDefinition {
type: MigrationType
name: MigrationName
}

6
packages/worker/src/api/controllers/global/users.ts
View File

@ -28,7 +28,6 @@ import {
LockType,
LookupAccountHolderResponse,
LookupTenantUserResponse,
MigrationType,
PlatformUserByEmail,
SaveUserResponse,
SearchUsersRequest,
@ -45,7 +44,6 @@ import {
cache,
ErrorCode,
events,
migrations,
platform,
tenancy,
db,
@ -187,10 +185,6 @@ export const adminUser = async (
if (env.MULTI_TENANCY) {
// store the new tenant record in the platform db
await platform.tenants.addTenant(tenantId)
await migrations.backPopulateMigrations({
type: MigrationType.GLOBAL,
tenantId,
})
}
await tenancy.doInTenant(tenantId, async () => {

23
packages/worker/src/api/controllers/system/migrations.ts
View File

@ -1,23 +0,0 @@
import {
FetchMigrationDefinitionsResponse,
RunGlobalMigrationRequest,
RunGlobalMigrationResponse,
UserCtx,
} from "@budibase/types"
const { migrate, MIGRATIONS } = require("../../../migrations")
export const runMigrations = async (
ctx: UserCtx<RunGlobalMigrationRequest, RunGlobalMigrationResponse>
) => {
const options = ctx.request.body
// don't await as can take a while, just return
migrate(options)
ctx.body = { message: "Migration started." }
}
export const fetchDefinitions = async (
ctx: UserCtx<void, FetchMigrationDefinitionsResponse>
) => {
ctx.body = MIGRATIONS
}

2
packages/worker/src/api/routes/index.ts
View File

@ -12,7 +12,6 @@ import tenantsRoutes from "./system/tenants"
import statusRoutes from "./system/status"
import selfRoutes from "./global/self"
import licenseRoutes from "./global/license"
import migrationRoutes from "./system/migrations"
import accountRoutes from "./system/accounts"
import restoreRoutes from "./system/restore"
import systemLogRoutes from "./system/logs"
@ -34,7 +33,6 @@ export const routes: Router[] = [
licenseRoutes,
pro.groups,
pro.auditLogs,
migrationRoutes,
accountRoutes,
restoreRoutes,
eventRoutes,

19
packages/worker/src/api/routes/system/migrations.ts
View File

@ -1,19 +0,0 @@
import Router from "@koa/router"
import * as migrationsController from "../../controllers/system/migrations"
import { auth } from "@budibase/backend-core"
const router: Router = new Router()
router
.post(
"/api/system/migrations/run",
auth.internalApi,
migrationsController.runMigrations
)
.get(
"/api/system/migrations/definitions",
auth.internalApi,
migrationsController.fetchDefinitions
)
export default router

63
packages/worker/src/api/routes/system/tests/migrations.spec.ts
View File

@ -1,63 +0,0 @@
const migrateFn = jest.fn()
import { TestConfiguration } from "../../../../tests"
jest.mock("../../../../migrations", () => {
return {
...jest.requireActual("../../../../migrations"),
migrate: migrateFn,
}
})
describe("/api/system/migrations", () => {
const config = new TestConfiguration()
beforeAll(async () => {
await config.beforeAll()
})
afterAll(async () => {
await config.afterAll()
})
afterEach(() => {
jest.clearAllMocks()
})
describe("POST /api/system/migrations/run", () => {
it("fails with no internal api key", async () => {
const res = await config.api.migrations.runMigrations({
headers: {},
status: 403,
})
expect(res.body).toEqual({ message: "Unauthorized", status: 403 })
expect(migrateFn).toHaveBeenCalledTimes(0)
})
it("runs migrations", async () => {
const res = await config.api.migrations.runMigrations()
expect(res.body.message).toBeDefined()
expect(migrateFn).toHaveBeenCalledTimes(1)
})
})
describe("DELETE /api/system/migrations/definitions", () => {
it("fails with no internal api key", async () => {
const res = await config.api.migrations.getMigrationDefinitions({
headers: {},
status: 403,
})
expect(res.body).toEqual({ message: "Unauthorized", status: 403 })
})
it("returns definitions", async () => {
const res = await config.api.migrations.getMigrationDefinitions()
expect(res.body).toEqual([
{
name: "global_info_sync_users",
type: "global",
},
])
})
})
})

62
packages/worker/src/migrations/index.ts
View File

@ -1,62 +0,0 @@
import { migrations, locks } from "@budibase/backend-core"
import {
Migration,
MigrationOptions,
MigrationName,
LockType,
LockName,
} from "@budibase/types"
import env from "../environment"
// migration functions
import * as syncUserInfo from "./functions/globalInfoSyncUsers"
/**
* Populate the migration function and additional configuration from
* the static migration definitions.
*/
export const buildMigrations = () => {
const definitions = migrations.DEFINITIONS
const workerMigrations: Migration[] = []
for (const definition of definitions) {
switch (definition.name) {
case MigrationName.GLOBAL_INFO_SYNC_USERS: {
// only needed in cloud
if (!env.SELF_HOSTED) {
workerMigrations.push({
...definition,
fn: syncUserInfo.run,
})
}
break
}
}
}
return workerMigrations
}
export const MIGRATIONS = buildMigrations()
export const migrate = async (options?: MigrationOptions) => {
if (env.SELF_HOSTED) {
await migrateWithLock(options)
} else {
await migrations.runMigrations(MIGRATIONS, options)
}
}
const migrateWithLock = async (options?: MigrationOptions) => {
await locks.doWithLock(
{
type: LockType.TRY_ONCE,
name: LockName.MIGRATIONS,
ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
systemLock: true,
},
async () => {
await migrations.runMigrations(MIGRATIONS, options)
}
)
}

3
packages/worker/src/tests/api/index.ts
View File

@ -6,7 +6,6 @@ import { EmailAPI } from "./email"
import { SelfAPI } from "./self"
import { UserAPI } from "./users"
import { EnvironmentAPI } from "./environment"
import { MigrationAPI } from "./migrations"
import { StatusAPI } from "./status"
import { RestoreAPI } from "./restore"
import { TenantAPI } from "./tenants"
@ -26,7 +25,6 @@ export default class API {
self: SelfAPI
users: UserAPI
environment: EnvironmentAPI
migrations: MigrationAPI
status: StatusAPI
restore: RestoreAPI
tenants: TenantAPI
@ -46,7 +44,6 @@ export default class API {
this.self = new SelfAPI(config)
this.users = new UserAPI(config)
this.environment = new EnvironmentAPI(config)
this.migrations = new MigrationAPI(config)
this.status = new StatusAPI(config)
this.restore = new RestoreAPI(config)
this.tenants = new TenantAPI(config)

17
packages/worker/src/tests/api/migrations.ts
View File

@ -1,17 +0,0 @@
import { TestAPI, TestAPIOpts } from "./base"
export class MigrationAPI extends TestAPI {
runMigrations = (opts?: TestAPIOpts) => {
return this.request
.post(`/api/system/migrations/run`)
.set(opts?.headers ? opts.headers : this.config.internalAPIHeaders())
.expect(opts?.status ? opts.status : 200)
}
getMigrationDefinitions = (opts?: TestAPIOpts) => {
return this.request
.get(`/api/system/migrations/definitions`)
.set(opts?.headers ? opts.headers : this.config.internalAPIHeaders())
.expect(opts?.status ? opts.status : 200)
}
}