diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml
index cd43631992..d9def8c641 100644
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@@ -108,6 +108,8 @@ spec:
value: {{ .Values.globals.accountPortalApiKey | quote }}
- name: COOKIE_DOMAIN
value: {{ .Values.globals.cookieDomain | quote }}
+ - name: HTTP_MIGRATIONS
+ value: {{ .Values.globals.httpMigrations | quote }}
image: budibase/apps:{{ .Values.globals.appVersion }}
imagePullPolicy: Always
name: bbapps
diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml
index 9ea055c6c0..bb582f69c4 100644
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@@ -99,6 +99,7 @@ globals:
accountPortalApiKey: ""
cookieDomain: ""
platformUrl: ""
+ httpMigrations: "0"
createSecrets: true # creates an internal API key, JWT secrets and redis password for you
diff --git a/lerna.json b/lerna.json
index 6dde2921dd..b008fa64a2 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
{
- "version": "1.0.49-alpha.1",
+ "version": "1.0.49-alpha.2",
"npmClient": "yarn",
"packages": [
"packages/*"
diff --git a/packages/backend-core/db.js b/packages/backend-core/db.js
index 37b2ffbfa5..d2adf6c092 100644
--- a/packages/backend-core/db.js
+++ b/packages/backend-core/db.js
@@ -2,4 +2,5 @@ module.exports = {
...require("./src/db/utils"),
...require("./src/db/constants"),
...require("./src/db"),
+ ...require("./src/db/views"),
}
diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index 637ac47a3a..727951c8ee 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
- "version": "1.0.49-alpha.1",
+ "version": "1.0.49-alpha.2",
"description": "Budibase backend core libraries used in server and worker",
"main": "src/index.js",
"author": "Budibase",
diff --git a/packages/backend-core/src/auth.js b/packages/backend-core/src/auth.js
index 41d2bb1cc5..f6d53522d5 100644
--- a/packages/backend-core/src/auth.js
+++ b/packages/backend-core/src/auth.js
@@ -13,6 +13,7 @@ const {
appTenancy,
authError,
csrf,
+ internalApi,
} = require("./middleware")
// Strategies
@@ -44,4 +45,5 @@ module.exports = {
auditLog,
authError,
buildCsrfMiddleware: csrf,
+ internalApi,
}
diff --git a/packages/backend-core/src/middleware/index.js b/packages/backend-core/src/middleware/index.js
index 0d01fb3952..5878479152 100644
--- a/packages/backend-core/src/middleware/index.js
+++ b/packages/backend-core/src/middleware/index.js
@@ -7,6 +7,7 @@ const authenticated = require("./authenticated")
const auditLog = require("./auditLog")
const tenancy = require("./tenancy")
const appTenancy = require("./appTenancy")
+const internalApi = require("./internalApi")
const datasourceGoogle = require("./passport/datasource/google")
const csrf = require("./csrf")
@@ -20,6 +21,7 @@ module.exports = {
tenancy,
appTenancy,
authError,
+ internalApi,
datasource: {
google: datasourceGoogle,
},
diff --git a/packages/backend-core/src/middleware/internalApi.js b/packages/backend-core/src/middleware/internalApi.js
new file mode 100644
index 0000000000..275d559a9e
--- /dev/null
+++ b/packages/backend-core/src/middleware/internalApi.js
@@ -0,0 +1,14 @@
+const env = require("../environment")
+const { Headers } = require("../constants")
+
+/**
+ * API Key only endpoint.
+ */
+module.exports = async (ctx, next) => {
+ const apiKey = ctx.request.headers[Headers.API_KEY]
+ if (apiKey !== env.INTERNAL_API_KEY) {
+ ctx.throw(403, "Unauthorized")
+ }
+
+ return next()
+}
diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js
index 6b8eb3a95c..e2ed75d407 100644
--- a/packages/backend-core/src/migrations/index.js
+++ b/packages/backend-core/src/migrations/index.js
@@ -1,20 +1,17 @@
+const { DEFAULT_TENANT_ID } = require("../constants")
const { DocumentTypes } = require("../db/constants")
-const { getGlobalDB, getTenantId } = require("../tenancy")
+const { getAllApps } = require("../db/utils")
+const environment = require("../environment")
+const {
+ doInTenant,
+ getTenantIds,
+ getGlobalDBName,
+ getTenantId,
+} = require("../tenancy")
-exports.MIGRATION_DBS = {
- GLOBAL_DB: "GLOBAL_DB",
-}
-
-exports.MIGRATIONS = {
- USER_EMAIL_VIEW_CASING: "user_email_view_casing",
- QUOTAS_1: "quotas_1",
-}
-
-const DB_LOOKUP = {
- [exports.MIGRATION_DBS.GLOBAL_DB]: [
- exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
- exports.MIGRATIONS.QUOTAS_1,
- ],
+exports.MIGRATION_TYPES = {
+ GLOBAL: "global", // run once, recorded in global db, global db is provided as an argument
+ APP: "app", // run per app, recorded in each app db, app db is provided as an argument
}
exports.getMigrationsDoc = async db => {
@@ -28,40 +25,90 @@ exports.getMigrationsDoc = async db => {
}
}
-exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
+const runMigration = async (CouchDB, migration, options = {}) => {
const tenantId = getTenantId()
- try {
- let db
- if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) {
- db = getGlobalDB()
- } else {
- throw new Error(`Unrecognised migration db [${migrationDb}]`)
- }
+ const migrationType = migration.type
+ const migrationName = migration.name
- if (!DB_LOOKUP[migrationDb].includes(migrationName)) {
- throw new Error(
- `Unrecognised migration name [${migrationName}] for db [${migrationDb}]`
- )
- }
-
- const doc = await exports.getMigrationsDoc(db)
- // exit if the migration has been performed
- if (doc[migrationName]) {
- return
- }
-
- console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`)
- await migrateFn()
- console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
-
- // mark as complete
- doc[migrationName] = Date.now()
- await db.put(doc)
- } catch (err) {
- console.error(
- `[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `,
- err
+ // get the db to store the migration in
+ let dbNames
+ if (migrationType === exports.MIGRATION_TYPES.GLOBAL) {
+ dbNames = [getGlobalDBName()]
+ } else if (migrationType === exports.MIGRATION_TYPES.APP) {
+ const apps = await getAllApps(CouchDB, migration.opts)
+ dbNames = apps.map(app => app.appId)
+ } else {
+ throw new Error(
+ `[Tenant: ${tenantId}] Unrecognised migration type [${migrationType}]`
)
- throw err
+ }
+
+ // run the migration against each db
+ for (const dbName of dbNames) {
+ const db = new CouchDB(dbName)
+ try {
+ const doc = await exports.getMigrationsDoc(db)
+
+ // exit if the migration has been performed already
+ if (doc[migrationName]) {
+ if (
+ options.force &&
+ options.force[migrationType] &&
+ options.force[migrationType].includes(migrationName)
+ ) {
+ console.log(
+ `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
+ )
+ } else {
+ // the migration has already been performed
+ continue
+ }
+ }
+
+ console.log(
+ `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running`
+ )
+ // run the migration with tenant context
+ await migration.fn(db)
+ console.log(
+ `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
+ )
+
+ // mark as complete
+ doc[migrationName] = Date.now()
+ await db.put(doc)
+ } catch (err) {
+ console.error(
+ `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
+ err
+ )
+ throw err
+ }
}
}
+
+exports.runMigrations = async (CouchDB, migrations, options = {}) => {
+ console.log("Running migrations")
+ let tenantIds
+ if (environment.MULTI_TENANCY) {
+ if (!options.tenantIds || !options.tenantIds.length) {
+ // run for all tenants
+ tenantIds = await getTenantIds()
+ }
+ } else {
+ // single tenancy
+ tenantIds = [DEFAULT_TENANT_ID]
+ }
+
+ // for all tenants
+ for (const tenantId of tenantIds) {
+ // for all migrations
+ for (const migration of migrations) {
+ // run the migration
+ await doInTenant(tenantId, () =>
+ runMigration(CouchDB, migration, options)
+ )
+ }
+ }
+ console.log("Migrations complete")
+}
diff --git a/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap b/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap
index e9a18eadde..222c3b1228 100644
--- a/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap
+++ b/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap
@@ -3,7 +3,7 @@
exports[`migrations should match snapshot 1`] = `
Object {
"_id": "migrations",
- "_rev": "1-af6c272fe081efafecd2ea49a8fcbb40",
- "user_email_view_casing": 1487076708000,
+ "_rev": "1-6277abc4e3db950221768e5a2618a059",
+ "test": 1487076708000,
}
`;
diff --git a/packages/backend-core/src/migrations/tests/index.spec.js b/packages/backend-core/src/migrations/tests/index.spec.js
index 0ed16fc184..12a2e54cb3 100644
--- a/packages/backend-core/src/migrations/tests/index.spec.js
+++ b/packages/backend-core/src/migrations/tests/index.spec.js
@@ -1,7 +1,7 @@
require("../../tests/utilities/dbConfig")
-const { migrateIfRequired, MIGRATION_DBS, MIGRATIONS, getMigrationsDoc } = require("../index")
-const database = require("../../db")
+const { runMigrations, getMigrationsDoc } = require("../index")
+const CouchDB = require("../../db").getCouch()
const {
StaticDatabases,
} = require("../../db/utils")
@@ -13,8 +13,14 @@ describe("migrations", () => {
const migrationFunction = jest.fn()
+ const MIGRATIONS = [{
+ type: "global",
+ name: "test",
+ fn: migrationFunction
+ }]
+
beforeEach(() => {
- db = database.getDB(StaticDatabases.GLOBAL.name)
+ db = new CouchDB(StaticDatabases.GLOBAL.name)
})
afterEach(async () => {
@@ -22,39 +28,29 @@ describe("migrations", () => {
await db.destroy()
})
- const validMigration = () => {
- return migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction)
+ const migrate = () => {
+ return runMigrations(CouchDB, MIGRATIONS)
}
it("should run a new migration", async () => {
- await validMigration()
+ await migrate()
expect(migrationFunction).toHaveBeenCalled()
+ const doc = await getMigrationsDoc(db)
+ expect(doc.test).toBeDefined()
})
it("should match snapshot", async () => {
- await validMigration()
+ await migrate()
const doc = await getMigrationsDoc(db)
expect(doc).toMatchSnapshot()
})
it("should skip a previously run migration", async () => {
- await validMigration()
- await validMigration()
+ await migrate()
+ const previousMigrationTime = await getMigrationsDoc(db).test
+ await migrate()
+ const currentMigrationTime = await getMigrationsDoc(db).test
expect(migrationFunction).toHaveBeenCalledTimes(1)
+ expect(currentMigrationTime).toBe(previousMigrationTime)
})
-
- it("should reject an unknown migration name", async () => {
- expect(async () => {
- await migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, "bogus_name", migrationFunction)
- }).rejects.toThrow()
- expect(migrationFunction).not.toHaveBeenCalled()
- })
-
- it("should reject an unknown database name", async () => {
- expect(async () => {
- await migrateIfRequired("bogus_db", MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction)
- }).rejects.toThrow()
- expect(migrationFunction).not.toHaveBeenCalled()
- })
-
})
\ No newline at end of file
diff --git a/packages/backend-core/src/tenancy/tenancy.js b/packages/backend-core/src/tenancy/tenancy.js
index 2cd05ea925..de597eac01 100644
--- a/packages/backend-core/src/tenancy/tenancy.js
+++ b/packages/backend-core/src/tenancy/tenancy.js
@@ -148,3 +148,15 @@ exports.isUserInAppTenant = (appId, user = null) => {
const tenantId = exports.getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
return tenantId === userTenantId
}
+
+exports.getTenantIds = async () => {
+ const db = getDB(PLATFORM_INFO_DB)
+ let tenants
+ try {
+ tenants = await db.get(TENANT_DOC)
+ } catch (err) {
+ // if theres an error the doc doesn't exist, no tenants exist
+ return []
+ }
+ return (tenants && tenants.tenantIds) || []
+}
diff --git a/packages/backend-core/src/utils.js b/packages/backend-core/src/utils.js
index 8c00f2a8b8..6c71c51b9d 100644
--- a/packages/backend-core/src/utils.js
+++ b/packages/backend-core/src/utils.js
@@ -20,9 +20,6 @@ const { hash } = require("./hashing")
const userCache = require("./cache/user")
const env = require("./environment")
const { getUserSessions, invalidateSessions } = require("./security/sessions")
-const { migrateIfRequired } = require("./migrations")
-const { USER_EMAIL_VIEW_CASING } = require("./migrations").MIGRATIONS
-const { GLOBAL_DB } = require("./migrations").MIGRATION_DBS
const APP_PREFIX = DocumentTypes.APP + SEPARATOR
@@ -144,11 +141,6 @@ exports.getGlobalUserByEmail = async email => {
}
const db = getGlobalDB()
- await migrateIfRequired(GLOBAL_DB, USER_EMAIL_VIEW_CASING, async () => {
- // re-create the view with latest changes
- await createUserEmailView(db)
- })
-
try {
let users = (
await db.query(`database/${ViewNames.USER_BY_EMAIL}`, {
diff --git a/packages/bbui/package.json b/packages/bbui/package.json
index aab6f8f3a3..cb1ad164d9 100644
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
- "version": "1.0.49-alpha.1",
+ "version": "1.0.49-alpha.2",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",
diff --git a/packages/builder/package.json b/packages/builder/package.json
index 3bba72d314..000368e1a5 100644
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
- "version": "1.0.49-alpha.1",
+ "version": "1.0.49-alpha.2",
"license": "GPL-3.0",
"private": true,
"scripts": {
@@ -66,10 +66,10 @@
}
},
"dependencies": {
- "@budibase/bbui": "^1.0.49-alpha.1",
- "@budibase/client": "^1.0.49-alpha.1",
+ "@budibase/bbui": "^1.0.49-alpha.2",
+ "@budibase/client": "^1.0.49-alpha.2",
"@budibase/colorpicker": "1.1.2",
- "@budibase/string-templates": "^1.0.49-alpha.1",
+ "@budibase/string-templates": "^1.0.49-alpha.2",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",
diff --git a/packages/builder/src/pages/builder/_layout.svelte b/packages/builder/src/pages/builder/_layout.svelte
index 12a544096a..1d41af15e7 100644
--- a/packages/builder/src/pages/builder/_layout.svelte
+++ b/packages/builder/src/pages/builder/_layout.svelte
@@ -61,7 +61,7 @@
await auth.setInitInfo({ init_template: $params["?template"] })
}
- await auth.checkAuth()
+ await auth.getSelf()
await admin.init()
if (useAccountPortal && multiTenancyEnabled) {
diff --git a/packages/builder/src/pages/builder/auth/reset.svelte b/packages/builder/src/pages/builder/auth/reset.svelte
index f78dd19eb9..5e5b615d73 100644
--- a/packages/builder/src/pages/builder/auth/reset.svelte
+++ b/packages/builder/src/pages/builder/auth/reset.svelte
@@ -31,7 +31,7 @@
}
onMount(async () => {
- await auth.checkAuth()
+ await auth.getSelf()
await organisation.init()
})
diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte
index faa57e5df3..bf783fdb86 100644
--- a/packages/builder/src/pages/builder/portal/apps/index.svelte
+++ b/packages/builder/src/pages/builder/portal/apps/index.svelte
@@ -337,6 +337,14 @@
}}
class="template-card"
>
+
+
+