From 4813a634398afde2e3a1b581e6a6fde8a17f39c1 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Tue, 28 Sep 2021 09:48:00 +0100 Subject: [PATCH 1/3] Deprovisioning WIP --- packages/auth/src/tenancy/tenancy.js | 48 +++++++++++++++++++ .../api/controllers/row/ExternalRequest.ts | 2 +- packages/server/src/definitions/datasource.ts | 4 +- packages/server/src/integrations/base/sql.ts | 2 +- packages/server/src/integrations/mysql.ts | 12 +++-- packages/server/src/integrations/postgres.ts | 18 +++++-- packages/server/src/integrations/utils.ts | 6 ++- .../src/api/controllers/system/tenants.js | 11 +++++ .../worker/src/api/routes/system/tenants.js | 1 + 9 files changed, 91 insertions(+), 13 deletions(-) diff --git a/packages/auth/src/tenancy/tenancy.js b/packages/auth/src/tenancy/tenancy.js index ebd573496c..127ee96c6e 100644 --- a/packages/auth/src/tenancy/tenancy.js +++ b/packages/auth/src/tenancy/tenancy.js @@ -73,6 +73,54 @@ exports.tryAddTenant = async (tenantId, userId, email) => { await Promise.all(promises) } +const DocumentTypes = { + USER: "us", +} +const UNICODE_MAX = "\ufff0" + +/** + * Gets parameters for retrieving users. + * Duplicate of "../db/utils" due to circular dependency + */ +const getGlobalUserParams = (globalId, otherProps = {}) => { + if (!globalId) { + globalId = "" + } + return { + ...otherProps, + startkey: `${DocumentTypes.USER}${SEPARATOR}${globalId}`, + endkey: `${DocumentTypes.USER}${SEPARATOR}${globalId}${UNICODE_MAX}`, + } +} + +exports.deleteTenant = async tenantId => { + const globalDb = exports.getGlobalDB() + + let promises = [] + // remove the tenant entry from global info + const infoDb = getDB(PLATFORM_INFO_DB) + let tenants = await infoDb.get(TENANT_DOC) + tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId) + promises.push(infoDb.put(tenants)) + + // remove the users + const allUsers = await globalDb.allDocs( + getGlobalUserParams(null, { + include_docs: true, + }) + ) + allUsers.rows.map(row => { + promises.push(infoDb.remove(row.id, row.value.rev)) + promises.push(infoDb.remove(row.doc.email, row.value.rev)) + }) + + // remove the global db + promises.push(globalDb.destroy()) + + await Promise.all(promises) + // TODO: Delete all apps +} + exports.getGlobalDB = (tenantId = null) => { // tenant ID can be set externally, for example user API where // new tenants are being created, this may be the case diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 12db55efdc..75c3e9b492 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -546,7 +546,7 @@ module External { }, meta: { table, - } + }, } // can't really use response right now const response = await makeExternalQuery(appId, json) diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index d7d4e77961..2daef8eda7 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -1,4 +1,4 @@ -import {Table} from "./common"; +import { Table } from "./common" export enum Operation { CREATE = "CREATE", @@ -139,7 +139,7 @@ export interface QueryJson { paginate?: PaginationJson body?: object meta?: { - table?: Table, + table?: Table } extra?: { idFilter?: SearchFilters diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 91af3e1a85..c5e9bdb0bb 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -148,7 +148,7 @@ function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery { if (!resource) { resource = { fields: [] } } - let selectStatement: string|string[] = "*" + let selectStatement: string | string[] = "*" // handle select if (resource.fields && resource.fields.length > 0) { // select the resources as the format "table.columnName" - this is what is provided diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 11220afb46..c17cca0745 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -12,7 +12,11 @@ import { getSqlQuery } from "./utils" module MySQLModule { const mysql = require("mysql") const Sql = require("./base/sql") - const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") + const { + buildExternalTableId, + convertType, + copyExistingPropsOver, + } = require("./utils") const { FieldTypes } = require("../constants") interface MySQLConfig { @@ -104,7 +108,7 @@ module MySQLModule { client: any, query: SqlQuery, connect: boolean = true - ): Promise { + ): Promise { // Node MySQL is callback based, so we must wrap our call in a promise return new Promise((resolve, reject) => { if (connect) { @@ -248,9 +252,9 @@ module MySQLModule { json.extra = { idFilter: { equal: { - [primaryKey]: results.insertId + [primaryKey]: results.insertId, }, - } + }, } return json } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index e06e3936c8..db81e183d1 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -12,7 +12,11 @@ module PostgresModule { const { Pool } = require("pg") const Sql = require("./base/sql") const { FieldTypes } = require("../constants") - const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") + const { + buildExternalTableId, + convertType, + copyExistingPropsOver, + } = require("./utils") const { escapeDangerousCharacters } = require("../utilities") const JSON_REGEX = /'{.*}'::json/s @@ -193,10 +197,16 @@ module PostgresModule { } const type: string = convertType(column.data_type, TYPE_MAP) - const identity = !!(column.identity_generation || column.identity_start || column.identity_increment) - const hasDefault = typeof column.column_default === "string" && + const identity = !!( + column.identity_generation || + column.identity_start || + column.identity_increment + ) + const hasDefault = + typeof column.column_default === "string" && column.column_default.startsWith("nextval") - const isGenerated = column.is_generated && column.is_generated !== "NEVER" + const isGenerated = + column.is_generated && column.is_generated !== "NEVER" const isAuto: boolean = hasDefault || identity || isGenerated tables[tableName].schema[columnName] = { autocolumn: isAuto, diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 82c35bc2d9..6e3dc6f684 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -84,7 +84,11 @@ export function isIsoDateString(str: string) { } // add the existing relationships from the entities if they exist, to prevent them from being overridden -export function copyExistingPropsOver(tableName: string, tables: { [key: string]: any }, entities: { [key: string]: any }) { +export function copyExistingPropsOver( + tableName: string, + tables: { [key: string]: any }, + entities: { [key: string]: any } +) { if (entities && entities[tableName]) { if (entities[tableName].primaryDisplay) { tables[tableName].primaryDisplay = entities[tableName].primaryDisplay diff --git a/packages/worker/src/api/controllers/system/tenants.js b/packages/worker/src/api/controllers/system/tenants.js index e053216dd9..34f2ed9664 100644 --- a/packages/worker/src/api/controllers/system/tenants.js +++ b/packages/worker/src/api/controllers/system/tenants.js @@ -1,5 +1,6 @@ const CouchDB = require("../../../db") const { StaticDatabases } = require("@budibase/auth/db") +const { deleteTenant, getTenantId } = require("@budibase/auth/tenancy") exports.exists = async ctx => { const tenantId = ctx.request.params @@ -31,3 +32,13 @@ exports.fetch = async ctx => { } ctx.body = tenants } + +exports.delete = async ctx => { + const tenantId = getTenantId() + + if (ctx.params.tenantId !== tenantId) { + ctx.throw(403, "Unauthorized") + } + + await deleteTenant(tenantId) +} diff --git a/packages/worker/src/api/routes/system/tenants.js b/packages/worker/src/api/routes/system/tenants.js index 223ba9f26e..49c7509a67 100644 --- a/packages/worker/src/api/routes/system/tenants.js +++ b/packages/worker/src/api/routes/system/tenants.js @@ -7,5 +7,6 @@ const router = Router() router .get("/api/system/tenants/:tenantId/exists", controller.exists) .get("/api/system/tenants", adminOnly, controller.fetch) + .delete("/api/system/tenants/:tenantId", adminOnly, controller.delete) module.exports = router From 5be2a8489d4c13df9aca44d14429338144f3fc22 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 29 Sep 2021 11:36:24 +0100 Subject: [PATCH 2/3] Deprovisioning working minus apps --- packages/auth/src/tenancy/tenancy.js | 78 ++++++++++++++----- .../src/api/controllers/system/tenants.js | 8 +- 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/packages/auth/src/tenancy/tenancy.js b/packages/auth/src/tenancy/tenancy.js index 127ee96c6e..6010274442 100644 --- a/packages/auth/src/tenancy/tenancy.js +++ b/packages/auth/src/tenancy/tenancy.js @@ -93,32 +93,70 @@ const getGlobalUserParams = (globalId, otherProps = {}) => { } } -exports.deleteTenant = async tenantId => { - const globalDb = exports.getGlobalDB() +const removeTenantFromInfoDB = async tenantId => { + try { + const infoDb = getDB(PLATFORM_INFO_DB) + let tenants = await infoDb.get(TENANT_DOC) + tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId) - let promises = [] - // remove the tenant entry from global info - const infoDb = getDB(PLATFORM_INFO_DB) - let tenants = await infoDb.get(TENANT_DOC) - tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId) - promises.push(infoDb.put(tenants)) + await infoDb.put(tenants) + } catch (err) { + console.error(`Error removing tenant ${tenantId} from info db`, err) + throw err + } +} - // remove the users - const allUsers = await globalDb.allDocs( - getGlobalUserParams(null, { +const removeUsersFromInfoDB = async tenantId => { + try { + const globalDb = exports.getGlobalDB(tenantId) + const infoDb = getDB(PLATFORM_INFO_DB) + const allUsers = await globalDb.allDocs( + getGlobalUserParams(null, { + include_docs: true, + }) + ) + const allEmails = allUsers.rows.map(row => row.doc.email) + // get the id docs + let keys = allUsers.rows.map(row => row.id) + // and the email docs + keys = keys.concat(allEmails) + // retrieve the docs and delete them + const userDocs = await infoDb.allDocs({ + keys, include_docs: true, }) - ) - allUsers.rows.map(row => { - promises.push(infoDb.remove(row.id, row.value.rev)) - promises.push(infoDb.remove(row.doc.email, row.value.rev)) - }) + const toDelete = userDocs.rows.map(row => { + return { + ...row.doc, + _deleted: true, + } + }) + await infoDb.bulkDocs(toDelete) + } catch (err) { + console.error(`Error removing tenant ${tenantId} users from info db`, err) + throw err + } +} - // remove the global db - promises.push(globalDb.destroy()) +const removeGlobalDB = async tenantId => { + try { + const globalDb = exports.getGlobalDB(tenantId) + await globalDb.destroy() + } catch (err) { + console.error(`Error removing tenant ${tenantId} users from info db`, err) + throw err + } +} - await Promise.all(promises) - // TODO: Delete all apps +const removeTenantApps = async () => { + // TODO +} + +exports.deleteTenant = async tenantId => { + await removeTenantFromInfoDB(tenantId) + await removeUsersFromInfoDB(tenantId) + await removeGlobalDB(tenantId) + await removeTenantApps(tenantId) } exports.getGlobalDB = (tenantId = null) => { diff --git a/packages/worker/src/api/controllers/system/tenants.js b/packages/worker/src/api/controllers/system/tenants.js index 34f2ed9664..22a6967619 100644 --- a/packages/worker/src/api/controllers/system/tenants.js +++ b/packages/worker/src/api/controllers/system/tenants.js @@ -40,5 +40,11 @@ exports.delete = async ctx => { ctx.throw(403, "Unauthorized") } - await deleteTenant(tenantId) + try { + await deleteTenant(tenantId) + ctx.status = 204 + } catch (err) { + ctx.log.error(err) + throw err + } } From 5835716e9d8951265de71db467d4201ff7cfea98 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 29 Sep 2021 12:25:01 +0100 Subject: [PATCH 3/3] Deprovision complete --- packages/auth/deprovision.js | 1 + packages/auth/src/tenancy/deprovision.js | 81 +++++++++++++++++ packages/auth/src/tenancy/tenancy.js | 86 ------------------- .../src/api/controllers/system/tenants.js | 3 +- 4 files changed, 84 insertions(+), 87 deletions(-) create mode 100644 packages/auth/deprovision.js create mode 100644 packages/auth/src/tenancy/deprovision.js diff --git a/packages/auth/deprovision.js b/packages/auth/deprovision.js new file mode 100644 index 0000000000..b4b8dc6110 --- /dev/null +++ b/packages/auth/deprovision.js @@ -0,0 +1 @@ +module.exports = require("./src/tenancy/deprovision") diff --git a/packages/auth/src/tenancy/deprovision.js b/packages/auth/src/tenancy/deprovision.js new file mode 100644 index 0000000000..b8e5bc82cf --- /dev/null +++ b/packages/auth/src/tenancy/deprovision.js @@ -0,0 +1,81 @@ +const { getGlobalUserParams, getAllApps } = require("../db/utils") +const { getDB, getCouch } = require("../db") +const { getGlobalDB } = require("./tenancy") +const { StaticDatabases } = require("../db/constants") + +const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants +const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name + +const removeTenantFromInfoDB = async tenantId => { + try { + const infoDb = getDB(PLATFORM_INFO_DB) + let tenants = await infoDb.get(TENANT_DOC) + tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId) + + await infoDb.put(tenants) + } catch (err) { + console.error(`Error removing tenant ${tenantId} from info db`, err) + throw err + } +} + +const removeUsersFromInfoDB = async tenantId => { + try { + const globalDb = getGlobalDB(tenantId) + const infoDb = getDB(PLATFORM_INFO_DB) + const allUsers = await globalDb.allDocs( + getGlobalUserParams(null, { + include_docs: true, + }) + ) + const allEmails = allUsers.rows.map(row => row.doc.email) + // get the id docs + let keys = allUsers.rows.map(row => row.id) + // and the email docs + keys = keys.concat(allEmails) + // retrieve the docs and delete them + const userDocs = await infoDb.allDocs({ + keys, + include_docs: true, + }) + const toDelete = userDocs.rows.map(row => { + return { + ...row.doc, + _deleted: true, + } + }) + await infoDb.bulkDocs(toDelete) + } catch (err) { + console.error(`Error removing tenant ${tenantId} users from info db`, err) + throw err + } +} + +const removeGlobalDB = async tenantId => { + try { + const globalDb = getGlobalDB(tenantId) + await globalDb.destroy() + } catch (err) { + console.error(`Error removing tenant ${tenantId} users from info db`, err) + throw err + } +} + +const removeTenantApps = async tenantId => { + try { + const apps = await getAllApps(getCouch(), { all: true }) + const destroyPromises = apps.map(app => getDB(app.appId).destroy()) + await Promise.allSettled(destroyPromises) + } catch (err) { + console.error(`Error removing tenant ${tenantId} apps`, err) + throw err + } +} + +// can't live in tenancy package due to circular dependency on db/utils +exports.deleteTenant = async tenantId => { + await removeTenantFromInfoDB(tenantId) + await removeUsersFromInfoDB(tenantId) + await removeGlobalDB(tenantId) + await removeTenantApps(tenantId) +} diff --git a/packages/auth/src/tenancy/tenancy.js b/packages/auth/src/tenancy/tenancy.js index 6010274442..ebd573496c 100644 --- a/packages/auth/src/tenancy/tenancy.js +++ b/packages/auth/src/tenancy/tenancy.js @@ -73,92 +73,6 @@ exports.tryAddTenant = async (tenantId, userId, email) => { await Promise.all(promises) } -const DocumentTypes = { - USER: "us", -} -const UNICODE_MAX = "\ufff0" - -/** - * Gets parameters for retrieving users. - * Duplicate of "../db/utils" due to circular dependency - */ -const getGlobalUserParams = (globalId, otherProps = {}) => { - if (!globalId) { - globalId = "" - } - return { - ...otherProps, - startkey: `${DocumentTypes.USER}${SEPARATOR}${globalId}`, - endkey: `${DocumentTypes.USER}${SEPARATOR}${globalId}${UNICODE_MAX}`, - } -} - -const removeTenantFromInfoDB = async tenantId => { - try { - const infoDb = getDB(PLATFORM_INFO_DB) - let tenants = await infoDb.get(TENANT_DOC) - tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId) - - await infoDb.put(tenants) - } catch (err) { - console.error(`Error removing tenant ${tenantId} from info db`, err) - throw err - } -} - -const removeUsersFromInfoDB = async tenantId => { - try { - const globalDb = exports.getGlobalDB(tenantId) - const infoDb = getDB(PLATFORM_INFO_DB) - const allUsers = await globalDb.allDocs( - getGlobalUserParams(null, { - include_docs: true, - }) - ) - const allEmails = allUsers.rows.map(row => row.doc.email) - // get the id docs - let keys = allUsers.rows.map(row => row.id) - // and the email docs - keys = keys.concat(allEmails) - // retrieve the docs and delete them - const userDocs = await infoDb.allDocs({ - keys, - include_docs: true, - }) - const toDelete = userDocs.rows.map(row => { - return { - ...row.doc, - _deleted: true, - } - }) - await infoDb.bulkDocs(toDelete) - } catch (err) { - console.error(`Error removing tenant ${tenantId} users from info db`, err) - throw err - } -} - -const removeGlobalDB = async tenantId => { - try { - const globalDb = exports.getGlobalDB(tenantId) - await globalDb.destroy() - } catch (err) { - console.error(`Error removing tenant ${tenantId} users from info db`, err) - throw err - } -} - -const removeTenantApps = async () => { - // TODO -} - -exports.deleteTenant = async tenantId => { - await removeTenantFromInfoDB(tenantId) - await removeUsersFromInfoDB(tenantId) - await removeGlobalDB(tenantId) - await removeTenantApps(tenantId) -} - exports.getGlobalDB = (tenantId = null) => { // tenant ID can be set externally, for example user API where // new tenants are being created, this may be the case diff --git a/packages/worker/src/api/controllers/system/tenants.js b/packages/worker/src/api/controllers/system/tenants.js index 22a6967619..a96c5e5f9f 100644 --- a/packages/worker/src/api/controllers/system/tenants.js +++ b/packages/worker/src/api/controllers/system/tenants.js @@ -1,6 +1,7 @@ const CouchDB = require("../../../db") const { StaticDatabases } = require("@budibase/auth/db") -const { deleteTenant, getTenantId } = require("@budibase/auth/tenancy") +const { getTenantId } = require("@budibase/auth/tenancy") +const { deleteTenant } = require("@budibase/auth/deprovision") exports.exists = async ctx => { const tenantId = ctx.request.params