From ef737a7717cf69264e2b0e932c8dabca6b7f3916 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 15 Sep 2022 14:57:27 +0100 Subject: [PATCH 1/2] Adding budibase/pro integration for plugins, controlling quota limits. --- packages/backend-core/src/db/constants.ts | 1 + packages/backend-core/src/db/utils.ts | 17 ++- .../src/api/controllers/plugin/index.ts | 113 +++--------------- packages/server/src/db/utils.js | 8 -- .../server/src/utilities/fileSystem/index.js | 7 -- packages/server/src/watch.ts | 4 +- packages/types/src/documents/global/plugin.ts | 1 + 7 files changed, 36 insertions(+), 115 deletions(-) diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts index 2c2c29cee2..62f4e8820f 100644 --- a/packages/backend-core/src/db/constants.ts +++ b/packages/backend-core/src/db/constants.ts @@ -44,6 +44,7 @@ export enum DocumentType { DEV_INFO = "devinfo", AUTOMATION_LOG = "log_au", ACCOUNT_METADATA = "acc_metadata", + PLUGIN = "plg", } export const StaticDatabases = { diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index c93c7b5662..cc20b87a58 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -3,7 +3,7 @@ import { DEFAULT_TENANT_ID, Configs } from "../constants" import env from "../environment" import { SEPARATOR, DocumentType, UNICODE_MAX, ViewName } from "./constants" import { getTenantId, getGlobalDB } from "../context" -import { getGlobalDBName } from "../tenancy/utils" +import { getGlobalDBName } from "../tenancy" import fetch from "node-fetch" import { doWithDB, allDbs } from "./index" import { getCouchInfo } from "./pouch" @@ -367,6 +367,21 @@ export const generateDevInfoID = (userId: any) => { return `${DocumentType.DEV_INFO}${SEPARATOR}${userId}` } +/** + * Generates a new plugin ID - to be used in the global DB. + * @returns {string} The new plugin ID which a plugin metadata document can be stored under. + */ +export const generatePluginID = (name: string) => { + return `${DocumentType.PLUGIN}${SEPARATOR}${name}` +} + +/** + * Gets parameters for retrieving automations, this is a utility function for the getDocParams function. + */ +export const getPluginParams = (pluginId?: string | null, otherProps = {}) => { + return getDocParams(DocumentType.PLUGIN, pluginId, otherProps) +} + /** * Returns the most granular configuration document from the DB based on the type, workspace and userID passed. * @param {Object} db - db instance to query diff --git a/packages/server/src/api/controllers/plugin/index.ts b/packages/server/src/api/controllers/plugin/index.ts index f560572082..a674c5892e 100644 --- a/packages/server/src/api/controllers/plugin/index.ts +++ b/packages/server/src/api/controllers/plugin/index.ts @@ -1,22 +1,16 @@ -import { ObjectStoreBuckets } from "../../../constants" -import { loadJSFile } from "../../../utilities/fileSystem" import { npmUpload, urlUpload, githubUpload, fileUpload } from "./uploaders" import { getGlobalDB } from "@budibase/backend-core/tenancy" import { validate } from "@budibase/backend-core/plugins" -import { generatePluginID, getPluginParams } from "../../../db/utils" -import { - uploadDirectory, - deleteFolder, -} from "@budibase/backend-core/objectStore" -import { PluginType, FileType, PluginSource, Plugin } from "@budibase/types" +import { PluginType, FileType, PluginSource } from "@budibase/types" import env from "../../../environment" import { ClientAppSocket } from "../../../websocket" -import { events } from "@budibase/backend-core" +import { db as dbCore } from "@budibase/backend-core" +import { plugins } from "@budibase/pro" export async function getPlugins(type?: PluginType) { const db = getGlobalDB() const response = await db.allDocs( - getPluginParams(null, { + dbCore.getPluginParams(null, { include_docs: true, }) ) @@ -37,7 +31,7 @@ export async function upload(ctx: any) { let docs = [] // can do single or multiple plugins for (let plugin of plugins) { - const doc = await processPlugin(plugin, PluginSource.FILE) + const doc = await processUploadedPlugin(plugin, PluginSource.FILE) docs.push(doc) } ctx.body = { @@ -91,18 +85,19 @@ export async function create(ctx: any) { ) } - const doc = await storePlugin(metadata, directory, source) + const doc = await plugins.storePlugin(metadata, directory, source) + ClientAppSocket.emit("plugins-update", { name, hash: doc.hash }) ctx.body = { message: "Plugin uploaded successfully", plugins: [doc], } + ctx.body = { plugin: doc } } catch (err: any) { const errMsg = err?.message ? err?.message : err ctx.throw(400, `Failed to import plugin: ${errMsg}`) } - ctx.status = 200 } export async function fetch(ctx: any) { @@ -110,99 +105,21 @@ export async function fetch(ctx: any) { } export async function destroy(ctx: any) { - const db = getGlobalDB() const { pluginId } = ctx.params try { - const plugin: Plugin = await db.get(pluginId) - const bucketPath = `${plugin.name}/` - await deleteFolder(ObjectStoreBuckets.PLUGINS, bucketPath) + await plugins.deletePlugin(pluginId) - await db.remove(pluginId, plugin._rev) - await events.plugin.deleted(plugin) + ctx.body = { message: `Plugin ${ctx.params.pluginId} deleted.` } } catch (err: any) { - const errMsg = err?.message ? err?.message : err - - ctx.throw(400, `Failed to delete plugin: ${errMsg}`) + ctx.throw(400, err.message) } - - ctx.message = `Plugin ${ctx.params.pluginId} deleted.` - ctx.status = 200 } -export async function storePlugin( - metadata: any, - directory: any, +export async function processUploadedPlugin( + plugin: FileType, source?: PluginSource ) { - const db = getGlobalDB() - const version = metadata.package.version, - name = metadata.package.name, - description = metadata.package.description, - hash = metadata.schema.hash - - // first open the tarball into tmp directory - const bucketPath = `${name}/` - const files = await uploadDirectory( - ObjectStoreBuckets.PLUGINS, - directory, - bucketPath - ) - const jsFile = files.find((file: any) => file.name.endsWith(".js")) - if (!jsFile) { - throw new Error(`Plugin missing .js file.`) - } - // validate the JS for a datasource - if (metadata.schema.type === PluginType.DATASOURCE) { - const js = loadJSFile(directory, jsFile.name) - // TODO: this isn't safe - but we need full node environment - // in future we should do this in a thread for safety - try { - eval(js) - } catch (err: any) { - const message = err?.message ? err.message : JSON.stringify(err) - throw new Error(`JS invalid: ${message}`) - } - } - const jsFileName = jsFile.name - const pluginId = generatePluginID(name) - - // overwrite existing docs entirely if they exist - let rev - try { - const existing = await db.get(pluginId) - rev = existing._rev - } catch (err) { - rev = undefined - } - let doc: Plugin = { - _id: pluginId, - _rev: rev, - ...metadata, - name, - version, - hash, - description, - jsUrl: `${bucketPath}${jsFileName}`, - } - - if (source) { - doc = { - ...doc, - source, - } - } - - const response = await db.put(doc) - await events.plugin.imported(doc) - ClientAppSocket.emit("plugin-update", { name, hash }) - return { - ...doc, - _rev: response.rev, - } -} - -export async function processPlugin(plugin: FileType, source?: PluginSource) { const { metadata, directory } = await fileUpload(plugin) validate(metadata?.schema) @@ -211,5 +128,7 @@ export async function processPlugin(plugin: FileType, source?: PluginSource) { throw new Error("Only component plugins are supported outside of self-host") } - return await storePlugin(metadata, directory, source) + const doc = await plugins.storePlugin(metadata, directory, source) + ClientAppSocket.emit("plugins-update", { name, hash: doc.hash }) + return doc } diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js index 64d206aeb8..a97dcada59 100644 --- a/packages/server/src/db/utils.js +++ b/packages/server/src/db/utils.js @@ -42,7 +42,6 @@ const DocumentType = { MEM_VIEW: "view", USER_FLAG: "flag", AUTOMATION_METADATA: "meta_au", - PLUGIN: "plg", } const InternalTables = { @@ -384,10 +383,3 @@ exports.getMultiIDParams = ids => { include_docs: true, } } - -/** - * Gets parameters for retrieving automations, this is a utility function for the getDocParams function. - */ -exports.getPluginParams = (pluginId = null, otherProps = {}) => { - return getDocParams(DocumentType.PLUGIN, pluginId, otherProps) -} diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 96d8c85e0b..e85fe18f94 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -112,13 +112,6 @@ exports.loadHandlebarsFile = path => { return fs.readFileSync(path, "utf8") } -/** - * Same as above just with a different name. - */ -exports.loadJSFile = (directory, name) => { - return fs.readFileSync(join(directory, name), "utf8") -} - /** * When return a file from the API need to write the file to the system temporarily so we * can create a read stream to send. diff --git a/packages/server/src/watch.ts b/packages/server/src/watch.ts index a97fda4138..4beef5ddb8 100644 --- a/packages/server/src/watch.ts +++ b/packages/server/src/watch.ts @@ -4,7 +4,7 @@ import chokidar from "chokidar" import fs from "fs" import { tenancy } from "@budibase/backend-core" import { DEFAULT_TENANT_ID } from "@budibase/backend-core/constants" -import { processPlugin } from "./api/controllers/plugin" +import { processUploadedPlugin } from "./api/controllers/plugin" export function watch() { const watchPath = path.join(env.PLUGINS_DIR, "./**/*.tar.gz") @@ -28,7 +28,7 @@ export function watch() { const split = path.split("/") const name = split[split.length - 1] console.log("Importing plugin:", path) - await processPlugin({ name, path }) + await processUploadedPlugin({ name, path }) } catch (err: any) { const message = err?.message ? err?.message : err console.error("Failed to import plugin:", message) diff --git a/packages/types/src/documents/global/plugin.ts b/packages/types/src/documents/global/plugin.ts index 8b9607c41d..a374d5496c 100644 --- a/packages/types/src/documents/global/plugin.ts +++ b/packages/types/src/documents/global/plugin.ts @@ -23,6 +23,7 @@ export interface Plugin extends Document { jsUrl?: string source: PluginSource package: { [key: string]: any } + hash: string schema: { type: PluginType [key: string]: any From ef3d0c1098140f606c5a64ad1f547975514f3a52 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 15 Sep 2022 18:43:07 +0100 Subject: [PATCH 2/2] Some minor fixes, plus typescript conversion to avoid issue in future. --- .../{component.js => component.ts} | 26 ++++----- .../api/controllers/{screen.js => screen.ts} | 53 +++++++++---------- packages/types/src/documents/app/screen.ts | 13 +++++ 3 files changed, 52 insertions(+), 40 deletions(-) rename packages/server/src/api/controllers/{component.js => component.ts} (62%) rename packages/server/src/api/controllers/{screen.js => screen.ts} (67%) diff --git a/packages/server/src/api/controllers/component.js b/packages/server/src/api/controllers/component.ts similarity index 62% rename from packages/server/src/api/controllers/component.js rename to packages/server/src/api/controllers/component.ts index e949db3042..6d65d43db6 100644 --- a/packages/server/src/api/controllers/component.js +++ b/packages/server/src/api/controllers/component.ts @@ -1,15 +1,15 @@ -const { DocumentType, getPluginParams } = require("../../db/utils") -const { getComponentLibraryManifest } = require("../../utilities/fileSystem") -const { getAppDB } = require("@budibase/backend-core/context") -const { getGlobalDB } = require("@budibase/backend-core/tenancy") +import { DocumentType } from "../../db/utils" +import { Plugin } from "@budibase/types" +import { db as dbCore, context, tenancy } from "@budibase/backend-core" +import { getComponentLibraryManifest } from "../../utilities/fileSystem" -exports.fetchAppComponentDefinitions = async function (ctx) { +exports.fetchAppComponentDefinitions = async function (ctx: any) { try { - const db = getAppDB() + const db = context.getAppDB() const app = await db.get(DocumentType.APP_METADATA) let componentManifests = await Promise.all( - app.componentLibraries.map(async library => { + app.componentLibraries.map(async (library: any) => { let manifest = await getComponentLibraryManifest(library) return { manifest, @@ -17,7 +17,7 @@ exports.fetchAppComponentDefinitions = async function (ctx) { } }) ) - const definitions = {} + const definitions: { [key: string]: any } = {} for (let { manifest, library } of componentManifests) { for (let key of Object.keys(manifest)) { if (key === "features") { @@ -33,16 +33,16 @@ exports.fetchAppComponentDefinitions = async function (ctx) { } // Add custom components - const globalDB = getGlobalDB() + const globalDB = tenancy.getGlobalDB() const response = await globalDB.allDocs( - getPluginParams(null, { + dbCore.getPluginParams(null, { include_docs: true, }) ) response.rows - .map(row => row.doc) - .filter(plugin => plugin.schema.type === "component") - .forEach(plugin => { + .map((row: any) => row.doc) + .filter((plugin: Plugin) => plugin.schema.type === "component") + .forEach((plugin: Plugin) => { const fullComponentName = `plugin/${plugin.name}` definitions[fullComponentName] = { component: fullComponentName, diff --git a/packages/server/src/api/controllers/screen.js b/packages/server/src/api/controllers/screen.ts similarity index 67% rename from packages/server/src/api/controllers/screen.js rename to packages/server/src/api/controllers/screen.ts index ca25a72a8a..08040351dd 100644 --- a/packages/server/src/api/controllers/screen.js +++ b/packages/server/src/api/controllers/screen.ts @@ -1,17 +1,16 @@ -const { - getScreenParams, - generateScreenID, - getPluginParams, - DocumentType, -} = require("../../db/utils") -const { AccessController } = require("@budibase/backend-core/roles") -const { getAppDB } = require("@budibase/backend-core/context") -const { events } = require("@budibase/backend-core") -const { getGlobalDB } = require("@budibase/backend-core/tenancy") -const { updateAppPackage } = require("./application") +import { getScreenParams, generateScreenID, DocumentType } from "../../db/utils" +import { + events, + context, + tenancy, + db as dbCore, + roles, +} from "@budibase/backend-core" +import { updateAppPackage } from "./application" +import { Plugin, ScreenProps } from "@budibase/types" -exports.fetch = async ctx => { - const db = getAppDB() +exports.fetch = async (ctx: any) => { + const db = context.getAppDB() const screens = ( await db.allDocs( @@ -19,16 +18,16 @@ exports.fetch = async ctx => { include_docs: true, }) ) - ).rows.map(element => element.doc) + ).rows.map((el: any) => el.doc) - ctx.body = await new AccessController().checkScreensAccess( + ctx.body = await new roles.AccessController().checkScreensAccess( screens, ctx.user.role._id ) } -exports.save = async ctx => { - const db = getAppDB() +exports.save = async (ctx: any) => { + const db = context.getAppDB() let screen = ctx.request.body let eventFn @@ -40,19 +39,19 @@ exports.save = async ctx => { const response = await db.put(screen) // Find any custom components being used - let pluginNames = [] + let pluginNames: string[] = [] let pluginAdded = false findPlugins(screen.props, pluginNames) if (pluginNames.length) { - const globalDB = getGlobalDB() + const globalDB = tenancy.getGlobalDB() const pluginsResponse = await globalDB.allDocs( - getPluginParams(null, { + dbCore.getPluginParams(null, { include_docs: true, }) ) const requiredPlugins = pluginsResponse.rows - .map(row => row.doc) - .filter(plugin => { + .map((row: any) => row.doc) + .filter((plugin: Plugin) => { return ( plugin.schema.type === "component" && pluginNames.includes(`plugin/${plugin.name}`) @@ -63,8 +62,8 @@ exports.save = async ctx => { const application = await db.get(DocumentType.APP_METADATA) let usedPlugins = application.usedPlugins || [] - requiredPlugins.forEach(plugin => { - if (!usedPlugins.find(x => x._id === plugin._id)) { + requiredPlugins.forEach((plugin: Plugin) => { + if (!usedPlugins.find((x: Plugin) => x._id === plugin._id)) { pluginAdded = true usedPlugins.push({ _id: plugin._id, @@ -93,8 +92,8 @@ exports.save = async ctx => { } } -exports.destroy = async ctx => { - const db = getAppDB() +exports.destroy = async (ctx: any) => { + const db = context.getAppDB() const id = ctx.params.screenId const screen = await db.get(id) @@ -107,7 +106,7 @@ exports.destroy = async ctx => { ctx.status = 200 } -const findPlugins = (component, foundPlugins) => { +const findPlugins = (component: ScreenProps, foundPlugins: string[]) => { if (!component) { return } diff --git a/packages/types/src/documents/app/screen.ts b/packages/types/src/documents/app/screen.ts index 98db658aa6..6390c3b18c 100644 --- a/packages/types/src/documents/app/screen.ts +++ b/packages/types/src/documents/app/screen.ts @@ -1,5 +1,17 @@ import { Document } from "../document" +export interface ScreenProps extends Document { + _instanceName: string + _styles: { [key: string]: any } + _component: string + _children: ScreenProps[] + size?: string + gap?: string + direction?: string + vAlign?: string + hAlign?: string +} + export interface Screen extends Document { layoutId?: string showNavigation?: boolean @@ -9,4 +21,5 @@ export interface Screen extends Document { roleId: string homeScreen?: boolean } + props: ScreenProps }