Merge pull request #7678 from Budibase/feature/plugin-management-ui

Plugin management UI

packages/backend-core/plugins.js

@@ -0,0 +1,3 @@
+module.exports = {
+  ...require("./src/plugin"),
+}

packages/backend-core/src/index.ts

@@ -18,6 +18,7 @@ import * as dbConstants from "./db/constants"
 import logging from "./logging"
 import pino from "./pino"
 import * as middleware from "./middleware"
+import plugins from "./plugin"
 
 // mimic the outer package exports
 import * as db from "./pkg/db"
@@ -56,6 +57,7 @@ const core = {
   errors,
   logging,
   roles,
+  plugins,
   ...pino,
   ...errorClasses,
   middleware,

packages/backend-core/src/objectStore/index.ts

@@ -307,9 +307,13 @@ export const uploadDirectory = async (
   return files
 }
 
-exports.downloadTarballDirect = async (url: string, path: string) => {
+exports.downloadTarballDirect = async (
+  url: string,
+  path: string,
+  headers = {}
+) => {
   path = sanitizeKey(path)
-  const response = await fetch(url)
+  const response = await fetch(url, { headers })
   if (!response.ok) {
     throw new Error(`unexpected response ${response.statusText}`)
   }

packages/backend-core/src/plugin/index.ts

@@ -0,0 +1,7 @@
+import * as utils from "./utils"
+
+const pkg = {
+  ...utils,
+}
+
+export = pkg

packages/backend-core/src/plugin/utils.js

@@ -1,5 +1,8 @@
-const { PluginTypes } = require("./constants")
+const {
-const { DatasourceFieldType, QueryType } = require("@budibase/types")
+  DatasourceFieldType,
+  QueryType,
+  PluginType,
+} = require("@budibase/types")
 const joi = require("joi")
 
 const DATASOURCE_TYPES = [
@@ -78,11 +81,11 @@ function validateDatasource(schema) {
 }
 
 exports.validate = schema => {
-  switch (schema.type) {
+  switch (schema?.type) {
-    case PluginTypes.COMPONENT:
+    case PluginType.COMPONENT:
       validateComponent(schema)
       break
-    case PluginTypes.DATASOURCE:
+    case PluginType.DATASOURCE:
       validateDatasource(schema)
       break
     default:

packages/bbui/src/Label/Label.svelte

@@ -4,10 +4,15 @@
 
   export let size = "M"
   export let tooltip = ""
+  export let muted
 </script>
 
 <TooltipWrapper {tooltip} {size}>
-  <label for="" class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}>
+  <label
+    class:muted
+    for=""
+    class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}
+  >
     <slot />
   </label>
 </TooltipWrapper>
@@ -17,4 +22,8 @@
     padding: 0;
     white-space: nowrap;
   }
+
+  .muted {
+    opacity: 0.5;
+  }
 </style>

packages/bbui/src/Modal/ModalContent.svelte

@@ -24,7 +24,6 @@
   export let secondaryAction = undefined
   export let secondaryButtonWarning = false
   export let dataCy = null
-
   const { hide, cancel } = getContext(Context.Modal)
   let loading = false
   $: confirmDisabled = disabled || loading
@@ -88,12 +87,11 @@
     <section class="spectrum-Dialog-content content-grid">
       <slot />
     </section>
-    {#if showCancelButton || showConfirmButton}
+    {#if showCancelButton || showConfirmButton || $$slots.footer}
       <div
         class="spectrum-ButtonGroup spectrum-Dialog-buttonGroup spectrum-Dialog-buttonGroup--noFooter"
       >
         <slot name="footer" />
-
         {#if showSecondaryButton && secondaryButtonText && secondaryAction}
           <div class="secondary-action">
             <Button

packages/builder/src/constants/index.js

@@ -57,3 +57,10 @@ export const DefaultAppTheme = {
   navBackground: "var(--spectrum-global-color-gray-50)",
   navTextColor: "var(--spectrum-global-color-gray-800)",
 }
+
+export const PluginSource = {
+  URL: "URL",
+  NPM: "NPM",
+  GITHUB: "Github",
+  FILE: "File Upload",
+}

packages/builder/src/pages/builder/portal/_layout.svelte

@@ -55,6 +55,8 @@
         },
         { title: "Auth", href: "/builder/portal/manage/auth" },
         { title: "Email", href: "/builder/portal/manage/email" },
+        { title: "Plugins", href: "/builder/portal/manage/plugins" },
+
         {
           title: "Organisation",
           href: "/builder/portal/settings/organisation",

packages/builder/src/pages/builder/portal/manage/plugins/_components/AddPluginModal.svelte

@@ -0,0 +1,127 @@
+<script>
+  import {
+    ModalContent,
+    Label,
+    Input,
+    Select,
+    Dropzone,
+    Body,
+    notifications,
+  } from "@budibase/bbui"
+  import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
+  import { plugins } from "stores/portal"
+  import { PluginSource } from "constants"
+
+  function opt(name, optional) {
+    if (optional) {
+      return { name, optional }
+    }
+    return { name }
+  }
+
+  let authOptions = {
+    [PluginSource.URL]: [opt("URL"), opt("Headers", true)],
+    [PluginSource.NPM]: [opt("URL")],
+    [PluginSource.GITHUB]: [opt("URL"), opt("Github Token", true)],
+    [PluginSource.FILE]: [opt("File Upload")],
+  }
+  let file
+  let source = PluginSource.URL
+  let dynamicValues = {}
+
+  let validation
+  $: validation = source === "File Upload" ? file : dynamicValues["URL"]
+
+  function infoMessage(optionName) {
+    switch (optionName) {
+      case PluginSource.URL:
+        return "Please specify a URL which directs to a built plugin TAR archive, you can provide headers if authentication is required."
+      case PluginSource.NPM:
+        return "Please specify the URL to a public NPM package which contains the built version of the plugin you wish to install."
+      case PluginSource.GITHUB:
+        return "Please specify the URL to a Github repository which contains built plugin releases. If this is a private repo you can provide a token to access it."
+      case PluginSource.FILE:
+        return "Please provide a built plugin TAR archive, you can build a plugin locally using the Budibase CLI."
+    }
+  }
+
+  async function save() {
+    try {
+      if (source === PluginSource.FILE) {
+        await plugins.uploadPlugin(file)
+      } else {
+        const url = dynamicValues["URL"]
+        let auth =
+          source === PluginSource.GITHUB
+            ? dynamicValues["Github Token"]
+            : source === PluginSource.URL
+            ? dynamicValues["Headers"]
+            : undefined
+        await plugins.createPlugin(source, url, auth)
+      }
+      notifications.success("Plugin added successfully.")
+    } catch (err) {
+      const msg = err?.message ? err.message : JSON.stringify(err)
+      notifications.error(`Failed to add plugin: ${msg}`)
+    }
+  }
+</script>
+
+<ModalContent
+  confirmText={"Save"}
+  onConfirm={save}
+  disabled={!validation}
+  size="M"
+  title="Add new plugin"
+>
+  <div class="form-row">
+    <Label size="M">Source</Label>
+    <Select
+      placeholder={null}
+      bind:value={source}
+      options={Object.values(PluginSource)}
+    />
+  </div>
+  <Body size="S">{infoMessage(source)}</Body>
+  {#each authOptions[source] as option}
+    {#if option.name === PluginSource.FILE}
+      <div class="form-row">
+        <Label size="M">{option.name}</Label>
+        <Dropzone
+          gallery={false}
+          value={[file]}
+          on:change={e => {
+            if (!e.detail || e.detail.length === 0) {
+              file = null
+            } else {
+              file = e.detail[0]
+            }
+          }}
+        />
+      </div>
+    {:else}
+      <div class="form-row">
+        <div>
+          <Label size="M">{option.name}</Label>
+          {#if option.optional}
+            <Label size="S" muted><i>Optional</i></Label>
+          {/if}
+        </div>
+        {#if option.name === "Headers"}
+          <KeyValueBuilder bind:object={dynamicValues[option.name]} />
+        {:else}
+          <Input bind:value={dynamicValues[option.name]} />
+        {/if}
+      </div>
+    {/if}
+  {/each}
+</ModalContent>
+
+<style>
+  .form-row {
+    display: grid;
+    grid-template-columns: 60px 1fr;
+    grid-gap: var(--spacing-l);
+    align-items: center;
+  }
+</style>

packages/builder/src/pages/builder/portal/manage/plugins/_components/DeletePluginModal.svelte

@@ -0,0 +1,33 @@
+<script>
+  import { Body, ModalContent, notifications } from "@budibase/bbui"
+  import { plugins } from "stores/portal"
+  import { createEventDispatcher } from "svelte"
+
+  export let plugin
+
+  let dispatch = createEventDispatcher()
+
+  async function deletePlugin() {
+    try {
+      await plugins.deletePlugin(plugin._id)
+      notifications.success(`Plugin ${plugin?.name} deleted.`)
+      dispatch("deleted")
+    } catch (error) {
+      const msg = error?.message ? error.message : JSON.stringify(error)
+      notifications.error(`Error deleting plugin: ${msg}`)
+    }
+  }
+</script>
+
+<ModalContent
+  warning
+  onConfirm={deletePlugin}
+  title="Delete Plugin"
+  confirmText="Delete plugin"
+  cancelText="Cancel"
+  showCloseIcon={false}
+>
+  <Body>
+    Are you sure you want to delete <strong>{plugin?.name}</strong>
+  </Body>
+</ModalContent>

packages/builder/src/pages/builder/portal/manage/plugins/_components/PluginRow.svelte

@@ -0,0 +1,145 @@
+<script>
+  import {
+    Icon,
+    Body,
+    Modal,
+    ModalContent,
+    Button,
+    Label,
+    Input,
+  } from "@budibase/bbui"
+  import DeletePluginModal from "../_components/DeletePluginModal.svelte"
+
+  export let plugin
+
+  let detailsModal
+  let deleteModal
+
+  let icon =
+    plugin.schema.type === "component"
+      ? plugin.schema.schema.icon || "Book"
+      : plugin.schema.schema.icon || "Beaker"
+
+  function pluginDeleted() {
+    if (detailsModal) {
+      detailsModal.hide()
+    }
+  }
+</script>
+
+<div class="row" on:click={() => detailsModal.show()}>
+  <div class="title">
+    <div class="name">
+      <div>
+        <Icon size="M" name={icon} />
+      </div>
+      <div>
+        <Body
+          size="S"
+          color="var(--spectrum-global-color-gray-900)"
+          weight="800"
+        >
+          {plugin.name}
+        </Body>
+      </div>
+    </div>
+  </div>
+  <div class="desktop">{plugin.version}</div>
+  <div class="desktop">
+    {plugin.schema.type.charAt(0).toUpperCase() + plugin.schema.type.slice(1)}
+  </div>
+  <div>
+    <Icon name="ChevronRight" />
+  </div>
+</div>
+
+<Modal bind:this={detailsModal}>
+  <ModalContent
+    size="M"
+    title="Plugin details"
+    showConfirmButton={false}
+    showCancelButton={false}
+  >
+    <div class="details-row">
+      <Label size="M">Name</Label>
+      <Input disabled value={plugin.name} />
+    </div>
+
+    <div class="details-row">
+      <Label size="M">Type</Label>
+      <Input
+        disabled
+        value={plugin.schema.type.charAt(0).toUpperCase() +
+          plugin.schema.type.slice(1)}
+      />
+    </div>
+    <div class="details-row">
+      <Label size="M">Source</Label>
+      <Input disabled value={plugin.source || "N/A"} />
+    </div>
+    <div class="details-row">
+      <Label size="M">Version</Label>
+      <Input disabled value={plugin.version} />
+    </div>
+    <div class="details-row">
+      <Label size="M">License</Label>
+      <Input disabled value={plugin.package.license} />
+    </div>
+    <div class="details-row">
+      <Label size="M">Author</Label>
+      <Input disabled value={plugin.package.author || "N/A"} />
+    </div>
+
+    <div class="footer" slot="footer">
+      <Button newStyles on:click={deleteModal.show()} warning>Delete</Button>
+    </div>
+  </ModalContent>
+
+  <Modal bind:this={deleteModal}>
+    <DeletePluginModal {plugin} on:deleted={pluginDeleted} />
+  </Modal>
+</Modal>
+
+<style>
+  .row {
+    display: grid;
+    grid-template-columns: 1fr 110px 140px 20px;
+    align-items: center;
+    background: var(--background);
+    border-radius: 4px;
+    padding: 0 16px;
+    height: 56px;
+    background: var(--spectrum-global-color-gray-50);
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    transition: background 130ms ease-out;
+  }
+  .row:hover {
+    cursor: pointer;
+    background: var(--spectrum-global-color-gray-75);
+  }
+
+  .name {
+    grid-gap: var(--spacing-m);
+    grid-template-columns: 75px 75px;
+    align-items: center;
+    display: flex;
+  }
+
+  .details-row {
+    display: grid;
+    grid-template-columns: 60px 1fr;
+    grid-gap: var(--spacing-l) var(--spacing-l);
+    align-items: center;
+  }
+
+  @media (max-width: 640px) {
+    .desktop {
+      display: none !important;
+    }
+  }
+
+  .footer {
+    display: flex;
+    gap: var(--spacing-l);
+  }
+</style>

packages/builder/src/pages/builder/portal/manage/plugins/index.svelte

@@ -0,0 +1,95 @@
+<script>
+  import {
+    Layout,
+    Heading,
+    Body,
+    Button,
+    Select,
+    Divider,
+    Modal,
+    Search,
+  } from "@budibase/bbui"
+  import { onMount } from "svelte"
+  import { plugins } from "stores/portal"
+  import PluginRow from "./_components/PluginRow.svelte"
+  import AddPluginModal from "./_components/AddPluginModal.svelte"
+
+  let modal
+  let searchTerm = ""
+  let filter = "all"
+  let filterOptions = [
+    { label: "All plugins", value: "all" },
+    { label: "Components", value: "component" },
+    { label: "Datasources", value: "datasource" },
+  ]
+
+  $: filteredPlugins = $plugins
+    .filter(plugin => {
+      return filter === "all" || plugin.schema.type === filter
+    })
+    .filter(plugin => {
+      return (
+        !searchTerm ||
+        plugin?.name?.toLowerCase().includes(searchTerm.toLowerCase())
+      )
+    })
+
+  onMount(async () => {
+    await plugins.load()
+  })
+</script>
+
+<Layout noPadding>
+  <Layout gap="XS" noPadding>
+    <Heading size="M">Plugins</Heading>
+    <Body>Add your own custom datasources and components</Body>
+  </Layout>
+  <Divider size="S" />
+  <Layout noPadding>
+    <div class="controls">
+      <div>
+        <Button on:click={modal.show} newStyles cta icon={"Add"}>
+          Add plugin
+        </Button>
+      </div>
+      <div class="filters">
+        <div class="select">
+          <Select
+            bind:value={filter}
+            placeholder={null}
+            options={filterOptions}
+            autoWidth
+            quiet
+          />
+        </div>
+        <Search bind:value={searchTerm} placeholder="Search plugins" />
+      </div>
+    </div>
+    {#if filteredPlugins?.length}
+      <Layout noPadding gap="S">
+        {#each filteredPlugins as plugin}
+          <PluginRow {plugin} />
+        {/each}
+      </Layout>
+    {/if}
+  </Layout>
+</Layout>
+
+<Modal bind:this={modal}>
+  <AddPluginModal />
+</Modal>
+
+<style>
+  .filters {
+    display: flex;
+    gap: var(--spacing-xl);
+  }
+  .controls {
+    display: flex;
+    gap: var(--spacing-xl);
+    justify-content: space-between;
+  }
+  .controls :global(.spectrum-Search) {
+    width: 200px;
+  }
+</style>

packages/builder/src/pages/builder/portal/settings/organisation.svelte

@@ -15,7 +15,6 @@
   import { API } from "api"
   import { writable } from "svelte/store"
   import { redirect } from "@roxi/routify"
-  import { onMount } from "svelte"
 
   // Only admins allowed here
   $: {
@@ -34,12 +33,11 @@
   })
   let loading = false
 
-  async function uploadLogo() {
+  async function uploadLogo(file) {
     try {
       let data = new FormData()
-      data.append("file", $values.logo)
+      data.append("file", file)
-      await API.uploadPlugin(data)
+      await API.uploadLogo(data)
-      notifications.success("Plugin uploaded successfully")
     } catch (error) {
       notifications.error("Error uploading logo")
     }
@@ -73,11 +71,6 @@
     }
     loading = false
   }
-
-  onMount(async () => {
-    const plugins = await API.getPlugins()
-    console.log(plugins)
-  })
 </script>
 
 {#if $auth.isAdmin}
@@ -95,14 +88,14 @@
       <Heading size="S">Information</Heading>
       <Body size="S">Here you can update your logo and organization name.</Body>
     </Layout>
-    <div class="fields">
+    <div className="fields">
-      <div class="field">
+      <div className="field">
         <Label size="L">Org. name</Label>
         <Input thin bind:value={$values.company} />
       </div>
-      <div class="field logo">
+      <div className="field logo">
         <Label size="L">Logo</Label>
-        <div class="file">
+        <div className="file">
           <Dropzone
             value={[$values.logo]}
             on:change={e => {
@@ -113,7 +106,6 @@
               }
             }}
           />
-          <button on:click={uploadLogo}>Upload</button>
         </div>
       </div>
     </div>
@@ -123,8 +115,8 @@
         <Heading size="S">Platform</Heading>
         <Body size="S">Here you can set up general platform settings.</Body>
       </Layout>
-      <div class="fields">
+      <div className="fields">
-        <div class="field">
+        <div className="field">
           <Label
             size="L"
             tooltip={"Update the Platform URL to match your Budibase web URL. This keeps email templates and authentication configs up to date."}
@@ -158,15 +150,18 @@
     display: grid;
     grid-gap: var(--spacing-m);
   }
+
   .field {
     display: grid;
     grid-template-columns: 100px 1fr;
     grid-gap: var(--spacing-l);
     align-items: center;
   }
+
   .file {
     max-width: 30ch;
   }
+
   .logo {
     align-items: start;
   }

packages/builder/src/stores/portal/index.js

@@ -8,3 +8,4 @@ export { oidc } from "./oidc"
 export { templates } from "./templates"
 export { licensing } from "./licensing"
 export { groups } from "./groups"
+export { plugins } from "./plugins"

packages/builder/src/stores/portal/plugins.js

@@ -0,0 +1,77 @@
+import { writable } from "svelte/store"
+import { API } from "api"
+import { PluginSource } from "constants"
+
+export function createPluginsStore() {
+  const { subscribe, set, update } = writable([])
+
+  async function load() {
+    const plugins = await API.getPlugins()
+    set(plugins)
+  }
+
+  async function deletePlugin(pluginId) {
+    await API.deletePlugin(pluginId)
+    update(state => {
+      state = state.filter(existing => existing._id !== pluginId)
+      return state
+    })
+  }
+
+  async function createPlugin(source, url, auth = null) {
+    let pluginData = {
+      source,
+      url,
+    }
+
+    switch (source) {
+      case PluginSource.URL:
+        pluginData.headers = auth
+        break
+      case PluginSource.GITHUB:
+        pluginData.githubToken = auth
+        break
+    }
+
+    let res = await API.createPlugin(pluginData)
+
+    let newPlugin = res.plugins[0]
+    update(state => {
+      const currentIdx = state.findIndex(plugin => plugin._id === newPlugin._id)
+      if (currentIdx >= 0) {
+        state.splice(currentIdx, 1, newPlugin)
+      } else {
+        state.push(newPlugin)
+      }
+      return state
+    })
+  }
+
+  async function uploadPlugin(file) {
+    if (!file) {
+      return
+    }
+    let data = new FormData()
+    data.append("file", file)
+    let resp = await API.uploadPlugin(data)
+    let newPlugin = resp.plugins[0]
+    update(state => {
+      const currentIdx = state.findIndex(plugin => plugin._id === newPlugin._id)
+      if (currentIdx >= 0) {
+        state.splice(currentIdx, 1, newPlugin)
+      } else {
+        state.push(newPlugin)
+      }
+      return state
+    })
+  }
+  return {
+    subscribe,
+    load,
+    createPlugin,
+    deletePlugin,
+    uploadPlugin,
+  }
+}
+
+export const plugins = createPluginsStore()

packages/cli/src/plugins/constants.js

@@ -1,6 +0,0 @@
-exports.PluginTypes = {
-  COMPONENT: "component",
-  DATASOURCE: "datasource",
-}
-
-exports.PLUGIN_TYPES_ARR = Object.values(exports.PluginTypes)

packages/cli/src/plugins/index.js

@@ -3,8 +3,8 @@ const { CommandWords } = require("../constants")
 const { getSkeleton, fleshOutSkeleton } = require("./skeleton")
 const questions = require("../questions")
 const fs = require("fs")
-const { PLUGIN_TYPES_ARR } = require("./constants")
+const { PLUGIN_TYPE_ARR } = require("@budibase/types")
-const { validate } = require("./validate")
+const { validate } = require("@budibase/backend-core/plugins")
 const { runPkgCommand } = require("../exec")
 const { join } = require("path")
 const { success, error, info } = require("../utils")
@@ -24,7 +24,7 @@ function checkInPlugin() {
 
 async function init(opts) {
   const type = opts["init"] || opts
-  if (!type || !PLUGIN_TYPES_ARR.includes(type)) {
+  if (!type || !PLUGIN_TYPE_ARR.includes(type)) {
     console.log(
       error(
         "Please provide a type to init, either 'component' or 'datasource'."

packages/frontend-core/src/api/plugins.js

@@ -5,12 +5,22 @@ export const buildPluginEndpoints = API => ({
    */
   uploadPlugin: async data => {
     return await API.post({
-      url: "/api/plugin/upload",
+      url: `/api/plugin/upload`,
       body: data,
       json: false,
     })
   },
 
+  /**
+   * Creates a plugin from URL, Github or NPM
+   */
+  createPlugin: async data => {
+    return await API.post({
+      url: `/api/plugin`,
+      body: data,
+    })
+  },
+
   /**
    * Gets a list of all plugins
    */
@@ -19,4 +29,16 @@ export const buildPluginEndpoints = API => ({
       url: "/api/plugin",
     })
   },
+
+  /**
+   * Deletes a plugin.
+   * @param pluginId the ID of the plugin to delete
+   *
+   * * @param pluginId the revision of the plugin to delete
+   */
+  deletePlugin: async pluginId => {
+    return await API.delete({
+      url: `/api/plugin/${pluginId}`,
+    })
+  },
 })

packages/server/src/api/controllers/plugin.ts

@@ -1,110 +0,0 @@
-import { ObjectStoreBuckets } from "../../constants"
-import { extractPluginTarball, loadJSFile } from "../../utilities/fileSystem"
-import { getGlobalDB } from "@budibase/backend-core/tenancy"
-import { generatePluginID, getPluginParams } from "../../db/utils"
-import { uploadDirectory } from "@budibase/backend-core/objectStore"
-import { PluginType, FileType } from "@budibase/types"
-import env from "../../environment"
-
-export async function getPlugins(type?: PluginType) {
-  const db = getGlobalDB()
-  const response = await db.allDocs(
-    getPluginParams(null, {
-      include_docs: true,
-    })
-  )
-  const plugins = response.rows.map((row: any) => row.doc)
-  if (type) {
-    return plugins.filter((plugin: any) => plugin.schema?.type === type)
-  } else {
-    return plugins
-  }
-}
-
-export async function upload(ctx: any) {
-  const plugins: FileType[] =
-    ctx.request.files.file.length > 1
-      ? Array.from(ctx.request.files.file)
-      : [ctx.request.files.file]
-  try {
-    let docs = []
-    // can do single or multiple plugins
-    for (let plugin of plugins) {
-      const doc = await processPlugin(plugin)
-      docs.push(doc)
-    }
-    ctx.body = {
-      message: "Plugin(s) uploaded successfully",
-      plugins: docs,
-    }
-  } catch (err: any) {
-    const errMsg = err?.message ? err?.message : err
-    ctx.throw(400, `Failed to import plugin: ${errMsg}`)
-  }
-}
-
-export async function fetch(ctx: any) {
-  ctx.body = await getPlugins()
-}
-
-export async function destroy(ctx: any) {}
-
-export async function processPlugin(plugin: FileType) {
-  if (!env.SELF_HOSTED) {
-    throw new Error("Plugins not supported outside of self-host.")
-  }
-  const db = getGlobalDB()
-  const { metadata, directory } = await extractPluginTarball(plugin)
-  const version = metadata.package.version,
-    name = metadata.package.name,
-    description = metadata.package.description
-
-  // first open the tarball into tmp directory
-  const bucketPath = `${name}/`
-  const files = await uploadDirectory(
-    ObjectStoreBuckets.PLUGINS,
-    directory,
-    bucketPath
-  )
-  const jsFile = files.find((file: any) => file.name.endsWith(".js"))
-  if (!jsFile) {
-    throw new Error(`Plugin missing .js file.`)
-  }
-  // validate the JS for a datasource
-  if (metadata.schema.type === PluginType.DATASOURCE) {
-    const js = loadJSFile(directory, jsFile.name)
-    // TODO: this isn't safe - but we need full node environment
-    // in future we should do this in a thread for safety
-    try {
-      eval(js)
-    } catch (err: any) {
-      const message = err?.message ? err.message : JSON.stringify(err)
-      throw new Error(`JS invalid: ${message}`)
-    }
-  }
-  const jsFileName = jsFile.name
-  const pluginId = generatePluginID(name)
-
-  // overwrite existing docs entirely if they exist
-  let rev
-  try {
-    const existing = await db.get(pluginId)
-    rev = existing._rev
-  } catch (err) {
-    rev = undefined
-  }
-  const doc = {
-    _id: pluginId,
-    _rev: rev,
-    ...metadata,
-    name,
-    version,
-    description,
-    jsUrl: `${bucketPath}${jsFileName}`,
-  }
-  const response = await db.put(doc)
-  return {
-    ...doc,
-    _rev: response.rev,
-  }
-}

packages/server/src/api/controllers/plugin/file.ts

@@ -0,0 +1,15 @@
+import {
+  createTempFolder,
+  getPluginMetadata,
+  extractTarball,
+} from "../../../utilities/fileSystem"
+
+export async function fileUpload(file: { name: string; path: string }) {
+  if (!file.name.endsWith(".tar.gz")) {
+    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  }
+  const path = createTempFolder(file.name.split(".tar.gz")[0])
+  await extractTarball(file.path, path)
+
+  return await getPluginMetadata(path)
+}

packages/server/src/api/controllers/plugin/github.ts

@@ -0,0 +1,75 @@
+import { getPluginMetadata } from "../../../utilities/fileSystem"
+import fetch from "node-fetch"
+import { downloadUnzipTarball } from "./utils"
+
+export async function request(
+  url: string,
+  headers: { [key: string]: string },
+  err: string
+) {
+  const response = await fetch(url, { headers })
+  if (response.status >= 300) {
+    const respErr = await response.text()
+    throw new Error(`Error: ${err} - ${respErr}`)
+  }
+  return response.json()
+}
+
+export async function githubUpload(url: string, name = "", token = "") {
+  let githubUrl = url
+
+  if (!githubUrl.includes("https://github.com/")) {
+    throw new Error("The plugin origin must be from Github")
+  }
+
+  if (url.includes(".git")) {
+    githubUrl = url.replace(".git", "")
+  }
+
+  const githubApiUrl = githubUrl.replace(
+    "https://github.com/",
+    "https://api.github.com/repos/"
+  )
+  const headers: any = token ? { Authorization: `Bearer ${token}` } : {}
+  const pluginDetails = await request(
+    githubApiUrl,
+    headers,
+    "Repository not found"
+  )
+  const pluginName = pluginDetails.name || name
+  const pluginLatestReleaseUrl = pluginDetails?.["releases_url"]
+    ? pluginDetails?.["releases_url"].replace("{/id}", "/latest")
+    : undefined
+  if (!pluginLatestReleaseUrl) {
+    throw new Error("Github release not found")
+  }
+
+  const pluginReleaseDetails = await request(
+    pluginLatestReleaseUrl,
+    headers,
+    "Github latest release not found"
+  )
+  const pluginReleaseTarballAsset = pluginReleaseDetails?.assets?.find(
+    (x: any) => x?.["content_type"] === "application/gzip"
+  )
+  const pluginLastReleaseTarballUrl =
+    pluginReleaseTarballAsset?.["browser_download_url"]
+  if (!pluginLastReleaseTarballUrl) {
+    throw new Error("Github latest release url not found")
+  }
+
+  try {
+    const path = await downloadUnzipTarball(
+      pluginLastReleaseTarballUrl,
+      pluginName,
+      headers
+    )
+    return await getPluginMetadata(path)
+  } catch (err: any) {
+    let errMsg = err?.message || err
+    if (errMsg === "unexpected response Not Found") {
+      errMsg = "Github release tarball not found"
+    }
+    throw new Error(errMsg)
+  }
+}

packages/server/src/api/controllers/plugin/index.ts

@@ -0,0 +1,202 @@
+import { ObjectStoreBuckets } from "../../../constants"
+import { loadJSFile } from "../../../utilities/fileSystem"
+import { npmUpload, urlUpload, githubUpload, fileUpload } from "./uploaders"
+import { getGlobalDB } from "@budibase/backend-core/tenancy"
+import { validate } from "@budibase/backend-core/plugins"
+import { generatePluginID, getPluginParams } from "../../../db/utils"
+import {
+  uploadDirectory,
+  deleteFolder,
+} from "@budibase/backend-core/objectStore"
+import { PluginType, FileType, PluginSource } from "@budibase/types"
+import env from "../../../environment"
+
+export async function getPlugins(type?: PluginType) {
+  const db = getGlobalDB()
+  const response = await db.allDocs(
+    getPluginParams(null, {
+      include_docs: true,
+    })
+  )
+  const plugins = response.rows.map((row: any) => row.doc)
+  if (type) {
+    return plugins.filter((plugin: any) => plugin.schema?.type === type)
+  } else {
+    return plugins
+  }
+}
+
+export async function upload(ctx: any) {
+  const plugins: FileType[] =
+    ctx.request.files.file.length > 1
+      ? Array.from(ctx.request.files.file)
+      : [ctx.request.files.file]
+  try {
+    let docs = []
+    // can do single or multiple plugins
+    for (let plugin of plugins) {
+      const doc = await processPlugin(plugin, PluginSource.FILE)
+      docs.push(doc)
+    }
+    ctx.body = {
+      message: "Plugin(s) uploaded successfully",
+      plugins: docs,
+    }
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+
+    ctx.throw(400, `Failed to import plugin: ${errMsg}`)
+  }
+}
+
+export async function create(ctx: any) {
+  const { source, url, headers, githubToken } = ctx.request.body
+
+  if (!env.SELF_HOSTED) {
+    ctx.throw(400, "Plugins not supported outside of self-host.")
+  }
+
+  try {
+    let metadata
+    let directory
+    // Generating random name as a backup and needed for url
+    let name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000)
+
+    switch (source) {
+      case PluginSource.NPM:
+        const { metadata: metadataNpm, directory: directoryNpm } =
+          await npmUpload(url, name)
+        metadata = metadataNpm
+        directory = directoryNpm
+        break
+      case PluginSource.GITHUB:
+        const { metadata: metadataGithub, directory: directoryGithub } =
+          await githubUpload(url, name, githubToken)
+        metadata = metadataGithub
+        directory = directoryGithub
+        break
+      case PluginSource.URL:
+        const headersObj = headers || {}
+        const { metadata: metadataUrl, directory: directoryUrl } =
+          await urlUpload(url, name, headersObj)
+        metadata = metadataUrl
+        directory = directoryUrl
+        break
+    }
+
+    validate(metadata?.schema)
+
+    const doc = await storePlugin(metadata, directory, source)
+
+    ctx.body = {
+      message: "Plugin uploaded successfully",
+      plugins: [doc],
+    }
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+
+    ctx.throw(400, `Failed to import plugin: ${errMsg}`)
+  }
+  ctx.status = 200
+}
+
+export async function fetch(ctx: any) {
+  ctx.body = await getPlugins()
+}
+
+export async function destroy(ctx: any) {
+  const db = getGlobalDB()
+  const { pluginId } = ctx.params
+
+  try {
+    const plugin = await db.get(pluginId)
+    const bucketPath = `${plugin.name}/`
+    await deleteFolder(ObjectStoreBuckets.PLUGINS, bucketPath)
+
+    await db.remove(pluginId, plugin._rev)
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+
+    ctx.throw(400, `Failed to delete plugin: ${errMsg}`)
+  }
+
+  ctx.message = `Plugin ${ctx.params.pluginId} deleted.`
+  ctx.status = 200
+}
+
+export async function storePlugin(
+  metadata: any,
+  directory: any,
+  source?: string
+) {
+  const db = getGlobalDB()
+  const version = metadata.package.version,
+    name = metadata.package.name,
+    description = metadata.package.description
+
+  // first open the tarball into tmp directory
+  const bucketPath = `${name}/`
+  const files = await uploadDirectory(
+    ObjectStoreBuckets.PLUGINS,
+    directory,
+    bucketPath
+  )
+  const jsFile = files.find((file: any) => file.name.endsWith(".js"))
+  if (!jsFile) {
+    throw new Error(`Plugin missing .js file.`)
+  }
+  // validate the JS for a datasource
+  if (metadata.schema.type === PluginType.DATASOURCE) {
+    const js = loadJSFile(directory, jsFile.name)
+    // TODO: this isn't safe - but we need full node environment
+    // in future we should do this in a thread for safety
+    try {
+      eval(js)
+    } catch (err: any) {
+      const message = err?.message ? err.message : JSON.stringify(err)
+      throw new Error(`JS invalid: ${message}`)
+    }
+  }
+  const jsFileName = jsFile.name
+  const pluginId = generatePluginID(name)
+
+  // overwrite existing docs entirely if they exist
+  let rev
+  try {
+    const existing = await db.get(pluginId)
+    rev = existing._rev
+  } catch (err) {
+    rev = undefined
+  }
+  let doc = {
+    _id: pluginId,
+    _rev: rev,
+    ...metadata,
+    name,
+    version,
+    description,
+    jsUrl: `${bucketPath}${jsFileName}`,
+  }
+
+  if (source) {
+    doc = {
+      ...doc,
+      source,
+    }
+  }
+
+  const response = await db.put(doc)
+  return {
+    ...doc,
+    _rev: response.rev,
+  }
+}
+
+export async function processPlugin(plugin: FileType, source?: string) {
+  if (!env.SELF_HOSTED) {
+    throw new Error("Plugins not supported outside of self-host.")
+  }
+
+  const { metadata, directory } = await fileUpload(plugin)
+  return await storePlugin(metadata, directory, source)
+}

packages/server/src/api/controllers/plugin/npm.ts

@@ -0,0 +1,56 @@
+import {
+  getPluginMetadata,
+  findFileRec,
+  extractTarball,
+  deleteFolderFileSystem,
+} from "../../../utilities/fileSystem"
+import fetch from "node-fetch"
+import { join } from "path"
+import { downloadUnzipTarball } from "./utils"
+
+export async function npmUpload(url: string, name: string, headers = {}) {
+  let npmTarballUrl = url
+  let pluginName = name
+
+  if (
+    !npmTarballUrl.includes("https://www.npmjs.com") &&
+    !npmTarballUrl.includes("https://registry.npmjs.org")
+  ) {
+    throw new Error("The plugin origin must be from NPM")
+  }
+
+  if (!npmTarballUrl.includes(".tgz")) {
+    const npmPackageURl = url.replace(
+      "https://www.npmjs.com/package/",
+      "https://registry.npmjs.org/"
+    )
+    const response = await fetch(npmPackageURl)
+    if (response.status !== 200) {
+      throw new Error("NPM Package not found")
+    }
+
+    let npmDetails = await response.json()
+    pluginName = npmDetails.name
+    const npmVersion = npmDetails["dist-tags"].latest
+    npmTarballUrl = npmDetails?.versions?.[npmVersion]?.dist?.tarball
+
+    if (!npmTarballUrl) {
+      throw new Error("NPM tarball url not found")
+    }
+  }
+
+  const path = await downloadUnzipTarball(npmTarballUrl, pluginName, headers)
+  const tarballPluginFile = findFileRec(path, ".tar.gz")
+  if (!tarballPluginFile) {
+    throw new Error("Tarball plugin file not found")
+  }
+
+  try {
+    await extractTarball(tarballPluginFile, path)
+    deleteFolderFileSystem(join(path, "package"))
+  } catch (err: any) {
+    throw new Error(err)
+  }
+
+  return await getPluginMetadata(path)
+}

packages/server/src/api/controllers/plugin/uploaders.ts

@@ -0,0 +1,4 @@
+export { fileUpload } from "./file"
+export { githubUpload } from "./github"
+export { npmUpload } from "./npm"
+export { urlUpload } from "./url"

packages/server/src/api/controllers/plugin/url.ts

@@ -0,0 +1,12 @@
+import { downloadUnzipTarball } from "./utils"
+import { getPluginMetadata } from "../../../utilities/fileSystem"
+
+export async function urlUpload(url: string, name = "", headers = {}) {
+  if (!url.includes(".tar.gz")) {
+    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  }
+
+  const path = await downloadUnzipTarball(url, name, headers)
+
+  return await getPluginMetadata(path)
+}

packages/server/src/api/controllers/plugin/utils.ts

@@ -0,0 +1,19 @@
+import {
+  createTempFolder,
+  downloadTarballDirect,
+} from "../../../utilities/fileSystem"
+
+export async function downloadUnzipTarball(
+  url: string,
+  name: string,
+  headers = {}
+) {
+  try {
+    const path = createTempFolder(name)
+    await downloadTarballDirect(url, path, headers)
+
+    return path
+  } catch (e: any) {
+    throw new Error(e.message)
+  }
+}

packages/server/src/api/routes/plugin.ts

@@ -7,6 +7,7 @@ const router = new Router()
 
 router
   .post("/api/plugin/upload", authorized(BUILDER), controller.upload)
+  .post("/api/plugin", authorized(BUILDER), controller.create)
   .get("/api/plugin", authorized(BUILDER), controller.fetch)
   .delete("/api/plugin/:pluginId", authorized(BUILDER), controller.destroy)
 

packages/server/src/module.d.ts

@@ -11,3 +11,4 @@ declare module "@budibase/backend-core/encryption"
 declare module "@budibase/backend-core/utils"
 declare module "@budibase/backend-core/redis"
 declare module "@budibase/backend-core/objectStore"
+declare module "@budibase/backend-core/plugins"

packages/server/src/utilities/fileSystem/index.js

@@ -15,6 +15,7 @@ const {
   streamUpload,
   deleteFolder,
   downloadTarball,
+  downloadTarballDirect,
   deleteFiles,
 } = require("./utilities")
 const { updateClientLibrary } = require("./clientLibrary")
@@ -338,31 +339,57 @@ exports.cleanup = appIds => {
   }
 }
 
-exports.extractPluginTarball = async file => {
+const createTempFolder = item => {
-  if (!file.name.endsWith(".tar.gz")) {
+  const path = join(budibaseTempDir(), item)
-    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  try {
+    // remove old tmp directories automatically - don't combine
+    if (fs.existsSync(path)) {
+      fs.rmSync(path, { recursive: true, force: true })
+    }
+    fs.mkdirSync(path)
+  } catch (err) {
+    throw new Error(`Path cannot be created: ${err.message}`)
   }
-  const path = join(budibaseTempDir(), file.name.split(".tar.gz")[0])
+
-  // remove old tmp directories automatically - don't combine
+  return path
-  if (fs.existsSync(path)) {
+}
-    fs.rmSync(path, { recursive: true, force: true })
+exports.createTempFolder = createTempFolder
-  }
+
-  fs.mkdirSync(path)
+const extractTarball = async (fromFilePath, toPath) => {
   await tar.extract({
-    file: file.path,
+    file: fromFilePath,
-    C: path,
+    C: toPath,
   })
+}
+exports.extractTarball = extractTarball
+
+const getPluginMetadata = async path => {
   let metadata = {}
   try {
     const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
     const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
+
     metadata.schema = JSON.parse(schema)
     metadata.package = JSON.parse(pkg)
+
+    if (
+      !metadata.package.name ||
+      !metadata.package.version ||
+      !metadata.package.description
+    ) {
+      throw new Error(
+        "package.json is missing one of 'name', 'version' or 'description'."
+      )
+    }
   } catch (err) {
-    throw new Error("Unable to process schema.json/package.json in plugin.")
+    throw new Error(
+      `Unable to process schema.json/package.json in plugin. ${err.message}`
+    )
   }
+
   return { metadata, directory: path }
 }
+exports.getPluginMetadata = getPluginMetadata
 
 exports.getDatasourcePlugin = async (name, url, hash) => {
   if (!fs.existsSync(DATASOURCE_PATH)) {
@@ -396,6 +423,38 @@ exports.getDatasourcePlugin = async (name, url, hash) => {
   }
 }
 
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+exports.findFileRec = (startPath, filter) => {
+  if (!fs.existsSync(startPath)) {
+    return
+  }
+
+  const files = fs.readdirSync(startPath)
+  for (let i = 0, len = files.length; i < len; i++) {
+    const filename = join(startPath, files[i])
+    const stat = fs.lstatSync(filename)
+
+    if (stat.isDirectory()) {
+      return exports.findFileRec(filename, filter)
+    } else if (filename.endsWith(filter)) {
+      return filename
+    }
+  }
+}
+
+/**
+ * Remove a folder which is not empty from the file system
+ */
+exports.deleteFolderFileSystem = path => {
+  if (!fs.existsSync(path)) {
+    return
+  }
+
+  fs.rmSync(path, { recursive: true, force: true })
+}
+
 /**
  * Full function definition for below can be found in the utilities.
  */
@@ -403,5 +462,6 @@ exports.upload = upload
 exports.retrieve = retrieve
 exports.retrieveToTmp = retrieveToTmp
 exports.deleteFiles = deleteFiles
+exports.downloadTarballDirect = downloadTarballDirect
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH

packages/types/src/documents/plugin/index.ts

@@ -3,7 +3,15 @@ export enum PluginType {
   COMPONENT = "component",
 }
 
+export enum PluginSource {
+  NPM = "NPM",
+  GITHUB = "Github",
+  URL = "URL",
+  FILE = "File Upload",
+}
 export interface FileType {
   path: string
   name: string
 }
+
+export const PLUGIN_TYPE_ARR = Object.values(PluginType)