uploading plugin for github, npm and url
This commit is contained in:
NEOLPAR
parent
06b36315b6
commit
c2bca8a025
|
|
@ -1,11 +1,11 @@
|
|||
import { ObjectStoreBuckets } from "../../constants"
|
||||
import { loadJSFile } from "../../utilities/fileSystem"
|
||||
import {
|
||||
extractPluginTarball,
|
||||
createUrlPlugin,
|
||||
createGithubPlugin,
|
||||
loadJSFile,
|
||||
} from "../../utilities/fileSystem"
|
||||
import { createNpmPlugin } from "./plugin/utils"
|
||||
uploadedNpmPlugin,
|
||||
uploadedUrlPlugin,
|
||||
uploadedGithubPlugin,
|
||||
uploadedFilePlugin,
|
||||
} from "./plugin/utils"
|
||||
import { getGlobalDB } from "@budibase/backend-core/tenancy"
|
||||
import { generatePluginID, getPluginParams } from "../../db/utils"
|
||||
import {
|
||||
|
|
@ -60,22 +60,32 @@ export async function create(ctx: any) {
|
|||
// Generating random name as a backup and needed for url
|
||||
let name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000)
|
||||
|
||||
if (!env.SELF_HOSTED) {
|
||||
throw new Error("Plugins not supported outside of self-host.")
|
||||
}
|
||||
|
||||
switch (source) {
|
||||
case "npm":
|
||||
const { metadata: metadataNpm, directory: directoryNpm } =
|
||||
await createNpmPlugin(url, name)
|
||||
metadata = metadataNpm
|
||||
directory = directoryNpm
|
||||
try {
|
||||
const { metadata: metadataNpm, directory: directoryNpm } =
|
||||
await uploadedNpmPlugin(url, name)
|
||||
metadata = metadataNpm
|
||||
directory = directoryNpm
|
||||
} catch (err: any) {
|
||||
const errMsg = err?.message ? err?.message : err
|
||||
|
||||
ctx.throw(400, `Failed to import plugin: ${errMsg}`)
|
||||
}
|
||||
break
|
||||
case "github":
|
||||
const { metadata: metadataGithub, directory: directoryGithub } =
|
||||
await createGithubPlugin(ctx, url, name, githubToken)
|
||||
await uploadedGithubPlugin(ctx, url, name, githubToken)
|
||||
metadata = metadataGithub
|
||||
directory = directoryGithub
|
||||
break
|
||||
case "url":
|
||||
const { metadata: metadataUrl, directory: directoryUrl } =
|
||||
await createUrlPlugin(url, name, headers)
|
||||
await uploadedUrlPlugin(url, name, headers)
|
||||
metadata = metadataUrl
|
||||
directory = directoryUrl
|
||||
break
|
||||
|
|
@ -192,6 +202,6 @@ export async function processPlugin(plugin: FileType, source?: string) {
|
|||
throw new Error("Plugins not supported outside of self-host.")
|
||||
}
|
||||
|
||||
const { metadata, directory } = await extractPluginTarball(plugin)
|
||||
const { metadata, directory } = await uploadedFilePlugin(plugin)
|
||||
return await storePlugin(metadata, directory, source)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,32 +1,153 @@
|
|||
const fetch = require("node-fetch")
|
||||
import { downloadUnzipPlugin } from "../../../utilities/fileSystem"
|
||||
import fetch from "node-fetch"
|
||||
import {
|
||||
createTempFolder,
|
||||
getPluginMetadata,
|
||||
findFileRec,
|
||||
downloadTarballDirect,
|
||||
extractTarball,
|
||||
deleteFolderFileSystem,
|
||||
} from "../../../utilities/fileSystem"
|
||||
import { join } from "path"
|
||||
|
||||
export const createNpmPlugin = async (url, name = "") => {
|
||||
let npmTarball = url
|
||||
export const uploadedFilePlugin = async file => {
|
||||
if (!file.name.endsWith(".tar.gz")) {
|
||||
throw new Error("Plugin must be compressed into a gzipped tarball.")
|
||||
}
|
||||
const path = createTempFolder(file.name.split(".tar.gz")[0])
|
||||
await extractTarball(file.path, path)
|
||||
|
||||
return await getPluginMetadata(path)
|
||||
}
|
||||
|
||||
export const uploadedNpmPlugin = async (url, name, headers = {}) => {
|
||||
let npmTarballUrl = url
|
||||
let pluginName = name
|
||||
|
||||
if (
|
||||
!npmTarball.includes("https://www.npmjs.com") &&
|
||||
!npmTarball.includes("https://registry.npmjs.org")
|
||||
!npmTarballUrl.includes("https://www.npmjs.com") &&
|
||||
!npmTarballUrl.includes("https://registry.npmjs.org")
|
||||
) {
|
||||
throw "The plugin origin must be from NPM"
|
||||
throw new Error("The plugin origin must be from NPM")
|
||||
}
|
||||
|
||||
if (!npmTarball.includes(".tgz")) {
|
||||
if (!npmTarballUrl.includes(".tgz")) {
|
||||
const npmPackageURl = url.replace(
|
||||
"https://www.npmjs.com/package/",
|
||||
"https://registry.npmjs.org/"
|
||||
)
|
||||
const response = await fetch(npmPackageURl)
|
||||
if (response.status === 200) {
|
||||
let npmDetails = await response.json()
|
||||
pluginName = npmDetails.name
|
||||
const npmVersion = npmDetails["dist-tags"].latest
|
||||
npmTarball = npmDetails.versions[npmVersion].dist.tarball
|
||||
} else {
|
||||
throw "Cannot get package details"
|
||||
if (response.status !== 200) {
|
||||
throw new Error("NPM Package not found")
|
||||
}
|
||||
|
||||
let npmDetails = await response.json()
|
||||
pluginName = npmDetails.name
|
||||
const npmVersion = npmDetails["dist-tags"].latest
|
||||
npmTarballUrl = npmDetails?.versions?.[npmVersion]?.dist?.tarball
|
||||
|
||||
if (!npmTarballUrl) {
|
||||
throw new Error("NPM tarball url not found")
|
||||
}
|
||||
}
|
||||
|
||||
return await downloadUnzipPlugin(pluginName, npmTarball)
|
||||
const path = await downloadUnzipTarball(npmTarballUrl, pluginName, headers)
|
||||
const tarballPluginFile = findFileRec(path, ".tar.gz")
|
||||
if (!tarballPluginFile) {
|
||||
throw new Error("Tarball plugin file not found")
|
||||
}
|
||||
|
||||
try {
|
||||
await extractTarball(tarballPluginFile, path)
|
||||
deleteFolderFileSystem(join(path, "package"))
|
||||
} catch (err) {
|
||||
throw new Error(err)
|
||||
}
|
||||
|
||||
return await getPluginMetadata(path)
|
||||
}
|
||||
|
||||
export const uploadedUrlPlugin = async (url, name = "", headers = {}) => {
|
||||
if (!url.includes(".tar.gz")) {
|
||||
throw new Error("Plugin must be compressed into a gzipped tarball.")
|
||||
}
|
||||
|
||||
const path = await downloadUnzipTarball(url, name, headers)
|
||||
|
||||
return await getPluginMetadata(path)
|
||||
}
|
||||
|
||||
export const uploadedGithubPlugin = async (ctx, url, name = "", token = "") => {
|
||||
let githubUrl = url
|
||||
|
||||
if (!githubUrl.includes("https://github.com/")) {
|
||||
throw new Error("The plugin origin must be from Github")
|
||||
}
|
||||
|
||||
if (url.includes(".git")) {
|
||||
githubUrl = url.replace(".git", "")
|
||||
}
|
||||
|
||||
const githubApiUrl = githubUrl.replace(
|
||||
"https://github.com/",
|
||||
"https://api.github.com/repos/"
|
||||
)
|
||||
const headers = token ? { Authorization: `Bearer ${token}` } : {}
|
||||
try {
|
||||
const pluginRaw = await fetch(githubApiUrl, { headers })
|
||||
if (pluginRaw.status !== 200) {
|
||||
throw new Error(`Repository not found`)
|
||||
}
|
||||
|
||||
let pluginDetails = await pluginRaw.json()
|
||||
const pluginName = pluginDetails.name || name
|
||||
|
||||
const pluginLatestReleaseUrl = pluginDetails?.["releases_url"]
|
||||
? pluginDetails?.["releases_url"].replace("{/id}", "/latest")
|
||||
: undefined
|
||||
if (!pluginLatestReleaseUrl) {
|
||||
throw new Error("Github release not found")
|
||||
}
|
||||
|
||||
const pluginReleaseRaw = await fetch(pluginLatestReleaseUrl, { headers })
|
||||
if (pluginReleaseRaw.status !== 200) {
|
||||
throw new Error("Github latest release not found")
|
||||
}
|
||||
const pluginReleaseDetails = await pluginReleaseRaw.json()
|
||||
const pluginReleaseTarballAsset = pluginReleaseDetails?.assets?.find(
|
||||
x => x?.content_type === "application/gzip"
|
||||
)
|
||||
const pluginLastReleaseTarballUrl =
|
||||
pluginReleaseTarballAsset?.browser_download_url
|
||||
if (!pluginLastReleaseTarballUrl) {
|
||||
throw new Error("Github latest release url not found")
|
||||
}
|
||||
|
||||
const path = await downloadUnzipTarball(
|
||||
pluginLastReleaseTarballUrl,
|
||||
pluginName,
|
||||
headers
|
||||
)
|
||||
|
||||
return await getPluginMetadata(path)
|
||||
} catch (err) {
|
||||
let errMsg = err?.message || err
|
||||
|
||||
if (errMsg === "unexpected response Not Found") {
|
||||
errMsg = "Github release tarbal not found"
|
||||
}
|
||||
|
||||
throw new Error(errMsg)
|
||||
}
|
||||
}
|
||||
|
||||
export const downloadUnzipTarball = async (url, name, headers = {}) => {
|
||||
try {
|
||||
const path = createTempFolder(name)
|
||||
|
||||
await downloadTarballDirect(url, path, headers)
|
||||
|
||||
return path
|
||||
} catch (e) {
|
||||
throw new Error(e.message)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
const { budibaseTempDir } = require("../budibaseDir")
|
||||
const fs = require("fs")
|
||||
const { join } = require("path")
|
||||
const { promisify } = require("util")
|
||||
const streamPipeline = promisify(require("stream").pipeline)
|
||||
const uuid = require("uuid/v4")
|
||||
const {
|
||||
doWithDB,
|
||||
|
|
@ -17,6 +15,7 @@ const {
|
|||
streamUpload,
|
||||
deleteFolder,
|
||||
downloadTarball,
|
||||
downloadTarballDirect,
|
||||
deleteFiles,
|
||||
} = require("./utilities")
|
||||
const { updateClientLibrary } = require("./clientLibrary")
|
||||
|
|
@ -31,7 +30,6 @@ const MemoryStream = require("memorystream")
|
|||
const { getAppId } = require("@budibase/backend-core/context")
|
||||
const tar = require("tar")
|
||||
const fetch = require("node-fetch")
|
||||
const { NodeVM } = require("vm2")
|
||||
|
||||
const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
|
||||
const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
|
||||
|
|
@ -341,131 +339,52 @@ exports.cleanup = appIds => {
|
|||
}
|
||||
}
|
||||
|
||||
const extractPluginTarball = async (file, ext = ".tar.gz") => {
|
||||
if (!file.name.endsWith(ext)) {
|
||||
throw new Error("Plugin must be compressed into a gzipped tarball.")
|
||||
const createTempFolder = item => {
|
||||
const path = join(budibaseTempDir(), item)
|
||||
try {
|
||||
// remove old tmp directories automatically - don't combine
|
||||
if (fs.existsSync(path)) {
|
||||
fs.rmSync(path, { recursive: true, force: true })
|
||||
}
|
||||
fs.mkdirSync(path)
|
||||
} catch (err) {
|
||||
throw new Error(`Path cannot be created: ${err.message}`)
|
||||
}
|
||||
const path = join(budibaseTempDir(), file.name.split(ext)[0])
|
||||
// remove old tmp directories automatically - don't combine
|
||||
if (fs.existsSync(path)) {
|
||||
fs.rmSync(path, { recursive: true, force: true })
|
||||
}
|
||||
fs.mkdirSync(path)
|
||||
|
||||
return path
|
||||
}
|
||||
exports.createTempFolder = createTempFolder
|
||||
|
||||
const extractTarball = async (fromFilePath, toPath) => {
|
||||
await tar.extract({
|
||||
file: file.path,
|
||||
C: path,
|
||||
file: fromFilePath,
|
||||
C: toPath,
|
||||
})
|
||||
|
||||
return await getPluginMetadata(path)
|
||||
}
|
||||
exports.extractPluginTarball = extractPluginTarball
|
||||
|
||||
exports.createUrlPlugin = async (url, name = "", headers = {}) => {
|
||||
if (!url.includes(".tgz") && !url.includes(".tar.gz")) {
|
||||
throw new Error("Plugin must be compressed into a gzipped tarball.")
|
||||
}
|
||||
|
||||
return await downloadUnzipPlugin(name, url, headers)
|
||||
}
|
||||
|
||||
exports.createGithubPlugin = async (ctx, url, name = "", token = "") => {
|
||||
let githubRepositoryUrl
|
||||
let githubUrl
|
||||
|
||||
if (url.includes(".git")) {
|
||||
githubRepositoryUrl = token
|
||||
? url.replace("https://", `https://${token}@`)
|
||||
: url
|
||||
githubUrl = url.replace(".git", "")
|
||||
} else {
|
||||
githubRepositoryUrl = token
|
||||
? `${url}.git`.replace("https://", `https://${token}@`)
|
||||
: `${url}.git`
|
||||
githubUrl = url
|
||||
}
|
||||
|
||||
const githubApiUrl = githubUrl.replace(
|
||||
"https://github.com/",
|
||||
"https://api.github.com/repos/"
|
||||
)
|
||||
const headers = token ? { Authorization: `Bearer ${token}` } : {}
|
||||
try {
|
||||
const pluginRaw = await fetch(githubApiUrl, { headers })
|
||||
if (pluginRaw.status !== 200) {
|
||||
throw `Repository not found`
|
||||
}
|
||||
|
||||
let pluginDetails = await pluginRaw.json()
|
||||
const pluginName = pluginDetails.name || name
|
||||
|
||||
const path = join(budibaseTempDir(), pluginName)
|
||||
// Remove first if exists
|
||||
if (fs.existsSync(path)) {
|
||||
fs.rmSync(path, { recursive: true, force: true })
|
||||
}
|
||||
fs.mkdirSync(path)
|
||||
|
||||
const script = `
|
||||
module.exports = async () => {
|
||||
const child_process = require('child_process')
|
||||
child_process.execSync(\`git clone ${githubRepositoryUrl} ${join(
|
||||
budibaseTempDir(),
|
||||
pluginName
|
||||
)}\`);
|
||||
}
|
||||
`
|
||||
const scriptRunner = new NodeVM({
|
||||
require: {
|
||||
external: true,
|
||||
builtin: ["child_process"],
|
||||
root: "./",
|
||||
},
|
||||
}).run(script)
|
||||
|
||||
await scriptRunner()
|
||||
|
||||
return await getPluginMetadata(path)
|
||||
} catch (e) {
|
||||
throw e.message
|
||||
}
|
||||
}
|
||||
|
||||
const downloadUnzipPlugin = async (name, url, headers = {}) => {
|
||||
const path = join(budibaseTempDir(), name)
|
||||
try {
|
||||
// Remove first if exists
|
||||
if (fs.existsSync(path)) {
|
||||
fs.rmSync(path, { recursive: true, force: true })
|
||||
}
|
||||
fs.mkdirSync(path)
|
||||
|
||||
const response = await fetch(url, { headers })
|
||||
if (!response.ok)
|
||||
throw new Error(`Loading NPM plugin failed ${response.statusText}`)
|
||||
|
||||
await streamPipeline(
|
||||
response.body,
|
||||
tar.x({
|
||||
strip: 1,
|
||||
C: path,
|
||||
})
|
||||
)
|
||||
return await getPluginMetadata(path)
|
||||
} catch (e) {
|
||||
throw `Cannot store plugin locally: ${e.message}`
|
||||
}
|
||||
}
|
||||
exports.downloadUnzipPlugin = downloadUnzipPlugin
|
||||
exports.extractTarball = extractTarball
|
||||
|
||||
const getPluginMetadata = async path => {
|
||||
let metadata = {}
|
||||
try {
|
||||
const pkg = fs.readFileSync(join(path, "package.json"), "utf8")
|
||||
const schema = fs.readFileSync(join(path, "schema.json"), "utf8")
|
||||
|
||||
metadata.schema = JSON.parse(schema)
|
||||
metadata.package = JSON.parse(pkg)
|
||||
|
||||
if (
|
||||
!metadata.package.name ||
|
||||
!metadata.package.version ||
|
||||
!metadata.package.description
|
||||
) {
|
||||
throw new Error(
|
||||
"package.json is missing one of 'name', 'version' or 'description'."
|
||||
)
|
||||
}
|
||||
} catch (err) {
|
||||
throw new Error("Unable to process schema.json/package.json in plugin.")
|
||||
throw new Error(
|
||||
`Unable to process schema.json/package.json in plugin. ${err.message}`
|
||||
)
|
||||
}
|
||||
|
||||
return { metadata, directory: path }
|
||||
|
|
@ -504,6 +423,40 @@ exports.getDatasourcePlugin = async (name, url, hash) => {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find for a file recursively from start path applying filter, return first match
|
||||
*/
|
||||
const findFileRec = (startPath, filter) => {
|
||||
if (!fs.existsSync(startPath)) {
|
||||
return
|
||||
}
|
||||
|
||||
var files = fs.readdirSync(startPath)
|
||||
for (let i = 0, len = files.length; i < len; i++) {
|
||||
const filename = join(startPath, files[i])
|
||||
const stat = fs.lstatSync(filename)
|
||||
|
||||
if (stat.isDirectory()) {
|
||||
return findFileRec(filename, filter)
|
||||
} else if (filename.endsWith(filter)) {
|
||||
return filename
|
||||
}
|
||||
}
|
||||
}
|
||||
exports.findFileRec = findFileRec
|
||||
|
||||
/**
|
||||
* Remove a folder which is not empty from the file system
|
||||
*/
|
||||
const deleteFolderFileSystem = path => {
|
||||
if (!fs.existsSync(path)) {
|
||||
return
|
||||
}
|
||||
|
||||
fs.rmSync(path, { recursive: true, force: true })
|
||||
}
|
||||
exports.deleteFolderFileSystem = deleteFolderFileSystem
|
||||
|
||||
/**
|
||||
* Full function definition for below can be found in the utilities.
|
||||
*/
|
||||
|
|
@ -511,5 +464,6 @@ exports.upload = upload
|
|||
exports.retrieve = retrieve
|
||||
exports.retrieveToTmp = retrieveToTmp
|
||||
exports.deleteFiles = deleteFiles
|
||||
exports.downloadTarballDirect = downloadTarballDirect
|
||||
exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH
|
||||
exports.NODE_MODULES_PATH = NODE_MODULES_PATH
|
||||
|
|
|
|||
Loading…
Reference in New Issue