Merge pull request #645 from Budibase/template-export

Template export
This commit is contained in:
Martin McKeaveney 2020-10-02 17:19:03 +01:00 committed by GitHub
commit fc03d2e7a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 409 additions and 48 deletions

View File

@ -30,6 +30,7 @@
"test:e2e:ci": "lerna run cy:ci"
},
"dependencies": {
"@fortawesome/fontawesome": "^1.1.8"
"@fortawesome/fontawesome": "^1.1.8",
"pouchdb-replication-stream": "^1.2.9"
}
}

View File

@ -1,20 +1,44 @@
<script>
import AppCard from "./AppCard.svelte"
export let apps
import { Heading } from "@budibase/bbui"
import Spinner from "components/common/Spinner.svelte"
import { get } from "builderStore/api"
let promise = getApps()
async function getApps() {
const res = await get("/api/applications")
const json = await res.json()
if (res.ok) {
return json
} else {
throw new Error(json)
}
}
</script>
<div class="root">
<div class="inner">
<div>
<Heading medium black>Your Apps</Heading>
{#await promise}
<div class="spinner-container">
<Spinner size="30" />
</div>
{:then apps}
<div class="inner">
<div>
<div class="apps">
{#each apps as app}
<AppCard {...app} />
{/each}
<div>
<div class="apps">
{#each apps as app}
<AppCard {...app} />
{/each}
</div>
</div>
</div>
</div>
</div>
{:catch err}
<h1 style="color:red">{err}</h1>
{/await}
</div>
<style>

View File

@ -21,6 +21,7 @@
const createAppStore = writable({ currentStep: 0, values: {} })
export let hasKey
export let template
let isApiKeyValid
let lastApiKey
@ -142,11 +143,13 @@
// Create App
const appResp = await post("/api/applications", {
name: $createAppStore.values.applicationName,
template,
})
const appJson = await appResp.json()
analytics.captureEvent("App Created", {
name,
appId: appJson._id,
template,
})
// Select Correct Application/DB in prep for creating user
@ -222,6 +225,7 @@
<div class:hidden={$createAppStore.currentStep !== i}>
<svelte:component
this={step.component}
{template}
{validationErrors}
options={step.options}
name={step.name} />

View File

@ -1,10 +1,15 @@
<script>
import { Input } from "@budibase/bbui"
import { Input, Heading, Body } from "@budibase/bbui"
export let validationErrors
export let template
let blurred = { appName: false }
</script>
{#if template}
<Heading small black>Selected Template</Heading>
<Body>{template.name}</Body>
{/if}
<h2>Create your web app</h2>
<div class="container">
<Input

View File

@ -0,0 +1,75 @@
<script>
import { Button, Heading, Body } from "@budibase/bbui"
import AppCard from "./AppCard.svelte"
import Spinner from "components/common/Spinner.svelte"
import api from "builderStore/api"
export let onSelect
async function fetchTemplates() {
const response = await api.get("/api/templates?type=app")
return await response.json()
}
let templatesPromise = fetchTemplates()
</script>
<div class="root">
<Heading medium black>Start With a Template</Heading>
{#await templatesPromise}
<div class="spinner-container">
<Spinner size="30" />
</div>
{:then templates}
<div class="templates">
{#each templates as template}
<div class="templates-card">
<Heading black medium>{template.name}</Heading>
<Body medium grey>{template.category}</Body>
<Body small black>{template.description}</Body>
<div>
<img src={template.image} width="300" />
</div>
<div class="card-footer">
<Button secondary on:click={() => onSelect(template)}>
Create {template.name}
</Button>
</div>
</div>
{/each}
</div>
{:catch err}
<h1 style="color:red">{err}</h1>
{/await}
</div>
<style>
.templates {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(280px, 1fr));
grid-gap: var(--layout-m);
justify-content: start;
}
.templates-card {
background-color: var(--white);
padding: var(--spacing-xl);
border-radius: var(--border-radius-m);
border: var(--border-dark);
}
.card-footer {
margin-top: var(--spacing-m);
}
h3 {
font-size: var(--font-size-l);
font-weight: 600;
color: var(--ink);
text-transform: capitalize;
}
.root {
margin: 20px 80px;
}
</style>

View File

@ -5,25 +5,12 @@
import AppList from "components/start/AppList.svelte"
import { onMount } from "svelte"
import ActionButton from "components/common/ActionButton.svelte"
import { get } from "builderStore/api"
import Spinner from "components/common/Spinner.svelte"
import CreateAppModal from "components/start/CreateAppModal.svelte"
import TemplateList from "components/start/TemplateList.svelte"
import { Button } from "@budibase/bbui"
import analytics from "analytics"
let promise = getApps()
async function getApps() {
const res = await get("/api/applications")
const json = await res.json()
if (res.ok) {
return json
} else {
throw new Error(json)
}
}
let hasKey
async function fetchKeys() {
@ -47,11 +34,12 @@
// Handle create app modal
const { open } = getContext("simple-modal")
const showCreateAppModal = () => {
const showCreateAppModal = template => {
open(
CreateAppModal,
{
hasKey,
template,
},
{
closeButton: false,
@ -68,7 +56,7 @@
<div class="header">
<div class="welcome">Welcome to the Budibase Beta</div>
<Button primary purple on:click={showCreateAppModal}>
<Button primary purple on:click={() => showCreateAppModal()}>
Create New Web App
</Button>
</div>
@ -80,15 +68,8 @@
</div>
</div>
{#await promise}
<div class="spinner-container">
<Spinner />
</div>
{:then result}
<AppList apps={result} />
{:catch err}
<h1 style="color:red">{err}</h1>
{/await}
<TemplateList onSelect={showCreateAppModal} />
<AppList />
<style>
.header {
@ -127,12 +108,4 @@
color: var(--white);
font-weight: 500;
}
.spinner-container {
height: 100%;
width: 100%;
display: flex;
align-items: center;
justify-content: center;
}
</style>

Binary file not shown.

Binary file not shown.

Before

Width:  |  Height:  |  Size: 201 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 105 KiB

View File

@ -0,0 +1,40 @@
#!/usr/bin/env node
const { exportTemplateFromApp } = require("../src/utilities/templates")
const yargs = require("yargs")
// Script to export a chosen budibase app into a package
// Usage: ./scripts/exportAppTemplate.js export --name=Funky --instanceId=someInstanceId --appId=appId
yargs
.command(
"export",
"Export an existing budibase application to the .budibase/templates directory",
{
name: {
description: "The name of the newly exported template",
alias: "n",
type: "string",
},
instanceId: {
description: "The instanceId to dump the database for",
alias: "inst",
type: "string",
},
appId: {
description: "The appId of the application you want to export",
alias: "app",
type: "string",
},
},
async args => {
console.log("Exporting app..")
const exportPath = await exportTemplateFromApp({
templateName: args.name,
instanceId: args.instanceId,
appId: args.appId,
})
console.log(`Template ${args.name} exported to ${exportPath}`)
}
)
.help()
.alias("help", "h").argv

View File

@ -66,6 +66,7 @@ exports.create = async function(ctx) {
userInstanceMap: {},
componentLibraries: ["@budibase/standard-components"],
name: ctx.request.body.name,
template: ctx.request.body.template,
}
const { rev } = await db.put(newApplication)
@ -75,9 +76,13 @@ exports.create = async function(ctx) {
appId: newApplication._id,
},
request: {
body: { name: `dev-${clientId}` },
body: {
name: `dev-${clientId}`,
template: ctx.request.body.template,
},
},
}
await instanceController.create(createInstCtx)
newApplication.instances.push(createInstCtx.body)
@ -154,6 +159,19 @@ const createEmptyAppPackage = async (ctx, app) => {
name: npmFriendlyAppName(app.name),
})
// if this app is being created from a template,
// copy the frontend page definition files from
// the template directory.
if (app.template) {
const templatePageDefinitions = join(
appsFolder,
"templates",
app.template.key,
"pages"
)
await copy(templatePageDefinitions, join(appsFolder, app._id, "pages"))
}
const mainJson = await updateJsonFile(
join(appsFolder, app._id, "pages", "main", "page.json"),
app

View File

@ -1,9 +1,12 @@
const fs = require("fs")
const CouchDB = require("../../db")
const client = require("../../db/clientDb")
const newid = require("../../db/newid")
const { downloadTemplate } = require("../../utilities/templates")
exports.create = async function(ctx) {
const instanceName = ctx.request.body.name
const template = ctx.request.body.template
const { appId } = ctx.user
const appShortId = appId.substring(0, 7)
const instanceId = `inst_${appShortId}_${newid()}`
@ -51,6 +54,16 @@ exports.create = async function(ctx) {
budibaseApp.instances.push(instance)
await clientDb.put(budibaseApp)
// replicate the template data to the instance DB
if (template) {
const templatePath = await downloadTemplate(...template.key.split("/"))
const dbDumpReadStream = fs.createReadStream(`${templatePath}/db/dump.txt`)
const { ok } = await db.load(dbDumpReadStream)
if (!ok) {
ctx.throw(500, "Error loading database dump from template.")
}
}
ctx.status = 200
ctx.message = `Instance Database ${instanceName} successfully provisioned.`
ctx.body = instance

View File

@ -2,7 +2,7 @@ const send = require("koa-send")
const { resolve, join } = require("path")
const jwt = require("jsonwebtoken")
const fetch = require("node-fetch")
const fs = require("fs")
const fs = require("fs-extra")
const uuid = require("uuid")
const AWS = require("aws-sdk")
const { prepareUploadForS3 } = require("./deploy/aws")

View File

@ -0,0 +1,44 @@
const fetch = require("node-fetch")
const {
downloadTemplate,
exportTemplateFromApp,
} = require("../../utilities/templates")
const DEFAULT_TEMPLATES_BUCKET =
"prod-budi-templates.s3-eu-west-1.amazonaws.com"
exports.fetch = async function(ctx) {
const { type = "app" } = ctx.query
const response = await fetch(
`https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
)
const json = await response.json()
ctx.body = Object.values(json.templates[type])
}
exports.downloadTemplate = async function(ctx) {
const { type, name } = ctx.params
await downloadTemplate(type, name)
ctx.body = {
message: `template ${type}:${name} downloaded successfully.`,
}
}
exports.exportTemplateFromApp = async function(ctx) {
const { appId, instanceId } = ctx.user
const { templateName } = ctx.request.body
await exportTemplateFromApp({
appId,
instanceId,
templateName,
})
ctx.status = 200
ctx.body = {
message: `Created template: ${templateName}`,
}
}

View File

@ -19,6 +19,7 @@ const {
automationRoutes,
accesslevelRoutes,
apiKeysRoutes,
templatesRoutes,
analyticsRoutes,
} = require("./routes")
@ -90,6 +91,9 @@ router.use(automationRoutes.allowedMethods())
router.use(deployRoutes.routes())
router.use(deployRoutes.allowedMethods())
router.use(templatesRoutes.routes())
router.use(templatesRoutes.allowedMethods())
// end auth routes
router.use(pageRoutes.routes())

View File

@ -13,6 +13,7 @@ const automationRoutes = require("./automation")
const accesslevelRoutes = require("./accesslevel")
const deployRoutes = require("./deploy")
const apiKeysRoutes = require("./apikeys")
const templatesRoutes = require("./templates")
const analyticsRoutes = require("./analytics")
module.exports = {
@ -31,5 +32,6 @@ module.exports = {
automationRoutes,
accesslevelRoutes,
apiKeysRoutes,
templatesRoutes,
analyticsRoutes,
}

View File

@ -0,0 +1,17 @@
const Router = require("@koa/router")
const controller = require("../controllers/templates")
const authorized = require("../../middleware/authorized")
const { BUILDER } = require("../../utilities/accessLevels")
const router = Router()
router
.get("/api/templates", authorized(BUILDER), controller.fetch)
.get(
"/api/templates/:type/:name",
authorized(BUILDER),
controller.downloadTemplate
)
.post("/api/templates", authorized(BUILDER), controller.exportTemplateFromApp)
module.exports = router

View File

@ -1,4 +1,5 @@
const PouchDB = require("pouchdb")
const replicationStream = require("pouchdb-replication-stream")
const allDbs = require("pouchdb-all-dbs")
const { budibaseAppsDir } = require("../utilities/budibaseDir")
const env = require("../environment")
@ -6,6 +7,9 @@ const env = require("../environment")
const COUCH_DB_URL = env.COUCH_DB_URL || `leveldb://${budibaseAppsDir()}/.data/`
const isInMemory = env.NODE_ENV === "jest"
PouchDB.plugin(replicationStream.plugin)
PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
let POUCH_DB_DEFAULTS = {
prefix: COUCH_DB_URL,
}

View File

@ -0,0 +1,57 @@
const path = require("path")
const fs = require("fs-extra")
const os = require("os")
const fetch = require("node-fetch")
const stream = require("stream")
const tar = require("tar-fs")
const zlib = require("zlib")
const { promisify } = require("util")
const streamPipeline = promisify(stream.pipeline)
const { budibaseAppsDir } = require("./budibaseDir")
const CouchDB = require("../db")
const DEFAULT_TEMPLATES_BUCKET =
"prod-budi-templates.s3-eu-west-1.amazonaws.com"
exports.downloadTemplate = async function(type, name) {
const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`
const response = await fetch(templateUrl)
if (!response.ok) {
throw new Error(
`Error downloading template ${type}:${name}: ${response.statusText}`
)
}
// stream the response, unzip and extract
await streamPipeline(
response.body,
zlib.Unzip(),
tar.extract(path.join(budibaseAppsDir(), "templates", type))
)
return path.join(budibaseAppsDir(), "templates", type, name)
}
exports.exportTemplateFromApp = async function({
appId,
templateName,
instanceId,
}) {
// Copy frontend files
const appToExport = path.join(os.homedir(), ".budibase", appId, "pages")
const templatesDir = path.join(os.homedir(), ".budibase", "templates")
fs.ensureDirSync(templatesDir)
const templateOutputPath = path.join(templatesDir, templateName)
fs.copySync(appToExport, `${templateOutputPath}/pages`)
fs.ensureDirSync(path.join(templateOutputPath, "db"))
const writeStream = fs.createWriteStream(`${templateOutputPath}/db/dump.txt`)
// perform couch dump
const instanceDb = new CouchDB(instanceId)
await instanceDb.dump(writeStream)
return templateOutputPath
}

View File

@ -913,6 +913,11 @@ argparse@^1.0.7:
dependencies:
sprintf-js "~1.0.2"
argsarray@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/argsarray/-/argsarray-0.0.1.tgz#6e7207b4ecdb39b0af88303fa5ae22bda8df61cb"
integrity sha1-bnIHtOzbObCviDA/pa4ivajfYcs=
arr-diff@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/arr-diff/-/arr-diff-4.0.0.tgz#d6461074febfec71e7e15235761a329a5dc7c520"
@ -2331,6 +2336,11 @@ ignore@^4.0.6:
version "4.0.6"
resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc"
immediate@~3.0.5:
version "3.0.6"
resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=
import-fresh@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-2.0.0.tgz#d81355c15612d386c61f9ddd3922d4304822a546"
@ -2377,7 +2387,7 @@ inflight@^1.0.4:
once "^1.3.0"
wrappy "1"
inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3:
inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3:
version "2.0.4"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
@ -2628,6 +2638,11 @@ is-windows@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d"
isarray@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf"
integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=
isarray@1.0.0, isarray@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
@ -2790,6 +2805,13 @@ libnpmpublish@^1.1.1:
semver "^5.5.1"
ssri "^6.0.1"
lie@3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"
integrity sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=
dependencies:
immediate "~3.0.5"
load-json-file@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-1.1.0.tgz#956905708d58b4bab4c2261b04f59f31c99374c0"
@ -2839,6 +2861,11 @@ lodash.ismatch@^4.4.0:
version "4.4.0"
resolved "https://registry.yarnpkg.com/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz#756cb5150ca3ba6f11085a78849645f188f85f37"
lodash.pick@^4.0.0:
version "4.4.0"
resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3"
integrity sha1-UvBWEP/53tQiYRRB7R/BI6AwAbM=
lodash.set@^4.3.2:
version "4.3.2"
resolved "https://registry.yarnpkg.com/lodash.set/-/lodash.set-4.3.2.tgz#d8757b1da807dde24816b0d6a84bea1a76230b23"
@ -3177,6 +3204,16 @@ natural-compare@^1.4.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
ndjson@^1.4.3:
version "1.5.0"
resolved "https://registry.yarnpkg.com/ndjson/-/ndjson-1.5.0.tgz#ae603b36b134bcec347b452422b0bf98d5832ec8"
integrity sha1-rmA7NrE0vOw0e0UkIrC/mNWDLsg=
dependencies:
json-stringify-safe "^5.0.1"
minimist "^1.2.0"
split2 "^2.1.0"
through2 "^2.0.3"
neo-async@^2.6.0:
version "2.6.1"
resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.1.tgz#ac27ada66167fa8849a6addd837f6b189ad2081c"
@ -3680,6 +3717,34 @@ posix-character-classes@^0.1.0:
version "0.1.1"
resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"
pouch-stream@^0.4.0:
version "0.4.1"
resolved "https://registry.yarnpkg.com/pouch-stream/-/pouch-stream-0.4.1.tgz#0c6d8475c9307677627991a2f079b301c3b89bdd"
integrity sha1-DG2EdckwdndieZGi8HmzAcO4m90=
dependencies:
inherits "^2.0.1"
readable-stream "^1.0.27-1"
pouchdb-promise@^6.0.4:
version "6.4.3"
resolved "https://registry.yarnpkg.com/pouchdb-promise/-/pouchdb-promise-6.4.3.tgz#74516f4acf74957b54debd0fb2c0e5b5a68ca7b3"
integrity sha512-ruJaSFXwzsxRHQfwNHjQfsj58LBOY1RzGzde4PM5CWINZwFjCQAhZwfMrch2o/0oZT6d+Xtt0HTWhq35p3b0qw==
dependencies:
lie "3.1.1"
pouchdb-replication-stream@^1.2.9:
version "1.2.9"
resolved "https://registry.yarnpkg.com/pouchdb-replication-stream/-/pouchdb-replication-stream-1.2.9.tgz#aa4fa5d8f52df4825392f18e07c7e11acffc650a"
integrity sha1-qk+l2PUt9IJTkvGOB8fhGs/8ZQo=
dependencies:
argsarray "0.0.1"
inherits "^2.0.3"
lodash.pick "^4.0.0"
ndjson "^1.4.3"
pouch-stream "^0.4.0"
pouchdb-promise "^6.0.4"
through2 "^2.0.0"
prelude-ls@~1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"
@ -3862,6 +3927,16 @@ read@1, read@~1.0.1:
string_decoder "^1.1.1"
util-deprecate "^1.0.1"
readable-stream@^1.0.27-1:
version "1.1.14"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9"
integrity sha1-fPTFTvZI44EwhMY23SB54WbAgdk=
dependencies:
core-util-is "~1.0.0"
inherits "~2.0.1"
isarray "0.0.1"
string_decoder "~0.10.x"
readdir-scoped-modules@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz#8d45407b4f870a0dcaebc0e28670d18e74514309"
@ -4216,7 +4291,7 @@ split-string@^3.0.1, split-string@^3.0.2:
dependencies:
extend-shallow "^3.0.0"
split2@^2.0.0:
split2@^2.0.0, split2@^2.1.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/split2/-/split2-2.2.0.tgz#186b2575bcf83e85b7d18465756238ee4ee42493"
dependencies:
@ -4321,6 +4396,11 @@ string_decoder@^1.1.1:
dependencies:
safe-buffer "~5.2.0"
string_decoder@~0.10.x:
version "0.10.31"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=
string_decoder@~1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
@ -4441,7 +4521,7 @@ text-table@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
through2@^2.0.0, through2@^2.0.2:
through2@^2.0.0, through2@^2.0.2, through2@^2.0.3:
version "2.0.5"
resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd"
dependencies: