From 667052e0b4bdabaa9ff3e2fb1165c503c668f567 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 6 Nov 2020 12:30:30 +0000 Subject: [PATCH] Updating templates to be able to run locally with an environment variable LOCAL_TEMPLATES and making them work using the DB. Users are also no longer included in the db dump. --- .../src/builderStore/store/frontend.js | 28 ++--- packages/builder/src/pages/index.svelte | 2 +- .../builder/tests/buildCodeForScreen.spec.js | 63 ---------- packages/server/scripts/exportAppTemplate.js | 6 + .../server/src/api/controllers/application.js | 17 +-- packages/server/src/api/controllers/page.js | 6 +- .../server/src/api/controllers/templates.js | 21 +++- packages/server/src/api/controllers/user.js | 25 ++-- packages/server/src/api/routes/pages.old.js | 111 ------------------ packages/server/src/environment.js | 1 + packages/server/src/utilities/templates.js | 54 ++++++--- 11 files changed, 88 insertions(+), 246 deletions(-) delete mode 100644 packages/builder/tests/buildCodeForScreen.spec.js delete mode 100644 packages/server/src/api/routes/pages.old.js diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 7d788ff5ee..d06ff3744c 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -1,23 +1,15 @@ -import { writable, get } from "svelte/store" -import { cloneDeep } from "lodash/fp" -import { - createProps, - makePropsSafe, - getBuiltin, -} from "components/userInterface/pagesParsing/createProps" +import { get, writable } from "svelte/store" +import {cloneDeep} from "lodash/fp" +import { createProps, getBuiltin, makePropsSafe } from "components/userInterface/pagesParsing/createProps" import { getExactComponent } from "components/userInterface/pagesParsing/searchComponents" -import { backendUiStore, allScreens } from "builderStore" +import {allScreens, backendUiStore } from "builderStore" import { generate_screen_css } from "../generate_css" import { fetchComponentLibDefinitions } from "../loadComponentLibraries" import api from "../api" import { DEFAULT_PAGES_OBJECT } from "../../constants" import getNewComponentName from "../getNewComponentName" import analytics from "analytics" -import { - getParent, - generateNewIdsForComponent, - getComponentDefinition, findChildComponentType, -} from "../storeUtils" +import { findChildComponentType, generateNewIdsForComponent, getComponentDefinition, getParent } from "../storeUtils" const INITIAL_FRONTEND_STATE = { apps: [], @@ -332,8 +324,7 @@ export const getFrontendStore = () => { ...presetProps, _instanceId: instanceId, _instanceName: instanceName, - }, - state + } ) const currentComponent = @@ -367,8 +358,7 @@ export const getFrontendStore = () => { }, copy: (component, cut = false) => { store.update(state => { - const copiedComponent = cloneDeep(component) - state.componentToPaste = copiedComponent + state.componentToPaste = cloneDeep(component) state.componentToPaste.isCut = cut if (cut) { const parent = getParent( @@ -469,9 +459,7 @@ export const getFrontendStore = () => { const IdList = allComponents.map(c => c._id) // Construct ID Path: - const path = IdList.join("/") - - return path + return IdList.join("/") }, links: { save: async (url, title) => { diff --git a/packages/builder/src/pages/index.svelte b/packages/builder/src/pages/index.svelte index fcf0f5af52..5462494397 100644 --- a/packages/builder/src/pages/index.svelte +++ b/packages/builder/src/pages/index.svelte @@ -62,7 +62,7 @@ - + diff --git a/packages/builder/tests/buildCodeForScreen.spec.js b/packages/builder/tests/buildCodeForScreen.spec.js deleted file mode 100644 index 87f4c02fa3..0000000000 --- a/packages/builder/tests/buildCodeForScreen.spec.js +++ /dev/null @@ -1,63 +0,0 @@ -import { buildCodeForScreens } from "../src/builderStore/buildCodeForScreens" - -describe("buildCodeForScreen", () => { - it("should package _code into runnable function, for simple screen props", () => { - const screen = { - props: { - _id: "1234", - _code: "render('render argument');", - }, - } - - let renderArg - const render = arg => { - renderArg = arg - } - const uiFunctions = getFunctions(screen) - - const targetfunction = uiFunctions[screen.props._id] - expect(targetfunction).toBeDefined() - - targetfunction(render) - - expect(renderArg).toBe("render argument") - }) - - it("should package _code into runnable function, for _children ", () => { - const screen = { - props: { - _id: "parent", - _code: "render('parent argument');", - _children: [ - { - _id: "child1", - _code: "render('child 1 argument');", - }, - { - _id: "child2", - _code: "render('child 2 argument');", - }, - ], - }, - } - - let renderArg - const render = arg => { - renderArg = arg - } - const uiFunctions = getFunctions(screen) - - const targetfunction = uiFunctions["child2"] - expect(targetfunction).toBeDefined() - - targetfunction(render) - - expect(renderArg).toBe("child 2 argument") - }) -}) - -const getFunctions = screen => { - const code = buildCodeForScreens([screen]) - const func = new Function(`return ${code}`)() - return func -} diff --git a/packages/server/scripts/exportAppTemplate.js b/packages/server/scripts/exportAppTemplate.js index 72ad797183..8268b802d4 100755 --- a/packages/server/scripts/exportAppTemplate.js +++ b/packages/server/scripts/exportAppTemplate.js @@ -23,6 +23,12 @@ yargs }, async args => { console.log("Exporting app..") + if (args.name == null || args.appId == null) { + console.error( + "Unable to export without a name and app ID being specified, check help for more info." + ) + return + } const exportPath = await exportTemplateFromApp({ templateName: args.name, appId: args.appId, diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index cf26dfa766..3894ed85a0 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -1,7 +1,7 @@ const CouchDB = require("../../db") const { compileStaticAssetsForPage } = require("../../utilities/builder") const env = require("../../environment") -const { copy, existsSync } = require("fs-extra") +const { existsSync } = require("fs-extra") const { budibaseAppsDir } = require("../../utilities/budibaseDir") const setBuilderToken = require("../../utilities/builder/setBuilderToken") const fs = require("fs-extra") @@ -160,21 +160,6 @@ const createEmptyAppPackage = async (ctx, app) => { } fs.mkdirpSync(newAppFolder) - // if this app is being created from a template, - // copy the frontend page definition files from - // the template directory. - if (app.template) { - const templatePageDefinitions = join( - appsFolder, - "templates", - app.template.key, - "pages" - ) - // TODO: write the template page JSON to couch - // iterate over the pages and write them to the db - await copy(templatePageDefinitions, join(appsFolder, app._id, "pages")) - } - const mainPage = cloneDeep(MAIN) mainPage._id = generatePageID() mainPage.title = app.name diff --git a/packages/server/src/api/controllers/page.js b/packages/server/src/api/controllers/page.js index 009d167856..06bd887453 100644 --- a/packages/server/src/api/controllers/page.js +++ b/packages/server/src/api/controllers/page.js @@ -8,11 +8,7 @@ exports.save = async function(ctx) { const appPackage = ctx.request.body const page = await db.get(ctx.params.pageId) - await compileStaticAssetsForPage( - ctx.user.appId, - page.name, - ctx.request.body - ) + await compileStaticAssetsForPage(ctx.user.appId, page.name, ctx.request.body) // remove special doc props which couch will complain about delete appPackage.page._css diff --git a/packages/server/src/api/controllers/templates.js b/packages/server/src/api/controllers/templates.js index 8e21b3c18b..94243d3c75 100644 --- a/packages/server/src/api/controllers/templates.js +++ b/packages/server/src/api/controllers/templates.js @@ -2,25 +2,34 @@ const fetch = require("node-fetch") const { downloadTemplate, exportTemplateFromApp, + getLocalTemplates, } = require("../../utilities/templates") +const env = require("../../environment") +// development flag, can be used to test against templates exported locally const DEFAULT_TEMPLATES_BUCKET = "prod-budi-templates.s3-eu-west-1.amazonaws.com" exports.fetch = async function(ctx) { const { type = "app" } = ctx.query - const response = await fetch( - `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json` - ) - const json = await response.json() - ctx.body = Object.values(json.templates[type]) + if (env.LOCAL_TEMPLATES) { + ctx.body = Object.values(getLocalTemplates()[type]) + } else { + const response = await fetch( + `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json` + ) + const json = await response.json() + ctx.body = Object.values(json.templates[type]) + } } exports.downloadTemplate = async function(ctx) { const { type, name } = ctx.params - await downloadTemplate(type, name) + if (!env.LOCAL_TEMPLATES) { + await downloadTemplate(type, name) + } ctx.body = { message: `template ${type}:${name} downloaded successfully.`, diff --git a/packages/server/src/api/controllers/user.js b/packages/server/src/api/controllers/user.js index ba6adeb060..5e4f963f5b 100644 --- a/packages/server/src/api/controllers/user.js +++ b/packages/server/src/api/controllers/user.js @@ -37,15 +37,22 @@ exports.create = async function(ctx) { accessLevelId, } - const response = await db.post(user) - - ctx.status = 200 - ctx.message = "User created successfully." - ctx.userId = response._id - ctx.body = { - _rev: response.rev, - username, - name, + try { + const response = await db.post(user) + ctx.status = 200 + ctx.message = "User created successfully." + ctx.userId = response._id + ctx.body = { + _rev: response.rev, + username, + name, + } + } catch (err) { + if (err.status === 409) { + ctx.throw(400, "User exists already") + } else { + ctx.throw(err.status, err) + } } } diff --git a/packages/server/src/api/routes/pages.old.js b/packages/server/src/api/routes/pages.old.js deleted file mode 100644 index 5d23264c10..0000000000 --- a/packages/server/src/api/routes/pages.old.js +++ /dev/null @@ -1,111 +0,0 @@ -const Router = require("@koa/router") -const StatusCodes = require("../../utilities/statusCodes") -const joiValidator = require("../../middleware/joi-validator") -const Joi = require("joi") -const { - listScreens, - saveScreen, - compileStaticAssetsForPage, - deleteScreen, -} = require("../../utilities/builder") -const authorized = require("../../middleware/authorized") -const { BUILDER } = require("../../utilities/accessLevels") - -const router = Router() - -function generateSaveValidation() { - // prettier-ignore - return joiValidator.body(Joi.object({ - _css: Joi.string().allow(""), - name: Joi.string().required(), - route: Joi.string().required(), - props: Joi.object({ - _id: Joi.string().required(), - _component: Joi.string().required(), - _children: Joi.array().required(), - _instanceName: Joi.string().required(), - _styles: Joi.object().required(), - type: Joi.string().optional(), - table: Joi.string().optional(), - }).required().unknown(true), - }).unknown(true)) -} - -function generatePatchValidation() { - return joiValidator.body( - Joi.object({ - oldname: Joi.string().required(), - newname: Joi.string().required(), - }).unknown(true) - ) -} - -router.post( - "/_builder/api/:appId/pages/:pageName", - authorized(BUILDER), - async ctx => { - await compileStaticAssetsForPage( - ctx.params.appId, - ctx.params.pageName, - ctx.request.body - ) - ctx.response.status = StatusCodes.OK - } -) - -router.get( - "/_builder/api/:appId/pages/:pagename/screens", - authorized(BUILDER), - async ctx => { - ctx.body = await listScreens(ctx.params.appId, ctx.params.pagename) - ctx.response.status = StatusCodes.OK - } -) - -router.post( - "/_builder/api/:appId/pages/:pagename/screen", - authorized(BUILDER), - generateSaveValidation(), - async ctx => { - ctx.body = await saveScreen( - ctx.config, - ctx.params.appId, - ctx.params.pagename, - ctx.request.body - ) - ctx.response.status = StatusCodes.OK - } -) - -router.patch( - "/_builder/api/:appname/pages/:pagename/screen", - authorized(BUILDER), - generatePatchValidation(), - async ctx => { - await renameScreen( - ctx.config, - ctx.params.appname, - ctx.params.pagename, - ctx.request.body.oldname, - ctx.request.body.newname - ) - ctx.response.status = StatusCodes.OK - } -) - -router.delete( - "/_builder/api/pages/:pagename/screens/:id", - authorized(BUILDER), - async ctx => { - await deleteScreen( - ctx.config, - ctx.user.appId, - ctx.params.pagename, - ctx.params.id - ) - - ctx.response.status = StatusCodes.OK - } -) - -module.exports = router diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 28fa74ddcd..003390c502 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -34,6 +34,7 @@ module.exports = { USERID_API_KEY: process.env.USERID_API_KEY, ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, DEPLOYMENT_DB_URL: process.env.DEPLOYMENT_DB_URL, + LOCAL_TEMPLATES: process.env.LOCAL_TEMPLATES, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/server/src/utilities/templates.js b/packages/server/src/utilities/templates.js index 11150261ec..d5b5d5e257 100644 --- a/packages/server/src/utilities/templates.js +++ b/packages/server/src/utilities/templates.js @@ -8,12 +8,35 @@ const zlib = require("zlib") const { promisify } = require("util") const streamPipeline = promisify(stream.pipeline) const { budibaseAppsDir } = require("./budibaseDir") +const env = require("../environment") const CouchDB = require("../db") +const { DocumentTypes } = require("../db/utils") const DEFAULT_TEMPLATES_BUCKET = "prod-budi-templates.s3-eu-west-1.amazonaws.com" +exports.getLocalTemplates = function() { + const templatesDir = join(os.homedir(), ".budibase", "templates", "app") + const templateObj = { app: {} } + fs.ensureDirSync(templatesDir) + const templateNames = fs.readdirSync(templatesDir) + for (let name of templateNames) { + templateObj.app[name] = { + name, + category: "local", + description: "local template", + type: "app", + key: `app/${name}`, + } + } + return templateObj +} + exports.downloadTemplate = async function(type, name) { + const dirName = join(budibaseAppsDir(), "templates", type, name) + if (env.LOCAL_TEMPLATES) { + return dirName + } const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz` const response = await fetch(templateUrl) @@ -30,26 +53,27 @@ exports.downloadTemplate = async function(type, name) { tar.extract(join(budibaseAppsDir(), "templates", type)) ) - return join(budibaseAppsDir(), "templates", type, name) + return dirName } exports.exportTemplateFromApp = async function({ templateName, appId }) { // Copy frontend files - const appToExport = join(os.homedir(), ".budibase", appId, "pages") - const templatesDir = join(os.homedir(), ".budibase", "templates") - fs.ensureDirSync(templatesDir) - - const templateOutputPath = join(templatesDir, templateName) - fs.copySync(appToExport, join(templateOutputPath, "pages")) - - fs.ensureDirSync(join(templateOutputPath, "db")) - const writeStream = fs.createWriteStream( - join(templateOutputPath, "db", "dump.txt") + const templatesDir = join( + os.homedir(), + ".budibase", + "templates", + "app", + templateName, + "db" ) - + fs.ensureDirSync(templatesDir) + const writeStream = fs.createWriteStream(join(templatesDir, "dump.txt")) // perform couch dump const instanceDb = new CouchDB(appId) - - await instanceDb.dump(writeStream) - return templateOutputPath + await instanceDb.dump(writeStream, { + filter: doc => { + return !doc._id.startsWith(DocumentTypes.USER) + }, + }) + return templatesDir }