Working on moving file system access around, starting by centralising it then working out what needs to be moved to object store and how.

This commit is contained in:
mike12345567 2021-03-19 14:43:41 +00:00
parent 1bb2b45d67
commit 8821eba634
11 changed files with 1132 additions and 244 deletions

3
.gitignore vendored
View File

@ -84,3 +84,6 @@ typings/
# Nova Editor # Nova Editor
.nova .nova
# swap files (linux)
*.swp

1
packages/server/dev.env Normal file
View File

@ -0,0 +1 @@
PORT=4001

View File

@ -36,7 +36,7 @@
"test:integration": "jest --coverage --detectOpenHandles", "test:integration": "jest --coverage --detectOpenHandles",
"test:watch": "jest --watch", "test:watch": "jest --watch",
"run:docker": "node src/index", "run:docker": "node src/index",
"dev:builder": "cross-env PORT=4001 nodemon src/index.js", "dev:builder": "env-cmd -f dev.env nodemon src/index.js",
"electron": "electron src/electron.js", "electron": "electron src/electron.js",
"build:electron": "electron-builder --dir", "build:electron": "electron-builder --dir",
"publish:electron": "electron-builder -mwl --publish always", "publish:electron": "electron-builder -mwl --publish always",
@ -134,10 +134,10 @@
"devDependencies": { "devDependencies": {
"@budibase/standard-components": "^0.8.9", "@budibase/standard-components": "^0.8.9",
"@jest/test-sequencer": "^24.8.0", "@jest/test-sequencer": "^24.8.0",
"cross-env": "^7.0.3",
"electron": "10.1.3", "electron": "10.1.3",
"electron-builder": "^22.9.1", "electron-builder": "^22.9.1",
"electron-builder-notarize": "^1.1.2", "electron-builder-notarize": "^1.1.2",
"env-cmd": "^10.1.0",
"eslint": "^6.8.0", "eslint": "^6.8.0",
"jest": "^24.8.0", "jest": "^24.8.0",
"nodemon": "^2.0.4", "nodemon": "^2.0.4",

View File

@ -9,7 +9,7 @@ const { join, resolve } = require("../../utilities/centralPath")
const packageJson = require("../../../package.json") const packageJson = require("../../../package.json")
const { createLinkView } = require("../../db/linkedRows") const { createLinkView } = require("../../db/linkedRows")
const { createRoutingView } = require("../../utilities/routing") const { createRoutingView } = require("../../utilities/routing")
const { downloadTemplate } = require("../../utilities/templates") const { getTemplateStream } = require("../../utilities/fileSystem")
const { const {
generateAppID, generateAppID,
getLayoutParams, getLayoutParams,
@ -106,16 +106,7 @@ async function createInstance(template) {
// this is currently very hard to test, downloading and importing template files // this is currently very hard to test, downloading and importing template files
/* istanbul ignore next */ /* istanbul ignore next */
if (template && template.useTemplate === "true") { if (template && template.useTemplate === "true") {
let dbDumpReadStream const { ok } = await db.load(await getTemplateStream(template))
if (template.file) {
dbDumpReadStream = fs.createReadStream(template.file.path)
} else {
const templatePath = await downloadTemplate(...template.key.split("/"))
dbDumpReadStream = fs.createReadStream(
join(templatePath, "db", "dump.txt")
)
}
const { ok } = await db.load(dbDumpReadStream)
if (!ok) { if (!ok) {
throw "Error loading database dump from template." throw "Error loading database dump from template."
} }

View File

@ -50,17 +50,6 @@ exports.serveBuilder = async function(ctx) {
await send(ctx, ctx.file, { root: ctx.devPath || builderPath }) await send(ctx, ctx.file, { root: ctx.devPath || builderPath })
} }
exports.serveSelfHostPage = async function(ctx) {
const logo = fs.readFileSync(resolve(__dirname, "selfhost/logo.svg"), "utf8")
const hostingHbs = fs.readFileSync(
resolve(__dirname, "selfhost/index.hbs"),
"utf8"
)
ctx.body = await processString(hostingHbs, {
logo,
})
}
exports.uploadFile = async function(ctx) { exports.uploadFile = async function(ctx) {
let files let files
files = files =

View File

@ -1,173 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Budibase self hosting</title>
<style>
body {
font-family: Inter, -apple-system, BlinkMacSystemFont, Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
height: 100%;
width: 100%;
margin: 0;
padding: 0;
background: #fafafa;
}
.main {
padding: 0 20px;
margin: 30px auto;
width: 60%;
}
h2 {
font-size: clamp(24px, 1.5vw, 30px);
text-align: center;
line-height: 1.3;
font-weight: bold;
}
.card-grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 3rem;
}
.card {
display: grid;
background-color: #222222;
grid-template-columns: 1fr;
align-items: center;
padding: 2.5rem 1.75rem;
border-radius: 12px;
color: white;
}
.card h3 {
margin: 0;
font-size: 24px;
font-family: sans-serif;
font-weight: 600;
}
.card h3 b {
text-wrap: normal;
font-size: 36px;
padding-right: 14px;
}
.card p {
color: #ffffff;
opacity: 0.8;
font-size: 18px;
text-align: left;
line-height: 1.3;
margin-top: 1rem;
}
.logo {
width: 60px;
height: 60px;
margin: auto;
}
.top-text {
text-align: center;
color: #707070;
margin: 0 0 1.5rem 0;
}
.button {
cursor: pointer;
display: block;
background: #4285f4;
color: white;
padding: 12px 16px;
font-size: 16px;
font-weight: 600;
border-radius: 6px;
max-width: 120px;
text-align: center;
transition: 200ms background ease;
text-decoration: none;
}
.info {
background: #f5f5f5;
padding: 1rem 1rem 1rem 1rem;
border: #ccc 1px solid;
border-radius: 6px;
margin-top: 40px;
display: flex;
align-items: center;
}
.info p {
margin-left: 20px;
color: #222222;
font-family: sans-serif;
}
.info p {
margin-right: 20px;
}
.info svg {
margin-left: 20px;
}
.info a {
color: #4285f4;
}
</style>
</head>
<body>
<div class="main">
<div class="logo">
{{logo}}
</div>
<h2>Get started with Budibase Self Hosting</h2>
<p class="top-text">Use the address <b id="url"></b> in your Builder</p>
<div class="card-grid">
<div class="card">
<h3><b>📚</b>Documentation</h3>
<p>
Find out more about your self hosted platform.
</p>
<a class="button"
href="https://docs.budibase.com/self-hosting/introduction-to-self-hosting">
Documentation
</a>
</div>
<div class="card">
<h3><b>💻</b>Next steps</h3>
<p>
Find out how to make use of your self hosted Budibase platform.
</p>
<a class="button"
href="https://docs.budibase.com/self-hosting/builder-settings">
Next steps
</a>
</div>
</div>
<div class="info">
<svg preserveAspectRatio="xMidYMid meet" height="28px" width="28px" fill="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink" stroke="none" class="icon-7f6730be--text-3f89f380">
<g>
<path d="M12.2 8.98c.06-.01.12-.03.18-.06.06-.02.12-.05.18-.09l.15-.12c.18-.19.29-.45.29-.71 0-.06-.01-.13-.02-.19a.603.603 0 0 0-.06-.19.757.757 0 0 0-.09-.18c-.03-.05-.08-.1-.12-.15-.28-.27-.72-.37-1.09-.21-.13.05-.23.12-.33.21-.04.05-.09.1-.12.15-.04.06-.07.12-.09.18-.03.06-.05.12-.06.19-.01.06-.02.13-.02.19 0 .26.11.52.29.71.1.09.2.16.33.21.12.05.25.08.38.08.06 0 .13-.01.2-.02M13 16v-4a1 1 0 1 0-2 0v4a1 1 0 1 0 2 0M12 3c-4.962 0-9 4.038-9 9 0 4.963 4.038 9 9 9 4.963 0 9-4.037 9-9 0-4.962-4.037-9-9-9m0 20C5.935 23 1 18.065 1 12S5.935 1 12 1c6.066 0 11 4.935 11 11s-4.934 11-11 11" fill-rule="evenodd">
</path>
</g>
</svg>
<p>A <b>Hosting Key</b> will also be required, this can be found in your hosting properties, info found <a href="https://docs.budibase.com/self-hosting/hosting-settings">here</a>.</p>
</div>
</div>
<script>
window.addEventListener("load", () => {
let url = document.URL.split("//")[1]
if (url.substring(url.length - 1) === "/") {
url = url.substring(0, url.length - 1)
}
document.getElementById("url").innerHTML = url
})
</script>
</body>
</html>

View File

@ -1,17 +0,0 @@
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" x="0" y="0" viewBox="0 0 48 48" xml:space="preserve">
<style>
.st0{fill:#393c44}.st1{fill:#fff}
</style>
<path class="st0" d="M-152.17-24.17H-56V72h-96.17z"/>
<path class="st1" d="M-83.19 48h-41.79c-1.76 0-3.19-1.43-3.19-3.19V3.02c0-1.76 1.43-3.19 3.19-3.19h41.79c1.76 0 3.19 1.43 3.19 3.19v41.79c0 1.76-1.43 3.19-3.19 3.19z"/>
<path class="st0" d="M-99.62 12.57v9.94c1.15-1.21 2.59-1.81 4.32-1.81 1.03 0 1.97.19 2.82.58.86.39 1.59.91 2.19 1.57.6.66 1.08 1.43 1.42 2.32.34.89.51 1.84.51 2.85 0 1.03-.18 1.99-.53 2.89-.35.9-.84 1.68-1.47 2.35-.63.67-1.37 1.19-2.23 1.58-.86.39-1.78.58-2.77.58-1.8 0-3.22-.66-4.27-1.97V35h-4.89V12.57h4.9zm6.16 15.54c0-.43-.08-.84-.24-1.23-.16-.39-.39-.72-.68-1.01-.29-.29-.62-.52-1-.69-.38-.17-.79-.26-1.24-.26-.43 0-.84.08-1.22.24-.38.16-.71.39-.99.68-.28.29-.5.63-.68 1.01-.17.39-.26.8-.26 1.23 0 .43.08.84.24 1.22.16.38.39.71.68.99.29.28.63.5 1.01.68.39.17.8.26 1.23.26.43 0 .84-.08 1.22-.24.38-.16.71-.39.99-.68.28-.29.5-.62.68-1 .17-.39.26-.79.26-1.2z"/>
<path class="st0" d="M-114.76 12.57v9.94c1.15-1.21 2.59-1.81 4.32-1.81 1.03 0 1.97.19 2.82.58.86.39 1.59.91 2.19 1.57.6.66 1.08 1.43 1.42 2.32.34.89.51 1.84.51 2.85 0 1.03-.18 1.99-.53 2.89-.35.9-.84 1.68-1.47 2.35-.63.67-1.37 1.19-2.23 1.58-.86.39-1.78.58-2.77.58-1.8 0-3.22-.66-4.27-1.97V35h-4.89V12.57h4.9zm6.16 15.54c0-.43-.08-.84-.24-1.23-.16-.39-.39-.72-.68-1.01-.29-.29-.62-.52-1-.69-.38-.17-.79-.26-1.24-.26-.43 0-.84.08-1.22.24-.38.16-.71.39-.99.68-.28.29-.5.63-.68 1.01-.17.39-.26.8-.26 1.23 0 .43.08.84.24 1.22.16.38.39.71.68.99.29.28.63.5 1.01.68.39.17.8.26 1.23.26.43 0 .84-.08 1.22-.24.38-.16.71-.39.99-.68.28-.29.5-.62.68-1 .18-.39.26-.79.26-1.2z"/>
<path d="M44.81 159H3.02c-1.76 0-3.19-1.43-3.19-3.19v-41.79c0-1.76 1.43-3.19 3.19-3.19h41.79c1.76 0 3.19 1.43 3.19 3.19v41.79c0 1.76-1.43 3.19-3.19 3.19z" fill="#4285f4"/>
<path class="st1" d="M28.38 123.57v9.94c1.15-1.21 2.59-1.81 4.32-1.81 1.03 0 1.97.19 2.82.58.86.39 1.59.91 2.19 1.57.6.66 1.08 1.43 1.42 2.32.34.89.51 1.84.51 2.85 0 1.03-.18 1.99-.53 2.89-.35.9-.84 1.68-1.47 2.35-.63.67-1.37 1.19-2.23 1.58-.86.39-1.78.58-2.77.58-1.8 0-3.22-.66-4.27-1.97V146h-4.89v-22.43h4.9zm6.16 15.54c0-.43-.08-.84-.24-1.23-.16-.39-.39-.72-.68-1.01-.29-.29-.62-.52-1-.69-.38-.17-.79-.26-1.24-.26-.43 0-.84.08-1.22.24-.38.16-.71.39-.99.68-.28.29-.5.63-.68 1.01-.17.39-.26.8-.26 1.23 0 .43.08.84.24 1.22.16.38.39.71.68.99.29.28.63.5 1.01.68.39.17.8.26 1.23.26.43 0 .84-.08 1.22-.24.38-.16.71-.39.99-.68.28-.29.5-.62.68-1 .17-.39.26-.79.26-1.2z"/>
<path class="st1" d="M13.24 123.57v9.94c1.15-1.21 2.59-1.81 4.32-1.81 1.03 0 1.97.19 2.82.58.86.39 1.59.91 2.19 1.57.6.66 1.08 1.43 1.42 2.32.34.89.51 1.84.51 2.85 0 1.03-.18 1.99-.53 2.89-.35.9-.84 1.68-1.47 2.35-.63.67-1.37 1.19-2.23 1.58-.86.39-1.78.58-2.77.58-1.8 0-3.22-.66-4.27-1.97V146H8.35v-22.43h4.89zm6.16 15.54c0-.43-.08-.84-.24-1.23-.16-.39-.39-.72-.68-1.01-.29-.29-.62-.52-1-.69-.38-.17-.79-.26-1.24-.26-.43 0-.84.08-1.22.24-.38.16-.71.39-.99.68-.28.29-.5.63-.68 1.01-.17.39-.26.8-.26 1.23 0 .43.08.84.24 1.22.16.38.39.71.68.99.29.28.63.5 1.01.68.39.17.8.26 1.23.26.43 0 .84-.08 1.22-.24.38-.16.71-.39.99-.68.28-.29.5-.62.68-1 .18-.39.26-.79.26-1.2z"/>
<g>
<path class="st0" d="M44 48H4c-2.21 0-4-1.79-4-4V4c0-2.21 1.79-4 4-4h40c2.21 0 4 1.79 4 4v40c0 2.21-1.79 4-4 4z"/>
<path class="st1" d="M28.48 12v10.44c1.18-1.27 2.65-1.9 4.42-1.9 1.05 0 2.01.2 2.89.61.87.41 1.62.96 2.24 1.65.62.69 1.1 1.5 1.45 2.44.35.94.52 1.93.52 2.99 0 1.08-.18 2.09-.54 3.04-.36.95-.86 1.77-1.51 2.47-.64.7-1.4 1.25-2.28 1.66-.87.4-1.81.6-2.83.6-1.84 0-3.3-.69-4.37-2.07v1.62h-5V12h5.01zm6.3 16.31c0-.45-.08-.88-.25-1.29-.17-.41-.4-.76-.69-1.06-.3-.3-.64-.54-1.02-.72-.39-.18-.81-.27-1.27-.27-.44 0-.86.09-1.24.26-.39.17-.72.41-1.01.71-.29.3-.52.66-.69 1.06-.18.41-.26.84-.26 1.29s.08.88.25 1.28c.17.4.4.74.69 1.04.29.29.64.53 1.04.71.4.18.82.27 1.26.27.44 0 .86-.09 1.24-.26.39-.17.72-.41 1.01-.71.29-.3.52-.65.69-1.05.16-.41.25-.82.25-1.26z"/>
<path class="st1" d="M13 12v10.44c1.18-1.27 2.65-1.9 4.42-1.9 1.05 0 2.01.2 2.89.61.87.41 1.62.96 2.24 1.65.62.69 1.1 1.5 1.45 2.44.35.94.52 1.93.52 2.99 0 1.08-.18 2.09-.54 3.04-.36.95-.86 1.77-1.51 2.47-.64.7-1.4 1.25-2.28 1.66-.87.4-1.81.6-2.82.6-1.84 0-3.3-.69-4.37-2.07v1.62H8V12h5zm6.3 16.31c0-.45-.08-.88-.25-1.29-.17-.41-.4-.76-.69-1.06-.3-.3-.64-.54-1.02-.72-.39-.18-.81-.27-1.27-.27-.44 0-.86.09-1.24.26-.39.17-.72.41-1.01.71-.29.3-.52.66-.69 1.06-.18.41-.26.84-.26 1.29s.08.88.25 1.28c.17.4.4.74.69 1.04.29.29.64.53 1.04.71.4.18.82.27 1.26.27.44 0 .86-.09 1.24-.26.39-.17.72-.41 1.01-.71.29-.3.52-.65.69-1.05.16-.41.25-.82.25-1.26z"/>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 4.6 KiB

View File

@ -20,15 +20,9 @@ router.param("file", async (file, ctx, next) => {
await next() await next()
}) })
if (env.NODE_ENV !== "production") {
router.get("/_builder/:file*", controller.serveBuilder)
}
if (env.SELF_HOSTED) {
router.get("/", controller.serveSelfHostPage)
}
router router
// TODO: for now this _builder endpoint is not authorized/secured, will need to be
.get("/_builder/:file*", controller.serveBuilder)
.post("/api/attachments/process", authorized(BUILDER), controller.uploadFile) .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
.post("/api/attachments/upload", usage, controller.uploadFile) .post("/api/attachments/upload", usage, controller.uploadFile)
.get("/componentlibrary", controller.serveComponentLibrary) .get("/componentlibrary", controller.serveComponentLibrary)

View File

@ -1,17 +1,10 @@
const { budibaseTempDir } = require("./utilities/budibaseDir")
const { isDev } = require("./utilities")
const fixPath = require("fix-path") const fixPath = require("fix-path")
const fs = require("fs") const { checkDevelopmentEnvironment } = require("./utilities/fileSystem")
async function runServer() {
if (isDev() && !fs.existsSync(budibaseTempDir())) {
console.error(
"Please run a build before attempting to run server independently to fill 'tmp' directory."
)
process.exit(-1)
}
function runServer() {
// this will shutdown the system if development environment not ready
// will print an error explaining what to do
checkDevelopmentEnvironment()
fixPath() fixPath()
require("./app") require("./app")
} }

View File

@ -0,0 +1,40 @@
const { budibaseTempDir } = require("./budibaseDir")
const { isDev } = require("./index")
const fs = require("fs")
const { join } = require("path")
const { downloadTemplate } = require("./templates")
/**
* The single stack system (Cloud and Builder) should not make use of the file system where possible,
* this file handles all of the file access for the system with the intention of limiting it all to one
* place. Keeping all of this logic in one place means that when we need to do file system access (like
* downloading a package or opening a temporary file) in can be done in way that we can confirm it shouldn't
* be done through an object store instead.
*/
/**
* Checks if the system is currently in development mode and if it is makes sure
* everything required to function is ready.
*/
exports.checkDevelopmentEnvironment = () => {
if (isDev() && !fs.existsSync(budibaseTempDir())) {
console.error(
"Please run a build before attempting to run server independently to fill 'tmp' directory."
)
process.exit(-1)
}
}
/**
* This function manages temporary template files which are stored by Koa.
* @param {Object} template The template object retrieved from the Koa context object.
* @returns {Object} Returns an fs read stream which can be loaded into the database.
*/
exports.getTemplateStream = async template => {
if (template.file) {
return fs.createReadStream(template.file.path)
} else {
const templatePath = await downloadTemplate(...template.key.split("/"))
return fs.createReadStream(join(templatePath, "db", "dump.txt"))
}
}

File diff suppressed because it is too large Load Diff