General cleanup, doing away with the concept of hosting in the builder and the generally confusing difference between cloud, self hosting and running locally - server is simply always a server now.

This commit is contained in:
mike12345567 2021-05-11 17:49:26 +01:00
parent a32ae1db46
commit c9d903a92f
23 changed files with 72 additions and 491 deletions

View File

@ -14,3 +14,10 @@ exports.GlobalRoles = {
BUILDER: "builder",
GROUP_MANAGER: "group_manager",
}
exports.Configs = {
SETTINGS: "settings",
ACCOUNT: "account",
SMTP: "smtp",
GOOGLE: "google",
}

View File

@ -2,7 +2,6 @@ import { writable } from "svelte/store"
import api, { get } from "../api"
const INITIAL_HOSTING_UI_STATE = {
hostingInfo: {},
appUrl: "",
deployedApps: {},
deployedAppNames: [],
@ -13,28 +12,12 @@ export const getHostingStore = () => {
const store = writable({ ...INITIAL_HOSTING_UI_STATE })
store.actions = {
fetch: async () => {
const responses = await Promise.all([
api.get("/api/hosting/"),
api.get("/api/hosting/urls"),
])
const [info, urls] = await Promise.all(responses.map(resp => resp.json()))
const response = await api.get("/api/hosting/urls")
const urls = await response.json()
store.update(state => {
state.hostingInfo = info
state.appUrl = urls.app
return state
})
return info
},
save: async hostingInfo => {
const response = await api.post("/api/hosting", hostingInfo)
const revision = (await response.json()).rev
store.update(state => {
state.hostingInfo = {
...hostingInfo,
_rev: revision,
}
return state
})
},
fetchDeployedApps: async () => {
let deployments = await (await get("/api/hosting/apps")).json()

View File

@ -36,8 +36,7 @@
let errorReason
let poll
let deployments = []
let urlComponent =
$hostingStore.hostingInfo.type === "self" ? $store.url : `/${appId}`
let urlComponent = $store.url || `/${appId}`
let deploymentUrl = `${$hostingStore.appUrl}${urlComponent}`
const formatDate = (date, format) =>

View File

@ -49,27 +49,22 @@
onMount(async () => {
const nameError = "Your application must have a name.",
urlError = "Your application must have a URL."
let hostingInfo = await hostingStore.actions.fetch()
if (hostingInfo.type === "self") {
await hostingStore.actions.fetchDeployedApps()
const existingAppNames = get(hostingStore).deployedAppNames
const existingAppUrls = get(hostingStore).deployedAppUrls
const nameIdx = existingAppNames.indexOf(get(store).name)
const urlIdx = existingAppUrls.indexOf(get(store).url)
if (nameIdx !== -1) {
existingAppNames.splice(nameIdx, 1)
}
if (urlIdx !== -1) {
existingAppUrls.splice(urlIdx, 1)
}
nameValidation = {
name: string().required(nameError).notOneOf(existingAppNames),
}
urlValidation = {
url: string().required(urlError).notOneOf(existingAppUrls),
}
} else {
nameValidation = { name: string().required(nameError) }
await hostingStore.actions.fetchDeployedApps()
const existingAppNames = get(hostingStore).deployedAppNames
const existingAppUrls = get(hostingStore).deployedAppUrls
const nameIdx = existingAppNames.indexOf(get(store).name)
const urlIdx = existingAppUrls.indexOf(get(store).url)
if (nameIdx !== -1) {
existingAppNames.splice(nameIdx, 1)
}
if (urlIdx !== -1) {
existingAppUrls.splice(urlIdx, 1)
}
nameValidation = {
name: string().required(nameError).notOneOf(existingAppNames),
}
urlValidation = {
url: string().required(urlError).notOneOf(existingAppUrls),
}
})
</script>
@ -81,14 +76,12 @@
error={nameError}
label="App Name"
/>
{#if $hostingStore.hostingInfo.type === "self"}
<Input
on:change={e => updateApplication({ url: e.detail })}
value={$store.url}
error={urlError}
label="App URL"
/>
{/if}
<Input
on:change={e => updateApplication({ url: e.detail })}
value={$store.url}
error={urlError}
label="App URL"
/>
<TextArea
on:change={e => updateApplication({ description: e.detail })}
value={$store.description}

View File

@ -1,47 +1,19 @@
<script>
import { hostingStore } from "builderStore"
import { HostingTypes } from "constants/backend"
import {
Heading,
Divider,
notifications,
Input,
ModalContent,
Toggle,
Body,
} from "@budibase/bbui"
import ThemeEditor from "components/settings/ThemeEditor.svelte"
import analytics from "analytics"
import { onMount } from "svelte"
let hostingInfo
let selfhosted = false
$: analyticsDisabled = analytics.disabled()
async function save() {
hostingInfo.type = selfhosted ? HostingTypes.SELF : HostingTypes.CLOUD
if (!selfhosted && hostingInfo._rev) {
hostingInfo = {
type: hostingInfo.type,
_id: hostingInfo._id,
_rev: hostingInfo._rev,
}
}
try {
await hostingStore.actions.save(hostingInfo)
notifications.success(`Settings saved.`)
} catch (err) {
notifications.error(`Failed to update builder settings.`)
}
}
function updateSelfHosting(event) {
if (hostingInfo.type === HostingTypes.CLOUD && event.detail) {
hostingInfo.hostingUrl = "localhost:10000"
hostingInfo.useHttps = false
hostingInfo.selfHostKey = "budibase"
}
notifications.success(`Settings saved.`)
}
function toggleAnalytics() {
@ -51,33 +23,12 @@
analytics.optOut()
}
}
onMount(async () => {
hostingInfo = await hostingStore.actions.fetch()
selfhosted = hostingInfo.type === "self"
})
</script>
<ModalContent title="Builder settings" confirmText="Save" onConfirm={save}>
<Heading size="XS">Theme</Heading>
<ThemeEditor />
<Divider noMargin noGrid />
<Heading size="XS">Hosting</Heading>
<Body size="S">
This section contains settings that relate to the deployment and hosting of
apps made in this builder.
</Body>
<Toggle
text="Self hosted"
on:change={updateSelfHosting}
bind:value={selfhosted}
/>
{#if selfhosted}
<Input bind:value={hostingInfo.hostingUrl} label="Hosting URL" />
<Input bind:value={hostingInfo.selfHostKey} label="Hosting Key" />
<Toggle text="HTTPS" bind:value={hostingInfo.useHttps} />
{/if}
<Divider noMargin noGrid />
<Heading size="XS">Analytics</Heading>
<Body size="S">
If you would like to send analytics that help us make budibase better,

View File

@ -7,8 +7,6 @@
import Spinner from "components/common/Spinner.svelte"
import { Info, User } from "./Steps"
import Indicator from "./Indicator.svelte"
import { goto } from "@roxi/routify"
import { fade } from "svelte/transition"
import { post } from "builderStore/api"
import analytics from "analytics"
import { onMount } from "svelte"
@ -38,21 +36,18 @@
$: checkValidity($values, validators[$currentStep])
onMount(async () => {
const hostingInfo = await hostingStore.actions.fetch()
if (hostingInfo.type === "self") {
await hostingStore.actions.fetchDeployedApps()
const existingAppNames = svelteGet(hostingStore).deployedAppNames
validators[0].applicationName = string()
.required("Your application must have a name.")
.test(
"non-existing-app-name",
"App with same name already exists. Please try another app name.",
value =>
!existingAppNames.some(
appName => appName.toLowerCase() === value.toLowerCase()
)
)
}
await hostingStore.actions.fetchDeployedApps()
const existingAppNames = svelteGet(hostingStore).deployedAppNames
validators[0].applicationName = string()
.required("Your application must have a name.")
.test(
"non-existing-app-name",
"App with same name already exists. Please try another app name.",
value =>
!existingAppNames.some(
appName => appName.toLowerCase() === value.toLowerCase()
)
)
})
const checkValidity = async (values, validator) => {

View File

@ -1,6 +1,6 @@
<script>
import { Button, Modal, notifications, Heading } from "@budibase/bbui"
import { store, hostingStore } from "builderStore"
import { store } from "builderStore"
import api from "builderStore/api"
import DeploymentHistory from "components/deploy/DeploymentHistory.svelte"
import analytics from "analytics"
@ -15,18 +15,6 @@
$: appId = $store.appId
async function deployApp() {
// Must have cloud or self host API key to deploy
if (!$hostingStore.hostingInfo?.selfHostKey) {
const response = await api.get(`/api/keys/`)
const userKeys = await response.json()
if (!userKeys.budibase) {
notifications.error(
"No budibase API Keys configured. You must set either a self hosted or cloud API key to deploy your budibase app."
)
return
}
}
const DEPLOY_URL = `/api/deploy`
try {
@ -37,19 +25,10 @@
throw new Error()
}
analytics.captureEvent("Deployed App", {
appId,
hostingType: $hostingStore.hostingInfo?.type,
})
if (analytics.requestFeedbackOnDeploy()) {
feedbackModal.show()
}
} catch (err) {
analytics.captureEvent("Deploy App Failed", {
appId,
hostingType: $hostingStore.hostingInfo?.type,
})
analytics.captureException(err)
notifications.error("Deployment unsuccessful. Please try again later.")
}

View File

@ -30,7 +30,7 @@ const { cloneDeep } = require("lodash/fp")
const { processObject } = require("@budibase/string-templates")
const { getAllApps } = require("../../utilities")
const { USERS_TABLE_SCHEMA } = require("../../constants")
const { getDeployedApps } = require("../../utilities/builder/hosting")
const { getDeployedApps } = require("../../utilities/workerRequests")
const { clientLibraryPath } = require("../../utilities")
const URL_REGEX_SLASH = /\/|\\/g

View File

@ -1,5 +1,3 @@
const { getAppQuota } = require("./quota")
const env = require("../../../environment")
const newid = require("../../../db/newid")
/**
@ -11,24 +9,6 @@ class Deployment {
this._id = id || newid()
}
// purely so that we can do quota stuff outside the main deployment context
async init() {
if (!env.SELF_HOSTED) {
this.setQuota(await getAppQuota(this.appId))
}
}
setQuota(quota) {
if (!quota) {
return
}
this.quota = quota
}
getQuota() {
return this.quota
}
getAppId() {
return this.appId
}
@ -38,9 +18,6 @@ class Deployment {
return
}
this.verification = verification
if (this.verification.quota) {
this.quota = this.verification.quota
}
}
getVerification() {
@ -58,9 +35,6 @@ class Deployment {
if (json.verification) {
this.setVerification(json.verification)
}
if (json.quota) {
this.setQuota(json.quota)
}
if (json.status) {
this.setStatus(json.status, json.err)
}
@ -78,9 +52,6 @@ class Deployment {
if (this.verification && this.verification.cfDistribution) {
obj.cfDistribution = this.verification.cfDistribution
}
if (this.quota) {
obj.quota = this.quota
}
return obj
}
}

View File

@ -58,11 +58,6 @@ async function storeLocalDeploymentHistory(deployment) {
async function deployApp(deployment) {
const appId = deployment.getAppId()
try {
await deployment.init()
deployment.setVerification(
await deploymentService.preDeployment(deployment)
)
console.log(`Uploading assets for appID ${appId}..`)
await deploymentService.deploy(deployment)
@ -71,8 +66,6 @@ async function deployApp(deployment) {
console.log("Replicating local PouchDB to CouchDB..")
await deploymentService.replicateDb(deployment)
await deploymentService.postDeployment(deployment)
deployment.setStatus(DeploymentStatus.SUCCESS)
await storeLocalDeploymentHistory(deployment)
} catch (err) {

View File

@ -1,39 +0,0 @@
const PouchDB = require("../../../db")
const {
DocumentTypes,
SEPARATOR,
UNICODE_MAX,
ViewNames,
} = require("../../../db/utils")
exports.getAppQuota = async function (appId) {
const db = new PouchDB(appId)
const rows = await db.allDocs({
startkey: DocumentTypes.ROW + SEPARATOR,
endkey: DocumentTypes.ROW + SEPARATOR + UNICODE_MAX,
})
const users = await db.allDocs({
startkey: DocumentTypes.USER + SEPARATOR,
endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX,
})
const existingRows = rows.rows.length
const existingUsers = users.rows.length
const designDoc = await db.get("_design/database")
let views = 0
for (let viewName of Object.keys(designDoc.views)) {
if (Object.values(ViewNames).indexOf(viewName) === -1) {
views++
}
}
return {
rows: existingRows,
users: existingUsers,
views: views,
}
}

View File

@ -1,45 +1,7 @@
const AWS = require("aws-sdk")
const {
deployToObjectStore,
performReplication,
fetchCredentials,
} = require("./utils")
const {
getWorkerUrl,
getCouchUrl,
getSelfHostKey,
} = require("../../../utilities/builder/hosting")
exports.preDeployment = async function () {
const url = `${await getWorkerUrl()}/api/deploy`
try {
const json = await fetchCredentials(url, {
selfHostKey: await getSelfHostKey(),
})
// response contains:
// couchDbSession, bucket, objectStoreSession
// set credentials here, means any time we're verified we're ready to go
if (json.objectStoreSession) {
AWS.config.update({
accessKeyId: json.objectStoreSession.accessKeyId,
secretAccessKey: json.objectStoreSession.secretAccessKey,
})
}
return json
} catch (err) {
throw {
message: "Unauthorised to deploy, check self hosting key",
status: 401,
}
}
}
exports.postDeployment = async function () {
// we don't actively need to do anything after deployment in self hosting
}
exports.deploy = async function (deployment) {
const appId = deployment.getAppId()

View File

@ -1,41 +1,19 @@
const CouchDB = require("../../db")
const {
getHostingInfo,
getDeployedApps,
HostingTypes,
getAppUrl,
} = require("../../utilities/builder/hosting")
const { StaticDatabases } = require("../../db/utils")
exports.fetchInfo = async ctx => {
ctx.body = {
types: Object.values(HostingTypes),
}
}
exports.save = async ctx => {
const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name)
const { type } = ctx.request.body
if (type === HostingTypes.CLOUD && ctx.request.body._rev) {
ctx.body = await db.remove({
...ctx.request.body,
_id: StaticDatabases.BUILDER_HOSTING.baseDoc,
})
} else {
ctx.body = await db.put({
...ctx.request.body,
_id: StaticDatabases.BUILDER_HOSTING.baseDoc,
})
}
}
exports.fetch = async ctx => {
ctx.body = await getHostingInfo()
}
const { getDeployedApps } = require("../../utilities/workerRequests")
const { getScopedConfig } = require("@budibase/auth").db
const { Configs } = require("@budibase/auth").constants
const { checkSlashesInUrl } = require("../../utilities")
exports.fetchUrls = async ctx => {
const appId = ctx.appId
const db = new CouchDB(appId)
const settings = await getScopedConfig(db, { type: Configs.SETTINGS })
let appUrl = "http://localhost:10000/app"
if (settings && settings["platformUrl"]) {
appUrl = checkSlashesInUrl(`${settings["platformUrl"]}/app`)
}
ctx.body = {
app: await getAppUrl(ctx.appId),
app: appUrl,
}
}

View File

@ -9,14 +9,16 @@ class ScriptExecutor {
}
execute() {
const returnValue = this.script.runInContext(this.context)
return returnValue
return this.script.runInContext(this.context)
}
}
exports.execute = async function (ctx) {
const executor = new ScriptExecutor(ctx.request.body)
const result = executor.execute()
ctx.body = result
ctx.body = executor.execute()
}
exports.save = async function(ctx) {
ctx.throw(501, "Not currently implemented")
}

View File

@ -8,7 +8,7 @@ const { ObjectStoreBuckets } = require("../../../constants")
const { prepareUpload } = require("../deploy/utils")
const { processString } = require("@budibase/string-templates")
const { budibaseTempDir } = require("../../../utilities/budibaseDir")
const { getDeployedApps } = require("../../../utilities/builder/hosting")
const { getDeployedApps } = require("../../../utilities/workerRequests")
const CouchDB = require("../../../db")
const {
loadHandlebarsFile,

View File

@ -7,10 +7,7 @@ const { BUILDER } = require("../../utilities/security/permissions")
const router = Router()
router
.get("/api/hosting/info", authorized(BUILDER), controller.fetchInfo)
.get("/api/hosting/urls", authorized(BUILDER), controller.fetchUrls)
.get("/api/hosting", authorized(BUILDER), controller.fetch)
.post("/api/hosting", authorized(BUILDER), controller.save)
// this isn't risky, doesn't return anything about apps other than names and URLs
.get("/api/hosting/apps", selfhost, controller.getDeployedApps)

View File

@ -1,5 +1,5 @@
const Router = require("@koa/router")
const controller = require("../controllers/hosting")
const controller = require("../controllers/script")
const authorized = require("../../middleware/authorized")
const { BUILDER } = require("../../utilities/security/permissions")

View File

@ -15,25 +15,6 @@ describe("/hosting", () => {
app = await config.init()
})
describe("fetchInfo", () => {
it("should be able to fetch hosting information", async () => {
const res = await request
.get(`/api/hosting/info`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body).toEqual({ types: ["cloud", "self"]})
})
it("should apply authorization to endpoint", async () => {
await checkBuilderEndpoint({
config,
method: "GET",
url: `/api/hosting/info`,
})
})
})
describe("fetchUrls", () => {
it("should be able to fetch current app URLs", async () => {
const res = await request
@ -41,7 +22,7 @@ describe("/hosting", () => {
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body.app).toEqual(`https://${config.getAppId()}.app.budi.live`)
expect(res.body.app).toEqual(`http://localhost:10000/app`)
})
it("should apply authorization to endpoint", async () => {
@ -52,78 +33,4 @@ describe("/hosting", () => {
})
})
})
describe("fetch", () => {
it("should be able to fetch the current hosting information", async () => {
const res = await request
.get(`/api/hosting`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body._id).toBeDefined()
expect(res.body.hostingUrl).toBeDefined()
expect(res.body.type).toEqual("cloud")
})
it("should apply authorization to endpoint", async () => {
await checkBuilderEndpoint({
config,
method: "GET",
url: `/api/hosting`,
})
})
})
describe("save", () => {
it("should be able to update the hosting information", async () => {
const res = await request
.post(`/api/hosting`)
.send({
type: "self",
selfHostKey: "budibase",
hostingUrl: "localhost:10000",
useHttps: false,
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body.ok).toEqual(true)
// make sure URL updated
const urlRes = await request
.get(`/api/hosting/urls`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(urlRes.body.app).toEqual(`http://localhost:10000/app`)
})
it("should apply authorization to endpoint", async () => {
await checkBuilderEndpoint({
config,
method: "POST",
url: `/api/hosting`,
})
})
})
describe("getDeployedApps", () => {
it("should fail when not self hosted", async () => {
await request
.get(`/api/hosting/apps`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(400)
})
it("should get apps when in cloud", async () => {
await setup.switchToSelfHosted(async () => {
const res = await request
.get(`/api/hosting/apps`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body.app1).toEqual({url: "/app1"})
})
})
})
})

View File

@ -8,11 +8,6 @@ const StaticDatabases = {
name: "builder-db",
baseDoc: "builder-doc",
},
// TODO: needs removed
BUILDER_HOSTING: {
name: "builder-config-db",
baseDoc: "hosting-doc",
},
}
const DocumentTypes = {

View File

@ -1,86 +0,0 @@
const CouchDB = require("../../db")
const { StaticDatabases } = require("../../db/utils")
const { getDeployedApps } = require("../../utilities/workerRequests")
const PROD_HOSTING_URL = "app.budi.live"
function getProtocol(hostingInfo) {
return hostingInfo.useHttps ? "https://" : "http://"
}
async function getURLWithPath(pathIfSelfHosted) {
const hostingInfo = await exports.getHostingInfo()
const protocol = getProtocol(hostingInfo)
const path =
hostingInfo.type === exports.HostingTypes.SELF ? pathIfSelfHosted : ""
return `${protocol}${hostingInfo.hostingUrl}${path}`
}
exports.HostingTypes = {
CLOUD: "cloud",
SELF: "self",
}
exports.getHostingInfo = async () => {
const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name)
let doc
try {
doc = await db.get(StaticDatabases.BUILDER_HOSTING.baseDoc)
} catch (err) {
// don't write this doc, want to be able to update these default props
// for our servers with a new release without needing to worry about state of
// PouchDB in peoples installations
doc = {
_id: StaticDatabases.BUILDER_HOSTING.baseDoc,
type: exports.HostingTypes.CLOUD,
hostingUrl: PROD_HOSTING_URL,
selfHostKey: "",
templatesUrl: "prod-budi-templates.s3-eu-west-1.amazonaws.com",
useHttps: true,
}
}
return doc
}
exports.getAppUrl = async appId => {
const hostingInfo = await exports.getHostingInfo()
const protocol = getProtocol(hostingInfo)
let url
if (hostingInfo.type === exports.HostingTypes.CLOUD) {
url = `${protocol}${appId}.${hostingInfo.hostingUrl}`
} else {
url = `${protocol}${hostingInfo.hostingUrl}/app`
}
return url
}
exports.getWorkerUrl = async () => {
return getURLWithPath("/worker")
}
exports.getMinioUrl = async () => {
return getURLWithPath("/")
}
exports.getCouchUrl = async () => {
return getURLWithPath("/db")
}
exports.getSelfHostKey = async () => {
const hostingInfo = await exports.getHostingInfo()
return hostingInfo.selfHostKey
}
exports.getTemplatesUrl = async (appId, type, name) => {
const hostingInfo = await exports.getHostingInfo()
const protocol = getProtocol(hostingInfo)
let path
if (type && name) {
path = `templates/type/${name}.tar.gz`
} else {
path = "manifest.json"
}
return `${protocol}${hostingInfo.templatesUrl}/${path}`
}
exports.getDeployedApps = getDeployedApps

View File

@ -37,9 +37,6 @@ function request(ctx, request) {
exports.request = request
exports.getDeployedApps = async ctx => {
if (!env.SELF_HOSTED) {
throw "Can only check apps for self hosted environments"
}
try {
const response = await fetch(
checkSlashesInUrl(env.WORKER_URL + `/api/apps`),

View File

@ -2,7 +2,7 @@
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "0.8.16",
"description": "Budibase Deployment Server",
"description": "Budibase background service",
"main": "src/index.js",
"repository": {
"type": "git",

View File

@ -1,3 +1,5 @@
const { Configs } = require("@budibase/auth").constants
exports.LOGO_URL =
"https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg"
@ -10,12 +12,7 @@ exports.Groups = {
ALL_USERS: "all_users",
}
exports.Configs = {
SETTINGS: "settings",
ACCOUNT: "account",
SMTP: "smtp",
GOOGLE: "google",
}
exports.Configs = Configs
exports.ConfigUploads = {
LOGO: "logo",