MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1537 from Budibase/feature/app-updated-at 

Mike fixes + application updated at timestamps
This commit is contained in:
Michael Drury 2021-05-21 16:10:29 +01:00 committed by GitHub
parent cb274d33ed 8c8ac18ac0
commit 8ee874055e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 176 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hosting/docker-compose.yaml
View File

@ -35,6 +35,7 @@ services:
environment: environment:
SELF_HOSTED: 1 SELF_HOSTED: 1
PORT: 4003 PORT: 4003
CLUSTER_PORT: ${MAIN_PORT}
JWT_SECRET: ${JWT_SECRET} JWT_SECRET: ${JWT_SECRET}
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}

11
packages/auth/src/redis/index.js
View File

@ -6,6 +6,7 @@ const { addDbPrefix, removeDbPrefix, getRedisOptions } = require("./utils")
const CLUSTERED = false const CLUSTERED = false
// for testing just generate the client once // for testing just generate the client once
let CONNECTED = false
let CLIENT = env.isTest() ? new Redis(getRedisOptions()) : null let CLIENT = env.isTest() ? new Redis(getRedisOptions()) : null
/** /**
@ -16,14 +17,9 @@ let CLIENT = env.isTest() ? new Redis(getRedisOptions()) : null
function init() { function init() {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
// testing uses a single in memory client // testing uses a single in memory client
if (env.isTest()) { if (env.isTest() || (CLIENT && CONNECTED)) {
return resolve(CLIENT) return resolve(CLIENT)
} }
// if a connection existed, close it and re-create it
if (CLIENT) {
CLIENT.disconnect()
CLIENT = null
}
const { opts, host, port } = getRedisOptions(CLUSTERED) const { opts, host, port } = getRedisOptions(CLUSTERED)
if (CLUSTERED) { if (CLUSTERED) {
CLIENT = new Redis.Cluster([{ host, port }], opts) CLIENT = new Redis.Cluster([{ host, port }], opts)
@ -32,12 +28,15 @@ function init() {
} }
CLIENT.on("end", err => { CLIENT.on("end", err => {
reject(err) reject(err)
CONNECTED = false
}) })
CLIENT.on("error", err => { CLIENT.on("error", err => {
reject(err) reject(err)
CONNECTED = false
}) })
CLIENT.on("connect", () => { CLIENT.on("connect", () => {
resolve(CLIENT) resolve(CLIENT)
CONNECTED = true
}) })
}) })
} }

1
packages/auth/src/redis/utils.js
View File

@ -10,6 +10,7 @@ exports.Databases = {
PW_RESETS: "pwReset", PW_RESETS: "pwReset",
INVITATIONS: "invitation", INVITATIONS: "invitation",
DEV_LOCKS: "devLocks", DEV_LOCKS: "devLocks",
DEBOUNCE: "debounce",
} }
exports.getRedisOptions = (clustered = false) => { exports.getRedisOptions = (clustered = false) => {

11
packages/builder/src/components/start/AppCard.svelte
View File

@ -9,6 +9,9 @@
StatusLight, StatusLight,
} from "@budibase/bbui" } from "@budibase/bbui"
import { gradient } from "actions" import { gradient } from "actions"
import { auth } from "stores/portal"
import { AppStatus } from "constants"
import { processStringSync } from "@budibase/string-templates"
export let app export let app
export let exportApp export let exportApp
@ -60,7 +63,13 @@
</div> </div>
<div class="status"> <div class="status">
<Body size="S"> <Body size="S">
Updated {Math.floor(1 + Math.random() * 10)} months ago {#if app.updatedAt}
{processStringSync("Updated {{ duration time 'millisecond' }} ago", {
time: new Date().getTime() - new Date(app.updatedAt).getTime(),
})}
{:else}
Never updated
{/if}
</Body> </Body>
<StatusLight active={app.deployed} neutral={!app.deployed}> <StatusLight active={app.deployed} neutral={!app.deployed}>
{#if app.deployed}Published{:else}Unpublished{/if} {#if app.deployed}Published{:else}Unpublished{/if}

18
packages/server/__mocks__/pg.js
View File

@ -1,9 +1,6 @@
const pg = {} const pg = {}
// constructor const query = jest.fn(() => ({
function Client() {}
Client.prototype.query = jest.fn(() => ({
rows: [ rows: [
{ {
a: "string", a: "string",
@ -12,8 +9,21 @@ Client.prototype.query = jest.fn(() => ({
], ],
})) }))
// constructor
function Client() {}
Client.prototype.query = query
Client.prototype.connect = jest.fn() Client.prototype.connect = jest.fn()
Client.prototype.release = jest.fn()
function Pool() {}
Pool.prototype.query = query
Pool.prototype.connect = jest.fn(() => {
return new Client()
})
pg.Client = Client pg.Client = Client
pg.Pool = Pool
pg.queryMock = query
module.exports = pg module.exports = pg

3
packages/server/package.json
View File

@ -66,7 +66,8 @@
"!src/tests/**/*", "!src/tests/**/*",
"!src/automations/tests/**/*", "!src/automations/tests/**/*",
"!src/utilities/fileProcessor.js", "!src/utilities/fileProcessor.js",
"!src/utilities/fileSystem/**/*" "!src/utilities/fileSystem/**/*",
"!src/utilities/redis.js"
], ],
"coverageReporters": [ "coverageReporters": [
"lcov", "lcov",

5
packages/server/src/api/controllers/application.js
View File

@ -190,6 +190,8 @@ exports.create = async function (ctx) {
url: url, url: url,
template: ctx.request.body.template, template: ctx.request.body.template,
instance: instance, instance: instance,
updatedAt: new Date().toISOString(),
createdAt: new Date().toISOString(),
deployment: { deployment: {
type: "cloud", type: "cloud",
}, },
@ -214,6 +216,9 @@ exports.update = async function (ctx) {
const data = ctx.request.body const data = ctx.request.body
const newData = { ...application, ...data, url } const newData = { ...application, ...data, url }
if (ctx.request.body._rev !== application._rev) {
newData._rev = application._rev
}
// the locked by property is attached by server but generated from // the locked by property is attached by server but generated from
// Redis, shouldn't ever store it // Redis, shouldn't ever store it

31
packages/server/src/api/routes/tests/application.spec.js
View File

@ -1,12 +1,18 @@
const { clearAllApps, checkBuilderEndpoint } = require("./utilities/TestFunctions") const { clearAllApps, checkBuilderEndpoint } = require("./utilities/TestFunctions")
const setup = require("./utilities") const setup = require("./utilities")
const { AppStatus } = require("../../../db/utils")
jest.mock("../../../utilities/redis", () => ({ jest.mock("../../../utilities/redis", () => ({
init: jest.fn(), init: jest.fn(),
getAllLocks: () => { getAllLocks: () => {
return [] return []
}, },
doesUserHaveLock: () => {
return true
},
updateLock: jest.fn(), updateLock: jest.fn(),
setDebounce: jest.fn(),
checkDebounce: jest.fn(),
})) }))
describe("/applications", () => { describe("/applications", () => {
@ -47,7 +53,7 @@ describe("/applications", () => {
await config.createApp(request, "app2") await config.createApp(request, "app2")
const res = await request const res = await request
.get("/api/applications?status=dev") .get(`/api/applications?status=${AppStatus.DEV}`)
.set(config.defaultHeaders()) .set(config.defaultHeaders())
.expect('Content-Type', /json/) .expect('Content-Type', /json/)
.expect(200) .expect(200)
@ -104,4 +110,27 @@ describe("/applications", () => {
expect(res.body.rev).toBeDefined() expect(res.body.rev).toBeDefined()
}) })
}) })
describe("edited at", () => {
it("middleware should set edited at", async () => {
const headers = config.defaultHeaders()
headers["referer"] = `/${config.getAppId()}/test`
const res = await request
.put(`/api/applications/${config.getAppId()}`)
.send({
name: "UPDATED_NAME",
})
.set(headers)
.expect('Content-Type', /json/)
.expect(200)
expect(res.body.rev).toBeDefined()
// retrieve the app to check it
const getRes = await request
.get(`/api/applications/${config.getAppId()}/appPackage`)
.set(headers)
.expect('Content-Type', /json/)
.expect(200)
expect(getRes.body.application.updatedAt).toBeDefined()
})
})
}) })

3
packages/server/src/api/routes/tests/utilities/TestFunctions.js
View File

@ -1,6 +1,7 @@
const rowController = require("../../../controllers/row") const rowController = require("../../../controllers/row")
const appController = require("../../../controllers/application") const appController = require("../../../controllers/application")
const CouchDB = require("../../../../db") const CouchDB = require("../../../../db")
const { AppStatus } = require("../../../../db/utils")
function Request(appId, params) { function Request(appId, params) {
this.appId = appId this.appId = appId
@ -14,7 +15,7 @@ exports.getAllTableRows = async config => {
} }
exports.clearAllApps = async () => { exports.clearAllApps = async () => {
const req = { query: { status: "dev" } } const req = { query: { status: AppStatus.DEV } }
await appController.fetch(req) await appController.fetch(req)
const apps = req.body const apps = req.body
if (!apps || apps.length <= 0) { if (!apps || apps.length <= 0) {

12
packages/server/src/integrations/tests/postgres.spec.js
View File

@ -20,7 +20,7 @@ describe("Postgres Integration", () => {
const response = await config.integration.create({ const response = await config.integration.create({
sql sql
}) })
expect(config.integration.client.query).toHaveBeenCalledWith(sql) expect(pg.queryMock).toHaveBeenCalledWith(sql)
}) })
it("calls the read method with the correct params", async () => { it("calls the read method with the correct params", async () => {
@ -28,7 +28,7 @@ describe("Postgres Integration", () => {
const response = await config.integration.read({ const response = await config.integration.read({
sql sql
}) })
expect(config.integration.client.query).toHaveBeenCalledWith(sql) expect(pg.queryMock).toHaveBeenCalledWith(sql)
}) })
it("calls the update method with the correct params", async () => { it("calls the update method with the correct params", async () => {
@ -36,20 +36,20 @@ describe("Postgres Integration", () => {
const response = await config.integration.update({ const response = await config.integration.update({
sql sql
}) })
expect(config.integration.client.query).toHaveBeenCalledWith(sql) expect(pg.queryMock).toHaveBeenCalledWith(sql)
}) })
it("calls the delete method with the correct params", async () => { it("calls the delete method with the correct params", async () => {
const sql = "delete from users where name = 'todelete';" const sql = "delete from users where name = 'todelete';"
const response = await config.integration.delete({ await config.integration.delete({
sql sql
}) })
expect(config.integration.client.query).toHaveBeenCalledWith(sql) expect(pg.queryMock).toHaveBeenCalledWith(sql)
}) })
describe("no rows returned", () => { describe("no rows returned", () => {
beforeEach(() => { beforeEach(() => {
config.integration.client.query.mockImplementation(() => ({ rows: [] })) pg.queryMock.mockImplementation(() => ({ rows: [] }))
}) })
it("returns the correct response when the create response has no rows", async () => { it("returns the correct response when the create response has no rows", async () => {

36
packages/server/src/middleware/authorized.js
View File

@ -4,8 +4,7 @@ const {
doesHaveResourcePermission, doesHaveResourcePermission,
doesHaveBasePermission, doesHaveBasePermission,
} = require("@budibase/auth/permissions") } = require("@budibase/auth/permissions")
const { APP_DEV_PREFIX } = require("../db/utils") const builderMiddleware = require("./builder")
const { doesUserHaveLock, updateLock } = require("../utilities/redis")
function hasResource(ctx) { function hasResource(ctx) {
return ctx.resourceId != null return ctx.resourceId != null
@ -15,26 +14,6 @@ const WEBHOOK_ENDPOINTS = new RegExp(
["webhooks/trigger", "webhooks/schema"].join("|") ["webhooks/trigger", "webhooks/schema"].join("|")
) )
async function checkDevAppLocks(ctx) {
const appId = ctx.appId
// if any public usage, don't proceed
if (!ctx.user._id && !ctx.user.userId) {
return
}
// not a development app, don't need to do anything
if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
return
}
if (!(await doesUserHaveLock(appId, ctx.user))) {
ctx.throw(403, "User does not hold app lock.")
}
// they do have lock, update it
await updateLock(appId, ctx.user)
}
module.exports = (permType, permLevel = null) => async (ctx, next) => { module.exports = (permType, permLevel = null) => async (ctx, next) => {
// webhooks don't need authentication, each webhook unique // webhooks don't need authentication, each webhook unique
if (WEBHOOK_ENDPOINTS.test(ctx.request.url)) { if (WEBHOOK_ENDPOINTS.test(ctx.request.url)) {
@ -45,13 +24,9 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => {
return ctx.throw(403, "No user info found") return ctx.throw(403, "No user info found")
} }
const builderCall = permType === PermissionTypes.BUILDER // check general builder stuff, this middleware is a good way
const referer = ctx.headers["referer"] // to find API endpoints which are builder focused
const editingApp = referer ? referer.includes(ctx.appId) : false await builderMiddleware(ctx, permType)
// this makes sure that builder calls abide by dev locks
if (builderCall && editingApp) {
await checkDevAppLocks(ctx)
}
const isAuthed = ctx.isAuthenticated const isAuthed = ctx.isAuthenticated
const { basePermissions, permissions } = await getUserPermissions( const { basePermissions, permissions } = await getUserPermissions(
@ -62,9 +37,10 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => {
// builders for now have permission to do anything // builders for now have permission to do anything
// TODO: in future should consider separating permissions with an require("@budibase/auth").isClient check // TODO: in future should consider separating permissions with an require("@budibase/auth").isClient check
let isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global let isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
const isBuilderApi = permType === PermissionTypes.BUILDER
if (isBuilder) { if (isBuilder) {
return next() return next()
} else if (builderCall && !isBuilder) { } else if (isBuilderApi && !isBuilder) {
return ctx.throw(403, "Not Authorized") return ctx.throw(403, "Not Authorized")
} }

75
packages/server/src/middleware/builder.js Normal file
View File

@ -0,0 +1,75 @@
const { APP_DEV_PREFIX } = require("../db/utils")
const {
doesUserHaveLock,
updateLock,
checkDebounce,
setDebounce,
} = require("../utilities/redis")
const CouchDB = require("../db")
const { DocumentTypes } = require("../db/utils")
const { PermissionTypes } = require("@budibase/auth/permissions")
const DEBOUNCE_TIME_SEC = 30
/************************************************** *
* This middleware has been broken out of the *
* "authorized" middleware as it had nothing to do *
* with authorization, but requires the perms *
* imparted by it. This middleware shouldn't *
* be called directly, it should always be called *
* through the authorized middleware *
****************************************************/
async function checkDevAppLocks(ctx) {
const appId = ctx.appId
// if any public usage, don't proceed
if (!ctx.user._id && !ctx.user.userId) {
return
}
// not a development app, don't need to do anything
if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
return
}
if (!(await doesUserHaveLock(appId, ctx.user))) {
ctx.throw(403, "User does not hold app lock.")
}
// they do have lock, update it
await updateLock(appId, ctx.user)
}
async function updateAppUpdatedAt(ctx) {
const appId = ctx.appId
// if debouncing skip this update
// get methods also aren't updating
if (ctx.method === "GET" || (await checkDebounce(appId))) {
return
}
const db = new CouchDB(appId)
const metadata = await db.get(DocumentTypes.APP_METADATA)
metadata.updatedAt = new Date().toISOString()
await db.put(metadata)
// set a new debounce record with a short TTL
await setDebounce(appId, DEBOUNCE_TIME_SEC)
}
module.exports = async (ctx, permType) => {
const appId = ctx.appId
// this only functions within an app context
if (!appId) {
return
}
const isBuilderApi = permType === PermissionTypes.BUILDER
const referer = ctx.headers["referer"]
const editingApp = referer ? referer.includes(appId) : false
// check this is a builder call and editing
if (!isBuilderApi || !editingApp) {
return
}
// check locks
await checkDevAppLocks(ctx)
// set updated at time on app
await updateAppUpdatedAt(ctx)
}

3
packages/server/src/middleware/tests/authorized.spec.js
View File

@ -78,9 +78,6 @@ describe("Authorization middleware", () => {
}) })
describe("external web hook call", () => { describe("external web hook call", () => {
let ctx = {}
let middleware
beforeEach(() => { beforeEach(() => {
config = new TestConfiguration() config = new TestConfiguration()
config.setEnvironment(true) config.setEnvironment(true)

14
packages/server/src/utilities/redis.js
View File

@ -2,13 +2,13 @@ const { Client, utils } = require("@budibase/auth/redis")
const { getGlobalIDFromUserMetadataID } = require("../db/utils") const { getGlobalIDFromUserMetadataID } = require("../db/utils")
const APP_DEV_LOCK_SECONDS = 600 const APP_DEV_LOCK_SECONDS = 600
const DB_NAME = utils.Databases.DEV_LOCKS let devAppClient, debounceClient
let devAppClient
// we init this as we want to keep the connection open all the time // we init this as we want to keep the connection open all the time
// reduces the performance hit // reduces the performance hit
exports.init = async () => { exports.init = async () => {
devAppClient = await new Client(DB_NAME).init() devAppClient = await new Client(utils.Databases.DEV_LOCKS).init()
debounceClient = await new Client(utils.Databases.DEBOUNCE).init()
} }
exports.doesUserHaveLock = async (devAppId, user) => { exports.doesUserHaveLock = async (devAppId, user) => {
@ -52,3 +52,11 @@ exports.clearLock = async (devAppId, user) => {
} }
await devAppClient.delete(devAppId) await devAppClient.delete(devAppId)
} }
exports.checkDebounce = async id => {
return debounceClient.get(id)
}
exports.setDebounce = async (id, seconds) => {
await debounceClient.store(id, "debouncing", seconds)
}

1
packages/worker/scripts/dev/manage.js
View File

@ -7,6 +7,7 @@ async function init() {
const envFileJson = { const envFileJson = {
SELF_HOSTED: 1, SELF_HOSTED: 1,
PORT: 4002, PORT: 4002,
CLUSTER_PORT: 10000,
JWT_SECRET: "testsecret", JWT_SECRET: "testsecret",
INTERNAL_API_KEY: "budibase", INTERNAL_API_KEY: "budibase",
MINIO_ACCESS_KEY: "budibase", MINIO_ACCESS_KEY: "budibase",

4
packages/worker/src/api/controllers/admin/users.js
View File

@ -42,6 +42,10 @@ exports.save = async ctx => {
_id: _id || generateGlobalUserID(), _id: _id || generateGlobalUserID(),
password: hashedPassword, password: hashedPassword,
} }
// make sure the roles object is always present
if (!user.roles) {
user.roles = {}
}
// add the active status to a user if its not provided // add the active status to a user if its not provided
if (user.status == null) { if (user.status == null) {
user.status = UserStatus.ACTIVE user.status = UserStatus.ACTIVE

1
packages/worker/src/environment.js
View File

@ -19,6 +19,7 @@ if (!LOADED && isDev() && !isTest()) {
module.exports = { module.exports = {
SELF_HOSTED: process.env.SELF_HOSTED, SELF_HOSTED: process.env.SELF_HOSTED,
PORT: process.env.PORT, PORT: process.env.PORT,
CLUSTER_PORT: process.env.CLUSTER_PORT,
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
MINIO_URL: process.env.MINIO_URL, MINIO_URL: process.env.MINIO_URL,

3
packages/worker/src/utilities/templates.js
View File

@ -7,8 +7,9 @@ const {
EmailTemplatePurpose, EmailTemplatePurpose,
} = require("../constants") } = require("../constants")
const { checkSlashesInUrl } = require("./index") const { checkSlashesInUrl } = require("./index")
const env = require("../environment")
const LOCAL_URL = `http://localhost:10000` const LOCAL_URL = `http://localhost:${env.CLUSTER_PORT || 10000}`
const BASE_COMPANY = "Budibase" const BASE_COMPANY = "Budibase"
exports.getSettingsTemplateContext = async (purpose, code = null) => { exports.getSettingsTemplateContext = async (purpose, code = null) => {