diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index b609f5d962..37657ce009 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -35,6 +35,7 @@ services: environment: SELF_HOSTED: 1 PORT: 4003 + CLUSTER_PORT: ${MAIN_PORT} JWT_SECRET: ${JWT_SECRET} MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} diff --git a/packages/auth/src/redis/index.js b/packages/auth/src/redis/index.js index 5db80a216b..78e3ea7acd 100644 --- a/packages/auth/src/redis/index.js +++ b/packages/auth/src/redis/index.js @@ -6,6 +6,7 @@ const { addDbPrefix, removeDbPrefix, getRedisOptions } = require("./utils") const CLUSTERED = false // for testing just generate the client once +let CONNECTED = false let CLIENT = env.isTest() ? new Redis(getRedisOptions()) : null /** @@ -16,14 +17,9 @@ let CLIENT = env.isTest() ? new Redis(getRedisOptions()) : null function init() { return new Promise((resolve, reject) => { // testing uses a single in memory client - if (env.isTest()) { + if (env.isTest() || (CLIENT && CONNECTED)) { return resolve(CLIENT) } - // if a connection existed, close it and re-create it - if (CLIENT) { - CLIENT.disconnect() - CLIENT = null - } const { opts, host, port } = getRedisOptions(CLUSTERED) if (CLUSTERED) { CLIENT = new Redis.Cluster([{ host, port }], opts) @@ -32,12 +28,15 @@ function init() { } CLIENT.on("end", err => { reject(err) + CONNECTED = false }) CLIENT.on("error", err => { reject(err) + CONNECTED = false }) CLIENT.on("connect", () => { resolve(CLIENT) + CONNECTED = true }) }) } diff --git a/packages/auth/src/redis/utils.js b/packages/auth/src/redis/utils.js index efdd2aa48d..23702353d8 100644 --- a/packages/auth/src/redis/utils.js +++ b/packages/auth/src/redis/utils.js @@ -10,6 +10,7 @@ exports.Databases = { PW_RESETS: "pwReset", INVITATIONS: "invitation", DEV_LOCKS: "devLocks", + DEBOUNCE: "debounce", } exports.getRedisOptions = (clustered = false) => { diff --git a/packages/builder/src/components/start/AppCard.svelte b/packages/builder/src/components/start/AppCard.svelte index 0d7b155f2a..c5937c5637 100644 --- a/packages/builder/src/components/start/AppCard.svelte +++ b/packages/builder/src/components/start/AppCard.svelte @@ -9,6 +9,9 @@ StatusLight, } from "@budibase/bbui" import { gradient } from "actions" + import { auth } from "stores/portal" + import { AppStatus } from "constants" + import { processStringSync } from "@budibase/string-templates" export let app export let exportApp @@ -60,7 +63,13 @@
- Updated {Math.floor(1 + Math.random() * 10)} months ago + {#if app.updatedAt} + {processStringSync("Updated {{ duration time 'millisecond' }} ago", { + time: new Date().getTime() - new Date(app.updatedAt).getTime(), + })} + {:else} + Never updated + {/if} {#if app.deployed}Published{:else}Unpublished{/if} diff --git a/packages/server/__mocks__/pg.js b/packages/server/__mocks__/pg.js index 0d8b8cc26a..5d4e3793fd 100644 --- a/packages/server/__mocks__/pg.js +++ b/packages/server/__mocks__/pg.js @@ -1,9 +1,6 @@ const pg = {} -// constructor -function Client() {} - -Client.prototype.query = jest.fn(() => ({ +const query = jest.fn(() => ({ rows: [ { a: "string", @@ -12,8 +9,21 @@ Client.prototype.query = jest.fn(() => ({ ], })) +// constructor +function Client() {} + +Client.prototype.query = query Client.prototype.connect = jest.fn() +Client.prototype.release = jest.fn() + +function Pool() {} +Pool.prototype.query = query +Pool.prototype.connect = jest.fn(() => { + return new Client() +}) pg.Client = Client +pg.Pool = Pool +pg.queryMock = query module.exports = pg diff --git a/packages/server/package.json b/packages/server/package.json index 5afb4db968..2af49cd91d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -66,7 +66,8 @@ "!src/tests/**/*", "!src/automations/tests/**/*", "!src/utilities/fileProcessor.js", - "!src/utilities/fileSystem/**/*" + "!src/utilities/fileSystem/**/*", + "!src/utilities/redis.js" ], "coverageReporters": [ "lcov", diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index 386c0f1d7a..2511686bed 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -190,6 +190,8 @@ exports.create = async function (ctx) { url: url, template: ctx.request.body.template, instance: instance, + updatedAt: new Date().toISOString(), + createdAt: new Date().toISOString(), deployment: { type: "cloud", }, @@ -214,6 +216,9 @@ exports.update = async function (ctx) { const data = ctx.request.body const newData = { ...application, ...data, url } + if (ctx.request.body._rev !== application._rev) { + newData._rev = application._rev + } // the locked by property is attached by server but generated from // Redis, shouldn't ever store it diff --git a/packages/server/src/api/routes/tests/application.spec.js b/packages/server/src/api/routes/tests/application.spec.js index 9783079124..dc550cd237 100644 --- a/packages/server/src/api/routes/tests/application.spec.js +++ b/packages/server/src/api/routes/tests/application.spec.js @@ -1,12 +1,18 @@ const { clearAllApps, checkBuilderEndpoint } = require("./utilities/TestFunctions") const setup = require("./utilities") +const { AppStatus } = require("../../../db/utils") jest.mock("../../../utilities/redis", () => ({ init: jest.fn(), getAllLocks: () => { return [] }, + doesUserHaveLock: () => { + return true + }, updateLock: jest.fn(), + setDebounce: jest.fn(), + checkDebounce: jest.fn(), })) describe("/applications", () => { @@ -47,7 +53,7 @@ describe("/applications", () => { await config.createApp(request, "app2") const res = await request - .get("/api/applications?status=dev") + .get(`/api/applications?status=${AppStatus.DEV}`) .set(config.defaultHeaders()) .expect('Content-Type', /json/) .expect(200) @@ -104,4 +110,27 @@ describe("/applications", () => { expect(res.body.rev).toBeDefined() }) }) + + describe("edited at", () => { + it("middleware should set edited at", async () => { + const headers = config.defaultHeaders() + headers["referer"] = `/${config.getAppId()}/test` + const res = await request + .put(`/api/applications/${config.getAppId()}`) + .send({ + name: "UPDATED_NAME", + }) + .set(headers) + .expect('Content-Type', /json/) + .expect(200) + expect(res.body.rev).toBeDefined() + // retrieve the app to check it + const getRes = await request + .get(`/api/applications/${config.getAppId()}/appPackage`) + .set(headers) + .expect('Content-Type', /json/) + .expect(200) + expect(getRes.body.application.updatedAt).toBeDefined() + }) + }) }) diff --git a/packages/server/src/api/routes/tests/utilities/TestFunctions.js b/packages/server/src/api/routes/tests/utilities/TestFunctions.js index f10989bf56..c49e44c949 100644 --- a/packages/server/src/api/routes/tests/utilities/TestFunctions.js +++ b/packages/server/src/api/routes/tests/utilities/TestFunctions.js @@ -1,6 +1,7 @@ const rowController = require("../../../controllers/row") const appController = require("../../../controllers/application") const CouchDB = require("../../../../db") +const { AppStatus } = require("../../../../db/utils") function Request(appId, params) { this.appId = appId @@ -14,7 +15,7 @@ exports.getAllTableRows = async config => { } exports.clearAllApps = async () => { - const req = { query: { status: "dev" } } + const req = { query: { status: AppStatus.DEV } } await appController.fetch(req) const apps = req.body if (!apps || apps.length <= 0) { diff --git a/packages/server/src/integrations/tests/postgres.spec.js b/packages/server/src/integrations/tests/postgres.spec.js index 8a8876a556..dc34fcf6bb 100644 --- a/packages/server/src/integrations/tests/postgres.spec.js +++ b/packages/server/src/integrations/tests/postgres.spec.js @@ -20,7 +20,7 @@ describe("Postgres Integration", () => { const response = await config.integration.create({ sql }) - expect(config.integration.client.query).toHaveBeenCalledWith(sql) + expect(pg.queryMock).toHaveBeenCalledWith(sql) }) it("calls the read method with the correct params", async () => { @@ -28,7 +28,7 @@ describe("Postgres Integration", () => { const response = await config.integration.read({ sql }) - expect(config.integration.client.query).toHaveBeenCalledWith(sql) + expect(pg.queryMock).toHaveBeenCalledWith(sql) }) it("calls the update method with the correct params", async () => { @@ -36,20 +36,20 @@ describe("Postgres Integration", () => { const response = await config.integration.update({ sql }) - expect(config.integration.client.query).toHaveBeenCalledWith(sql) + expect(pg.queryMock).toHaveBeenCalledWith(sql) }) it("calls the delete method with the correct params", async () => { const sql = "delete from users where name = 'todelete';" - const response = await config.integration.delete({ + await config.integration.delete({ sql }) - expect(config.integration.client.query).toHaveBeenCalledWith(sql) + expect(pg.queryMock).toHaveBeenCalledWith(sql) }) describe("no rows returned", () => { beforeEach(() => { - config.integration.client.query.mockImplementation(() => ({ rows: [] })) + pg.queryMock.mockImplementation(() => ({ rows: [] })) }) it("returns the correct response when the create response has no rows", async () => { diff --git a/packages/server/src/middleware/authorized.js b/packages/server/src/middleware/authorized.js index b22fe245d5..8ed4c41db7 100644 --- a/packages/server/src/middleware/authorized.js +++ b/packages/server/src/middleware/authorized.js @@ -4,8 +4,7 @@ const { doesHaveResourcePermission, doesHaveBasePermission, } = require("@budibase/auth/permissions") -const { APP_DEV_PREFIX } = require("../db/utils") -const { doesUserHaveLock, updateLock } = require("../utilities/redis") +const builderMiddleware = require("./builder") function hasResource(ctx) { return ctx.resourceId != null @@ -15,26 +14,6 @@ const WEBHOOK_ENDPOINTS = new RegExp( ["webhooks/trigger", "webhooks/schema"].join("|") ) -async function checkDevAppLocks(ctx) { - const appId = ctx.appId - - // if any public usage, don't proceed - if (!ctx.user._id && !ctx.user.userId) { - return - } - - // not a development app, don't need to do anything - if (!appId || !appId.startsWith(APP_DEV_PREFIX)) { - return - } - if (!(await doesUserHaveLock(appId, ctx.user))) { - ctx.throw(403, "User does not hold app lock.") - } - - // they do have lock, update it - await updateLock(appId, ctx.user) -} - module.exports = (permType, permLevel = null) => async (ctx, next) => { // webhooks don't need authentication, each webhook unique if (WEBHOOK_ENDPOINTS.test(ctx.request.url)) { @@ -45,13 +24,9 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => { return ctx.throw(403, "No user info found") } - const builderCall = permType === PermissionTypes.BUILDER - const referer = ctx.headers["referer"] - const editingApp = referer ? referer.includes(ctx.appId) : false - // this makes sure that builder calls abide by dev locks - if (builderCall && editingApp) { - await checkDevAppLocks(ctx) - } + // check general builder stuff, this middleware is a good way + // to find API endpoints which are builder focused + await builderMiddleware(ctx, permType) const isAuthed = ctx.isAuthenticated const { basePermissions, permissions } = await getUserPermissions( @@ -62,9 +37,10 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => { // builders for now have permission to do anything // TODO: in future should consider separating permissions with an require("@budibase/auth").isClient check let isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global + const isBuilderApi = permType === PermissionTypes.BUILDER if (isBuilder) { return next() - } else if (builderCall && !isBuilder) { + } else if (isBuilderApi && !isBuilder) { return ctx.throw(403, "Not Authorized") } diff --git a/packages/server/src/middleware/builder.js b/packages/server/src/middleware/builder.js new file mode 100644 index 0000000000..240a2d1912 --- /dev/null +++ b/packages/server/src/middleware/builder.js @@ -0,0 +1,75 @@ +const { APP_DEV_PREFIX } = require("../db/utils") +const { + doesUserHaveLock, + updateLock, + checkDebounce, + setDebounce, +} = require("../utilities/redis") +const CouchDB = require("../db") +const { DocumentTypes } = require("../db/utils") +const { PermissionTypes } = require("@budibase/auth/permissions") + +const DEBOUNCE_TIME_SEC = 30 + +/************************************************** * + * This middleware has been broken out of the * + * "authorized" middleware as it had nothing to do * + * with authorization, but requires the perms * + * imparted by it. This middleware shouldn't * + * be called directly, it should always be called * + * through the authorized middleware * + ****************************************************/ + +async function checkDevAppLocks(ctx) { + const appId = ctx.appId + + // if any public usage, don't proceed + if (!ctx.user._id && !ctx.user.userId) { + return + } + + // not a development app, don't need to do anything + if (!appId || !appId.startsWith(APP_DEV_PREFIX)) { + return + } + if (!(await doesUserHaveLock(appId, ctx.user))) { + ctx.throw(403, "User does not hold app lock.") + } + + // they do have lock, update it + await updateLock(appId, ctx.user) +} + +async function updateAppUpdatedAt(ctx) { + const appId = ctx.appId + // if debouncing skip this update + // get methods also aren't updating + if (ctx.method === "GET" || (await checkDebounce(appId))) { + return + } + const db = new CouchDB(appId) + const metadata = await db.get(DocumentTypes.APP_METADATA) + metadata.updatedAt = new Date().toISOString() + await db.put(metadata) + // set a new debounce record with a short TTL + await setDebounce(appId, DEBOUNCE_TIME_SEC) +} + +module.exports = async (ctx, permType) => { + const appId = ctx.appId + // this only functions within an app context + if (!appId) { + return + } + const isBuilderApi = permType === PermissionTypes.BUILDER + const referer = ctx.headers["referer"] + const editingApp = referer ? referer.includes(appId) : false + // check this is a builder call and editing + if (!isBuilderApi || !editingApp) { + return + } + // check locks + await checkDevAppLocks(ctx) + // set updated at time on app + await updateAppUpdatedAt(ctx) +} diff --git a/packages/server/src/middleware/tests/authorized.spec.js b/packages/server/src/middleware/tests/authorized.spec.js index d51ce4cc4d..50e1b1dcf2 100644 --- a/packages/server/src/middleware/tests/authorized.spec.js +++ b/packages/server/src/middleware/tests/authorized.spec.js @@ -78,9 +78,6 @@ describe("Authorization middleware", () => { }) describe("external web hook call", () => { - let ctx = {} - let middleware - beforeEach(() => { config = new TestConfiguration() config.setEnvironment(true) diff --git a/packages/server/src/utilities/redis.js b/packages/server/src/utilities/redis.js index d1fd3de469..8e0f774f42 100644 --- a/packages/server/src/utilities/redis.js +++ b/packages/server/src/utilities/redis.js @@ -2,13 +2,13 @@ const { Client, utils } = require("@budibase/auth/redis") const { getGlobalIDFromUserMetadataID } = require("../db/utils") const APP_DEV_LOCK_SECONDS = 600 -const DB_NAME = utils.Databases.DEV_LOCKS -let devAppClient +let devAppClient, debounceClient // we init this as we want to keep the connection open all the time // reduces the performance hit exports.init = async () => { - devAppClient = await new Client(DB_NAME).init() + devAppClient = await new Client(utils.Databases.DEV_LOCKS).init() + debounceClient = await new Client(utils.Databases.DEBOUNCE).init() } exports.doesUserHaveLock = async (devAppId, user) => { @@ -52,3 +52,11 @@ exports.clearLock = async (devAppId, user) => { } await devAppClient.delete(devAppId) } + +exports.checkDebounce = async id => { + return debounceClient.get(id) +} + +exports.setDebounce = async (id, seconds) => { + await debounceClient.store(id, "debouncing", seconds) +} diff --git a/packages/worker/scripts/dev/manage.js b/packages/worker/scripts/dev/manage.js index f7216befb5..b9d28b6278 100644 --- a/packages/worker/scripts/dev/manage.js +++ b/packages/worker/scripts/dev/manage.js @@ -7,6 +7,7 @@ async function init() { const envFileJson = { SELF_HOSTED: 1, PORT: 4002, + CLUSTER_PORT: 10000, JWT_SECRET: "testsecret", INTERNAL_API_KEY: "budibase", MINIO_ACCESS_KEY: "budibase", diff --git a/packages/worker/src/api/controllers/admin/users.js b/packages/worker/src/api/controllers/admin/users.js index d7b198dfdb..e1aa8c1150 100644 --- a/packages/worker/src/api/controllers/admin/users.js +++ b/packages/worker/src/api/controllers/admin/users.js @@ -42,6 +42,10 @@ exports.save = async ctx => { _id: _id || generateGlobalUserID(), password: hashedPassword, } + // make sure the roles object is always present + if (!user.roles) { + user.roles = {} + } // add the active status to a user if its not provided if (user.status == null) { user.status = UserStatus.ACTIVE diff --git a/packages/worker/src/environment.js b/packages/worker/src/environment.js index 11eed33982..384230b9b3 100644 --- a/packages/worker/src/environment.js +++ b/packages/worker/src/environment.js @@ -19,6 +19,7 @@ if (!LOADED && isDev() && !isTest()) { module.exports = { SELF_HOSTED: process.env.SELF_HOSTED, PORT: process.env.PORT, + CLUSTER_PORT: process.env.CLUSTER_PORT, MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, MINIO_URL: process.env.MINIO_URL, diff --git a/packages/worker/src/utilities/templates.js b/packages/worker/src/utilities/templates.js index 86436a2f29..3ac897c10f 100644 --- a/packages/worker/src/utilities/templates.js +++ b/packages/worker/src/utilities/templates.js @@ -7,8 +7,9 @@ const { EmailTemplatePurpose, } = require("../constants") const { checkSlashesInUrl } = require("./index") +const env = require("../environment") -const LOCAL_URL = `http://localhost:10000` +const LOCAL_URL = `http://localhost:${env.CLUSTER_PORT || 10000}` const BASE_COMPANY = "Budibase" exports.getSettingsTemplateContext = async (purpose, code = null) => {