From 5ee6aa24b20ce958612038e5443450bd58ff44ba Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Jan 2022 11:32:14 +0000 Subject: [PATCH 1/3] Refactoring core library usage in monorepo, make it a bit cleaner/easier to search/more standardised. --- packages/backend-core/auth.js | 1 + packages/backend-core/objectStore.js | 4 + packages/backend-core/src/auth.js | 45 +++++++++++ packages/backend-core/src/index.js | 74 +++---------------- packages/backend-core/utils.js | 4 + packages/server/src/api/controllers/dev.js | 2 +- .../server/src/api/controllers/hosting.js | 2 +- packages/server/src/api/index.js | 2 +- packages/server/src/constants/index.js | 4 +- packages/server/src/middleware/currentapp.js | 10 ++- .../src/middleware/tests/currentapp.spec.js | 2 +- .../src/tests/utilities/TestConfiguration.js | 4 +- packages/server/src/threads/automation.js | 2 +- packages/server/src/utilities/budibaseDir.js | 2 +- .../src/utilities/fileSystem/utilities.js | 2 +- .../src/api/controllers/global/configs.js | 6 +- .../src/api/controllers/global/users.js | 8 +- packages/worker/src/api/index.js | 7 +- .../worker/src/api/routes/tests/auth.spec.js | 4 +- .../tests/utilities/TestConfiguration.js | 6 +- packages/worker/src/constants/index.js | 2 +- packages/worker/src/index.js | 2 +- .../worker/src/middleware/cloudRestricted.js | 2 +- packages/worker/src/utilities/redis.js | 2 +- 24 files changed, 106 insertions(+), 93 deletions(-) create mode 100644 packages/backend-core/auth.js create mode 100644 packages/backend-core/objectStore.js create mode 100644 packages/backend-core/src/auth.js create mode 100644 packages/backend-core/utils.js diff --git a/packages/backend-core/auth.js b/packages/backend-core/auth.js new file mode 100644 index 0000000000..bbfe3d41dd --- /dev/null +++ b/packages/backend-core/auth.js @@ -0,0 +1 @@ +module.exports = require("./src/auth") diff --git a/packages/backend-core/objectStore.js b/packages/backend-core/objectStore.js new file mode 100644 index 0000000000..3ee433f224 --- /dev/null +++ b/packages/backend-core/objectStore.js @@ -0,0 +1,4 @@ +module.exports = { + ...require("./src/objectStore"), + ...require("./src/objectStore/utils"), +} diff --git a/packages/backend-core/src/auth.js b/packages/backend-core/src/auth.js new file mode 100644 index 0000000000..7f66d887ae --- /dev/null +++ b/packages/backend-core/src/auth.js @@ -0,0 +1,45 @@ +const passport = require("koa-passport") +const LocalStrategy = require("passport-local").Strategy +const JwtStrategy = require("passport-jwt").Strategy +const { getGlobalDB } = require("./tenancy") +const { + jwt, + local, + authenticated, + google, + oidc, + auditLog, + tenancy, + appTenancy, + authError, +} = require("./middleware") + +// Strategies +passport.use(new LocalStrategy(local.options, local.authenticate)) +passport.use(new JwtStrategy(jwt.options, jwt.authenticate)) + +passport.serializeUser((user, done) => done(null, user)) + +passport.deserializeUser(async (user, done) => { + const db = getGlobalDB() + + try { + const user = await db.get(user._id) + return done(null, user) + } catch (err) { + console.error("User not found", err) + return done(null, false, { message: "User not found" }) + } +}) + +module.exports = { + buildAuthMiddleware: authenticated, + passport, + google, + oidc, + jwt: require("jsonwebtoken"), + buildTenancyMiddleware: tenancy, + buildAppTenancyMiddleware: appTenancy, + auditLog, + authError, +} diff --git a/packages/backend-core/src/index.js b/packages/backend-core/src/index.js index 4aa2c8ab96..cd3a3f5c97 100644 --- a/packages/backend-core/src/index.js +++ b/packages/backend-core/src/index.js @@ -1,71 +1,17 @@ -const passport = require("koa-passport") -const LocalStrategy = require("passport-local").Strategy -const JwtStrategy = require("passport-jwt").Strategy -const { StaticDatabases } = require("./db/utils") -const { getGlobalDB } = require("./tenancy") -const { - jwt, - local, - authenticated, - google, - oidc, - auditLog, - tenancy, - appTenancy, - authError, -} = require("./middleware") const { setDB } = require("./db") -const userCache = require("./cache/user") - -// Strategies -passport.use(new LocalStrategy(local.options, local.authenticate)) -passport.use(new JwtStrategy(jwt.options, jwt.authenticate)) - -passport.serializeUser((user, done) => done(null, user)) - -passport.deserializeUser(async (user, done) => { - const db = getGlobalDB() - - try { - const user = await db.get(user._id) - return done(null, user) - } catch (err) { - console.error("User not found", err) - return done(null, false, { message: "User not found" }) - } -}) module.exports = { init(pouch) { setDB(pouch) }, - db: require("./db/utils"), - redis: { - Client: require("./redis"), - utils: require("./redis/utils"), - }, - objectStore: { - ...require("./objectStore"), - ...require("./objectStore/utils"), - }, - utils: { - ...require("./utils"), - ...require("./hashing"), - }, - auth: { - buildAuthMiddleware: authenticated, - passport, - google, - oidc, - jwt: require("jsonwebtoken"), - buildTenancyMiddleware: tenancy, - buildAppTenancyMiddleware: appTenancy, - auditLog, - authError, - }, - cache: { - user: userCache, - }, - StaticDatabases, - constants: require("./constants"), + // some default exports from the library, however these ideally shouldn't + // be used, instead the syntax require("@budibase/backend-core/db") should be used + StaticDatabases: require("./db/utils").StaticDatabases, + db: require("../db"), + redis: require("../redis"), + objectStore: require("../objectStore"), + utils: require("../utils"), + cache: require("../cache"), + auth: require("../auth"), + constants: require("../constants"), } diff --git a/packages/backend-core/utils.js b/packages/backend-core/utils.js new file mode 100644 index 0000000000..2ef920e103 --- /dev/null +++ b/packages/backend-core/utils.js @@ -0,0 +1,4 @@ +module.exports = { + ...require("./src/utils"), + ...require("./src/hashing"), +} diff --git a/packages/server/src/api/controllers/dev.js b/packages/server/src/api/controllers/dev.js index c7dbbc1e8f..3126454a6b 100644 --- a/packages/server/src/api/controllers/dev.js +++ b/packages/server/src/api/controllers/dev.js @@ -4,7 +4,7 @@ const env = require("../../environment") const { checkSlashesInUrl } = require("../../utilities") const { request } = require("../../utilities/workerRequests") const { clearLock } = require("../../utilities/redis") -const { Replication } = require("@budibase/backend-core").db +const { Replication } = require("@budibase/backend-core/db") const { DocumentTypes } = require("../../db/utils") const { app: appCache } = require("@budibase/backend-core/cache") diff --git a/packages/server/src/api/controllers/hosting.js b/packages/server/src/api/controllers/hosting.js index bff37193bd..0360643942 100644 --- a/packages/server/src/api/controllers/hosting.js +++ b/packages/server/src/api/controllers/hosting.js @@ -1,7 +1,7 @@ const CouchDB = require("../../db") const { getDeployedApps } = require("../../utilities/workerRequests") const { getScopedConfig } = require("@budibase/backend-core/db") -const { Configs } = require("@budibase/backend-core").constants +const { Configs } = require("@budibase/backend-core/constants") const { checkSlashesInUrl } = require("../../utilities") exports.fetchUrls = async ctx => { diff --git a/packages/server/src/api/index.js b/packages/server/src/api/index.js index 1c8684ad87..4a94063e31 100644 --- a/packages/server/src/api/index.js +++ b/packages/server/src/api/index.js @@ -4,7 +4,7 @@ const { auditLog, buildTenancyMiddleware, buildAppTenancyMiddleware, -} = require("@budibase/backend-core").auth +} = require("@budibase/backend-core/auth") const currentApp = require("../middleware/currentapp") const compress = require("koa-compress") const zlib = require("zlib") diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index 5d04ab870a..b63b71d2c2 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -1,6 +1,6 @@ const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles") -const { UserStatus } = require("@budibase/backend-core").constants -const { ObjectStoreBuckets } = require("@budibase/backend-core").objectStore +const { UserStatus } = require("@budibase/backend-core/constants") +const { ObjectStoreBuckets } = require("@budibase/backend-core/objectStore") exports.JobQueues = { AUTOMATIONS: "automationQueue", diff --git a/packages/server/src/middleware/currentapp.js b/packages/server/src/middleware/currentapp.js index 5eeaac8a33..69f80c895b 100644 --- a/packages/server/src/middleware/currentapp.js +++ b/packages/server/src/middleware/currentapp.js @@ -1,6 +1,10 @@ -const { getAppId, setCookie, getCookie, clearCookie } = - require("@budibase/backend-core").utils -const { Cookies } = require("@budibase/backend-core").constants +const { + getAppId, + setCookie, + getCookie, + clearCookie, +} = require("@budibase/backend-core/utils") +const { Cookies } = require("@budibase/backend-core/constants") const { getRole } = require("@budibase/backend-core/roles") const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles") const { generateUserMetadataID, isDevAppID } = require("../db/utils") diff --git a/packages/server/src/middleware/tests/currentapp.spec.js b/packages/server/src/middleware/tests/currentapp.spec.js index a76f70e754..893a508620 100644 --- a/packages/server/src/middleware/tests/currentapp.spec.js +++ b/packages/server/src/middleware/tests/currentapp.spec.js @@ -121,7 +121,7 @@ describe("Current app middleware", () => { async function checkExpected(setCookie) { config.setUser() await config.executeMiddleware() - const cookieFn = require("@budibase/backend-core").utils.setCookie + let { setCookie: cookieFn } = require("@budibase/backend-core/utils") if (setCookie) { expect(cookieFn).toHaveBeenCalled() } else { diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js index eb1280306d..06d81593a1 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.js +++ b/packages/server/src/tests/utilities/TestConfiguration.js @@ -15,8 +15,8 @@ const { const controllers = require("./controllers") const supertest = require("supertest") const { cleanup } = require("../../utilities/fileSystem") -const { Cookies, Headers } = require("@budibase/backend-core").constants -const { jwt } = require("@budibase/backend-core").auth +const { Cookies, Headers } = require("@budibase/backend-core/constants") +const { jwt } = require("@budibase/backend-core/auth") const core = require("@budibase/backend-core") const { getGlobalDB } = require("@budibase/backend-core/tenancy") const { createASession } = require("@budibase/backend-core/sessions") diff --git a/packages/server/src/threads/automation.js b/packages/server/src/threads/automation.js index 65a8a82492..2a39773520 100644 --- a/packages/server/src/threads/automation.js +++ b/packages/server/src/threads/automation.js @@ -4,7 +4,7 @@ const actions = require("../automations/actions") const automationUtils = require("../automations/automationUtils") const AutomationEmitter = require("../events/AutomationEmitter") const { processObject } = require("@budibase/string-templates") -const { DEFAULT_TENANT_ID } = require("@budibase/backend-core").constants +const { DEFAULT_TENANT_ID } = require("@budibase/backend-core/constants") const CouchDB = require("../db") const { DocumentTypes, isDevAppID } = require("../db/utils") const { doInTenant } = require("@budibase/backend-core/tenancy") diff --git a/packages/server/src/utilities/budibaseDir.js b/packages/server/src/utilities/budibaseDir.js index 37d83d48ce..48ae7eb0e7 100644 --- a/packages/server/src/utilities/budibaseDir.js +++ b/packages/server/src/utilities/budibaseDir.js @@ -1,7 +1,7 @@ const { join } = require("./centralPath") const { homedir } = require("os") const env = require("../environment") -const { budibaseTempDir } = require("@budibase/backend-core").objectStore +const { budibaseTempDir } = require("@budibase/backend-core/objectStore") module.exports.budibaseAppsDir = function () { return env.BUDIBASE_DIR || join(homedir(), ".budibase") diff --git a/packages/server/src/utilities/fileSystem/utilities.js b/packages/server/src/utilities/fileSystem/utilities.js index 84a701ee95..e85f4f9c79 100644 --- a/packages/server/src/utilities/fileSystem/utilities.js +++ b/packages/server/src/utilities/fileSystem/utilities.js @@ -9,7 +9,7 @@ const { deleteFolder, uploadDirectory, downloadTarball, -} = require("@budibase/backend-core").objectStore +} = require("@budibase/backend-core/objectStore") /*********************************** * NOTE * diff --git a/packages/worker/src/api/controllers/global/configs.js b/packages/worker/src/api/controllers/global/configs.js index fff4e277f7..fc0aa868a3 100644 --- a/packages/worker/src/api/controllers/global/configs.js +++ b/packages/worker/src/api/controllers/global/configs.js @@ -7,8 +7,10 @@ const { } = require("@budibase/backend-core/db") const { Configs } = require("../../../constants") const email = require("../../../utilities/email") -const { upload, ObjectStoreBuckets } = - require("@budibase/backend-core").objectStore +const { + upload, + ObjectStoreBuckets, +} = require("@budibase/backend-core/objectStore") const CouchDB = require("../../../db") const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") const env = require("../../../environment") diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index 3016255d70..152bc9f5ae 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -3,8 +3,12 @@ const { StaticDatabases, generateNewUsageQuotaDoc, } = require("@budibase/backend-core/db") -const { hash, getGlobalUserByEmail, saveUser, platformLogout } = - require("@budibase/backend-core").utils +const { + hash, + getGlobalUserByEmail, + saveUser, + platformLogout, +} = require("@budibase/backend-core/utils") const { EmailTemplatePurpose } = require("../../../constants") const { checkInviteCode } = require("../../../utilities/redis") const { sendEmail } = require("../../../utilities/email") diff --git a/packages/worker/src/api/index.js b/packages/worker/src/api/index.js index dd9d1c3a53..a83b39e6cf 100644 --- a/packages/worker/src/api/index.js +++ b/packages/worker/src/api/index.js @@ -2,8 +2,11 @@ const Router = require("@koa/router") const compress = require("koa-compress") const zlib = require("zlib") const { routes } = require("./routes") -const { buildAuthMiddleware, auditLog, buildTenancyMiddleware } = - require("@budibase/backend-core").auth +const { + buildAuthMiddleware, + auditLog, + buildTenancyMiddleware, +} = require("@budibase/backend-core/auth") const PUBLIC_ENDPOINTS = [ // old deprecated endpoints kept for backwards compat diff --git a/packages/worker/src/api/routes/tests/auth.spec.js b/packages/worker/src/api/routes/tests/auth.spec.js index f9dafc45da..e26af12e01 100644 --- a/packages/worker/src/api/routes/tests/auth.spec.js +++ b/packages/worker/src/api/routes/tests/auth.spec.js @@ -54,7 +54,7 @@ describe("/api/global/auth", () => { }) describe("oidc", () => { - const auth = require("@budibase/backend-core").auth + const auth = require("@budibase/backend-core/auth") // mock the oidc strategy implementation and return value strategyFactory = jest.fn() @@ -104,4 +104,4 @@ describe("/api/global/auth", () => { }) }) -}) \ No newline at end of file +}) diff --git a/packages/worker/src/api/routes/tests/utilities/TestConfiguration.js b/packages/worker/src/api/routes/tests/utilities/TestConfiguration.js index a8824b3db0..34ce01263d 100644 --- a/packages/worker/src/api/routes/tests/utilities/TestConfiguration.js +++ b/packages/worker/src/api/routes/tests/utilities/TestConfiguration.js @@ -1,10 +1,10 @@ const env = require("../../../../environment") const controllers = require("./controllers") const supertest = require("supertest") -const { jwt } = require("@budibase/backend-core").auth -const { Cookies } = require("@budibase/backend-core").constants +const { jwt } = require("@budibase/backend-core/auth") +const { Cookies } = require("@budibase/backend-core/constants") const { Configs, LOGO_URL } = require("../../../../constants") -const { getGlobalUserByEmail } = require("@budibase/backend-core").utils +const { getGlobalUserByEmail } = require("@budibase/backend-core/utils") const { createASession } = require("@budibase/backend-core/sessions") const { newid } = require("@budibase/backend-core/src/hashing") const { TENANT_ID } = require("./structures") diff --git a/packages/worker/src/constants/index.js b/packages/worker/src/constants/index.js index 02393bd17c..5affaaccb3 100644 --- a/packages/worker/src/constants/index.js +++ b/packages/worker/src/constants/index.js @@ -1,4 +1,4 @@ -const { Configs } = require("@budibase/backend-core").constants +const { Configs } = require("@budibase/backend-core/constants") exports.LOGO_URL = "https://d33wubrfki0l68.cloudfront.net/aac32159d7207b5085e74a7ef67afbb7027786c5/2b1fd/img/logo/bb-emblem.svg" diff --git a/packages/worker/src/index.js b/packages/worker/src/index.js index fc0ab04873..7568784f09 100644 --- a/packages/worker/src/index.js +++ b/packages/worker/src/index.js @@ -6,7 +6,7 @@ const Koa = require("koa") const destroyable = require("server-destroy") const koaBody = require("koa-body") const koaSession = require("koa-session") -const { passport } = require("@budibase/backend-core").auth +const { passport } = require("@budibase/backend-core/auth") const logger = require("koa-pino-logger") const http = require("http") const api = require("./api") diff --git a/packages/worker/src/middleware/cloudRestricted.js b/packages/worker/src/middleware/cloudRestricted.js index 817e193f8d..b42ce4f3a4 100644 --- a/packages/worker/src/middleware/cloudRestricted.js +++ b/packages/worker/src/middleware/cloudRestricted.js @@ -1,5 +1,5 @@ const env = require("../environment") -const { Headers } = require("@budibase/backend-core").constants +const { Headers } = require("@budibase/backend-core/constants") /** * This is a restricted endpoint in the cloud. diff --git a/packages/worker/src/utilities/redis.js b/packages/worker/src/utilities/redis.js index 2954fcba0a..cfed97cd18 100644 --- a/packages/worker/src/utilities/redis.js +++ b/packages/worker/src/utilities/redis.js @@ -1,5 +1,5 @@ const { Client, utils } = require("@budibase/backend-core/redis") -const { newid } = require("@budibase/backend-core").utils +const { newid } = require("@budibase/backend-core/utils") function getExpirySecondsForDB(db) { switch (db) { From 398e87e72a736f38316503549513cd0497ecae6b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Jan 2022 11:50:14 +0000 Subject: [PATCH 2/3] Fixing test mocking. --- .../src/middleware/tests/currentapp.spec.js | 72 +++++++++---------- 1 file changed, 33 insertions(+), 39 deletions(-) diff --git a/packages/server/src/middleware/tests/currentapp.spec.js b/packages/server/src/middleware/tests/currentapp.spec.js index 893a508620..27c88f3b48 100644 --- a/packages/server/src/middleware/tests/currentapp.spec.js +++ b/packages/server/src/middleware/tests/currentapp.spec.js @@ -32,34 +32,30 @@ function mockAuthWithNoCookie() { }, }, })) - jest.mock("@budibase/backend-core", () => ({ - utils: { - getAppId: jest.fn(), - setCookie: jest.fn(), - getCookie: jest.fn(), - }, - constants: { - Cookies: {}, - }, + jest.mock("@budibase/backend-core/utils", () => ({ + getAppId: jest.fn(), + setCookie: jest.fn(), + getCookie: jest.fn(), + })) + jest.mock("@budibase/backend-core/constants", () => ({ + Cookies: {}, })) } function mockAuthWithCookie() { jest.resetModules() mockWorker() - jest.mock("@budibase/backend-core", () => ({ - utils: { - getAppId: () => { - return "app_test" - }, - setCookie: jest.fn(), - getCookie: () => ({appId: "app_different", roleId: "PUBLIC"}), + jest.mock("@budibase/backend-core/utils", () => ({ + getAppId: () => { + return "app_test" }, - constants: { - Cookies: { - Auth: "auth", - CurrentApp: "currentapp", - }, + setCookie: jest.fn(), + getCookie: () => ({appId: "app_different", roleId: "PUBLIC"}), + })) + jest.mock("@budibase/backend-core/constants", () => ({ + Cookies: { + Auth: "auth", + CurrentApp: "currentapp", }, })) } @@ -140,32 +136,30 @@ describe("Current app middleware", () => { it("should perform correct when no cookie exists", async () => { mockReset() - jest.mock("@budibase/backend-core", () => ({ - utils: { - getAppId: () => { - return "app_test" - }, - setCookie: jest.fn(), - getCookie: jest.fn(), - }, - constants: { - Cookies: {}, + jest.mock("@budibase/backend-core/utils", () => ({ + getAppId: () => { + return "app_test" }, + setCookie: jest.fn(), + getCookie: jest.fn(), + })) + jest.mock("@budibase/backend-core/constants", () => ({ + Cookies: {}, })) await checkExpected(true) }) it("lastly check what occurs when cookie doesn't need updated", async () => { mockReset() - jest.mock("@budibase/backend-core", () => ({ - utils: { - getAppId: () => { - return "app_test" - }, - setCookie: jest.fn(), - getCookie: () => ({appId: "app_test", roleId: "PUBLIC"}), + jest.mock("@budibase/backend-core/utils", () => ({ + getAppId: () => { + return "app_test" }, - constants: { Cookies: {} }, + setCookie: jest.fn(), + getCookie: () => ({appId: "app_test", roleId: "PUBLIC"}), + })) + jest.mock("@budibase/backend-core/constants", () => ({ + Cookies: {}, })) await checkExpected(false) }) From 97c6f3298a9c3ed17123e9c5a44333fce9baec39 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Jan 2022 12:54:25 +0000 Subject: [PATCH 3/3] Removing use of direct package access. --- packages/backend-core/middleware.js | 1 + packages/server/src/utilities/index.js | 2 +- packages/worker/src/api/controllers/global/auth.js | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 packages/backend-core/middleware.js diff --git a/packages/backend-core/middleware.js b/packages/backend-core/middleware.js new file mode 100644 index 0000000000..30fec96239 --- /dev/null +++ b/packages/backend-core/middleware.js @@ -0,0 +1 @@ +module.exports = require("./src/middleware") diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index 8b51640c97..77d3d46282 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -1,6 +1,6 @@ const env = require("../environment") const { OBJ_STORE_DIRECTORY } = require("../constants") -const { sanitizeKey } = require("@budibase/backend-core/src/objectStore") +const { sanitizeKey } = require("@budibase/backend-core/objectStore") const CouchDB = require("../db") const { generateMetadataID } = require("../db/utils") const Readable = require("stream").Readable diff --git a/packages/worker/src/api/controllers/global/auth.js b/packages/worker/src/api/controllers/global/auth.js index ef7f6fdfdc..2ba12194ca 100644 --- a/packages/worker/src/api/controllers/global/auth.js +++ b/packages/worker/src/api/controllers/global/auth.js @@ -1,7 +1,7 @@ const core = require("@budibase/backend-core") const { getScopedConfig } = require("@budibase/backend-core/db") -const { google } = require("@budibase/backend-core/src/middleware") -const { oidc } = require("@budibase/backend-core/src/middleware") +const { google } = require("@budibase/backend-core/middleware") +const { oidc } = require("@budibase/backend-core/middleware") const { Configs, EmailTemplatePurpose } = require("../../../constants") const { sendEmail, isEmailConfigured } = require("../../../utilities/email") const {