General work after running some tests in CI - general refactoring as well.

packages/backend-core/src/auth/auth.ts

@@ -1,16 +1,14 @@
-const passport = require("koa-passport")
+const _passport = require("koa-passport")
 const LocalStrategy = require("passport-local").Strategy
 const JwtStrategy = require("passport-jwt").Strategy
-import { getGlobalDB } from "./tenancy"
+import { getGlobalDB } from "../tenancy"
 const refresh = require("passport-oauth2-refresh")
-import { Config } from "./constants"
+import { Config } from "../constants"
-import { getScopedConfig } from "./db"
+import { getScopedConfig } from "../db"
 import {
-  jwt,
+  jwt as jwtPassport,
   local,
   authenticated,
-  google,
-  oidc,
   auditLog,
   tenancy,
   authError,
@@ -21,22 +19,41 @@ import {
   builderOnly,
   builderOrAdmin,
   joiValidator,
-} from "./middleware"
+  oidc,
-import { invalidateUser } from "./cache/user"
+  google,
+} from "../middleware"
+import { invalidateUser } from "../cache/user"
 import { User } from "@budibase/types"
-import { logAlert } from "./logging"
+import { logAlert } from "../logging"
+export {
+  auditLog,
+  authError,
+  internalApi,
+  ssoCallbackUrl,
+  adminOnly,
+  builderOnly,
+  builderOrAdmin,
+  joiValidator,
+  google,
+  oidc,
+} from "../middleware"
+export const buildAuthMiddleware = authenticated
+export const buildTenancyMiddleware = tenancy
+export const buildCsrfMiddleware = csrf
+export const passport = _passport
+export const jwt = require("jsonwebtoken")
 
 // Strategies
-passport.use(new LocalStrategy(local.options, local.authenticate))
+_passport.use(new LocalStrategy(local.options, local.authenticate))
-if (jwt.options.secretOrKey) {
+if (jwtPassport.options.secretOrKey) {
-  passport.use(new JwtStrategy(jwt.options, jwt.authenticate))
+  _passport.use(new JwtStrategy(jwtPassport.options, jwtPassport.authenticate))
 } else {
   logAlert("No JWT Secret supplied, cannot configure JWT strategy")
 }
 
-passport.serializeUser((user: User, done: any) => done(null, user))
+_passport.serializeUser((user: User, done: any) => done(null, user))
 
-passport.deserializeUser(async (user: User, done: any) => {
+_passport.deserializeUser(async (user: User, done: any) => {
   const db = getGlobalDB()
 
   try {
@@ -115,7 +132,7 @@ async function refreshGoogleAccessToken(
   })
 }
 
-async function refreshOAuthToken(
+export async function refreshOAuthToken(
   refreshToken: string,
   configType: string,
   configId: string
@@ -152,7 +169,7 @@ async function refreshOAuthToken(
   return refreshResponse
 }
 
-async function updateUserOAuth(userId: string, oAuthConfig: any) {
+export async function updateUserOAuth(userId: string, oAuthConfig: any) {
   const details = {
     accessToken: oAuthConfig.accessToken,
     refreshToken: oAuthConfig.refreshToken,
@@ -179,23 +196,3 @@ async function updateUserOAuth(userId: string, oAuthConfig: any) {
     console.error("Could not update OAuth details for current user", e)
   }
 }
-
-export = {
-  buildAuthMiddleware: authenticated,
-  passport,
-  google,
-  oidc,
-  jwt: require("jsonwebtoken"),
-  buildTenancyMiddleware: tenancy,
-  auditLog,
-  authError,
-  buildCsrfMiddleware: csrf,
-  internalApi,
-  refreshOAuthToken,
-  updateUserOAuth,
-  ssoCallbackUrl,
-  adminOnly,
-  builderOnly,
-  builderOrAdmin,
-  joiValidator,
-}

packages/backend-core/src/auth/index.ts

@@ -0,0 +1 @@
+export * from "./auth"

packages/backend-core/src/constants/index.ts

@@ -0,0 +1,2 @@
+export * from "./db"
+export * from "./misc"

packages/backend-core/src/context/Context.ts

@@ -1,18 +1,17 @@
 import { AsyncLocalStorage } from "async_hooks"
-import { ContextMap } from "./constants"
 
 export default class Context {
-  static storage = new AsyncLocalStorage<ContextMap>()
+  static storage = new AsyncLocalStorage<Record<string, any>>()
 
-  static run(context: ContextMap, func: any) {
+  static run(context: Record<string, any>, func: any) {
     return Context.storage.run(context, () => func())
   }
 
-  static get(): ContextMap {
+  static get(): Record<string, any> {
-    return Context.storage.getStore() as ContextMap
+    return Context.storage.getStore() as Record<string, any>
   }
 
-  static set(context: ContextMap) {
+  static set(context: Record<string, any>) {
     Context.storage.enterWith(context)
   }
 }

packages/backend-core/src/context/constants.ts

@@ -1,7 +0,0 @@
-import { IdentityContext } from "@budibase/types"
-
-export type ContextMap = {
-  tenantId?: string
-  appId?: string
-  identity?: IdentityContext
-}

packages/backend-core/src/context/mainContext.ts

@@ -2,20 +2,43 @@
 // store an app ID to pretend there is a context
 import env from "../environment"
 import Context from "./Context"
+import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
+import { getDB } from "../db/db"
 import {
-  baseGlobalDBName,
   DocumentType,
-  getDB,
-  getDevelopmentAppID,
-  getProdAppID,
   SEPARATOR,
-} from "../db"
+  StaticDatabases,
-import { ContextMap } from "./constants"
+  DEFAULT_TENANT_ID,
+} from "../constants"
 import { Database, IdentityContext } from "@budibase/types"
-import { DEFAULT_TENANT_ID } from "./index"
+
+export type ContextMap = {
+  tenantId?: string
+  appId?: string
+  identity?: IdentityContext
+}
 
 let TEST_APP_ID: string | null = null
 
+export function getGlobalDBName(tenantId?: string) {
+  // tenant ID can be set externally, for example user API where
+  // new tenants are being created, this may be the case
+  if (!tenantId) {
+    tenantId = getTenantId()
+  }
+  return baseGlobalDBName(tenantId)
+}
+
+export function baseGlobalDBName(tenantId: string | undefined | null) {
+  let dbName
+  if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
+    dbName = StaticDatabases.GLOBAL.name
+  } else {
+    dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
+  }
+  return dbName
+}
+
 export function isMultiTenant() {
   return env.MULTI_TENANCY
 }

packages/backend-core/src/db/Replication.ts

@@ -1,5 +1,5 @@
-import { getPouchDB, closePouchDB } from "./couch/pouchDB"
+import { getPouchDB, closePouchDB } from "./couch"
-import { DocumentType } from "./constants"
+import { DocumentType } from "../constants"
 
 class Replication {
   source: any

packages/backend-core/src/db/conversions.ts

@@ -1,4 +1,4 @@
-import { APP_DEV_PREFIX, APP_PREFIX } from "./constants"
+import { APP_DEV_PREFIX, APP_PREFIX } from "../constants"
 import { App } from "@budibase/types"
 const NO_APP_ERROR = "No app provided"
 

packages/backend-core/src/db/index.ts

@@ -2,6 +2,8 @@ export * from "./couch"
 export * from "./db"
 export * from "./utils"
 export * from "./views"
-export * from "./constants"
 export * from "./conversions"
-export * from "./tenancy"
+export { default as Replication } from "./Replication"
+// exports to support old export structure
+export * from "../constants/db"
+export { getGlobalDBName, baseGlobalDBName } from "../context"

packages/backend-core/src/db/tenancy.ts

@@ -1,22 +0,0 @@
-import { DEFAULT_TENANT_ID } from "../constants"
-import { StaticDatabases, SEPARATOR } from "./constants"
-import { getTenantId } from "../context"
-
-export const getGlobalDBName = (tenantId?: string) => {
-  // tenant ID can be set externally, for example user API where
-  // new tenants are being created, this may be the case
-  if (!tenantId) {
-    tenantId = getTenantId()
-  }
-  return baseGlobalDBName(tenantId)
-}
-
-export const baseGlobalDBName = (tenantId: string | undefined | null) => {
-  let dbName
-  if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
-    dbName = StaticDatabases.GLOBAL.name
-  } else {
-    dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
-  }
-  return dbName
-}

packages/backend-core/src/db/tests/utils.spec.js

@@ -1,10 +1,12 @@
 require("../../../tests")
 const {
-  generateAppID,
   getDevelopmentAppID,
   getProdAppID,
   isDevAppID,
   isProdAppID,
+} = require("../conversions")
+const {
+  generateAppID,
   getPlatformUrl,
   getScopedConfig
 } = require("../utils")

packages/backend-core/src/db/utils.ts

@@ -1,26 +1,20 @@
-import { newid } from "../utils/hashing"
+import { newid } from "../newid"
-import { DEFAULT_TENANT_ID, Config } from "../constants"
 import env from "../environment"
 import {
+  DEFAULT_TENANT_ID,
   SEPARATOR,
   DocumentType,
   UNICODE_MAX,
   ViewName,
   InternalTable,
-} from "./constants"
+  APP_PREFIX,
-import { getTenantId, getGlobalDB } from "../context"
+} from "../constants"
-import { getGlobalDBName } from "./tenancy"
+import { getTenantId, getGlobalDB, getGlobalDBName } from "../context"
 import { doWithDB, allDbs, directCouchAllDbs } from "./db"
 import { getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "./conversions"
-import { APP_PREFIX } from "./constants"
 import * as events from "../events"
-import { App, Database } from "@budibase/types"
+import { App, Database, ConfigType } from "@budibase/types"
-
-export * from "./constants"
-export * from "./conversions"
-export { default as Replication } from "./Replication"
-export * from "./tenancy"
 
 /**
  * Generates a new app ID.
@@ -494,7 +488,7 @@ export const getScopedFullConfig = async function (
   )[0]
 
   // custom logic for settings doc
-  if (type === Config.SETTINGS) {
+  if (type === ConfigType.SETTINGS) {
     if (scopedConfig && scopedConfig.doc) {
       // overrides affected by environment variables
       scopedConfig.doc.config.platformUrl = await getPlatformUrl({
@@ -533,7 +527,7 @@ export const getPlatformUrl = async (opts = { tenantAware: true }) => {
     // get the doc directly instead of with getScopedConfig to prevent loop
     let settings
     try {
-      settings = await db.get(generateConfigID({ type: Config.SETTINGS }))
+      settings = await db.get(generateConfigID({ type: ConfigType.SETTINGS }))
     } catch (e: any) {
       if (e.status !== 404) {
         throw e

packages/backend-core/src/db/views.ts

@@ -1,6 +1,11 @@
-import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils"
+import {
+  DocumentType,
+  ViewName,
+  DeprecatedViews,
+  SEPARATOR,
+  StaticDatabases,
+} from "../constants"
 import { getGlobalDB } from "../context"
-import { StaticDatabases } from "./constants"
 import { doWithDB } from "./"
 import { Database, DatabaseQueryOpts } from "@budibase/types"
 

packages/backend-core/src/environment.ts

@@ -75,7 +75,8 @@ const environment = {
     process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
   _set(key: any, value: any) {
     process.env[key] = value
-    module.exports[key] = value
+    // @ts-ignore
+    environment[key] = value
   },
 }
 

packages/backend-core/src/events/analytics.ts

@@ -1,8 +1,8 @@
 import env from "../environment"
-import tenancy from "../tenancy"
+import * as tenancy from "../tenancy"
 import * as dbUtils from "../db/utils"
 import { Config } from "../constants"
-import { withCache, TTL, CacheKey } from "../cache/generic"
+import { withCache, TTL, CacheKey } from "../cache"
 
 export const enabled = async () => {
   // cloud - always use the environment variable

packages/backend-core/src/featureFlags/index.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-import tenancy from "../tenancy"
+import * as tenancy from "../tenancy"
 
 /**
  * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.

packages/backend-core/src/index.ts

@@ -8,17 +8,16 @@ import * as permissions from "./security/permissions"
 import * as accounts from "./cloud/accounts"
 import * as installation from "./installation"
 import env from "./environment"
-import tenancy from "./tenancy"
+import * as tenancy from "./tenancy"
 import * as featureFlags from "./featureFlags"
 import * as sessions from "./security/sessions"
 import * as deprovisioning from "./context/deprovision"
-import auth from "./auth"
+import * as auth from "./auth"
 import * as constants from "./constants"
-import * as dbConstants from "./db/constants"
 import * as logging from "./logging"
 import * as pino from "./pino"
 import * as middleware from "./middleware"
-import plugins from "./plugin"
+import * as plugins from "./plugin"
 import * as encryption from "./security/encryption"
 import * as queue from "./queue"
 import * as db from "./db"
@@ -35,7 +34,7 @@ const init = (opts: any = {}) => {
 const core = {
   init,
   db,
-  ...dbConstants,
+  ...constants,
   redis,
   locks: redis.redlock,
   objectStore,
@@ -44,7 +43,6 @@ const core = {
   cache,
   auth,
   constants,
-  ...constants,
   migrations,
   env,
   accounts,

packages/backend-core/src/migrations/tests/index.spec.js

@@ -3,7 +3,7 @@ const { runMigrations, getMigrationsDoc } = require("../index")
 const { getDB } = require("../../db")
 const {
   StaticDatabases,
-} = require("../../db/utils")
+} = require("../../constants")
 
 let db
 

packages/backend-core/src/plugin/index.ts

@@ -1,7 +1 @@
-import * as utils from "./utils"
+export * from "./utils"
-
-const pkg = {
-  ...utils,
-}
-
-export = pkg

packages/backend-core/src/tenancy/index.ts

@@ -1,9 +1,2 @@
-import * as context from "../context"
+export * from "../context"
-import * as tenancy from "./tenancy"
+export * from "./tenancy"
-
-const pkg = {
-  ...context,
-  ...tenancy,
-}
-
-export = pkg

packages/backend-core/src/tenancy/tenancy.ts

@@ -1,10 +1,4 @@
-import {
+import { doWithDB, queryPlatformView, getGlobalDBName } from "../db"
-  doWithDB,
-  queryPlatformView,
-  StaticDatabases,
-  getGlobalDBName,
-  ViewName,
-} from "../db"
 import {
   DEFAULT_TENANT_ID,
   getTenantId,
@@ -18,7 +12,7 @@ import {
   TenantResolutionStrategy,
   GetTenantIdOptions,
 } from "@budibase/types"
-import { Header } from "../constants"
+import { Header, StaticDatabases, ViewName } from "../constants"
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
 const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name

packages/backend-core/src/tests/utils.spec.js

@@ -1,7 +1,8 @@
 const { structures } = require("../../tests")
 const utils = require("../utils")
 const events = require("../events")
-const { doInTenant, DEFAULT_TENANT_ID }= require("../context")
+const { DEFAULT_TENANT_ID } = require("../constants")
+const { doInTenant } = require("../context")
 
 describe("utils", () => {
   describe("platformLogout", () => {

packages/backend-core/src/utils/utils.ts

@@ -1,17 +1,11 @@
-import {
+import { getAllApps, queryGlobalView } from "../db"
-  DocumentType,
-  SEPARATOR,
-  ViewName,
-  getAllApps,
-  queryGlobalView,
-} from "../db"
 import { options } from "../middleware/passport/jwt"
 import { Header, Cookie, MAX_VALID_DATE } from "../constants"
 import env from "../environment"
 import * as userCache from "../cache/user"
 import { getSessionsForUser, invalidateSessions } from "../security/sessions"
 import * as events from "../events"
-import tenancy from "../tenancy"
+import * as tenancy from "../tenancy"
 import {
   App,
   BBContext,
@@ -19,6 +13,7 @@ import {
   TenantResolutionStrategy,
 } from "@budibase/types"
 import { SetOption } from "cookies"
+import { DocumentType, SEPARATOR, ViewName } from "../constants"
 const jwt = require("jsonwebtoken")
 
 const APP_PREFIX = DocumentType.APP + SEPARATOR

packages/server/src/app.ts

@@ -12,18 +12,18 @@ if (process.env.ELASTIC_APM_ENABLED) {
 import { ExtendableContext } from "koa"
 import * as db from "./db"
 db.init()
-const Koa = require("koa")
+import Koa from "koa"
-const destroyable = require("server-destroy")
+import koaBody from "koa-body"
-const koaBody = require("koa-body")
+import http from "http"
-const http = require("http")
+import * as api from "./api"
-const api = require("./api")
+import * as automations from "./automations"
-const automations = require("./automations/index")
+import { Thread } from "./threads"
-const Sentry = require("@sentry/node")
-const { Thread } = require("./threads")
 import * as redis from "./utilities/redis"
 import { events, logging } from "@budibase/backend-core"
 import { initialise as initialiseWebsockets } from "./websocket"
 import { startup } from "./startup"
+const Sentry = require("@sentry/node")
+const destroyable = require("server-destroy")
 
 const app = new Koa()
 
@@ -34,6 +34,7 @@ app.use(
     formLimit: "10mb",
     jsonLimit: "10mb",
     textLimit: "10mb",
+    // @ts-ignore
     enableTypes: ["json", "form", "text"],
     parsedMethods: ["POST", "PUT", "PATCH", "DELETE"],
   })
@@ -76,12 +77,13 @@ server.on("close", async () => {
   }
 })
 
-module.exports = server.listen(env.PORT || 0, async () => {
+export = server.listen(env.PORT || 0, async () => {
   await startup(app, server)
 })
 
 const shutdown = () => {
   server.close()
+  // @ts-ignore
   server.destroy()
 }
 

packages/server/src/environment.ts

@@ -88,7 +88,8 @@ const environment = {
   CLIENT_ID: process.env.CLIENT_ID,
   _set(key: string, value: any) {
     process.env[key] = value
-    module.exports[key] = value
+    // @ts-ignore
+    environment[key] = value
   },
   isTest,
   isJest,

packages/worker/src/api/index.ts

@@ -163,4 +163,4 @@ for (let route of routes) {
   router.use(route.allowedMethods())
 }
 
-export = router
+export default router

packages/worker/src/environment.ts

@@ -68,7 +68,7 @@ const environment = {
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore
-    env[key] = value
+    environment[key] = value
   },
   isDev,
   isTest,

packages/worker/src/index.ts

@@ -19,7 +19,7 @@ db.init()
 import Koa from "koa"
 import koaBody from "koa-body"
 import http from "http"
-import * as api from "./api"
+import api from "./api"
 import * as redis from "./utilities/redis"
 import Sentry from "@sentry/node"
 const koaSession = require("koa-session")

packages/worker/src/tests/TestConfiguration.ts

@@ -2,7 +2,7 @@ import "./mocks"
 import * as dbConfig from "../db"
 dbConfig.init()
 import env from "../environment"
-import controllers from "./controllers"
+import * as controllers from "./controllers"
 const supertest = require("supertest")
 import { Config } from "../constants"
 import {

packages/worker/src/tests/controllers.js

@@ -1,7 +0,0 @@
-module.exports = {
-  email: require("../api/controllers/global/email"),
-  workspaces: require("../api/controllers/global/workspaces"),
-  config: require("../api/controllers/global/configs"),
-  templates: require("../api/controllers/global/templates"),
-  users: require("../api/controllers/global/users"),
-}

packages/worker/src/tests/controllers.ts

@@ -0,0 +1,5 @@
+export * as email from "../api/controllers/global/email"
+export * as workspaces from "../api/controllers/global/workspaces"
+export * as config from "../api/controllers/global/configs"
+export * as templates from "../api/controllers/global/templates"
+export * as users from "../api/controllers/global/users"

packages/worker/src/tests/structures/configs.ts

@@ -1,7 +1,7 @@
 const { Config } = require("../../constants")
 const { utils } = require("@budibase/backend-core")
 
-exports.oidc = conf => {
+export function oidc(conf?: any) {
   return {
     type: Config.OIDC,
     config: {
@@ -21,7 +21,7 @@ exports.oidc = conf => {
   }
 }
 
-exports.google = conf => {
+export function google(conf?: any) {
   return {
     type: Config.GOOGLE,
     config: {
@@ -33,7 +33,7 @@ exports.google = conf => {
   }
 }
 
-exports.smtp = conf => {
+export function smtp(conf?: any) {
   return {
     type: Config.SMTP,
     config: {
@@ -47,7 +47,7 @@ exports.smtp = conf => {
   }
 }
 
-exports.smtpEthereal = () => {
+export function smtpEthereal() {
   return {
     type: Config.SMTP,
     config: {
@@ -63,7 +63,7 @@ exports.smtpEthereal = () => {
   }
 }
 
-exports.settings = conf => {
+export function settings(conf?: any) {
   return {
     type: Config.SETTINGS,
     config: {

packages/worker/src/tests/structures/index.ts

@@ -1,5 +1,5 @@
 import { structures } from "@budibase/backend-core/tests"
-import configs from "./configs"
+import * as configs from "./configs"
 import * as users from "./users"
 import * as groups from "./groups"
 import { v4 as uuid } from "uuid"