diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index e0263546ff..c64adb010f 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -10,7 +10,7 @@ on: pull_request: branches: - master - - develop + - develop workflow_dispatch: env: @@ -64,6 +64,20 @@ jobs: name: codecov-umbrella verbose: true + test-pro: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Use Node.js 14.x + uses: actions/setup-node@v1 + with: + node-version: 14.x + - name: Install Pro + run: yarn install:pro $BRANCH $BASE_BRANCH + - run: yarn + - run: yarn bootstrap + - run: yarn test:pro + integration-test: runs-on: ubuntu-latest services: diff --git a/hosting/docker-compose.test.yaml b/hosting/docker-compose.test.yaml index dfd78621c5..f059173d2d 100644 --- a/hosting/docker-compose.test.yaml +++ b/hosting/docker-compose.test.yaml @@ -8,8 +8,8 @@ services: # Last version that supports the "fs" backend image: minio/minio:RELEASE.2022-10-24T18-35-07Z ports: - - 9000 - - 9001 + - "9000" + - "9001" environment: MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} @@ -28,9 +28,9 @@ services: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - - 5984 - - 4369 - - 9100 + - "5984" + - "4369" + - "9100" healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5984/_up"] interval: 30s @@ -42,6 +42,6 @@ services: image: redis command: redis-server --requirepass ${REDIS_PASSWORD} ports: - - 6379 + - "6379" healthcheck: - test: ["CMD", "redis-cli", "ping"] + test: ["CMD", "redis-cli", "ping"] \ No newline at end of file diff --git a/lerna.json b/lerna.json index 5145222dfa..39a4de05fa 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.3.18-alpha.12", + "version": "2.3.18-alpha.14", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 3ead7d5553..815e470916 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "js-yaml": "^4.1.0", "kill-port": "^1.6.1", "lerna": "3.14.1", - "madge": "^5.0.1", + "madge": "^6.0.0", "prettier": "^2.3.1", "prettier-plugin-svelte": "^2.3.0", "rimraf": "^3.0.2", @@ -44,7 +44,7 @@ "dev": "yarn run kill-all && lerna link && lerna run --parallel dev:builder --concurrency 1", "dev:noserver": "yarn run kill-builder && lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server", - "test": "lerna run test && yarn test:pro", + "test": "lerna run test", "test:pro": "bash scripts/pro/test.sh", "lint:eslint": "eslint packages && eslint qa-core", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", @@ -84,4 +84,4 @@ "install:pro": "bash scripts/pro/install.sh", "dep:clean": "yarn clean && yarn bootstrap" } -} \ No newline at end of file +} diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 428d785a44..658641bb1b 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.3.18-alpha.12", + "version": "2.3.18-alpha.14", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -24,7 +24,7 @@ "dependencies": { "@budibase/nano": "10.1.1", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "2.3.18-alpha.12", + "@budibase/types": "2.3.18-alpha.14", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/backend-core/src/auth/auth.ts b/packages/backend-core/src/auth/auth.ts index bee245a3ae..936d06ddff 100644 --- a/packages/backend-core/src/auth/auth.ts +++ b/packages/backend-core/src/auth/auth.ts @@ -2,25 +2,34 @@ const _passport = require("koa-passport") const LocalStrategy = require("passport-local").Strategy const JwtStrategy = require("passport-jwt").Strategy import { getGlobalDB } from "../context" -const refresh = require("passport-oauth2-refresh") -import { Config, Cookie } from "../constants" -import { getScopedConfig } from "../db" +import { Cookie } from "../constants" import { getSessionsForUser, invalidateSessions } from "../security/sessions" import { + authenticated, + csrf, + google, jwt as jwtPassport, local, - authenticated, - tenancy, - csrf, oidc, - google, + tenancy, } from "../middleware" +import * as userCache from "../cache/user" import { invalidateUser } from "../cache/user" -import { PlatformLogoutOpts, User } from "@budibase/types" +import { + ConfigType, + GoogleInnerConfig, + OIDCInnerConfig, + PlatformLogoutOpts, + SSOProviderType, + User, +} from "@budibase/types" import { logAlert } from "../logging" import * as events from "../events" -import * as userCache from "../cache/user" +import * as configs from "../configs" import { clearCookie, getCookie } from "../utils" +import { ssoSaveUserNoOp } from "../middleware/passport/sso/sso" + +const refresh = require("passport-oauth2-refresh") export { auditLog, authError, @@ -33,7 +42,6 @@ export { google, oidc, } from "../middleware" -import { ssoSaveUserNoOp } from "../middleware/passport/sso/sso" export const buildAuthMiddleware = authenticated export const buildTenancyMiddleware = tenancy export const buildCsrfMiddleware = csrf @@ -63,11 +71,10 @@ _passport.deserializeUser(async (user: User, done: any) => { }) async function refreshOIDCAccessToken( - db: any, - chosenConfig: any, + chosenConfig: OIDCInnerConfig, refreshToken: string -) { - const callbackUrl = await oidc.getCallbackUrl(db, chosenConfig) +): Promise { + const callbackUrl = await oidc.getCallbackUrl() let enrichedConfig: any let strategy: any @@ -90,7 +97,7 @@ async function refreshOIDCAccessToken( return new Promise(resolve => { refresh.requestNewAccessToken( - Config.OIDC, + ConfigType.OIDC, refreshToken, (err: any, accessToken: string, refreshToken: any, params: any) => { resolve({ err, accessToken, refreshToken, params }) @@ -100,11 +107,10 @@ async function refreshOIDCAccessToken( } async function refreshGoogleAccessToken( - db: any, - config: any, + config: GoogleInnerConfig, refreshToken: any -) { - let callbackUrl = await google.getCallbackUrl(db, config) +): Promise { + let callbackUrl = await google.getCallbackUrl(config) let strategy try { @@ -124,7 +130,7 @@ async function refreshGoogleAccessToken( return new Promise(resolve => { refresh.requestNewAccessToken( - Config.GOOGLE, + ConfigType.GOOGLE, refreshToken, (err: any, accessToken: string, refreshToken: string, params: any) => { resolve({ err, accessToken, refreshToken, params }) @@ -133,41 +139,37 @@ async function refreshGoogleAccessToken( }) } +interface RefreshResponse { + err?: { + data?: string + } + accessToken?: string + refreshToken?: string + params?: any +} + export async function refreshOAuthToken( refreshToken: string, - configType: string, - configId: string -) { - const db = getGlobalDB() - - const config = await getScopedConfig(db, { - type: configType, - group: {}, - }) - - let chosenConfig = {} - let refreshResponse - if (configType === Config.OIDC) { - // configId - retrieved from cookie. - chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0] - if (!chosenConfig) { - throw new Error("Invalid OIDC configuration") - } - refreshResponse = await refreshOIDCAccessToken( - db, - chosenConfig, - refreshToken - ) - } else { - chosenConfig = config - refreshResponse = await refreshGoogleAccessToken( - db, - chosenConfig, - refreshToken - ) + providerType: SSOProviderType, + configId?: string +): Promise { + switch (providerType) { + case SSOProviderType.OIDC: + if (!configId) { + return { err: { data: "OIDC config id not provided" } } + } + const oidcConfig = await configs.getOIDCConfigById(configId) + if (!oidcConfig) { + return { err: { data: "OIDC configuration not found" } } + } + return refreshOIDCAccessToken(oidcConfig, refreshToken) + case SSOProviderType.GOOGLE: + let googleConfig = await configs.getGoogleConfig() + if (!googleConfig) { + return { err: { data: "Google configuration not found" } } + } + return refreshGoogleAccessToken(googleConfig, refreshToken) } - - return refreshResponse } // TODO: Refactor to use user save function instead to prevent the need for @@ -225,6 +227,6 @@ export async function platformLogout(opts: PlatformLogoutOpts) { const sessionIds = sessions.map(({ sessionId }) => sessionId) await invalidateSessions(userId, { sessionIds, reason: "logout" }) - await events.auth.logout() + await events.auth.logout(ctx.user?.email) await userCache.invalidateUser(userId) } diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts new file mode 100644 index 0000000000..882c37ceb9 --- /dev/null +++ b/packages/backend-core/src/configs/configs.ts @@ -0,0 +1,224 @@ +import { + Config, + ConfigType, + GoogleConfig, + GoogleInnerConfig, + OIDCConfig, + OIDCInnerConfig, + SettingsConfig, + SettingsInnerConfig, + SMTPConfig, + SMTPInnerConfig, +} from "@budibase/types" +import { DocumentType, SEPARATOR } from "../constants" +import { CacheKey, TTL, withCache } from "../cache" +import * as context from "../context" +import env from "../environment" +import environment from "../environment" + +// UTILS + +/** + * Generates a new configuration ID. + * @returns {string} The new configuration ID which the config doc can be stored under. + */ +export function generateConfigID(type: ConfigType) { + return `${DocumentType.CONFIG}${SEPARATOR}${type}` +} + +export async function getConfig( + type: ConfigType +): Promise { + const db = context.getGlobalDB() + try { + // await to catch error + const config = (await db.get(generateConfigID(type))) as T + return config + } catch (e: any) { + if (e.status === 404) { + return + } + throw e + } +} + +export async function save(config: Config) { + const db = context.getGlobalDB() + return db.put(config) +} + +// SETTINGS + +export async function getSettingsConfigDoc(): Promise { + let config = await getConfig(ConfigType.SETTINGS) + + if (!config) { + config = { + _id: generateConfigID(ConfigType.GOOGLE), + type: ConfigType.SETTINGS, + config: {}, + } + } + + // overridden fields + config.config.platformUrl = await getPlatformUrl({ + tenantAware: true, + config: config.config, + }) + config.config.analyticsEnabled = await analyticsEnabled({ + config: config.config, + }) + + return config +} + +export async function getSettingsConfig(): Promise { + return (await getSettingsConfigDoc()).config +} + +export async function getPlatformUrl( + opts: { tenantAware: boolean; config?: SettingsInnerConfig } = { + tenantAware: true, + } +) { + let platformUrl = env.PLATFORM_URL || "http://localhost:10000" + + if (!env.SELF_HOSTED && env.MULTI_TENANCY && opts.tenantAware) { + // cloud and multi tenant - add the tenant to the default platform url + const tenantId = context.getTenantId() + if (!platformUrl.includes("localhost:")) { + platformUrl = platformUrl.replace("://", `://${tenantId}.`) + } + } else if (env.SELF_HOSTED) { + const config = opts?.config + ? opts.config + : // direct to db to prevent infinite loop + (await getConfig(ConfigType.SETTINGS))?.config + if (config?.platformUrl) { + platformUrl = config.platformUrl + } + } + + return platformUrl +} + +export const analyticsEnabled = async (opts?: { + config?: SettingsInnerConfig +}) => { + // cloud - always use the environment variable + if (!env.SELF_HOSTED) { + return !!env.ENABLE_ANALYTICS + } + + // self host - prefer the settings doc + // use cache as events have high throughput + const enabledInDB = await withCache( + CacheKey.ANALYTICS_ENABLED, + TTL.ONE_DAY, + async () => { + const config = opts?.config + ? opts.config + : // direct to db to prevent infinite loop + (await getConfig(ConfigType.SETTINGS))?.config + + // need to do explicit checks in case the field is not set + if (config?.analyticsEnabled === false) { + return false + } else if (config?.analyticsEnabled === true) { + return true + } + } + ) + + if (enabledInDB !== undefined) { + return enabledInDB + } + + // fallback to the environment variable + // explicitly check for 0 or false here, undefined or otherwise is treated as true + const envEnabled: any = env.ENABLE_ANALYTICS + if (envEnabled === 0 || envEnabled === false) { + return false + } else { + return true + } +} + +// GOOGLE + +async function getGoogleConfigDoc(): Promise { + return await getConfig(ConfigType.GOOGLE) +} + +export async function getGoogleConfig(): Promise< + GoogleInnerConfig | undefined +> { + const config = await getGoogleConfigDoc() + if (config) { + return config.config + } + + // Use google fallback configuration from env variables + if (environment.GOOGLE_CLIENT_ID && environment.GOOGLE_CLIENT_SECRET) { + return { + clientID: environment.GOOGLE_CLIENT_ID!, + clientSecret: environment.GOOGLE_CLIENT_SECRET!, + activated: true, + } + } +} + +// OIDC + +async function getOIDCConfigDoc(): Promise { + return getConfig(ConfigType.OIDC) +} + +export async function getOIDCConfig(): Promise { + const config = (await getOIDCConfigDoc())?.config + // default to the 0th config + return config?.configs && config.configs[0] +} + +/** + * @param configId The config id of the inner config to retrieve + */ +export async function getOIDCConfigById( + configId: string +): Promise { + const config = (await getConfig(ConfigType.OIDC))?.config + return config && config.configs.filter((c: any) => c.uuid === configId)[0] +} + +// SMTP + +export async function getSMTPConfigDoc(): Promise { + return getConfig(ConfigType.SMTP) +} + +export async function getSMTPConfig( + isAutomation?: boolean +): Promise { + const config = await getSMTPConfigDoc() + if (config) { + return config.config + } + + // always allow fallback in self host + // in cloud don't allow for automations + const allowFallback = env.SELF_HOSTED || !isAutomation + + // Use an SMTP fallback configuration from env variables + if (env.SMTP_FALLBACK_ENABLED && allowFallback) { + return { + port: env.SMTP_PORT, + host: env.SMTP_HOST!, + secure: false, + from: env.SMTP_FROM_ADDRESS!, + auth: { + user: env.SMTP_USER!, + pass: env.SMTP_PASSWORD!, + }, + } + } +} diff --git a/packages/backend-core/src/configs/index.ts b/packages/backend-core/src/configs/index.ts new file mode 100644 index 0000000000..783f22a0b9 --- /dev/null +++ b/packages/backend-core/src/configs/index.ts @@ -0,0 +1 @@ +export * from "./configs" diff --git a/packages/backend-core/src/configs/tests/configs.spec.ts b/packages/backend-core/src/configs/tests/configs.spec.ts new file mode 100644 index 0000000000..079f2ab681 --- /dev/null +++ b/packages/backend-core/src/configs/tests/configs.spec.ts @@ -0,0 +1,116 @@ +import { DBTestConfiguration, generator, testEnv } from "../../../tests" +import { ConfigType } from "@budibase/types" +import env from "../../environment" +import * as configs from "../configs" + +const DEFAULT_URL = "http://localhost:10000" +const ENV_URL = "http://env.com" + +describe("configs", () => { + const config = new DBTestConfiguration() + + const setDbPlatformUrl = async (dbUrl: string) => { + const settingsConfig = { + _id: configs.generateConfigID(ConfigType.SETTINGS), + type: ConfigType.SETTINGS, + config: { + platformUrl: dbUrl, + }, + } + await configs.save(settingsConfig) + } + + beforeEach(async () => { + config.newTenant() + }) + + describe("getPlatformUrl", () => { + describe("self host", () => { + beforeEach(async () => { + testEnv.selfHosted() + }) + + it("gets the default url", async () => { + await config.doInTenant(async () => { + const url = await configs.getPlatformUrl() + expect(url).toBe(DEFAULT_URL) + }) + }) + + it("gets the platform url from the environment", async () => { + await config.doInTenant(async () => { + env._set("PLATFORM_URL", ENV_URL) + const url = await configs.getPlatformUrl() + expect(url).toBe(ENV_URL) + }) + }) + + it("gets the platform url from the database", async () => { + await config.doInTenant(async () => { + const dbUrl = generator.url() + await setDbPlatformUrl(dbUrl) + const url = await configs.getPlatformUrl() + expect(url).toBe(dbUrl) + }) + }) + }) + + describe("cloud", () => { + function getTenantAwareUrl() { + return `http://${config.tenantId}.env.com` + } + + beforeEach(async () => { + testEnv.cloudHosted() + testEnv.multiTenant() + + env._set("PLATFORM_URL", ENV_URL) + }) + + it("gets the platform url from the environment without tenancy", async () => { + await config.doInTenant(async () => { + const url = await configs.getPlatformUrl({ tenantAware: false }) + expect(url).toBe(ENV_URL) + }) + }) + + it("gets the platform url from the environment with tenancy", async () => { + await config.doInTenant(async () => { + const url = await configs.getPlatformUrl() + expect(url).toBe(getTenantAwareUrl()) + }) + }) + + it("never gets the platform url from the database", async () => { + await config.doInTenant(async () => { + await setDbPlatformUrl(generator.url()) + const url = await configs.getPlatformUrl() + expect(url).toBe(getTenantAwareUrl()) + }) + }) + }) + }) + + describe("getSettingsConfig", () => { + beforeAll(async () => { + testEnv.selfHosted() + env._set("PLATFORM_URL", "") + }) + + it("returns the platform url with an existing config", async () => { + await config.doInTenant(async () => { + const dbUrl = generator.url() + await setDbPlatformUrl(dbUrl) + const config = await configs.getSettingsConfig() + expect(config.platformUrl).toBe(dbUrl) + }) + }) + + it("returns the platform url without an existing config", async () => { + await config.doInTenant(async () => { + const config = await configs.getSettingsConfig() + expect(config.platformUrl).toBe(DEFAULT_URL) + }) + }) + }) +}) diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index f7d15b3880..d41098c405 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -68,6 +68,7 @@ export enum DocumentType { MEM_VIEW = "view", USER_FLAG = "flag", AUTOMATION_METADATA = "meta_au", + AUDIT_LOG = "al", } export const StaticDatabases = { @@ -88,6 +89,9 @@ export const StaticDatabases = { install: "install", }, }, + AUDIT_LOGS: { + name: "audit-logs", + }, } export const APP_PREFIX = DocumentType.APP + SEPARATOR diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index 0bf3df4094..e25c90575f 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -41,5 +41,6 @@ export enum Config { OIDC_LOGOS = "logos_oidc", } +export const MIN_VALID_DATE = new Date(-2147483647000) export const MAX_VALID_DATE = new Date(2147483647000) export const DEFAULT_TENANT_ID = "default" diff --git a/packages/backend-core/src/context/Context.ts b/packages/backend-core/src/context/Context.ts index 02b7713764..d29b6935a8 100644 --- a/packages/backend-core/src/context/Context.ts +++ b/packages/backend-core/src/context/Context.ts @@ -1,5 +1,5 @@ import { AsyncLocalStorage } from "async_hooks" -import { ContextMap } from "./mainContext" +import { ContextMap } from "./types" export default class Context { static storage = new AsyncLocalStorage() diff --git a/packages/backend-core/src/context/identity.ts b/packages/backend-core/src/context/identity.ts index 648dd1b5fd..84de3b68c9 100644 --- a/packages/backend-core/src/context/identity.ts +++ b/packages/backend-core/src/context/identity.ts @@ -5,6 +5,8 @@ import { isCloudAccount, Account, AccountUserContext, + UserContext, + Ctx, } from "@budibase/types" import * as context from "." @@ -16,15 +18,22 @@ export function doInIdentityContext(identity: IdentityContext, task: any) { return context.doInIdentityContext(identity, task) } -export function doInUserContext(user: User, task: any) { - const userContext: any = { +// used in server/worker +export function doInUserContext(user: User, ctx: Ctx, task: any) { + const userContext: UserContext = { ...user, _id: user._id as string, type: IdentityType.USER, + hostInfo: { + ipAddress: ctx.request.ip, + // filled in by koa-useragent package + userAgent: ctx.userAgent._agent.source, + }, } return doInIdentityContext(userContext, task) } +// used in account portal export function doInAccountContext(account: Account, task: any) { const _id = getAccountUserId(account) const tenantId = account.tenantId diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 9884d25d5a..02ba16aa8c 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -11,13 +11,7 @@ import { DEFAULT_TENANT_ID, } from "../constants" import { Database, IdentityContext } from "@budibase/types" - -export type ContextMap = { - tenantId?: string - appId?: string - identity?: IdentityContext - environmentVariables?: Record -} +import { ContextMap } from "./types" let TEST_APP_ID: string | null = null @@ -30,14 +24,23 @@ export function getGlobalDBName(tenantId?: string) { return baseGlobalDBName(tenantId) } -export function baseGlobalDBName(tenantId: string | undefined | null) { - let dbName - if (!tenantId || tenantId === DEFAULT_TENANT_ID) { - dbName = StaticDatabases.GLOBAL.name - } else { - dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}` +export function getAuditLogDBName(tenantId?: string) { + if (!tenantId) { + tenantId = getTenantId() + } + if (tenantId === DEFAULT_TENANT_ID) { + return StaticDatabases.AUDIT_LOGS.name + } else { + return `${tenantId}${SEPARATOR}${StaticDatabases.AUDIT_LOGS.name}` + } +} + +export function baseGlobalDBName(tenantId: string | undefined | null) { + if (!tenantId || tenantId === DEFAULT_TENANT_ID) { + return StaticDatabases.GLOBAL.name + } else { + return `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}` } - return dbName } export function isMultiTenant() { @@ -228,6 +231,13 @@ export function getGlobalDB(): Database { return getDB(baseGlobalDBName(context?.tenantId)) } +export function getAuditLogsDB(): Database { + if (!getTenantId()) { + throw new Error("No tenant ID found - cannot open audit log DB") + } + return getDB(getAuditLogDBName()) +} + /** * Gets the app database based on whatever the request * contained, dev or prod. diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts new file mode 100644 index 0000000000..78197ed528 --- /dev/null +++ b/packages/backend-core/src/context/types.ts @@ -0,0 +1,9 @@ +import { IdentityContext } from "@budibase/types" + +// keep this out of Budibase types, don't want to expose context info +export type ContextMap = { + tenantId?: string + appId?: string + identity?: IdentityContext + environmentVariables?: Record +} diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 0d9f75fa18..a569b17b36 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -7,3 +7,4 @@ export { default as Replication } from "./Replication" // exports to support old export structure export * from "../constants/db" export { getGlobalDBName, baseGlobalDBName } from "../context" +export * from "./lucene" diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts new file mode 100644 index 0000000000..cba2f0138a --- /dev/null +++ b/packages/backend-core/src/db/lucene.ts @@ -0,0 +1,624 @@ +import fetch from "node-fetch" +import { getCouchInfo } from "./couch" +import { SearchFilters, Row } from "@budibase/types" + +const QUERY_START_REGEX = /\d[0-9]*:/g + +interface SearchResponse { + rows: T[] | any[] + bookmark: string +} + +interface PaginatedSearchResponse extends SearchResponse { + hasNextPage: boolean +} + +export type SearchParams = { + tableId?: string + sort?: string + sortOrder?: string + sortType?: string + limit?: number + bookmark?: string + version?: string + indexer?: () => Promise + disableEscaping?: boolean + rows?: T | Row[] +} + +export function removeKeyNumbering(key: any): string { + if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) { + const parts = key.split(":") + // remove the number + parts.shift() + return parts.join(":") + } else { + return key + } +} + +/** + * Class to build lucene query URLs. + * Optionally takes a base lucene query object. + */ +export class QueryBuilder { + dbName: string + index: string + query: SearchFilters + limit: number + sort?: string + bookmark?: string + sortOrder: string + sortType: string + includeDocs: boolean + version?: string + indexBuilder?: () => Promise + noEscaping = false + + constructor(dbName: string, index: string, base?: SearchFilters) { + this.dbName = dbName + this.index = index + this.query = { + allOr: false, + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + oneOf: {}, + contains: {}, + notContains: {}, + containsAny: {}, + ...base, + } + this.limit = 50 + this.sortOrder = "ascending" + this.sortType = "string" + this.includeDocs = true + } + + disableEscaping() { + this.noEscaping = true + return this + } + + setIndexBuilder(builderFn: () => Promise) { + this.indexBuilder = builderFn + return this + } + + setVersion(version?: string) { + if (version != null) { + this.version = version + } + return this + } + + setTable(tableId: string) { + this.query.equal!.tableId = tableId + return this + } + + setLimit(limit?: number) { + if (limit != null) { + this.limit = limit + } + return this + } + + setSort(sort?: string) { + if (sort != null) { + this.sort = sort + } + return this + } + + setSortOrder(sortOrder?: string) { + if (sortOrder != null) { + this.sortOrder = sortOrder + } + return this + } + + setSortType(sortType?: string) { + if (sortType != null) { + this.sortType = sortType + } + return this + } + + setBookmark(bookmark?: string) { + if (bookmark != null) { + this.bookmark = bookmark + } + return this + } + + excludeDocs() { + this.includeDocs = false + return this + } + + addString(key: string, partial: string) { + this.query.string![key] = partial + return this + } + + addFuzzy(key: string, fuzzy: string) { + this.query.fuzzy![key] = fuzzy + return this + } + + addRange(key: string, low: string | number, high: string | number) { + this.query.range![key] = { + low, + high, + } + return this + } + + addEqual(key: string, value: any) { + this.query.equal![key] = value + return this + } + + addNotEqual(key: string, value: any) { + this.query.notEqual![key] = value + return this + } + + addEmpty(key: string, value: any) { + this.query.empty![key] = value + return this + } + + addNotEmpty(key: string, value: any) { + this.query.notEmpty![key] = value + return this + } + + addOneOf(key: string, value: any) { + this.query.oneOf![key] = value + return this + } + + addContains(key: string, value: any) { + this.query.contains![key] = value + return this + } + + addNotContains(key: string, value: any) { + this.query.notContains![key] = value + return this + } + + addContainsAny(key: string, value: any) { + this.query.containsAny![key] = value + return this + } + + handleSpaces(input: string) { + if (this.noEscaping) { + return input + } else { + return input.replace(/ /g, "_") + } + } + + /** + * Preprocesses a value before going into a lucene search. + * Transforms strings to lowercase and wraps strings and bools in quotes. + * @param value The value to process + * @param options The preprocess options + * @returns {string|*} + */ + preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) { + const hasVersion = !!this.version + // Determine if type needs wrapped + const originalType = typeof value + // Convert to lowercase + if (value && lowercase) { + value = value.toLowerCase ? value.toLowerCase() : value + } + // Escape characters + if (!this.noEscaping && escape && originalType === "string") { + value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&") + } + + // Wrap in quotes + if (originalType === "string" && !isNaN(value) && !type) { + value = `"${value}"` + } else if (hasVersion && wrap) { + value = originalType === "number" ? value : `"${value}"` + } + return value + } + + buildSearchQuery() { + const builder = this + let allOr = this.query && this.query.allOr + let query = allOr ? "" : "*:*" + const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true } + let tableId + if (this.query.equal!.tableId) { + tableId = this.query.equal!.tableId + delete this.query.equal!.tableId + } + + const equal = (key: string, value: any) => { + // 0 evaluates to false, which means we would return all rows if we don't check it + if (!value && value !== 0) { + return null + } + return `${key}:${builder.preprocess(value, allPreProcessingOpts)}` + } + + const contains = (key: string, value: any, mode = "AND") => { + if (Array.isArray(value) && value.length === 0) { + return null + } + if (!Array.isArray(value)) { + return `${key}:${value}` + } + let statement = `${builder.preprocess(value[0], { escape: true })}` + for (let i = 1; i < value.length; i++) { + statement += ` ${mode} ${builder.preprocess(value[i], { + escape: true, + })}` + } + return `${key}:(${statement})` + } + + const notContains = (key: string, value: any) => { + // @ts-ignore + const allPrefix = allOr === "" ? "*:* AND" : "" + return allPrefix + "NOT " + contains(key, value) + } + + const containsAny = (key: string, value: any) => { + return contains(key, value, "OR") + } + + const oneOf = (key: string, value: any) => { + if (!Array.isArray(value)) { + if (typeof value === "string") { + value = value.split(",") + } else { + return "" + } + } + let orStatement = `${builder.preprocess(value[0], allPreProcessingOpts)}` + for (let i = 1; i < value.length; i++) { + orStatement += ` OR ${builder.preprocess( + value[i], + allPreProcessingOpts + )}` + } + return `${key}:(${orStatement})` + } + + function build(structure: any, queryFn: any) { + for (let [key, value] of Object.entries(structure)) { + // check for new format - remove numbering if needed + key = removeKeyNumbering(key) + key = builder.preprocess(builder.handleSpaces(key), { + escape: true, + }) + const expression = queryFn(key, value) + if (expression == null) { + continue + } + if (query.length > 0) { + query += ` ${allOr ? "OR" : "AND"} ` + } + query += expression + } + } + + // Construct the actual lucene search query string from JSON structure + if (this.query.string) { + build(this.query.string, (key: string, value: any) => { + if (!value) { + return null + } + value = builder.preprocess(value, { + escape: true, + lowercase: true, + type: "string", + }) + return `${key}:${value}*` + }) + } + if (this.query.range) { + build(this.query.range, (key: string, value: any) => { + if (!value) { + return null + } + if (value.low == null || value.low === "") { + return null + } + if (value.high == null || value.high === "") { + return null + } + const low = builder.preprocess(value.low, allPreProcessingOpts) + const high = builder.preprocess(value.high, allPreProcessingOpts) + return `${key}:[${low} TO ${high}]` + }) + } + if (this.query.fuzzy) { + build(this.query.fuzzy, (key: string, value: any) => { + if (!value) { + return null + } + value = builder.preprocess(value, { + escape: true, + lowercase: true, + type: "fuzzy", + }) + return `${key}:${value}~` + }) + } + if (this.query.equal) { + build(this.query.equal, equal) + } + if (this.query.notEqual) { + build(this.query.notEqual, (key: string, value: any) => { + if (!value) { + return null + } + return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}` + }) + } + if (this.query.empty) { + build(this.query.empty, (key: string) => `!${key}:["" TO *]`) + } + if (this.query.notEmpty) { + build(this.query.notEmpty, (key: string) => `${key}:["" TO *]`) + } + if (this.query.oneOf) { + build(this.query.oneOf, oneOf) + } + if (this.query.contains) { + build(this.query.contains, contains) + } + if (this.query.notContains) { + build(this.query.notContains, notContains) + } + if (this.query.containsAny) { + build(this.query.containsAny, containsAny) + } + // make sure table ID is always added as an AND + if (tableId) { + query = `(${query})` + allOr = false + build({ tableId }, equal) + } + return query + } + + buildSearchBody() { + let body: any = { + q: this.buildSearchQuery(), + limit: Math.min(this.limit, 200), + include_docs: this.includeDocs, + } + if (this.bookmark) { + body.bookmark = this.bookmark + } + if (this.sort) { + const order = this.sortOrder === "descending" ? "-" : "" + const type = `<${this.sortType}>` + body.sort = `${order}${this.handleSpaces(this.sort)}${type}` + } + return body + } + + async run() { + const { url, cookie } = getCouchInfo() + const fullPath = `${url}/${this.dbName}/_design/database/_search/${this.index}` + const body = this.buildSearchBody() + try { + return await runQuery(fullPath, body, cookie) + } catch (err: any) { + if (err.status === 404 && this.indexBuilder) { + await this.indexBuilder() + return await runQuery(fullPath, body, cookie) + } else { + throw err + } + } + } +} + +/** + * Executes a lucene search query. + * @param url The query URL + * @param body The request body defining search criteria + * @param cookie The auth cookie for CouchDB + * @returns {Promise<{rows: []}>} + */ +async function runQuery( + url: string, + body: any, + cookie: string +): Promise> { + const response = await fetch(url, { + body: JSON.stringify(body), + method: "POST", + headers: { + Authorization: cookie, + }, + }) + + if (response.status === 404) { + throw response + } + const json = await response.json() + + let output: any = { + rows: [], + } + if (json.rows != null && json.rows.length > 0) { + output.rows = json.rows.map((row: any) => row.doc) + } + if (json.bookmark) { + output.bookmark = json.bookmark + } + return output +} + +/** + * Gets round the fixed limit of 200 results from a query by fetching as many + * pages as required and concatenating the results. This recursively operates + * until enough results have been found. + * @param dbName {string} Which database to run a lucene query on + * @param index {string} Which search index to utilise + * @param query {object} The JSON query structure + * @param params {object} The search params including: + * tableId {string} The table ID to search + * sort {string} The sort column + * sortOrder {string} The sort order ("ascending" or "descending") + * sortType {string} Whether to treat sortable values as strings or + * numbers. ("string" or "number") + * limit {number} The number of results to fetch + * bookmark {string|null} Current bookmark in the recursive search + * rows {array|null} Current results in the recursive search + * @returns {Promise<*[]|*>} + */ +async function recursiveSearch( + dbName: string, + index: string, + query: any, + params: any +): Promise { + const bookmark = params.bookmark + const rows = params.rows || [] + if (rows.length >= params.limit) { + return rows + } + let pageSize = 200 + if (rows.length > params.limit - 200) { + pageSize = params.limit - rows.length + } + const page = await new QueryBuilder(dbName, index, query) + .setVersion(params.version) + .setTable(params.tableId) + .setBookmark(bookmark) + .setLimit(pageSize) + .setSort(params.sort) + .setSortOrder(params.sortOrder) + .setSortType(params.sortType) + .run() + if (!page.rows.length) { + return rows + } + if (page.rows.length < 200) { + return [...rows, ...page.rows] + } + const newParams = { + ...params, + bookmark: page.bookmark, + rows: [...rows, ...page.rows], + } + return await recursiveSearch(dbName, index, query, newParams) +} + +/** + * Performs a paginated search. A bookmark will be returned to allow the next + * page to be fetched. There is a max limit off 200 results per page in a + * paginated search. + * @param dbName {string} Which database to run a lucene query on + * @param index {string} Which search index to utilise + * @param query {object} The JSON query structure + * @param params {object} The search params including: + * tableId {string} The table ID to search + * sort {string} The sort column + * sortOrder {string} The sort order ("ascending" or "descending") + * sortType {string} Whether to treat sortable values as strings or + * numbers. ("string" or "number") + * limit {number} The desired page size + * bookmark {string} The bookmark to resume from + * @returns {Promise<{hasNextPage: boolean, rows: *[]}>} + */ +export async function paginatedSearch( + dbName: string, + index: string, + query: SearchFilters, + params: SearchParams +) { + let limit = params.limit + if (limit == null || isNaN(limit) || limit < 0) { + limit = 50 + } + limit = Math.min(limit, 200) + const search = new QueryBuilder(dbName, index, query) + if (params.version) { + search.setVersion(params.version) + } + if (params.tableId) { + search.setTable(params.tableId) + } + if (params.sort) { + search + .setSort(params.sort) + .setSortOrder(params.sortOrder) + .setSortType(params.sortType) + } + if (params.indexer) { + search.setIndexBuilder(params.indexer) + } + if (params.disableEscaping) { + search.disableEscaping() + } + const searchResults = await search + .setBookmark(params.bookmark) + .setLimit(limit) + .run() + + // Try fetching 1 row in the next page to see if another page of results + // exists or not + search.setBookmark(searchResults.bookmark).setLimit(1) + if (params.tableId) { + search.setTable(params.tableId) + } + const nextResults = await search.run() + + return { + ...searchResults, + hasNextPage: nextResults.rows && nextResults.rows.length > 0, + } +} + +/** + * Performs a full search, fetching multiple pages if required to return the + * desired amount of results. There is a limit of 1000 results to avoid + * heavy performance hits, and to avoid client components breaking from + * handling too much data. + * @param dbName {string} Which database to run a lucene query on + * @param index {string} Which search index to utilise + * @param query {object} The JSON query structure + * @param params {object} The search params including: + * tableId {string} The table ID to search + * sort {string} The sort column + * sortOrder {string} The sort order ("ascending" or "descending") + * sortType {string} Whether to treat sortable values as strings or + * numbers. ("string" or "number") + * limit {number} The desired number of results + * @returns {Promise<{rows: *}>} + */ +export async function fullSearch( + dbName: string, + index: string, + query: SearchFilters, + params: SearchParams +) { + let limit = params.limit + if (limit == null || isNaN(limit) || limit < 0) { + limit = 1000 + } + params.limit = Math.min(limit, 1000) + const rows = await recursiveSearch(dbName, index, query, params) + return { rows } +} diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts new file mode 100644 index 0000000000..23b01e18df --- /dev/null +++ b/packages/backend-core/src/db/tests/lucene.spec.ts @@ -0,0 +1,161 @@ +import { newid } from "../../newid" +import { getDB } from "../db" +import { Database } from "@budibase/types" +import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene" + +const INDEX_NAME = "main" + +const index = `function(doc) { + let props = ["property", "number"] + for (let key of props) { + if (doc[key]) { + index(key, doc[key]) + } + } +}` + +describe("lucene", () => { + let db: Database, dbName: string + + beforeAll(async () => { + dbName = `db-${newid()}` + // create the DB for testing + db = getDB(dbName) + await db.put({ _id: newid(), property: "word" }) + await db.put({ _id: newid(), property: "word2" }) + await db.put({ _id: newid(), property: "word3", number: 1 }) + }) + + it("should be able to create a lucene index", async () => { + const response = await db.put({ + _id: "_design/database", + indexes: { + [INDEX_NAME]: { + index: index, + analyzer: "standard", + }, + }, + }) + expect(response.ok).toBe(true) + }) + + describe("query builder", () => { + it("should be able to perform a basic query", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.setSort("property") + builder.setSortOrder("desc") + builder.setSortType("string") + const resp = await builder.run() + expect(resp.rows.length).toBe(3) + }) + + it("should handle limits", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.setLimit(1) + const resp = await builder.run() + expect(resp.rows.length).toBe(1) + }) + + it("should be able to perform a string search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addString("property", "wo") + const resp = await builder.run() + expect(resp.rows.length).toBe(3) + }) + + it("should be able to perform a range search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addRange("number", 0, 1) + const resp = await builder.run() + expect(resp.rows.length).toBe(1) + }) + + it("should be able to perform an equal search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addEqual("property", "word2") + const resp = await builder.run() + expect(resp.rows.length).toBe(1) + }) + + it("should be able to perform a not equal search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addNotEqual("property", "word2") + const resp = await builder.run() + expect(resp.rows.length).toBe(2) + }) + + it("should be able to perform an empty search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addEmpty("number", true) + const resp = await builder.run() + expect(resp.rows.length).toBe(2) + }) + + it("should be able to perform a not empty search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addNotEmpty("number", true) + const resp = await builder.run() + expect(resp.rows.length).toBe(1) + }) + + it("should be able to perform a one of search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addOneOf("property", ["word", "word2"]) + const resp = await builder.run() + expect(resp.rows.length).toBe(2) + }) + + it("should be able to perform a contains search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addContains("property", ["word"]) + const resp = await builder.run() + expect(resp.rows.length).toBe(1) + }) + + it("should be able to perform a not contains search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addNotContains("property", ["word2"]) + const resp = await builder.run() + expect(resp.rows.length).toBe(2) + }) + }) + + describe("paginated search", () => { + it("should be able to perform a paginated search", async () => { + const page = await paginatedSearch( + dbName, + INDEX_NAME, + { + string: { + property: "wo", + }, + }, + { + limit: 1, + sort: "property", + sortType: "string", + sortOrder: "desc", + } + ) + expect(page.rows.length).toBe(1) + expect(page.hasNextPage).toBe(true) + expect(page.bookmark).toBeDefined() + }) + }) + + describe("full search", () => { + it("should be able to perform a full search", async () => { + const page = await fullSearch( + dbName, + INDEX_NAME, + { + string: { + property: "wo", + }, + }, + {} + ) + expect(page.rows.length).toBe(3) + }) + }) +}) diff --git a/packages/backend-core/src/db/tests/utils.spec.ts b/packages/backend-core/src/db/tests/utils.spec.ts index 7bdca5ae8b..138457c65e 100644 --- a/packages/backend-core/src/db/tests/utils.spec.ts +++ b/packages/backend-core/src/db/tests/utils.spec.ts @@ -1,19 +1,13 @@ -import { generator, DBTestConfiguration, testEnv } from "../../../tests" import { getDevelopmentAppID, getProdAppID, isDevAppID, isProdAppID, } from "../conversions" -import { generateAppID, getPlatformUrl, getScopedConfig } from "../utils" -import * as context from "../../context" -import { Config } from "../../constants" -import env from "../../environment" +import { generateAppID } from "../utils" describe("utils", () => { - const config = new DBTestConfiguration() - - describe("app ID manipulation", () => { + describe("generateAppID", () => { function getID() { const appId = generateAppID() const split = appId.split("_") @@ -66,127 +60,4 @@ describe("utils", () => { expect(isProdAppID(devAppId)).toEqual(false) }) }) - - const DEFAULT_URL = "http://localhost:10000" - const ENV_URL = "http://env.com" - - const setDbPlatformUrl = async (dbUrl: string) => { - const db = context.getGlobalDB() - await db.put({ - _id: "config_settings", - type: Config.SETTINGS, - config: { - platformUrl: dbUrl, - }, - }) - } - - const clearSettingsConfig = async () => { - await config.doInTenant(async () => { - const db = context.getGlobalDB() - try { - const config = await db.get("config_settings") - await db.remove("config_settings", config._rev) - } catch (e: any) { - if (e.status !== 404) { - throw e - } - } - }) - } - - describe("getPlatformUrl", () => { - describe("self host", () => { - beforeEach(async () => { - testEnv.selfHosted() - await clearSettingsConfig() - }) - - it("gets the default url", async () => { - await config.doInTenant(async () => { - const url = await getPlatformUrl() - expect(url).toBe(DEFAULT_URL) - }) - }) - - it("gets the platform url from the environment", async () => { - await config.doInTenant(async () => { - env._set("PLATFORM_URL", ENV_URL) - const url = await getPlatformUrl() - expect(url).toBe(ENV_URL) - }) - }) - - it("gets the platform url from the database", async () => { - await config.doInTenant(async () => { - const dbUrl = generator.url() - await setDbPlatformUrl(dbUrl) - const url = await getPlatformUrl() - expect(url).toBe(dbUrl) - }) - }) - }) - - describe("cloud", () => { - const TENANT_AWARE_URL = `http://${config.tenantId}.env.com` - - beforeEach(async () => { - testEnv.cloudHosted() - testEnv.multiTenant() - - env._set("PLATFORM_URL", ENV_URL) - await clearSettingsConfig() - }) - - it("gets the platform url from the environment without tenancy", async () => { - await config.doInTenant(async () => { - const url = await getPlatformUrl({ tenantAware: false }) - expect(url).toBe(ENV_URL) - }) - }) - - it("gets the platform url from the environment with tenancy", async () => { - await config.doInTenant(async () => { - const url = await getPlatformUrl() - expect(url).toBe(TENANT_AWARE_URL) - }) - }) - - it("never gets the platform url from the database", async () => { - await config.doInTenant(async () => { - await setDbPlatformUrl(generator.url()) - const url = await getPlatformUrl() - expect(url).toBe(TENANT_AWARE_URL) - }) - }) - }) - }) - - describe("getScopedConfig", () => { - describe("settings config", () => { - beforeEach(async () => { - env._set("SELF_HOSTED", 1) - env._set("PLATFORM_URL", "") - await clearSettingsConfig() - }) - - it("returns the platform url with an existing config", async () => { - await config.doInTenant(async () => { - const dbUrl = generator.url() - await setDbPlatformUrl(dbUrl) - const db = context.getGlobalDB() - const config = await getScopedConfig(db, { type: Config.SETTINGS }) - expect(config.platformUrl).toBe(dbUrl) - }) - }) - - it("returns the platform url without an existing config", async () => { - await config.doInTenant(async () => { - const db = context.getGlobalDB() - const config = await getScopedConfig(db, { type: Config.SETTINGS }) - expect(config.platformUrl).toBe(DEFAULT_URL) - }) - }) - }) - }) }) diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 233d044eaa..76c52d08ad 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -9,12 +9,11 @@ import { InternalTable, APP_PREFIX, } from "../constants" -import { getTenantId, getGlobalDB, getGlobalDBName } from "../context" +import { getTenantId, getGlobalDBName } from "../context" import { doWithDB, directCouchAllDbs } from "./db" import { getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" -import * as events from "../events" -import { App, Database, ConfigType, isSettingsConfig } from "@budibase/types" +import { App, Database } from "@budibase/types" /** * Generates a new app ID. @@ -366,6 +365,16 @@ export async function getAllApps({ } } +export async function getAppsByIDs(appIds: string[]) { + const settled = await Promise.allSettled( + appIds.map(appId => getAppMetadata(appId)) + ) + // have to list the apps which exist, some may have been deleted + return settled + .filter(promise => promise.status === "fulfilled") + .map(promise => (promise as PromiseFulfilledResult).value) +} + /** * Utility function for getAllApps but filters to production apps only. */ @@ -382,6 +391,16 @@ export async function getDevAppIDs() { return apps.filter((id: any) => isDevAppID(id)) } +export function isSameAppID( + appId1: string | undefined, + appId2: string | undefined +) { + if (appId1 == undefined || appId2 == undefined) { + return false + } + return getProdAppID(appId1) === getProdAppID(appId2) +} + export async function dbExists(dbName: any) { return doWithDB( dbName, @@ -392,32 +411,6 @@ export async function dbExists(dbName: any) { ) } -/** - * Generates a new configuration ID. - * @returns {string} The new configuration ID which the config doc can be stored under. - */ -export const generateConfigID = ({ type, workspace, user }: any) => { - const scope = [type, workspace, user].filter(Boolean).join(SEPARATOR) - - return `${DocumentType.CONFIG}${SEPARATOR}${scope}` -} - -/** - * Gets parameters for retrieving configurations. - */ -export const getConfigParams = ( - { type, workspace, user }: any, - otherProps = {} -) => { - const scope = [type, workspace, user].filter(Boolean).join(SEPARATOR) - - return { - ...otherProps, - startkey: `${DocumentType.CONFIG}${SEPARATOR}${scope}`, - endkey: `${DocumentType.CONFIG}${SEPARATOR}${scope}${UNICODE_MAX}`, - } -} - /** * Generates a new dev info document ID - this is scoped to a user. * @returns {string} The new dev info ID which info for dev (like api key) can be stored under. @@ -441,109 +434,6 @@ export const getPluginParams = (pluginId?: string | null, otherProps = {}) => { return getDocParams(DocumentType.PLUGIN, pluginId, otherProps) } -/** - * Returns the most granular configuration document from the DB based on the type, workspace and userID passed. - * @param {Object} db - db instance to query - * @param {Object} scopes - the type, workspace and userID scopes of the configuration. - * @returns The most granular configuration document based on the scope. - */ -export const getScopedFullConfig = async function ( - db: any, - { type, user, workspace }: any -) { - const response = await db.allDocs( - getConfigParams( - { type, user, workspace }, - { - include_docs: true, - } - ) - ) - - function determineScore(row: any) { - const config = row.doc - - // Config is specific to a user and a workspace - if (config._id.includes(generateConfigID({ type, user, workspace }))) { - return 4 - } else if (config._id.includes(generateConfigID({ type, user }))) { - // Config is specific to a user only - return 3 - } else if (config._id.includes(generateConfigID({ type, workspace }))) { - // Config is specific to a workspace only - return 2 - } else if (config._id.includes(generateConfigID({ type }))) { - // Config is specific to a type only - return 1 - } - return 0 - } - - // Find the config with the most granular scope based on context - let scopedConfig = response.rows.sort( - (a: any, b: any) => determineScore(a) - determineScore(b) - )[0] - - // custom logic for settings doc - if (type === ConfigType.SETTINGS) { - if (!scopedConfig || !scopedConfig.doc) { - // defaults - scopedConfig = { - doc: { - _id: generateConfigID({ type, user, workspace }), - type: ConfigType.SETTINGS, - config: { - platformUrl: await getPlatformUrl({ tenantAware: true }), - analyticsEnabled: await events.analytics.enabled(), - }, - }, - } - } - - // will always be true - use assertion function to get type access - if (isSettingsConfig(scopedConfig.doc)) { - // overrides affected by environment - scopedConfig.doc.config.platformUrl = await getPlatformUrl({ - tenantAware: true, - }) - scopedConfig.doc.config.analyticsEnabled = - await events.analytics.enabled() - } - } - - return scopedConfig && scopedConfig.doc -} - -export const getPlatformUrl = async (opts = { tenantAware: true }) => { - let platformUrl = env.PLATFORM_URL || "http://localhost:10000" - - if (!env.SELF_HOSTED && env.MULTI_TENANCY && opts.tenantAware) { - // cloud and multi tenant - add the tenant to the default platform url - const tenantId = getTenantId() - if (!platformUrl.includes("localhost:")) { - platformUrl = platformUrl.replace("://", `://${tenantId}.`) - } - } else if (env.SELF_HOSTED) { - const db = getGlobalDB() - // get the doc directly instead of with getScopedConfig to prevent loop - let settings - try { - settings = await db.get(generateConfigID({ type: ConfigType.SETTINGS })) - } catch (e: any) { - if (e.status !== 404) { - throw e - } - } - - // self hosted - check for platform url override - if (settings && settings.config && settings.config.platformUrl) { - platformUrl = settings.config.platformUrl - } - } - - return platformUrl -} - export function pagination( data: any[], pageSize: number, @@ -577,8 +467,3 @@ export function pagination( nextPage, } } - -export async function getScopedConfig(db: any, params: any) { - const configDoc = await getScopedFullConfig(db, params) - return configDoc && configDoc.config ? configDoc.config : configDoc -} diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index ed7a161160..cd7bcca11d 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -28,6 +28,8 @@ const DefaultBucketName = { PLUGINS: "plugins", } +const selfHosted = !!parseInt(process.env.SELF_HOSTED || "") + const environment = { isTest, isJest, @@ -58,7 +60,7 @@ const environment = { process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app", ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY || "", DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL, - SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED || ""), + SELF_HOSTED: selfHosted, COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, PLATFORM_URL: process.env.PLATFORM_URL || "", POSTHOG_TOKEN: process.env.POSTHOG_TOKEN, @@ -84,6 +86,22 @@ const environment = { DEPLOYMENT_ENVIRONMENT: process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose", ENABLE_4XX_HTTP_LOGGING: process.env.ENABLE_4XX_HTTP_LOGGING || true, + ENABLE_AUDIT_LOG_IP_ADDR: process.env.ENABLE_AUDIT_LOG_IP_ADDR, + // smtp + SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED, + SMTP_USER: process.env.SMTP_USER, + SMTP_PASSWORD: process.env.SMTP_PASSWORD, + SMTP_HOST: process.env.SMTP_HOST, + SMTP_PORT: parseInt(process.env.SMTP_PORT || ""), + SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS, + /** + * Enable to allow an admin user to login using a password. + * This can be useful to prevent lockout when configuring SSO. + * However, this should be turned OFF by default for security purposes. + */ + ENABLE_SSO_MAINTENANCE_MODE: selfHosted + ? process.env.ENABLE_SSO_MAINTENANCE_MODE + : false, _set(key: any, value: any) { process.env[key] = value // @ts-ignore diff --git a/packages/backend-core/src/events/analytics.ts b/packages/backend-core/src/events/analytics.ts index 7fbc6d9c2b..dcfd6d5104 100644 --- a/packages/backend-core/src/events/analytics.ts +++ b/packages/backend-core/src/events/analytics.ts @@ -1,55 +1,6 @@ -import env from "../environment" -import * as context from "../context" -import * as dbUtils from "../db/utils" -import { Config } from "../constants" -import { withCache, TTL, CacheKey } from "../cache" +import * as configs from "../configs" +// wrapper utility function export const enabled = async () => { - // cloud - always use the environment variable - if (!env.SELF_HOSTED) { - return !!env.ENABLE_ANALYTICS - } - - // self host - prefer the settings doc - // use cache as events have high throughput - const enabledInDB = await withCache( - CacheKey.ANALYTICS_ENABLED, - TTL.ONE_DAY, - async () => { - const settings = await getSettingsDoc() - - // need to do explicit checks in case the field is not set - if (settings?.config?.analyticsEnabled === false) { - return false - } else if (settings?.config?.analyticsEnabled === true) { - return true - } - } - ) - - if (enabledInDB !== undefined) { - return enabledInDB - } - - // fallback to the environment variable - // explicitly check for 0 or false here, undefined or otherwise is treated as true - const envEnabled: any = env.ENABLE_ANALYTICS - if (envEnabled === 0 || envEnabled === false) { - return false - } else { - return true - } -} - -const getSettingsDoc = async () => { - const db = context.getGlobalDB() - let settings - try { - settings = await db.get(dbUtils.generateConfigID({ type: Config.SETTINGS })) - } catch (e: any) { - if (e.status !== 404) { - throw e - } - } - return settings + return configs.analyticsEnabled() } diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index 01928221a0..c2f7cf66ec 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -1,4 +1,4 @@ -import { Event } from "@budibase/types" +import { Event, AuditedEventFriendlyName } from "@budibase/types" import { processors } from "./processors" import identification from "./identification" import * as backfill from "./backfill" diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index 7cade9e14b..9534fb293d 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -10,7 +10,6 @@ import { isCloudAccount, isSSOAccount, TenantGroup, - SettingsConfig, CloudAccount, UserIdentity, InstallationGroup, @@ -19,10 +18,9 @@ import { isSSOUser, } from "@budibase/types" import { processors } from "./processors" -import * as dbUtils from "../db/utils" -import { Config } from "../constants" import { newid } from "../utils" import * as installation from "../installation" +import * as configs from "../configs" import { withCache, TTL, CacheKey } from "../cache/generic" const pkg = require("../../package.json") @@ -89,6 +87,7 @@ const getCurrentIdentity = async (): Promise => { installationId, tenantId, environment, + hostInfo: userContext.hostInfo, } } else { throw new Error("Unknown identity type") @@ -270,9 +269,7 @@ const getUniqueTenantId = async (tenantId: string): Promise => { return context.doInTenant(tenantId, () => { return withCache(CacheKey.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => { const db = context.getGlobalDB() - const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, { - type: Config.SETTINGS, - }) + const config = await configs.getSettingsConfigDoc() let uniqueTenantId: string if (config.config.uniqueTenantId) { diff --git a/packages/backend-core/src/events/processors/AuditLogsProcessor.ts b/packages/backend-core/src/events/processors/AuditLogsProcessor.ts new file mode 100644 index 0000000000..fd68b66871 --- /dev/null +++ b/packages/backend-core/src/events/processors/AuditLogsProcessor.ts @@ -0,0 +1,90 @@ +import { + Event, + Identity, + Group, + IdentityType, + AuditLogQueueEvent, + AuditLogFn, + HostInfo, +} from "@budibase/types" +import { EventProcessor } from "./types" +import { getAppId } from "../../context" +import BullQueue from "bull" +import { createQueue, JobQueue } from "../../queue" +import { isAudited } from "../../utils" +import env from "../../environment" + +export default class AuditLogsProcessor implements EventProcessor { + static auditLogsEnabled = false + static auditLogQueue: BullQueue.Queue + + // can't use constructor as need to return promise + static init(fn: AuditLogFn) { + AuditLogsProcessor.auditLogsEnabled = true + const writeAuditLogs = fn + AuditLogsProcessor.auditLogQueue = createQueue( + JobQueue.AUDIT_LOG + ) + return AuditLogsProcessor.auditLogQueue.process(async job => { + let properties = job.data.properties + if (properties.audited) { + properties = { + ...properties, + ...properties.audited, + } + delete properties.audited + } + + // this feature is disabled by default due to privacy requirements + // in some countries - available as env var in-case it is desired + // in self host deployments + let hostInfo: HostInfo | undefined = {} + if (env.ENABLE_AUDIT_LOG_IP_ADDR) { + hostInfo = job.data.opts.hostInfo + } + + await writeAuditLogs(job.data.event, properties, { + userId: job.data.opts.userId, + timestamp: job.data.opts.timestamp, + appId: job.data.opts.appId, + hostInfo, + }) + }) + } + + async processEvent( + event: Event, + identity: Identity, + properties: any, + timestamp?: string + ): Promise { + if (AuditLogsProcessor.auditLogsEnabled && isAudited(event)) { + // only audit log actual events, don't include backfills + const userId = + identity.type === IdentityType.USER ? identity.id : undefined + // add to the event queue, rather than just writing immediately + await AuditLogsProcessor.auditLogQueue.add({ + event, + properties, + opts: { + userId, + timestamp, + appId: getAppId(), + hostInfo: identity.hostInfo, + }, + }) + } + } + + async identify(identity: Identity, timestamp?: string | number) { + // no-op + } + + async identifyGroup(group: Group, timestamp?: string | number) { + // no-op + } + + shutdown(): void { + AuditLogsProcessor.auditLogQueue?.close() + } +} diff --git a/packages/backend-core/src/events/processors/index.ts b/packages/backend-core/src/events/processors/index.ts index 0e75f050db..6646764e47 100644 --- a/packages/backend-core/src/events/processors/index.ts +++ b/packages/backend-core/src/events/processors/index.ts @@ -1,8 +1,19 @@ import AnalyticsProcessor from "./AnalyticsProcessor" import LoggingProcessor from "./LoggingProcessor" +import AuditLogsProcessor from "./AuditLogsProcessor" import Processors from "./Processors" +import { AuditLogFn } from "@budibase/types" export const analyticsProcessor = new AnalyticsProcessor() const loggingProcessor = new LoggingProcessor() +const auditLogsProcessor = new AuditLogsProcessor() -export const processors = new Processors([analyticsProcessor, loggingProcessor]) +export function init(auditingFn: AuditLogFn) { + return AuditLogsProcessor.init(auditingFn) +} + +export const processors = new Processors([ + analyticsProcessor, + loggingProcessor, + auditLogsProcessor, +]) diff --git a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts index 593e5ff082..0dbe70d543 100644 --- a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts @@ -47,6 +47,8 @@ export default class PosthogProcessor implements EventProcessor { return } + properties = this.clearPIIProperties(properties) + properties.version = pkg.version properties.service = env.SERVICE properties.environment = identity.environment @@ -79,6 +81,16 @@ export default class PosthogProcessor implements EventProcessor { this.posthog.capture(payload) } + clearPIIProperties(properties: any) { + if (properties.email) { + delete properties.email + } + if (properties.audited) { + delete properties.audited + } + return properties + } + async identify(identity: Identity, timestamp?: string | number) { const payload: any = { distinctId: identity.id, properties: identity } if (timestamp) { diff --git a/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts index 2c1340d36e..8df4e40bcf 100644 --- a/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts +++ b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts @@ -49,6 +49,25 @@ describe("PosthogProcessor", () => { expect(processor.posthog.capture).toHaveBeenCalledTimes(0) }) + it("removes audited information", async () => { + const processor = new PosthogProcessor("test") + + const identity = newIdentity() + const properties = { + email: "test", + audited: { + name: "test", + }, + } + + await processor.processEvent(Event.USER_CREATED, identity, properties) + expect(processor.posthog.capture).toHaveBeenCalled() + // @ts-ignore + const call = processor.posthog.capture.mock.calls[0][0] + expect(call.properties.audited).toBeUndefined() + expect(call.properties.email).toBeUndefined() + }) + describe("rate limiting", () => { it("sends daily event once in same day", async () => { const processor = new PosthogProcessor("test") diff --git a/packages/backend-core/src/events/publishers/app.ts b/packages/backend-core/src/events/publishers/app.ts index 90da21f3f5..d08d59b5f1 100644 --- a/packages/backend-core/src/events/publishers/app.ts +++ b/packages/backend-core/src/events/publishers/app.ts @@ -19,6 +19,9 @@ const created = async (app: App, timestamp?: string | number) => { const properties: AppCreatedEvent = { appId: app.appId, version: app.version, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_CREATED, properties, timestamp) } @@ -27,6 +30,9 @@ async function updated(app: App) { const properties: AppUpdatedEvent = { appId: app.appId, version: app.version, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_UPDATED, properties) } @@ -34,6 +40,9 @@ async function updated(app: App) { async function deleted(app: App) { const properties: AppDeletedEvent = { appId: app.appId, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_DELETED, properties) } @@ -41,6 +50,9 @@ async function deleted(app: App) { async function published(app: App, timestamp?: string | number) { const properties: AppPublishedEvent = { appId: app.appId, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_PUBLISHED, properties, timestamp) } @@ -48,6 +60,9 @@ async function published(app: App, timestamp?: string | number) { async function unpublished(app: App) { const properties: AppUnpublishedEvent = { appId: app.appId, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_UNPUBLISHED, properties) } @@ -55,6 +70,9 @@ async function unpublished(app: App) { async function fileImported(app: App) { const properties: AppFileImportedEvent = { appId: app.appId, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_FILE_IMPORTED, properties) } @@ -63,6 +81,9 @@ async function templateImported(app: App, templateKey: string) { const properties: AppTemplateImportedEvent = { appId: app.appId, templateKey, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_TEMPLATE_IMPORTED, properties) } @@ -76,6 +97,9 @@ async function versionUpdated( appId: app.appId, currentVersion, updatedToVersion, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_VERSION_UPDATED, properties) } @@ -89,6 +113,9 @@ async function versionReverted( appId: app.appId, currentVersion, revertedToVersion, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_VERSION_REVERTED, properties) } @@ -96,6 +123,9 @@ async function versionReverted( async function reverted(app: App) { const properties: AppRevertedEvent = { appId: app.appId, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_REVERTED, properties) } @@ -103,6 +133,9 @@ async function reverted(app: App) { async function exported(app: App) { const properties: AppExportedEvent = { appId: app.appId, + audited: { + name: app.name, + }, } await publishEvent(Event.APP_EXPORTED, properties) } diff --git a/packages/backend-core/src/events/publishers/auditLog.ts b/packages/backend-core/src/events/publishers/auditLog.ts new file mode 100644 index 0000000000..7cfb76147a --- /dev/null +++ b/packages/backend-core/src/events/publishers/auditLog.ts @@ -0,0 +1,26 @@ +import { + Event, + AuditLogSearchParams, + AuditLogFilteredEvent, + AuditLogDownloadedEvent, +} from "@budibase/types" +import { publishEvent } from "../events" + +async function filtered(search: AuditLogSearchParams) { + const properties: AuditLogFilteredEvent = { + filters: search, + } + await publishEvent(Event.AUDIT_LOGS_FILTERED, properties) +} + +async function downloaded(search: AuditLogSearchParams) { + const properties: AuditLogDownloadedEvent = { + filters: search, + } + await publishEvent(Event.AUDIT_LOGS_DOWNLOADED, properties) +} + +export default { + filtered, + downloaded, +} diff --git a/packages/backend-core/src/events/publishers/auth.ts b/packages/backend-core/src/events/publishers/auth.ts index 4436045599..e275d2dbb0 100644 --- a/packages/backend-core/src/events/publishers/auth.ts +++ b/packages/backend-core/src/events/publishers/auth.ts @@ -12,19 +12,25 @@ import { } from "@budibase/types" import { identification } from ".." -async function login(source: LoginSource) { +async function login(source: LoginSource, email: string) { const identity = await identification.getCurrentIdentity() const properties: LoginEvent = { userId: identity.id, source, + audited: { + email, + }, } await publishEvent(Event.AUTH_LOGIN, properties) } -async function logout() { +async function logout(email?: string) { const identity = await identification.getCurrentIdentity() const properties: LogoutEvent = { userId: identity.id, + audited: { + email, + }, } await publishEvent(Event.AUTH_LOGOUT, properties) } diff --git a/packages/backend-core/src/events/publishers/automation.ts b/packages/backend-core/src/events/publishers/automation.ts index 6eb36ab067..419d4136bd 100644 --- a/packages/backend-core/src/events/publishers/automation.ts +++ b/packages/backend-core/src/events/publishers/automation.ts @@ -18,6 +18,9 @@ async function created(automation: Automation, timestamp?: string | number) { automationId: automation._id as string, triggerId: automation.definition?.trigger?.id, triggerType: automation.definition?.trigger?.stepId, + audited: { + name: automation.name, + }, } await publishEvent(Event.AUTOMATION_CREATED, properties, timestamp) } @@ -38,6 +41,9 @@ async function deleted(automation: Automation) { automationId: automation._id as string, triggerId: automation.definition?.trigger?.id, triggerType: automation.definition?.trigger?.stepId, + audited: { + name: automation.name, + }, } await publishEvent(Event.AUTOMATION_DELETED, properties) } @@ -71,6 +77,9 @@ async function stepCreated( triggerType: automation.definition?.trigger?.stepId, stepId: step.id!, stepType: step.stepId, + audited: { + name: automation.name, + }, } await publishEvent(Event.AUTOMATION_STEP_CREATED, properties, timestamp) } @@ -83,6 +92,9 @@ async function stepDeleted(automation: Automation, step: AutomationStep) { triggerType: automation.definition?.trigger?.stepId, stepId: step.id!, stepType: step.stepId, + audited: { + name: automation.name, + }, } await publishEvent(Event.AUTOMATION_STEP_DELETED, properties) } diff --git a/packages/backend-core/src/events/publishers/backup.ts b/packages/backend-core/src/events/publishers/backup.ts index 12263fe1ff..d7d87f09f1 100644 --- a/packages/backend-core/src/events/publishers/backup.ts +++ b/packages/backend-core/src/events/publishers/backup.ts @@ -13,6 +13,7 @@ async function appBackupRestored(backup: AppBackup) { appId: backup.appId, restoreId: backup._id!, backupCreatedAt: backup.timestamp, + name: backup.name as string, } await publishEvent(Event.APP_BACKUP_RESTORED, properties) @@ -22,13 +23,15 @@ async function appBackupTriggered( appId: string, backupId: string, type: AppBackupType, - trigger: AppBackupTrigger + trigger: AppBackupTrigger, + name: string ) { const properties: AppBackupTriggeredEvent = { appId: appId, backupId, type, trigger, + name, } await publishEvent(Event.APP_BACKUP_TRIGGERED, properties) } diff --git a/packages/backend-core/src/events/publishers/group.ts b/packages/backend-core/src/events/publishers/group.ts index d79920562b..a000b880a2 100644 --- a/packages/backend-core/src/events/publishers/group.ts +++ b/packages/backend-core/src/events/publishers/group.ts @@ -8,12 +8,16 @@ import { GroupUsersAddedEvent, GroupUsersDeletedEvent, GroupAddedOnboardingEvent, + GroupPermissionsEditedEvent, UserGroupRoles, } from "@budibase/types" async function created(group: UserGroup, timestamp?: number) { const properties: GroupCreatedEvent = { groupId: group._id as string, + audited: { + name: group.name, + }, } await publishEvent(Event.USER_GROUP_CREATED, properties, timestamp) } @@ -21,6 +25,9 @@ async function created(group: UserGroup, timestamp?: number) { async function updated(group: UserGroup) { const properties: GroupUpdatedEvent = { groupId: group._id as string, + audited: { + name: group.name, + }, } await publishEvent(Event.USER_GROUP_UPDATED, properties) } @@ -28,6 +35,9 @@ async function updated(group: UserGroup) { async function deleted(group: UserGroup) { const properties: GroupDeletedEvent = { groupId: group._id as string, + audited: { + name: group.name, + }, } await publishEvent(Event.USER_GROUP_DELETED, properties) } @@ -36,6 +46,9 @@ async function usersAdded(count: number, group: UserGroup) { const properties: GroupUsersAddedEvent = { count, groupId: group._id as string, + audited: { + name: group.name, + }, } await publishEvent(Event.USER_GROUP_USERS_ADDED, properties) } @@ -44,6 +57,9 @@ async function usersDeleted(count: number, group: UserGroup) { const properties: GroupUsersDeletedEvent = { count, groupId: group._id as string, + audited: { + name: group.name, + }, } await publishEvent(Event.USER_GROUP_USERS_REMOVED, properties) } @@ -56,9 +72,13 @@ async function createdOnboarding(groupId: string) { await publishEvent(Event.USER_GROUP_ONBOARDING, properties) } -async function permissionsEdited(roles: UserGroupRoles) { - const properties: UserGroupRoles = { - ...roles, +async function permissionsEdited(group: UserGroup) { + const properties: GroupPermissionsEditedEvent = { + permissions: group.roles!, + groupId: group._id as string, + audited: { + name: group.name, + }, } await publishEvent(Event.USER_GROUP_PERMISSIONS_EDITED, properties) } diff --git a/packages/backend-core/src/events/publishers/index.ts b/packages/backend-core/src/events/publishers/index.ts index 34e47b2990..87a34bf3f1 100644 --- a/packages/backend-core/src/events/publishers/index.ts +++ b/packages/backend-core/src/events/publishers/index.ts @@ -21,3 +21,4 @@ export { default as group } from "./group" export { default as plugin } from "./plugin" export { default as backup } from "./backup" export { default as environmentVariable } from "./environmentVariable" +export { default as auditLog } from "./auditLog" diff --git a/packages/backend-core/src/events/publishers/screen.ts b/packages/backend-core/src/events/publishers/screen.ts index 27264b5847..df486029e8 100644 --- a/packages/backend-core/src/events/publishers/screen.ts +++ b/packages/backend-core/src/events/publishers/screen.ts @@ -11,6 +11,9 @@ async function created(screen: Screen, timestamp?: string | number) { layoutId: screen.layoutId, screenId: screen._id as string, roleId: screen.routing.roleId, + audited: { + name: screen.routing?.route, + }, } await publishEvent(Event.SCREEN_CREATED, properties, timestamp) } @@ -20,6 +23,9 @@ async function deleted(screen: Screen) { layoutId: screen.layoutId, screenId: screen._id as string, roleId: screen.routing.roleId, + audited: { + name: screen.routing?.route, + }, } await publishEvent(Event.SCREEN_DELETED, properties) } diff --git a/packages/backend-core/src/events/publishers/table.ts b/packages/backend-core/src/events/publishers/table.ts index d50f4df0e1..dc3200291a 100644 --- a/packages/backend-core/src/events/publishers/table.ts +++ b/packages/backend-core/src/events/publishers/table.ts @@ -13,6 +13,9 @@ import { async function created(table: Table, timestamp?: string | number) { const properties: TableCreatedEvent = { tableId: table._id as string, + audited: { + name: table.name, + }, } await publishEvent(Event.TABLE_CREATED, properties, timestamp) } @@ -20,6 +23,9 @@ async function created(table: Table, timestamp?: string | number) { async function updated(table: Table) { const properties: TableUpdatedEvent = { tableId: table._id as string, + audited: { + name: table.name, + }, } await publishEvent(Event.TABLE_UPDATED, properties) } @@ -27,6 +33,9 @@ async function updated(table: Table) { async function deleted(table: Table) { const properties: TableDeletedEvent = { tableId: table._id as string, + audited: { + name: table.name, + }, } await publishEvent(Event.TABLE_DELETED, properties) } @@ -35,6 +44,9 @@ async function exported(table: Table, format: TableExportFormat) { const properties: TableExportedEvent = { tableId: table._id as string, format, + audited: { + name: table.name, + }, } await publishEvent(Event.TABLE_EXPORTED, properties) } @@ -42,6 +54,9 @@ async function exported(table: Table, format: TableExportFormat) { async function imported(table: Table) { const properties: TableImportedEvent = { tableId: table._id as string, + audited: { + name: table.name, + }, } await publishEvent(Event.TABLE_IMPORTED, properties) } diff --git a/packages/backend-core/src/events/publishers/user.ts b/packages/backend-core/src/events/publishers/user.ts index 1fe50149b5..8dbc494d1e 100644 --- a/packages/backend-core/src/events/publishers/user.ts +++ b/packages/backend-core/src/events/publishers/user.ts @@ -19,6 +19,9 @@ import { async function created(user: User, timestamp?: number) { const properties: UserCreatedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_CREATED, properties, timestamp) } @@ -26,6 +29,9 @@ async function created(user: User, timestamp?: number) { async function updated(user: User) { const properties: UserUpdatedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_UPDATED, properties) } @@ -33,6 +39,9 @@ async function updated(user: User) { async function deleted(user: User) { const properties: UserDeletedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_DELETED, properties) } @@ -40,6 +49,9 @@ async function deleted(user: User) { export async function onboardingComplete(user: User) { const properties: UserOnboardingEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_ONBOARDING_COMPLETE, properties) } @@ -49,6 +61,9 @@ export async function onboardingComplete(user: User) { async function permissionAdminAssigned(user: User, timestamp?: number) { const properties: UserPermissionAssignedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent( Event.USER_PERMISSION_ADMIN_ASSIGNED, @@ -60,6 +75,9 @@ async function permissionAdminAssigned(user: User, timestamp?: number) { async function permissionAdminRemoved(user: User) { const properties: UserPermissionRemovedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_PERMISSION_ADMIN_REMOVED, properties) } @@ -67,6 +85,9 @@ async function permissionAdminRemoved(user: User) { async function permissionBuilderAssigned(user: User, timestamp?: number) { const properties: UserPermissionAssignedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent( Event.USER_PERMISSION_BUILDER_ASSIGNED, @@ -78,20 +99,30 @@ async function permissionBuilderAssigned(user: User, timestamp?: number) { async function permissionBuilderRemoved(user: User) { const properties: UserPermissionRemovedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_PERMISSION_BUILDER_REMOVED, properties) } // INVITE -async function invited() { - const properties: UserInvitedEvent = {} +async function invited(email: string) { + const properties: UserInvitedEvent = { + audited: { + email, + }, + } await publishEvent(Event.USER_INVITED, properties) } async function inviteAccepted(user: User) { const properties: UserInviteAcceptedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_INVITED_ACCEPTED, properties) } @@ -101,6 +132,9 @@ async function inviteAccepted(user: User) { async function passwordForceReset(user: User) { const properties: UserPasswordForceResetEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_PASSWORD_FORCE_RESET, properties) } @@ -108,6 +142,9 @@ async function passwordForceReset(user: User) { async function passwordUpdated(user: User) { const properties: UserPasswordUpdatedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_PASSWORD_UPDATED, properties) } @@ -115,6 +152,9 @@ async function passwordUpdated(user: User) { async function passwordResetRequested(user: User) { const properties: UserPasswordResetRequestedEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_PASSWORD_RESET_REQUESTED, properties) } @@ -122,6 +162,9 @@ async function passwordResetRequested(user: User) { async function passwordReset(user: User) { const properties: UserPasswordResetEvent = { userId: user._id as string, + audited: { + email: user.email, + }, } await publishEvent(Event.USER_PASSWORD_RESET, properties) } diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index d507d8175f..48569548e3 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -1,3 +1,4 @@ +export * as configs from "./configs" export * as events from "./events" export * as migrations from "./migrations" export * as users from "./users" @@ -20,11 +21,11 @@ export * as context from "./context" export * as cache from "./cache" export * as objectStore from "./objectStore" export * as redis from "./redis" -export * as locks from "./redis/redlock" +export * as locks from "./redis/redlockImpl" export * as utils from "./utils" export * as errors from "./errors" export { default as env } from "./environment" - +export { SearchParams } from "./db" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal // circular dependencies diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 4bb2aaba76..d7e6346b3f 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -8,7 +8,7 @@ import { getGlobalDB, doInTenant } from "../context" import { decrypt } from "../security/encryption" import * as identity from "../context/identity" import env from "../environment" -import { BBContext, EndpointMatcher } from "@budibase/types" +import { Ctx, EndpointMatcher } from "@budibase/types" const ONE_MINUTE = env.SESSION_UPDATE_PERIOD ? parseInt(env.SESSION_UPDATE_PERIOD) @@ -73,7 +73,7 @@ export default function ( } ) { const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : [] - return async (ctx: BBContext | any, next: any) => { + return async (ctx: Ctx | any, next: any) => { let publicEndpoint = false const version = ctx.request.headers[Header.API_VER] // the path is not authenticated @@ -115,7 +115,8 @@ export default function ( authenticated = true } catch (err: any) { authenticated = false - console.error("Auth Error", err?.message || err) + console.error(`Auth Error: ${err.message}`) + console.error(err) // remove the cookie as the user does not exist anymore clearCookie(ctx, Cookie.Auth) } @@ -148,7 +149,7 @@ export default function ( finalise(ctx, { authenticated, user, internal, version, publicEndpoint }) if (user && user.email) { - return identity.doInUserContext(user, next) + return identity.doInUserContext(user, ctx, next) } else { return next() } diff --git a/packages/backend-core/src/middleware/errorHandling.ts b/packages/backend-core/src/middleware/errorHandling.ts index 5ac70c33e5..36aff2cdbc 100644 --- a/packages/backend-core/src/middleware/errorHandling.ts +++ b/packages/backend-core/src/middleware/errorHandling.ts @@ -11,6 +11,7 @@ export async function errorHandling(ctx: any, next: any) { if (status > 499 || env.ENABLE_4XX_HTTP_LOGGING) { ctx.log.error(err) + console.trace(err) } const error = errors.getPublicError(err) diff --git a/packages/backend-core/src/middleware/index.ts b/packages/backend-core/src/middleware/index.ts index de609f9a3e..addeac6a1a 100644 --- a/packages/backend-core/src/middleware/index.ts +++ b/packages/backend-core/src/middleware/index.ts @@ -17,4 +17,5 @@ export { default as builderOrAdmin } from "./builderOrAdmin" export { default as builderOnly } from "./builderOnly" export { default as logging } from "./logging" export { default as errorHandling } from "./errorHandling" +export { default as querystringToBody } from "./querystringToBody" export * as joiValidator from "./joi-validator" diff --git a/packages/backend-core/src/middleware/passport/datasource/google.ts b/packages/backend-core/src/middleware/passport/datasource/google.ts index 112f8d2096..ff5224e437 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.ts +++ b/packages/backend-core/src/middleware/passport/datasource/google.ts @@ -1,9 +1,8 @@ import * as google from "../sso/google" -import { Cookie, Config } from "../../../constants" +import { Cookie } from "../../../constants" import { clearCookie, getCookie } from "../../../utils" -import { getScopedConfig, getPlatformUrl, doWithDB } from "../../../db" -import environment from "../../../environment" -import { getGlobalDB } from "../../../context" +import { doWithDB } from "../../../db" +import * as configs from "../../../configs" import { BBContext, Database, SSOProfile } from "@budibase/types" import { ssoSaveUserNoOp } from "../sso/sso" const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy @@ -13,18 +12,11 @@ type Passport = { } async function fetchGoogleCreds() { - // try and get the config from the tenant - const db = getGlobalDB() - const googleConfig = await getScopedConfig(db, { - type: Config.GOOGLE, - }) - // or fall back to env variables - return ( - googleConfig || { - clientID: environment.GOOGLE_CLIENT_ID, - clientSecret: environment.GOOGLE_CLIENT_SECRET, - } - ) + const config = await configs.getGoogleConfig() + if (!config) { + throw new Error("No google configuration found") + } + return config } export async function preAuth( @@ -34,7 +26,7 @@ export async function preAuth( ) { // get the relevant config const googleConfig = await fetchGoogleCreds() - const platformUrl = await getPlatformUrl({ tenantAware: false }) + const platformUrl = await configs.getPlatformUrl({ tenantAware: false }) let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback` const strategy = await google.strategyFactory( @@ -61,7 +53,7 @@ export async function postAuth( ) { // get the relevant config const config = await fetchGoogleCreds() - const platformUrl = await getPlatformUrl({ tenantAware: false }) + const platformUrl = await configs.getPlatformUrl({ tenantAware: false }) let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback` const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth) diff --git a/packages/backend-core/src/middleware/passport/sso/google.ts b/packages/backend-core/src/middleware/passport/sso/google.ts index d26d7d6a8d..ad7593e63d 100644 --- a/packages/backend-core/src/middleware/passport/sso/google.ts +++ b/packages/backend-core/src/middleware/passport/sso/google.ts @@ -2,12 +2,11 @@ import { ssoCallbackUrl } from "../utils" import * as sso from "./sso" import { ConfigType, - GoogleConfig, - Database, SSOProfile, SSOAuthDetails, SSOProviderType, SaveSSOUserFunction, + GoogleInnerConfig, } from "@budibase/types" const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy @@ -45,7 +44,7 @@ export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) { * @returns Dynamically configured Passport Google Strategy */ export async function strategyFactory( - config: GoogleConfig["config"], + config: GoogleInnerConfig, callbackUrl: string, saveUserFn: SaveSSOUserFunction ) { @@ -73,9 +72,6 @@ export async function strategyFactory( } } -export async function getCallbackUrl( - db: Database, - config: { callbackURL?: string } -) { - return ssoCallbackUrl(db, config, ConfigType.GOOGLE) +export async function getCallbackUrl(config: GoogleInnerConfig) { + return ssoCallbackUrl(ConfigType.GOOGLE, config) } diff --git a/packages/backend-core/src/middleware/passport/sso/oidc.ts b/packages/backend-core/src/middleware/passport/sso/oidc.ts index 1fb44b84a3..b6d5eb52e9 100644 --- a/packages/backend-core/src/middleware/passport/sso/oidc.ts +++ b/packages/backend-core/src/middleware/passport/sso/oidc.ts @@ -4,7 +4,6 @@ import { ssoCallbackUrl } from "../utils" import { ConfigType, OIDCInnerConfig, - Database, SSOProfile, OIDCStrategyConfiguration, SSOAuthDetails, @@ -157,9 +156,6 @@ export async function fetchStrategyConfig( } } -export async function getCallbackUrl( - db: Database, - config: { callbackURL?: string } -) { - return ssoCallbackUrl(db, config, ConfigType.OIDC) +export async function getCallbackUrl() { + return ssoCallbackUrl(ConfigType.OIDC) } diff --git a/packages/backend-core/src/middleware/passport/utils.ts b/packages/backend-core/src/middleware/passport/utils.ts index 6eb3bc29d1..7e0d3863a0 100644 --- a/packages/backend-core/src/middleware/passport/utils.ts +++ b/packages/backend-core/src/middleware/passport/utils.ts @@ -1,6 +1,6 @@ -import { isMultiTenant, getTenantId } from "../../context" -import { getScopedConfig } from "../../db" -import { ConfigType, Database } from "@budibase/types" +import { getTenantId, isMultiTenant } from "../../context" +import * as configs from "../../configs" +import { ConfigType, GoogleInnerConfig } from "@budibase/types" /** * Utility to handle authentication errors. @@ -19,17 +19,14 @@ export function authError(done: Function, message: string, err?: any) { } export async function ssoCallbackUrl( - db: Database, - config?: { callbackURL?: string }, - type?: ConfigType + type: ConfigType, + config?: GoogleInnerConfig ) { // incase there is a callback URL from before - if (config && config.callbackURL) { - return config.callbackURL + if (config && (config as GoogleInnerConfig).callbackURL) { + return (config as GoogleInnerConfig).callbackURL as string } - const publicConfig = await getScopedConfig(db, { - type: ConfigType.SETTINGS, - }) + const settingsConfig = await configs.getSettingsConfig() let callbackUrl = `/api/global/auth` if (isMultiTenant()) { @@ -37,5 +34,5 @@ export async function ssoCallbackUrl( } callbackUrl += `/${type}/callback` - return `${publicConfig.platformUrl}${callbackUrl}` + return `${settingsConfig.platformUrl}${callbackUrl}` } diff --git a/packages/backend-core/src/middleware/querystringToBody.ts b/packages/backend-core/src/middleware/querystringToBody.ts new file mode 100644 index 0000000000..b6f109231a --- /dev/null +++ b/packages/backend-core/src/middleware/querystringToBody.ts @@ -0,0 +1,28 @@ +import { Ctx } from "@budibase/types" + +/** + * Expects a standard "query" query string property which is the JSON body + * of the request, which has to be sent via query string due to the requirement + * of making an endpoint a GET request e.g. downloading a file stream. + */ +export default function (ctx: Ctx, next: any) { + const queryString = ctx.request.query?.query as string | undefined + if (ctx.request.method.toLowerCase() !== "get") { + ctx.throw( + 500, + "Query to download middleware can only be used for get requests." + ) + } + if (!queryString) { + return next() + } + const decoded = decodeURIComponent(queryString) + let json + try { + json = JSON.parse(decoded) + } catch (err) { + return next() + } + ctx.request.body = json + return next() +} diff --git a/packages/backend-core/src/platform/tenants.ts b/packages/backend-core/src/platform/tenants.ts index b9f946a735..b6bc3410d8 100644 --- a/packages/backend-core/src/platform/tenants.ts +++ b/packages/backend-core/src/platform/tenants.ts @@ -1,7 +1,7 @@ import { StaticDatabases } from "../constants" import { getPlatformDB } from "./platformDb" import { LockName, LockOptions, LockType, Tenants } from "@budibase/types" -import * as locks from "../redis/redlock" +import * as locks from "../redis/redlockImpl" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts index e8323dacb8..9261ed1176 100644 --- a/packages/backend-core/src/queue/constants.ts +++ b/packages/backend-core/src/queue/constants.ts @@ -1,4 +1,5 @@ export enum JobQueue { AUTOMATION = "automationQueue", APP_BACKUP = "appBackupQueue", + AUDIT_LOG = "auditLogQueue", } diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 8e1fc1fbf3..c57ebafb1f 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -40,8 +40,10 @@ export function createQueue( } export async function shutdown() { - if (QUEUES.length) { + if (cleanupInterval) { clearInterval(cleanupInterval) + } + if (QUEUES.length) { for (let queue of QUEUES) { await queue.close() } diff --git a/packages/backend-core/src/redis/index.ts b/packages/backend-core/src/redis/index.ts index 5bf2c65c39..6585d6e4fa 100644 --- a/packages/backend-core/src/redis/index.ts +++ b/packages/backend-core/src/redis/index.ts @@ -3,4 +3,4 @@ export { default as Client } from "./redis" export * as utils from "./utils" export * as clients from "./init" -export * as locks from "./redlock" +export * as locks from "./redlockImpl" diff --git a/packages/backend-core/src/redis/redlock.ts b/packages/backend-core/src/redis/redlockImpl.ts similarity index 100% rename from packages/backend-core/src/redis/redlock.ts rename to packages/backend-core/src/redis/redlockImpl.ts diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index ad226e29d8..8963f7c141 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -11,14 +11,38 @@ import { BulkDocsResponse, User } from "@budibase/types" import { getGlobalDB } from "./context" import * as context from "./context" -export const bulkGetGlobalUsersById = async (userIds: string[]) => { +type GetOpts = { cleanup?: boolean } + +function removeUserPassword(users: User | User[]) { + if (Array.isArray(users)) { + return users.map(user => { + if (user) { + delete user.password + return user + } + }) + } else if (users) { + delete users.password + return users + } + return users +} + +export const bulkGetGlobalUsersById = async ( + userIds: string[], + opts?: GetOpts +) => { const db = getGlobalDB() - return ( + let users = ( await db.allDocs({ keys: userIds, include_docs: true, }) ).rows.map(row => row.doc) as User[] + if (opts?.cleanup) { + users = removeUserPassword(users) as User[] + } + return users } export const bulkUpdateGlobalUsers = async (users: User[]) => { @@ -26,18 +50,22 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => { return (await db.bulkDocs(users)) as BulkDocsResponse } -export async function getById(id: string): Promise { +export async function getById(id: string, opts?: GetOpts): Promise { const db = context.getGlobalDB() - return db.get(id) + let user = await db.get(id) + if (opts?.cleanup) { + user = removeUserPassword(user) + } + return user } /** * Given an email address this will use a view to search through * all the users to find one with this email address. - * @param {string} email the email to lookup the user by. */ export const getGlobalUserByEmail = async ( - email: String + email: String, + opts?: GetOpts ): Promise => { if (email == null) { throw "Must supply an email address to view" @@ -53,10 +81,19 @@ export const getGlobalUserByEmail = async ( throw new Error(`Multiple users found with email address: ${email}`) } - return response + let user = response as User + if (opts?.cleanup) { + user = removeUserPassword(user) as User + } + + return user } -export const searchGlobalUsersByApp = async (appId: any, opts: any) => { +export const searchGlobalUsersByApp = async ( + appId: any, + opts: any, + getOpts?: GetOpts +) => { if (typeof appId !== "string") { throw new Error("Must provide a string based app ID") } @@ -69,7 +106,11 @@ export const searchGlobalUsersByApp = async (appId: any, opts: any) => { if (!response) { response = [] } - return Array.isArray(response) ? response : [response] + let users: User[] = Array.isArray(response) ? response : [response] + if (getOpts?.cleanup) { + users = removeUserPassword(users) as User[] + } + return users } /* @@ -121,7 +162,11 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => { /** * Performs a starts with search on the global email view. */ -export const searchGlobalUsersByEmail = async (email: string, opts: any) => { +export const searchGlobalUsersByEmail = async ( + email: string, + opts: any, + getOpts?: GetOpts +) => { if (typeof email !== "string") { throw new Error("Must provide a string to search by") } @@ -136,5 +181,9 @@ export const searchGlobalUsersByEmail = async (email: string, opts: any) => { if (!response) { response = [] } - return Array.isArray(response) ? response : [response] + let users: User[] = Array.isArray(response) ? response : [response] + if (getOpts?.cleanup) { + users = removeUserPassword(users) as User[] + } + return users } diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 3731e134ad..3efd40ca80 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -10,7 +10,13 @@ import { import env from "../environment" import * as tenancy from "../tenancy" import * as context from "../context" -import { App, Ctx, TenantResolutionStrategy } from "@budibase/types" +import { + App, + AuditedEventFriendlyName, + Ctx, + Event, + TenantResolutionStrategy, +} from "@budibase/types" import { SetOption } from "cookies" const jwt = require("jsonwebtoken") @@ -217,3 +223,7 @@ export async function getBuildersCount() { export function timeout(timeMs: number) { return new Promise(resolve => setTimeout(resolve, timeMs)) } + +export function isAudited(event: Event) { + return !!AuditedEventFriendlyName[event] +} diff --git a/packages/backend-core/tests/utilities/mocks/licenses.ts b/packages/backend-core/tests/utilities/mocks/licenses.ts index e374612f5f..2ca41616e4 100644 --- a/packages/backend-core/tests/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/utilities/mocks/licenses.ts @@ -70,6 +70,10 @@ export const useBackups = () => { return useFeature(Feature.APP_BACKUPS) } +export const useEnforceableSSO = () => { + return useFeature(Feature.ENFORCEABLE_SSO) +} + export const useGroups = () => { return useFeature(Feature.USER_GROUPS) } @@ -78,6 +82,10 @@ export const useEnvironmentVariables = () => { return useFeature(Feature.ENVIRONMENT_VARIABLES) } +export const useAuditLogs = () => { + return useFeature(Feature.AUDIT_LOGS) +} + // QUOTAS export const setAutomationLogsQuota = (value: number) => { diff --git a/packages/backend-core/tests/utilities/structures/generator.ts b/packages/backend-core/tests/utilities/structures/generator.ts new file mode 100644 index 0000000000..51567b152e --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/generator.ts @@ -0,0 +1,2 @@ +import Chance from "chance" +export const generator = new Chance() diff --git a/packages/backend-core/tests/utilities/structures/index.ts b/packages/backend-core/tests/utilities/structures/index.ts index d0073ba851..ca77f476d0 100644 --- a/packages/backend-core/tests/utilities/structures/index.ts +++ b/packages/backend-core/tests/utilities/structures/index.ts @@ -1,8 +1,4 @@ export * from "./common" - -import Chance from "chance" -export const generator = new Chance() - export * as accounts from "./accounts" export * as apps from "./apps" export * as db from "./db" @@ -12,3 +8,4 @@ export * as plugins from "./plugins" export * as sso from "./sso" export * as tenant from "./tenants" export * as users from "./users" +export { generator } from "./generator" diff --git a/packages/backend-core/tests/utilities/structures/shared.ts b/packages/backend-core/tests/utilities/structures/shared.ts new file mode 100644 index 0000000000..de0e19486c --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/shared.ts @@ -0,0 +1,19 @@ +import { User } from "@budibase/types" +import { generator } from "./generator" +import { uuid } from "./common" + +export const newEmail = () => { + return `${uuid()}@test.com` +} + +export const user = (userProps?: any): User => { + return { + email: newEmail(), + password: "test", + roles: { app_test: "admin" }, + firstName: generator.first(), + lastName: generator.last(), + pictureUrl: "http://test.com", + ...userProps, + } +} diff --git a/packages/backend-core/tests/utilities/structures/sso.ts b/packages/backend-core/tests/utilities/structures/sso.ts index ad5e8e87ef..a5957c9233 100644 --- a/packages/backend-core/tests/utilities/structures/sso.ts +++ b/packages/backend-core/tests/utilities/structures/sso.ts @@ -8,8 +8,36 @@ import { SSOProviderType, User, } from "@budibase/types" -import { uuid, generator, users, email } from "./index" +import { generator } from "./generator" +import { uuid, email } from "./common" +import * as shared from "./shared" import _ from "lodash" +import { user } from "./shared" + +export function authDetails(userDoc?: User): SSOAuthDetails { + if (!userDoc) { + userDoc = user() + } + + const userId = userDoc._id || uuid() + const provider = generator.string() + + const profile = ssoProfile(userDoc) + profile.provider = provider + profile.id = userId + + return { + email: userDoc.email, + oauth2: { + refreshToken: generator.string(), + accessToken: generator.string(), + }, + profile, + provider, + providerType: providerType(), + userId, + } +} export function providerType(): SSOProviderType { return _.sample(Object.values(SSOProviderType)) as SSOProviderType @@ -17,7 +45,7 @@ export function providerType(): SSOProviderType { export function ssoProfile(user?: User): SSOProfile { if (!user) { - user = users.user() + user = shared.user() } return { id: user._id!, @@ -33,31 +61,6 @@ export function ssoProfile(user?: User): SSOProfile { } } -export function authDetails(user?: User): SSOAuthDetails { - if (!user) { - user = users.user() - } - - const userId = user._id || uuid() - const provider = generator.string() - - const profile = ssoProfile(user) - profile.provider = provider - profile.id = userId - - return { - email: user.email, - oauth2: { - refreshToken: generator.string(), - accessToken: generator.string(), - }, - profile, - provider, - providerType: providerType(), - userId, - } -} - // OIDC export function oidcConfig(): OIDCInnerConfig { @@ -69,6 +72,7 @@ export function oidcConfig(): OIDCInnerConfig { configUrl: "http://someconfigurl", clientID: generator.string(), clientSecret: generator.string(), + scopes: [], } } diff --git a/packages/backend-core/tests/utilities/structures/users.ts b/packages/backend-core/tests/utilities/structures/users.ts index 332c27ca12..7a6b4f0d80 100644 --- a/packages/backend-core/tests/utilities/structures/users.ts +++ b/packages/backend-core/tests/utilities/structures/users.ts @@ -1,29 +1,13 @@ -import { generator } from "../" import { AdminUser, BuilderUser, SSOAuthDetails, SSOUser, - User, } from "@budibase/types" -import { v4 as uuid } from "uuid" -import * as sso from "./sso" +import { user } from "./shared" +import { authDetails } from "./sso" -export const newEmail = () => { - return `${uuid()}@test.com` -} - -export const user = (userProps?: any): User => { - return { - email: newEmail(), - password: "test", - roles: { app_test: "admin" }, - firstName: generator.first(), - lastName: generator.last(), - pictureUrl: "http://test.com", - ...userProps, - } -} +export { user, newEmail } from "./shared" export const adminUser = (userProps?: any): AdminUser => { return { @@ -53,7 +37,7 @@ export function ssoUser( delete base.password if (!opts.details) { - opts.details = sso.authDetails(base) + opts.details = authDetails(base) } return { diff --git a/packages/backend-core/tests/utilities/testContainerUtils.ts b/packages/backend-core/tests/utilities/testContainerUtils.ts index 11c5fca806..f6c702f7ef 100644 --- a/packages/backend-core/tests/utilities/testContainerUtils.ts +++ b/packages/backend-core/tests/utilities/testContainerUtils.ts @@ -1,3 +1,31 @@ +import { execSync } from "child_process" + +let dockerPsResult: string | undefined + +function formatDockerPsResult(serverName: string, port: number) { + const lines = dockerPsResult?.split("\n") + let first = true + if (!lines) { + return null + } + for (let line of lines) { + if (first) { + first = false + continue + } + let toLookFor = serverName.split("-service")[0] + if (!line.includes(toLookFor)) { + continue + } + const regex = new RegExp(`0.0.0.0:([0-9]*)->${port}`, "g") + const found = line.match(regex) + if (found) { + return found[0].split(":")[1].split("->")[0] + } + } + return null +} + function getTestContainerSettings( serverName: string, key: string @@ -14,10 +42,22 @@ function getTestContainerSettings( } function getContainerInfo(containerName: string, port: number) { - const assignedPort = getTestContainerSettings( + let assignedPort = getTestContainerSettings( containerName.toUpperCase(), `PORT_${port}` ) + if (!dockerPsResult) { + try { + const outputBuffer = execSync("docker ps") + dockerPsResult = outputBuffer.toString("utf8") + } catch (err) { + //no-op + } + } + const possiblePort = formatDockerPsResult(containerName, port) + if (possiblePort) { + assignedPort = possiblePort + } const host = getTestContainerSettings(containerName.toUpperCase(), "IP") return { port: assignedPort, @@ -39,12 +79,15 @@ function getRedisConfig() { } export function setupEnv(...envs: any[]) { + const couch = getCouchConfig(), + minio = getCouchConfig(), + redis = getRedisConfig() const configs = [ - { key: "COUCH_DB_PORT", value: getCouchConfig().port }, - { key: "COUCH_DB_URL", value: getCouchConfig().url }, - { key: "MINIO_PORT", value: getMinioConfig().port }, - { key: "MINIO_URL", value: getMinioConfig().url }, - { key: "REDIS_URL", value: getRedisConfig().url }, + { key: "COUCH_DB_PORT", value: couch.port }, + { key: "COUCH_DB_URL", value: couch.url }, + { key: "MINIO_PORT", value: minio.port }, + { key: "MINIO_URL", value: minio.url }, + { key: "REDIS_URL", value: redis.url }, ] for (const config of configs.filter(x => !!x.value)) { diff --git a/packages/bbui/package.json b/packages/bbui/package.json index b697a532ad..ebf2790cb5 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.3.18-alpha.12", + "version": "2.3.18-alpha.14", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/string-templates": "2.3.18-alpha.12", + "@budibase/string-templates": "2.3.18-alpha.14", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", diff --git a/packages/bbui/src/Form/Core/Multiselect.svelte b/packages/bbui/src/Form/Core/Multiselect.svelte index d6c4dc23ac..8860fcbf0b 100644 --- a/packages/bbui/src/Form/Core/Multiselect.svelte +++ b/packages/bbui/src/Form/Core/Multiselect.svelte @@ -14,6 +14,8 @@ export let autocomplete = false export let sort = false export let autoWidth = false + export let fetchTerm = null + export let customPopoverHeight const dispatch = createEventDispatcher() @@ -83,10 +85,12 @@ {options} isPlaceholder={!value?.length} {autocomplete} + bind:fetchTerm {isOptionSelected} {getOptionLabel} {getOptionValue} onSelectOption={toggleOption} {sort} {autoWidth} + {customPopoverHeight} /> diff --git a/packages/bbui/src/Form/Core/Picker.svelte b/packages/bbui/src/Form/Core/Picker.svelte index 32cfcf3310..5cef0f9213 100644 --- a/packages/bbui/src/Form/Core/Picker.svelte +++ b/packages/bbui/src/Form/Core/Picker.svelte @@ -31,7 +31,8 @@ export let autoWidth = false export let autocomplete = false export let sort = false - + export let fetchTerm = null + export let customPopoverHeight const dispatch = createEventDispatcher() let searchTerm = null @@ -71,7 +72,7 @@ } const getFilteredOptions = (options, term, getLabel) => { - if (autocomplete && term) { + if (autocomplete && term && !fetchTerm) { const lowerCaseTerm = term.toLowerCase() return options.filter(option => { return `${getLabel(option)}`.toLowerCase().includes(lowerCaseTerm) @@ -136,6 +137,7 @@ on:close={() => (open = false)} useAnchorWidth={!autoWidth} maxWidth={autoWidth ? 400 : null} + customHeight={customPopoverHeight} >
{#if autocomplete} (searchTerm = event.detail)} + value={fetchTerm ? fetchTerm : searchTerm} + on:change={event => + fetchTerm ? (fetchTerm = event.detail) : (searchTerm = event.detail)} {disabled} placeholder="Search" /> @@ -247,7 +250,7 @@ } .popover-content.auto-width .spectrum-Menu-itemLabel { white-space: nowrap; - overflow: hidden; + overflow: none; text-overflow: ellipsis; } .popover-content:not(.auto-width) .spectrum-Menu-itemLabel { diff --git a/packages/bbui/src/Form/Multiselect.svelte b/packages/bbui/src/Form/Multiselect.svelte index 7bcf22aa06..2237cd1dce 100644 --- a/packages/bbui/src/Form/Multiselect.svelte +++ b/packages/bbui/src/Form/Multiselect.svelte @@ -15,6 +15,10 @@ export let getOptionValue = option => option export let sort = false export let autoWidth = false + export let autocomplete = false + export let fetchTerm = null + export let customPopoverHeight + const dispatch = createEventDispatcher() const onChange = e => { value = e.detail @@ -34,6 +38,9 @@ {getOptionLabel} {getOptionValue} {autoWidth} + {autocomplete} + {customPopoverHeight} + bind:fetchTerm on:change={onChange} on:click /> diff --git a/packages/bbui/src/Form/Select.svelte b/packages/bbui/src/Form/Select.svelte index 69126e648d..76fe613c92 100644 --- a/packages/bbui/src/Form/Select.svelte +++ b/packages/bbui/src/Form/Select.svelte @@ -20,6 +20,8 @@ export let autoWidth = false export let sort = false export let tooltip = "" + export let autocomplete = false + export let customPopoverHeight const dispatch = createEventDispatcher() const onChange = e => { @@ -51,6 +53,8 @@ {getOptionIcon} {getOptionColour} {isOptionEnabled} + {autocomplete} + {customPopoverHeight} on:change={onChange} on:click /> diff --git a/packages/bbui/src/Popover/Popover.svelte b/packages/bbui/src/Popover/Popover.svelte index 8f6ef06591..081e3a34df 100644 --- a/packages/bbui/src/Popover/Popover.svelte +++ b/packages/bbui/src/Popover/Popover.svelte @@ -18,6 +18,7 @@ export let useAnchorWidth = false export let dismissible = true export let offset = 5 + export let customHeight $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum" @@ -74,6 +75,7 @@ on:keydown={handleEscape} class="spectrum-Popover is-open" role="presentation" + style="height: {customHeight}" transition:fly|local={{ y: -20, duration: 200 }} > diff --git a/packages/builder/package.json b/packages/builder/package.json index b3afb4de2a..d5fc757ca3 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.3.18-alpha.12", + "version": "2.3.18-alpha.14", "license": "GPL-3.0", "private": true, "scripts": { @@ -58,10 +58,10 @@ } }, "dependencies": { - "@budibase/bbui": "2.3.18-alpha.12", - "@budibase/client": "2.3.18-alpha.12", - "@budibase/frontend-core": "2.3.18-alpha.12", - "@budibase/string-templates": "2.3.18-alpha.12", + "@budibase/bbui": "2.3.18-alpha.14", + "@budibase/client": "2.3.18-alpha.14", + "@budibase/frontend-core": "2.3.18-alpha.14", + "@budibase/string-templates": "2.3.18-alpha.14", "@fortawesome/fontawesome-svg-core": "^6.2.1", "@fortawesome/free-brands-svg-icons": "^6.2.1", "@fortawesome/free-solid-svg-icons": "^6.2.1", diff --git a/packages/builder/src/pages/builder/auth/login.svelte b/packages/builder/src/pages/builder/auth/login.svelte index 80122c23a5..06e09e4fee 100644 --- a/packages/builder/src/pages/builder/auth/login.svelte +++ b/packages/builder/src/pages/builder/auth/login.svelte @@ -79,67 +79,71 @@ - {/if} - - { - formData = { - ...formData, - username: e.detail, - } - }} - validate={() => { - let fieldError = { - username: !formData.username - ? "Please enter a valid email" - : undefined, - } - errors = handleError({ ...errors, ...fieldError }) - }} - error={errors.username} - /> - { - formData = { - ...formData, - password: e.detail, - } - }} - validate={() => { - let fieldError = { - password: !formData.password - ? "Please enter your password" - : undefined, - } - errors = handleError({ ...errors, ...fieldError }) - }} - error={errors.password} - /> - - - - - - - + {#if !$organisation.isSSOEnforced} + + + { + formData = { + ...formData, + username: e.detail, + } + }} + validate={() => { + let fieldError = { + username: !formData.username + ? "Please enter a valid email" + : undefined, + } + errors = handleError({ ...errors, ...fieldError }) + }} + error={errors.username} + /> + { + formData = { + ...formData, + password: e.detail, + } + }} + validate={() => { + let fieldError = { + password: !formData.password + ? "Please enter your password" + : undefined, + } + errors = handleError({ ...errors, ...fieldError }) + }} + error={errors.password} + /> + + {/if} + {#if !$organisation.isSSOEnforced} + + + + + + + {/if} {#if cloud} diff --git a/packages/builder/src/pages/builder/portal/_components/LockedFeature.svelte b/packages/builder/src/pages/builder/portal/_components/LockedFeature.svelte new file mode 100644 index 0000000000..e6f4075e2e --- /dev/null +++ b/packages/builder/src/pages/builder/portal/_components/LockedFeature.svelte @@ -0,0 +1,76 @@ + + + + +
+ {title} + {#if !enabled} + + {planType} + + {/if} +
+ {description} + + + + {#if enabled} + + {:else} +
+ + + +
+ {/if} + + + diff --git a/packages/builder/src/pages/builder/portal/account/_layout.svelte b/packages/builder/src/pages/builder/portal/account/_layout.svelte index 3083b574a8..892e853aad 100644 --- a/packages/builder/src/pages/builder/portal/account/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/account/_layout.svelte @@ -4,12 +4,13 @@ import { Content, SideNav, SideNavItem } from "components/portal/page" import { menu } from "stores/portal" + $: wide = $isActive("./auditLogs") $: pages = $menu.find(x => x.title === "Account")?.subPages || [] $: !pages.length && $goto("../") - +
{#each pages as { title, href }} diff --git a/packages/builder/src/pages/builder/portal/account/auditLogs/_components/AppColumnRenderer.svelte b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/AppColumnRenderer.svelte new file mode 100644 index 0000000000..08260ecc5c --- /dev/null +++ b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/AppColumnRenderer.svelte @@ -0,0 +1,5 @@ + + +
{value?.name || ""}
diff --git a/packages/builder/src/pages/builder/portal/account/auditLogs/_components/TimeRenderer.svelte b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/TimeRenderer.svelte new file mode 100644 index 0000000000..b6c0262b47 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/TimeRenderer.svelte @@ -0,0 +1,11 @@ + + +
+ {dayjs(row.timestamp).fromNow()} +
diff --git a/packages/builder/src/pages/builder/portal/account/auditLogs/_components/UserRenderer.svelte b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/UserRenderer.svelte new file mode 100644 index 0000000000..16e2a7be04 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/UserRenderer.svelte @@ -0,0 +1,45 @@ + + +
(showTooltip = true)} + on:focus={() => (showTooltip = true)} + on:mouseleave={() => (showTooltip = false)} +> + +
+{#if showTooltip} +
+ +
+{/if} + + diff --git a/packages/builder/src/pages/builder/portal/account/auditLogs/_components/ViewDetailsRenderer.svelte b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/ViewDetailsRenderer.svelte new file mode 100644 index 0000000000..b62ee48cb8 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/account/auditLogs/_components/ViewDetailsRenderer.svelte @@ -0,0 +1,13 @@ + + +Details diff --git a/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte b/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte new file mode 100644 index 0000000000..739877db26 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte @@ -0,0 +1,501 @@ + + + + { + $licensing.goToUpgradePage() + }} +> +
+
+ user._id} + getOptionLabel={user => user.email} + options={sortedUsers} + /> +
+
+ app.instance._id} + getOptionLabel={app => app.name} + options={sortedApps} + bind:value={selectedApps} + /> +
+
+ event.id} + getOptionLabel={event => event.label} + options={sortedEvents} + placeholder="All events" + label="Events" + bind:value={selectedEvents} + /> +
+ +
+ { + if (e.detail[0]?.length === 1) { + startDate = e.detail[0][0].toISOString() + endDate = "" + } else if (e.detail[0]?.length > 1) { + startDate = e.detail[0][0].toISOString() + endDate = e.detail[0][1].toISOString() + } else { + startDate = "" + endDate = "" + } + }} + /> +
+
+ debounce(e.detail)} /> +
+ +
+ downloadLogs()} /> +
+
+ + viewDetails(detail)} + {customRenderers} + data={$auditLogs.logs.data} + allowEditColumns={false} + allowEditRows={false} + allowSelectRows={false} + {schema} + /> + + + + +{#if selectedLog} +
{ + sidePanelVisible = false + }} + > +
+ Audit Log +
+ { + wideSidePanel = !wideSidePanel + }} + /> + { + sidePanelVisible = false + }} + /> +
+
+ + +
+
copyToClipboard(JSON.stringify(selectedLog.metadata))} + class="copy-icon" + > + +
+ +
+
+{/if} + + diff --git a/packages/builder/src/pages/builder/portal/overview/[appId]/backups/index.svelte b/packages/builder/src/pages/builder/portal/overview/[appId]/backups/index.svelte index f2592ec805..4ff9ea386a 100644 --- a/packages/builder/src/pages/builder/portal/overview/[appId]/backups/index.svelte +++ b/packages/builder/src/pages/builder/portal/overview/[appId]/backups/index.svelte @@ -185,7 +185,7 @@ {#if !$licensing.backupsEnabled} - {#if !$auth.accountPortalAccess && !$licensing.groupsEnabled && $admin.cloud} + {#if !$auth.accountPortalAccess && $admin.cloud} Contact your account holder to upgrade your plan. {/if}
diff --git a/packages/builder/src/pages/builder/portal/settings/auth/index.svelte b/packages/builder/src/pages/builder/portal/settings/auth/index.svelte index 608ca271c0..e5196a1cbf 100644 --- a/packages/builder/src/pages/builder/portal/settings/auth/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/auth/index.svelte @@ -22,10 +22,11 @@ Tags, Icon, Helpers, + Link, } from "@budibase/bbui" import { onMount } from "svelte" import { API } from "api" - import { organisation, admin } from "stores/portal" + import { organisation, admin, licensing } from "stores/portal" const ConfigTypes = { Google: "google", @@ -34,6 +35,8 @@ const HasSpacesRegex = /[\\"\s]/ + $: enforcedSSO = $organisation.isSSOEnforced + // Some older google configs contain a manually specified value - retain the functionality to edit the field // When there is no value or we are in the cloud - prohibit editing the field, must use platform url to change $: googleCallbackUrl = undefined @@ -154,6 +157,11 @@ iconDropdownOptions.unshift({ label: fileName, value: fileName }) } + async function toggleIsSSOEnforced() { + const value = $organisation.isSSOEnforced + await organisation.save({ isSSOEnforced: !value }) + } + async function save(docs) { let calls = [] // Only if the user has provided an image, upload it @@ -316,6 +324,49 @@ Authentication Add additional authentication methods from the options below + + + Single Sign-On URL + + Use the following link to access your configured identity provider. + + +
+ {$organisation.platformUrl} +
+ +
+
+ + + + +
+
+
+ Enforce Single Sign-On +
+ {#if !$licensing.enforceableSSO} + + Business plan + + {/if} +
+ {#if $licensing.enforceableSSO} + + {/if} +
+ + Require SSO authentication for all users. It is recommended to read the + help documentation before enabling this feature. + + {#if providers.google} @@ -546,7 +597,24 @@ input[type="file"] { display: none; } - + .sso-link-icon { + padding-top: 4px; + margin-left: 3px; + } + .sso-link { + margin-top: 12px; + display: flex; + flex-direction: row; + align-items: center; + } + .enforce-sso-title { + margin-right: 10px; + } + .enforce-sso-heading-container { + display: flex; + flex-direction: row; + align-items: start; + } .provider-title { display: flex; flex-direction: row; diff --git a/packages/builder/src/pages/builder/portal/settings/environment/index.svelte b/packages/builder/src/pages/builder/portal/settings/environment/index.svelte index 3c170235e9..cff578febd 100644 --- a/packages/builder/src/pages/builder/portal/settings/environment/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/environment/index.svelte @@ -1,21 +1,17 @@ - - -
- Environment Variables - {#if !$licensing.environmentVariablesEnabled} - - Business plan - - {/if} -
- Add and manage environment variables for development and production - - - - {#if $licensing.environmentVariablesEnabled} - {#if noEncryptionKey} - - {/if} -
- -
- - -
- - {:else} -
- - - -
+ { + await environment.upgradePanelOpened() + $licensing.goToUpgradePage() + }} +> + {#if noEncryptionKey} + {/if} - +
+ +
+ +
+ + diff --git a/packages/builder/src/pages/builder/portal/settings/organisation.svelte b/packages/builder/src/pages/builder/portal/settings/organisation.svelte index 8e4d6e738c..bba046ce10 100644 --- a/packages/builder/src/pages/builder/portal/settings/organisation.svelte +++ b/packages/builder/src/pages/builder/portal/settings/organisation.svelte @@ -24,6 +24,7 @@ } const values = writable({ + isSSOEnforced: $organisation.isSSOEnforced, company: $organisation.company, platformUrl: $organisation.platformUrl, analyticsEnabled: $organisation.analyticsEnabled, @@ -54,6 +55,7 @@ } const config = { + isSSOEnforced: $values.isSSOEnforced, company: $values.company ?? "", platformUrl: $values.platformUrl ?? "", analyticsEnabled: $values.analyticsEnabled, diff --git a/packages/builder/src/stores/portal/auditLogs.js b/packages/builder/src/stores/portal/auditLogs.js new file mode 100644 index 0000000000..9abf8ec11b --- /dev/null +++ b/packages/builder/src/stores/portal/auditLogs.js @@ -0,0 +1,43 @@ +import { writable, get } from "svelte/store" +import { API } from "api" +import { licensing } from "stores/portal" + +export function createAuditLogsStore() { + const { subscribe, update } = writable({ + events: {}, + logs: {}, + }) + + async function search(opts = {}) { + if (get(licensing).auditLogsEnabled) { + const paged = await API.searchAuditLogs(opts) + + update(state => { + return { ...state, logs: { ...paged, opts } } + }) + + return paged + } + } + + async function getEventDefinitions() { + const events = await API.getEventDefinitions() + + update(state => { + return { ...state, ...events } + }) + } + + function getDownloadUrl(opts = {}) { + return API.getDownloadUrl(opts) + } + + return { + subscribe, + search, + getEventDefinitions, + getDownloadUrl, + } +} + +export const auditLogs = createAuditLogsStore() diff --git a/packages/builder/src/stores/portal/index.js b/packages/builder/src/stores/portal/index.js index 8d704a5e15..e8f7853165 100644 --- a/packages/builder/src/stores/portal/index.js +++ b/packages/builder/src/stores/portal/index.js @@ -13,3 +13,4 @@ export { backups } from "./backups" export { overview } from "./overview" export { environment } from "./environment" export { menu } from "./menu" +export { auditLogs } from "./auditLogs" diff --git a/packages/builder/src/stores/portal/licensing.js b/packages/builder/src/stores/portal/licensing.js index 6f5c80e03c..df63e9455b 100644 --- a/packages/builder/src/stores/portal/licensing.js +++ b/packages/builder/src/stores/portal/licensing.js @@ -63,7 +63,13 @@ export const createLicensingStore = () => { const environmentVariablesEnabled = license.features.includes( Constants.Features.ENVIRONMENT_VARIABLES ) + const enforceableSSO = license.features.includes( + Constants.Features.ENFORCEABLE_SSO + ) + const auditLogsEnabled = license.features.includes( + Constants.Features.AUDIT_LOGS + ) store.update(state => { return { ...state, @@ -72,6 +78,8 @@ export const createLicensingStore = () => { groupsEnabled, backupsEnabled, environmentVariablesEnabled, + auditLogsEnabled, + enforceableSSO, } }) }, diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js index 8eea36c08c..56956fc330 100644 --- a/packages/builder/src/stores/portal/menu.js +++ b/packages/builder/src/stores/portal/menu.js @@ -75,6 +75,10 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => { title: "Usage", href: "/builder/portal/account/usage", }, + { + title: "Audit Logs", + href: "/builder/portal/account/auditLogs", + }, ] if ($admin.cloud && $auth?.user?.accountPortalAccess) { accountSubPages.push({ @@ -87,6 +91,7 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => { href: "/builder/portal/account/upgrade", }) } + // add license check here if ( $auth?.user?.accountPortalAccess && $auth.user.account.stripeCustomerId diff --git a/packages/builder/src/stores/portal/organisation.js b/packages/builder/src/stores/portal/organisation.js index 9709578fa2..c8b62bb2bc 100644 --- a/packages/builder/src/stores/portal/organisation.js +++ b/packages/builder/src/stores/portal/organisation.js @@ -11,6 +11,7 @@ const DEFAULT_CONFIG = { google: undefined, oidcCallbackUrl: "", googleCallbackUrl: "", + isSSOEnforced: false, } export function createOrganisationStore() { @@ -19,8 +20,8 @@ export function createOrganisationStore() { async function init() { const tenantId = get(auth).tenantId - const tenant = await API.getTenantConfig(tenantId) - set({ ...DEFAULT_CONFIG, ...tenant.config, _rev: tenant._rev }) + const settingsConfigDoc = await API.getTenantConfig(tenantId) + set({ ...DEFAULT_CONFIG, ...settingsConfigDoc.config }) } async function save(config) { @@ -33,7 +34,6 @@ export function createOrganisationStore() { await API.saveConfig({ type: "settings", config: { ...get(store), ...config }, - _rev: get(store)._rev, }) await init() } diff --git a/packages/cli/package.json b/packages/cli/package.json index 5bc10322c1..e3afdf555c 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.3.18-alpha.12", + "version": "2.3.18-alpha.14", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "2.3.18-alpha.12", - "@budibase/string-templates": "2.3.18-alpha.12", - "@budibase/types": "2.3.18-alpha.12", + "@budibase/backend-core": "2.3.18-alpha.14", + "@budibase/string-templates": "2.3.18-alpha.14", + "@budibase/types": "2.3.18-alpha.14", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index c18aa46812..6d46da138c 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.3.18-alpha.12", + "version": "2.3.18-alpha.14", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "2.3.18-alpha.12", - "@budibase/frontend-core": "2.3.18-alpha.12", - "@budibase/string-templates": "2.3.18-alpha.12", + "@budibase/bbui": "2.3.18-alpha.14", + "@budibase/frontend-core": "2.3.18-alpha.14", + "@budibase/string-templates": "2.3.18-alpha.14", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index bc7a82d0a6..938dca82aa 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.3.18-alpha.12", + "version": "2.3.18-alpha.14", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "2.3.18-alpha.12", + "@budibase/bbui": "2.3.18-alpha.14", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/frontend-core/src/api/auditLogs.js b/packages/frontend-core/src/api/auditLogs.js new file mode 100644 index 0000000000..c4230df6d9 --- /dev/null +++ b/packages/frontend-core/src/api/auditLogs.js @@ -0,0 +1,63 @@ +const buildOpts = ({ + bookmark, + userIds, + appIds, + startDate, + endDate, + fullSearch, + events, +}) => { + const opts = {} + + if (bookmark) { + opts.bookmark = bookmark + } + + if (startDate && endDate) { + opts.startDate = startDate + opts.endDate = endDate + } else if (startDate && !endDate) { + opts.startDate = startDate + } + + if (fullSearch) { + opts.fullSearch = fullSearch + } + + if (events.length) { + opts.events = events + } + + if (userIds.length) { + opts.userIds = userIds + } + + if (appIds.length) { + opts.appIds = appIds + } + + return opts +} + +export const buildAuditLogsEndpoints = API => ({ + /** + * Gets a list of users in the current tenant. + */ + searchAuditLogs: async opts => { + return await API.post({ + url: `/api/global/auditlogs/search`, + body: buildOpts(opts), + }) + }, + + getEventDefinitions: async () => { + return await API.get({ + url: `/api/global/auditlogs/definitions`, + }) + }, + + getDownloadUrl: opts => { + const query = encodeURIComponent(JSON.stringify(opts)) + return `/api/global/auditlogs/download?query=${query}` + }, +}) diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js index e2935b416b..f8eee45cb8 100644 --- a/packages/frontend-core/src/api/index.js +++ b/packages/frontend-core/src/api/index.js @@ -28,6 +28,8 @@ import { buildPluginEndpoints } from "./plugins" import { buildBackupsEndpoints } from "./backups" import { buildEnvironmentVariableEndpoints } from "./environmentVariables" import { buildEventEndpoints } from "./events" +import { buildAuditLogsEndpoints } from "./auditLogs" + const defaultAPIClientConfig = { /** * Certain definitions can't change at runtime for client apps, such as the @@ -250,5 +252,6 @@ export const createAPIClient = config => { ...buildBackupsEndpoints(API), ...buildEnvironmentVariableEndpoints(API), ...buildEventEndpoints(API), + ...buildAuditLogsEndpoints(API), } } diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js index 3a16013df2..09b7a00aec 100644 --- a/packages/frontend-core/src/constants.js +++ b/packages/frontend-core/src/constants.js @@ -115,6 +115,8 @@ export const Features = { USER_GROUPS: "userGroups", BACKUPS: "appBackups", ENVIRONMENT_VARIABLES: "environmentVariables", + AUDIT_LOGS: "auditLogs", + ENFORCEABLE_SSO: "enforceableSSO", } // Role IDs diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 8199e84495..051af9233e 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.3.18-alpha.12", + "version": "2.3.18-alpha.14", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/jest.config.ts b/packages/server/jest.config.ts index 331912aa19..f247e90bd1 100644 --- a/packages/server/jest.config.ts +++ b/packages/server/jest.config.ts @@ -39,6 +39,7 @@ const config: Config.InitialOptions = { ], collectCoverageFrom: [ "src/**/*.{js,ts}", + "../backend-core/src/**/*.{js,ts}", // The use of coverage with couchdb view functions breaks tests "!src/db/views/staticViews.*", ], diff --git a/packages/server/package.json b/packages/server/package.json index aa3754ee94..84337f0113 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.3.18-alpha.12", + "version": "2.3.18-alpha.14", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -43,11 +43,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "2.3.18-alpha.12", - "@budibase/client": "2.3.18-alpha.12", - "@budibase/pro": "2.3.18-alpha.12", - "@budibase/string-templates": "2.3.18-alpha.12", - "@budibase/types": "2.3.18-alpha.12", + "@budibase/backend-core": "2.3.18-alpha.14", + "@budibase/client": "2.3.18-alpha.14", + "@budibase/pro": "2.3.18-alpha.14", + "@budibase/string-templates": "2.3.18-alpha.14", + "@budibase/types": "2.3.18-alpha.14", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", @@ -87,6 +87,7 @@ "koa-send": "5.0.0", "koa-session": "5.12.0", "koa-static": "5.0.0", + "koa-useragent": "^4.1.0", "koa2-ratelimit": "1.1.1", "lodash": "4.17.21", "memorystream": "0.3.1", diff --git a/packages/server/src/api/controllers/cloud.ts b/packages/server/src/api/controllers/cloud.ts index 7f43f21fc9..7be00e3a1d 100644 --- a/packages/server/src/api/controllers/cloud.ts +++ b/packages/server/src/api/controllers/cloud.ts @@ -58,7 +58,7 @@ export async function exportApps(ctx: Ctx) { } async function checkHasBeenImported() { - if (!env.SELF_HOSTED || env.MULTI_TENANCY) { + if (!env.SELF_HOSTED) { return true } const apps = await dbCore.getAllApps({ all: true }) @@ -72,7 +72,7 @@ export async function hasBeenImported(ctx: Ctx) { } export async function importApps(ctx: Ctx) { - if (!env.SELF_HOSTED || env.MULTI_TENANCY) { + if (!env.SELF_HOSTED) { ctx.throw(400, "Importing only allowed in self hosted environments.") } const beenImported = await checkHasBeenImported() diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index c3b7636636..bdf8d485f2 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -24,8 +24,7 @@ import { breakExternalTableId, isSQL } from "../../../integrations/utils" import { processObjectSync } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" import { processFormulas, processDates } from "../../../utilities/rowProcessor" -import { context } from "@budibase/backend-core" -import { removeKeyNumbering } from "./utils" +import { db as dbCore } from "@budibase/backend-core" import sdk from "../../../sdk" export interface ManyRelationship { @@ -61,7 +60,7 @@ function buildFilters( let prefix = 1 for (let operator of Object.values(filters)) { for (let field of Object.keys(operator || {})) { - if (removeKeyNumbering(field) === "_id") { + if (dbCore.removeKeyNumbering(field) === "_id") { if (primary) { const parts = breakRowIdField(operator[field]) for (let field of primary) { diff --git a/packages/server/src/api/controllers/row/internalSearch.ts b/packages/server/src/api/controllers/row/internalSearch.ts index 7068aabc5a..9dc12342d6 100644 --- a/packages/server/src/api/controllers/row/internalSearch.ts +++ b/packages/server/src/api/controllers/row/internalSearch.ts @@ -1,531 +1,18 @@ -import { SearchIndexes } from "../../../db/utils" -import { removeKeyNumbering } from "./utils" -import fetch from "node-fetch" -import { db as dbCore, context } from "@budibase/backend-core" -import { SearchFilters, Row } from "@budibase/types" +import { db as dbCore, context, SearchParams } from "@budibase/backend-core" +import { SearchFilters, Row, SearchIndex } from "@budibase/types" -type SearchParams = { - tableId: string - sort?: string - sortOrder?: string - sortType?: string - limit?: number - bookmark?: string - version?: string - rows?: Row[] -} - -/** - * Class to build lucene query URLs. - * Optionally takes a base lucene query object. - */ -export class QueryBuilder { - query: SearchFilters - limit: number - sort?: string - bookmark?: string - sortOrder: string - sortType: string - includeDocs: boolean - version?: string - - constructor(base?: SearchFilters) { - this.query = { - allOr: false, - string: {}, - fuzzy: {}, - range: {}, - equal: {}, - notEqual: {}, - empty: {}, - notEmpty: {}, - oneOf: {}, - contains: {}, - notContains: {}, - containsAny: {}, - ...base, - } - this.limit = 50 - this.sortOrder = "ascending" - this.sortType = "string" - this.includeDocs = true - } - - setVersion(version?: string) { - if (version != null) { - this.version = version - } - return this - } - - setTable(tableId: string) { - this.query.equal!.tableId = tableId - return this - } - - setLimit(limit?: number) { - if (limit != null) { - this.limit = limit - } - return this - } - - setSort(sort?: string) { - if (sort != null) { - this.sort = sort - } - return this - } - - setSortOrder(sortOrder?: string) { - if (sortOrder != null) { - this.sortOrder = sortOrder - } - return this - } - - setSortType(sortType?: string) { - if (sortType != null) { - this.sortType = sortType - } - return this - } - - setBookmark(bookmark?: string) { - if (bookmark != null) { - this.bookmark = bookmark - } - return this - } - - excludeDocs() { - this.includeDocs = false - return this - } - - addString(key: string, partial: string) { - this.query.string![key] = partial - return this - } - - addFuzzy(key: string, fuzzy: string) { - this.query.fuzzy![key] = fuzzy - return this - } - - addRange(key: string, low: string | number, high: string | number) { - this.query.range![key] = { - low, - high, - } - return this - } - - addEqual(key: string, value: any) { - this.query.equal![key] = value - return this - } - - addNotEqual(key: string, value: any) { - this.query.notEqual![key] = value - return this - } - - addEmpty(key: string, value: any) { - this.query.empty![key] = value - return this - } - - addNotEmpty(key: string, value: any) { - this.query.notEmpty![key] = value - return this - } - - addOneOf(key: string, value: any) { - this.query.oneOf![key] = value - return this - } - - addContains(key: string, value: any) { - this.query.contains![key] = value - return this - } - - addNotContains(key: string, value: any) { - this.query.notContains![key] = value - return this - } - - addContainsAny(key: string, value: any) { - this.query.containsAny![key] = value - return this - } - - /** - * Preprocesses a value before going into a lucene search. - * Transforms strings to lowercase and wraps strings and bools in quotes. - * @param value The value to process - * @param options The preprocess options - * @returns {string|*} - */ - preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) { - const hasVersion = !!this.version - // Determine if type needs wrapped - const originalType = typeof value - // Convert to lowercase - if (value && lowercase) { - value = value.toLowerCase ? value.toLowerCase() : value - } - // Escape characters - if (escape && originalType === "string") { - value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&") - } - - // Wrap in quotes - if (originalType === "string" && !isNaN(value) && !type) { - value = `"${value}"` - } else if (hasVersion && wrap) { - value = originalType === "number" ? value : `"${value}"` - } - return value - } - - buildSearchQuery() { - const builder = this - let allOr = this.query && this.query.allOr - let query = allOr ? "" : "*:*" - const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true } - let tableId - if (this.query.equal!.tableId) { - tableId = this.query.equal!.tableId - delete this.query.equal!.tableId - } - - const equal = (key: string, value: any) => { - // 0 evaluates to false, which means we would return all rows if we don't check it - if (!value && value !== 0) { - return null - } - return `${key}:${builder.preprocess(value, allPreProcessingOpts)}` - } - - const contains = (key: string, value: any, mode = "AND") => { - if (Array.isArray(value) && value.length === 0) { - return null - } - if (!Array.isArray(value)) { - return `${key}:${value}` - } - let statement = `${builder.preprocess(value[0], { escape: true })}` - for (let i = 1; i < value.length; i++) { - statement += ` ${mode} ${builder.preprocess(value[i], { - escape: true, - })}` - } - return `${key}:(${statement})` - } - - const notContains = (key: string, value: any) => { - // @ts-ignore - const allPrefix = allOr === "" ? "*:* AND" : "" - return allPrefix + "NOT " + contains(key, value) - } - - const containsAny = (key: string, value: any) => { - return contains(key, value, "OR") - } - - const oneOf = (key: string, value: any) => { - if (!Array.isArray(value)) { - if (typeof value === "string") { - value = value.split(",") - } else { - return "" - } - } - let orStatement = `${builder.preprocess(value[0], allPreProcessingOpts)}` - for (let i = 1; i < value.length; i++) { - orStatement += ` OR ${builder.preprocess( - value[i], - allPreProcessingOpts - )}` - } - return `${key}:(${orStatement})` - } - - function build(structure: any, queryFn: any) { - for (let [key, value] of Object.entries(structure)) { - // check for new format - remove numbering if needed - key = removeKeyNumbering(key) - key = builder.preprocess(key.replace(/ /g, "_"), { - escape: true, - }) - const expression = queryFn(key, value) - if (expression == null) { - continue - } - if (query.length > 0) { - query += ` ${allOr ? "OR" : "AND"} ` - } - query += expression - } - } - - // Construct the actual lucene search query string from JSON structure - if (this.query.string) { - build(this.query.string, (key: string, value: any) => { - if (!value) { - return null - } - value = builder.preprocess(value, { - escape: true, - lowercase: true, - type: "string", - }) - return `${key}:${value}*` - }) - } - if (this.query.range) { - build(this.query.range, (key: string, value: any) => { - if (!value) { - return null - } - if (value.low == null || value.low === "") { - return null - } - if (value.high == null || value.high === "") { - return null - } - const low = builder.preprocess(value.low, allPreProcessingOpts) - const high = builder.preprocess(value.high, allPreProcessingOpts) - return `${key}:[${low} TO ${high}]` - }) - } - if (this.query.fuzzy) { - build(this.query.fuzzy, (key: string, value: any) => { - if (!value) { - return null - } - value = builder.preprocess(value, { - escape: true, - lowercase: true, - type: "fuzzy", - }) - return `${key}:${value}~` - }) - } - if (this.query.equal) { - build(this.query.equal, equal) - } - if (this.query.notEqual) { - build(this.query.notEqual, (key: string, value: any) => { - if (!value) { - return null - } - return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}` - }) - } - if (this.query.empty) { - build(this.query.empty, (key: string) => `!${key}:["" TO *]`) - } - if (this.query.notEmpty) { - build(this.query.notEmpty, (key: string) => `${key}:["" TO *]`) - } - if (this.query.oneOf) { - build(this.query.oneOf, oneOf) - } - if (this.query.contains) { - build(this.query.contains, contains) - } - if (this.query.notContains) { - build(this.query.notContains, notContains) - } - if (this.query.containsAny) { - build(this.query.containsAny, containsAny) - } - // make sure table ID is always added as an AND - if (tableId) { - query = `(${query})` - allOr = false - build({ tableId }, equal) - } - return query - } - - buildSearchBody() { - let body: any = { - q: this.buildSearchQuery(), - limit: Math.min(this.limit, 200), - include_docs: this.includeDocs, - } - if (this.bookmark) { - body.bookmark = this.bookmark - } - if (this.sort) { - const order = this.sortOrder === "descending" ? "-" : "" - const type = `<${this.sortType}>` - body.sort = `${order}${this.sort.replace(/ /g, "_")}${type}` - } - return body - } - - async run() { - const appId = context.getAppId() - const { url, cookie } = dbCore.getCouchInfo() - const fullPath = `${url}/${appId}/_design/database/_search/${SearchIndexes.ROWS}` - const body = this.buildSearchBody() - return await runQuery(fullPath, body, cookie) - } -} - -/** - * Executes a lucene search query. - * @param url The query URL - * @param body The request body defining search criteria - * @param cookie The auth cookie for CouchDB - * @returns {Promise<{rows: []}>} - */ -const runQuery = async (url: string, body: any, cookie: string) => { - const response = await fetch(url, { - body: JSON.stringify(body), - method: "POST", - headers: { - Authorization: cookie, - }, - }) - const json = await response.json() - - let output: any = { - rows: [], - } - if (json.rows != null && json.rows.length > 0) { - output.rows = json.rows.map((row: any) => row.doc) - } - if (json.bookmark) { - output.bookmark = json.bookmark - } - return output -} - -/** - * Gets round the fixed limit of 200 results from a query by fetching as many - * pages as required and concatenating the results. This recursively operates - * until enough results have been found. - * @param query {object} The JSON query structure - * @param params {object} The search params including: - * tableId {string} The table ID to search - * sort {string} The sort column - * sortOrder {string} The sort order ("ascending" or "descending") - * sortType {string} Whether to treat sortable values as strings or - * numbers. ("string" or "number") - * limit {number} The number of results to fetch - * bookmark {string|null} Current bookmark in the recursive search - * rows {array|null} Current results in the recursive search - * @returns {Promise<*[]|*>} - */ -async function recursiveSearch(query: any, params: any): Promise { - const bookmark = params.bookmark - const rows = params.rows || [] - if (rows.length >= params.limit) { - return rows - } - let pageSize = 200 - if (rows.length > params.limit - 200) { - pageSize = params.limit - rows.length - } - const page = await new QueryBuilder(query) - .setVersion(params.version) - .setTable(params.tableId) - .setBookmark(bookmark) - .setLimit(pageSize) - .setSort(params.sort) - .setSortOrder(params.sortOrder) - .setSortType(params.sortType) - .run() - if (!page.rows.length) { - return rows - } - if (page.rows.length < 200) { - return [...rows, ...page.rows] - } - const newParams = { - ...params, - bookmark: page.bookmark, - rows: [...rows, ...page.rows], - } - return await recursiveSearch(query, newParams) -} - -/** - * Performs a paginated search. A bookmark will be returned to allow the next - * page to be fetched. There is a max limit off 200 results per page in a - * paginated search. - * @param query {object} The JSON query structure - * @param params {object} The search params including: - * tableId {string} The table ID to search - * sort {string} The sort column - * sortOrder {string} The sort order ("ascending" or "descending") - * sortType {string} Whether to treat sortable values as strings or - * numbers. ("string" or "number") - * limit {number} The desired page size - * bookmark {string} The bookmark to resume from - * @returns {Promise<{hasNextPage: boolean, rows: *[]}>} - */ export async function paginatedSearch( query: SearchFilters, - params: SearchParams + params: SearchParams ) { - let limit = params.limit - if (limit == null || isNaN(limit) || limit < 0) { - limit = 50 - } - limit = Math.min(limit, 200) - const search = new QueryBuilder(query) - .setVersion(params.version) - .setTable(params.tableId) - .setSort(params.sort) - .setSortOrder(params.sortOrder) - .setSortType(params.sortType) - const searchResults = await search - .setBookmark(params.bookmark) - .setLimit(limit) - .run() - - // Try fetching 1 row in the next page to see if another page of results - // exists or not - const nextResults = await search - .setTable(params.tableId) - .setBookmark(searchResults.bookmark) - .setLimit(1) - .run() - - return { - ...searchResults, - hasNextPage: nextResults.rows && nextResults.rows.length > 0, - } + const appId = context.getAppId() + return dbCore.paginatedSearch(appId!, SearchIndex.ROWS, query, params) } -/** - * Performs a full search, fetching multiple pages if required to return the - * desired amount of results. There is a limit of 1000 results to avoid - * heavy performance hits, and to avoid client components breaking from - * handling too much data. - * @param query {object} The JSON query structure - * @param params {object} The search params including: - * tableId {string} The table ID to search - * sort {string} The sort column - * sortOrder {string} The sort order ("ascending" or "descending") - * sortType {string} Whether to treat sortable values as strings or - * numbers. ("string" or "number") - * limit {number} The desired number of results - * @returns {Promise<{rows: *}>} - */ -export async function fullSearch(query: SearchFilters, params: SearchParams) { - let limit = params.limit - if (limit == null || isNaN(limit) || limit < 0) { - limit = 1000 - } - params.limit = Math.min(limit, 1000) - const rows = await recursiveSearch(query, params) - return { rows } +export async function fullSearch( + query: SearchFilters, + params: SearchParams +) { + const appId = context.getAppId() + return dbCore.fullSearch(appId!, SearchIndex.ROWS, query, params) } diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index f0f1075205..82232b7f98 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -3,8 +3,7 @@ import * as userController from "../user" import { FieldTypes } from "../../../constants" import { context } from "@budibase/backend-core" import { makeExternalQuery } from "../../../integrations/base/query" -import { BBContext, Row, Table } from "@budibase/types" -export { removeKeyNumbering } from "../../../integrations/base/utils" +import { Row, Table } from "@budibase/types" const validateJs = require("validate.js") const { cloneDeep } = require("lodash/fp") import { Format } from "../view/exporters" diff --git a/packages/server/src/api/routes/tests/cloud.seq.spec.ts b/packages/server/src/api/routes/tests/cloud.spec.ts similarity index 70% rename from packages/server/src/api/routes/tests/cloud.seq.spec.ts rename to packages/server/src/api/routes/tests/cloud.spec.ts index d9bd6221ad..aad1214a31 100644 --- a/packages/server/src/api/routes/tests/cloud.seq.spec.ts +++ b/packages/server/src/api/routes/tests/cloud.spec.ts @@ -1,3 +1,5 @@ +import { App } from "@budibase/types" + jest.setTimeout(30000) import { AppStatus } from "../../../db/utils" @@ -5,6 +7,7 @@ import { AppStatus } from "../../../db/utils" import * as setup from "./utilities" import { wipeDb } from "./utilities/TestFunctions" +import { tenancy } from "@budibase/backend-core" describe("/cloud", () => { let request = setup.getRequest()! @@ -12,18 +15,10 @@ describe("/cloud", () => { afterAll(setup.afterAll) - beforeAll(() => { + beforeAll(async () => { // Importing is only allowed in self hosted environments - config.modeSelf() - }) - - beforeEach(async () => { await config.init() - }) - - afterEach(async () => { - // clear all mocks - jest.clearAllMocks() + config.modeSelf() }) describe("import", () => { @@ -32,30 +27,28 @@ describe("/cloud", () => { // import will not run await wipeDb() - // get a count of apps before the import - const preImportApps = await request - .get(`/api/applications?status=${AppStatus.ALL}`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - // Perform the import const res = await request .post(`/api/cloud/import`) + .set(config.publicHeaders()) .attach("importFile", "src/api/routes/tests/data/export-test.tar.gz") - .set(config.defaultHeaders()) .expect(200) expect(res.body.message).toEqual("Apps successfully imported.") // get a count of apps after the import const postImportApps = await request .get(`/api/applications?status=${AppStatus.ALL}`) - .set(config.defaultHeaders()) + .set(config.publicHeaders()) .expect("Content-Type", /json/) .expect(200) + const apps = postImportApps.body as App[] // There are two apps in the file that was imported so check for this - expect(postImportApps.body.length).toEqual(2) + expect(apps.length).toEqual(2) + // The new tenant id was assigned to the imported apps + expect(tenancy.getTenantIDFromAppID(apps[0].appId)).toBe( + config.getTenantId() + ) }) }) }) diff --git a/packages/server/src/api/routes/tests/utilities/TestFunctions.ts b/packages/server/src/api/routes/tests/utilities/TestFunctions.ts index 3f57c1e8ef..5a1ed5210e 100644 --- a/packages/server/src/api/routes/tests/utilities/TestFunctions.ts +++ b/packages/server/src/api/routes/tests/utilities/TestFunctions.ts @@ -2,7 +2,6 @@ import * as rowController from "../../../controllers/row" import * as appController from "../../../controllers/application" import { AppStatus } from "../../../../db/utils" import { roles, tenancy, context } from "@budibase/backend-core" -import { TENANT_ID } from "../../../../tests/utilities/structures" import env from "../../../../environment" import { db } from "@budibase/backend-core" import Nano from "@budibase/nano" @@ -33,7 +32,7 @@ export const getAllTableRows = async (config: any) => { } export const clearAllApps = async ( - tenantId = TENANT_ID, + tenantId: string, exceptions: Array = [] ) => { await tenancy.doInTenant(tenantId, async () => { diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 03dce4f875..00f2aca7fc 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -32,6 +32,7 @@ import { initialise as initialiseWebsockets } from "./websocket" import { startup } from "./startup" const Sentry = require("@sentry/node") const destroyable = require("server-destroy") +const { userAgent } = require("koa-useragent") const app = new Koa() @@ -53,6 +54,7 @@ app.use( ) app.use(middleware.logging) +app.use(userAgent) if (env.isProd()) { env._set("NODE_ENV", "production") diff --git a/packages/server/src/db/index.ts b/packages/server/src/db/index.ts index fa8027dcc1..157c2f4fb3 100644 --- a/packages/server/src/db/index.ts +++ b/packages/server/src/db/index.ts @@ -1,4 +1,4 @@ -import { init as coreInit } from "@budibase/backend-core" +import * as core from "@budibase/backend-core" import env from "../environment" export function init() { @@ -12,5 +12,5 @@ export function init() { dbConfig.allDbs = true } - coreInit({ db: dbConfig }) + core.init({ db: dbConfig }) } diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index ac5a6162b9..50341e4abc 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -9,10 +9,6 @@ export const AppStatus = { DEPLOYED: "published", } -export const SearchIndexes = { - ROWS: "rows", -} - export const BudibaseInternalDB = { _id: "bb_internal", type: dbCore.BUDIBASE_DATASOURCE_TYPE, diff --git a/packages/server/src/db/views/staticViews.ts b/packages/server/src/db/views/staticViews.ts index 4bccfebeee..730e5b1e66 100644 --- a/packages/server/src/db/views/staticViews.ts +++ b/packages/server/src/db/views/staticViews.ts @@ -1,6 +1,6 @@ import { context } from "@budibase/backend-core" -import { DocumentType, SEPARATOR, ViewName, SearchIndexes } from "../utils" -import { LinkDocument, Row } from "@budibase/types" +import { DocumentType, SEPARATOR, ViewName } from "../utils" +import { LinkDocument, Row, SearchIndex } from "@budibase/types" const SCREEN_PREFIX = DocumentType.SCREEN + SEPARATOR /************************************************** @@ -91,7 +91,7 @@ async function searchIndex(indexName: string, fnString: string) { export async function createAllSearchIndex() { await searchIndex( - SearchIndexes.ROWS, + SearchIndex.ROWS, function (doc: Row) { function idx(input: Row, prev?: string) { for (let key of Object.keys(input)) { diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 6871d7b0ba..00e62d3ce4 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -6,11 +6,11 @@ import { SearchFilters, SortDirection, } from "@budibase/types" +import { db as dbCore } from "@budibase/backend-core" import { QueryOptions } from "../../definitions/datasource" import { isIsoDateString, SqlClient } from "../utils" import SqlTableQueryBuilder from "./sqlTable" import environment from "../../environment" -import { removeKeyNumbering } from "./utils" const envLimit = environment.SQL_MAX_ROWS ? parseInt(environment.SQL_MAX_ROWS) @@ -136,7 +136,7 @@ class InternalBuilder { fn: (key: string, value: any) => void ) { for (let [key, value] of Object.entries(structure)) { - const updatedKey = removeKeyNumbering(key) + const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") if (!opts.relationship && !isRelationshipField) { fn(`${opts.tableName}.${updatedKey}`, value) diff --git a/packages/server/src/integrations/base/utils.ts b/packages/server/src/integrations/base/utils.ts deleted file mode 100644 index 54efdb91a0..0000000000 --- a/packages/server/src/integrations/base/utils.ts +++ /dev/null @@ -1,12 +0,0 @@ -const QUERY_START_REGEX = /\d[0-9]*:/g - -export function removeKeyNumbering(key: any): string { - if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) { - const parts = key.split(":") - // remove the number - parts.shift() - return parts.join(":") - } else { - return key - } -} diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index d3caf0b944..2b54123bdb 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -11,8 +11,7 @@ import { OAuth2Client } from "google-auth-library" import { buildExternalTableId } from "./utils" import { DataSourceOperation, FieldTypes } from "../constants" import { GoogleSpreadsheet } from "google-spreadsheet" -import env from "../environment" -import { tenancy, db as dbCore, constants } from "@budibase/backend-core" +import { configs, HTTPError } from "@budibase/backend-core" const fetch = require("node-fetch") interface GoogleSheetsConfig { @@ -173,16 +172,9 @@ class GoogleSheetsIntegration implements DatasourcePlus { async connect() { try { // Initialise oAuth client - const db = tenancy.getGlobalDB() - let googleConfig = await dbCore.getScopedConfig(db, { - type: constants.Config.GOOGLE, - }) - + let googleConfig = await configs.getGoogleConfig() if (!googleConfig) { - googleConfig = { - clientID: env.GOOGLE_CLIENT_ID, - clientSecret: env.GOOGLE_CLIENT_SECRET, - } + throw new HTTPError("Google config not found", 400) } const oauthClient = new OAuth2Client({ diff --git a/packages/server/src/migrations/functions/backfill/global/configs.ts b/packages/server/src/migrations/functions/backfill/global/configs.ts index 7eaa987bc7..1b76727bbe 100644 --- a/packages/server/src/migrations/functions/backfill/global/configs.ts +++ b/packages/server/src/migrations/functions/backfill/global/configs.ts @@ -1,4 +1,9 @@ -import { events, db as dbUtils } from "@budibase/backend-core" +import { + events, + DocumentType, + SEPARATOR, + UNICODE_MAX, +} from "@budibase/backend-core" import { Config, isSMTPConfig, @@ -9,15 +14,16 @@ import { } from "@budibase/types" import env from "./../../../../environment" +export const getConfigParams = () => { + return { + include_docs: true, + startkey: `${DocumentType.CONFIG}${SEPARATOR}`, + endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`, + } +} + const getConfigs = async (globalDb: any): Promise => { - const response = await globalDb.allDocs( - dbUtils.getConfigParams( - {}, - { - include_docs: true, - } - ) - ) + const response = await globalDb.allDocs(getConfigParams()) return response.rows.map((row: any) => row.doc) } diff --git a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js index c0066b1c71..15ddcfd1ef 100644 --- a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js +++ b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js @@ -8,9 +8,8 @@ jest.mock("@budibase/backend-core", () => { } } }) -const { tenancy, db: dbCore } = require("@budibase/backend-core") +const { context, db: dbCore } = require("@budibase/backend-core") const TestConfig = require("../../../tests/utilities/TestConfiguration") -const { TENANT_ID } = require("../../../tests/utilities/structures") // mock email view creation @@ -26,8 +25,8 @@ describe("run", () => { afterAll(config.end) it("runs successfully", async () => { - await tenancy.doInTenant(TENANT_ID, async () => { - const globalDb = tenancy.getGlobalDB() + await config.doInTenant(async () => { + const globalDb = context.getGlobalDB() await migration.run(globalDb) expect(dbCore.createNewUserEmailView).toHaveBeenCalledTimes(1) }) diff --git a/packages/server/src/migrations/tests/helpers.ts b/packages/server/src/migrations/tests/helpers.ts index 17658d5290..35831a2fd0 100644 --- a/packages/server/src/migrations/tests/helpers.ts +++ b/packages/server/src/migrations/tests/helpers.ts @@ -1,7 +1,7 @@ // Mimic configs test configuration from worker, creation configs directly in database import * as structures from "./structures" -import { db } from "@budibase/backend-core" +import { configs } from "@budibase/backend-core" import { Config } from "@budibase/types" export const saveSettingsConfig = async (globalDb: any) => { @@ -25,7 +25,7 @@ export const saveSmtpConfig = async (globalDb: any) => { } const saveConfig = async (config: Config, globalDb: any) => { - config._id = db.generateConfigID({ type: config.type }) + config._id = configs.generateConfigID(config.type) let response try { diff --git a/packages/server/src/migrations/tests/structures.ts b/packages/server/src/migrations/tests/structures.ts index bd48bf63cd..b075c04f5c 100644 --- a/packages/server/src/migrations/tests/structures.ts +++ b/packages/server/src/migrations/tests/structures.ts @@ -20,6 +20,7 @@ export const oidc = (conf?: OIDCConfig): OIDCConfig => { name: "Active Directory", uuid: utils.newid(), activated: true, + scopes: [], ...conf, }, ], diff --git a/packages/server/src/startup.ts b/packages/server/src/startup.ts index f70694226d..6cdbf87c2c 100644 --- a/packages/server/src/startup.ts +++ b/packages/server/src/startup.ts @@ -5,7 +5,7 @@ import { generateApiKey, getChecklist, } from "./utilities/workerRequests" -import { installation, tenancy, logging } from "@budibase/backend-core" +import { installation, tenancy, logging, events } from "@budibase/backend-core" import fs from "fs" import { watch } from "./watch" import * as automations from "./automations" @@ -124,6 +124,9 @@ export async function startup(app?: any, server?: any) { // get the references to the queue promises, don't await as // they will never end, unless the processing stops let queuePromises = [] + // configure events to use the pro audit log write + // can't integrate directly into backend-core due to cyclic issues + queuePromises.push(events.processors.init(pro.sdk.auditLogs.write)) queuePromises.push(automations.init()) queuePromises.push(initPro()) if (app) { diff --git a/packages/server/src/tests/jestEnv.ts b/packages/server/src/tests/jestEnv.ts index c567b260b3..7727bb6007 100644 --- a/packages/server/src/tests/jestEnv.ts +++ b/packages/server/src/tests/jestEnv.ts @@ -8,3 +8,4 @@ process.env.BUDIBASE_DIR = tmpdir("budibase-unittests") process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error" process.env.ENABLE_4XX_HTTP_LOGGING = "0" process.env.MOCK_REDIS = "1" +process.env.PLATFORM_URL = "http://localhost:10000" diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 88545dbcbb..e9b770229f 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -21,7 +21,6 @@ import { basicScreen, basicLayout, basicWebhook, - TENANT_ID, } from "./structures" import { constants, @@ -41,8 +40,8 @@ import { generateUserMetadataID } from "../../db/utils" import { startup } from "../../startup" import supertest from "supertest" import { + App, AuthToken, - Database, Datasource, Row, SourceName, @@ -63,7 +62,7 @@ class TestConfiguration { started: boolean appId: string | null allApps: any[] - app: any + app?: App prodApp: any prodAppId: any user: any @@ -73,7 +72,7 @@ class TestConfiguration { linkedTable: any automation: any datasource: any - tenantId: string | null + tenantId?: string defaultUserValues: DefaultUserValues constructor(openServer = true) { @@ -89,7 +88,6 @@ class TestConfiguration { } this.appId = null this.allApps = [] - this.tenantId = null this.defaultUserValues = this.populateDefaultUserValues() } @@ -154,19 +152,10 @@ class TestConfiguration { // use a new id as the name to avoid name collisions async init(appName = newid()) { - this.defaultUserValues = this.populateDefaultUserValues() - if (context.isMultiTenant()) { - this.tenantId = structures.tenant.id() - } - if (!this.started) { await startup() } - this.user = await this.globalUser() - this.globalUserId = this.user._id - this.userMetadataId = generateUserMetadataID(this.globalUserId) - - return this.createApp(appName) + return this.newTenant(appName) } end() { @@ -182,24 +171,22 @@ class TestConfiguration { } // MODES - #setMultiTenancy = (value: boolean) => { + setMultiTenancy = (value: boolean) => { env._set("MULTI_TENANCY", value) coreEnv._set("MULTI_TENANCY", value) } - #setSelfHosted = (value: boolean) => { + setSelfHosted = (value: boolean) => { env._set("SELF_HOSTED", value) coreEnv._set("SELF_HOSTED", value) } modeCloud = () => { - this.#setSelfHosted(false) - this.#setMultiTenancy(true) + this.setSelfHosted(false) } modeSelf = () => { - this.#setSelfHosted(true) - this.#setMultiTenancy(false) + this.setSelfHosted(true) } // UTILS @@ -354,6 +341,8 @@ class TestConfiguration { }) } + // HEADERS + defaultHeaders(extras = {}) { const tenantId = this.getTenantId() const authObj: AuthToken = { @@ -374,6 +363,7 @@ class TestConfiguration { `${constants.Cookie.CurrentApp}=${appToken}`, ], [constants.Header.CSRF_TOKEN]: this.defaultUserValues.csrfToken, + Host: this.tenantHost(), ...extras, } @@ -383,10 +373,6 @@ class TestConfiguration { return headers } - getTenantId() { - return this.tenantId || TENANT_ID - } - publicHeaders({ prodApp = true } = {}) { const appId = prodApp ? this.prodAppId : this.appId @@ -397,9 +383,7 @@ class TestConfiguration { headers[constants.Header.APP_ID] = appId } - if (this.tenantId) { - headers[constants.Header.TENANT_ID] = this.tenantId - } + headers[constants.Header.TENANT_ID] = this.getTenantId() return headers } @@ -413,6 +397,34 @@ class TestConfiguration { return this.login({ email, roleId, builder, prodApp }) } + // TENANCY + + tenantHost() { + const tenantId = this.getTenantId() + const platformHost = new URL(coreEnv.PLATFORM_URL).host.split(":")[0] + return `${tenantId}.${platformHost}` + } + + getTenantId() { + if (!this.tenantId) { + throw new Error("no test tenant id - init has not been called") + } + return this.tenantId + } + + async newTenant(appName = newid()): Promise { + this.defaultUserValues = this.populateDefaultUserValues() + this.tenantId = structures.tenant.id() + this.user = await this.globalUser() + this.globalUserId = this.user._id + this.userMetadataId = generateUserMetadataID(this.globalUserId) + return this.createApp(appName) + } + + doInTenant(task: any) { + return context.doInTenant(this.getTenantId(), task) + } + // API async generateApiKey(userId = this.defaultUserValues.globalUserId) { @@ -432,7 +444,7 @@ class TestConfiguration { } // APP - async createApp(appName: string) { + async createApp(appName: string): Promise { // create dev app // clear any old app this.appId = null @@ -442,7 +454,7 @@ class TestConfiguration { null, controllers.app.create ) - this.appId = this.app.appId + this.appId = this.app?.appId! }) return await context.doInAppContext(this.appId, async () => { // create production app diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index c1959dc791..e38c0a5275 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -13,8 +13,6 @@ import { const { v4: uuidv4 } = require("uuid") -export const TENANT_ID = "default" - export function basicTable() { return { name: "TestTable", diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 0526603777..f39c5387d7 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1278,14 +1278,14 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.3.18-alpha.12": - version "2.3.18-alpha.12" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.18-alpha.12.tgz#ad1b16be64b78b596af2b5f75647c32e8f6f101a" - integrity sha512-E1NEO+/sNkkRqn/xk9XQmFBO9/dl27w9EB0QGztti/16JV9NgxyDQCJIdGwlD08s1y/lUwOKk0TkSZJs+CTYDw== +"@budibase/backend-core@2.3.18-alpha.14": + version "2.3.18-alpha.14" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.18-alpha.14.tgz#84c10d5840a61437c77c62cb27aa52a48ebea34c" + integrity sha512-8MlNAJNFhct4CwN49wu7EBZJQyToSnUlhZeBlvv94AQxKF6iTzTq40CoNIMCv69nzbeTbDw/ImG7dPW8kBHjpg== dependencies: "@budibase/nano" "10.1.1" "@budibase/pouchdb-replication-stream" "1.2.10" - "@budibase/types" "2.3.18-alpha.12" + "@budibase/types" "2.3.18-alpha.14" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-cloudfront-sign "2.2.0" @@ -1392,13 +1392,13 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@2.3.18-alpha.12": - version "2.3.18-alpha.12" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.18-alpha.12.tgz#be552b3a9f5850e746081540d6586aae69147bec" - integrity sha512-M3b0njzSi47KH6uaQfYPoA2KWrjPiwcU3ONyaVWXHIktVrIKtYaFwOLBr/dmWGfMrL2297SSqg7V4DTaLyAhnw== +"@budibase/pro@2.3.18-alpha.14": + version "2.3.18-alpha.14" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.18-alpha.14.tgz#037045c7c315c23d2981abdcd35fb18bc1a4727d" + integrity sha512-15OFi/Kycp8lYTP424fMoF1l+l3M/0pcDOHpkn0sHix4KhQz2mnqE2iu3k1FR/w9tpPEGHwdd4++BHiDw6h1ZQ== dependencies: - "@budibase/backend-core" "2.3.18-alpha.12" - "@budibase/types" "2.3.18-alpha.12" + "@budibase/backend-core" "2.3.18-alpha.14" + "@budibase/types" "2.3.18-alpha.14" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" @@ -1424,10 +1424,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.3.18-alpha.12": - version "2.3.18-alpha.12" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.18-alpha.12.tgz#a63eb978ccc7e55c209b3e9d71f9aecf7facc0d1" - integrity sha512-27o2BmI/HXIR3frZ8FtqHgAe1hd8jPIzgPaEhKrQiYJ/opUVccqupx9ld75Hyk9E6cdXu0UF0/+LxPpUmMugag== +"@budibase/types@2.3.18-alpha.14": + version "2.3.18-alpha.14" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.18-alpha.14.tgz#3fa32f0b262169c4c8679f38f5e0e321f43f54dc" + integrity sha512-mkp0GAqB7zAeLSNV8//dBjSH9dP9z8Q/Sxv29zlExKAxBHlUcYIB442QZJ8Z2V8Tzb+DlRIK7SLTG622cfyUgg== "@bull-board/api@3.7.0": version "3.7.0" @@ -6122,20 +6122,13 @@ dir-glob@^3.0.1: dependencies: path-type "^4.0.0" -docker-compose@0.23.17: +docker-compose@0.23.17, docker-compose@^0.23.5: version "0.23.17" resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.17.tgz#8816bef82562d9417dc8c790aa4871350f93a2ba" integrity sha512-YJV18YoYIcxOdJKeFcCFihE6F4M2NExWM/d4S1ITcS9samHKnNUihz9kjggr0dNtsrbpFNc7/Yzd19DWs+m1xg== dependencies: yaml "^1.10.2" -docker-compose@^0.23.5: - version "0.23.19" - resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.19.tgz#9947726e2fe67bdfa9e8efe1ff15aa0de2e10eb8" - integrity sha512-v5vNLIdUqwj4my80wxFDkNH+4S85zsRuH29SO7dCWVWPCMt/ohZBsGN6g6KXWifT0pzQ7uOxqEKCYCDPJ8Vz4g== - dependencies: - yaml "^1.10.2" - docker-modem@^3.0.0: version "3.0.6" resolved "https://registry.yarnpkg.com/docker-modem/-/docker-modem-3.0.6.tgz#8c76338641679e28ec2323abb65b3276fb1ce597" @@ -6983,6 +6976,11 @@ expose-loader@^3.1.0: resolved "https://registry.yarnpkg.com/expose-loader/-/expose-loader-3.1.0.tgz#7a0bdecb345b921ca238a8c4715a4ea7e227213f" integrity sha512-2RExSo0yJiqP+xiUue13jQa2IHE8kLDzTI7b6kn+vUlBVvlzNSiLDzo4e5Pp5J039usvTUnxZ8sUOhv0Kg15NA== +express-useragent@^1.0.15: + version "1.0.15" + resolved "https://registry.yarnpkg.com/express-useragent/-/express-useragent-1.0.15.tgz#cefda5fa4904345d51d3368b117a8dd4124985d9" + integrity sha512-eq5xMiYCYwFPoekffMjvEIk+NWdlQY9Y38OsTyl13IvA728vKT+q/CSERYWzcw93HGBJcIqMIsZC5CZGARPVdg== + ext-list@^2.0.0: version "2.2.2" resolved "https://registry.yarnpkg.com/ext-list/-/ext-list-2.2.2.tgz#0b98e64ed82f5acf0f2931babf69212ef52ddd37" @@ -10243,6 +10241,13 @@ koa-static@5.0.0, koa-static@^5.0.0: debug "^3.1.0" koa-send "^5.0.0" +koa-useragent@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/koa-useragent/-/koa-useragent-4.1.0.tgz#d3f128b552c6da3e5e9e9e9c887b2922b16e4468" + integrity sha512-x/HUDZ1zAmNNh5hA9hHbPm9p3UVg2prlpHzxCXQCzbibrNS0kmj7MkCResCbAbG7ZT6FVxNSMjR94ZGamdMwxA== + dependencies: + express-useragent "^1.0.15" + koa-views@^7.0.1: version "7.0.2" resolved "https://registry.yarnpkg.com/koa-views/-/koa-views-7.0.2.tgz#c96fd9e2143ef00c29dc5160c5ed639891aa723d" @@ -10706,7 +10711,12 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" -lru-cache@^7.14.0, lru-cache@^7.14.1: +lru-cache@^7.14.0: + version "7.16.1" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.16.1.tgz#7acea16fecd9ed11430e78443c2bb81a06d3dea9" + integrity sha512-9kkuMZHnLH/8qXARvYSjNvq8S1GYFFzynQTAfKeaJ0sIrR3PUPuu37Z+EiIANiZBvpfTf2B5y8ecDLSMWlLv+w== + +lru-cache@^7.14.1: version "7.14.1" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.14.1.tgz#8da8d2f5f59827edb388e63e459ac23d6d408fea" integrity sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA== diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 1baef9fcf0..a8c2e0ccc2 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.3.18-alpha.12", + "version": "2.3.18-alpha.14", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index e3972ab5ea..eac644ab11 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.3.18-alpha.12", + "version": "2.3.18-alpha.14", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/types/src/api/web/global/auditLogs.ts b/packages/types/src/api/web/global/auditLogs.ts new file mode 100644 index 0000000000..8ce0e742d2 --- /dev/null +++ b/packages/types/src/api/web/global/auditLogs.ts @@ -0,0 +1,44 @@ +import { Event, AuditedEventFriendlyName } from "../../../sdk" +import { + PaginationResponse, + PaginationRequest, + BasicPaginationRequest, +} from "../" +import { User, App } from "../../../" + +export interface AuditLogSearchParams { + userIds?: string[] + appIds?: string[] + events?: Event[] + startDate?: string + endDate?: string + fullSearch?: string + bookmark?: string +} + +export interface DownloadAuditLogsRequest extends AuditLogSearchParams {} + +export interface SearchAuditLogsRequest + extends BasicPaginationRequest, + AuditLogSearchParams {} + +export enum AuditLogResourceStatus { + DELETED = "deleted", +} + +export interface AuditLogEnriched { + app?: App | { _id: string; status: AuditLogResourceStatus } + user: User | { _id: string; status: AuditLogResourceStatus } + event: Event + timestamp: string + name: string + metadata: any +} + +export interface SearchAuditLogsResponse extends PaginationResponse { + data: AuditLogEnriched[] +} + +export interface DefinitionsAuditLogsResponse { + events: Record +} diff --git a/packages/types/src/api/web/global/configs.ts b/packages/types/src/api/web/global/configs.ts new file mode 100644 index 0000000000..1476d13cee --- /dev/null +++ b/packages/types/src/api/web/global/configs.ts @@ -0,0 +1,23 @@ +import { SettingsConfig, SettingsInnerConfig } from "../../../documents" + +/** + * Settings that aren't stored in the database - enriched at runtime. + */ +export interface PublicSettingsInnerConfig extends SettingsInnerConfig { + google: boolean + oidc: boolean + oidcCallbackUrl: string + googleCallbackUrl: string +} + +export interface GetPublicSettingsResponse extends SettingsConfig { + config: PublicSettingsInnerConfig +} + +export interface PublicOIDCConfig { + logo?: string + name?: string + uuid?: string +} + +export type GetPublicOIDCConfigResponse = PublicOIDCConfig[] diff --git a/packages/types/src/api/web/global/index.ts b/packages/types/src/api/web/global/index.ts index 415ed55ab1..21a5de3727 100644 --- a/packages/types/src/api/web/global/index.ts +++ b/packages/types/src/api/web/global/index.ts @@ -1,2 +1,4 @@ export * from "./environmentVariables" +export * from "./auditLogs" export * from "./events" +export * from "./configs" diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index 8ed5b0aad4..141119c837 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -5,3 +5,4 @@ export * from "./errors" export * from "./schedule" export * from "./app" export * from "./global" +export * from "./pagination" diff --git a/packages/types/src/api/web/pagination.ts b/packages/types/src/api/web/pagination.ts new file mode 100644 index 0000000000..ae4c56971a --- /dev/null +++ b/packages/types/src/api/web/pagination.ts @@ -0,0 +1,27 @@ +export enum SortOrder { + ASCENDING = "ascending", + DESCENDING = "descending", +} + +export enum SortType { + STRING = "string", + number = "number", +} + +export interface BasicPaginationRequest { + bookmark?: string +} + +export interface PaginationRequest extends BasicPaginationRequest { + limit?: number + sort?: { + order: SortOrder + column: string + type: SortType + } +} + +export interface PaginationResponse { + bookmark: string + hasNextPage: boolean +} diff --git a/packages/types/src/documents/global/auditLogs.ts b/packages/types/src/documents/global/auditLogs.ts new file mode 100644 index 0000000000..7902997b39 --- /dev/null +++ b/packages/types/src/documents/global/auditLogs.ts @@ -0,0 +1,13 @@ +import { Document } from "../document" +import { Event } from "../../sdk" + +export const AuditLogSystemUser = "SYSTEM" + +export interface AuditLogDoc extends Document { + appId?: string + event: Event + userId: string + timestamp: string + metadata: any + name: string +} diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts index 99dec534b6..f4bba21e0f 100644 --- a/packages/types/src/documents/global/config.ts +++ b/packages/types/src/documents/global/config.ts @@ -5,32 +5,45 @@ export interface Config extends Document { config: any } -export interface SMTPConfig extends Config { - config: { - port: number - host: string - from: string - subject: string - secure: boolean +export interface SMTPInnerConfig { + port: number + host: string + from: string + subject?: string + secure: boolean + auth?: { + user: string + pass: string } + connectionTimeout?: any +} + +export interface SMTPConfig extends Config { + config: SMTPInnerConfig +} + +export interface SettingsInnerConfig { + platformUrl?: string + company?: string + logoUrl?: string // Populated on read + logoUrlEtag?: string + uniqueTenantId?: string + analyticsEnabled?: boolean + isSSOEnforced?: boolean } export interface SettingsConfig extends Config { - config: { - company: string - // Populated on read - logoUrl?: string - logoUrlEtag?: boolean - platformUrl: string - uniqueTenantId?: string - analyticsEnabled?: boolean - } + config: SettingsInnerConfig } export interface GoogleInnerConfig { clientID: string clientSecret: string activated: boolean + /** + * @deprecated read only + */ + callbackURL?: string } export interface GoogleConfig extends Config { @@ -55,6 +68,7 @@ export interface OIDCInnerConfig { name: string uuid: string activated: boolean + scopes: string[] } export interface OIDCConfig extends Config { diff --git a/packages/types/src/documents/global/index.ts b/packages/types/src/documents/global/index.ts index 11ce7513f2..b728439dd6 100644 --- a/packages/types/src/documents/global/index.ts +++ b/packages/types/src/documents/global/index.ts @@ -6,3 +6,4 @@ export * from "./quotas" export * from "./schedule" export * from "./templates" export * from "./environmentVariables" +export * from "./auditLogs" diff --git a/packages/types/src/sdk/auditLogs.ts b/packages/types/src/sdk/auditLogs.ts new file mode 100644 index 0000000000..0322d2e862 --- /dev/null +++ b/packages/types/src/sdk/auditLogs.ts @@ -0,0 +1,21 @@ +import { Event, HostInfo } from "./events" +import { AuditLogDoc } from "../documents" + +export type AuditWriteOpts = { + appId?: string + timestamp?: string | number + userId?: string + hostInfo?: HostInfo +} + +export type AuditLogFn = ( + event: Event, + metadata: any, + opts: AuditWriteOpts +) => Promise + +export type AuditLogQueueEvent = { + event: Event + properties: any + opts: AuditWriteOpts +} diff --git a/packages/types/src/sdk/context.ts b/packages/types/src/sdk/context.ts index b3403df8af..c8345de196 100644 --- a/packages/types/src/sdk/context.ts +++ b/packages/types/src/sdk/context.ts @@ -1,5 +1,5 @@ import { User, Account } from "../documents" -import { IdentityType } from "./events" +import { IdentityType, HostInfo } from "./events" export interface BaseContext { _id: string @@ -16,6 +16,7 @@ export interface UserContext extends BaseContext, User { _id: string tenantId: string account?: Account + hostInfo: HostInfo } export type IdentityContext = BaseContext | AccountUserContext | UserContext diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 35d198ccb2..6e213b5831 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -1,5 +1,11 @@ import Nano from "@budibase/nano" import { AllDocsResponse, AnyDocument, Document } from "../" +import { Writable } from "stream" + +export enum SearchIndex { + ROWS = "rows", + AUDIT = "audit", +} export type PouchOptions = { inMemory?: boolean @@ -63,6 +69,18 @@ export const isDocument = (doc: any): doc is Document => { return typeof doc === "object" && doc._id && doc._rev } +export interface DatabaseDumpOpts { + filter?: (doc: AnyDocument) => boolean + batch_size?: number + batch_limit?: number + style?: "main_only" | "all_docs" + timeout?: number + doc_ids?: string[] + query_params?: any + view?: string + selector?: any +} + export interface Database { name: string @@ -87,7 +105,7 @@ export interface Database { compact(): Promise // these are all PouchDB related functions that are rarely used - in future // should be replaced by better typed/non-pouch implemented methods - dump(...args: any[]): Promise + dump(stream: Writable, opts?: DatabaseDumpOpts): Promise load(...args: any[]): Promise createIndex(...args: any[]): Promise deleteIndex(...args: any[]): Promise diff --git a/packages/types/src/sdk/events/app.ts b/packages/types/src/sdk/events/app.ts index 73d491070f..602dd8b6a5 100644 --- a/packages/types/src/sdk/events/app.ts +++ b/packages/types/src/sdk/events/app.ts @@ -3,50 +3,83 @@ import { BaseEvent } from "./event" export interface AppCreatedEvent extends BaseEvent { appId: string version: string + audited: { + name: string + } } export interface AppUpdatedEvent extends BaseEvent { appId: string version: string + audited: { + name: string + } } export interface AppDeletedEvent extends BaseEvent { appId: string + audited: { + name: string + } } export interface AppPublishedEvent extends BaseEvent { appId: string + audited: { + name: string + } } export interface AppUnpublishedEvent extends BaseEvent { appId: string + audited: { + name: string + } } export interface AppFileImportedEvent extends BaseEvent { appId: string + audited: { + name: string + } } export interface AppTemplateImportedEvent extends BaseEvent { appId: string templateKey: string + audited: { + name: string + } } export interface AppVersionUpdatedEvent extends BaseEvent { appId: string currentVersion: string updatedToVersion: string + audited: { + name: string + } } export interface AppVersionRevertedEvent extends BaseEvent { appId: string currentVersion: string revertedToVersion: string + audited: { + name: string + } } export interface AppRevertedEvent extends BaseEvent { appId: string + audited: { + name: string + } } export interface AppExportedEvent extends BaseEvent { appId: string + audited: { + name: string + } } diff --git a/packages/types/src/sdk/events/auditLog.ts b/packages/types/src/sdk/events/auditLog.ts new file mode 100644 index 0000000000..5f3edfb826 --- /dev/null +++ b/packages/types/src/sdk/events/auditLog.ts @@ -0,0 +1,10 @@ +import { BaseEvent } from "./event" +import { AuditLogSearchParams } from "../../api" + +export interface AuditLogFilteredEvent extends BaseEvent { + filters: AuditLogSearchParams +} + +export interface AuditLogDownloadedEvent extends BaseEvent { + filters: AuditLogSearchParams +} diff --git a/packages/types/src/sdk/events/auth.ts b/packages/types/src/sdk/events/auth.ts index eb9f3148a3..c7171d923d 100644 --- a/packages/types/src/sdk/events/auth.ts +++ b/packages/types/src/sdk/events/auth.ts @@ -7,10 +7,16 @@ export type SSOType = ConfigType.OIDC | ConfigType.GOOGLE export interface LoginEvent extends BaseEvent { userId: string source: LoginSource + audited: { + email: string + } } export interface LogoutEvent extends BaseEvent { userId: string + audited: { + email?: string + } } export interface SSOCreatedEvent extends BaseEvent { diff --git a/packages/types/src/sdk/events/automation.ts b/packages/types/src/sdk/events/automation.ts index beabdbb9eb..a3f4d15186 100644 --- a/packages/types/src/sdk/events/automation.ts +++ b/packages/types/src/sdk/events/automation.ts @@ -5,6 +5,9 @@ export interface AutomationCreatedEvent extends BaseEvent { automationId: string triggerId: string triggerType: string + audited: { + name: string + } } export interface AutomationTriggerUpdatedEvent extends BaseEvent { @@ -19,6 +22,9 @@ export interface AutomationDeletedEvent extends BaseEvent { automationId: string triggerId: string triggerType: string + audited: { + name: string + } } export interface AutomationTestedEvent extends BaseEvent { @@ -35,6 +41,9 @@ export interface AutomationStepCreatedEvent extends BaseEvent { triggerType: string stepId: string stepType: string + audited: { + name: string + } } export interface AutomationStepDeletedEvent extends BaseEvent { @@ -44,6 +53,9 @@ export interface AutomationStepDeletedEvent extends BaseEvent { triggerType: string stepId: string stepType: string + audited: { + name: string + } } export interface AutomationsRunEvent extends BaseEvent { diff --git a/packages/types/src/sdk/events/backup.ts b/packages/types/src/sdk/events/backup.ts index 1dddc109cc..23863cf8ac 100644 --- a/packages/types/src/sdk/events/backup.ts +++ b/packages/types/src/sdk/events/backup.ts @@ -5,6 +5,7 @@ export interface AppBackupRestoreEvent extends BaseEvent { appId: string restoreId: string backupCreatedAt: string + name: string } export interface AppBackupTriggeredEvent extends BaseEvent { @@ -12,4 +13,5 @@ export interface AppBackupTriggeredEvent extends BaseEvent { appId: string trigger: AppBackupTrigger type: AppBackupType + name: string } diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index f509682add..3d0d3122b5 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -180,6 +180,190 @@ export enum Event { ENVIRONMENT_VARIABLE_CREATED = "environment_variable:created", ENVIRONMENT_VARIABLE_DELETED = "environment_variable:deleted", ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED = "environment_variable:upgrade_panel_opened", + + // AUDIT LOG + AUDIT_LOGS_FILTERED = "audit_log:filtered", + AUDIT_LOGS_DOWNLOADED = "audit_log:downloaded", +} + +// all events that are not audited have been added to this record as undefined, this means +// that Typescript can protect us against new events being added and auditing of those +// events not being considered. This might be a little ugly, but provides a level of +// Typescript build protection for the audit log feature, any new event also needs to be +// added to this map, during which the developer will need to consider if it should be +// a user facing event or not. +export const AuditedEventFriendlyName: Record = { + // USER + [Event.USER_CREATED]: `User "{{ email }}" created`, + [Event.USER_UPDATED]: `User "{{ email }}" updated`, + [Event.USER_DELETED]: `User "{{ email }}" deleted`, + [Event.USER_PERMISSION_ADMIN_ASSIGNED]: `User "{{ email }}" admin role assigned`, + [Event.USER_PERMISSION_ADMIN_REMOVED]: `User "{{ email }}" admin role removed`, + [Event.USER_PERMISSION_BUILDER_ASSIGNED]: `User "{{ email }}" builder role assigned`, + [Event.USER_PERMISSION_BUILDER_REMOVED]: `User "{{ email }}" builder role removed`, + [Event.USER_INVITED]: `User "{{ email }}" invited`, + [Event.USER_INVITED_ACCEPTED]: `User "{{ email }}" accepted invite`, + [Event.USER_PASSWORD_UPDATED]: `User "{{ email }}" password updated`, + [Event.USER_PASSWORD_RESET_REQUESTED]: `User "{{ email }}" password reset requested`, + [Event.USER_PASSWORD_RESET]: `User "{{ email }}" password reset`, + [Event.USER_GROUP_CREATED]: `User group "{{ name }}" created`, + [Event.USER_GROUP_UPDATED]: `User group "{{ name }}" updated`, + [Event.USER_GROUP_DELETED]: `User group "{{ name }}" deleted`, + [Event.USER_GROUP_USERS_ADDED]: `User group "{{ name }}" {{ count }} users added`, + [Event.USER_GROUP_USERS_REMOVED]: `User group "{{ name }}" {{ count }} users removed`, + [Event.USER_GROUP_PERMISSIONS_EDITED]: `User group "{{ name }}" permissions edited`, + [Event.USER_PASSWORD_FORCE_RESET]: undefined, + [Event.USER_GROUP_ONBOARDING]: undefined, + [Event.USER_ONBOARDING_COMPLETE]: undefined, + + // EMAIL + [Event.EMAIL_SMTP_CREATED]: `Email configuration created`, + [Event.EMAIL_SMTP_UPDATED]: `Email configuration updated`, + + // AUTH + [Event.AUTH_SSO_CREATED]: `SSO configuration created`, + [Event.AUTH_SSO_UPDATED]: `SSO configuration updated`, + [Event.AUTH_SSO_ACTIVATED]: `SSO configuration activated`, + [Event.AUTH_SSO_DEACTIVATED]: `SSO configuration deactivated`, + [Event.AUTH_LOGIN]: `User "{{ email }}" logged in`, + [Event.AUTH_LOGOUT]: `User "{{ email }}" logged out`, + + // ORG + [Event.ORG_NAME_UPDATED]: `Organisation name updated`, + [Event.ORG_LOGO_UPDATED]: `Organisation logo updated`, + [Event.ORG_PLATFORM_URL_UPDATED]: `Organisation platform URL updated`, + + // APP + [Event.APP_CREATED]: `App "{{ name }}" created`, + [Event.APP_UPDATED]: `App "{{ name }}" updated`, + [Event.APP_DELETED]: `App "{{ name }}" deleted`, + [Event.APP_PUBLISHED]: `App "{{ name }}" published`, + [Event.APP_UNPUBLISHED]: `App "{{ name }}" unpublished`, + [Event.APP_TEMPLATE_IMPORTED]: `App "{{ name }}" template imported`, + [Event.APP_FILE_IMPORTED]: `App "{{ name }}" file imported`, + [Event.APP_VERSION_UPDATED]: `App "{{ name }}" version updated`, + [Event.APP_VERSION_REVERTED]: `App "{{ name }}" version reverted`, + [Event.APP_REVERTED]: `App "{{ name }}" reverted`, + [Event.APP_EXPORTED]: `App "{{ name }}" exported`, + [Event.APP_BACKUP_RESTORED]: `App backup "{{ name }}" restored`, + [Event.APP_BACKUP_TRIGGERED]: `App backup "{{ name }}" triggered`, + + // DATASOURCE + [Event.DATASOURCE_CREATED]: `Datasource created`, + [Event.DATASOURCE_UPDATED]: `Datasource updated`, + [Event.DATASOURCE_DELETED]: `Datasource deleted`, + + // QUERY + [Event.QUERY_CREATED]: `Query created`, + [Event.QUERY_UPDATED]: `Query updated`, + [Event.QUERY_DELETED]: `Query deleted`, + [Event.QUERY_IMPORT]: `Query import`, + [Event.QUERIES_RUN]: undefined, + [Event.QUERY_PREVIEWED]: undefined, + + // TABLE + [Event.TABLE_CREATED]: `Table "{{ name }}" created`, + [Event.TABLE_UPDATED]: `Table "{{ name }}" updated`, + [Event.TABLE_DELETED]: `Table "{{ name }}" deleted`, + [Event.TABLE_EXPORTED]: `Table "{{ name }}" exported`, + [Event.TABLE_IMPORTED]: `Table "{{ name }}" imported`, + [Event.TABLE_DATA_IMPORTED]: `Data imported to table`, + + // ROWS + [Event.ROWS_CREATED]: `Rows created`, + [Event.ROWS_IMPORTED]: `Rows imported`, + + // AUTOMATION + [Event.AUTOMATION_CREATED]: `Automation "{{ name }}" created`, + [Event.AUTOMATION_DELETED]: `Automation "{{ name }}" deleted`, + [Event.AUTOMATION_STEP_CREATED]: `Automation "{{ name }}" step added`, + [Event.AUTOMATION_STEP_DELETED]: `Automation "{{ name }}" step removed`, + [Event.AUTOMATION_TESTED]: undefined, + [Event.AUTOMATIONS_RUN]: undefined, + [Event.AUTOMATION_TRIGGER_UPDATED]: undefined, + + // SCREEN + [Event.SCREEN_CREATED]: `Screen "{{ name }}" created`, + [Event.SCREEN_DELETED]: `Screen "{{ name }}" deleted`, + + // COMPONENT + [Event.COMPONENT_CREATED]: `Component created`, + [Event.COMPONENT_DELETED]: `Component deleted`, + + // ENVIRONMENT VARIABLE + [Event.ENVIRONMENT_VARIABLE_CREATED]: `Environment variable created`, + [Event.ENVIRONMENT_VARIABLE_DELETED]: `Environment variable deleted`, + [Event.ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED]: undefined, + + // PLUGIN + [Event.PLUGIN_IMPORTED]: `Plugin imported`, + [Event.PLUGIN_DELETED]: `Plugin deleted`, + [Event.PLUGIN_INIT]: undefined, + + // ROLE - NOT AUDITED + [Event.ROLE_CREATED]: undefined, + [Event.ROLE_UPDATED]: undefined, + [Event.ROLE_DELETED]: undefined, + [Event.ROLE_ASSIGNED]: undefined, + [Event.ROLE_UNASSIGNED]: undefined, + + // LICENSE - NOT AUDITED + [Event.LICENSE_PLAN_CHANGED]: undefined, + [Event.LICENSE_TIER_CHANGED]: undefined, + [Event.LICENSE_ACTIVATED]: undefined, + [Event.LICENSE_PAYMENT_FAILED]: undefined, + [Event.LICENSE_PAYMENT_RECOVERED]: undefined, + [Event.LICENSE_CHECKOUT_OPENED]: undefined, + [Event.LICENSE_CHECKOUT_SUCCESS]: undefined, + [Event.LICENSE_PORTAL_OPENED]: undefined, + + // ACCOUNT - NOT AUDITED + [Event.ACCOUNT_CREATED]: undefined, + [Event.ACCOUNT_DELETED]: undefined, + [Event.ACCOUNT_VERIFIED]: undefined, + + // BACKFILL - NOT AUDITED + [Event.APP_BACKFILL_SUCCEEDED]: undefined, + [Event.APP_BACKFILL_FAILED]: undefined, + [Event.TENANT_BACKFILL_SUCCEEDED]: undefined, + [Event.TENANT_BACKFILL_FAILED]: undefined, + [Event.INSTALLATION_BACKFILL_SUCCEEDED]: undefined, + [Event.INSTALLATION_BACKFILL_FAILED]: undefined, + + // LAYOUT - NOT AUDITED + [Event.LAYOUT_CREATED]: undefined, + [Event.LAYOUT_DELETED]: undefined, + + // VIEW - NOT AUDITED + [Event.VIEW_CREATED]: undefined, + [Event.VIEW_UPDATED]: undefined, + [Event.VIEW_DELETED]: undefined, + [Event.VIEW_EXPORTED]: undefined, + [Event.VIEW_FILTER_CREATED]: undefined, + [Event.VIEW_FILTER_UPDATED]: undefined, + [Event.VIEW_FILTER_DELETED]: undefined, + [Event.VIEW_CALCULATION_CREATED]: undefined, + [Event.VIEW_CALCULATION_UPDATED]: undefined, + [Event.VIEW_CALCULATION_DELETED]: undefined, + + // SERVED - NOT AUDITED + [Event.SERVED_BUILDER]: undefined, + [Event.SERVED_APP]: undefined, + [Event.SERVED_APP_PREVIEW]: undefined, + + // ANALYTICS - NOT AUDITED + [Event.ANALYTICS_OPT_OUT]: undefined, + [Event.ANALYTICS_OPT_IN]: undefined, + + // INSTALLATION - NOT AUDITED + [Event.INSTALLATION_VERSION_CHECKED]: undefined, + [Event.INSTALLATION_VERSION_UPGRADED]: undefined, + [Event.INSTALLATION_VERSION_DOWNGRADED]: undefined, + [Event.INSTALLATION_FIRST_STARTUP]: undefined, + + // AUDIT LOG - NOT AUDITED + [Event.AUDIT_LOGS_FILTERED]: undefined, + [Event.AUDIT_LOGS_DOWNLOADED]: undefined, } // properties added at the final stage of the event pipeline @@ -191,6 +375,11 @@ export interface BaseEvent { installationId?: string tenantId?: string hosting?: Hosting + // any props in the audited section will be removed before passing events + // up out of system (purely for use with auditing) + audited?: { + [key: string]: any + } } export type TableExportFormat = "json" | "csv" diff --git a/packages/types/src/sdk/events/identification.ts b/packages/types/src/sdk/events/identification.ts index 3f4e7ec9d4..627254882e 100644 --- a/packages/types/src/sdk/events/identification.ts +++ b/packages/types/src/sdk/events/identification.ts @@ -34,6 +34,11 @@ export enum IdentityType { INSTALLATION = "installation", } +export interface HostInfo { + ipAddress?: string + userAgent?: string +} + export interface Identity { id: string type: IdentityType @@ -41,6 +46,7 @@ export interface Identity { environment: string installationId?: string tenantId?: string + hostInfo?: HostInfo } export interface UserIdentity extends Identity { diff --git a/packages/types/src/sdk/events/index.ts b/packages/types/src/sdk/events/index.ts index 009d9beac4..745f84d2a3 100644 --- a/packages/types/src/sdk/events/index.ts +++ b/packages/types/src/sdk/events/index.ts @@ -22,3 +22,4 @@ export * from "./userGroup" export * from "./plugin" export * from "./backup" export * from "./environmentVariable" +export * from "./auditLog" diff --git a/packages/types/src/sdk/events/screen.ts b/packages/types/src/sdk/events/screen.ts index 23c468b7ad..e6b1d4d360 100644 --- a/packages/types/src/sdk/events/screen.ts +++ b/packages/types/src/sdk/events/screen.ts @@ -4,10 +4,16 @@ export interface ScreenCreatedEvent extends BaseEvent { screenId: string layoutId?: string roleId: string + audited: { + name: string + } } export interface ScreenDeletedEvent extends BaseEvent { screenId: string layoutId?: string roleId: string + audited: { + name: string + } } diff --git a/packages/types/src/sdk/events/table.ts b/packages/types/src/sdk/events/table.ts index da3c7edf47..8df2a95796 100644 --- a/packages/types/src/sdk/events/table.ts +++ b/packages/types/src/sdk/events/table.ts @@ -2,21 +2,36 @@ import { BaseEvent, TableExportFormat } from "./event" export interface TableCreatedEvent extends BaseEvent { tableId: string + audited: { + name: string + } } export interface TableUpdatedEvent extends BaseEvent { tableId: string + audited: { + name: string + } } export interface TableDeletedEvent extends BaseEvent { tableId: string + audited: { + name: string + } } export interface TableExportedEvent extends BaseEvent { tableId: string format: TableExportFormat + audited: { + name: string + } } export interface TableImportedEvent extends BaseEvent { tableId: string + audited: { + name: string + } } diff --git a/packages/types/src/sdk/events/user.ts b/packages/types/src/sdk/events/user.ts index 3f8f72801c..ab4b4d9724 100644 --- a/packages/types/src/sdk/events/user.ts +++ b/packages/types/src/sdk/events/user.ts @@ -2,47 +2,84 @@ import { BaseEvent } from "./event" export interface UserCreatedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserUpdatedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserDeletedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserOnboardingEvent extends BaseEvent { userId: string step?: string + audited: { + email: string + } } export interface UserPermissionAssignedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserPermissionRemovedEvent extends BaseEvent { userId: string + audited: { + email: string + } } -export interface UserInvitedEvent extends BaseEvent {} +export interface UserInvitedEvent extends BaseEvent { + audited: { + email: string + } +} export interface UserInviteAcceptedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserPasswordForceResetEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserPasswordUpdatedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserPasswordResetRequestedEvent extends BaseEvent { userId: string + audited: { + email: string + } } export interface UserPasswordResetEvent extends BaseEvent { userId: string + audited: { + email: string + } } diff --git a/packages/types/src/sdk/events/userGroup.ts b/packages/types/src/sdk/events/userGroup.ts index 2ce642e274..d82ab70b4c 100644 --- a/packages/types/src/sdk/events/userGroup.ts +++ b/packages/types/src/sdk/events/userGroup.ts @@ -2,27 +2,50 @@ import { BaseEvent } from "./event" export interface GroupCreatedEvent extends BaseEvent { groupId: string + audited: { + name: string + } } export interface GroupUpdatedEvent extends BaseEvent { groupId: string + audited: { + name: string + } } export interface GroupDeletedEvent extends BaseEvent { groupId: string + audited: { + name: string + } } export interface GroupUsersAddedEvent extends BaseEvent { count: number groupId: string + audited: { + name: string + } } export interface GroupUsersDeletedEvent extends BaseEvent { count: number groupId: string + audited: { + name: string + } } export interface GroupAddedOnboardingEvent extends BaseEvent { groupId: string onboarding: boolean } + +export interface GroupPermissionsEditedEvent extends BaseEvent { + permissions: Record + groupId: string + audited: { + name: string + } +} diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index be12d45527..ccd59bf720 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -12,5 +12,6 @@ export * from "./db" export * from "./middleware" export * from "./featureFlag" export * from "./environmentVariables" +export * from "./auditLogs" export * from "./sso" export * from "./user" diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts index a39bcab18b..9f6090623b 100644 --- a/packages/types/src/sdk/licensing/feature.ts +++ b/packages/types/src/sdk/licensing/feature.ts @@ -2,4 +2,6 @@ export enum Feature { USER_GROUPS = "userGroups", APP_BACKUPS = "appBackups", ENVIRONMENT_VARIABLES = "environmentVariables", + AUDIT_LOGS = "auditLogs", + ENFORCEABLE_SSO = "enforceableSSO", } diff --git a/packages/worker/jest.config.ts b/packages/worker/jest.config.ts index cdacfa411a..3655479d82 100644 --- a/packages/worker/jest.config.ts +++ b/packages/worker/jest.config.ts @@ -7,7 +7,7 @@ const config: Config.InitialOptions = { preset: "@trendyol/jest-testcontainers", setupFiles: ["./src/tests/jestEnv.ts"], setupFilesAfterEnv: ["./src/tests/jestSetup.ts"], - collectCoverageFrom: ["src/**/*.{js,ts}"], + collectCoverageFrom: ["src/**/*.{js,ts}", "../backend-core/src/**/*.{js,ts}"], coverageReporters: ["lcov", "json", "clover"], transform: { "^.+\\.ts?$": "@swc/jest", diff --git a/packages/worker/package.json b/packages/worker/package.json index 6d7cae05a6..84b5d69ee0 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.3.18-alpha.12", + "version": "2.3.18-alpha.14", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "2.3.18-alpha.12", - "@budibase/pro": "2.3.18-alpha.12", - "@budibase/string-templates": "2.3.18-alpha.12", - "@budibase/types": "2.3.18-alpha.12", + "@budibase/backend-core": "2.3.18-alpha.14", + "@budibase/pro": "2.3.18-alpha.14", + "@budibase/string-templates": "2.3.18-alpha.14", + "@budibase/types": "2.3.18-alpha.14", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", @@ -60,6 +60,7 @@ "koa-send": "5.0.1", "koa-session": "5.13.1", "koa-static": "5.0.0", + "koa-useragent": "^4.1.0", "node-fetch": "2.6.7", "nodemailer": "6.7.2", "passport-google-oauth": "2.0.0", diff --git a/packages/worker/src/api/controllers/global/auth.ts b/packages/worker/src/api/controllers/global/auth.ts index 948a98cf3a..92cf014a48 100644 --- a/packages/worker/src/api/controllers/global/auth.ts +++ b/packages/worker/src/api/controllers/global/auth.ts @@ -2,10 +2,9 @@ import { auth as authCore, constants, context, - db as dbCore, events, - tenancy, utils as utilsCore, + configs, } from "@budibase/backend-core" import { ConfigType, @@ -15,6 +14,7 @@ import { SSOUser, PasswordResetRequest, PasswordResetUpdateRequest, + GoogleInnerConfig, } from "@budibase/types" import env from "../../../environment" @@ -61,16 +61,16 @@ export const login = async (ctx: Ctx, next: any) => { const email = ctx.request.body.username const user = await userSdk.getUserByEmail(email) - if (user && (await userSdk.isPreventSSOPasswords(user))) { - ctx.throw(400, "SSO user cannot login using password") + if (user && (await userSdk.isPreventPasswordActions(user))) { + ctx.throw(400, "Password login is disabled for this user") } return passport.authenticate( "local", async (err: any, user: User, info: any) => { await passportCallback(ctx, user, err, info) - await context.identity.doInUserContext(user, async () => { - await events.auth.login("local") + await context.identity.doInUserContext(user, ctx, async () => { + await events.auth.login("local", user.email) }) ctx.status = 200 } @@ -163,8 +163,8 @@ export const datasourceAuth = async (ctx: any, next: any) => { // GOOGLE SSO -export async function googleCallbackUrl(config?: { callbackURL?: string }) { - return ssoCallbackUrl(tenancy.getGlobalDB(), config, ConfigType.GOOGLE) +export async function googleCallbackUrl(config?: GoogleInnerConfig) { + return ssoCallbackUrl(ConfigType.GOOGLE, config) } /** @@ -172,12 +172,10 @@ export async function googleCallbackUrl(config?: { callbackURL?: string }) { * On a successful login, you will be redirected to the googleAuth callback route. */ export const googlePreAuth = async (ctx: any, next: any) => { - const db = tenancy.getGlobalDB() - - const config = await dbCore.getScopedConfig(db, { - type: ConfigType.GOOGLE, - workspace: ctx.query.workspace, - }) + const config = await configs.getGoogleConfig() + if (!config) { + return ctx.throw(400, "Google config not found") + } let callbackUrl = await googleCallbackUrl(config) const strategy = await google.strategyFactory( config, @@ -193,12 +191,10 @@ export const googlePreAuth = async (ctx: any, next: any) => { } export const googleCallback = async (ctx: any, next: any) => { - const db = tenancy.getGlobalDB() - - const config = await dbCore.getScopedConfig(db, { - type: ConfigType.GOOGLE, - workspace: ctx.query.workspace, - }) + const config = await configs.getGoogleConfig() + if (!config) { + return ctx.throw(400, "Google config not found") + } const callbackUrl = await googleCallbackUrl(config) const strategy = await google.strategyFactory( config, @@ -211,8 +207,8 @@ export const googleCallback = async (ctx: any, next: any) => { { successRedirect: "/", failureRedirect: "/error" }, async (err: any, user: SSOUser, info: any) => { await passportCallback(ctx, user, err, info) - await context.identity.doInUserContext(user, async () => { - await events.auth.login("google-internal") + await context.identity.doInUserContext(user, ctx, async () => { + await events.auth.login("google-internal", user.email) }) ctx.redirect("/") } @@ -221,25 +217,20 @@ export const googleCallback = async (ctx: any, next: any) => { // OIDC SSO -export async function oidcCallbackUrl(config?: { callbackURL?: string }) { - return ssoCallbackUrl(tenancy.getGlobalDB(), config, ConfigType.OIDC) +export async function oidcCallbackUrl() { + return ssoCallbackUrl(ConfigType.OIDC) } export const oidcStrategyFactory = async (ctx: any, configId: any) => { - const db = tenancy.getGlobalDB() - const config = await dbCore.getScopedConfig(db, { - type: ConfigType.OIDC, - group: ctx.query.group, - }) + const config = await configs.getOIDCConfig() + if (!config) { + return ctx.throw(400, "OIDC config not found") + } - const chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0] - let callbackUrl = await oidcCallbackUrl(chosenConfig) + let callbackUrl = await oidcCallbackUrl() //Remote Config - const enrichedConfig = await oidc.fetchStrategyConfig( - chosenConfig, - callbackUrl - ) + const enrichedConfig = await oidc.fetchStrategyConfig(config, callbackUrl) return oidc.strategyFactory(enrichedConfig, userSdk.save) } @@ -247,23 +238,23 @@ export const oidcStrategyFactory = async (ctx: any, configId: any) => { * The initial call that OIDC authentication makes to take you to the configured OIDC login screen. * On a successful login, you will be redirected to the oidcAuth callback route. */ -export const oidcPreAuth = async (ctx: any, next: any) => { +export const oidcPreAuth = async (ctx: Ctx, next: any) => { const { configId } = ctx.params + if (!configId) { + ctx.throw(400, "OIDC config id is required") + } const strategy = await oidcStrategyFactory(ctx, configId) setCookie(ctx, configId, Cookie.OIDC_CONFIG) - const db = tenancy.getGlobalDB() - const config = await dbCore.getScopedConfig(db, { - type: ConfigType.OIDC, - group: ctx.query.group, - }) - - const chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0] + const config = await configs.getOIDCConfigById(configId) + if (!config) { + return ctx.throw(400, "OIDC config not found") + } let authScopes = - chosenConfig.scopes?.length > 0 - ? chosenConfig.scopes + config.scopes?.length > 0 + ? config.scopes : ["profile", "email", "offline_access"] return passport.authenticate(strategy, { @@ -281,8 +272,8 @@ export const oidcCallback = async (ctx: any, next: any) => { { successRedirect: "/", failureRedirect: "/error" }, async (err: any, user: SSOUser, info: any) => { await passportCallback(ctx, user, err, info) - await context.identity.doInUserContext(user, async () => { - await events.auth.login("oidc") + await context.identity.doInUserContext(user, ctx, async () => { + await events.auth.login("oidc", user.email) }) ctx.redirect("/") } diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index 855d766a87..1ea0f52c96 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -2,38 +2,32 @@ import * as email from "../../../utilities/email" import env from "../../../environment" import { googleCallbackUrl, oidcCallbackUrl } from "./auth" import { - events, cache, - objectStore, - tenancy, + configs, db as dbCore, env as coreEnv, + events, + objectStore, + tenancy, } from "@budibase/backend-core" import { checkAnyUserExists } from "../../../utilities/users" import { - Database, - Config as ConfigDoc, + Config, ConfigType, - SSOType, - GoogleConfig, - OIDCConfig, - SettingsConfig, + Ctx, + GetPublicOIDCConfigResponse, + GetPublicSettingsResponse, isGoogleConfig, isOIDCConfig, isSettingsConfig, isSMTPConfig, - Ctx, UserCtx, } from "@budibase/types" +import * as pro from "@budibase/pro" -const getEventFns = async (db: Database, config: ConfigDoc) => { +const getEventFns = async (config: Config, existing?: Config) => { const fns = [] - let existing - if (config._id) { - existing = await db.get(config._id) - } - if (!existing) { if (isSMTPConfig(config)) { fns.push(events.email.SMTPCreated) @@ -125,21 +119,21 @@ const getEventFns = async (db: Database, config: ConfigDoc) => { return fns } -export async function save(ctx: UserCtx) { - const db = tenancy.getGlobalDB() - const { type, workspace, user, config } = ctx.request.body - let eventFns = await getEventFns(db, ctx.request.body) - // Config does not exist yet - if (!ctx.request.body._id) { - ctx.request.body._id = dbCore.generateConfigID({ - type, - workspace, - user, - }) +export async function save(ctx: UserCtx) { + const body = ctx.request.body + const type = body.type + const config = body.config + + const existingConfig = await configs.getConfig(type) + let eventFns = await getEventFns(ctx.request.body, existingConfig) + + if (existingConfig) { + body._rev = existingConfig._rev } + try { // verify the configuration - switch (type) { + switch (config.type) { case ConfigType.SMTP: await email.verifyConfig(config) break @@ -149,7 +143,8 @@ export async function save(ctx: UserCtx) { } try { - const response = await db.put(ctx.request.body) + body._id = configs.generateConfigID(type) + const response = await configs.save(body) await cache.bustCache(cache.CacheKey.CHECKLIST) await cache.bustCache(cache.CacheKey.ANALYTICS_ENABLED) @@ -167,44 +162,11 @@ export async function save(ctx: UserCtx) { } } -export async function fetch(ctx: UserCtx) { - const db = tenancy.getGlobalDB() - const response = await db.allDocs( - dbCore.getConfigParams( - { type: ctx.params.type }, - { - include_docs: true, - } - ) - ) - ctx.body = response.rows.map(row => row.doc) -} - -/** - * Gets the most granular config for a particular configuration type. - * The hierarchy is type -> workspace -> user. - */ export async function find(ctx: UserCtx) { - const db = tenancy.getGlobalDB() - - const { userId, workspaceId } = ctx.query - if (workspaceId && userId) { - const workspace = await db.get(workspaceId as string) - const userInWorkspace = workspace.users.some( - (workspaceUser: any) => workspaceUser === userId - ) - if (!ctx.user!.admin && !userInWorkspace) { - ctx.throw(400, `User is not in specified workspace: ${workspace}.`) - } - } - try { // Find the config with the most granular scope based on context - const scopedConfig = await dbCore.getScopedFullConfig(db, { - type: ctx.params.type, - user: userId, - workspace: workspaceId, - }) + const type = ctx.params.type + const scopedConfig = await configs.getConfig(type) if (scopedConfig) { ctx.body = scopedConfig @@ -217,85 +179,70 @@ export async function find(ctx: UserCtx) { } } -export async function publicOidc(ctx: Ctx) { - const db = tenancy.getGlobalDB() +export async function publicOidc(ctx: Ctx) { try { // Find the config with the most granular scope based on context - const oidcConfig: OIDCConfig = await dbCore.getScopedFullConfig(db, { - type: ConfigType.OIDC, - }) + const config = await configs.getOIDCConfig() - if (!oidcConfig) { - ctx.body = {} + if (!config) { + ctx.body = [] } else { - ctx.body = oidcConfig.config.configs.map(config => ({ - logo: config.logo, - name: config.name, - uuid: config.uuid, - })) + ctx.body = [ + { + logo: config.logo, + name: config.name, + uuid: config.uuid, + }, + ] } } catch (err: any) { ctx.throw(err.status, err) } } -export async function publicSettings(ctx: Ctx) { - const db = tenancy.getGlobalDB() - +export async function publicSettings( + ctx: Ctx +) { try { - // Find the config with the most granular scope based on context - const publicConfig = await dbCore.getScopedFullConfig(db, { - type: ConfigType.SETTINGS, - }) - - const googleConfig = await dbCore.getScopedFullConfig(db, { - type: ConfigType.GOOGLE, - }) - - const oidcConfig = await dbCore.getScopedFullConfig(db, { - type: ConfigType.OIDC, - }) - - let config - if (!publicConfig) { - config = { - config: {}, - } - } else { - config = publicConfig - } - - // enrich the logo url - // empty url means deleted - if (config.config.logoUrl && config.config.logoUrl !== "") { - config.config.logoUrl = objectStore.getGlobalFileUrl( + // settings + const configDoc = await configs.getSettingsConfigDoc() + const config = configDoc.config + // enrich the logo url - empty url means deleted + if (config.logoUrl && config.logoUrl !== "") { + config.logoUrl = objectStore.getGlobalFileUrl( "settings", "logoUrl", - config.config.logoUrlEtag + config.logoUrlEtag ) } - // google button flag - if (googleConfig && googleConfig.config) { - // activated by default for configs pre-activated flag - config.config.google = - googleConfig.config.activated == null || googleConfig.config.activated - } else { - config.config.google = false + // google + const googleConfig = await configs.getGoogleConfig() + const preActivated = googleConfig?.activated == null + const google = preActivated || !!googleConfig?.activated + const _googleCallbackUrl = await googleCallbackUrl(googleConfig) + + // oidc + const oidcConfig = await configs.getOIDCConfig() + const oidc = oidcConfig?.activated || false + const _oidcCallbackUrl = await oidcCallbackUrl() + + // sso enforced + const isSSOEnforced = await pro.features.isSSOEnforced({ config }) + + ctx.body = { + type: ConfigType.SETTINGS, + _id: configDoc._id, + _rev: configDoc._rev, + config: { + ...config, + google, + oidc, + isSSOEnforced, + oidcCallbackUrl: _oidcCallbackUrl, + googleCallbackUrl: _googleCallbackUrl, + }, } - - // callback urls - config.config.oidcCallbackUrl = await oidcCallbackUrl() - config.config.googleCallbackUrl = await googleCallbackUrl() - - // oidc button flag - if (oidcConfig && oidcConfig.config) { - config.config.oidc = oidcConfig.config.configs[0].activated - } else { - config.config.oidc = false - } - - ctx.body = config } catch (err: any) { ctx.throw(err.status, err) } @@ -319,12 +266,11 @@ export async function upload(ctx: UserCtx) { }) // add to configuration structure - // TODO: right now this only does a global level - const db = tenancy.getGlobalDB() - let cfgStructure = await dbCore.getScopedFullConfig(db, { type }) - if (!cfgStructure) { - cfgStructure = { - _id: dbCore.generateConfigID({ type }), + let config = await configs.getConfig(type) + if (!config) { + config = { + _id: configs.generateConfigID(type), + type, config: {}, } } @@ -332,14 +278,14 @@ export async function upload(ctx: UserCtx) { // save the Etag for cache bursting const etag = result.ETag if (etag) { - cfgStructure.config[`${name}Etag`] = etag.replace(/"/g, "") + config.config[`${name}Etag`] = etag.replace(/"/g, "") } // save the file key - cfgStructure.config[`${name}`] = key + config.config[`${name}`] = key // write back to db - await db.put(cfgStructure) + await configs.save(config) ctx.body = { message: "File has been uploaded and url stored to config.", @@ -360,7 +306,6 @@ export async function destroy(ctx: UserCtx) { } export async function configChecklist(ctx: Ctx) { - const db = tenancy.getGlobalDB() const tenantId = tenancy.getTenantId() try { @@ -375,19 +320,13 @@ export async function configChecklist(ctx: Ctx) { } // They have set up SMTP - const smtpConfig = await dbCore.getScopedFullConfig(db, { - type: ConfigType.SMTP, - }) + const smtpConfig = await configs.getSMTPConfig() // They have set up Google Auth - const googleConfig = await dbCore.getScopedFullConfig(db, { - type: ConfigType.GOOGLE, - }) + const googleConfig = await configs.getGoogleConfig() // They have set up OIDC - const oidcConfig = await dbCore.getScopedFullConfig(db, { - type: ConfigType.OIDC, - }) + const oidcConfig = await configs.getOIDCConfig() // They have set up a global user const userExists = await checkAnyUserExists() diff --git a/packages/worker/src/api/routes/global/configs.ts b/packages/worker/src/api/routes/global/configs.ts index 38a31a28e6..922bcea212 100644 --- a/packages/worker/src/api/routes/global/configs.ts +++ b/packages/worker/src/api/routes/global/configs.ts @@ -104,13 +104,7 @@ router controller.save ) .delete("/api/global/configs/:id/:rev", auth.adminOnly, controller.destroy) - .get("/api/global/configs", controller.fetch) .get("/api/global/configs/checklist", controller.configChecklist) - .get( - "/api/global/configs/all/:type", - buildConfigGetValidation(), - controller.fetch - ) .get("/api/global/configs/public", controller.publicSettings) .get("/api/global/configs/public/oidc", controller.publicOidc) .get("/api/global/configs/:type", buildConfigGetValidation(), controller.find) diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts new file mode 100644 index 0000000000..19e3cd64b4 --- /dev/null +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -0,0 +1,111 @@ +import { mocks, structures } from "@budibase/backend-core/tests" +import { context, events } from "@budibase/backend-core" +import { Event, IdentityType } from "@budibase/types" +import { TestConfiguration } from "../../../../tests" + +mocks.licenses.useAuditLogs() + +const BASE_IDENTITY = { + account: undefined, + type: IdentityType.USER, +} +const USER_AUDIT_LOG_COUNT = 3 +const APP_ID = "app_1" + +describe("/api/global/auditlogs", () => { + const config = new TestConfiguration() + + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + + describe("POST /api/global/auditlogs/search", () => { + it("should be able to fire some events (create audit logs)", async () => { + await context.doInTenant(config.tenantId, async () => { + const userId = config.user!._id! + const identity = { + ...BASE_IDENTITY, + _id: userId, + tenantId: config.tenantId, + } + await context.doInIdentityContext(identity, async () => { + for (let i = 0; i < USER_AUDIT_LOG_COUNT; i++) { + await events.user.created(structures.users.user()) + } + await context.doInAppContext(APP_ID, async () => { + await events.app.created(structures.apps.app(APP_ID)) + }) + // fetch the user created events + const response = await config.api.auditLogs.search({ + events: [Event.USER_CREATED], + }) + expect(response.data).toBeDefined() + // there will be an initial event which comes from the default user creation + expect(response.data.length).toBe(USER_AUDIT_LOG_COUNT + 1) + }) + }) + }) + + it("should be able to search by event", async () => { + const response = await config.api.auditLogs.search({ + events: [Event.USER_CREATED], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.event).toBe(Event.USER_CREATED) + } + }) + + it("should be able to search by time range (frozen)", async () => { + // this is frozen, only need to add 1 and minus 1 + const now = new Date() + const start = new Date() + start.setSeconds(now.getSeconds() - 1) + const end = new Date() + end.setSeconds(now.getSeconds() + 1) + const response = await config.api.auditLogs.search({ + startDate: start.toISOString(), + endDate: end.toISOString(), + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.timestamp).toBe(now.toISOString()) + } + }) + + it("should be able to search by user ID", async () => { + const userId = config.user!._id! + const response = await config.api.auditLogs.search({ + userIds: [userId], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.user._id).toBe(userId) + } + }) + + it("should be able to search by app ID", async () => { + const response = await config.api.auditLogs.search({ + appIds: [APP_ID], + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.app?._id).toBe(APP_ID) + } + }) + + it("should be able to search by full string", async () => { + const response = await config.api.auditLogs.search({ + fullSearch: "User", + }) + expect(response.data.length).toBeGreaterThan(0) + for (let log of response.data) { + expect(log.name.includes("User")).toBe(true) + } + }) + }) +}) diff --git a/packages/worker/src/api/routes/global/tests/auth.spec.ts b/packages/worker/src/api/routes/global/tests/auth.spec.ts index 84f8ce1b0a..9b5392fc73 100644 --- a/packages/worker/src/api/routes/global/tests/auth.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auth.spec.ts @@ -110,7 +110,7 @@ describe("/api/global/auth", () => { ) expect(response.body).toEqual({ - message: "SSO user cannot login using password", + message: "Password login is disabled for this user", status: 400, }) } @@ -175,7 +175,7 @@ describe("/api/global/auth", () => { ) expect(res.body).toEqual({ - message: "SSO user cannot reset password", + message: "Password reset is disabled for this user", status: 400, error: { code: "http", @@ -367,7 +367,7 @@ describe("/api/global/auth", () => { const res = await config.api.configs.OIDCCallback(configId, preAuthRes) - expect(events.auth.login).toBeCalledWith("oidc") + expect(events.auth.login).toBeCalledWith("oidc", "oauth@example.com") expect(events.auth.login).toBeCalledTimes(1) expect(res.status).toBe(302) const location: string = res.get("location") diff --git a/packages/worker/src/api/routes/global/tests/configs.spec.ts b/packages/worker/src/api/routes/global/tests/configs.spec.ts index 39ad74d295..892fe8a67b 100644 --- a/packages/worker/src/api/routes/global/tests/configs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/configs.spec.ts @@ -2,7 +2,8 @@ jest.mock("nodemailer") import { TestConfiguration, structures, mocks } from "../../../../tests" mocks.email.mock() -import { Config, events } from "@budibase/backend-core" +import { events } from "@budibase/backend-core" +import { GetPublicSettingsResponse, Config, ConfigType } from "@budibase/types" describe("configs", () => { const config = new TestConfiguration() @@ -19,22 +20,29 @@ describe("configs", () => { await config.afterAll() }) - describe("post /api/global/configs", () => { - const saveConfig = async (conf: any, _id?: string, _rev?: string) => { - const data = { - ...conf, - _id, - _rev, - } - - const res = await config.api.configs.saveConfig(data) - - return { - ...data, - ...res.body, - } + const saveConfig = async (conf: Config, _id?: string, _rev?: string) => { + const data = { + ...conf, + _id, + _rev, } + const res = await config.api.configs.saveConfig(data) + return { + ...data, + ...res.body, + } + } + const saveSettingsConfig = async ( + conf?: any, + _id?: string, + _rev?: string + ) => { + const settingsConfig = structures.configs.settings(conf) + return saveConfig(settingsConfig, _id, _rev) + } + + describe("POST /api/global/configs", () => { describe("google", () => { const saveGoogleConfig = async ( conf?: any, @@ -49,20 +57,20 @@ describe("configs", () => { it("should create activated google config", async () => { await saveGoogleConfig() expect(events.auth.SSOCreated).toBeCalledTimes(1) - expect(events.auth.SSOCreated).toBeCalledWith(Config.GOOGLE) + expect(events.auth.SSOCreated).toBeCalledWith(ConfigType.GOOGLE) expect(events.auth.SSODeactivated).not.toBeCalled() expect(events.auth.SSOActivated).toBeCalledTimes(1) - expect(events.auth.SSOActivated).toBeCalledWith(Config.GOOGLE) - await config.deleteConfig(Config.GOOGLE) + expect(events.auth.SSOActivated).toBeCalledWith(ConfigType.GOOGLE) + await config.deleteConfig(ConfigType.GOOGLE) }) it("should create deactivated google config", async () => { await saveGoogleConfig({ activated: false }) expect(events.auth.SSOCreated).toBeCalledTimes(1) - expect(events.auth.SSOCreated).toBeCalledWith(Config.GOOGLE) + expect(events.auth.SSOCreated).toBeCalledWith(ConfigType.GOOGLE) expect(events.auth.SSOActivated).not.toBeCalled() expect(events.auth.SSODeactivated).not.toBeCalled() - await config.deleteConfig(Config.GOOGLE) + await config.deleteConfig(ConfigType.GOOGLE) }) }) @@ -76,11 +84,11 @@ describe("configs", () => { googleConf._rev ) expect(events.auth.SSOUpdated).toBeCalledTimes(1) - expect(events.auth.SSOUpdated).toBeCalledWith(Config.GOOGLE) + expect(events.auth.SSOUpdated).toBeCalledWith(ConfigType.GOOGLE) expect(events.auth.SSOActivated).not.toBeCalled() expect(events.auth.SSODeactivated).toBeCalledTimes(1) - expect(events.auth.SSODeactivated).toBeCalledWith(Config.GOOGLE) - await config.deleteConfig(Config.GOOGLE) + expect(events.auth.SSODeactivated).toBeCalledWith(ConfigType.GOOGLE) + await config.deleteConfig(ConfigType.GOOGLE) }) it("should update google config to activated", async () => { @@ -92,11 +100,11 @@ describe("configs", () => { googleConf._rev ) expect(events.auth.SSOUpdated).toBeCalledTimes(1) - expect(events.auth.SSOUpdated).toBeCalledWith(Config.GOOGLE) + expect(events.auth.SSOUpdated).toBeCalledWith(ConfigType.GOOGLE) expect(events.auth.SSODeactivated).not.toBeCalled() expect(events.auth.SSOActivated).toBeCalledTimes(1) - expect(events.auth.SSOActivated).toBeCalledWith(Config.GOOGLE) - await config.deleteConfig(Config.GOOGLE) + expect(events.auth.SSOActivated).toBeCalledWith(ConfigType.GOOGLE) + await config.deleteConfig(ConfigType.GOOGLE) }) }) }) @@ -115,20 +123,20 @@ describe("configs", () => { it("should create activated OIDC config", async () => { await saveOIDCConfig() expect(events.auth.SSOCreated).toBeCalledTimes(1) - expect(events.auth.SSOCreated).toBeCalledWith(Config.OIDC) + expect(events.auth.SSOCreated).toBeCalledWith(ConfigType.OIDC) expect(events.auth.SSODeactivated).not.toBeCalled() expect(events.auth.SSOActivated).toBeCalledTimes(1) - expect(events.auth.SSOActivated).toBeCalledWith(Config.OIDC) - await config.deleteConfig(Config.OIDC) + expect(events.auth.SSOActivated).toBeCalledWith(ConfigType.OIDC) + await config.deleteConfig(ConfigType.OIDC) }) it("should create deactivated OIDC config", async () => { await saveOIDCConfig({ activated: false }) expect(events.auth.SSOCreated).toBeCalledTimes(1) - expect(events.auth.SSOCreated).toBeCalledWith(Config.OIDC) + expect(events.auth.SSOCreated).toBeCalledWith(ConfigType.OIDC) expect(events.auth.SSOActivated).not.toBeCalled() expect(events.auth.SSODeactivated).not.toBeCalled() - await config.deleteConfig(Config.OIDC) + await config.deleteConfig(ConfigType.OIDC) }) }) @@ -142,11 +150,11 @@ describe("configs", () => { oidcConf._rev ) expect(events.auth.SSOUpdated).toBeCalledTimes(1) - expect(events.auth.SSOUpdated).toBeCalledWith(Config.OIDC) + expect(events.auth.SSOUpdated).toBeCalledWith(ConfigType.OIDC) expect(events.auth.SSOActivated).not.toBeCalled() expect(events.auth.SSODeactivated).toBeCalledTimes(1) - expect(events.auth.SSODeactivated).toBeCalledWith(Config.OIDC) - await config.deleteConfig(Config.OIDC) + expect(events.auth.SSODeactivated).toBeCalledWith(ConfigType.OIDC) + await config.deleteConfig(ConfigType.OIDC) }) it("should update OIDC config to activated", async () => { @@ -158,11 +166,11 @@ describe("configs", () => { oidcConf._rev ) expect(events.auth.SSOUpdated).toBeCalledTimes(1) - expect(events.auth.SSOUpdated).toBeCalledWith(Config.OIDC) + expect(events.auth.SSOUpdated).toBeCalledWith(ConfigType.OIDC) expect(events.auth.SSODeactivated).not.toBeCalled() expect(events.auth.SSOActivated).toBeCalledTimes(1) - expect(events.auth.SSOActivated).toBeCalledWith(Config.OIDC) - await config.deleteConfig(Config.OIDC) + expect(events.auth.SSOActivated).toBeCalledWith(ConfigType.OIDC) + await config.deleteConfig(ConfigType.OIDC) }) }) }) @@ -179,11 +187,11 @@ describe("configs", () => { describe("create", () => { it("should create SMTP config", async () => { - await config.deleteConfig(Config.SMTP) + await config.deleteConfig(ConfigType.SMTP) await saveSMTPConfig() expect(events.email.SMTPUpdated).not.toBeCalled() expect(events.email.SMTPCreated).toBeCalledTimes(1) - await config.deleteConfig(Config.SMTP) + await config.deleteConfig(ConfigType.SMTP) }) }) @@ -194,24 +202,15 @@ describe("configs", () => { await saveSMTPConfig(smtpConf.config, smtpConf._id, smtpConf._rev) expect(events.email.SMTPCreated).not.toBeCalled() expect(events.email.SMTPUpdated).toBeCalledTimes(1) - await config.deleteConfig(Config.SMTP) + await config.deleteConfig(ConfigType.SMTP) }) }) }) describe("settings", () => { - const saveSettingsConfig = async ( - conf?: any, - _id?: string, - _rev?: string - ) => { - const settingsConfig = structures.configs.settings(conf) - return saveConfig(settingsConfig, _id, _rev) - } - describe("create", () => { it("should create settings config with default settings", async () => { - await config.deleteConfig(Config.SETTINGS) + await config.deleteConfig(ConfigType.SETTINGS) await saveSettingsConfig() @@ -222,7 +221,7 @@ describe("configs", () => { it("should create settings config with non-default settings", async () => { config.selfHosted() - await config.deleteConfig(Config.SETTINGS) + await config.deleteConfig(ConfigType.SETTINGS) const conf = { company: "acme", logoUrl: "http://example.com", @@ -241,7 +240,7 @@ describe("configs", () => { describe("update", () => { it("should update settings config", async () => { config.selfHosted() - await config.deleteConfig(Config.SETTINGS) + await config.deleteConfig(ConfigType.SETTINGS) const settingsConfig = await saveSettingsConfig() settingsConfig.config.company = "acme" settingsConfig.config.logoUrl = "http://example.com" @@ -262,14 +261,43 @@ describe("configs", () => { }) }) - it("should return the correct checklist status based on the state of the budibase installation", async () => { - await config.saveSmtpConfig() + describe("GET /api/global/configs/checklist", () => { + it("should return the correct checklist", async () => { + await config.saveSmtpConfig() - const res = await config.api.configs.getConfigChecklist() - const checklist = res.body + const res = await config.api.configs.getConfigChecklist() + const checklist = res.body - expect(checklist.apps.checked).toBeFalsy() - expect(checklist.smtp.checked).toBeTruthy() - expect(checklist.adminUser.checked).toBeTruthy() + expect(checklist.apps.checked).toBeFalsy() + expect(checklist.smtp.checked).toBeTruthy() + expect(checklist.adminUser.checked).toBeTruthy() + }) + }) + + describe("GET /api/global/configs/public", () => { + it("should return the expected public settings", async () => { + await saveSettingsConfig() + + const res = await config.api.configs.getPublicSettings() + const body = res.body as GetPublicSettingsResponse + + const expected = { + _id: "config_settings", + type: "settings", + config: { + company: "Budibase", + logoUrl: "", + analyticsEnabled: false, + google: true, + googleCallbackUrl: `http://localhost:10000/api/global/auth/${config.tenantId}/google/callback`, + isSSOEnforced: false, + oidc: false, + oidcCallbackUrl: `http://localhost:10000/api/global/auth/${config.tenantId}/oidc/callback`, + platformUrl: "http://localhost:10000", + }, + } + delete body._rev + expect(body).toEqual(expected) + }) }) }) diff --git a/packages/worker/src/api/routes/global/tests/realEmail.spec.ts b/packages/worker/src/api/routes/global/tests/realEmail.spec.ts index 1c180be75d..8ad7363c85 100644 --- a/packages/worker/src/api/routes/global/tests/realEmail.spec.ts +++ b/packages/worker/src/api/routes/global/tests/realEmail.spec.ts @@ -1,3 +1,4 @@ +jest.unmock("node-fetch") import { TestConfiguration } from "../../../../tests" import { EmailTemplatePurpose } from "../../../../constants" const nodemailer = require("nodemailer") diff --git a/packages/worker/src/api/routes/index.ts b/packages/worker/src/api/routes/index.ts index 3aa9422238..c64ad44423 100644 --- a/packages/worker/src/api/routes/index.ts +++ b/packages/worker/src/api/routes/index.ts @@ -18,6 +18,8 @@ import accountRoutes from "./system/accounts" import restoreRoutes from "./system/restore" let userGroupRoutes = api.groups +let auditLogRoutes = api.auditLogs + export const routes: Router[] = [ configRoutes, userRoutes, @@ -32,6 +34,7 @@ export const routes: Router[] = [ selfRoutes, licenseRoutes, userGroupRoutes, + auditLogRoutes, migrationRoutes, accountRoutes, restoreRoutes, diff --git a/packages/worker/src/db/index.ts b/packages/worker/src/db/index.ts index d74d00d910..157c2f4fb3 100644 --- a/packages/worker/src/db/index.ts +++ b/packages/worker/src/db/index.ts @@ -1,10 +1,16 @@ import * as core from "@budibase/backend-core" import env from "../environment" -export const init = () => { - const dbConfig: any = {} +export function init() { + const dbConfig: any = { + replication: true, + find: true, + } + if (env.isTest() && !env.COUCH_DB_URL) { dbConfig.inMemory = true + dbConfig.allDbs = true } + core.init({ db: dbConfig }) } diff --git a/packages/worker/src/environment.ts b/packages/worker/src/environment.ts index 71fd89f276..cd5360f7f7 100644 --- a/packages/worker/src/environment.ts +++ b/packages/worker/src/environment.ts @@ -26,8 +26,6 @@ function parseIntSafe(number: any) { } } -const selfHosted = !!parseInt(process.env.SELF_HOSTED || "") - const environment = { // auth MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, @@ -51,7 +49,7 @@ const environment = { CLUSTER_PORT: process.env.CLUSTER_PORT, // flags NODE_ENV: process.env.NODE_ENV, - SELF_HOSTED: selfHosted, + SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED || ""), LOG_LEVEL: process.env.LOG_LEVEL, MULTI_TENANCY: process.env.MULTI_TENANCY, DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL, @@ -71,14 +69,6 @@ const environment = { * Mock the email service in use - links to ethereal hosted emails are logged instead. */ ENABLE_EMAIL_TEST_MODE: process.env.ENABLE_EMAIL_TEST_MODE, - /** - * Enable to allow an admin user to login using a password. - * This can be useful to prevent lockout when configuring SSO. - * However, this should be turned OFF by default for security purposes. - */ - ENABLE_SSO_MAINTENANCE_MODE: selfHosted - ? process.env.ENABLE_SSO_MAINTENANCE_MODE - : false, _set(key: any, value: any) { process.env[key] = value // @ts-ignore diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 1e3ff3cbdf..04413e8429 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -13,7 +13,15 @@ import { Event } from "@sentry/types/dist/event" import Application from "koa" import { bootstrap } from "global-agent" import * as db from "./db" -import { auth, logging, events, middleware } from "@budibase/backend-core" +import { sdk as proSdk } from "@budibase/pro" +import { + auth, + logging, + events, + middleware, + queue, + env as coreEnv, +} from "@budibase/backend-core" db.init() import Koa from "koa" import koaBody from "koa-body" @@ -23,9 +31,15 @@ import * as redis from "./utilities/redis" const Sentry = require("@sentry/node") const koaSession = require("koa-session") const logger = require("koa-pino-logger") +const { userAgent } = require("koa-useragent") + import destroyable from "server-destroy" -if (env.ENABLE_SSO_MAINTENANCE_MODE) { +// configure events to use the pro audit log write +// can't integrate directly into backend-core due to cyclic issues +events.processors.init(proSdk.auditLogs.write) + +if (coreEnv.ENABLE_SSO_MAINTENANCE_MODE) { console.warn( "Warning: ENABLE_SSO_MAINTENANCE_MODE is set. It is recommended this flag is disabled if maintenance is not in progress" ) @@ -43,6 +57,7 @@ app.use(koaBody({ multipart: true })) app.use(koaSession(app)) app.use(middleware.logging) app.use(logger(logging.pinoSettings())) +app.use(userAgent) // authentication app.use(auth.passport.initialize()) @@ -78,6 +93,7 @@ server.on("close", async () => { console.log("Server Closed") await redis.shutdown() await events.shutdown() + await queue.shutdown() if (!env.isTest()) { process.exit(errCode) } diff --git a/packages/worker/src/sdk/auth/auth.ts b/packages/worker/src/sdk/auth/auth.ts index 15a4f3c7e7..8e9cff18dd 100644 --- a/packages/worker/src/sdk/auth/auth.ts +++ b/packages/worker/src/sdk/auth/auth.ts @@ -58,8 +58,8 @@ export const reset = async (email: string) => { } // exit if user has sso - if (await userSdk.isPreventSSOPasswords(user)) { - throw new HTTPError("SSO user cannot reset password", 400) + if (await userSdk.isPreventPasswordActions(user)) { + throw new HTTPError("Password reset is disabled for this user", 400) } // send password reset diff --git a/packages/worker/src/sdk/users/tests/users.spec.ts b/packages/worker/src/sdk/users/tests/users.spec.ts index 41d9298997..77f02eec7a 100644 --- a/packages/worker/src/sdk/users/tests/users.spec.ts +++ b/packages/worker/src/sdk/users/tests/users.spec.ts @@ -1,26 +1,50 @@ import { structures } from "../../../tests" -import * as users from "../users" -import env from "../../../environment" import { mocks } from "@budibase/backend-core/tests" +import { env } from "@budibase/backend-core" +import * as users from "../users" import { CloudAccount } from "@budibase/types" +import { isPreventPasswordActions } from "../users" + +jest.mock("@budibase/pro") +import * as _pro from "@budibase/pro" +const pro = jest.mocked(_pro, true) describe("users", () => { - describe("isPreventSSOPasswords", () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + describe("isPreventPasswordActions", () => { + it("returns false for non sso user", async () => { + const user = structures.users.user() + const result = await users.isPreventPasswordActions(user) + expect(result).toBe(false) + }) + it("returns true for sso account user", async () => { const user = structures.users.user() mocks.accounts.getAccount.mockReturnValue( Promise.resolve(structures.accounts.ssoAccount() as CloudAccount) ) - const result = await users.isPreventSSOPasswords(user) + const result = await users.isPreventPasswordActions(user) expect(result).toBe(true) }) it("returns true for sso user", async () => { const user = structures.users.ssoUser() - const result = await users.isPreventSSOPasswords(user) + const result = await users.isPreventPasswordActions(user) expect(result).toBe(true) }) + describe("enforced sso", () => { + it("returns true for all users when sso is enforced", async () => { + const user = structures.users.user() + pro.features.isSSOEnforced.mockReturnValue(Promise.resolve(true)) + const result = await users.isPreventPasswordActions(user) + expect(result).toBe(true) + }) + }) + describe("sso maintenance mode", () => { beforeEach(() => { env._set("ENABLE_SSO_MAINTENANCE_MODE", true) @@ -33,7 +57,7 @@ describe("users", () => { describe("non-admin user", () => { it("returns true", async () => { const user = structures.users.ssoUser() - const result = await users.isPreventSSOPasswords(user) + const result = await users.isPreventPasswordActions(user) expect(result).toBe(true) }) }) @@ -43,7 +67,7 @@ describe("users", () => { const user = structures.users.ssoUser({ user: structures.users.adminUser(), }) - const result = await users.isPreventSSOPasswords(user) + const result = await users.isPreventPasswordActions(user) expect(result).toBe(false) }) }) diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts index de1ea27546..9de90eae03 100644 --- a/packages/worker/src/sdk/users/users.ts +++ b/packages/worker/src/sdk/users/users.ts @@ -14,6 +14,7 @@ import { users as usersCore, utils, ViewName, + env as coreEnv, } from "@budibase/backend-core" import { AccountMetadata, @@ -34,7 +35,7 @@ import { } from "@budibase/types" import { sendEmail } from "../../utilities/email" import { EmailTemplatePurpose } from "../../constants" -import { groups as groupsSdk } from "@budibase/pro" +import * as pro from "@budibase/pro" import * as accountSdk from "../accounts" const PAGE_LIMIT = 8 @@ -133,8 +134,8 @@ const buildUser = async ( let hashedPassword if (password) { - if (await isPreventSSOPasswords(user)) { - throw new HTTPError("SSO user cannot set password", 400) + if (await isPreventPasswordActions(user)) { + throw new HTTPError("Password change is disabled for this user", 400) } hashedPassword = opts.hashPassword ? await utils.hash(password) : password } else if (dbUser) { @@ -199,13 +200,18 @@ const validateUniqueUser = async (email: string, tenantId: string) => { } } -export async function isPreventSSOPasswords(user: User) { +export async function isPreventPasswordActions(user: User) { // when in maintenance mode we allow sso users with the admin role // to perform any password action - this prevents lockout - if (env.ENABLE_SSO_MAINTENANCE_MODE && user.admin?.global) { + if (coreEnv.ENABLE_SSO_MAINTENANCE_MODE && user.admin?.global) { return false } + // SSO is enforced for all users + if (await pro.features.isSSOEnforced()) { + return true + } + // Check local sso if (isSSOUser(user)) { return true @@ -293,7 +299,7 @@ export const save = async ( if (userGroups.length > 0) { for (let groupId of userGroups) { - groupPromises.push(groupsSdk.addUsers(groupId, [_id])) + groupPromises.push(pro.groups.addUsers(groupId, [_id])) } } } @@ -471,7 +477,7 @@ export const bulkCreate = async ( const groupPromises = [] const createdUserIds = saved.map(user => user._id) for (let groupId of groups) { - groupPromises.push(groupsSdk.addUsers(groupId, createdUserIds)) + groupPromises.push(pro.groups.addUsers(groupId, createdUserIds)) } await Promise.all(groupPromises) } @@ -646,7 +652,7 @@ export const invite = async ( } await sendEmail(user.email, EmailTemplatePurpose.INVITATION, opts) response.successful.push({ email: user.email }) - await events.user.invited() + await events.user.invited(user.email) } catch (e) { console.error(`Failed to send email invitation email=${user.email}`, e) response.unsuccessful.push({ diff --git a/packages/worker/src/tests/api/auditLogs.ts b/packages/worker/src/tests/api/auditLogs.ts new file mode 100644 index 0000000000..d7bc4d99fb --- /dev/null +++ b/packages/worker/src/tests/api/auditLogs.ts @@ -0,0 +1,26 @@ +import { AuditLogSearchParams, SearchAuditLogsResponse } from "@budibase/types" +import TestConfiguration from "../TestConfiguration" +import { TestAPI } from "./base" + +export class AuditLogAPI extends TestAPI { + constructor(config: TestConfiguration) { + super(config) + } + + search = async (search: AuditLogSearchParams) => { + const res = await this.request + .post("/api/global/auditlogs/search") + .send(search) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + return res.body as SearchAuditLogsResponse + } + + download = (search: AuditLogSearchParams) => { + const query = encodeURIComponent(JSON.stringify(search)) + return this.request + .get(`/api/global/auditlogs/download?query=${query}`) + .set(this.config.defaultHeaders()) + } +} diff --git a/packages/worker/src/tests/api/configs.ts b/packages/worker/src/tests/api/configs.ts index 76a6f31415..74cef2bf8b 100644 --- a/packages/worker/src/tests/api/configs.ts +++ b/packages/worker/src/tests/api/configs.ts @@ -14,6 +14,14 @@ export class ConfigAPI extends TestAPI { .expect("Content-Type", /json/) } + getPublicSettings = () => { + return this.request + .get(`/api/global/configs/public`) + .set(this.config.defaultHeaders()) + .expect(200) + .expect("Content-Type", /json/) + } + saveConfig = (data: any) => { return this.request .post(`/api/global/configs`) diff --git a/packages/worker/src/tests/api/email.ts b/packages/worker/src/tests/api/email.ts index ba7c7dbec0..fd3c622cfa 100644 --- a/packages/worker/src/tests/api/email.ts +++ b/packages/worker/src/tests/api/email.ts @@ -13,6 +13,7 @@ export class EmailAPI extends TestAPI { email: "test@test.com", purpose, tenantId: this.config.getTenantId(), + userId: this.config.user?._id!, }) .set(this.config.defaultHeaders()) .expect("Content-Type", /json/) diff --git a/packages/worker/src/tests/api/index.ts b/packages/worker/src/tests/api/index.ts index 0bd0308e2f..166996e792 100644 --- a/packages/worker/src/tests/api/index.ts +++ b/packages/worker/src/tests/api/index.ts @@ -14,6 +14,7 @@ import { GroupsAPI } from "./groups" import { RolesAPI } from "./roles" import { TemplatesAPI } from "./templates" import { LicenseAPI } from "./license" +import { AuditLogAPI } from "./auditLogs" export default class API { accounts: AccountAPI auth: AuthAPI @@ -30,6 +31,7 @@ export default class API { roles: RolesAPI templates: TemplatesAPI license: LicenseAPI + auditLogs: AuditLogAPI constructor(config: TestConfiguration) { this.accounts = new AccountAPI(config) @@ -47,5 +49,6 @@ export default class API { this.roles = new RolesAPI(config) this.templates = new TemplatesAPI(config) this.license = new LicenseAPI(config) + this.auditLogs = new AuditLogAPI(config) } } diff --git a/packages/worker/src/tests/structures/configs.ts b/packages/worker/src/tests/structures/configs.ts index 2c76f271c4..d50f5ebc72 100644 --- a/packages/worker/src/tests/structures/configs.ts +++ b/packages/worker/src/tests/structures/configs.ts @@ -1,9 +1,15 @@ -import { Config } from "../../constants" import { utils } from "@budibase/backend-core" +import { + SettingsConfig, + ConfigType, + SMTPConfig, + GoogleConfig, + OIDCConfig, +} from "@budibase/types" -export function oidc(conf?: any) { +export function oidc(conf?: any): OIDCConfig { return { - type: Config.OIDC, + type: ConfigType.OIDC, config: { configs: [ { @@ -21,9 +27,9 @@ export function oidc(conf?: any) { } } -export function google(conf?: any) { +export function google(conf?: any): GoogleConfig { return { - type: Config.GOOGLE, + type: ConfigType.GOOGLE, config: { clientID: "clientId", clientSecret: "clientSecret", @@ -33,9 +39,9 @@ export function google(conf?: any) { } } -export function smtp(conf?: any) { +export function smtp(conf?: any): SMTPConfig { return { - type: Config.SMTP, + type: ConfigType.SMTP, config: { port: 12345, host: "smtptesthost.com", @@ -47,25 +53,26 @@ export function smtp(conf?: any) { } } -export function smtpEthereal() { +export function smtpEthereal(): SMTPConfig { return { - type: Config.SMTP, + type: ConfigType.SMTP, config: { port: 587, host: "smtp.ethereal.email", + from: "testfrom@test.com", secure: false, auth: { - user: "don.bahringer@ethereal.email", - pass: "yCKSH8rWyUPbnhGYk9", + user: "wyatt.zulauf29@ethereal.email", + pass: "tEwDtHBWWxusVWAPfa", }, connectionTimeout: 1000, // must be less than the jest default of 5000 }, } } -export function settings(conf?: any) { +export function settings(conf?: any): SettingsConfig { return { - type: Config.SETTINGS, + type: ConfigType.SETTINGS, config: { platformUrl: "http://localhost:10000", logoUrl: "", diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts index 66e860edcb..69861684eb 100644 --- a/packages/worker/src/utilities/email.ts +++ b/packages/worker/src/utilities/email.ts @@ -1,11 +1,11 @@ import env from "../environment" -import { EmailTemplatePurpose, TemplateType, Config } from "../constants" +import { EmailTemplatePurpose, TemplateType } from "../constants" import { getTemplateByPurpose } from "../constants/templates" import { getSettingsTemplateContext } from "./templates" import { processString } from "@budibase/string-templates" import { getResetPasswordCode, getInviteCode } from "./redis" -import { User, Database } from "@budibase/types" -import { tenancy, db as dbCore } from "@budibase/backend-core" +import { User, SMTPInnerConfig } from "@budibase/types" +import { configs } from "@budibase/backend-core" const nodemailer = require("nodemailer") type SendEmailOpts = { @@ -36,24 +36,24 @@ const FULL_EMAIL_PURPOSES = [ EmailTemplatePurpose.CUSTOM, ] -function createSMTPTransport(config: any) { +function createSMTPTransport(config?: SMTPInnerConfig) { let options: any - let secure = config.secure + let secure = config?.secure // default it if not specified if (secure == null) { - secure = config.port === 465 + secure = config?.port === 465 } if (!TEST_MODE) { options = { - port: config.port, - host: config.host, + port: config?.port, + host: config?.host, secure: secure, - auth: config.auth, + auth: config?.auth, } options.tls = { rejectUnauthorized: false, } - if (config.connectionTimeout) { + if (config?.connectionTimeout) { options.connectionTimeout = config.connectionTimeout } } else { @@ -134,57 +134,16 @@ async function buildEmail( }) } -/** - * Utility function for finding most valid SMTP configuration. - * @param {object} db The CouchDB database which is to be looked up within. - * @param {string|null} workspaceId If using finer grain control of configs a workspace can be used. - * @param {boolean|null} automation Whether or not the configuration is being fetched for an email automation. - * @return {Promise} returns the SMTP configuration if it exists - */ -async function getSmtpConfiguration( - db: Database, - workspaceId?: string, - automation?: boolean -) { - const params: any = { - type: Config.SMTP, - } - if (workspaceId) { - params.workspace = workspaceId - } - - const customConfig = await dbCore.getScopedConfig(db, params) - - if (customConfig) { - return customConfig - } - - // Use an SMTP fallback configuration from env variables - if (!automation && env.SMTP_FALLBACK_ENABLED) { - return { - port: env.SMTP_PORT, - host: env.SMTP_HOST, - secure: false, - from: env.SMTP_FROM_ADDRESS, - auth: { - user: env.SMTP_USER, - pass: env.SMTP_PASSWORD, - }, - } - } -} - /** * Checks if a SMTP config exists based on passed in parameters. * @return {Promise} returns true if there is a configuration that can be used. */ -export async function isEmailConfigured(workspaceId?: string) { +export async function isEmailConfigured() { // when "testing" or smtp fallback is enabled simply return true if (TEST_MODE || env.SMTP_FALLBACK_ENABLED) { return true } - const db = tenancy.getGlobalDB() - const config = await getSmtpConfiguration(db, workspaceId) + const config = await configs.getSMTPConfig() return config != null } @@ -202,22 +161,17 @@ export async function sendEmail( purpose: EmailTemplatePurpose, opts: SendEmailOpts ) { - const db = tenancy.getGlobalDB() - let config = - (await getSmtpConfiguration(db, opts?.workspaceId, opts?.automation)) || {} - if (Object.keys(config).length === 0 && !TEST_MODE) { + const config = await configs.getSMTPConfig(opts?.automation) + if (!config && !TEST_MODE) { throw "Unable to find SMTP configuration." } const transport = createSMTPTransport(config) // if there is a link code needed this will retrieve it const code = await getLinkCode(purpose, email, opts.user, opts?.info) - let context - if (code) { - context = await getSettingsTemplateContext(purpose, code) - } + let context = await getSettingsTemplateContext(purpose, code) let message: any = { - from: opts?.from || config.from, + from: opts?.from || config?.from, html: await buildEmail(purpose, email, context, { user: opts?.user, contents: opts?.contents, @@ -231,9 +185,9 @@ export async function sendEmail( bcc: opts?.bcc, } - if (opts?.subject || config.subject) { + if (opts?.subject || config?.subject) { message.subject = await processString( - opts?.subject || config.subject, + (opts?.subject || config?.subject) as string, context ) } diff --git a/packages/worker/src/utilities/templates.ts b/packages/worker/src/utilities/templates.ts index ede95dbe4a..1597325c7c 100644 --- a/packages/worker/src/utilities/templates.ts +++ b/packages/worker/src/utilities/templates.ts @@ -1,6 +1,5 @@ -import { db as dbCore, tenancy } from "@budibase/backend-core" +import { tenancy, configs } from "@budibase/backend-core" import { - Config, InternalTemplateBinding, LOGO_URL, EmailTemplatePurpose, @@ -10,20 +9,16 @@ const BASE_COMPANY = "Budibase" export async function getSettingsTemplateContext( purpose: EmailTemplatePurpose, - code?: string + code?: string | null ) { - const db = tenancy.getGlobalDB() - // TODO: use more granular settings in the future if required - let settings = - (await dbCore.getScopedConfig(db, { type: Config.SETTINGS })) || {} + let settings = await configs.getSettingsConfig() const URL = settings.platformUrl const context: any = { [InternalTemplateBinding.LOGO_URL]: checkSlashesInUrl(`${URL}/${settings.logoUrl}`) || LOGO_URL, [InternalTemplateBinding.PLATFORM_URL]: URL, [InternalTemplateBinding.COMPANY]: settings.company || BASE_COMPANY, - [InternalTemplateBinding.DOCS_URL]: - settings.docsUrl || "https://docs.budibase.com/", + [InternalTemplateBinding.DOCS_URL]: "https://docs.budibase.com/", [InternalTemplateBinding.LOGIN_URL]: checkSlashesInUrl( tenancy.addTenantToUrl(`${URL}/login`) ), diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 99f46ba5fb..dc2b635259 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -475,14 +475,14 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.3.18-alpha.12": - version "2.3.18-alpha.12" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.18-alpha.12.tgz#ad1b16be64b78b596af2b5f75647c32e8f6f101a" - integrity sha512-E1NEO+/sNkkRqn/xk9XQmFBO9/dl27w9EB0QGztti/16JV9NgxyDQCJIdGwlD08s1y/lUwOKk0TkSZJs+CTYDw== +"@budibase/backend-core@2.3.18-alpha.14": + version "2.3.18-alpha.14" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.18-alpha.14.tgz#84c10d5840a61437c77c62cb27aa52a48ebea34c" + integrity sha512-8MlNAJNFhct4CwN49wu7EBZJQyToSnUlhZeBlvv94AQxKF6iTzTq40CoNIMCv69nzbeTbDw/ImG7dPW8kBHjpg== dependencies: "@budibase/nano" "10.1.1" "@budibase/pouchdb-replication-stream" "1.2.10" - "@budibase/types" "2.3.18-alpha.12" + "@budibase/types" "2.3.18-alpha.14" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-cloudfront-sign "2.2.0" @@ -539,13 +539,13 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@2.3.18-alpha.12": - version "2.3.18-alpha.12" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.18-alpha.12.tgz#be552b3a9f5850e746081540d6586aae69147bec" - integrity sha512-M3b0njzSi47KH6uaQfYPoA2KWrjPiwcU3ONyaVWXHIktVrIKtYaFwOLBr/dmWGfMrL2297SSqg7V4DTaLyAhnw== +"@budibase/pro@2.3.18-alpha.14": + version "2.3.18-alpha.14" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.18-alpha.14.tgz#037045c7c315c23d2981abdcd35fb18bc1a4727d" + integrity sha512-15OFi/Kycp8lYTP424fMoF1l+l3M/0pcDOHpkn0sHix4KhQz2mnqE2iu3k1FR/w9tpPEGHwdd4++BHiDw6h1ZQ== dependencies: - "@budibase/backend-core" "2.3.18-alpha.12" - "@budibase/types" "2.3.18-alpha.12" + "@budibase/backend-core" "2.3.18-alpha.14" + "@budibase/types" "2.3.18-alpha.14" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" @@ -553,10 +553,10 @@ lru-cache "^7.14.1" node-fetch "^2.6.1" -"@budibase/types@2.3.18-alpha.12": - version "2.3.18-alpha.12" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.18-alpha.12.tgz#a63eb978ccc7e55c209b3e9d71f9aecf7facc0d1" - integrity sha512-27o2BmI/HXIR3frZ8FtqHgAe1hd8jPIzgPaEhKrQiYJ/opUVccqupx9ld75Hyk9E6cdXu0UF0/+LxPpUmMugag== +"@budibase/types@2.3.18-alpha.14": + version "2.3.18-alpha.14" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.18-alpha.14.tgz#3fa32f0b262169c4c8679f38f5e0e321f43f54dc" + integrity sha512-mkp0GAqB7zAeLSNV8//dBjSH9dP9z8Q/Sxv29zlExKAxBHlUcYIB442QZJ8Z2V8Tzb+DlRIK7SLTG622cfyUgg== "@cspotcode/source-map-support@^0.8.0": version "0.8.1" @@ -3653,6 +3653,11 @@ expect@^28.1.3: jest-message-util "^28.1.3" jest-util "^28.1.3" +express-useragent@^1.0.15: + version "1.0.15" + resolved "https://registry.yarnpkg.com/express-useragent/-/express-useragent-1.0.15.tgz#cefda5fa4904345d51d3368b117a8dd4124985d9" + integrity sha512-eq5xMiYCYwFPoekffMjvEIk+NWdlQY9Y38OsTyl13IvA728vKT+q/CSERYWzcw93HGBJcIqMIsZC5CZGARPVdg== + extend@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" @@ -5467,6 +5472,13 @@ koa-static@5.0.0: debug "^3.1.0" koa-send "^5.0.0" +koa-useragent@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/koa-useragent/-/koa-useragent-4.1.0.tgz#d3f128b552c6da3e5e9e9e9c887b2922b16e4468" + integrity sha512-x/HUDZ1zAmNNh5hA9hHbPm9p3UVg2prlpHzxCXQCzbibrNS0kmj7MkCResCbAbG7ZT6FVxNSMjR94ZGamdMwxA== + dependencies: + express-useragent "^1.0.15" + koa@2.13.4, koa@^2.13.4: version "2.13.4" resolved "https://registry.yarnpkg.com/koa/-/koa-2.13.4.tgz#ee5b0cb39e0b8069c38d115139c774833d32462e" diff --git a/scripts/link-dependencies.sh b/scripts/link-dependencies.sh index d2a501162b..9926f3dd2b 100755 --- a/scripts/link-dependencies.sh +++ b/scripts/link-dependencies.sh @@ -44,6 +44,9 @@ if [ -d "../budibase-pro" ]; then echo "Linking types to pro" yarn link '@budibase/types' + echo "Linking string-templates to pro" + yarn link '@budibase/string-templates' + cd ../../../budibase echo "Linking pro to worker" diff --git a/yarn.lock b/yarn.lock index f80cde1b6f..bcd33ad2b6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1064,6 +1064,11 @@ resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.45.0.tgz#794760b9037ee4154c09549ef5a96599621109c5" integrity sha512-QQij+u/vgskA66azc9dCmx+rev79PzX8uDHpsqSjEFtfF2gBUTRCpvYMh2gw2ghkJabNkPlSUCimsyBEQZd1DA== +"@typescript-eslint/types@5.53.0": + version "5.53.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-5.53.0.tgz#f79eca62b97e518ee124086a21a24f3be267026f" + integrity sha512-5kcDL9ZUIP756K6+QOAfPkigJmCPHcLN7Zjdz76lQWWDdzfOhZDTj1irs6gPBKiXx5/6O3L0+AvupAut3z7D2A== + "@typescript-eslint/typescript-estree@5.45.0": version "5.45.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.45.0.tgz#f70a0d646d7f38c0dfd6936a5e171a77f1e5291d" @@ -1090,6 +1095,19 @@ semver "^7.3.5" tsutils "^3.21.0" +"@typescript-eslint/typescript-estree@^5.13.0": + version "5.53.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.53.0.tgz#bc651dc28cf18ab248ecd18a4c886c744aebd690" + integrity sha512-eKmipH7QyScpHSkhbptBBYh9v8FxtngLquq292YTEQ1pxVs39yFBlLC1xeIZcPPz1RWGqb7YgERJRGkjw8ZV7w== + dependencies: + "@typescript-eslint/types" "5.53.0" + "@typescript-eslint/visitor-keys" "5.53.0" + debug "^4.3.4" + globby "^11.1.0" + is-glob "^4.0.3" + semver "^7.3.7" + tsutils "^3.21.0" + "@typescript-eslint/visitor-keys@4.33.0": version "4.33.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-4.33.0.tgz#2a22f77a41604289b7a186586e9ec48ca92ef1dd" @@ -1106,6 +1124,14 @@ "@typescript-eslint/types" "5.45.0" eslint-visitor-keys "^3.3.0" +"@typescript-eslint/visitor-keys@5.53.0": + version "5.53.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.53.0.tgz#8a5126623937cdd909c30d8fa72f79fa56cc1a9f" + integrity sha512-JqNLnX3leaHFZEN0gCh81sIvgrp/2GOACZNgO4+Tkf64u51kTpAyWFOY8XHx8XuXr3N2C9zgPPHtcpMg6z1g0w== + dependencies: + "@typescript-eslint/types" "5.53.0" + eslint-visitor-keys "^3.3.0" + JSONStream@^1.0.4, JSONStream@^1.3.4: version "1.3.5" resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.5.tgz#3208c1f08d3a4d99261ab64f92302bc15e111ca0" @@ -1214,6 +1240,11 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0: dependencies: color-convert "^2.0.1" +any-promise@^1.1.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" + integrity sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A== + app-module-path@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/app-module-path/-/app-module-path-2.2.0.tgz#641aa55dfb7d6a6f0a8141c4b9c0aa50b6c24dd5" @@ -1793,6 +1824,11 @@ commander@^7.2.0: resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== +commander@^9.1.0: + version "9.5.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-9.5.0.tgz#bc08d1eb5cedf7ccb797a96199d41c7bc3e60d30" + integrity sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ== + commondir@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" @@ -2139,16 +2175,16 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ== -dependency-tree@^8.1.1: - version "8.1.2" - resolved "https://registry.yarnpkg.com/dependency-tree/-/dependency-tree-8.1.2.tgz#c9e652984f53bd0239bc8a3e50cbd52f05b2e770" - integrity sha512-c4CL1IKxkKng0oT5xrg4uNiiMVFqTGOXqHSFx7XEFdgSsp6nw3AGGruICppzJUrfad/r7GLqt26rmWU4h4j39A== +dependency-tree@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/dependency-tree/-/dependency-tree-9.0.0.tgz#9288dd6daf35f6510c1ea30d9894b75369aa50a2" + integrity sha512-osYHZJ1fBSon3lNLw70amAXsQ+RGzXsPvk9HbBgTLbp/bQBmpH5mOmsUvqXU+YEWVU0ZLewsmzOET/8jWswjDQ== dependencies: commander "^2.20.3" debug "^4.3.1" filing-cabinet "^3.0.1" - precinct "^8.0.0" - typescript "^3.9.7" + precinct "^9.0.0" + typescript "^4.0.0" deprecation@^2.0.0, deprecation@^2.3.1: version "2.3.1" @@ -2170,6 +2206,16 @@ detective-amd@^3.1.0: get-amd-module-type "^3.0.0" node-source-walk "^4.2.0" +detective-amd@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/detective-amd/-/detective-amd-4.0.1.tgz#1a827d9e4fa2f832506bd87aa392f124155bca3a" + integrity sha512-bDo22IYbJ8yzALB0Ow5CQLtyhU1BpDksLB9dsWHI9Eh0N3OQR6aQqhjPsNDd69ncYwRfL1sTo7OA9T3VRVSe2Q== + dependencies: + ast-module-types "^3.0.0" + escodegen "^2.0.0" + get-amd-module-type "^4.0.0" + node-source-walk "^5.0.0" + detective-cjs@^3.1.1: version "3.1.3" resolved "https://registry.yarnpkg.com/detective-cjs/-/detective-cjs-3.1.3.tgz#50e107d67b37f459b0ec02966ceb7e20a73f268b" @@ -2178,13 +2224,28 @@ detective-cjs@^3.1.1: ast-module-types "^3.0.0" node-source-walk "^4.0.0" -detective-es6@^2.2.0, detective-es6@^2.2.1: +detective-cjs@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/detective-cjs/-/detective-cjs-4.0.0.tgz#316e2c7ae14276a5dcfe0c43dc05d8cf9a0e5cf9" + integrity sha512-VsD6Yo1+1xgxJWoeDRyut7eqZ8EWaJI70C5eanSAPcBHzenHZx0uhjxaaEfIm0cHII7dBiwU98Orh44bwXN2jg== + dependencies: + ast-module-types "^3.0.0" + node-source-walk "^5.0.0" + +detective-es6@^2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/detective-es6/-/detective-es6-2.2.2.tgz#ee5f880981d9fecae9a694007029a2f6f26d8d28" integrity sha512-eZUKCUsbHm8xoeoCM0z6JFwvDfJ5Ww5HANo+jPR7AzkFpW9Mun3t/TqIF2jjeWa2TFbAiGaWESykf2OQp3oeMw== dependencies: node-source-walk "^4.0.0" +detective-es6@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/detective-es6/-/detective-es6-3.0.0.tgz#78c9ef5492d5b59748b411aecaaf52b5ca0f0bc2" + integrity sha512-Uv2b5Uih7vorYlqGzCX+nTPUb4CMzUAn3VPHTV5p5lBkAN4cAApLGgUz4mZE2sXlBfv4/LMmeP7qzxHV/ZcfWA== + dependencies: + node-source-walk "^5.0.0" + detective-less@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/detective-less/-/detective-less-1.0.2.tgz#a68af9ca5f69d74b7d0aa190218b211d83b4f7e3" @@ -2204,14 +2265,14 @@ detective-postcss@^4.0.0: postcss "^8.1.7" postcss-values-parser "^2.0.1" -detective-postcss@^5.0.0: - version "5.1.1" - resolved "https://registry.yarnpkg.com/detective-postcss/-/detective-postcss-5.1.1.tgz#ec23ac3818f8be95ac3a38a8b9f3b6d43103ef87" - integrity sha512-YJMsvA0Y6/ST9abMNcQytl9iFQ2bfu4I7B74IUiAvyThfaI9Y666yipL+SrqfReoIekeIEwmGH72oeqX63mwUw== +detective-postcss@^6.0.1, detective-postcss@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/detective-postcss/-/detective-postcss-6.1.0.tgz#01ca6f83dbc3158540fb0e6a6c631f3998946e54" + integrity sha512-ZFZnEmUrL2XHAC0j/4D1fdwZbo/anAcK84soJh7qc7xfx2Kc8gFO5Bk5I9jU7NLC/OAF1Yho1GLxEDnmQnRH2A== dependencies: is-url "^1.2.4" - postcss "^8.4.6" - postcss-values-parser "^5.0.0" + postcss "^8.4.12" + postcss-values-parser "^6.0.2" detective-sass@^3.0.1: version "3.0.2" @@ -2221,6 +2282,14 @@ detective-sass@^3.0.1: gonzales-pe "^4.3.0" node-source-walk "^4.0.0" +detective-sass@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/detective-sass/-/detective-sass-4.0.1.tgz#2a9e303a0bd472d00aaa79512334845e3acbaffc" + integrity sha512-80zfpxux1krOrkxCHbtwvIs2gNHUBScnSqlGl0FvUuHVz8HD6vD2ov66OroMctyvzhM67fxhuEeVjIk18s6yTQ== + dependencies: + gonzales-pe "^4.3.0" + node-source-walk "^5.0.0" + detective-scss@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/detective-scss/-/detective-scss-2.0.2.tgz#7d2a642616d44bf677963484fa8754d9558b8235" @@ -2229,11 +2298,24 @@ detective-scss@^2.0.1: gonzales-pe "^4.3.0" node-source-walk "^4.0.0" +detective-scss@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/detective-scss/-/detective-scss-3.0.0.tgz#c3c7bc4799f51515a4f0ed1e8ca491151364230f" + integrity sha512-37MB/mhJyS45ngqfzd6eTbuLMoDgdZnH03ZOMW2m9WqJ/Rlbuc8kZAr0Ypovaf1DJiTRzy5mmxzOTja85jbzlA== + dependencies: + gonzales-pe "^4.3.0" + node-source-walk "^5.0.0" + detective-stylus@^1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/detective-stylus/-/detective-stylus-1.0.3.tgz#20a702936c9fd7d4203fd7a903314b5dd43ac713" integrity sha512-4/bfIU5kqjwugymoxLXXLltzQNeQfxGoLm2eIaqtnkWxqbhap9puDVpJPVDx96hnptdERzS5Cy6p9N8/08A69Q== +detective-stylus@^2.0.0, detective-stylus@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/detective-stylus/-/detective-stylus-2.0.1.tgz#d528dfa7ef3c4eb2fbc9a7249d54906ec4e05d09" + integrity sha512-/Tvs1pWLg8eYwwV6kZQY5IslGaYqc/GACxjcaGudiNtN5nKCH6o2WnJK3j0gA3huCnoQcbv8X7oz/c1lnvE3zQ== + detective-typescript@^7.0.0: version "7.0.2" resolved "https://registry.yarnpkg.com/detective-typescript/-/detective-typescript-7.0.2.tgz#c6e00b4c28764741ef719662250e6b014a5f3c8e" @@ -2244,6 +2326,16 @@ detective-typescript@^7.0.0: node-source-walk "^4.2.0" typescript "^3.9.10" +detective-typescript@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/detective-typescript/-/detective-typescript-9.0.0.tgz#57d674cec49ec775460ab975b5bcbb5c2d32ff8e" + integrity sha512-lR78AugfUSBojwlSRZBeEqQ1l8LI7rbxOl1qTUnGLcjZQDjZmrZCb7R46rK8U8B5WzFvJrxa7fEBA8FoD/n5fA== + dependencies: + "@typescript-eslint/typescript-estree" "^5.13.0" + ast-module-types "^3.0.0" + node-source-walk "^5.0.0" + typescript "^4.5.5" + dezalgo@^1.0.0: version "1.0.4" resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.4.tgz#751235260469084c132157dfa857f386d4c33d81" @@ -2937,6 +3029,14 @@ get-amd-module-type@^3.0.0: ast-module-types "^3.0.0" node-source-walk "^4.2.2" +get-amd-module-type@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/get-amd-module-type/-/get-amd-module-type-4.0.0.tgz#3d4e5b44eec81f8337157d7c52c4fa9389aff78b" + integrity sha512-GbBawUCuA2tY8ztiMiVo3e3P95gc2TVrfYFfpUHdHQA8WyxMCckK29bQsVKhYX8SUf+w6JLhL2LG8tSC0ANt9Q== + dependencies: + ast-module-types "^3.0.0" + node-source-walk "^5.0.0" + get-caller-file@^1.0.1: version "1.0.3" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.3.tgz#f978fa4c90d1dfe7ff2d6beda2a515e713bdcf4a" @@ -3148,13 +3248,6 @@ graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6 resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== -graphviz@0.0.9: - version "0.0.9" - resolved "https://registry.yarnpkg.com/graphviz/-/graphviz-0.0.9.tgz#0bbf1df588c6a92259282da35323622528c4bbc4" - integrity sha512-SmoY2pOtcikmMCqCSy2NO1YsRfu9OO0wpTlOYW++giGjfX1a6gax/m1Fo8IdUd0/3H15cTOfR1SMKwohj4LKsg== - dependencies: - temp "~0.4.0" - handlebars@^4.7.6: version "4.7.7" resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.7.tgz#9ce33416aad02dbd6c8fafa8240d5d98004945a1" @@ -4167,31 +4260,32 @@ macos-release@^2.2.0: resolved "https://registry.yarnpkg.com/macos-release/-/macos-release-2.5.0.tgz#067c2c88b5f3fb3c56a375b2ec93826220fa1ff2" integrity sha512-EIgv+QZ9r+814gjJj0Bt5vSLJLzswGmSUbUpbi9AIr/fsN2IWFBl2NucV9PAiek+U1STK468tEkxmVYUtuAN3g== -madge@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/madge/-/madge-5.0.1.tgz#2096d9006558ea0669b3ade89c2cda708a24e22b" - integrity sha512-krmSWL9Hkgub74bOjnjWRoFPAJvPwSG6Dbta06qhWOq6X/n/FPzO3ESZvbFYVIvG2g4UHXvCJN1b+RZLaSs9nA== +madge@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/madge/-/madge-6.0.0.tgz#d17d68d1023376318cae89abd16629d329f4ed0a" + integrity sha512-dddxP62sj5pL+l9MJnq9C34VFqmRj+2+uSOdn/7lOTSliLRH0WyQ8uCEF3VxkPRNOBvMKK2xumnIE15HRSAL9A== dependencies: chalk "^4.1.1" commander "^7.2.0" commondir "^1.0.1" debug "^4.3.1" - dependency-tree "^8.1.1" - detective-amd "^3.1.0" - detective-cjs "^3.1.1" - detective-es6 "^2.2.0" + dependency-tree "^9.0.0" + detective-amd "^4.0.1" + detective-cjs "^4.0.0" + detective-es6 "^3.0.0" detective-less "^1.0.2" - detective-postcss "^5.0.0" - detective-sass "^3.0.1" - detective-scss "^2.0.1" - detective-stylus "^1.0.0" - detective-typescript "^7.0.0" - graphviz "0.0.9" + detective-postcss "^6.1.0" + detective-sass "^4.0.1" + detective-scss "^3.0.0" + detective-stylus "^2.0.1" + detective-typescript "^9.0.0" ora "^5.4.1" pluralize "^8.0.0" precinct "^8.1.0" pretty-ms "^7.0.1" rc "^1.2.7" + stream-to-array "^2.3.0" + ts-graphviz "^1.5.0" typescript "^3.9.5" walkdir "^0.4.1" @@ -4481,6 +4575,14 @@ module-definition@^3.3.1: ast-module-types "^3.0.0" node-source-walk "^4.0.0" +module-definition@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/module-definition/-/module-definition-4.0.0.tgz#5b39cca9be28c5b0fec768eb2d9fd8de08a2550b" + integrity sha512-wntiAHV4lDn24BQn2kX6LKq0y85phHLHiv3aOPDF+lIs06kVjEMTe/ZTdrbVLnQV5FQsjik21taknvMhKY1Cug== + dependencies: + ast-module-types "^3.0.0" + node-source-walk "^5.0.0" + module-lookup-amd@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/module-lookup-amd/-/module-lookup-amd-7.0.1.tgz#d67c1a93f2ff8e38b8774b99a638e9a4395774b2" @@ -4616,6 +4718,13 @@ node-source-walk@^4.0.0, node-source-walk@^4.2.0, node-source-walk@^4.2.2: dependencies: "@babel/parser" "^7.0.0" +node-source-walk@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/node-source-walk/-/node-source-walk-5.0.0.tgz#7cf93a0d12408081531fc440a00d7019eb3d5665" + integrity sha512-58APXoMXpmmU+oVBJFajhTCoD8d/OGtngnVAWzIo2A8yn0IXwBzvIVIsTzoie/SrA37u+1hnpNz2HMWx/VIqlw== + dependencies: + "@babel/parser" "^7.0.0" + "nopt@2 || 3": version "3.0.6" resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9" @@ -5246,16 +5355,16 @@ postcss-values-parser@^2.0.1: indexes-of "^1.0.1" uniq "^1.0.1" -postcss-values-parser@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/postcss-values-parser/-/postcss-values-parser-5.0.0.tgz#10c61ac3f488e4de25746b829ea8d8894e9ac3d2" - integrity sha512-2viDDjMMrt21W2izbeiJxl3kFuD/+asgB0CBwPEgSyhCmBnDIa/y+pLaoyX+q3I3DHH0oPPL3cgjVTQvlS1Maw== +postcss-values-parser@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/postcss-values-parser/-/postcss-values-parser-6.0.2.tgz#636edc5b86c953896f1bb0d7a7a6615df00fb76f" + integrity sha512-YLJpK0N1brcNJrs9WatuJFtHaV9q5aAOj+S4DI5S7jgHlRfm0PIbDCAFRYMQD5SHq7Fy6xsDhyutgS0QOAs0qw== dependencies: color-name "^1.1.4" is-url-superb "^4.0.0" quote-unquote "^1.0.0" -postcss@^8.1.7, postcss@^8.4.6: +postcss@^8.1.7: version "8.4.16" resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.16.tgz#33a1d675fac39941f5f445db0de4db2b6e01d43c" integrity sha512-ipHE1XBvKzm5xI7hiHCZJCSugxvsdq2mPnsq5+UF+VHCjiBvtDrlxJfMBToWaP9D5XlgNmcFGqoHmUn0EYEaRQ== @@ -5264,7 +5373,16 @@ postcss@^8.1.7, postcss@^8.4.6: picocolors "^1.0.0" source-map-js "^1.0.2" -precinct@^8.0.0, precinct@^8.1.0: +postcss@^8.4.12: + version "8.4.21" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.21.tgz#c639b719a57efc3187b13a1d765675485f4134f4" + integrity sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg== + dependencies: + nanoid "^3.3.4" + picocolors "^1.0.0" + source-map-js "^1.0.2" + +precinct@^8.1.0: version "8.3.1" resolved "https://registry.yarnpkg.com/precinct/-/precinct-8.3.1.tgz#94b99b623df144eed1ce40e0801c86078466f0dc" integrity sha512-pVppfMWLp2wF68rwHqBIpPBYY8Kd12lDhk8LVQzOwqllifVR15qNFyod43YLyFpurKRZQKnE7E4pofAagDOm2Q== @@ -5283,6 +5401,24 @@ precinct@^8.0.0, precinct@^8.1.0: module-definition "^3.3.1" node-source-walk "^4.2.0" +precinct@^9.0.0: + version "9.0.1" + resolved "https://registry.yarnpkg.com/precinct/-/precinct-9.0.1.tgz#64e7ea0de4bea1b73572e50531dfe297c7b8a7c7" + integrity sha512-hVNS6JvfvlZ64B3ezKeGAcVhIuOvuAiSVzagHX/+KjVPkYWoCNkfyMgCl1bjDtAFQSlzi95NcS9ykUWrl1L1vA== + dependencies: + commander "^9.1.0" + detective-amd "^4.0.1" + detective-cjs "^4.0.0" + detective-es6 "^3.0.0" + detective-less "^1.0.2" + detective-postcss "^6.0.1" + detective-sass "^4.0.1" + detective-scss "^3.0.0" + detective-stylus "^2.0.0" + detective-typescript "^9.0.0" + module-definition "^4.0.0" + node-source-walk "^5.0.0" + prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" @@ -6168,6 +6304,13 @@ stream-shift@^1.0.0: resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== +stream-to-array@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/stream-to-array/-/stream-to-array-2.3.0.tgz#bbf6b39f5f43ec30bc71babcb37557acecf34353" + integrity sha512-UsZtOYEn4tWU2RGLOXr/o/xjRBftZRlG3dEWoaHr8j4GuypJ3isitGbVyjQKAuMu+xbiop8q224TjiZWc4XTZA== + dependencies: + any-promise "^1.1.0" + strict-uri-encode@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz#b9c7330c7042862f6b142dc274bbcc5866ce3546" @@ -6401,11 +6544,6 @@ temp-write@^3.4.0: temp-dir "^1.0.0" uuid "^3.0.1" -temp@~0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/temp/-/temp-0.4.0.tgz#671ad63d57be0fe9d7294664b3fc400636678a60" - integrity sha512-IsFisGgDKk7qzK9erMIkQe/XwiSUdac7z3wYOsjcLkhPBy3k1SlvLoIh2dAHIlEpgA971CgguMrx9z8fFg7tSA== - text-extensions@^1.0.0: version "1.9.0" resolved "https://registry.yarnpkg.com/text-extensions/-/text-extensions-1.9.0.tgz#1853e45fee39c945ce6f6c36b2d659b5aabc2a26" @@ -6523,6 +6661,11 @@ trim-newlines@^3.0.0: resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144" integrity sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw== +ts-graphviz@^1.5.0: + version "1.5.4" + resolved "https://registry.yarnpkg.com/ts-graphviz/-/ts-graphviz-1.5.4.tgz#61a3059afeac4f6d4be3c6729a4d88546ca9e095" + integrity sha512-oxhI6wfPQBJC8WSiP0AjDI8z+9hcc9yl1etaynQJmQ2Yivn0KlT0oImLzJct7Es0TR/6xphzvJBAhGzgOjTGmQ== + tsconfig-paths@^3.10.1: version "3.14.1" resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz#ba0734599e8ea36c862798e920bcf163277b137a" @@ -6606,6 +6749,11 @@ typescript@^3.9.10, typescript@^3.9.5, typescript@^3.9.7: resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.10.tgz#70f3910ac7a51ed6bef79da7800690b19bf778b8" integrity sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q== +typescript@^4.0.0, typescript@^4.5.5: + version "4.9.5" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a" + integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== + uglify-js@^3.1.4: version "3.16.1" resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.16.1.tgz#0e7ec928b3d0b1e1d952bce634c384fd56377317"