diff --git a/artifacthub-repo.yml b/artifacthub-repo.yml new file mode 100644 index 0000000000..304b2bc79b --- /dev/null +++ b/artifacthub-repo.yml @@ -0,0 +1,10 @@ +# Artifact Hub repository metadata file +# This file is used to verify ownership of the budibase Helm chart repo +# so that we appear as a verified owner on artifacthub.io + +repositoryID: a7536764-e72e-4961-87d8-efe7c8dedfa3 +owners: # (optional, used to claim repository ownership) + - name: Martin + email: budimaster@budibase.com + - name: DevOps + email: devops@budibase.com \ No newline at end of file diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 72d9fc93a9..2f8242a030 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -158,6 +158,10 @@ spec: {{ end }} - name: CDN_URL value: {{ .Values.globals.cdnUrl }} + {{ if .Values.services.tlsRejectUnauthorized }} + - name: NODE_TLS_REJECT_UNAUTHORIZED + value: {{ .Values.services.tlsRejectUnauthorized }} + {{ end }} image: budibase/apps:{{ .Values.globals.appVersion }} imagePullPolicy: Always diff --git a/charts/budibase/templates/minio-service-deployment.yaml b/charts/budibase/templates/minio-service-deployment.yaml index 144dbe539a..d0a367653d 100644 --- a/charts/budibase/templates/minio-service-deployment.yaml +++ b/charts/budibase/templates/minio-service-deployment.yaml @@ -42,6 +42,7 @@ spec: secretKeyRef: name: {{ template "budibase.fullname" . }} key: objectStoreSecret + image: minio/minio imagePullPolicy: "" livenessProbe: diff --git a/charts/budibase/templates/redis-service-deployment.yaml b/charts/budibase/templates/redis-service-deployment.yaml index d94e4d70f8..5916c6d3f9 100644 --- a/charts/budibase/templates/redis-service-deployment.yaml +++ b/charts/budibase/templates/redis-service-deployment.yaml @@ -60,5 +60,6 @@ spec: - name: redis-data persistentVolumeClaim: claimName: redis-data + status: {} {{- end }} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index df692a0723..5e8578212d 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -149,6 +149,10 @@ spec: {{ end }} - name: CDN_URL value: {{ .Values.globals.cdnUrl }} + {{ if .Values.services.tlsRejectUnauthorized }} + - name: NODE_TLS_REJECT_UNAUTHORIZED + value: {{ .Values.services.tlsRejectUnauthorized }} + {{ end }} image: budibase/worker:{{ .Values.globals.appVersion }} imagePullPolicy: Always diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index a2a761aa86..2cf2767f12 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -110,6 +110,7 @@ globals: services: budibaseVersion: latest dns: cluster.local + # tlsRejectUnauthorized: 0 proxy: port: 10000 diff --git a/lerna.json b/lerna.json index e26819bf33..1d8589b8f4 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.1.32-alpha.11", + "version": "2.1.43-alpha.5", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index af513fc8dd..6c147698ad 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "private": true, "devDependencies": { "@rollup/plugin-json": "^4.0.2", - "@typescript-eslint/parser": "4.28.0", + "@typescript-eslint/parser": "5.45.0", "babel-eslint": "^10.0.3", "eslint": "^7.28.0", "eslint-plugin-cypress": "^2.11.3", @@ -87,4 +87,4 @@ "install:pro": "bash scripts/pro/install.sh", "dep:clean": "yarn clean && yarn bootstrap" } -} \ No newline at end of file +} diff --git a/packages/backend-core/accounts.js b/packages/backend-core/accounts.js deleted file mode 100644 index 47ad03456a..0000000000 --- a/packages/backend-core/accounts.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./src/cloud/accounts") diff --git a/packages/backend-core/auth.js b/packages/backend-core/auth.js deleted file mode 100644 index bbfe3d41dd..0000000000 --- a/packages/backend-core/auth.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./src/auth") diff --git a/packages/backend-core/cache.js b/packages/backend-core/cache.js deleted file mode 100644 index c8bd3c9b6f..0000000000 --- a/packages/backend-core/cache.js +++ /dev/null @@ -1,9 +0,0 @@ -const generic = require("./src/cache/generic") - -module.exports = { - user: require("./src/cache/user"), - app: require("./src/cache/appMetadata"), - writethrough: require("./src/cache/writethrough"), - ...generic, - cache: generic, -} diff --git a/packages/backend-core/constants.js b/packages/backend-core/constants.js deleted file mode 100644 index 4abb7703db..0000000000 --- a/packages/backend-core/constants.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./src/constants") diff --git a/packages/backend-core/context.js b/packages/backend-core/context.js deleted file mode 100644 index c6fa87a337..0000000000 --- a/packages/backend-core/context.js +++ /dev/null @@ -1,24 +0,0 @@ -const { - getAppDB, - getDevAppDB, - getProdAppDB, - getAppId, - updateAppId, - doInAppContext, - doInTenant, - doInContext, -} = require("./src/context") - -const identity = require("./src/context/identity") - -module.exports = { - getAppDB, - getDevAppDB, - getProdAppDB, - getAppId, - updateAppId, - doInAppContext, - doInTenant, - identity, - doInContext, -} diff --git a/packages/backend-core/db.js b/packages/backend-core/db.js deleted file mode 100644 index f7004972d5..0000000000 --- a/packages/backend-core/db.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./src/db") diff --git a/packages/backend-core/deprovision.js b/packages/backend-core/deprovision.js deleted file mode 100644 index 672da214ff..0000000000 --- a/packages/backend-core/deprovision.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./src/context/deprovision") diff --git a/packages/backend-core/encryption.js b/packages/backend-core/encryption.js deleted file mode 100644 index 4ccb6e3a99..0000000000 --- a/packages/backend-core/encryption.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./src/security/encryption") diff --git a/packages/backend-core/logging.js b/packages/backend-core/logging.js deleted file mode 100644 index da40fe3100..0000000000 --- a/packages/backend-core/logging.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./src/logging") diff --git a/packages/backend-core/middleware.js b/packages/backend-core/middleware.js deleted file mode 100644 index 30fec96239..0000000000 --- a/packages/backend-core/middleware.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./src/middleware") diff --git a/packages/backend-core/migrations.js b/packages/backend-core/migrations.js deleted file mode 100644 index 2de19ebf65..0000000000 --- a/packages/backend-core/migrations.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./src/migrations") diff --git a/packages/backend-core/objectStore.js b/packages/backend-core/objectStore.js deleted file mode 100644 index 3ee433f224..0000000000 --- a/packages/backend-core/objectStore.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = { - ...require("./src/objectStore"), - ...require("./src/objectStore/utils"), -} diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index fdc41293eb..9544034e8f 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.1.32-alpha.11", + "version": "2.1.43-alpha.5", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -16,11 +16,11 @@ "prepack": "cp package.json dist", "build": "tsc -p tsconfig.build.json", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", - "test": "jest --coverage", + "test": "jest --coverage --maxWorkers=2", "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "2.1.32-alpha.11", + "@budibase/types": "2.1.43-alpha.5", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", diff --git a/packages/backend-core/permissions.js b/packages/backend-core/permissions.js deleted file mode 100644 index 42f37c9c7e..0000000000 --- a/packages/backend-core/permissions.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./src/security/permissions") diff --git a/packages/backend-core/plugins.js b/packages/backend-core/plugins.js deleted file mode 100644 index 018e214dcb..0000000000 --- a/packages/backend-core/plugins.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - ...require("./src/plugin"), -} diff --git a/packages/backend-core/redis.js b/packages/backend-core/redis.js deleted file mode 100644 index 1f7a48540a..0000000000 --- a/packages/backend-core/redis.js +++ /dev/null @@ -1,5 +0,0 @@ -module.exports = { - Client: require("./src/redis"), - utils: require("./src/redis/utils"), - clients: require("./src/redis/init"), -} diff --git a/packages/backend-core/roles.js b/packages/backend-core/roles.js deleted file mode 100644 index 158bcdb6b8..0000000000 --- a/packages/backend-core/roles.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./src/security/roles") diff --git a/packages/backend-core/sessions.js b/packages/backend-core/sessions.js deleted file mode 100644 index c07efa2380..0000000000 --- a/packages/backend-core/sessions.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./src/security/sessions") diff --git a/packages/backend-core/src/auth.ts b/packages/backend-core/src/auth/auth.ts similarity index 79% rename from packages/backend-core/src/auth.ts rename to packages/backend-core/src/auth/auth.ts index 5e1959e0c8..75e425bd0f 100644 --- a/packages/backend-core/src/auth.ts +++ b/packages/backend-core/src/auth/auth.ts @@ -1,16 +1,14 @@ -const passport = require("koa-passport") +const _passport = require("koa-passport") const LocalStrategy = require("passport-local").Strategy const JwtStrategy = require("passport-jwt").Strategy -import { getGlobalDB } from "./tenancy" +import { getGlobalDB } from "../tenancy" const refresh = require("passport-oauth2-refresh") -import { Config } from "./constants" -import { getScopedConfig } from "./db/utils" +import { Config } from "../constants" +import { getScopedConfig } from "../db" import { - jwt, + jwt as jwtPassport, local, authenticated, - google, - oidc, auditLog, tenancy, authError, @@ -21,22 +19,41 @@ import { builderOnly, builderOrAdmin, joiValidator, -} from "./middleware" -import { invalidateUser } from "./cache/user" + oidc, + google, +} from "../middleware" +import { invalidateUser } from "../cache/user" import { User } from "@budibase/types" -import { logAlert } from "./logging" +import { logAlert } from "../logging" +export { + auditLog, + authError, + internalApi, + ssoCallbackUrl, + adminOnly, + builderOnly, + builderOrAdmin, + joiValidator, + google, + oidc, +} from "../middleware" +export const buildAuthMiddleware = authenticated +export const buildTenancyMiddleware = tenancy +export const buildCsrfMiddleware = csrf +export const passport = _passport +export const jwt = require("jsonwebtoken") // Strategies -passport.use(new LocalStrategy(local.options, local.authenticate)) -if (jwt.options.secretOrKey) { - passport.use(new JwtStrategy(jwt.options, jwt.authenticate)) +_passport.use(new LocalStrategy(local.options, local.authenticate)) +if (jwtPassport.options.secretOrKey) { + _passport.use(new JwtStrategy(jwtPassport.options, jwtPassport.authenticate)) } else { logAlert("No JWT Secret supplied, cannot configure JWT strategy") } -passport.serializeUser((user: User, done: any) => done(null, user)) +_passport.serializeUser((user: User, done: any) => done(null, user)) -passport.deserializeUser(async (user: User, done: any) => { +_passport.deserializeUser(async (user: User, done: any) => { const db = getGlobalDB() try { @@ -115,7 +132,7 @@ async function refreshGoogleAccessToken( }) } -async function refreshOAuthToken( +export async function refreshOAuthToken( refreshToken: string, configType: string, configId: string @@ -152,7 +169,7 @@ async function refreshOAuthToken( return refreshResponse } -async function updateUserOAuth(userId: string, oAuthConfig: any) { +export async function updateUserOAuth(userId: string, oAuthConfig: any) { const details = { accessToken: oAuthConfig.accessToken, refreshToken: oAuthConfig.refreshToken, @@ -179,23 +196,3 @@ async function updateUserOAuth(userId: string, oAuthConfig: any) { console.error("Could not update OAuth details for current user", e) } } - -export = { - buildAuthMiddleware: authenticated, - passport, - google, - oidc, - jwt: require("jsonwebtoken"), - buildTenancyMiddleware: tenancy, - auditLog, - authError, - buildCsrfMiddleware: csrf, - internalApi, - refreshOAuthToken, - updateUserOAuth, - ssoCallbackUrl, - adminOnly, - builderOnly, - builderOrAdmin, - joiValidator, -} diff --git a/packages/backend-core/src/auth/index.ts b/packages/backend-core/src/auth/index.ts new file mode 100644 index 0000000000..306751af96 --- /dev/null +++ b/packages/backend-core/src/auth/index.ts @@ -0,0 +1 @@ +export * from "./auth" diff --git a/packages/backend-core/src/cache/appMetadata.js b/packages/backend-core/src/cache/appMetadata.ts similarity index 79% rename from packages/backend-core/src/cache/appMetadata.js rename to packages/backend-core/src/cache/appMetadata.ts index a7ff0d2fc1..d24c4a3140 100644 --- a/packages/backend-core/src/cache/appMetadata.js +++ b/packages/backend-core/src/cache/appMetadata.ts @@ -1,6 +1,6 @@ -const redis = require("../redis/init") -const { doWithDB } = require("../db") -const { DocumentType } = require("../db/constants") +import { getAppClient } from "../redis/init" +import { doWithDB, DocumentType } from "../db" +import { Database } from "@budibase/types" const AppState = { INVALID: "invalid", @@ -10,17 +10,17 @@ const EXPIRY_SECONDS = 3600 /** * The default populate app metadata function */ -const populateFromDB = async appId => { +async function populateFromDB(appId: string) { return doWithDB( appId, - db => { + (db: Database) => { return db.get(DocumentType.APP_METADATA) }, { skip_setup: true } ) } -const isInvalid = metadata => { +function isInvalid(metadata?: { state: string }) { return !metadata || metadata.state === AppState.INVALID } @@ -31,15 +31,15 @@ const isInvalid = metadata => { * @param {string} appId the id of the app to get metadata from. * @returns {object} the app metadata. */ -exports.getAppMetadata = async appId => { - const client = await redis.getAppClient() +export async function getAppMetadata(appId: string) { + const client = await getAppClient() // try cache let metadata = await client.get(appId) if (!metadata) { - let expiry = EXPIRY_SECONDS + let expiry: number | undefined = EXPIRY_SECONDS try { metadata = await populateFromDB(appId) - } catch (err) { + } catch (err: any) { // app DB left around, but no metadata, it is invalid if (err && err.status === 404) { metadata = { state: AppState.INVALID } @@ -74,11 +74,11 @@ exports.getAppMetadata = async appId => { * @param newMetadata {object|undefined} optional - can simply provide the new metadata to update with. * @return {Promise} will respond with success when cache is updated. */ -exports.invalidateAppMetadata = async (appId, newMetadata = null) => { +export async function invalidateAppMetadata(appId: string, newMetadata?: any) { if (!appId) { throw "Cannot invalidate if no app ID provided." } - const client = await redis.getAppClient() + const client = await getAppClient() await client.delete(appId) if (newMetadata) { await client.store(appId, newMetadata, EXPIRY_SECONDS) diff --git a/packages/backend-core/src/cache/base/index.ts b/packages/backend-core/src/cache/base/index.ts index f3216531f4..ab620a900e 100644 --- a/packages/backend-core/src/cache/base/index.ts +++ b/packages/backend-core/src/cache/base/index.ts @@ -1,6 +1,6 @@ import { getTenantId } from "../../context" -import redis from "../../redis/init" -import RedisWrapper from "../../redis" +import * as redis from "../../redis/init" +import { Client } from "../../redis" function generateTenantKey(key: string) { const tenantId = getTenantId() @@ -8,9 +8,9 @@ function generateTenantKey(key: string) { } export = class BaseCache { - client: RedisWrapper | undefined + client: Client | undefined - constructor(client: RedisWrapper | undefined = undefined) { + constructor(client: Client | undefined = undefined) { this.client = client } diff --git a/packages/backend-core/src/cache/generic.js b/packages/backend-core/src/cache/generic.js deleted file mode 100644 index 26ef0c6bb0..0000000000 --- a/packages/backend-core/src/cache/generic.js +++ /dev/null @@ -1,30 +0,0 @@ -const BaseCache = require("./base") - -const GENERIC = new BaseCache() - -exports.CacheKeys = { - CHECKLIST: "checklist", - INSTALLATION: "installation", - ANALYTICS_ENABLED: "analyticsEnabled", - UNIQUE_TENANT_ID: "uniqueTenantId", - EVENTS: "events", - BACKFILL_METADATA: "backfillMetadata", - EVENTS_RATE_LIMIT: "eventsRateLimit", -} - -exports.TTL = { - ONE_MINUTE: 600, - ONE_HOUR: 3600, - ONE_DAY: 86400, -} - -function performExport(funcName) { - return (...args) => GENERIC[funcName](...args) -} - -exports.keys = performExport("keys") -exports.get = performExport("get") -exports.store = performExport("store") -exports.delete = performExport("delete") -exports.withCache = performExport("withCache") -exports.bustCache = performExport("bustCache") diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts new file mode 100644 index 0000000000..d8a54e4a3f --- /dev/null +++ b/packages/backend-core/src/cache/generic.ts @@ -0,0 +1,30 @@ +const BaseCache = require("./base") + +const GENERIC = new BaseCache() + +export enum CacheKey { + CHECKLIST = "checklist", + INSTALLATION = "installation", + ANALYTICS_ENABLED = "analyticsEnabled", + UNIQUE_TENANT_ID = "uniqueTenantId", + EVENTS = "events", + BACKFILL_METADATA = "backfillMetadata", + EVENTS_RATE_LIMIT = "eventsRateLimit", +} + +export enum TTL { + ONE_MINUTE = 600, + ONE_HOUR = 3600, + ONE_DAY = 86400, +} + +function performExport(funcName: string) { + return (...args: any) => GENERIC[funcName](...args) +} + +export const keys = performExport("keys") +export const get = performExport("get") +export const store = performExport("store") +export const destroy = performExport("delete") +export const withCache = performExport("withCache") +export const bustCache = performExport("bustCache") diff --git a/packages/backend-core/src/cache/index.ts b/packages/backend-core/src/cache/index.ts new file mode 100644 index 0000000000..58928c271a --- /dev/null +++ b/packages/backend-core/src/cache/index.ts @@ -0,0 +1,5 @@ +export * as generic from "./generic" +export * as user from "./user" +export * as app from "./appMetadata" +export * as writethrough from "./writethrough" +export * from "./generic" diff --git a/packages/backend-core/src/cache/user.js b/packages/backend-core/src/cache/user.ts similarity index 68% rename from packages/backend-core/src/cache/user.js rename to packages/backend-core/src/cache/user.ts index 130da1915e..a128465cd6 100644 --- a/packages/backend-core/src/cache/user.js +++ b/packages/backend-core/src/cache/user.ts @@ -1,15 +1,16 @@ -const redis = require("../redis/init") -const { getTenantId, lookupTenantId, doWithGlobalDB } = require("../tenancy") -const env = require("../environment") -const accounts = require("../cloud/accounts") +import * as redis from "../redis/init" +import { getTenantId, lookupTenantId, doWithGlobalDB } from "../tenancy" +import env from "../environment" +import * as accounts from "../cloud/accounts" +import { Database } from "@budibase/types" const EXPIRY_SECONDS = 3600 /** * The default populate user function */ -const populateFromDB = async (userId, tenantId) => { - const user = await doWithGlobalDB(tenantId, db => db.get(userId)) +async function populateFromDB(userId: string, tenantId: string) { + const user = await doWithGlobalDB(tenantId, (db: Database) => db.get(userId)) user.budibaseAccess = true if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { const account = await accounts.getAccount(user.email) @@ -31,7 +32,11 @@ const populateFromDB = async (userId, tenantId) => { * @param {*} populateUser function to provide the user for re-caching. default to couch db * @returns */ -exports.getUser = async (userId, tenantId = null, populateUser = null) => { +export async function getUser( + userId: string, + tenantId?: string, + populateUser?: any +) { if (!populateUser) { populateUser = populateFromDB } @@ -47,7 +52,7 @@ exports.getUser = async (userId, tenantId = null, populateUser = null) => { let user = await client.get(userId) if (!user) { user = await populateUser(userId, tenantId) - client.store(userId, user, EXPIRY_SECONDS) + await client.store(userId, user, EXPIRY_SECONDS) } if (user && !user.tenantId && tenantId) { // make sure the tenant ID is always correct/set @@ -56,7 +61,7 @@ exports.getUser = async (userId, tenantId = null, populateUser = null) => { return user } -exports.invalidateUser = async userId => { +export async function invalidateUser(userId: string) { const client = await redis.getUserClient() await client.delete(userId) } diff --git a/packages/backend-core/src/cloud/api.js b/packages/backend-core/src/cloud/api.js deleted file mode 100644 index d4d4b6c8bb..0000000000 --- a/packages/backend-core/src/cloud/api.js +++ /dev/null @@ -1,42 +0,0 @@ -const fetch = require("node-fetch") -class API { - constructor(host) { - this.host = host - } - - apiCall = - method => - async (url = "", options = {}) => { - if (!options.headers) { - options.headers = {} - } - - if (!options.headers["Content-Type"]) { - options.headers = { - "Content-Type": "application/json", - Accept: "application/json", - ...options.headers, - } - } - - let json = options.headers["Content-Type"] === "application/json" - - const requestOptions = { - method: method, - body: json ? JSON.stringify(options.body) : options.body, - headers: options.headers, - // TODO: See if this is necessary - credentials: "include", - } - - return await fetch(`${this.host}${url}`, requestOptions) - } - - post = this.apiCall("POST") - get = this.apiCall("GET") - patch = this.apiCall("PATCH") - del = this.apiCall("DELETE") - put = this.apiCall("PUT") -} - -module.exports = API diff --git a/packages/backend-core/src/cloud/api.ts b/packages/backend-core/src/cloud/api.ts new file mode 100644 index 0000000000..287c447271 --- /dev/null +++ b/packages/backend-core/src/cloud/api.ts @@ -0,0 +1,55 @@ +import fetch from "node-fetch" + +export = class API { + host: string + + constructor(host: string) { + this.host = host + } + + async apiCall(method: string, url: string, options?: any) { + if (!options.headers) { + options.headers = {} + } + + if (!options.headers["Content-Type"]) { + options.headers = { + "Content-Type": "application/json", + Accept: "application/json", + ...options.headers, + } + } + + let json = options.headers["Content-Type"] === "application/json" + + const requestOptions = { + method: method, + body: json ? JSON.stringify(options.body) : options.body, + headers: options.headers, + // TODO: See if this is necessary + credentials: "include", + } + + return await fetch(`${this.host}${url}`, requestOptions) + } + + async post(url: string, options?: any) { + return this.apiCall("POST", url, options) + } + + async get(url: string, options?: any) { + return this.apiCall("GET", url, options) + } + + async patch(url: string, options?: any) { + return this.apiCall("PATCH", url, options) + } + + async del(url: string, options?: any) { + return this.apiCall("DELETE", url, options) + } + + async put(url: string, options?: any) { + return this.apiCall("PUT", url, options) + } +} diff --git a/packages/backend-core/src/constants.js b/packages/backend-core/src/constants.js deleted file mode 100644 index 7fda17f6f2..0000000000 --- a/packages/backend-core/src/constants.js +++ /dev/null @@ -1,44 +0,0 @@ -exports.UserStatus = { - ACTIVE: "active", - INACTIVE: "inactive", -} - -exports.Cookie = { - CurrentApp: "budibase:currentapp", - Auth: "budibase:auth", - Init: "budibase:init", - ACCOUNT_RETURN_URL: "budibase:account:returnurl", - DatasourceAuth: "budibase:datasourceauth", - OIDC_CONFIG: "budibase:oidc:config", -} - -exports.Header = { - API_KEY: "x-budibase-api-key", - LICENSE_KEY: "x-budibase-license-key", - API_VER: "x-budibase-api-version", - APP_ID: "x-budibase-app-id", - TYPE: "x-budibase-type", - PREVIEW_ROLE: "x-budibase-role", - TENANT_ID: "x-budibase-tenant-id", - TOKEN: "x-budibase-token", - CSRF_TOKEN: "x-csrf-token", -} - -exports.GlobalRoles = { - OWNER: "owner", - ADMIN: "admin", - BUILDER: "builder", - WORKSPACE_MANAGER: "workspace_manager", -} - -exports.Config = { - SETTINGS: "settings", - ACCOUNT: "account", - SMTP: "smtp", - GOOGLE: "google", - OIDC: "oidc", - OIDC_LOGOS: "logos_oidc", -} - -exports.MAX_VALID_DATE = new Date(2147483647000) -exports.DEFAULT_TENANT_ID = "default" diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/constants/db.ts similarity index 100% rename from packages/backend-core/src/db/constants.ts rename to packages/backend-core/src/constants/db.ts diff --git a/packages/backend-core/src/constants/index.ts b/packages/backend-core/src/constants/index.ts new file mode 100644 index 0000000000..62d5e08e63 --- /dev/null +++ b/packages/backend-core/src/constants/index.ts @@ -0,0 +1,2 @@ +export * from "./db" +export * from "./misc" diff --git a/packages/backend-core/src/constants.ts b/packages/backend-core/src/constants/misc.ts similarity index 100% rename from packages/backend-core/src/constants.ts rename to packages/backend-core/src/constants/misc.ts diff --git a/packages/backend-core/src/context/Context.ts b/packages/backend-core/src/context/Context.ts index 6ffb57e44e..f0ccdb97a8 100644 --- a/packages/backend-core/src/context/Context.ts +++ b/packages/backend-core/src/context/Context.ts @@ -1,18 +1,17 @@ import { AsyncLocalStorage } from "async_hooks" -import { ContextMap } from "./constants" export default class Context { - static storage = new AsyncLocalStorage() + static storage = new AsyncLocalStorage>() - static run(context: ContextMap, func: any) { + static run(context: Record, func: any) { return Context.storage.run(context, () => func()) } - static get(): ContextMap { - return Context.storage.getStore() as ContextMap + static get(): Record { + return Context.storage.getStore() as Record } - static set(context: ContextMap) { + static set(context: Record) { Context.storage.enterWith(context) } } diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts deleted file mode 100644 index 64fdb45dec..0000000000 --- a/packages/backend-core/src/context/constants.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { IdentityContext } from "@budibase/types" - -export type ContextMap = { - tenantId?: string - appId?: string - identity?: IdentityContext -} diff --git a/packages/backend-core/src/context/identity.ts b/packages/backend-core/src/context/identity.ts index 37e1ecf40a..648dd1b5fd 100644 --- a/packages/backend-core/src/context/identity.ts +++ b/packages/backend-core/src/context/identity.ts @@ -2,23 +2,22 @@ import { IdentityContext, IdentityType, User, - UserContext, isCloudAccount, Account, AccountUserContext, } from "@budibase/types" import * as context from "." -export const getIdentity = (): IdentityContext | undefined => { +export function getIdentity(): IdentityContext | undefined { return context.getIdentity() } -export const doInIdentityContext = (identity: IdentityContext, task: any) => { +export function doInIdentityContext(identity: IdentityContext, task: any) { return context.doInIdentityContext(identity, task) } -export const doInUserContext = (user: User, task: any) => { - const userContext: UserContext = { +export function doInUserContext(user: User, task: any) { + const userContext: any = { ...user, _id: user._id as string, type: IdentityType.USER, @@ -26,7 +25,7 @@ export const doInUserContext = (user: User, task: any) => { return doInIdentityContext(userContext, task) } -export const doInAccountContext = (account: Account, task: any) => { +export function doInAccountContext(account: Account, task: any) { const _id = getAccountUserId(account) const tenantId = account.tenantId const accountContext: AccountUserContext = { @@ -38,12 +37,12 @@ export const doInAccountContext = (account: Account, task: any) => { return doInIdentityContext(accountContext, task) } -export const getAccountUserId = (account: Account) => { +export function getAccountUserId(account: Account) { let userId: string if (isCloudAccount(account)) { userId = account.budibaseUserId } else { - // use account id as user id for self hosting + // use account id as user id for self-hosting userId = account.accountId } return userId diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index ce37d4f0b4..9c70363170 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -1,223 +1,3 @@ -import env from "../environment" -import { - SEPARATOR, - DocumentType, - getDevelopmentAppID, - getProdAppID, - baseGlobalDBName, - getDB, -} from "../db" -import Context from "./Context" -import { IdentityContext, Database } from "@budibase/types" -import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" -import { ContextMap } from "./constants" -export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID - -// some test cases call functions directly, need to -// store an app ID to pretend there is a context -let TEST_APP_ID: string | null = null - -export function isMultiTenant() { - return env.MULTI_TENANCY -} - -export function isTenantIdSet() { - const context = Context.get() - return !!context?.tenantId -} - -export function isTenancyEnabled() { - return env.MULTI_TENANCY -} - -/** - * Given an app ID this will attempt to retrieve the tenant ID from it. - * @return {null|string} The tenant ID found within the app ID. - */ -export function getTenantIDFromAppID(appId: string) { - if (!appId) { - return undefined - } - if (!isMultiTenant()) { - return DEFAULT_TENANT_ID - } - const split = appId.split(SEPARATOR) - const hasDev = split[1] === DocumentType.DEV - if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { - return undefined - } - if (hasDev) { - return split[2] - } else { - return split[1] - } -} - -function updateContext(updates: ContextMap) { - let context: ContextMap - try { - context = Context.get() - } catch (err) { - // no context, start empty - context = {} - } - context = { - ...context, - ...updates, - } - return context -} - -async function newContext(updates: ContextMap, task: any) { - // see if there already is a context setup - let context: ContextMap = updateContext(updates) - return Context.run(context, task) -} - -export async function doInContext(appId: string, task: any): Promise { - const tenantId = getTenantIDFromAppID(appId) - return newContext( - { - tenantId, - appId, - }, - task - ) -} - -export async function doInTenant( - tenantId: string | null, - task: any -): Promise { - // make sure default always selected in single tenancy - if (!env.MULTI_TENANCY) { - tenantId = tenantId || DEFAULT_TENANT_ID - } - - const updates = tenantId ? { tenantId } : {} - return newContext(updates, task) -} - -export async function doInAppContext(appId: string, task: any): Promise { - if (!appId) { - throw new Error("appId is required") - } - - const tenantId = getTenantIDFromAppID(appId) - const updates: ContextMap = { appId } - if (tenantId) { - updates.tenantId = tenantId - } - return newContext(updates, task) -} - -export async function doInIdentityContext( - identity: IdentityContext, - task: any -): Promise { - if (!identity) { - throw new Error("identity is required") - } - - const context: ContextMap = { - identity, - } - if (identity.tenantId) { - context.tenantId = identity.tenantId - } - return newContext(context, task) -} - -export function getIdentity(): IdentityContext | undefined { - try { - const context = Context.get() - return context?.identity - } catch (e) { - // do nothing - identity is not in context - } -} - -export function getTenantId(): string { - if (!isMultiTenant()) { - return DEFAULT_TENANT_ID - } - const context = Context.get() - const tenantId = context?.tenantId - if (!tenantId) { - throw new Error("Tenant id not found") - } - return tenantId -} - -export function getAppId(): string | undefined { - const context = Context.get() - const foundId = context?.appId - if (!foundId && env.isTest() && TEST_APP_ID) { - return TEST_APP_ID - } else { - return foundId - } -} - -export function updateTenantId(tenantId?: string) { - let context: ContextMap = updateContext({ - tenantId, - }) - Context.set(context) -} - -export function updateAppId(appId: string) { - let context: ContextMap = updateContext({ - appId, - }) - try { - Context.set(context) - } catch (err) { - if (env.isTest()) { - TEST_APP_ID = appId - } else { - throw err - } - } -} - -export function getGlobalDB(): Database { - const context = Context.get() - if (!context || (env.MULTI_TENANCY && !context.tenantId)) { - throw new Error("Global DB not found") - } - return getDB(baseGlobalDBName(context?.tenantId)) -} - -/** - * Gets the app database based on whatever the request - * contained, dev or prod. - */ -export function getAppDB(opts?: any): Database { - const appId = getAppId() - return getDB(appId, opts) -} - -/** - * This specifically gets the prod app ID, if the request - * contained a development app ID, this will get the prod one. - */ -export function getProdAppDB(opts?: any): Database { - const appId = getAppId() - if (!appId) { - throw new Error("Unable to retrieve prod DB - no app ID.") - } - return getDB(getProdAppID(appId), opts) -} - -/** - * This specifically gets the dev app ID, if the request - * contained a prod app ID, this will get the dev one. - */ -export function getDevAppDB(opts?: any): Database { - const appId = getAppId() - if (!appId) { - throw new Error("Unable to retrieve dev DB - no app ID.") - } - return getDB(getDevelopmentAppID(appId), opts) -} +export { DEFAULT_TENANT_ID } from "../constants" +export * as identity from "./identity" +export * from "./mainContext" diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts new file mode 100644 index 0000000000..d743d2f49b --- /dev/null +++ b/packages/backend-core/src/context/mainContext.ts @@ -0,0 +1,245 @@ +// some test cases call functions directly, need to +// store an app ID to pretend there is a context +import env from "../environment" +import Context from "./Context" +import { getDevelopmentAppID, getProdAppID } from "../db/conversions" +import { getDB } from "../db/db" +import { + DocumentType, + SEPARATOR, + StaticDatabases, + DEFAULT_TENANT_ID, +} from "../constants" +import { Database, IdentityContext } from "@budibase/types" + +export type ContextMap = { + tenantId?: string + appId?: string + identity?: IdentityContext +} + +let TEST_APP_ID: string | null = null + +export function getGlobalDBName(tenantId?: string) { + // tenant ID can be set externally, for example user API where + // new tenants are being created, this may be the case + if (!tenantId) { + tenantId = getTenantId() + } + return baseGlobalDBName(tenantId) +} + +export function baseGlobalDBName(tenantId: string | undefined | null) { + let dbName + if (!tenantId || tenantId === DEFAULT_TENANT_ID) { + dbName = StaticDatabases.GLOBAL.name + } else { + dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}` + } + return dbName +} + +export function isMultiTenant() { + return env.MULTI_TENANCY +} + +export function isTenantIdSet() { + const context = Context.get() + return !!context?.tenantId +} + +export function isTenancyEnabled() { + return env.MULTI_TENANCY +} + +/** + * Given an app ID this will attempt to retrieve the tenant ID from it. + * @return {null|string} The tenant ID found within the app ID. + */ +export function getTenantIDFromAppID(appId: string) { + if (!appId) { + return undefined + } + if (!isMultiTenant()) { + return DEFAULT_TENANT_ID + } + const split = appId.split(SEPARATOR) + const hasDev = split[1] === DocumentType.DEV + if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { + return undefined + } + if (hasDev) { + return split[2] + } else { + return split[1] + } +} + +function updateContext(updates: ContextMap) { + let context: ContextMap + try { + context = Context.get() + } catch (err) { + // no context, start empty + context = {} + } + context = { + ...context, + ...updates, + } + return context +} + +async function newContext(updates: ContextMap, task: any) { + // see if there already is a context setup + let context: ContextMap = updateContext(updates) + return Context.run(context, task) +} + +export async function doInContext(appId: string, task: any): Promise { + const tenantId = getTenantIDFromAppID(appId) + return newContext( + { + tenantId, + appId, + }, + task + ) +} + +export async function doInTenant( + tenantId: string | null, + task: any +): Promise { + // make sure default always selected in single tenancy + if (!env.MULTI_TENANCY) { + tenantId = tenantId || DEFAULT_TENANT_ID + } + + const updates = tenantId ? { tenantId } : {} + return newContext(updates, task) +} + +export async function doInAppContext(appId: string, task: any): Promise { + if (!appId) { + throw new Error("appId is required") + } + + const tenantId = getTenantIDFromAppID(appId) + const updates: ContextMap = { appId } + if (tenantId) { + updates.tenantId = tenantId + } + return newContext(updates, task) +} + +export async function doInIdentityContext( + identity: IdentityContext, + task: any +): Promise { + if (!identity) { + throw new Error("identity is required") + } + + const context: ContextMap = { + identity, + } + if (identity.tenantId) { + context.tenantId = identity.tenantId + } + return newContext(context, task) +} + +export function getIdentity(): IdentityContext | undefined { + try { + const context = Context.get() + return context?.identity + } catch (e) { + // do nothing - identity is not in context + } +} + +export function getTenantId(): string { + if (!isMultiTenant()) { + return DEFAULT_TENANT_ID + } + const context = Context.get() + const tenantId = context?.tenantId + if (!tenantId) { + throw new Error("Tenant id not found") + } + return tenantId +} + +export function getAppId(): string | undefined { + const context = Context.get() + const foundId = context?.appId + if (!foundId && env.isTest() && TEST_APP_ID) { + return TEST_APP_ID + } else { + return foundId + } +} + +export function updateTenantId(tenantId?: string) { + let context: ContextMap = updateContext({ + tenantId, + }) + Context.set(context) +} + +export function updateAppId(appId: string) { + let context: ContextMap = updateContext({ + appId, + }) + try { + Context.set(context) + } catch (err) { + if (env.isTest()) { + TEST_APP_ID = appId + } else { + throw err + } + } +} + +export function getGlobalDB(): Database { + const context = Context.get() + if (!context || (env.MULTI_TENANCY && !context.tenantId)) { + throw new Error("Global DB not found") + } + return getDB(baseGlobalDBName(context?.tenantId)) +} + +/** + * Gets the app database based on whatever the request + * contained, dev or prod. + */ +export function getAppDB(opts?: any): Database { + const appId = getAppId() + return getDB(appId, opts) +} + +/** + * This specifically gets the prod app ID, if the request + * contained a development app ID, this will get the prod one. + */ +export function getProdAppDB(opts?: any): Database { + const appId = getAppId() + if (!appId) { + throw new Error("Unable to retrieve prod DB - no app ID.") + } + return getDB(getProdAppID(appId), opts) +} + +/** + * This specifically gets the dev app ID, if the request + * contained a prod app ID, this will get the dev one. + */ +export function getDevAppDB(opts?: any): Database { + const appId = getAppId() + if (!appId) { + throw new Error("Unable to retrieve dev DB - no app ID.") + } + return getDB(getDevelopmentAppID(appId), opts) +} diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index 12f6001a70..eb9d613a58 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,5 +1,5 @@ -import { getPouchDB, closePouchDB } from "./couch/pouchDB" -import { DocumentType } from "./constants" +import { getPouchDB, closePouchDB } from "./couch" +import { DocumentType } from "../constants" class Replication { source: any diff --git a/packages/backend-core/src/db/conversions.ts b/packages/backend-core/src/db/conversions.ts index 48eaf31844..381c5cb90f 100644 --- a/packages/backend-core/src/db/conversions.ts +++ b/packages/backend-core/src/db/conversions.ts @@ -1,4 +1,4 @@ -import { APP_DEV_PREFIX, APP_PREFIX } from "./constants" +import { APP_DEV_PREFIX, APP_PREFIX } from "../constants" import { App } from "@budibase/types" const NO_APP_ERROR = "No app provided" diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index 7269aa8f92..0d9f75fa18 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -2,6 +2,8 @@ export * from "./couch" export * from "./db" export * from "./utils" export * from "./views" -export * from "./constants" export * from "./conversions" -export * from "./tenancy" +export { default as Replication } from "./Replication" +// exports to support old export structure +export * from "../constants/db" +export { getGlobalDBName, baseGlobalDBName } from "../context" diff --git a/packages/backend-core/src/db/tenancy.ts b/packages/backend-core/src/db/tenancy.ts deleted file mode 100644 index d920f7cd41..0000000000 --- a/packages/backend-core/src/db/tenancy.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { DEFAULT_TENANT_ID } from "../constants" -import { StaticDatabases, SEPARATOR } from "./constants" -import { getTenantId } from "../context" - -export const getGlobalDBName = (tenantId?: string) => { - // tenant ID can be set externally, for example user API where - // new tenants are being created, this may be the case - if (!tenantId) { - tenantId = getTenantId() - } - return baseGlobalDBName(tenantId) -} - -export const baseGlobalDBName = (tenantId: string | undefined | null) => { - let dbName - if (!tenantId || tenantId === DEFAULT_TENANT_ID) { - dbName = StaticDatabases.GLOBAL.name - } else { - dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}` - } - return dbName -} diff --git a/packages/backend-core/src/db/tests/utils.spec.js b/packages/backend-core/src/db/tests/utils.spec.js index 0d16e2dec2..f95889c1cc 100644 --- a/packages/backend-core/src/db/tests/utils.spec.js +++ b/packages/backend-core/src/db/tests/utils.spec.js @@ -1,10 +1,12 @@ require("../../../tests") const { - generateAppID, getDevelopmentAppID, getProdAppID, isDevAppID, isProdAppID, +} = require("../conversions") +const { + generateAppID, getPlatformUrl, getScopedConfig } = require("../utils") diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 04feafa008..590c3eeef8 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -1,26 +1,20 @@ -import { newid } from "../hashing" -import { DEFAULT_TENANT_ID, Config } from "../constants" +import { newid } from "../newid" import env from "../environment" import { + DEFAULT_TENANT_ID, SEPARATOR, DocumentType, UNICODE_MAX, ViewName, InternalTable, -} from "./constants" -import { getTenantId, getGlobalDB } from "../context" -import { getGlobalDBName } from "./tenancy" + APP_PREFIX, +} from "../constants" +import { getTenantId, getGlobalDB, getGlobalDBName } from "../context" import { doWithDB, allDbs, directCouchAllDbs } from "./db" import { getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" -import { APP_PREFIX } from "./constants" import * as events from "../events" -import { App, Database } from "@budibase/types" - -export * from "./constants" -export * from "./conversions" -export { default as Replication } from "./Replication" -export * from "./tenancy" +import { App, Database, ConfigType } from "@budibase/types" /** * Generates a new app ID. @@ -494,7 +488,7 @@ export const getScopedFullConfig = async function ( )[0] // custom logic for settings doc - if (type === Config.SETTINGS) { + if (type === ConfigType.SETTINGS) { if (scopedConfig && scopedConfig.doc) { // overrides affected by environment variables scopedConfig.doc.config.platformUrl = await getPlatformUrl({ @@ -533,7 +527,7 @@ export const getPlatformUrl = async (opts = { tenantAware: true }) => { // get the doc directly instead of with getScopedConfig to prevent loop let settings try { - settings = await db.get(generateConfigID({ type: Config.SETTINGS })) + settings = await db.get(generateConfigID({ type: ConfigType.SETTINGS })) } catch (e: any) { if (e.status !== 404) { throw e diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index c563d55be3..4a87be0a68 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -1,6 +1,11 @@ -import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils" +import { + DocumentType, + ViewName, + DeprecatedViews, + SEPARATOR, + StaticDatabases, +} from "../constants" import { getGlobalDB } from "../context" -import { StaticDatabases } from "./constants" import { doWithDB } from "./" import { Database, DatabaseQueryOpts } from "@budibase/types" diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 2443287d5a..51ab101b3c 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -25,7 +25,7 @@ const DefaultBucketName = { PLUGINS: "plugins", } -const env = { +const environment = { isTest, isDev, JS_BCRYPT: process.env.JS_BCRYPT, @@ -75,17 +75,18 @@ const env = { process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose", _set(key: any, value: any) { process.env[key] = value - module.exports[key] = value + // @ts-ignore + environment[key] = value }, } // clean up any environment variable edge cases -for (let [key, value] of Object.entries(env)) { +for (let [key, value] of Object.entries(environment)) { // handle the edge case of "0" to disable an environment variable if (value === "0") { // @ts-ignore - env[key] = 0 + environment[key] = 0 } } -export = env +export = environment diff --git a/packages/backend-core/src/events/analytics.ts b/packages/backend-core/src/events/analytics.ts index 228805ef82..f621a9c98b 100644 --- a/packages/backend-core/src/events/analytics.ts +++ b/packages/backend-core/src/events/analytics.ts @@ -1,8 +1,8 @@ import env from "../environment" -import tenancy from "../tenancy" +import * as tenancy from "../tenancy" import * as dbUtils from "../db/utils" import { Config } from "../constants" -import { withCache, TTL, CacheKeys } from "../cache/generic" +import { withCache, TTL, CacheKey } from "../cache" export const enabled = async () => { // cloud - always use the environment variable @@ -13,7 +13,7 @@ export const enabled = async () => { // self host - prefer the settings doc // use cache as events have high throughput const enabledInDB = await withCache( - CacheKeys.ANALYTICS_ENABLED, + CacheKey.ANALYTICS_ENABLED, TTL.ONE_DAY, async () => { const settings = await getSettingsDoc() diff --git a/packages/backend-core/src/events/backfill.ts b/packages/backend-core/src/events/backfill.ts index e4577c5ab4..c8025a8e4e 100644 --- a/packages/backend-core/src/events/backfill.ts +++ b/packages/backend-core/src/events/backfill.ts @@ -21,7 +21,7 @@ import { AppCreatedEvent, } from "@budibase/types" import * as context from "../context" -import { CacheKeys } from "../cache/generic" +import { CacheKey } from "../cache/generic" import * as cache from "../cache/generic" // LIFECYCLE @@ -48,18 +48,18 @@ export const end = async () => { // CRUD const getBackfillMetadata = async (): Promise => { - return cache.get(CacheKeys.BACKFILL_METADATA) + return cache.get(CacheKey.BACKFILL_METADATA) } const saveBackfillMetadata = async ( backfill: BackfillMetadata ): Promise => { // no TTL - deleted by backfill - return cache.store(CacheKeys.BACKFILL_METADATA, backfill) + return cache.store(CacheKey.BACKFILL_METADATA, backfill) } const deleteBackfillMetadata = async (): Promise => { - await cache.delete(CacheKeys.BACKFILL_METADATA) + await cache.destroy(CacheKey.BACKFILL_METADATA) } const clearEvents = async () => { @@ -70,7 +70,7 @@ const clearEvents = async () => { for (const key of keys) { // delete each key // don't use tenancy, already in the key - await cache.delete(key, { useTenancy: false }) + await cache.destroy(key, { useTenancy: false }) } } @@ -167,7 +167,7 @@ const getEventKey = (event?: Event, properties?: any) => { const tenantId = context.getTenantId() if (event) { - eventKey = `${CacheKeys.EVENTS}:${tenantId}:${event}` + eventKey = `${CacheKey.EVENTS}:${tenantId}:${event}` // use some properties to make the key more unique const custom = CUSTOM_PROPERTY_SUFFIX[event] @@ -176,7 +176,7 @@ const getEventKey = (event?: Event, properties?: any) => { eventKey = `${eventKey}:${suffix}` } } else { - eventKey = `${CacheKeys.EVENTS}:${tenantId}:*` + eventKey = `${CacheKey.EVENTS}:${tenantId}:*` } return eventKey diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index 0b4b043837..b93bd44968 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -20,9 +20,9 @@ import { import { processors } from "./processors" import * as dbUtils from "../db/utils" import { Config } from "../constants" -import * as hashing from "../hashing" +import { newid } from "../utils" import * as installation from "../installation" -import { withCache, TTL, CacheKeys } from "../cache/generic" +import { withCache, TTL, CacheKey } from "../cache/generic" const pkg = require("../../package.json") @@ -270,7 +270,7 @@ const getEventTenantId = async (tenantId: string): Promise => { const getUniqueTenantId = async (tenantId: string): Promise => { // make sure this tenantId always matches the tenantId in context return context.doInTenant(tenantId, () => { - return withCache(CacheKeys.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => { + return withCache(CacheKey.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => { const db = context.getGlobalDB() const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, { type: Config.SETTINGS, @@ -280,7 +280,7 @@ const getUniqueTenantId = async (tenantId: string): Promise => { if (config.config.uniqueTenantId) { return config.config.uniqueTenantId } else { - uniqueTenantId = `${hashing.newid()}_${tenantId}` + uniqueTenantId = `${newid()}_${tenantId}` config.config.uniqueTenantId = uniqueTenantId await db.put(config) return uniqueTenantId diff --git a/packages/backend-core/src/events/processors/posthog/rateLimiting.ts b/packages/backend-core/src/events/processors/posthog/rateLimiting.ts index 9c7b7876d6..89da10defa 100644 --- a/packages/backend-core/src/events/processors/posthog/rateLimiting.ts +++ b/packages/backend-core/src/events/processors/posthog/rateLimiting.ts @@ -1,5 +1,5 @@ import { Event } from "@budibase/types" -import { CacheKeys, TTL } from "../../../cache/generic" +import { CacheKey, TTL } from "../../../cache/generic" import * as cache from "../../../cache/generic" import * as context from "../../../context" @@ -74,7 +74,7 @@ export const limited = async (event: Event): Promise => { } const eventKey = (event: RateLimitedEvent) => { - let key = `${CacheKeys.EVENTS_RATE_LIMIT}:${event}` + let key = `${CacheKey.EVENTS_RATE_LIMIT}:${event}` if (isPerApp(event)) { key = key + ":" + context.getAppId() } diff --git a/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts index c9c4ceffe3..349a0427ac 100644 --- a/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts +++ b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts @@ -3,7 +3,7 @@ import PosthogProcessor from "../PosthogProcessor" import { Event, IdentityType, Hosting } from "@budibase/types" const tk = require("timekeeper") import * as cache from "../../../../cache/generic" -import { CacheKeys } from "../../../../cache/generic" +import { CacheKey } from "../../../../cache/generic" import * as context from "../../../../context" const newIdentity = () => { @@ -19,7 +19,7 @@ describe("PosthogProcessor", () => { beforeEach(async () => { jest.clearAllMocks() await cache.bustCache( - `${CacheKeys.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}` + `${CacheKey.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}` ) }) @@ -89,7 +89,7 @@ describe("PosthogProcessor", () => { await processor.processEvent(Event.SERVED_BUILDER, identity, properties) await cache.bustCache( - `${CacheKeys.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}` + `${CacheKey.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}` ) tk.freeze(new Date(2022, 0, 1, 14, 0)) diff --git a/packages/backend-core/src/events/publishers/automation.ts b/packages/backend-core/src/events/publishers/automation.ts index 95f9cb8db6..8b2574b739 100644 --- a/packages/backend-core/src/events/publishers/automation.ts +++ b/packages/backend-core/src/events/publishers/automation.ts @@ -72,7 +72,7 @@ export async function stepCreated( automationId: automation._id as string, triggerId: automation.definition?.trigger?.id, triggerType: automation.definition?.trigger?.stepId, - stepId: step.id, + stepId: step.id!, stepType: step.stepId, } await publishEvent(Event.AUTOMATION_STEP_CREATED, properties, timestamp) @@ -87,7 +87,7 @@ export async function stepDeleted( automationId: automation._id as string, triggerId: automation.definition?.trigger?.id, triggerType: automation.definition?.trigger?.stepId, - stepId: step.id, + stepId: step.id!, stepType: step.stepId, } await publishEvent(Event.AUTOMATION_STEP_DELETED, properties) diff --git a/packages/backend-core/src/featureFlags/index.js b/packages/backend-core/src/featureFlags/index.ts similarity index 70% rename from packages/backend-core/src/featureFlags/index.js rename to packages/backend-core/src/featureFlags/index.ts index 8a8162d0ba..71e226c976 100644 --- a/packages/backend-core/src/featureFlags/index.js +++ b/packages/backend-core/src/featureFlags/index.ts @@ -1,17 +1,17 @@ -const env = require("../environment") -const tenancy = require("../tenancy") +import env from "../environment" +import * as tenancy from "../tenancy" /** * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant. * The env var is formatted as: * tenant1:feature1:feature2,tenant2:feature1 */ -const getFeatureFlags = () => { +function getFeatureFlags() { if (!env.TENANT_FEATURE_FLAGS) { return } - const tenantFeatureFlags = {} + const tenantFeatureFlags: Record = {} env.TENANT_FEATURE_FLAGS.split(",").forEach(tenantToFeatures => { const [tenantId, ...features] = tenantToFeatures.split(":") @@ -29,13 +29,13 @@ const getFeatureFlags = () => { const TENANT_FEATURE_FLAGS = getFeatureFlags() -exports.isEnabled = featureFlag => { +export function isEnabled(featureFlag: string) { const tenantId = tenancy.getTenantId() - const flags = exports.getTenantFeatureFlags(tenantId) + const flags = getTenantFeatureFlags(tenantId) return flags.includes(featureFlag) } -exports.getTenantFeatureFlags = tenantId => { +export function getTenantFeatureFlags(tenantId: string) { const flags = [] if (TENANT_FEATURE_FLAGS) { @@ -53,8 +53,8 @@ exports.getTenantFeatureFlags = tenantId => { return flags } -exports.TenantFeatureFlag = { - LICENSING: "LICENSING", - GOOGLE_SHEETS: "GOOGLE_SHEETS", - USER_GROUPS: "USER_GROUPS", +export enum TenantFeatureFlag { + LICENSING = "LICENSING", + GOOGLE_SHEETS = "GOOGLE_SHEETS", + USER_GROUPS = "USER_GROUPS", } diff --git a/packages/worker/src/utilities/index.js b/packages/backend-core/src/helpers.ts similarity index 85% rename from packages/worker/src/utilities/index.js rename to packages/backend-core/src/helpers.ts index b402a82cf3..e1e065bd4e 100644 --- a/packages/worker/src/utilities/index.js +++ b/packages/backend-core/src/helpers.ts @@ -4,6 +4,6 @@ * @param {string} url The URL to test and remove any extra double slashes. * @return {string} The updated url. */ -exports.checkSlashesInUrl = url => { +export function checkSlashesInUrl(url: string) { return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2") } diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index c68c8f0927..a4d4ad0a80 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -8,27 +8,24 @@ import * as permissions from "./security/permissions" import * as accounts from "./cloud/accounts" import * as installation from "./installation" import env from "./environment" -import tenancy from "./tenancy" -import featureFlags from "./featureFlags" +import * as tenancy from "./tenancy" +import * as featureFlags from "./featureFlags" import * as sessions from "./security/sessions" import * as deprovisioning from "./context/deprovision" -import auth from "./auth" +import * as auth from "./auth" import * as constants from "./constants" -import * as dbConstants from "./db/constants" import * as logging from "./logging" -import pino from "./pino" +import * as pino from "./pino" import * as middleware from "./middleware" -import plugins from "./plugin" -import encryption from "./security/encryption" +import * as plugins from "./plugin" +import * as encryption from "./security/encryption" import * as queue from "./queue" import * as db from "./db" - -// mimic the outer package exports -import * as objectStore from "./pkg/objectStore" -import * as utils from "./pkg/utils" -import redis from "./pkg/redis" -import cache from "./pkg/cache" -import context from "./pkg/context" +import * as context from "./context" +import * as cache from "./cache" +import * as objectStore from "./objectStore" +import * as redis from "./redis" +import * as utils from "./utils" const init = (opts: any = {}) => { db.init(opts.db) @@ -37,7 +34,7 @@ const init = (opts: any = {}) => { const core = { init, db, - ...dbConstants, + ...constants, redis, locks: redis.redlock, objectStore, @@ -46,7 +43,6 @@ const core = { cache, auth, constants, - ...constants, migrations, env, accounts, diff --git a/packages/backend-core/src/installation.ts b/packages/backend-core/src/installation.ts index da9b6c5b76..4e78a508a5 100644 --- a/packages/backend-core/src/installation.ts +++ b/packages/backend-core/src/installation.ts @@ -1,16 +1,16 @@ -import * as hashing from "./hashing" +import { newid } from "./utils" import * as events from "./events" -import { StaticDatabases } from "./db/constants" +import { StaticDatabases } from "./db" import { doWithDB } from "./db" import { Installation, IdentityType } from "@budibase/types" import * as context from "./context" import semver from "semver" -import { bustCache, withCache, TTL, CacheKeys } from "./cache/generic" +import { bustCache, withCache, TTL, CacheKey } from "./cache/generic" const pkg = require("../package.json") export const getInstall = async (): Promise => { - return withCache(CacheKeys.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, { + return withCache(CacheKey.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, { useTenancy: false, }) } @@ -28,7 +28,7 @@ const getInstallFromDB = async (): Promise => { if (e.status === 404) { install = { _id: StaticDatabases.PLATFORM_INFO.docs.install, - installId: hashing.newid(), + installId: newid(), version: pkg.version, } const resp = await platformDb.put(install) @@ -50,7 +50,7 @@ const updateVersion = async (version: string): Promise => { const install = await getInstall() install.version = version await platformDb.put(install) - await bustCache(CacheKeys.INSTALLATION) + await bustCache(CacheKey.INSTALLATION) } ) } catch (e: any) { diff --git a/packages/backend-core/src/middleware/adminOnly.js b/packages/backend-core/src/middleware/adminOnly.ts similarity index 63% rename from packages/backend-core/src/middleware/adminOnly.js rename to packages/backend-core/src/middleware/adminOnly.ts index 4bfdf83848..30fdf2907b 100644 --- a/packages/backend-core/src/middleware/adminOnly.js +++ b/packages/backend-core/src/middleware/adminOnly.ts @@ -1,4 +1,6 @@ -module.exports = async (ctx, next) => { +import { BBContext } from "@budibase/types" + +export = async (ctx: BBContext, next: any) => { if ( !ctx.internal && (!ctx.user || !ctx.user.admin || !ctx.user.admin.global) diff --git a/packages/backend-core/src/middleware/auditLog.js b/packages/backend-core/src/middleware/auditLog.js deleted file mode 100644 index c9063ae2e0..0000000000 --- a/packages/backend-core/src/middleware/auditLog.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = async (ctx, next) => { - // Placeholder for audit log middleware - return next() -} diff --git a/packages/backend-core/src/middleware/auditLog.ts b/packages/backend-core/src/middleware/auditLog.ts new file mode 100644 index 0000000000..a2c30ade8a --- /dev/null +++ b/packages/backend-core/src/middleware/auditLog.ts @@ -0,0 +1,6 @@ +import { BBContext } from "@budibase/types" + +export = async (ctx: BBContext | any, next: any) => { + // Placeholder for audit log middleware + return next() +} diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 8a1e52f414..fcf07c50a5 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -6,10 +6,13 @@ import { buildMatcherRegex, matches } from "./matchers" import { SEPARATOR, queryGlobalView, ViewName } from "../db" import { getGlobalDB, doInTenant } from "../tenancy" import { decrypt } from "../security/encryption" -const identity = require("../context/identity") -const env = require("../environment") +import * as identity from "../context/identity" +import env from "../environment" +import { BBContext, EndpointMatcher } from "@budibase/types" -const ONE_MINUTE = env.SESSION_UPDATE_PERIOD || 60 * 1000 +const ONE_MINUTE = env.SESSION_UPDATE_PERIOD + ? parseInt(env.SESSION_UPDATE_PERIOD) + : 60 * 1000 interface FinaliseOpts { authenticated?: boolean @@ -40,13 +43,13 @@ async function checkApiKey(apiKey: string, populateUser?: Function) { return doInTenant(tenantId, async () => { const db = getGlobalDB() // api key is encrypted in the database - const userId = await queryGlobalView( + const userId = (await queryGlobalView( ViewName.BY_API_KEY, { key: apiKey, }, db - ) + )) as string if (userId) { return { valid: true, @@ -63,14 +66,14 @@ async function checkApiKey(apiKey: string, populateUser?: Function) { * The tenancy modules should not be used here and it should be assumed that the tenancy context * has not yet been populated. */ -export = ( - noAuthPatterns = [], - opts: { publicAllowed: boolean; populateUser?: Function } = { +export = function ( + noAuthPatterns: EndpointMatcher[] = [], + opts: { publicAllowed?: boolean; populateUser?: Function } = { publicAllowed: false, } -) => { +) { const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : [] - return async (ctx: any, next: any) => { + return async (ctx: BBContext | any, next: any) => { let publicEndpoint = false const version = ctx.request.headers[Header.API_VER] // the path is not authenticated diff --git a/packages/backend-core/src/middleware/builderOnly.js b/packages/backend-core/src/middleware/builderOnly.ts similarity index 64% rename from packages/backend-core/src/middleware/builderOnly.js rename to packages/backend-core/src/middleware/builderOnly.ts index 2128626db4..e13882d7f6 100644 --- a/packages/backend-core/src/middleware/builderOnly.js +++ b/packages/backend-core/src/middleware/builderOnly.ts @@ -1,4 +1,6 @@ -module.exports = async (ctx, next) => { +import { BBContext } from "@budibase/types" + +export = async (ctx: BBContext, next: any) => { if ( !ctx.internal && (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) diff --git a/packages/backend-core/src/middleware/builderOrAdmin.js b/packages/backend-core/src/middleware/builderOrAdmin.ts similarity index 71% rename from packages/backend-core/src/middleware/builderOrAdmin.js rename to packages/backend-core/src/middleware/builderOrAdmin.ts index 6440766298..26664695f8 100644 --- a/packages/backend-core/src/middleware/builderOrAdmin.js +++ b/packages/backend-core/src/middleware/builderOrAdmin.ts @@ -1,4 +1,6 @@ -module.exports = async (ctx, next) => { +import { BBContext } from "@budibase/types" + +export = async (ctx: BBContext, next: any) => { if ( !ctx.internal && (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) && diff --git a/packages/backend-core/src/middleware/csrf.js b/packages/backend-core/src/middleware/csrf.ts similarity index 86% rename from packages/backend-core/src/middleware/csrf.js rename to packages/backend-core/src/middleware/csrf.ts index 1557740cd6..654ba47e07 100644 --- a/packages/backend-core/src/middleware/csrf.js +++ b/packages/backend-core/src/middleware/csrf.ts @@ -1,5 +1,6 @@ -const { Header } = require("../constants") -const { buildMatcherRegex, matches } = require("./matchers") +import { Header } from "../constants" +import { buildMatcherRegex, matches } from "./matchers" +import { BBContext, EndpointMatcher } from "@budibase/types" /** * GET, HEAD and OPTIONS methods are considered safe operations @@ -31,9 +32,11 @@ const INCLUDED_CONTENT_TYPES = [ * https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html#synchronizer-token-pattern * */ -module.exports = (opts = { noCsrfPatterns: [] }) => { +export = function ( + opts: { noCsrfPatterns: EndpointMatcher[] } = { noCsrfPatterns: [] } +) { const noCsrfOptions = buildMatcherRegex(opts.noCsrfPatterns) - return async (ctx, next) => { + return async (ctx: BBContext | any, next: any) => { // don't apply for excluded paths const found = matches(ctx, noCsrfOptions) if (found) { @@ -62,7 +65,7 @@ module.exports = (opts = { noCsrfPatterns: [] }) => { // apply csrf when there is a token in the session (new logins) // in future there should be a hard requirement that the token is present - const userToken = ctx.user.csrfToken + const userToken = ctx.user?.csrfToken if (!userToken) { return next() } diff --git a/packages/backend-core/src/middleware/index.ts b/packages/backend-core/src/middleware/index.ts index 998c231b3d..2b332f5c49 100644 --- a/packages/backend-core/src/middleware/index.ts +++ b/packages/backend-core/src/middleware/index.ts @@ -1,18 +1,18 @@ -const jwt = require("./passport/jwt") -const local = require("./passport/local") -const google = require("./passport/google") -const oidc = require("./passport/oidc") -const { authError, ssoCallbackUrl } = require("./passport/utils") -const authenticated = require("./authenticated") -const auditLog = require("./auditLog") -const tenancy = require("./tenancy") -const internalApi = require("./internalApi") -const datasourceGoogle = require("./passport/datasource/google") -const csrf = require("./csrf") -const adminOnly = require("./adminOnly") -const builderOrAdmin = require("./builderOrAdmin") -const builderOnly = require("./builderOnly") -const joiValidator = require("./joi-validator") +import * as jwt from "./passport/jwt" +import * as local from "./passport/local" +import * as google from "./passport/google" +import * as oidc from "./passport/oidc" +import { authError, ssoCallbackUrl } from "./passport/utils" +import authenticated from "./authenticated" +import auditLog from "./auditLog" +import tenancy from "./tenancy" +import internalApi from "./internalApi" +import * as datasourceGoogle from "./passport/datasource/google" +import csrf from "./csrf" +import adminOnly from "./adminOnly" +import builderOrAdmin from "./builderOrAdmin" +import builderOnly from "./builderOnly" +import * as joiValidator from "./joi-validator" const pkg = { google, diff --git a/packages/backend-core/src/middleware/internalApi.js b/packages/backend-core/src/middleware/internalApi.ts similarity index 53% rename from packages/backend-core/src/middleware/internalApi.js rename to packages/backend-core/src/middleware/internalApi.ts index 05833842ce..f4f08ec2dd 100644 --- a/packages/backend-core/src/middleware/internalApi.js +++ b/packages/backend-core/src/middleware/internalApi.ts @@ -1,10 +1,11 @@ -const env = require("../environment") -const { Header } = require("../constants") +import env from "../environment" +import { Header } from "../constants" +import { BBContext } from "@budibase/types" /** * API Key only endpoint. */ -module.exports = async (ctx, next) => { +export = async (ctx: BBContext, next: any) => { const apiKey = ctx.request.headers[Header.API_KEY] if (apiKey !== env.INTERNAL_API_KEY) { ctx.throw(403, "Unauthorized") diff --git a/packages/backend-core/src/middleware/joi-validator.js b/packages/backend-core/src/middleware/joi-validator.ts similarity index 50% rename from packages/backend-core/src/middleware/joi-validator.js rename to packages/backend-core/src/middleware/joi-validator.ts index 6812dbdd54..fcc8316886 100644 --- a/packages/backend-core/src/middleware/joi-validator.js +++ b/packages/backend-core/src/middleware/joi-validator.ts @@ -1,21 +1,27 @@ -const Joi = require("joi") +import Joi, { ObjectSchema } from "joi" +import { BBContext } from "@budibase/types" -function validate(schema, property) { +function validate( + schema: Joi.ObjectSchema | Joi.ArraySchema, + property: string +) { // Return a Koa middleware function - return (ctx, next) => { + return (ctx: BBContext, next: any) => { if (!schema) { return next() } let params = null + // @ts-ignore + let reqProp = ctx.request?.[property] if (ctx[property] != null) { params = ctx[property] - } else if (ctx.request[property] != null) { - params = ctx.request[property] + } else if (reqProp != null) { + params = reqProp } // not all schemas have the append property e.g. array schemas - if (schema.append) { - schema = schema.append({ + if ((schema as Joi.ObjectSchema).append) { + schema = (schema as Joi.ObjectSchema).append({ createdAt: Joi.any().optional(), updatedAt: Joi.any().optional(), }) @@ -30,10 +36,10 @@ function validate(schema, property) { } } -module.exports.body = schema => { +export function body(schema: Joi.ObjectSchema | Joi.ArraySchema) { return validate(schema, "body") } -module.exports.params = schema => { +export function params(schema: Joi.ObjectSchema | Joi.ArraySchema) { return validate(schema, "params") } diff --git a/packages/backend-core/src/middleware/passport/datasource/google.js b/packages/backend-core/src/middleware/passport/datasource/google.ts similarity index 71% rename from packages/backend-core/src/middleware/passport/datasource/google.js rename to packages/backend-core/src/middleware/passport/datasource/google.ts index 7cfd7f55f6..65620d7aa3 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.js +++ b/packages/backend-core/src/middleware/passport/datasource/google.ts @@ -1,11 +1,15 @@ -const google = require("../google") +import * as google from "../google" +import { Cookie, Config } from "../../../constants" +import { clearCookie, getCookie } from "../../../utils" +import { getScopedConfig, getPlatformUrl, doWithDB } from "../../../db" +import environment from "../../../environment" +import { getGlobalDB } from "../../../tenancy" +import { BBContext, Database, SSOProfile } from "@budibase/types" const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy -const { Cookie, Config } = require("../../../constants") -const { clearCookie, getCookie } = require("../../../utils") -const { getScopedConfig, getPlatformUrl } = require("../../../db/utils") -const { doWithDB } = require("../../../db") -const environment = require("../../../environment") -const { getGlobalDB } = require("../../../tenancy") + +type Passport = { + authenticate: any +} async function fetchGoogleCreds() { // try and get the config from the tenant @@ -22,7 +26,11 @@ async function fetchGoogleCreds() { ) } -async function preAuth(passport, ctx, next) { +export async function preAuth( + passport: Passport, + ctx: BBContext, + next: Function +) { // get the relevant config const googleConfig = await fetchGoogleCreds() const platformUrl = await getPlatformUrl({ tenantAware: false }) @@ -41,7 +49,11 @@ async function preAuth(passport, ctx, next) { })(ctx, next) } -async function postAuth(passport, ctx, next) { +export async function postAuth( + passport: Passport, + ctx: BBContext, + next: Function +) { // get the relevant config const config = await fetchGoogleCreds() const platformUrl = await getPlatformUrl({ tenantAware: false }) @@ -56,15 +68,20 @@ async function postAuth(passport, ctx, next) { clientSecret: config.clientSecret, callbackURL: callbackUrl, }, - (accessToken, refreshToken, profile, done) => { + ( + accessToken: string, + refreshToken: string, + profile: SSOProfile, + done: Function + ) => { clearCookie(ctx, Cookie.DatasourceAuth) done(null, { accessToken, refreshToken }) } ), { successRedirect: "/", failureRedirect: "/error" }, - async (err, tokens) => { + async (err: any, tokens: string[]) => { // update the DB for the datasource with all the user info - await doWithDB(authStateCookie.appId, async db => { + await doWithDB(authStateCookie.appId, async (db: Database) => { const datasource = await db.get(authStateCookie.datasourceId) if (!datasource.config) { datasource.config = {} @@ -78,6 +95,3 @@ async function postAuth(passport, ctx, next) { } )(ctx, next) } - -exports.preAuth = preAuth -exports.postAuth = postAuth diff --git a/packages/backend-core/src/middleware/passport/google.js b/packages/backend-core/src/middleware/passport/google.ts similarity index 63% rename from packages/backend-core/src/middleware/passport/google.js rename to packages/backend-core/src/middleware/passport/google.ts index 7eb1215c1f..deba849233 100644 --- a/packages/backend-core/src/middleware/passport/google.js +++ b/packages/backend-core/src/middleware/passport/google.ts @@ -1,10 +1,15 @@ +import { ssoCallbackUrl } from "./utils" +import { authenticateThirdParty } from "./third-party-common" +import { ConfigType, GoogleConfig, Database, SSOProfile } from "@budibase/types" const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy -const { ssoCallbackUrl } = require("./utils") -const { authenticateThirdParty } = require("./third-party-common") -const { Config } = require("../../../constants") -const buildVerifyFn = saveUserFn => { - return (accessToken, refreshToken, profile, done) => { +export function buildVerifyFn(saveUserFn?: Function) { + return ( + accessToken: string, + refreshToken: string, + profile: SSOProfile, + done: Function + ) => { const thirdPartyUser = { provider: profile.provider, // should always be 'google' providerType: "google", @@ -31,7 +36,11 @@ const buildVerifyFn = saveUserFn => { * from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport. * @returns Dynamically configured Passport Google Strategy */ -exports.strategyFactory = async function (config, callbackUrl, saveUserFn) { +export async function strategyFactory( + config: GoogleConfig["config"], + callbackUrl: string, + saveUserFn?: Function +) { try { const { clientID, clientSecret } = config @@ -50,18 +59,15 @@ exports.strategyFactory = async function (config, callbackUrl, saveUserFn) { }, verify ) - } catch (err) { + } catch (err: any) { console.error(err) - throw new Error( - `Error constructing google authentication strategy: ${err}`, - err - ) + throw new Error(`Error constructing google authentication strategy: ${err}`) } } -exports.getCallbackUrl = async function (db, config) { - return ssoCallbackUrl(db, config, Config.GOOGLE) +export async function getCallbackUrl( + db: Database, + config: { callbackURL?: string } +) { + return ssoCallbackUrl(db, config, ConfigType.GOOGLE) } - -// expose for testing -exports.buildVerifyFn = buildVerifyFn diff --git a/packages/backend-core/src/middleware/passport/jwt.js b/packages/backend-core/src/middleware/passport/jwt.js deleted file mode 100644 index 36316264b0..0000000000 --- a/packages/backend-core/src/middleware/passport/jwt.js +++ /dev/null @@ -1,18 +0,0 @@ -const { Cookie } = require("../../constants") -const env = require("../../environment") -const { authError } = require("./utils") - -exports.options = { - secretOrKey: env.JWT_SECRET, - jwtFromRequest: function (ctx) { - return ctx.cookies.get(Cookie.Auth) - }, -} - -exports.authenticate = async function (jwt, done) { - try { - return done(null, jwt) - } catch (err) { - return authError(done, "JWT invalid", err) - } -} diff --git a/packages/backend-core/src/middleware/passport/jwt.ts b/packages/backend-core/src/middleware/passport/jwt.ts new file mode 100644 index 0000000000..95dc8f2656 --- /dev/null +++ b/packages/backend-core/src/middleware/passport/jwt.ts @@ -0,0 +1,19 @@ +import { Cookie } from "../../constants" +import env from "../../environment" +import { authError } from "./utils" +import { BBContext } from "@budibase/types" + +export const options = { + secretOrKey: env.JWT_SECRET, + jwtFromRequest: function (ctx: BBContext) { + return ctx.cookies.get(Cookie.Auth) + }, +} + +export async function authenticate(jwt: Function, done: Function) { + try { + return done(null, jwt) + } catch (err) { + return authError(done, "JWT invalid", err) + } +} diff --git a/packages/backend-core/src/middleware/passport/local.js b/packages/backend-core/src/middleware/passport/local.ts similarity index 73% rename from packages/backend-core/src/middleware/passport/local.js rename to packages/backend-core/src/middleware/passport/local.ts index b955d29102..8b85d3734c 100644 --- a/packages/backend-core/src/middleware/passport/local.js +++ b/packages/backend-core/src/middleware/passport/local.ts @@ -1,18 +1,18 @@ +import { UserStatus } from "../../constants" +import { compare, newid } from "../../utils" +import env from "../../environment" +import * as users from "../../users" +import { authError } from "./utils" +import { createASession } from "../../security/sessions" +import { getTenantId } from "../../tenancy" +import { BBContext } from "@budibase/types" const jwt = require("jsonwebtoken") -const { UserStatus } = require("../../constants") -const { compare } = require("../../hashing") -const env = require("../../environment") -const users = require("../../users") -const { authError } = require("./utils") -const { newid } = require("../../hashing") -const { createASession } = require("../../security/sessions") -const { getTenantId } = require("../../tenancy") const INVALID_ERR = "Invalid credentials" const SSO_NO_PASSWORD = "SSO user does not have a password set" const EXPIRED = "This account has expired. Please reset your password" -exports.options = { +export const options = { passReqToCallback: true, } @@ -24,7 +24,12 @@ exports.options = { * @param {*} done callback from passport to return user information and errors * @returns The authenticated user, or errors if they occur */ -exports.authenticate = async function (ctx, email, password, done) { +export async function authenticate( + ctx: BBContext, + email: string, + password: string, + done: Function +) { if (!email) return authError(done, "Email Required") if (!password) return authError(done, "Password Required") @@ -56,9 +61,9 @@ exports.authenticate = async function (ctx, email, password, done) { const sessionId = newid() const tenantId = getTenantId() - await createASession(dbUser._id, { sessionId, tenantId }) + await createASession(dbUser._id!, { sessionId, tenantId }) - dbUser.token = jwt.sign( + const token = jwt.sign( { userId: dbUser._id, sessionId, @@ -69,7 +74,10 @@ exports.authenticate = async function (ctx, email, password, done) { // Remove users password in payload delete dbUser.password - return done(null, dbUser) + return done(null, { + ...dbUser, + token, + }) } else { return authError(done, INVALID_ERR) } diff --git a/packages/backend-core/src/middleware/passport/oidc.js b/packages/backend-core/src/middleware/passport/oidc.ts similarity index 72% rename from packages/backend-core/src/middleware/passport/oidc.js rename to packages/backend-core/src/middleware/passport/oidc.ts index 55a7033e40..27c3c647b7 100644 --- a/packages/backend-core/src/middleware/passport/oidc.js +++ b/packages/backend-core/src/middleware/passport/oidc.ts @@ -1,10 +1,23 @@ -const fetch = require("node-fetch") +import fetch from "node-fetch" +import { authenticateThirdParty } from "./third-party-common" +import { ssoCallbackUrl } from "./utils" +import { + Config, + ConfigType, + OIDCInnerCfg, + Database, + SSOProfile, + ThirdPartyUser, + OIDCConfiguration, +} from "@budibase/types" const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy -const { authenticateThirdParty } = require("./third-party-common") -const { ssoCallbackUrl } = require("./utils") -const { Config } = require("../../../constants") -const buildVerifyFn = saveUserFn => { +type JwtClaims = { + preferred_username: string + email: string +} + +export function buildVerifyFn(saveUserFn?: Function) { /** * @param {*} issuer The identity provider base URL * @param {*} sub The user ID @@ -17,17 +30,17 @@ const buildVerifyFn = saveUserFn => { * @param {*} done The passport callback: err, user, info */ return async ( - issuer, - sub, - profile, - jwtClaims, - accessToken, - refreshToken, - idToken, - params, - done + issuer: string, + sub: string, + profile: SSOProfile, + jwtClaims: JwtClaims, + accessToken: string, + refreshToken: string, + idToken: string, + params: any, + done: Function ) => { - const thirdPartyUser = { + const thirdPartyUser: ThirdPartyUser = { // store the issuer info to enable sync in future provider: issuer, providerType: "oidc", @@ -53,7 +66,7 @@ const buildVerifyFn = saveUserFn => { * @param {*} profile The structured profile created by passport using the user info endpoint * @param {*} jwtClaims The claims returned in the id token */ -function getEmail(profile, jwtClaims) { +function getEmail(profile: SSOProfile, jwtClaims: JwtClaims) { // profile not guaranteed to contain email e.g. github connected azure ad account if (profile._json.email) { return profile._json.email @@ -77,7 +90,7 @@ function getEmail(profile, jwtClaims) { ) } -function validEmail(value) { +function validEmail(value: string) { return ( value && !!value.match( @@ -91,19 +104,25 @@ function validEmail(value) { * from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport. * @returns Dynamically configured Passport OIDC Strategy */ -exports.strategyFactory = async function (config, saveUserFn) { +export async function strategyFactory( + config: OIDCConfiguration, + saveUserFn?: Function +) { try { const verify = buildVerifyFn(saveUserFn) const strategy = new OIDCStrategy(config, verify) strategy.name = "oidc" return strategy - } catch (err) { + } catch (err: any) { console.error(err) - throw new Error("Error constructing OIDC authentication strategy", err) + throw new Error(`Error constructing OIDC authentication strategy - ${err}`) } } -exports.fetchStrategyConfig = async function (enrichedConfig, callbackUrl) { +export async function fetchStrategyConfig( + enrichedConfig: OIDCInnerCfg, + callbackUrl?: string +): Promise { try { const { clientID, clientSecret, configUrl } = enrichedConfig @@ -135,13 +154,15 @@ exports.fetchStrategyConfig = async function (enrichedConfig, callbackUrl) { } } catch (err) { console.error(err) - throw new Error("Error constructing OIDC authentication configuration", err) + throw new Error( + `Error constructing OIDC authentication configuration - ${err}` + ) } } -exports.getCallbackUrl = async function (db, config) { - return ssoCallbackUrl(db, config, Config.OIDC) +export async function getCallbackUrl( + db: Database, + config: { callbackURL?: string } +) { + return ssoCallbackUrl(db, config, ConfigType.OIDC) } - -// expose for testing -exports.buildVerifyFn = buildVerifyFn diff --git a/packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js b/packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js index 9799045ffc..d377d602f1 100644 --- a/packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js +++ b/packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js @@ -4,7 +4,7 @@ const { data } = require("./utilities/mock-data") const { DEFAULT_TENANT_ID } = require("../../../constants") const { generateGlobalUserID } = require("../../../db/utils") -const { newid } = require("../../../hashing") +const { newid } = require("../../../utils") const { doWithGlobalDB, doInTenant } = require("../../../tenancy") const done = jest.fn() diff --git a/packages/backend-core/src/middleware/passport/third-party-common.js b/packages/backend-core/src/middleware/passport/third-party-common.ts similarity index 77% rename from packages/backend-core/src/middleware/passport/third-party-common.js rename to packages/backend-core/src/middleware/passport/third-party-common.ts index 1c5891fce7..8798ce5298 100644 --- a/packages/backend-core/src/middleware/passport/third-party-common.js +++ b/packages/backend-core/src/middleware/passport/third-party-common.ts @@ -1,21 +1,22 @@ -const env = require("../../environment") +import env from "../../environment" +import { generateGlobalUserID } from "../../db" +import { authError } from "./utils" +import { newid } from "../../utils" +import { createASession } from "../../security/sessions" +import * as users from "../../users" +import { getGlobalDB, getTenantId } from "../../tenancy" +import fetch from "node-fetch" +import { ThirdPartyUser } from "@budibase/types" const jwt = require("jsonwebtoken") -const { generateGlobalUserID } = require("../../db/utils") -const { authError } = require("./utils") -const { newid } = require("../../hashing") -const { createASession } = require("../../security/sessions") -const users = require("../../users") -const { getGlobalDB, getTenantId } = require("../../tenancy") -const fetch = require("node-fetch") /** * Common authentication logic for third parties. e.g. OAuth, OIDC. */ -exports.authenticateThirdParty = async function ( - thirdPartyUser, - requireLocalAccount = true, - done, - saveUserFn +export async function authenticateThirdParty( + thirdPartyUser: ThirdPartyUser, + requireLocalAccount: boolean = true, + done: Function, + saveUserFn?: Function ) { if (!saveUserFn) { throw new Error("Save user function must be provided") @@ -39,7 +40,7 @@ exports.authenticateThirdParty = async function ( // try to load by id try { dbUser = await db.get(userId) - } catch (err) { + } catch (err: any) { // abort when not 404 error if (!err.status || err.status !== 404) { return authError( @@ -81,7 +82,7 @@ exports.authenticateThirdParty = async function ( // create or sync the user try { await saveUserFn(dbUser, false, false) - } catch (err) { + } catch (err: any) { return authError(done, err) } @@ -104,13 +105,16 @@ exports.authenticateThirdParty = async function ( return done(null, dbUser) } -async function syncProfilePicture(user, thirdPartyUser) { - const pictureUrl = thirdPartyUser.profile._json.picture +async function syncProfilePicture( + user: ThirdPartyUser, + thirdPartyUser: ThirdPartyUser +) { + const pictureUrl = thirdPartyUser.profile?._json.picture if (pictureUrl) { const response = await fetch(pictureUrl) if (response.status === 200) { - const type = response.headers.get("content-type") + const type = response.headers.get("content-type") as string if (type.startsWith("image/")) { user.pictureUrl = pictureUrl } @@ -123,7 +127,7 @@ async function syncProfilePicture(user, thirdPartyUser) { /** * @returns a user that has been sync'd with third party information */ -async function syncUser(user, thirdPartyUser) { +async function syncUser(user: ThirdPartyUser, thirdPartyUser: ThirdPartyUser) { // provider user.provider = thirdPartyUser.provider user.providerType = thirdPartyUser.providerType diff --git a/packages/backend-core/src/middleware/passport/utils.js b/packages/backend-core/src/middleware/passport/utils.ts similarity index 64% rename from packages/backend-core/src/middleware/passport/utils.js rename to packages/backend-core/src/middleware/passport/utils.ts index ab199b9f2f..3d79aada28 100644 --- a/packages/backend-core/src/middleware/passport/utils.js +++ b/packages/backend-core/src/middleware/passport/utils.ts @@ -1,6 +1,6 @@ -const { isMultiTenant, getTenantId } = require("../../tenancy") -const { getScopedConfig } = require("../../db/utils") -const { Config } = require("../../constants") +import { isMultiTenant, getTenantId } from "../../tenancy" +import { getScopedConfig } from "../../db" +import { ConfigType, Database, Config } from "@budibase/types" /** * Utility to handle authentication errors. @@ -10,7 +10,7 @@ const { Config } = require("../../constants") * @param {*} err (Optional) error that will be logged */ -exports.authError = function (done, message, err = null) { +export function authError(done: Function, message: string, err?: any) { return done( err, null, // never return a user @@ -18,13 +18,17 @@ exports.authError = function (done, message, err = null) { ) } -exports.ssoCallbackUrl = async (db, config, type) => { +export async function ssoCallbackUrl( + db: Database, + config?: { callbackURL?: string }, + type?: ConfigType +) { // incase there is a callback URL from before if (config && config.callbackURL) { return config.callbackURL } const publicConfig = await getScopedConfig(db, { - type: Config.SETTINGS, + type: ConfigType.SETTINGS, }) let callbackUrl = `/api/global/auth` diff --git a/packages/backend-core/src/middleware/tenancy.ts b/packages/backend-core/src/middleware/tenancy.ts index 0aaacef139..78da2bb3e8 100644 --- a/packages/backend-core/src/middleware/tenancy.ts +++ b/packages/backend-core/src/middleware/tenancy.ts @@ -8,15 +8,15 @@ import { TenantResolutionStrategy, } from "@budibase/types" -const tenancy = ( +export = function ( allowQueryStringPatterns: EndpointMatcher[], noTenancyPatterns: EndpointMatcher[], - opts = { noTenancyRequired: false } -) => { + opts: { noTenancyRequired?: boolean } = { noTenancyRequired: false } +) { const allowQsOptions = buildMatcherRegex(allowQueryStringPatterns) const noTenancyOptions = buildMatcherRegex(noTenancyPatterns) - return async function (ctx: BBContext, next: any) { + return async function (ctx: BBContext | any, next: any) { const allowNoTenant = opts.noTenancyRequired || !!matches(ctx, noTenancyOptions) const tenantOpts: GetTenantIdOptions = { @@ -33,5 +33,3 @@ const tenancy = ( return doInTenant(tenantId, next) } } - -export = tenancy diff --git a/packages/backend-core/src/migrations/definitions.ts b/packages/backend-core/src/migrations/definitions.ts index 6eba56ab43..0dd57fe639 100644 --- a/packages/backend-core/src/migrations/definitions.ts +++ b/packages/backend-core/src/migrations/definitions.ts @@ -21,6 +21,10 @@ export const DEFINITIONS: MigrationDefinition[] = [ type: MigrationType.APP, name: MigrationName.EVENT_APP_BACKFILL, }, + { + type: MigrationType.APP, + name: MigrationName.TABLE_SETTINGS_LINKS_TO_ACTIONS, + }, { type: MigrationType.GLOBAL, name: MigrationName.EVENT_GLOBAL_BACKFILL, diff --git a/packages/backend-core/src/migrations/tests/index.spec.js b/packages/backend-core/src/migrations/tests/index.spec.js index 8fbc244cd6..b7d2e14ea5 100644 --- a/packages/backend-core/src/migrations/tests/index.spec.js +++ b/packages/backend-core/src/migrations/tests/index.spec.js @@ -3,7 +3,7 @@ const { runMigrations, getMigrationsDoc } = require("../index") const { getDB } = require("../../db") const { StaticDatabases, -} = require("../../db/utils") +} = require("../../constants") let db diff --git a/packages/backend-core/src/newid.ts b/packages/backend-core/src/newid.ts new file mode 100644 index 0000000000..5676c23f48 --- /dev/null +++ b/packages/backend-core/src/newid.ts @@ -0,0 +1,5 @@ +import { v4 } from "uuid" + +export function newid() { + return v4().replace(/-/g, "") +} diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts index a1193c0303..2971834f0e 100644 --- a/packages/backend-core/src/objectStore/index.ts +++ b/packages/backend-core/src/objectStore/index.ts @@ -1,426 +1,2 @@ -const sanitize = require("sanitize-s3-objectkey") -import AWS from "aws-sdk" -import stream from "stream" -import fetch from "node-fetch" -import tar from "tar-fs" -const zlib = require("zlib") -import { promisify } from "util" -import { join } from "path" -import fs from "fs" -import env from "../environment" -import { budibaseTempDir, ObjectStoreBuckets } from "./utils" -import { v4 } from "uuid" -import { APP_PREFIX, APP_DEV_PREFIX } from "../db/utils" - -const streamPipeline = promisify(stream.pipeline) -// use this as a temporary store of buckets that are being created -const STATE = { - bucketCreationPromises: {}, -} - -type ListParams = { - ContinuationToken?: string -} - -type UploadParams = { - bucket: string - filename: string - path: string - type?: string - // can be undefined, we will remove it - metadata?: { - [key: string]: string | undefined - } -} - -const CONTENT_TYPE_MAP: any = { - txt: "text/plain", - html: "text/html", - css: "text/css", - js: "application/javascript", - json: "application/json", - gz: "application/gzip", -} -const STRING_CONTENT_TYPES = [ - CONTENT_TYPE_MAP.html, - CONTENT_TYPE_MAP.css, - CONTENT_TYPE_MAP.js, - CONTENT_TYPE_MAP.json, -] - -// does normal sanitization and then swaps dev apps to apps -export function sanitizeKey(input: string) { - return sanitize(sanitizeBucket(input)).replace(/\\/g, "/") -} - -// simply handles the dev app to app conversion -export function sanitizeBucket(input: string) { - return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX) -} - -function publicPolicy(bucketName: string) { - return { - Version: "2012-10-17", - Statement: [ - { - Effect: "Allow", - Principal: { - AWS: ["*"], - }, - Action: "s3:GetObject", - Resource: [`arn:aws:s3:::${bucketName}/*`], - }, - ], - } -} - -const PUBLIC_BUCKETS = [ - ObjectStoreBuckets.APPS, - ObjectStoreBuckets.GLOBAL, - ObjectStoreBuckets.PLUGINS, -] - -/** - * Gets a connection to the object store using the S3 SDK. - * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from. - * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage. - * @constructor - */ -export const ObjectStore = (bucket: string) => { - const config: any = { - s3ForcePathStyle: true, - signatureVersion: "v4", - apiVersion: "2006-03-01", - accessKeyId: env.MINIO_ACCESS_KEY, - secretAccessKey: env.MINIO_SECRET_KEY, - region: env.AWS_REGION, - } - if (bucket) { - config.params = { - Bucket: sanitizeBucket(bucket), - } - } - if (env.MINIO_URL) { - config.endpoint = env.MINIO_URL - } - return new AWS.S3(config) -} - -/** - * Given an object store and a bucket name this will make sure the bucket exists, - * if it does not exist then it will create it. - */ -export const makeSureBucketExists = async (client: any, bucketName: string) => { - bucketName = sanitizeBucket(bucketName) - try { - await client - .headBucket({ - Bucket: bucketName, - }) - .promise() - } catch (err: any) { - const promises: any = STATE.bucketCreationPromises - const doesntExist = err.statusCode === 404, - noAccess = err.statusCode === 403 - if (promises[bucketName]) { - await promises[bucketName] - } else if (doesntExist || noAccess) { - if (doesntExist) { - // bucket doesn't exist create it - promises[bucketName] = client - .createBucket({ - Bucket: bucketName, - }) - .promise() - await promises[bucketName] - delete promises[bucketName] - } - // public buckets are quite hidden in the system, make sure - // no bucket is set accidentally - if (PUBLIC_BUCKETS.includes(bucketName)) { - await client - .putBucketPolicy({ - Bucket: bucketName, - Policy: JSON.stringify(publicPolicy(bucketName)), - }) - .promise() - } - } else { - throw new Error("Unable to write to object store bucket.") - } - } -} - -/** - * Uploads the contents of a file given the required parameters, useful when - * temp files in use (for example file uploaded as an attachment). - */ -export const upload = async ({ - bucket: bucketName, - filename, - path, - type, - metadata, -}: UploadParams) => { - const extension = filename.split(".").pop() - const fileBytes = fs.readFileSync(path) - - const objectStore = ObjectStore(bucketName) - await makeSureBucketExists(objectStore, bucketName) - - let contentType = type - if (!contentType) { - contentType = extension - ? CONTENT_TYPE_MAP[extension.toLowerCase()] - : CONTENT_TYPE_MAP.txt - } - const config: any = { - // windows file paths need to be converted to forward slashes for s3 - Key: sanitizeKey(filename), - Body: fileBytes, - ContentType: contentType, - } - if (metadata && typeof metadata === "object") { - // remove any nullish keys from the metadata object, as these may be considered invalid - for (let key of Object.keys(metadata)) { - if (!metadata[key] || typeof metadata[key] !== "string") { - delete metadata[key] - } - } - config.Metadata = metadata - } - return objectStore.upload(config).promise() -} - -/** - * Similar to the upload function but can be used to send a file stream - * through to the object store. - */ -export const streamUpload = async ( - bucketName: string, - filename: string, - stream: any, - extra = {} -) => { - const objectStore = ObjectStore(bucketName) - await makeSureBucketExists(objectStore, bucketName) - - // Set content type for certain known extensions - if (filename?.endsWith(".js")) { - extra = { - ...extra, - ContentType: "application/javascript", - } - } else if (filename?.endsWith(".svg")) { - extra = { - ...extra, - ContentType: "image", - } - } - - const params = { - Bucket: sanitizeBucket(bucketName), - Key: sanitizeKey(filename), - Body: stream, - ...extra, - } - return objectStore.upload(params).promise() -} - -/** - * retrieves the contents of a file from the object store, if it is a known content type it - * will be converted, otherwise it will be returned as a buffer stream. - */ -export const retrieve = async (bucketName: string, filepath: string) => { - const objectStore = ObjectStore(bucketName) - const params = { - Bucket: sanitizeBucket(bucketName), - Key: sanitizeKey(filepath), - } - const response: any = await objectStore.getObject(params).promise() - // currently these are all strings - if (STRING_CONTENT_TYPES.includes(response.ContentType)) { - return response.Body.toString("utf8") - } else { - return response.Body - } -} - -export const listAllObjects = async (bucketName: string, path: string) => { - const objectStore = ObjectStore(bucketName) - const list = (params: ListParams = {}) => { - return objectStore - .listObjectsV2({ - ...params, - Bucket: sanitizeBucket(bucketName), - Prefix: sanitizeKey(path), - }) - .promise() - } - let isTruncated = false, - token, - objects: AWS.S3.Types.Object[] = [] - do { - let params: ListParams = {} - if (token) { - params.ContinuationToken = token - } - const response = await list(params) - if (response.Contents) { - objects = objects.concat(response.Contents) - } - isTruncated = !!response.IsTruncated - } while (isTruncated) - return objects -} - -/** - * Same as retrieval function but puts to a temporary file. - */ -export const retrieveToTmp = async (bucketName: string, filepath: string) => { - bucketName = sanitizeBucket(bucketName) - filepath = sanitizeKey(filepath) - const data = await retrieve(bucketName, filepath) - const outputPath = join(budibaseTempDir(), v4()) - fs.writeFileSync(outputPath, data) - return outputPath -} - -export const retrieveDirectory = async (bucketName: string, path: string) => { - let writePath = join(budibaseTempDir(), v4()) - fs.mkdirSync(writePath) - const objects = await listAllObjects(bucketName, path) - let fullObjects = await Promise.all( - objects.map(obj => retrieve(bucketName, obj.Key!)) - ) - let count = 0 - for (let obj of objects) { - const filename = obj.Key! - const data = fullObjects[count++] - const possiblePath = filename.split("/") - if (possiblePath.length > 1) { - const dirs = possiblePath.slice(0, possiblePath.length - 1) - fs.mkdirSync(join(writePath, ...dirs), { recursive: true }) - } - fs.writeFileSync(join(writePath, ...possiblePath), data) - } - return writePath -} - -/** - * Delete a single file. - */ -export const deleteFile = async (bucketName: string, filepath: string) => { - const objectStore = ObjectStore(bucketName) - await makeSureBucketExists(objectStore, bucketName) - const params = { - Bucket: bucketName, - Key: filepath, - } - return objectStore.deleteObject(params) -} - -export const deleteFiles = async (bucketName: string, filepaths: string[]) => { - const objectStore = ObjectStore(bucketName) - await makeSureBucketExists(objectStore, bucketName) - const params = { - Bucket: bucketName, - Delete: { - Objects: filepaths.map((path: any) => ({ Key: path })), - }, - } - return objectStore.deleteObjects(params).promise() -} - -/** - * Delete a path, including everything within. - */ -export const deleteFolder = async ( - bucketName: string, - folder: string -): Promise => { - bucketName = sanitizeBucket(bucketName) - folder = sanitizeKey(folder) - const client = ObjectStore(bucketName) - const listParams = { - Bucket: bucketName, - Prefix: folder, - } - - let response: any = await client.listObjects(listParams).promise() - if (response.Contents.length === 0) { - return - } - const deleteParams: any = { - Bucket: bucketName, - Delete: { - Objects: [], - }, - } - - response.Contents.forEach((content: any) => { - deleteParams.Delete.Objects.push({ Key: content.Key }) - }) - - response = await client.deleteObjects(deleteParams).promise() - // can only empty 1000 items at once - if (response.Deleted.length === 1000) { - return deleteFolder(bucketName, folder) - } -} - -export const uploadDirectory = async ( - bucketName: string, - localPath: string, - bucketPath: string -) => { - bucketName = sanitizeBucket(bucketName) - let uploads = [] - const files = fs.readdirSync(localPath, { withFileTypes: true }) - for (let file of files) { - const path = sanitizeKey(join(bucketPath, file.name)) - const local = join(localPath, file.name) - if (file.isDirectory()) { - uploads.push(uploadDirectory(bucketName, local, path)) - } else { - uploads.push(streamUpload(bucketName, path, fs.createReadStream(local))) - } - } - await Promise.all(uploads) - return files -} - -export const downloadTarballDirect = async ( - url: string, - path: string, - headers = {} -) => { - path = sanitizeKey(path) - const response = await fetch(url, { headers }) - if (!response.ok) { - throw new Error(`unexpected response ${response.statusText}`) - } - - await streamPipeline(response.body, zlib.Unzip(), tar.extract(path)) -} - -export const downloadTarball = async ( - url: string, - bucketName: string, - path: string -) => { - bucketName = sanitizeBucket(bucketName) - path = sanitizeKey(path) - const response = await fetch(url) - if (!response.ok) { - throw new Error(`unexpected response ${response.statusText}`) - } - - const tmpPath = join(budibaseTempDir(), path) - await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath)) - if (!env.isTest() && env.SELF_HOSTED) { - await uploadDirectory(bucketName, tmpPath, path) - } - // return the temporary path incase there is a use for it - return tmpPath -} +export * from "./objectStore" +export * from "./utils" diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts new file mode 100644 index 0000000000..2ae8848c53 --- /dev/null +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -0,0 +1,426 @@ +const sanitize = require("sanitize-s3-objectkey") +import AWS from "aws-sdk" +import stream from "stream" +import fetch from "node-fetch" +import tar from "tar-fs" +const zlib = require("zlib") +import { promisify } from "util" +import { join } from "path" +import fs from "fs" +import env from "../environment" +import { budibaseTempDir, ObjectStoreBuckets } from "./utils" +import { v4 } from "uuid" +import { APP_PREFIX, APP_DEV_PREFIX } from "../db" + +const streamPipeline = promisify(stream.pipeline) +// use this as a temporary store of buckets that are being created +const STATE = { + bucketCreationPromises: {}, +} + +type ListParams = { + ContinuationToken?: string +} + +type UploadParams = { + bucket: string + filename: string + path: string + type?: string + // can be undefined, we will remove it + metadata?: { + [key: string]: string | undefined + } +} + +const CONTENT_TYPE_MAP: any = { + txt: "text/plain", + html: "text/html", + css: "text/css", + js: "application/javascript", + json: "application/json", + gz: "application/gzip", +} +const STRING_CONTENT_TYPES = [ + CONTENT_TYPE_MAP.html, + CONTENT_TYPE_MAP.css, + CONTENT_TYPE_MAP.js, + CONTENT_TYPE_MAP.json, +] + +// does normal sanitization and then swaps dev apps to apps +export function sanitizeKey(input: string) { + return sanitize(sanitizeBucket(input)).replace(/\\/g, "/") +} + +// simply handles the dev app to app conversion +export function sanitizeBucket(input: string) { + return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX) +} + +function publicPolicy(bucketName: string) { + return { + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Principal: { + AWS: ["*"], + }, + Action: "s3:GetObject", + Resource: [`arn:aws:s3:::${bucketName}/*`], + }, + ], + } +} + +const PUBLIC_BUCKETS = [ + ObjectStoreBuckets.APPS, + ObjectStoreBuckets.GLOBAL, + ObjectStoreBuckets.PLUGINS, +] + +/** + * Gets a connection to the object store using the S3 SDK. + * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from. + * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage. + * @constructor + */ +export const ObjectStore = (bucket: string) => { + const config: any = { + s3ForcePathStyle: true, + signatureVersion: "v4", + apiVersion: "2006-03-01", + accessKeyId: env.MINIO_ACCESS_KEY, + secretAccessKey: env.MINIO_SECRET_KEY, + region: env.AWS_REGION, + } + if (bucket) { + config.params = { + Bucket: sanitizeBucket(bucket), + } + } + if (env.MINIO_URL) { + config.endpoint = env.MINIO_URL + } + return new AWS.S3(config) +} + +/** + * Given an object store and a bucket name this will make sure the bucket exists, + * if it does not exist then it will create it. + */ +export const makeSureBucketExists = async (client: any, bucketName: string) => { + bucketName = sanitizeBucket(bucketName) + try { + await client + .headBucket({ + Bucket: bucketName, + }) + .promise() + } catch (err: any) { + const promises: any = STATE.bucketCreationPromises + const doesntExist = err.statusCode === 404, + noAccess = err.statusCode === 403 + if (promises[bucketName]) { + await promises[bucketName] + } else if (doesntExist || noAccess) { + if (doesntExist) { + // bucket doesn't exist create it + promises[bucketName] = client + .createBucket({ + Bucket: bucketName, + }) + .promise() + await promises[bucketName] + delete promises[bucketName] + } + // public buckets are quite hidden in the system, make sure + // no bucket is set accidentally + if (PUBLIC_BUCKETS.includes(bucketName)) { + await client + .putBucketPolicy({ + Bucket: bucketName, + Policy: JSON.stringify(publicPolicy(bucketName)), + }) + .promise() + } + } else { + throw new Error("Unable to write to object store bucket.") + } + } +} + +/** + * Uploads the contents of a file given the required parameters, useful when + * temp files in use (for example file uploaded as an attachment). + */ +export const upload = async ({ + bucket: bucketName, + filename, + path, + type, + metadata, +}: UploadParams) => { + const extension = filename.split(".").pop() + const fileBytes = fs.readFileSync(path) + + const objectStore = ObjectStore(bucketName) + await makeSureBucketExists(objectStore, bucketName) + + let contentType = type + if (!contentType) { + contentType = extension + ? CONTENT_TYPE_MAP[extension.toLowerCase()] + : CONTENT_TYPE_MAP.txt + } + const config: any = { + // windows file paths need to be converted to forward slashes for s3 + Key: sanitizeKey(filename), + Body: fileBytes, + ContentType: contentType, + } + if (metadata && typeof metadata === "object") { + // remove any nullish keys from the metadata object, as these may be considered invalid + for (let key of Object.keys(metadata)) { + if (!metadata[key] || typeof metadata[key] !== "string") { + delete metadata[key] + } + } + config.Metadata = metadata + } + return objectStore.upload(config).promise() +} + +/** + * Similar to the upload function but can be used to send a file stream + * through to the object store. + */ +export const streamUpload = async ( + bucketName: string, + filename: string, + stream: any, + extra = {} +) => { + const objectStore = ObjectStore(bucketName) + await makeSureBucketExists(objectStore, bucketName) + + // Set content type for certain known extensions + if (filename?.endsWith(".js")) { + extra = { + ...extra, + ContentType: "application/javascript", + } + } else if (filename?.endsWith(".svg")) { + extra = { + ...extra, + ContentType: "image", + } + } + + const params = { + Bucket: sanitizeBucket(bucketName), + Key: sanitizeKey(filename), + Body: stream, + ...extra, + } + return objectStore.upload(params).promise() +} + +/** + * retrieves the contents of a file from the object store, if it is a known content type it + * will be converted, otherwise it will be returned as a buffer stream. + */ +export const retrieve = async (bucketName: string, filepath: string) => { + const objectStore = ObjectStore(bucketName) + const params = { + Bucket: sanitizeBucket(bucketName), + Key: sanitizeKey(filepath), + } + const response: any = await objectStore.getObject(params).promise() + // currently these are all strings + if (STRING_CONTENT_TYPES.includes(response.ContentType)) { + return response.Body.toString("utf8") + } else { + return response.Body + } +} + +export const listAllObjects = async (bucketName: string, path: string) => { + const objectStore = ObjectStore(bucketName) + const list = (params: ListParams = {}) => { + return objectStore + .listObjectsV2({ + ...params, + Bucket: sanitizeBucket(bucketName), + Prefix: sanitizeKey(path), + }) + .promise() + } + let isTruncated = false, + token, + objects: AWS.S3.Types.Object[] = [] + do { + let params: ListParams = {} + if (token) { + params.ContinuationToken = token + } + const response = await list(params) + if (response.Contents) { + objects = objects.concat(response.Contents) + } + isTruncated = !!response.IsTruncated + } while (isTruncated) + return objects +} + +/** + * Same as retrieval function but puts to a temporary file. + */ +export const retrieveToTmp = async (bucketName: string, filepath: string) => { + bucketName = sanitizeBucket(bucketName) + filepath = sanitizeKey(filepath) + const data = await retrieve(bucketName, filepath) + const outputPath = join(budibaseTempDir(), v4()) + fs.writeFileSync(outputPath, data) + return outputPath +} + +export const retrieveDirectory = async (bucketName: string, path: string) => { + let writePath = join(budibaseTempDir(), v4()) + fs.mkdirSync(writePath) + const objects = await listAllObjects(bucketName, path) + let fullObjects = await Promise.all( + objects.map(obj => retrieve(bucketName, obj.Key!)) + ) + let count = 0 + for (let obj of objects) { + const filename = obj.Key! + const data = fullObjects[count++] + const possiblePath = filename.split("/") + if (possiblePath.length > 1) { + const dirs = possiblePath.slice(0, possiblePath.length - 1) + fs.mkdirSync(join(writePath, ...dirs), { recursive: true }) + } + fs.writeFileSync(join(writePath, ...possiblePath), data) + } + return writePath +} + +/** + * Delete a single file. + */ +export const deleteFile = async (bucketName: string, filepath: string) => { + const objectStore = ObjectStore(bucketName) + await makeSureBucketExists(objectStore, bucketName) + const params = { + Bucket: bucketName, + Key: filepath, + } + return objectStore.deleteObject(params) +} + +export const deleteFiles = async (bucketName: string, filepaths: string[]) => { + const objectStore = ObjectStore(bucketName) + await makeSureBucketExists(objectStore, bucketName) + const params = { + Bucket: bucketName, + Delete: { + Objects: filepaths.map((path: any) => ({ Key: path })), + }, + } + return objectStore.deleteObjects(params).promise() +} + +/** + * Delete a path, including everything within. + */ +export const deleteFolder = async ( + bucketName: string, + folder: string +): Promise => { + bucketName = sanitizeBucket(bucketName) + folder = sanitizeKey(folder) + const client = ObjectStore(bucketName) + const listParams = { + Bucket: bucketName, + Prefix: folder, + } + + let response: any = await client.listObjects(listParams).promise() + if (response.Contents.length === 0) { + return + } + const deleteParams: any = { + Bucket: bucketName, + Delete: { + Objects: [], + }, + } + + response.Contents.forEach((content: any) => { + deleteParams.Delete.Objects.push({ Key: content.Key }) + }) + + response = await client.deleteObjects(deleteParams).promise() + // can only empty 1000 items at once + if (response.Deleted.length === 1000) { + return deleteFolder(bucketName, folder) + } +} + +export const uploadDirectory = async ( + bucketName: string, + localPath: string, + bucketPath: string +) => { + bucketName = sanitizeBucket(bucketName) + let uploads = [] + const files = fs.readdirSync(localPath, { withFileTypes: true }) + for (let file of files) { + const path = sanitizeKey(join(bucketPath, file.name)) + const local = join(localPath, file.name) + if (file.isDirectory()) { + uploads.push(uploadDirectory(bucketName, local, path)) + } else { + uploads.push(streamUpload(bucketName, path, fs.createReadStream(local))) + } + } + await Promise.all(uploads) + return files +} + +export const downloadTarballDirect = async ( + url: string, + path: string, + headers = {} +) => { + path = sanitizeKey(path) + const response = await fetch(url, { headers }) + if (!response.ok) { + throw new Error(`unexpected response ${response.statusText}`) + } + + await streamPipeline(response.body, zlib.Unzip(), tar.extract(path)) +} + +export const downloadTarball = async ( + url: string, + bucketName: string, + path: string +) => { + bucketName = sanitizeBucket(bucketName) + path = sanitizeKey(path) + const response = await fetch(url) + if (!response.ok) { + throw new Error(`unexpected response ${response.statusText}`) + } + + const tmpPath = join(budibaseTempDir(), path) + await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath)) + if (!env.isTest() && env.SELF_HOSTED) { + await uploadDirectory(bucketName, tmpPath, path) + } + // return the temporary path incase there is a use for it + return tmpPath +} diff --git a/packages/backend-core/src/objectStore/utils.js b/packages/backend-core/src/objectStore/utils.ts similarity index 71% rename from packages/backend-core/src/objectStore/utils.js rename to packages/backend-core/src/objectStore/utils.ts index 2d4faf55d1..f3c9e93943 100644 --- a/packages/backend-core/src/objectStore/utils.js +++ b/packages/backend-core/src/objectStore/utils.ts @@ -1,14 +1,15 @@ -const { join } = require("path") -const { tmpdir } = require("os") -const fs = require("fs") -const env = require("../environment") +import { join } from "path" +import { tmpdir } from "os" +import fs from "fs" +import env from "../environment" /**************************************************** * NOTE: When adding a new bucket - name * * sure that S3 usages (like budibase-infra) * * have been updated to have a unique bucket name. * ****************************************************/ -exports.ObjectStoreBuckets = { +// can't be an enum - only numbers can be used for computed types +export const ObjectStoreBuckets = { BACKUPS: env.BACKUPS_BUCKET_NAME, APPS: env.APPS_BUCKET_NAME, TEMPLATES: env.TEMPLATES_BUCKET_NAME, @@ -22,6 +23,6 @@ if (!fs.existsSync(bbTmp)) { fs.mkdirSync(bbTmp) } -exports.budibaseTempDir = function () { +export function budibaseTempDir() { return bbTmp } diff --git a/packages/backend-core/src/pino.js b/packages/backend-core/src/pino.js deleted file mode 100644 index 69962b3841..0000000000 --- a/packages/backend-core/src/pino.js +++ /dev/null @@ -1,11 +0,0 @@ -const env = require("./environment") - -exports.pinoSettings = () => ({ - prettyPrint: { - levelFirst: true, - }, - level: env.LOG_LEVEL || "error", - autoLogging: { - ignore: req => req.url.includes("/health"), - }, -}) diff --git a/packages/backend-core/src/pino.ts b/packages/backend-core/src/pino.ts new file mode 100644 index 0000000000..4140f428e1 --- /dev/null +++ b/packages/backend-core/src/pino.ts @@ -0,0 +1,13 @@ +import env from "./environment" + +export function pinoSettings() { + return { + prettyPrint: { + levelFirst: true, + }, + level: env.LOG_LEVEL || "error", + autoLogging: { + ignore: (req: { url: string }) => req.url.includes("/health"), + }, + } +} diff --git a/packages/backend-core/src/pkg/cache.ts b/packages/backend-core/src/pkg/cache.ts deleted file mode 100644 index c40a686260..0000000000 --- a/packages/backend-core/src/pkg/cache.ts +++ /dev/null @@ -1,13 +0,0 @@ -// Mimic the outer package export for usage in index.ts -// The outer exports can't be used as they now reference dist directly -import * as generic from "../cache/generic" -import * as user from "../cache/user" -import * as app from "../cache/appMetadata" -import * as writethrough from "../cache/writethrough" - -export = { - app, - user, - writethrough, - ...generic, -} diff --git a/packages/backend-core/src/pkg/context.ts b/packages/backend-core/src/pkg/context.ts deleted file mode 100644 index 4915cc6e41..0000000000 --- a/packages/backend-core/src/pkg/context.ts +++ /dev/null @@ -1,26 +0,0 @@ -// Mimic the outer package export for usage in index.ts -// The outer exports can't be used as they now reference dist directly -import { - getAppDB, - getDevAppDB, - getProdAppDB, - getAppId, - updateAppId, - doInAppContext, - doInTenant, - doInContext, -} from "../context" - -import * as identity from "../context/identity" - -export = { - getAppDB, - getDevAppDB, - getProdAppDB, - getAppId, - updateAppId, - doInAppContext, - doInTenant, - doInContext, - identity, -} diff --git a/packages/backend-core/src/pkg/objectStore.ts b/packages/backend-core/src/pkg/objectStore.ts deleted file mode 100644 index 0447c6b3c2..0000000000 --- a/packages/backend-core/src/pkg/objectStore.ts +++ /dev/null @@ -1,4 +0,0 @@ -// Mimic the outer package export for usage in index.ts -// The outer exports can't be used as they now reference dist directly -export * from "../objectStore" -export * from "../objectStore/utils" diff --git a/packages/backend-core/src/pkg/redis.ts b/packages/backend-core/src/pkg/redis.ts deleted file mode 100644 index 297c2b54f4..0000000000 --- a/packages/backend-core/src/pkg/redis.ts +++ /dev/null @@ -1,13 +0,0 @@ -// Mimic the outer package export for usage in index.ts -// The outer exports can't be used as they now reference dist directly -import Client from "../redis" -import utils from "../redis/utils" -import clients from "../redis/init" -import * as redlock from "../redis/redlock" - -export = { - Client, - utils, - clients, - redlock, -} diff --git a/packages/backend-core/src/pkg/utils.ts b/packages/backend-core/src/pkg/utils.ts deleted file mode 100644 index 5272046524..0000000000 --- a/packages/backend-core/src/pkg/utils.ts +++ /dev/null @@ -1,4 +0,0 @@ -// Mimic the outer package export for usage in index.ts -// The outer exports can't be used as they now reference dist directly -export * from "../utils" -export * from "../hashing" diff --git a/packages/backend-core/src/plugin/index.ts b/packages/backend-core/src/plugin/index.ts index a6d1853007..3eeaeaa90c 100644 --- a/packages/backend-core/src/plugin/index.ts +++ b/packages/backend-core/src/plugin/index.ts @@ -1,7 +1 @@ -import * as utils from "./utils" - -const pkg = { - ...utils, -} - -export = pkg +export * from "./utils" diff --git a/packages/backend-core/src/plugin/utils.js b/packages/backend-core/src/plugin/utils.ts similarity index 89% rename from packages/backend-core/src/plugin/utils.js rename to packages/backend-core/src/plugin/utils.ts index b943747483..7b62248bb5 100644 --- a/packages/backend-core/src/plugin/utils.js +++ b/packages/backend-core/src/plugin/utils.ts @@ -1,9 +1,5 @@ -const { - DatasourceFieldType, - QueryType, - PluginType, -} = require("@budibase/types") -const joi = require("joi") +import { DatasourceFieldType, QueryType, PluginType } from "@budibase/types" +import joi from "joi" const DATASOURCE_TYPES = [ "Relational", @@ -14,14 +10,14 @@ const DATASOURCE_TYPES = [ "API", ] -function runJoi(validator, schema) { +function runJoi(validator: joi.Schema, schema: any) { const { error } = validator.validate(schema) if (error) { throw error } } -function validateComponent(schema) { +function validateComponent(schema: any) { const validator = joi.object({ type: joi.string().allow("component").required(), metadata: joi.object().unknown(true).required(), @@ -37,7 +33,7 @@ function validateComponent(schema) { runJoi(validator, schema) } -function validateDatasource(schema) { +function validateDatasource(schema: any) { const fieldValidator = joi.object({ type: joi .string() @@ -86,7 +82,7 @@ function validateDatasource(schema) { runJoi(validator, schema) } -exports.validate = schema => { +export function validate(schema: any) { switch (schema?.type) { case PluginType.COMPONENT: validateComponent(schema) diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index eb054766d7..acfff1c7b8 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -1,5 +1,5 @@ import events from "events" -import { timeout } from "../../utils" +import { timeout } from "../utils" /** * Bull works with a Job wrapper around all messages that contains a lot more information about diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index b4eeeb31aa..b34d46e463 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -39,7 +39,7 @@ export function createQueue( return queue } -exports.shutdown = async () => { +export async function shutdown() { if (QUEUES.length) { clearInterval(cleanupInterval) for (let queue of QUEUES) { diff --git a/packages/backend-core/src/redis/index.ts b/packages/backend-core/src/redis/index.ts index 8a15320ff3..ea4379f048 100644 --- a/packages/backend-core/src/redis/index.ts +++ b/packages/backend-core/src/redis/index.ts @@ -1,278 +1,6 @@ -import RedisWrapper from "../redis" -const env = require("../environment") -// ioredis mock is all in memory -const Redis = env.isTest() ? require("ioredis-mock") : require("ioredis") -const { - addDbPrefix, - removeDbPrefix, - getRedisOptions, - SEPARATOR, - SelectableDatabases, -} = require("./utils") - -const RETRY_PERIOD_MS = 2000 -const STARTUP_TIMEOUT_MS = 5000 -const CLUSTERED = false -const DEFAULT_SELECT_DB = SelectableDatabases.DEFAULT - -// for testing just generate the client once -let CLOSED = false -let CLIENTS: { [key: number]: any } = {} -// if in test always connected -let CONNECTED = env.isTest() - -function pickClient(selectDb: number): any { - return CLIENTS[selectDb] -} - -function connectionError( - selectDb: number, - timeout: NodeJS.Timeout, - err: Error | string -) { - // manually shut down, ignore errors - if (CLOSED) { - return - } - pickClient(selectDb).disconnect() - CLOSED = true - // always clear this on error - clearTimeout(timeout) - CONNECTED = false - console.error("Redis connection failed - " + err) - setTimeout(() => { - init() - }, RETRY_PERIOD_MS) -} - -/** - * Inits the system, will error if unable to connect to redis cluster (may take up to 10 seconds) otherwise - * will return the ioredis client which will be ready to use. - */ -function init(selectDb = DEFAULT_SELECT_DB) { - let timeout: NodeJS.Timeout - CLOSED = false - let client = pickClient(selectDb) - // already connected, ignore - if (client && CONNECTED) { - return - } - // testing uses a single in memory client - if (env.isTest()) { - CLIENTS[selectDb] = new Redis(getRedisOptions()) - } - // start the timer - only allowed 5 seconds to connect - timeout = setTimeout(() => { - if (!CONNECTED) { - connectionError( - selectDb, - timeout, - "Did not successfully connect in timeout" - ) - } - }, STARTUP_TIMEOUT_MS) - - // disconnect any lingering client - if (client) { - client.disconnect() - } - const { redisProtocolUrl, opts, host, port } = getRedisOptions(CLUSTERED) - - if (CLUSTERED) { - client = new Redis.Cluster([{ host, port }], opts) - } else if (redisProtocolUrl) { - client = new Redis(redisProtocolUrl) - } else { - client = new Redis(opts) - } - // attach handlers - client.on("end", (err: Error) => { - connectionError(selectDb, timeout, err) - }) - client.on("error", (err: Error) => { - connectionError(selectDb, timeout, err) - }) - client.on("connect", () => { - clearTimeout(timeout) - CONNECTED = true - }) - CLIENTS[selectDb] = client -} - -function waitForConnection(selectDb: number = DEFAULT_SELECT_DB) { - return new Promise(resolve => { - if (pickClient(selectDb) == null) { - init() - } else if (CONNECTED) { - resolve("") - return - } - // check if the connection is ready - const interval = setInterval(() => { - if (CONNECTED) { - clearInterval(interval) - resolve("") - } - }, 500) - }) -} - -/** - * Utility function, takes a redis stream and converts it to a promisified response - - * this can only be done with redis streams because they will have an end. - * @param stream A redis stream, specifically as this type of stream will have an end. - * @param client The client to use for further lookups. - * @return {Promise} The final output of the stream - */ -function promisifyStream(stream: any, client: RedisWrapper) { - return new Promise((resolve, reject) => { - const outputKeys = new Set() - stream.on("data", (keys: string[]) => { - keys.forEach(key => { - outputKeys.add(key) - }) - }) - stream.on("error", (err: Error) => { - reject(err) - }) - stream.on("end", async () => { - const keysArray: string[] = Array.from(outputKeys) as string[] - try { - let getPromises = [] - for (let key of keysArray) { - getPromises.push(client.get(key)) - } - const jsonArray = await Promise.all(getPromises) - resolve( - keysArray.map(key => ({ - key: removeDbPrefix(key), - value: JSON.parse(jsonArray.shift()), - })) - ) - } catch (err) { - reject(err) - } - }) - }) -} - -export = class RedisWrapper { - _db: string - _select: number - - constructor(db: string, selectDb: number | null = null) { - this._db = db - this._select = selectDb || DEFAULT_SELECT_DB - } - - getClient() { - return pickClient(this._select) - } - - async init() { - CLOSED = false - init(this._select) - await waitForConnection(this._select) - return this - } - - async finish() { - CLOSED = true - this.getClient().disconnect() - } - - async scan(key = ""): Promise { - const db = this._db - key = `${db}${SEPARATOR}${key}` - let stream - if (CLUSTERED) { - let node = this.getClient().nodes("master") - stream = node[0].scanStream({ match: key + "*", count: 100 }) - } else { - stream = this.getClient().scanStream({ match: key + "*", count: 100 }) - } - return promisifyStream(stream, this.getClient()) - } - - async keys(pattern: string) { - const db = this._db - return this.getClient().keys(addDbPrefix(db, pattern)) - } - - async get(key: string) { - const db = this._db - let response = await this.getClient().get(addDbPrefix(db, key)) - // overwrite the prefixed key - if (response != null && response.key) { - response.key = key - } - // if its not an object just return the response - try { - return JSON.parse(response) - } catch (err) { - return response - } - } - - async bulkGet(keys: string[]) { - const db = this._db - if (keys.length === 0) { - return {} - } - const prefixedKeys = keys.map(key => addDbPrefix(db, key)) - let response = await this.getClient().mget(prefixedKeys) - if (Array.isArray(response)) { - let final: any = {} - let count = 0 - for (let result of response) { - if (result) { - let parsed - try { - parsed = JSON.parse(result) - } catch (err) { - parsed = result - } - final[keys[count]] = parsed - } - count++ - } - return final - } else { - throw new Error(`Invalid response: ${response}`) - } - } - - async store(key: string, value: any, expirySeconds: number | null = null) { - const db = this._db - if (typeof value === "object") { - value = JSON.stringify(value) - } - const prefixedKey = addDbPrefix(db, key) - await this.getClient().set(prefixedKey, value) - if (expirySeconds) { - await this.getClient().expire(prefixedKey, expirySeconds) - } - } - - async getTTL(key: string) { - const db = this._db - const prefixedKey = addDbPrefix(db, key) - return this.getClient().ttl(prefixedKey) - } - - async setExpiry(key: string, expirySeconds: number | null) { - const db = this._db - const prefixedKey = addDbPrefix(db, key) - await this.getClient().expire(prefixedKey, expirySeconds) - } - - async delete(key: string) { - const db = this._db - await this.getClient().del(addDbPrefix(db, key)) - } - - async clear() { - let items = await this.scan() - await Promise.all(items.map((obj: any) => this.delete(obj.key))) - } -} +// Mimic the outer package export for usage in index.ts +// The outer exports can't be used as they now reference dist directly +export { default as Client } from "./redis" +export * as utils from "./utils" +export * as clients from "./init" +export * as redlock from "./redlock" diff --git a/packages/backend-core/src/redis/init.js b/packages/backend-core/src/redis/init.js deleted file mode 100644 index 3150ef2c1c..0000000000 --- a/packages/backend-core/src/redis/init.js +++ /dev/null @@ -1,69 +0,0 @@ -const Client = require("./index") -const utils = require("./utils") - -let userClient, - sessionClient, - appClient, - cacheClient, - writethroughClient, - lockClient - -async function init() { - userClient = await new Client(utils.Databases.USER_CACHE).init() - sessionClient = await new Client(utils.Databases.SESSIONS).init() - appClient = await new Client(utils.Databases.APP_METADATA).init() - cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() - lockClient = await new Client(utils.Databases.LOCKS).init() - writethroughClient = await new Client( - utils.Databases.WRITE_THROUGH, - utils.SelectableDatabases.WRITE_THROUGH - ).init() -} - -process.on("exit", async () => { - if (userClient) await userClient.finish() - if (sessionClient) await sessionClient.finish() - if (appClient) await appClient.finish() - if (cacheClient) await cacheClient.finish() - if (writethroughClient) await writethroughClient.finish() - if (lockClient) await lockClient.finish() -}) - -module.exports = { - getUserClient: async () => { - if (!userClient) { - await init() - } - return userClient - }, - getSessionClient: async () => { - if (!sessionClient) { - await init() - } - return sessionClient - }, - getAppClient: async () => { - if (!appClient) { - await init() - } - return appClient - }, - getCacheClient: async () => { - if (!cacheClient) { - await init() - } - return cacheClient - }, - getWritethroughClient: async () => { - if (!writethroughClient) { - await init() - } - return writethroughClient - }, - getLockClient: async () => { - if (!lockClient) { - await init() - } - return lockClient - }, -} diff --git a/packages/backend-core/src/redis/init.ts b/packages/backend-core/src/redis/init.ts new file mode 100644 index 0000000000..00329ffb84 --- /dev/null +++ b/packages/backend-core/src/redis/init.ts @@ -0,0 +1,72 @@ +import Client from "./redis" +import * as utils from "./utils" + +let userClient: Client, + sessionClient: Client, + appClient: Client, + cacheClient: Client, + writethroughClient: Client, + lockClient: Client + +async function init() { + userClient = await new Client(utils.Databases.USER_CACHE).init() + sessionClient = await new Client(utils.Databases.SESSIONS).init() + appClient = await new Client(utils.Databases.APP_METADATA).init() + cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() + lockClient = await new Client(utils.Databases.LOCKS).init() + writethroughClient = await new Client( + utils.Databases.WRITE_THROUGH, + utils.SelectableDatabase.WRITE_THROUGH + ).init() +} + +process.on("exit", async () => { + if (userClient) await userClient.finish() + if (sessionClient) await sessionClient.finish() + if (appClient) await appClient.finish() + if (cacheClient) await cacheClient.finish() + if (writethroughClient) await writethroughClient.finish() + if (lockClient) await lockClient.finish() +}) + +export async function getUserClient() { + if (!userClient) { + await init() + } + return userClient +} + +export async function getSessionClient() { + if (!sessionClient) { + await init() + } + return sessionClient +} + +export async function getAppClient() { + if (!appClient) { + await init() + } + return appClient +} + +export async function getCacheClient() { + if (!cacheClient) { + await init() + } + return cacheClient +} + +export async function getWritethroughClient() { + if (!writethroughClient) { + await init() + } + return writethroughClient +} + +export async function getLockClient() { + if (!lockClient) { + await init() + } + return lockClient +} diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts new file mode 100644 index 0000000000..58734fc4f1 --- /dev/null +++ b/packages/backend-core/src/redis/redis.ts @@ -0,0 +1,279 @@ +import env from "../environment" +// ioredis mock is all in memory +const Redis = env.isTest() ? require("ioredis-mock") : require("ioredis") +import { + addDbPrefix, + removeDbPrefix, + getRedisOptions, + SEPARATOR, + SelectableDatabase, +} from "./utils" + +const RETRY_PERIOD_MS = 2000 +const STARTUP_TIMEOUT_MS = 5000 +const CLUSTERED = false +const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT + +// for testing just generate the client once +let CLOSED = false +let CLIENTS: { [key: number]: any } = {} +// if in test always connected +let CONNECTED = env.isTest() + +function pickClient(selectDb: number): any { + return CLIENTS[selectDb] +} + +function connectionError( + selectDb: number, + timeout: NodeJS.Timeout, + err: Error | string +) { + // manually shut down, ignore errors + if (CLOSED) { + return + } + pickClient(selectDb).disconnect() + CLOSED = true + // always clear this on error + clearTimeout(timeout) + CONNECTED = false + console.error("Redis connection failed - " + err) + setTimeout(() => { + init() + }, RETRY_PERIOD_MS) +} + +/** + * Inits the system, will error if unable to connect to redis cluster (may take up to 10 seconds) otherwise + * will return the ioredis client which will be ready to use. + */ +function init(selectDb = DEFAULT_SELECT_DB) { + let timeout: NodeJS.Timeout + CLOSED = false + let client = pickClient(selectDb) + // already connected, ignore + if (client && CONNECTED) { + return + } + // testing uses a single in memory client + if (env.isTest()) { + CLIENTS[selectDb] = new Redis(getRedisOptions()) + } + // start the timer - only allowed 5 seconds to connect + timeout = setTimeout(() => { + if (!CONNECTED) { + connectionError( + selectDb, + timeout, + "Did not successfully connect in timeout" + ) + } + }, STARTUP_TIMEOUT_MS) + + // disconnect any lingering client + if (client) { + client.disconnect() + } + const { redisProtocolUrl, opts, host, port } = getRedisOptions(CLUSTERED) + + if (CLUSTERED) { + client = new Redis.Cluster([{ host, port }], opts) + } else if (redisProtocolUrl) { + client = new Redis(redisProtocolUrl) + } else { + client = new Redis(opts) + } + // attach handlers + client.on("end", (err: Error) => { + connectionError(selectDb, timeout, err) + }) + client.on("error", (err: Error) => { + connectionError(selectDb, timeout, err) + }) + client.on("connect", () => { + clearTimeout(timeout) + CONNECTED = true + }) + CLIENTS[selectDb] = client +} + +function waitForConnection(selectDb: number = DEFAULT_SELECT_DB) { + return new Promise(resolve => { + if (pickClient(selectDb) == null) { + init() + } else if (CONNECTED) { + resolve("") + return + } + // check if the connection is ready + const interval = setInterval(() => { + if (CONNECTED) { + clearInterval(interval) + resolve("") + } + }, 500) + }) +} + +/** + * Utility function, takes a redis stream and converts it to a promisified response - + * this can only be done with redis streams because they will have an end. + * @param stream A redis stream, specifically as this type of stream will have an end. + * @param client The client to use for further lookups. + * @return {Promise} The final output of the stream + */ +function promisifyStream(stream: any, client: RedisWrapper) { + return new Promise((resolve, reject) => { + const outputKeys = new Set() + stream.on("data", (keys: string[]) => { + keys.forEach(key => { + outputKeys.add(key) + }) + }) + stream.on("error", (err: Error) => { + reject(err) + }) + stream.on("end", async () => { + const keysArray: string[] = Array.from(outputKeys) as string[] + try { + let getPromises = [] + for (let key of keysArray) { + getPromises.push(client.get(key)) + } + const jsonArray = await Promise.all(getPromises) + resolve( + keysArray.map(key => ({ + key: removeDbPrefix(key), + value: JSON.parse(jsonArray.shift()), + })) + ) + } catch (err) { + reject(err) + } + }) + }) +} + +class RedisWrapper { + _db: string + _select: number + + constructor(db: string, selectDb: number | null = null) { + this._db = db + this._select = selectDb || DEFAULT_SELECT_DB + } + + getClient() { + return pickClient(this._select) + } + + async init() { + CLOSED = false + init(this._select) + await waitForConnection(this._select) + return this + } + + async finish() { + CLOSED = true + this.getClient().disconnect() + } + + async scan(key = ""): Promise { + const db = this._db + key = `${db}${SEPARATOR}${key}` + let stream + if (CLUSTERED) { + let node = this.getClient().nodes("master") + stream = node[0].scanStream({ match: key + "*", count: 100 }) + } else { + stream = this.getClient().scanStream({ match: key + "*", count: 100 }) + } + return promisifyStream(stream, this.getClient()) + } + + async keys(pattern: string) { + const db = this._db + return this.getClient().keys(addDbPrefix(db, pattern)) + } + + async get(key: string) { + const db = this._db + let response = await this.getClient().get(addDbPrefix(db, key)) + // overwrite the prefixed key + if (response != null && response.key) { + response.key = key + } + // if its not an object just return the response + try { + return JSON.parse(response) + } catch (err) { + return response + } + } + + async bulkGet(keys: string[]) { + const db = this._db + if (keys.length === 0) { + return {} + } + const prefixedKeys = keys.map(key => addDbPrefix(db, key)) + let response = await this.getClient().mget(prefixedKeys) + if (Array.isArray(response)) { + let final: any = {} + let count = 0 + for (let result of response) { + if (result) { + let parsed + try { + parsed = JSON.parse(result) + } catch (err) { + parsed = result + } + final[keys[count]] = parsed + } + count++ + } + return final + } else { + throw new Error(`Invalid response: ${response}`) + } + } + + async store(key: string, value: any, expirySeconds: number | null = null) { + const db = this._db + if (typeof value === "object") { + value = JSON.stringify(value) + } + const prefixedKey = addDbPrefix(db, key) + await this.getClient().set(prefixedKey, value) + if (expirySeconds) { + await this.getClient().expire(prefixedKey, expirySeconds) + } + } + + async getTTL(key: string) { + const db = this._db + const prefixedKey = addDbPrefix(db, key) + return this.getClient().ttl(prefixedKey) + } + + async setExpiry(key: string, expirySeconds: number | null) { + const db = this._db + const prefixedKey = addDbPrefix(db, key) + await this.getClient().expire(prefixedKey, expirySeconds) + } + + async delete(key: string) { + const db = this._db + await this.getClient().del(addDbPrefix(db, key)) + } + + async clear() { + let items = await this.scan() + await Promise.all(items.map((obj: any) => this.delete(obj.key))) + } +} + +export = RedisWrapper diff --git a/packages/backend-core/src/redis/utils.js b/packages/backend-core/src/redis/utils.ts similarity index 68% rename from packages/backend-core/src/redis/utils.js rename to packages/backend-core/src/redis/utils.ts index af719197b5..4c556ebd54 100644 --- a/packages/backend-core/src/redis/utils.js +++ b/packages/backend-core/src/redis/utils.ts @@ -1,10 +1,10 @@ -const env = require("../environment") +import env from "../environment" const SLOT_REFRESH_MS = 2000 const CONNECT_TIMEOUT_MS = 10000 -const SEPARATOR = "-" const REDIS_URL = !env.REDIS_URL ? "localhost:6379" : env.REDIS_URL const REDIS_PASSWORD = !env.REDIS_PASSWORD ? "budibase" : env.REDIS_PASSWORD +export const SEPARATOR = "-" /** * These Redis databases help us to segment up a Redis keyspace by prepending the @@ -12,23 +12,23 @@ const REDIS_PASSWORD = !env.REDIS_PASSWORD ? "budibase" : env.REDIS_PASSWORD * can be split up a bit; allowing us to use scans on small databases to find some particular * keys within. * If writing a very large volume of keys is expected (say 10K+) then it is better to keep these out - * of the default keyspace and use a separate one - the SelectableDatabases can be used for this. + * of the default keyspace and use a separate one - the SelectableDatabase can be used for this. */ -exports.Databases = { - PW_RESETS: "pwReset", - VERIFICATIONS: "verification", - INVITATIONS: "invitation", - DEV_LOCKS: "devLocks", - DEBOUNCE: "debounce", - SESSIONS: "session", - USER_CACHE: "users", - FLAGS: "flags", - APP_METADATA: "appMetadata", - QUERY_VARS: "queryVars", - LICENSES: "license", - GENERIC_CACHE: "data_cache", - WRITE_THROUGH: "writeThrough", - LOCKS: "locks", +export enum Databases { + PW_RESETS = "pwReset", + VERIFICATIONS = "verification", + INVITATIONS = "invitation", + DEV_LOCKS = "devLocks", + DEBOUNCE = "debounce", + SESSIONS = "session", + USER_CACHE = "users", + FLAGS = "flags", + APP_METADATA = "appMetadata", + QUERY_VARS = "queryVars", + LICENSES = "license", + GENERIC_CACHE = "data_cache", + WRITE_THROUGH = "writeThrough", + LOCKS = "locks", } /** @@ -40,30 +40,28 @@ exports.Databases = { * but if you need to walk through all values in a database periodically then a separate selectable * keyspace should be used. */ -exports.SelectableDatabases = { - DEFAULT: 0, - WRITE_THROUGH: 1, - UNUSED_1: 2, - UNUSED_2: 3, - UNUSED_3: 4, - UNUSED_4: 5, - UNUSED_5: 6, - UNUSED_6: 7, - UNUSED_7: 8, - UNUSED_8: 9, - UNUSED_9: 10, - UNUSED_10: 11, - UNUSED_11: 12, - UNUSED_12: 13, - UNUSED_13: 14, - UNUSED_14: 15, +export enum SelectableDatabase { + DEFAULT = 0, + WRITE_THROUGH = 1, + UNUSED_1 = 2, + UNUSED_2 = 3, + UNUSED_3 = 4, + UNUSED_4 = 5, + UNUSED_5 = 6, + UNUSED_6 = 7, + UNUSED_7 = 8, + UNUSED_8 = 9, + UNUSED_9 = 10, + UNUSED_10 = 11, + UNUSED_11 = 12, + UNUSED_12 = 13, + UNUSED_13 = 14, + UNUSED_14 = 15, } -exports.SEPARATOR = SEPARATOR - -exports.getRedisOptions = (clustered = false) => { +export function getRedisOptions(clustered = false) { let password = REDIS_PASSWORD - let url = REDIS_URL.split("//") + let url: string[] | string = REDIS_URL.split("//") // get rid of the protocol url = url.length > 1 ? url[1] : url[0] // check for a password etc @@ -84,7 +82,7 @@ exports.getRedisOptions = (clustered = false) => { redisProtocolUrl = REDIS_URL } - const opts = { + const opts: any = { connectTimeout: CONNECT_TIMEOUT_MS, } if (clustered) { @@ -92,7 +90,7 @@ exports.getRedisOptions = (clustered = false) => { opts.redisOptions.tls = {} opts.redisOptions.password = password opts.slotsRefreshTimeout = SLOT_REFRESH_MS - opts.dnsLookup = (address, callback) => callback(null, address) + opts.dnsLookup = (address: string, callback: any) => callback(null, address) } else { opts.host = host opts.port = port @@ -101,14 +99,14 @@ exports.getRedisOptions = (clustered = false) => { return { opts, host, port, redisProtocolUrl } } -exports.addDbPrefix = (db, key) => { +export function addDbPrefix(db: string, key: string) { if (key.includes(db)) { return key } return `${db}${SEPARATOR}${key}` } -exports.removeDbPrefix = key => { +export function removeDbPrefix(key: string) { let parts = key.split(SEPARATOR) if (parts.length >= 2) { parts.shift() diff --git a/packages/backend-core/src/security/apiKeys.js b/packages/backend-core/src/security/apiKeys.js deleted file mode 100644 index e90418abb8..0000000000 --- a/packages/backend-core/src/security/apiKeys.js +++ /dev/null @@ -1 +0,0 @@ -exports.lookupApiKey = async () => {} diff --git a/packages/backend-core/src/security/encryption.js b/packages/backend-core/src/security/encryption.ts similarity index 73% rename from packages/backend-core/src/security/encryption.js rename to packages/backend-core/src/security/encryption.ts index c31f597652..a9006f302d 100644 --- a/packages/backend-core/src/security/encryption.js +++ b/packages/backend-core/src/security/encryption.ts @@ -1,5 +1,5 @@ -const crypto = require("crypto") -const env = require("../environment") +import crypto from "crypto" +import env from "../environment" const ALGO = "aes-256-ctr" const SECRET = env.JWT_SECRET @@ -8,13 +8,13 @@ const ITERATIONS = 10000 const RANDOM_BYTES = 16 const STRETCH_LENGTH = 32 -function stretchString(string, salt) { +function stretchString(string: string, salt: Buffer) { return crypto.pbkdf2Sync(string, salt, ITERATIONS, STRETCH_LENGTH, "sha512") } -exports.encrypt = input => { +export function encrypt(input: string) { const salt = crypto.randomBytes(RANDOM_BYTES) - const stretched = stretchString(SECRET, salt) + const stretched = stretchString(SECRET!, salt) const cipher = crypto.createCipheriv(ALGO, stretched, salt) const base = cipher.update(input) const final = cipher.final() @@ -22,10 +22,10 @@ exports.encrypt = input => { return `${salt.toString("hex")}${SEPARATOR}${encrypted}` } -exports.decrypt = input => { +export function decrypt(input: string) { const [salt, encrypted] = input.split(SEPARATOR) const saltBuffer = Buffer.from(salt, "hex") - const stretched = stretchString(SECRET, saltBuffer) + const stretched = stretchString(SECRET!, saltBuffer) const decipher = crypto.createDecipheriv(ALGO, stretched, saltBuffer) const base = decipher.update(Buffer.from(encrypted, "hex")) const final = decipher.final() diff --git a/packages/backend-core/src/tenancy/index.ts b/packages/backend-core/src/tenancy/index.ts index e0006abab2..1618a136dd 100644 --- a/packages/backend-core/src/tenancy/index.ts +++ b/packages/backend-core/src/tenancy/index.ts @@ -1,9 +1,2 @@ -import * as context from "../context" -import * as tenancy from "./tenancy" - -const pkg = { - ...context, - ...tenancy, -} - -export = pkg +export * from "../context" +export * from "./tenancy" diff --git a/packages/backend-core/src/tenancy/tenancy.ts b/packages/backend-core/src/tenancy/tenancy.ts index cc1088ab08..e0e0703433 100644 --- a/packages/backend-core/src/tenancy/tenancy.ts +++ b/packages/backend-core/src/tenancy/tenancy.ts @@ -1,10 +1,4 @@ -import { - doWithDB, - queryPlatformView, - StaticDatabases, - getGlobalDBName, - ViewName, -} from "../db" +import { doWithDB, queryPlatformView, getGlobalDBName } from "../db" import { DEFAULT_TENANT_ID, getTenantId, @@ -18,7 +12,7 @@ import { TenantResolutionStrategy, GetTenantIdOptions, } from "@budibase/types" -import { Header } from "../constants" +import { Header, StaticDatabases, ViewName } from "../constants" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name diff --git a/packages/backend-core/src/tests/utils.spec.js b/packages/backend-core/src/tests/utils.spec.js index 76fc7b4481..fb3828921d 100644 --- a/packages/backend-core/src/tests/utils.spec.js +++ b/packages/backend-core/src/tests/utils.spec.js @@ -1,7 +1,8 @@ const { structures } = require("../../tests") const utils = require("../utils") const events = require("../events") -const { doInTenant, DEFAULT_TENANT_ID }= require("../context") +const { DEFAULT_TENANT_ID } = require("../constants") +const { doInTenant } = require("../context") describe("utils", () => { describe("platformLogout", () => { diff --git a/packages/backend-core/src/hashing.js b/packages/backend-core/src/utils/hashing.ts similarity index 53% rename from packages/backend-core/src/hashing.js rename to packages/backend-core/src/utils/hashing.ts index 7524e66043..220ffea47f 100644 --- a/packages/backend-core/src/hashing.js +++ b/packages/backend-core/src/utils/hashing.ts @@ -1,18 +1,14 @@ -const env = require("./environment") +import env from "../environment" +export * from "../newid" const bcrypt = env.JS_BCRYPT ? require("bcryptjs") : require("bcrypt") -const { v4 } = require("uuid") const SALT_ROUNDS = env.SALT_ROUNDS || 10 -exports.hash = async data => { +export async function hash(data: string) { const salt = await bcrypt.genSalt(SALT_ROUNDS) return bcrypt.hash(data, salt) } -exports.compare = async (data, encrypted) => { +export async function compare(data: string, encrypted: string) { return bcrypt.compare(data, encrypted) } - -exports.newid = function () { - return v4().replace(/-/g, "") -} diff --git a/packages/backend-core/src/utils/index.ts b/packages/backend-core/src/utils/index.ts new file mode 100644 index 0000000000..8e663bce52 --- /dev/null +++ b/packages/backend-core/src/utils/index.ts @@ -0,0 +1,2 @@ +export * from "./hashing" +export * from "./utils" diff --git a/packages/backend-core/src/utils.ts b/packages/backend-core/src/utils/utils.ts similarity index 92% rename from packages/backend-core/src/utils.ts rename to packages/backend-core/src/utils/utils.ts index c04d6196b3..3e9fbb177a 100644 --- a/packages/backend-core/src/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -1,17 +1,11 @@ -import { - DocumentType, - SEPARATOR, - ViewName, - getAllApps, - queryGlobalView, -} from "./db" -import { options } from "./middleware/passport/jwt" -import { Header, Cookie, MAX_VALID_DATE } from "./constants" -import env from "./environment" -import userCache from "./cache/user" -import { getSessionsForUser, invalidateSessions } from "./security/sessions" -import * as events from "./events" -import tenancy from "./tenancy" +import { getAllApps, queryGlobalView } from "../db" +import { options } from "../middleware/passport/jwt" +import { Header, Cookie, MAX_VALID_DATE } from "../constants" +import env from "../environment" +import * as userCache from "../cache/user" +import { getSessionsForUser, invalidateSessions } from "../security/sessions" +import * as events from "../events" +import * as tenancy from "../tenancy" import { App, BBContext, @@ -19,6 +13,7 @@ import { TenantResolutionStrategy, } from "@budibase/types" import { SetOption } from "cookies" +import { DocumentType, SEPARATOR, ViewName } from "../constants" const jwt = require("jsonwebtoken") const APP_PREFIX = DocumentType.APP + SEPARATOR diff --git a/packages/backend-core/tenancy.js b/packages/backend-core/tenancy.js deleted file mode 100644 index 9ca808b74e..0000000000 --- a/packages/backend-core/tenancy.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require("./src/tenancy") diff --git a/packages/backend-core/tsconfig.build.json b/packages/backend-core/tsconfig.build.json index f5b16eda1a..9682f3e32f 100644 --- a/packages/backend-core/tsconfig.build.json +++ b/packages/backend-core/tsconfig.build.json @@ -3,7 +3,6 @@ "target": "es6", "module": "commonjs", "lib": ["es2020"], - "allowJs": true, "strict": true, "noImplicitAny": true, "esModuleInterop": true, diff --git a/packages/backend-core/utils.js b/packages/backend-core/utils.js deleted file mode 100644 index 2ef920e103..0000000000 --- a/packages/backend-core/utils.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = { - ...require("./src/utils"), - ...require("./src/hashing"), -} diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 6ba9f7b5ae..0a25d5fb43 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -1521,13 +1521,6 @@ assert-plus@1.0.0, assert-plus@^1.0.0: resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" integrity sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw== -async@~2.1.4: - version "2.1.5" - resolved "https://registry.yarnpkg.com/async/-/async-2.1.5.tgz#e587c68580994ac67fc56ff86d3ac56bdbe810bc" - integrity sha512-+g/Ncjbx0JSq2Mk03WQkyKvNh5q9Qvyo/RIqIqnmC5feJY70PNl2ESwZU2BhAB+AZPkHNzzyC2Dq2AS5VnTKhQ== - dependencies: - lodash "^4.14.0" - asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -2669,32 +2662,6 @@ globals@^11.1.0: resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== -google-auth-library@~0.10.0: - version "0.10.0" - resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-0.10.0.tgz#6e15babee85fd1dd14d8d128a295b6838d52136e" - integrity sha512-KM54Y9GhdAzfXUHmWEoYmaOykSLuMG7W4HvVLYqyogxOyE6px8oSS8W13ngqW0oDGZ915GFW3V6OM6+qcdvPOA== - dependencies: - gtoken "^1.2.1" - jws "^3.1.4" - lodash.noop "^3.0.1" - request "^2.74.0" - -google-p12-pem@^0.1.0: - version "0.1.2" - resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-0.1.2.tgz#33c46ab021aa734fa0332b3960a9a3ffcb2f3177" - integrity sha512-puhMlJ2+E/rgvxWaqgN/nC7x623OAE8MR9vBUqxF0inCE7HoVfCHvTeQ9+BR+rj9KM0fIg6XV6tmbt7XHHssoQ== - dependencies: - node-forge "^0.7.1" - -googleapis@^16.0.0: - version "16.1.0" - resolved "https://registry.yarnpkg.com/googleapis/-/googleapis-16.1.0.tgz#0f19f2d70572d918881a0f626e3b1a2fa8629576" - integrity sha512-5czmF7xkIlJKc1+/+5tltrI1skoR3HKtkDOld9rk+DOucTpZRjOhCoJzoSjxB3M8rP2tEb1VIr1TPyzR3V2PUQ== - dependencies: - async "~2.1.4" - google-auth-library "~0.10.0" - string-template "~1.0.0" - got@^9.6.0: version "9.6.0" resolved "https://registry.yarnpkg.com/got/-/got-9.6.0.tgz#edf45e7d67f99545705de1f7bbeeeb121765ed85" @@ -2717,16 +2684,6 @@ graceful-fs@^4.1.2, graceful-fs@^4.2.9: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== -gtoken@^1.2.1: - version "1.2.3" - resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-1.2.3.tgz#5509571b8afd4322e124cf66cf68115284c476d8" - integrity sha512-wQAJflfoqSgMWrSBk9Fg86q+sd6s7y6uJhIvvIPz++RElGlMtEqsdAR2oWwZ/WTEtp7P9xFbJRrT976oRgzJ/w== - dependencies: - google-p12-pem "^0.1.0" - jws "^3.0.0" - mime "^1.4.1" - request "^2.72.0" - har-schema@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" @@ -3609,7 +3566,7 @@ jwa@^1.4.1: ecdsa-sig-formatter "1.0.11" safe-buffer "^5.0.1" -jws@^3.0.0, jws@^3.1.4, jws@^3.2.2: +jws@^3.2.2: version "3.2.2" resolved "https://registry.yarnpkg.com/jws/-/jws-3.2.2.tgz#001099f3639468c9414000e99995fa52fb478304" integrity sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA== @@ -3855,11 +3812,6 @@ lodash.memoize@4.x: resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe" integrity sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag== -lodash.noop@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/lodash.noop/-/lodash.noop-3.0.1.tgz#38188f4d650a3a474258439b96ec45b32617133c" - integrity sha512-TmYdmu/pebrdTIBDK/FDx9Bmfzs9x0sZG6QIJuMDTqEPfeciLcN13ij+cOd0i9vwJfBtbG9UQ+C7MkXgYxrIJg== - lodash.once@^4.0.0: version "4.1.1" resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" @@ -3870,7 +3822,7 @@ lodash.pick@^4.0.0: resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3" integrity sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q== -lodash@4.17.21, lodash@^4.14.0, lodash@^4.17.21: +lodash@4.17.21, lodash@^4.17.21: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -3982,7 +3934,7 @@ mime-types@^2.1.12, mime-types@^2.1.18, mime-types@~2.1.19, mime-types@~2.1.24, dependencies: mime-db "1.52.0" -mime@^1.3.4, mime@^1.4.1: +mime@^1.3.4: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== @@ -4129,11 +4081,6 @@ node-fetch@2.6.7, node-fetch@^2.6.7: dependencies: whatwg-url "^5.0.0" -node-forge@^0.7.1: - version "0.7.6" - resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.7.6.tgz#fdf3b418aee1f94f0ef642cd63486c77ca9724ac" - integrity sha512-sol30LUpz1jQFBjOKwbjxijiE3b6pjd74YwfD0fJOKPjF+fONKb2Yg8rYgS6+bK6VDl+/wfr4IYpC7jDzLUIfw== - node-gyp-build-optional-packages@5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17" @@ -4356,14 +4303,6 @@ parseurl@^1.3.2, parseurl@^1.3.3: resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== -passport-google-auth@1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/passport-google-auth/-/passport-google-auth-1.0.2.tgz#8b300b5aa442ef433de1d832ed3112877d0b2938" - integrity sha512-cfAqna6jZLyMEwUdd4PIwAh2mQKQVEDAaRIaom1pG6h4x4Gwjllf/Jflt3TkR1Sen5Rkvr3l7kSXCWE1EKkh8g== - dependencies: - googleapis "^16.0.0" - passport-strategy "1.x" - passport-google-oauth1@1.x.x: version "1.0.0" resolved "https://registry.yarnpkg.com/passport-google-oauth1/-/passport-google-oauth1-1.0.0.tgz#af74a803df51ec646f66a44d82282be6f108e0cc" @@ -4426,7 +4365,7 @@ passport-oauth2@1.x.x: uid2 "0.0.x" utils-merge "1.x.x" -passport-strategy@1.x, passport-strategy@1.x.x, passport-strategy@^1.0.0: +passport-strategy@1.x.x, passport-strategy@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== @@ -4938,7 +4877,7 @@ remove-trailing-slash@^0.1.1: resolved "https://registry.yarnpkg.com/remove-trailing-slash/-/remove-trailing-slash-0.1.1.tgz#be2285a59f39c74d1bce4f825950061915e3780d" integrity sha512-o4S4Qh6L2jpnCy83ysZDau+VORNvnFw07CKSAymkd6ICNVEPisMyzlc00KlvvicsxKck94SEwhDnMNdICzO+tA== -request@^2.72.0, request@^2.74.0, request@^2.88.0: +request@^2.88.0: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== @@ -5204,11 +5143,6 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -string-template@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/string-template/-/string-template-1.0.0.tgz#9e9f2233dc00f218718ec379a28a5673ecca8b96" - integrity sha512-SLqR3GBUXuoPP5MmYtD7ompvXiG87QjT6lzOszyXjTM86Uu7At7vNnt2xgyTLq5o9T4IxTYFyGxcULqpsmsfdg== - "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 2e2a9f7f36..0568fb3085 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.1.32-alpha.11", + "version": "2.1.43-alpha.5", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "2.1.32-alpha.11", + "@budibase/string-templates": "2.1.43-alpha.5", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/bbui/src/Actions/click_outside.js b/packages/bbui/src/Actions/click_outside.js index 7fd2879071..3a08484635 100644 --- a/packages/bbui/src/Actions/click_outside.js +++ b/packages/bbui/src/Actions/click_outside.js @@ -1,18 +1,53 @@ -export default function clickOutside(element, callbackFunction) { - function onClick(event) { - if (!element.contains(event.target)) { - callbackFunction(event) +const ignoredClasses = [".flatpickr-calendar", ".modal-container"] +let clickHandlers = [] + +/** + * Handle a body click event + */ +const handleClick = event => { + // Ignore click if needed + for (let className of ignoredClasses) { + if (event.target.closest(className)) { + return } } - document.body.addEventListener("click", onClick, true) + // Process handlers + clickHandlers.forEach(handler => { + if (!handler.element.contains(event.target)) { + handler.callback?.(event) + } + }) +} +document.documentElement.addEventListener("click", handleClick, true) - return { - update(newCallbackFunction) { - callbackFunction = newCallbackFunction - }, - destroy() { - document.body.removeEventListener("click", onClick, true) - }, +/** + * Adds or updates a click handler + */ +const updateHandler = (id, element, callback) => { + let existingHandler = clickHandlers.find(x => x.id === id) + if (!existingHandler) { + clickHandlers.push({ id, element, callback }) + } else { + existingHandler.callback = callback + } +} + +/** + * Removes a click handler + */ +const removeHandler = id => { + clickHandlers = clickHandlers.filter(x => x.id !== id) +} + +/** + * Svelte action to apply a click outside handler for a certain element + */ +export default (element, callback) => { + const id = Math.random() + updateHandler(id, element, callback) + return { + update: newCallback => updateHandler(id, element, newCallback), + destroy: () => removeHandler(id), } } diff --git a/packages/bbui/src/DetailSummary/DetailSummary.svelte b/packages/bbui/src/DetailSummary/DetailSummary.svelte index 518c615504..f7e2611792 100644 --- a/packages/bbui/src/DetailSummary/DetailSummary.svelte +++ b/packages/bbui/src/DetailSummary/DetailSummary.svelte @@ -19,13 +19,19 @@
-
-
{name}
- {#if collapsible} - - {/if} -
-
+ {#if name} +
+
{name}
+ {#if collapsible} + + {/if} +
+ {/if} +
@@ -72,6 +78,9 @@ padding: var(--spacing-s) var(--spacing-xl) var(--spacing-xl) var(--spacing-xl); } + .property-panel.no-title { + padding: var(--spacing-xl); + } .show { display: flex; diff --git a/packages/bbui/src/Form/Core/DatePicker.svelte b/packages/bbui/src/Form/Core/DatePicker.svelte index 379f41b284..6996525a76 100644 --- a/packages/bbui/src/Form/Core/DatePicker.svelte +++ b/packages/bbui/src/Form/Core/DatePicker.svelte @@ -23,6 +23,15 @@ let open = false let flatpickr, flatpickrOptions + // Another classic flatpickr issue. Errors were randomly being thrown due to + // flatpickr internal code. Making sure that "destroy" is a valid function + // fixes it. The sooner we remove flatpickr the better. + $: { + if (flatpickr && !flatpickr.destroy) { + flatpickr.destroy = () => {} + } + } + const resolveTimeStamp = timestamp => { let maskedDate = new Date(`0-${timestamp}`) @@ -197,6 +206,7 @@ {/if} {/if}
-
{message || ""}
+
{message || ""}
{#if action}
{actionMessage}
@@ -53,6 +53,10 @@
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte index f6cd6af758..45269a365c 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte @@ -7,9 +7,9 @@ let name = "" let submitted = false - $: valid = name && name.length > 0 && !datasource?.entities[name] + $: valid = name && name.length > 0 && !datasource?.entities?.[name] $: error = - !submitted && name && datasource?.entities[name] + !submitted && name && datasource?.entities?.[name] ? "Table name already in use." : null diff --git a/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte b/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte index ae0023f682..165ed18ad8 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/popovers/EditDatasourcePopover.svelte @@ -5,6 +5,7 @@ import { ActionMenu, MenuItem, Icon } from "@budibase/bbui" import ConfirmDialog from "components/common/ConfirmDialog.svelte" import UpdateDatasourceModal from "components/backend/DatasourceNavigator/modals/UpdateDatasourceModal.svelte" + import { BUDIBASE_DATASOURCE_TYPE } from "constants/backend" export let datasource @@ -42,7 +43,9 @@
- Edit + {#if datasource.type !== BUDIBASE_DATASOURCE_TYPE} + Edit + {/if} Delete diff --git a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte index 230748b577..4defcbafab 100644 --- a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte +++ b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte @@ -45,6 +45,23 @@ const touched = writable({}) + function invalidThroughTable({ through, throughTo, throughFrom }) { + // need to know the foreign key columns to check error + if (!through || !throughTo || !throughFrom) { + return false + } + const throughTable = plusTables.find(tbl => tbl._id === through) + const otherColumns = Object.values(throughTable.schema).filter( + col => col.name !== throughFrom && col.name !== throughTo + ) + for (let col of otherColumns) { + if (col.constraints?.presence && !col.autocolumn) { + return true + } + } + return false + } + function checkForErrors(fromRelate, toRelate) { const isMany = fromRelate.relationshipType === RelationshipTypes.MANY_TO_MANY @@ -59,6 +76,10 @@ if ($touched.through && isMany && !fromRelate.through) { errObj.through = tableNotSet } + if ($touched.through && invalidThroughTable(fromRelate)) { + errObj.through = + "Ensure all columns in table are nullable or auto generated" + } if ($touched.foreign && !isMany && !fromRelate.fieldName) { errObj.foreign = "Please pick the foreign key" } diff --git a/packages/builder/src/components/design/settings/componentSettings.js b/packages/builder/src/components/design/settings/componentSettings.js index 1d8df7a052..f9f4295c17 100644 --- a/packages/builder/src/components/design/settings/componentSettings.js +++ b/packages/builder/src/components/design/settings/componentSettings.js @@ -1,4 +1,4 @@ -import { Checkbox, Select, Stepper } from "@budibase/bbui" +import { Checkbox, Select, RadioGroup, Stepper } from "@budibase/bbui" import DataSourceSelect from "./controls/DataSourceSelect.svelte" import S3DataSourceSelect from "./controls/S3DataSourceSelect.svelte" import DataProviderSelect from "./controls/DataProviderSelect.svelte" @@ -25,6 +25,7 @@ import BarButtonList from "./controls/BarButtonList.svelte" const componentMap = { text: DrawerBindableCombobox, select: Select, + radio: RadioGroup, dataSource: DataSourceSelect, "dataSource/s3": S3DataSourceSelect, dataProvider: DataProviderSelect, diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/CloseSidePanel.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/CloseSidePanel.svelte new file mode 100644 index 0000000000..ed0ca2c72b --- /dev/null +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/CloseSidePanel.svelte @@ -0,0 +1,8 @@ +
This action doesn't require any settings.
+ + diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte index 0ca5bfaa76..109eb9a956 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte @@ -27,6 +27,11 @@ />