MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/environment-variables' of github.com:Budibase/budibase into feature/environment-variables

This commit is contained in:
Peter Clement 2023-01-17 10:15:03 +00:00
parent e38bde79ac 95e8371b1d
commit 81a6286f42
48 changed files with 498 additions and 345 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,5 +1,5 @@
{ {
"version": "2.2.12-alpha.16", "version": "2.2.12-alpha.20",
"npmClient": "yarn", "npmClient": "yarn",
"packages": [ "packages": [
"packages/*" "packages/*"

4
packages/backend-core/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@budibase/backend-core", "name": "@budibase/backend-core",
"version": "2.2.12-alpha.16", "version": "2.2.12-alpha.20",
"description": "Budibase backend core libraries used in server and worker", "description": "Budibase backend core libraries used in server and worker",
"main": "dist/src/index.js", "main": "dist/src/index.js",
"types": "dist/src/index.d.ts", "types": "dist/src/index.d.ts",
@ -23,7 +23,7 @@
}, },
"dependencies": { "dependencies": {
"@budibase/nano": "10.1.1", "@budibase/nano": "10.1.1",
"@budibase/types": "2.2.12-alpha.16", "@budibase/types": "2.2.12-alpha.20",
"@shopify/jest-koa-mocks": "5.0.1", "@shopify/jest-koa-mocks": "5.0.1",
"@techpass/passport-openidconnect": "0.3.2", "@techpass/passport-openidconnect": "0.3.2",
"aws-cloudfront-sign": "2.2.0", "aws-cloudfront-sign": "2.2.0",

13
packages/backend-core/src/context/Context.ts
View File

@ -1,17 +1,14 @@
import { AsyncLocalStorage } from "async_hooks" import { AsyncLocalStorage } from "async_hooks"
import { ContextMap } from "./mainContext"
export default class Context { export default class Context {
static storage = new AsyncLocalStorage<Record<string, any>>() static storage = new AsyncLocalStorage<ContextMap>()
static run(context: Record<string, any>, func: any) { static run(context: ContextMap, func: any) {
return Context.storage.run(context, () => func()) return Context.storage.run(context, () => func())
} }
static get(): Record<string, any> { static get(): ContextMap {
return Context.storage.getStore() as Record<string, any> return Context.storage.getStore() as ContextMap
}
static set(context: Record<string, any>) {
Context.storage.enterWith(context)
} }
} }

57
packages/backend-core/src/context/mainContext.ts
View File

@ -16,6 +16,7 @@ export type ContextMap = {
tenantId?: string tenantId?: string
appId?: string appId?: string
identity?: IdentityContext identity?: IdentityContext
environmentVariables?: Record<string, string>
} }
let TEST_APP_ID: string | null = null let TEST_APP_ID: string | null = null
@ -75,7 +76,7 @@ export function getTenantIDFromAppID(appId: string) {
} }
} }
function updateContext(updates: ContextMap) { function updateContext(updates: ContextMap): ContextMap {
let context: ContextMap let context: ContextMap
try { try {
context = Context.get() context = Context.get()
@ -120,15 +121,23 @@ export async function doInTenant(
return newContext(updates, task) return newContext(updates, task)
} }
export async function doInAppContext(appId: string, task: any): Promise<any> { export async function doInAppContext(
if (!appId) { appId: string | null,
task: any
): Promise<any> {
if (!appId && !env.isTest()) {
throw new Error("appId is required") throw new Error("appId is required")
} }
const tenantId = getTenantIDFromAppID(appId) let updates: ContextMap
const updates: ContextMap = { appId } if (!appId) {
if (tenantId) { updates = { appId: "" }
updates.tenantId = tenantId } else {
const tenantId = getTenantIDFromAppID(appId)
updates = { appId }
if (tenantId) {
updates.tenantId = tenantId
}
} }
return newContext(updates, task) return newContext(updates, task)
} }
@ -189,25 +198,25 @@ export const getProdAppId = () => {
return conversions.getProdAppID(appId) return conversions.getProdAppID(appId)
} }
export function updateTenantId(tenantId?: string) { export function doInEnvironmentContext(
let context: ContextMap = updateContext({ values: Record<string, string>,
tenantId, task: any
}) ) {
Context.set(context) if (!values) {
throw new Error("Must supply environment variables.")
}
const updates = {
environmentVariables: values,
}
return newContext(updates, task)
} }
export function updateAppId(appId: string) { export function getEnvironmentVariables() {
let context: ContextMap = updateContext({ const context = Context.get()
appId, if (!context.environmentVariables) {
}) return null
try { } else {
Context.set(context) return context.environmentVariables
} catch (err) {
if (env.isTest()) {
TEST_APP_ID = appId
} else {
throw err
}
} }
} }

1
packages/backend-core/src/environment.ts
View File

@ -37,6 +37,7 @@ const environment = {
}, },
JS_BCRYPT: process.env.JS_BCRYPT, JS_BCRYPT: process.env.JS_BCRYPT,
JWT_SECRET: process.env.JWT_SECRET, JWT_SECRET: process.env.JWT_SECRET,
ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,

39
packages/backend-core/src/security/encryption.ts
View File

@ -2,19 +2,45 @@ import crypto from "crypto"
import env from "../environment" import env from "../environment"
const ALGO = "aes-256-ctr" const ALGO = "aes-256-ctr"
const SECRET = env.JWT_SECRET
const SEPARATOR = "-" const SEPARATOR = "-"
const ITERATIONS = 10000 const ITERATIONS = 10000
const RANDOM_BYTES = 16 const RANDOM_BYTES = 16
const STRETCH_LENGTH = 32 const STRETCH_LENGTH = 32
export enum SecretOption {
JWT = "jwt",
ENCRYPTION = "encryption",
}
function getSecret(secretOption: SecretOption): string {
let secret, secretName
switch (secretOption) {
case SecretOption.ENCRYPTION:
secret = env.ENCRYPTION_KEY
secretName = "ENCRYPTION_KEY"
break
case SecretOption.JWT:
default:
secret = env.JWT_SECRET
secretName = "JWT_SECRET"
break
}
if (!secret) {
throw new Error(`Secret "${secretName}" has not been set in environment.`)
}
return secret
}
function stretchString(string: string, salt: Buffer) { function stretchString(string: string, salt: Buffer) {
return crypto.pbkdf2Sync(string, salt, ITERATIONS, STRETCH_LENGTH, "sha512") return crypto.pbkdf2Sync(string, salt, ITERATIONS, STRETCH_LENGTH, "sha512")
} }
export function encrypt(input: string, secret: string | undefined = SECRET) { export function encrypt(
input: string,
secretOption: SecretOption = SecretOption.JWT
) {
const salt = crypto.randomBytes(RANDOM_BYTES) const salt = crypto.randomBytes(RANDOM_BYTES)
const stretched = stretchString(secret!, salt) const stretched = stretchString(getSecret(secretOption), salt)
const cipher = crypto.createCipheriv(ALGO, stretched, salt) const cipher = crypto.createCipheriv(ALGO, stretched, salt)
const base = cipher.update(input) const base = cipher.update(input)
const final = cipher.final() const final = cipher.final()
@ -22,10 +48,13 @@ export function encrypt(input: string, secret: string | undefined = SECRET) {
return `${salt.toString("hex")}${SEPARATOR}${encrypted}` return `${salt.toString("hex")}${SEPARATOR}${encrypted}`
} }
export function decrypt(input: string, secret: string | undefined = SECRET) { export function decrypt(
input: string,
secretOption: SecretOption = SecretOption.JWT
) {
const [salt, encrypted] = input.split(SEPARATOR) const [salt, encrypted] = input.split(SEPARATOR)
const saltBuffer = Buffer.from(salt, "hex") const saltBuffer = Buffer.from(salt, "hex")
const stretched = stretchString(secret!, saltBuffer) const stretched = stretchString(getSecret(secretOption), saltBuffer)
const decipher = crypto.createDecipheriv(ALGO, stretched, saltBuffer) const decipher = crypto.createDecipheriv(ALGO, stretched, saltBuffer)
const base = decipher.update(Buffer.from(encrypted, "hex")) const base = decipher.update(Buffer.from(encrypted, "hex"))
const final = decipher.final() const final = decipher.final()

4
packages/bbui/package.json
View File

@ -1,7 +1,7 @@
{ {
"name": "@budibase/bbui", "name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.", "description": "A UI solution used in the different Budibase projects.",
"version": "2.2.12-alpha.16", "version": "2.2.12-alpha.20",
"license": "MPL-2.0", "license": "MPL-2.0",
"svelte": "src/index.js", "svelte": "src/index.js",
"module": "dist/bbui.es.js", "module": "dist/bbui.es.js",
@ -38,7 +38,7 @@
], ],
"dependencies": { "dependencies": {
"@adobe/spectrum-css-workflow-icons": "1.2.1", "@adobe/spectrum-css-workflow-icons": "1.2.1",
"@budibase/string-templates": "2.2.12-alpha.16", "@budibase/string-templates": "2.2.12-alpha.20",
"@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actionbutton": "1.0.1",
"@spectrum-css/actiongroup": "1.0.1", "@spectrum-css/actiongroup": "1.0.1",
"@spectrum-css/avatar": "3.0.2", "@spectrum-css/avatar": "3.0.2",

12
packages/builder/cypress/integration/createScreen.spec.js
View File

@ -2,7 +2,7 @@ import filterTests from "../support/filterTests"
const interact = require('../support/interact') const interact = require('../support/interact')
filterTests(["smoke", "all"], () => { filterTests(["smoke", "all"], () => {
context("Screen Tests", () => { xcontext("Screen Tests", () => {
before(() => { before(() => {
cy.login() cy.login()
cy.createTestApp() cy.createTestApp()
@ -25,7 +25,7 @@ filterTests(["smoke", "all"], () => {
it.skip("should delete all screens then create first screen via button", () => { it.skip("should delete all screens then create first screen via button", () => {
cy.deleteAllScreens() cy.deleteAllScreens()
cy.contains("Create first screen").click() cy.contains("Create first screen").click()
cy.get(interact.BODY, { timeout: 2000 }).should('contain', '/home') cy.get(interact.BODY, { timeout: 2000 }).should('contain', '/home')
}) })
@ -33,7 +33,7 @@ filterTests(["smoke", "all"], () => {
it("Should create and filter screens by access level", () => { it("Should create and filter screens by access level", () => {
const accessLevels = ["Basic", "Admin", "Public", "Power"] const accessLevels = ["Basic", "Admin", "Public", "Power"]
for (const access of accessLevels){ for (const access of accessLevels) {
// Create screen with specified access level // Create screen with specified access level
cy.createScreen(access, access) cy.createScreen(access, access)
// Filter by access level and confirm screen visible // Filter by access level and confirm screen visible
@ -46,9 +46,9 @@ filterTests(["smoke", "all"], () => {
// Filter by All screens - Confirm all screens visible // Filter by All screens - Confirm all screens visible
cy.filterScreensAccessLevel("All screens") cy.filterScreensAccessLevel("All screens")
cy.get(interact.BODY).should('contain', accessLevels[0]) cy.get(interact.BODY).should('contain', accessLevels[0])
.and('contain', accessLevels[1]) .and('contain', accessLevels[1])
.and('contain', accessLevels[2]) .and('contain', accessLevels[2])
.and('contain', accessLevels[3]) .and('contain', accessLevels[3])
}) })
}) })
}) })

10
packages/builder/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@budibase/builder", "name": "@budibase/builder",
"version": "2.2.12-alpha.16", "version": "2.2.12-alpha.20",
"license": "GPL-3.0", "license": "GPL-3.0",
"private": true, "private": true,
"scripts": { "scripts": {
@ -71,10 +71,10 @@
} }
}, },
"dependencies": { "dependencies": {
"@budibase/bbui": "2.2.12-alpha.16", "@budibase/bbui": "2.2.12-alpha.20",
"@budibase/client": "2.2.12-alpha.16", "@budibase/client": "2.2.12-alpha.20",
"@budibase/frontend-core": "2.2.12-alpha.16", "@budibase/frontend-core": "2.2.12-alpha.20",
"@budibase/string-templates": "2.2.12-alpha.16", "@budibase/string-templates": "2.2.12-alpha.20",
"@sentry/browser": "5.19.1", "@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1", "@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1", "@spectrum-css/vars": "^3.0.1",

4
packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte
View File

@ -340,9 +340,7 @@
{:else if isManyToOne && toTable} {:else if isManyToOne && toTable}
<Select <Select
label={`Foreign Key (${toTable?.name})`} label={`Foreign Key (${toTable?.name})`}
options={Object.keys(toTable?.schema).filter( options={Object.keys(toTable?.schema)}
field => toTable?.primary.indexOf(field) === -1
)}
on:change={() => ($touched.foreign = true)} on:change={() => ($touched.foreign = true)}
bind:error={errors.foreign} bind:error={errors.foreign}
bind:value={fromRelationship.fieldName} bind:value={fromRelationship.fieldName}

8
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@budibase/cli", "name": "@budibase/cli",
"version": "2.2.12-alpha.16", "version": "2.2.12-alpha.20",
"description": "Budibase CLI, for developers, self hosting and migrations.", "description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js", "main": "src/index.js",
"bin": { "bin": {
@ -26,9 +26,9 @@
"outputPath": "build" "outputPath": "build"
}, },
"dependencies": { "dependencies": {
"@budibase/backend-core": "2.2.12-alpha.16", "@budibase/backend-core": "2.2.12-alpha.20",
"@budibase/string-templates": "2.2.12-alpha.16", "@budibase/string-templates": "2.2.12-alpha.20",
"@budibase/types": "2.2.12-alpha.16", "@budibase/types": "2.2.12-alpha.20",
"axios": "0.21.2", "axios": "0.21.2",
"chalk": "4.1.0", "chalk": "4.1.0",
"cli-progress": "3.11.2", "cli-progress": "3.11.2",

8
packages/client/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@budibase/client", "name": "@budibase/client",
"version": "2.2.12-alpha.16", "version": "2.2.12-alpha.20",
"license": "MPL-2.0", "license": "MPL-2.0",
"module": "dist/budibase-client.js", "module": "dist/budibase-client.js",
"main": "dist/budibase-client.js", "main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw" "dev:builder": "rollup -cw"
}, },
"dependencies": { "dependencies": {
"@budibase/bbui": "2.2.12-alpha.16", "@budibase/bbui": "2.2.12-alpha.20",
"@budibase/frontend-core": "2.2.12-alpha.16", "@budibase/frontend-core": "2.2.12-alpha.20",
"@budibase/string-templates": "2.2.12-alpha.16", "@budibase/string-templates": "2.2.12-alpha.20",
"@spectrum-css/button": "^3.0.3", "@spectrum-css/button": "^3.0.3",
"@spectrum-css/card": "^3.0.3", "@spectrum-css/card": "^3.0.3",
"@spectrum-css/divider": "^1.0.3", "@spectrum-css/divider": "^1.0.3",

4
packages/frontend-core/package.json
View File

@ -1,12 +1,12 @@
{ {
"name": "@budibase/frontend-core", "name": "@budibase/frontend-core",
"version": "2.2.12-alpha.16", "version": "2.2.12-alpha.20",
"description": "Budibase frontend core libraries used in builder and client", "description": "Budibase frontend core libraries used in builder and client",
"author": "Budibase", "author": "Budibase",
"license": "MPL-2.0", "license": "MPL-2.0",
"svelte": "src/index.js", "svelte": "src/index.js",
"dependencies": { "dependencies": {
"@budibase/bbui": "2.2.12-alpha.16", "@budibase/bbui": "2.2.12-alpha.20",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"svelte": "^3.46.2" "svelte": "^3.46.2"
} }

2
packages/sdk/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@budibase/sdk", "name": "@budibase/sdk",
"version": "2.2.12-alpha.16", "version": "2.2.12-alpha.20",
"description": "Budibase Public API SDK", "description": "Budibase Public API SDK",
"author": "Budibase", "author": "Budibase",
"license": "MPL-2.0", "license": "MPL-2.0",

12
packages/server/package.json
View File

@ -1,7 +1,7 @@
{ {
"name": "@budibase/server", "name": "@budibase/server",
"email": "hi@budibase.com", "email": "hi@budibase.com",
"version": "2.2.12-alpha.16", "version": "2.2.12-alpha.20",
"description": "Budibase Web Server", "description": "Budibase Web Server",
"main": "src/index.ts", "main": "src/index.ts",
"repository": { "repository": {
@ -43,11 +43,11 @@
"license": "GPL-3.0", "license": "GPL-3.0",
"dependencies": { "dependencies": {
"@apidevtools/swagger-parser": "10.0.3", "@apidevtools/swagger-parser": "10.0.3",
"@budibase/backend-core": "2.2.12-alpha.16", "@budibase/backend-core": "2.2.12-alpha.20",
"@budibase/client": "2.2.12-alpha.16", "@budibase/client": "2.2.12-alpha.20",
"@budibase/pro": "2.2.12-alpha.16", "@budibase/pro": "2.2.12-alpha.20",
"@budibase/string-templates": "2.2.12-alpha.16", "@budibase/string-templates": "2.2.12-alpha.20",
"@budibase/types": "2.2.12-alpha.16", "@budibase/types": "2.2.12-alpha.20",
"@bull-board/api": "3.7.0", "@bull-board/api": "3.7.0",
"@bull-board/koa": "3.9.4", "@bull-board/koa": "3.9.4",
"@elastic/elasticsearch": "7.10.0", "@elastic/elasticsearch": "7.10.0",

1
packages/server/scripts/dev/manage.js
View File

@ -29,6 +29,7 @@ async function init() {
ACCOUNT_PORTAL_URL: "http://localhost:10001", ACCOUNT_PORTAL_URL: "http://localhost:10001",
ACCOUNT_PORTAL_API_KEY: "budibase", ACCOUNT_PORTAL_API_KEY: "budibase",
JWT_SECRET: "testsecret", JWT_SECRET: "testsecret",
ENCRYPTION_KEY: "testsecret",
REDIS_PASSWORD: "budibase", REDIS_PASSWORD: "budibase",
MINIO_ACCESS_KEY: "budibase", MINIO_ACCESS_KEY: "budibase",
MINIO_SECRET_KEY: "budibase", MINIO_SECRET_KEY: "budibase",

68
packages/server/src/api/controllers/application.ts
View File

@ -116,42 +116,42 @@ async function createInstance(template: any, includeSampleData: boolean) {
const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
const baseAppId = generateAppID(tenantId) const baseAppId = generateAppID(tenantId)
const appId = generateDevAppID(baseAppId) const appId = generateDevAppID(baseAppId)
await context.updateAppId(appId) return await context.doInAppContext(appId, async () => {
const db = context.getAppDB()
await db.put({
_id: "_design/database",
// view collation information, read before writing any complex views:
// https://docs.couchdb.org/en/master/ddocs/views/collation.html#collation-specification
views: {},
})
const db = context.getAppDB() // NOTE: indexes need to be created before any tables/templates
await db.put({ // add view for linked rows
_id: "_design/database", await createLinkView()
// view collation information, read before writing any complex views: await createRoutingView()
// https://docs.couchdb.org/en/master/ddocs/views/collation.html#collation-specification await createAllSearchIndex()
views: {},
// replicate the template data to the instance DB
// this is currently very hard to test, downloading and importing template files
if (template && template.templateString) {
const { ok } = await db.load(stringToReadStream(template.templateString))
if (!ok) {
throw "Error loading database dump from memory."
}
} else if (template && template.useTemplate === "true") {
await sdk.backups.importApp(appId, db, template)
} else {
// create the users table
await db.put(USERS_TABLE_SCHEMA)
if (includeSampleData) {
// create ootb stock db
await addDefaultTables(db)
}
}
return { _id: appId }
}) })
// NOTE: indexes need to be created before any tables/templates
// add view for linked rows
await createLinkView()
await createRoutingView()
await createAllSearchIndex()
// replicate the template data to the instance DB
// this is currently very hard to test, downloading and importing template files
if (template && template.templateString) {
const { ok } = await db.load(stringToReadStream(template.templateString))
if (!ok) {
throw "Error loading database dump from memory."
}
} else if (template && template.useTemplate === "true") {
await sdk.backups.importApp(appId, db, template)
} else {
// create the users table
await db.put(USERS_TABLE_SCHEMA)
if (includeSampleData) {
// create ootb stock db
await addDefaultTables(db)
}
}
return { _id: appId }
} }
async function addDefaultTables(db: Database) { async function addDefaultTables(db: Database) {

4
packages/server/src/api/controllers/datasource.ts
View File

@ -15,6 +15,7 @@ import { db as dbCore, context, events } from "@budibase/backend-core"
import { BBContext, Datasource, Row } from "@budibase/types" import { BBContext, Datasource, Row } from "@budibase/types"
import sdk from "../../sdk" import sdk from "../../sdk"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import { enrich } from "../../sdk/app/datasources/datasources"
export async function fetch(ctx: BBContext) { export async function fetch(ctx: BBContext) {
// Get internal tables // Get internal tables
@ -315,8 +316,7 @@ function updateError(error: any, newError: any, tables: string[]) {
async function buildSchemaHelper(datasource: Datasource) { async function buildSchemaHelper(datasource: Datasource) {
const Connector = await getIntegration(datasource.source) const Connector = await getIntegration(datasource.source)
datasource = await sdk.datasources.enrichDatasourceWithValues(datasource) datasource = await sdk.datasources.enrich(datasource)
// Connect to the DB and build the schema // Connect to the DB and build the schema
const connector = new Connector(datasource.config) const connector = new Connector(datasource.config)
await connector.buildSchema(datasource._id, datasource.entities) await connector.buildSchema(datasource._id, datasource.entities)

75
packages/server/src/api/controllers/query/index.ts
View File

@ -8,6 +8,7 @@ import env from "../../../environment"
import { quotas } from "@budibase/pro" import { quotas } from "@budibase/pro"
import { events, context, utils, constants } from "@budibase/backend-core" import { events, context, utils, constants } from "@budibase/backend-core"
import sdk from "../../../sdk" import sdk from "../../../sdk"
import { QueryEvent } from "../../../threads/definitions"
const Runner = new Thread(ThreadType.QUERY, { const Runner = new Thread(ThreadType.QUERY, {
timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000, timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000,
@ -127,9 +128,9 @@ function getAuthConfig(ctx: any) {
} }
export async function preview(ctx: any) { export async function preview(ctx: any) {
const datasource = await sdk.datasources.get(ctx.request.body.datasourceId, { const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
withEnvVars: true, ctx.request.body.datasourceId
}) )
const query = ctx.request.body const query = ctx.request.body
// preview may not have a queryId as it hasn't been saved, but if it does // preview may not have a queryId as it hasn't been saved, but if it does
// this stops dynamic variables from calling the same query // this stops dynamic variables from calling the same query
@ -138,20 +139,22 @@ export async function preview(ctx: any) {
const authConfigCtx: any = getAuthConfig(ctx) const authConfigCtx: any = getAuthConfig(ctx)
try { try {
const runFn = () => const inputs: QueryEvent = {
Runner.run({ appId: ctx.appId,
appId: ctx.appId, datasource,
datasource, queryVerb,
queryVerb, fields,
fields, parameters,
parameters, transformer,
transformer, queryId,
queryId, // have to pass down to the thread runner - can't put into context now
ctx: { environmentVariables: envVars,
user: ctx.user, ctx: {
auth: { ...authConfigCtx }, user: ctx.user,
}, auth: { ...authConfigCtx },
}) },
}
const runFn = () => Runner.run(inputs)
const { rows, keys, info, extra } = await quotas.addQuery(runFn, { const { rows, keys, info, extra } = await quotas.addQuery(runFn, {
datasourceId: datasource._id, datasourceId: datasource._id,
@ -202,9 +205,9 @@ async function execute(
const db = context.getAppDB() const db = context.getAppDB()
const query = await db.get(ctx.params.queryId) const query = await db.get(ctx.params.queryId)
const datasource = await sdk.datasources.get(query.datasourceId, { const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
withEnvVars: true, query.datasourceId
}) )
let authConfigCtx: any = {} let authConfigCtx: any = {}
if (!opts.isAutomation) { if (!opts.isAutomation) {
@ -222,21 +225,23 @@ async function execute(
// call the relevant CRUD method on the integration class // call the relevant CRUD method on the integration class
try { try {
const runFn = () => const inputs: QueryEvent = {
Runner.run({ appId: ctx.appId,
appId: ctx.appId, datasource,
datasource, queryVerb: query.queryVerb,
queryVerb: query.queryVerb, fields: query.fields,
fields: query.fields, pagination: ctx.request.body.pagination,
pagination: ctx.request.body.pagination, parameters: enrichedParameters,
parameters: enrichedParameters, transformer: query.transformer,
transformer: query.transformer, queryId: ctx.params.queryId,
queryId: ctx.params.queryId, // have to pass down to the thread runner - can't put into context now
ctx: { environmentVariables: envVars,
user: ctx.user, ctx: {
auth: { ...authConfigCtx }, user: ctx.user,
}, auth: { ...authConfigCtx },
}) },
}
const runFn = () => Runner.run(inputs)
const { rows, pagination, extra } = await quotas.addQuery(runFn, { const { rows, pagination, extra } = await quotas.addQuery(runFn, {
datasourceId: datasource._id, datasourceId: datasource._id,

2
packages/server/src/api/controllers/static/index.ts
View File

@ -155,7 +155,7 @@ export const getSignedUploadURL = async function (ctx: any) {
let datasource let datasource
try { try {
const { datasourceId } = ctx.params const { datasourceId } = ctx.params
datasource = await sdk.datasources.get(datasourceId, { withEnvVars: true }) datasource = await sdk.datasources.get(datasourceId, { enriched: true })
if (!datasource) { if (!datasource) {
ctx.throw(400, "The specified datasource could not be found") ctx.throw(400, "The specified datasource could not be found")
} }

34
packages/server/src/api/controllers/user.ts
View File

@ -4,19 +4,21 @@ import { getGlobalUsers, getRawGlobalUser } from "../../utilities/global"
import { getFullUser } from "../../utilities/users" import { getFullUser } from "../../utilities/users"
import { import {
context, context,
constants,
roles as rolesCore, roles as rolesCore,
db as dbCore, db as dbCore,
} from "@budibase/backend-core" } from "@budibase/backend-core"
import { BBContext, User } from "@budibase/types" import { BBContext, Ctx, SyncUserRequest, User } from "@budibase/types"
import sdk from "../../sdk" import sdk from "../../sdk"
export async function syncUser(ctx: BBContext) { export async function syncUser(ctx: Ctx<SyncUserRequest>) {
let deleting = false, let deleting = false,
user: User | any user: User | any
const userId = ctx.params.id const userId = ctx.params.id
const previousUser = ctx.request.body?.previousUser
try { try {
user = await getRawGlobalUser(userId) user = (await getRawGlobalUser(userId)) as User
} catch (err: any) { } catch (err: any) {
if (err && err.status === 404) { if (err && err.status === 404) {
user = {} user = {}
@ -25,6 +27,11 @@ export async function syncUser(ctx: BBContext) {
throw err throw err
} }
} }
let previousApps = previousUser
? Object.keys(previousUser.roles).map(appId => appId)
: []
const roles = deleting ? {} : user.roles const roles = deleting ? {} : user.roles
// remove props which aren't useful to metadata // remove props which aren't useful to metadata
delete user.password delete user.password
@ -40,8 +47,9 @@ export async function syncUser(ctx: BBContext) {
.filter(entry => entry[1] !== rolesCore.BUILTIN_ROLE_IDS.PUBLIC) .filter(entry => entry[1] !== rolesCore.BUILTIN_ROLE_IDS.PUBLIC)
.map(([appId]) => appId) .map(([appId]) => appId)
} }
for (let prodAppId of prodAppIds) { for (let prodAppId of new Set([...prodAppIds, ...previousApps])) {
const roleId = roles[prodAppId] const roleId = roles[prodAppId]
const deleteFromApp = !roleId
const devAppId = dbCore.getDevelopmentAppID(prodAppId) const devAppId = dbCore.getDevelopmentAppID(prodAppId)
for (let appId of [prodAppId, devAppId]) { for (let appId of [prodAppId, devAppId]) {
if (!(await dbCore.dbExists(appId))) { if (!(await dbCore.dbExists(appId))) {
@ -54,24 +62,24 @@ export async function syncUser(ctx: BBContext) {
try { try {
metadata = await db.get(metadataId) metadata = await db.get(metadataId)
} catch (err) { } catch (err) {
if (deleting) { if (deleteFromApp) {
return return
} }
metadata = { metadata = {
tableId: InternalTables.USER_METADATA, tableId: InternalTables.USER_METADATA,
} }
} }
if (deleteFromApp) {
await db.remove(metadata)
return
}
// assign the roleId for the metadata doc // assign the roleId for the metadata doc
if (roleId) { if (roleId) {
metadata.roleId = roleId metadata.roleId = roleId
} }
let combined = !deleting let combined = sdk.users.combineMetadataAndUser(user, metadata)
? sdk.users.combineMetadataAndUser(user, metadata)
: {
...metadata,
status: constants.UserStatus.INACTIVE,
metadata: rolesCore.BUILTIN_ROLE_IDS.PUBLIC,
}
// if its null then there was no updates required // if its null then there was no updates required
if (combined) { if (combined) {
await db.put(combined) await db.put(combined)

94
packages/server/src/api/controllers/webhook.ts
View File

@ -39,60 +39,62 @@ export async function destroy(ctx: BBContext) {
} }
export async function buildSchema(ctx: BBContext) { export async function buildSchema(ctx: BBContext) {
await context.updateAppId(ctx.params.instance) await context.doInAppContext(ctx.params.instance, async () => {
const db = context.getAppDB() const db = context.getAppDB()
const webhook = (await db.get(ctx.params.id)) as Webhook const webhook = (await db.get(ctx.params.id)) as Webhook
webhook.bodySchema = toJsonSchema(ctx.request.body) webhook.bodySchema = toJsonSchema(ctx.request.body)
// update the automation outputs // update the automation outputs
if (webhook.action.type === WebhookActionType.AUTOMATION) { if (webhook.action.type === WebhookActionType.AUTOMATION) {
let automation = (await db.get(webhook.action.target)) as Automation let automation = (await db.get(webhook.action.target)) as Automation
const autoOutputs = automation.definition.trigger.schema.outputs const autoOutputs = automation.definition.trigger.schema.outputs
let properties = webhook.bodySchema.properties let properties = webhook.bodySchema.properties
// reset webhook outputs // reset webhook outputs
autoOutputs.properties = { autoOutputs.properties = {
body: autoOutputs.properties.body, body: autoOutputs.properties.body,
}
for (let prop of Object.keys(properties)) {
autoOutputs.properties[prop] = {
type: properties[prop].type,
description: AUTOMATION_DESCRIPTION,
} }
for (let prop of Object.keys(properties)) {
autoOutputs.properties[prop] = {
type: properties[prop].type,
description: AUTOMATION_DESCRIPTION,
}
}
await db.put(automation)
} }
await db.put(automation) ctx.body = await db.put(webhook)
} })
ctx.body = await db.put(webhook)
} }
export async function trigger(ctx: BBContext) { export async function trigger(ctx: BBContext) {
const prodAppId = dbCore.getProdAppID(ctx.params.instance) const prodAppId = dbCore.getProdAppID(ctx.params.instance)
await context.updateAppId(prodAppId) await context.doInAppContext(prodAppId, async () => {
try { try {
const db = context.getAppDB() const db = context.getAppDB()
const webhook = (await db.get(ctx.params.id)) as Webhook const webhook = (await db.get(ctx.params.id)) as Webhook
// validate against the schema // validate against the schema
if (webhook.bodySchema) { if (webhook.bodySchema) {
validate(ctx.request.body, webhook.bodySchema) validate(ctx.request.body, webhook.bodySchema)
} }
const target = await db.get(webhook.action.target) const target = await db.get(webhook.action.target)
if (webhook.action.type === WebhookActionType.AUTOMATION) { if (webhook.action.type === WebhookActionType.AUTOMATION) {
// trigger with both the pure request and then expand it // trigger with both the pure request and then expand it
// incase the user has produced a schema to bind to // incase the user has produced a schema to bind to
await triggers.externalTrigger(target, { await triggers.externalTrigger(target, {
body: ctx.request.body, body: ctx.request.body,
...ctx.request.body, ...ctx.request.body,
appId: prodAppId, appId: prodAppId,
}) })
} }
ctx.status = 200
ctx.body = {
message: "Webhook trigger fired successfully",
}
} catch (err: any) {
if (err.status === 404) {
ctx.status = 200 ctx.status = 200
ctx.body = { ctx.body = {
message: "Application not deployed yet.", message: "Webhook trigger fired successfully",
}
} catch (err: any) {
if (err.status === 404) {
ctx.status = 200
ctx.body = {
message: "Application not deployed yet.",
}
} }
} }
} })
} }

27
packages/server/src/api/routes/tests/user.spec.js
View File

@ -171,9 +171,28 @@ describe("/users", () => {
.expect("Content-Type", /json/) .expect("Content-Type", /json/)
expect(res.body.message).toEqual('User synced.') expect(res.body.message).toEqual('User synced.')
}) })
it("should sync the user when a previous user is specified", async () => {
const app1 = await config.createApp('App 1')
const app2 = await config.createApp('App 2')
let user = await config.createUser(
undefined,
undefined,
undefined,
undefined,
false,
true,
{ [app1.appId]: 'ADMIN' })
let res = await request
.post(`/api/users/metadata/sync/${user._id}`)
.set(config.defaultHeaders())
.send({ previousUser: { ...user, roles: { ...user.roles, [app2.appId]: 'BASIC' } } })
.expect(200)
.expect("Content-Type", /json/)
expect(res.body.message).toEqual('User synced.')
})
}) })
}) })

1
packages/server/src/definitions/automations.ts
View File

@ -24,6 +24,7 @@ export interface TriggerOutput {
export interface AutomationContext extends AutomationResults { export interface AutomationContext extends AutomationResults {
steps: any[] steps: any[]
env?: Record<string, string>
trigger: any trigger: any
} }

2
packages/server/src/integrations/base/query.ts
View File

@ -6,7 +6,7 @@ export async function makeExternalQuery(
datasource: Datasource, datasource: Datasource,
json: QueryJson json: QueryJson
) { ) {
datasource = await sdk.datasources.enrichDatasourceWithValues(datasource) datasource = await sdk.datasources.enrich(datasource)
const Integration = await getIntegration(datasource.source) const Integration = await getIntegration(datasource.source)
// query is the opinionated function // query is the opinionated function
if (Integration.prototype.query) { if (Integration.prototype.query) {

55
packages/server/src/integrations/queries/sql.ts
View File

@ -1,55 +1,10 @@
import { findHBSBlocks, processStringSync } from "@budibase/string-templates" import { findHBSBlocks } from "@budibase/string-templates"
import { DatasourcePlus } from "@budibase/types" import { DatasourcePlus } from "@budibase/types"
import sdk from "../../sdk"
const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g") const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")
export function enrichQueryFields( export async function interpolateSQL(
fields: { [key: string]: any },
parameters = {}
) {
const enrichedQuery: { [key: string]: any } = Array.isArray(fields) ? [] : {}
if (!fields || !parameters) {
return enrichedQuery
}
// enrich the fields with dynamic parameters
for (let key of Object.keys(fields)) {
if (fields[key] == null) {
continue
}
if (typeof fields[key] === "object") {
// enrich nested fields object
enrichedQuery[key] = enrichQueryFields(fields[key], parameters)
} else if (typeof fields[key] === "string") {
// enrich string value as normal
enrichedQuery[key] = processStringSync(fields[key], parameters, {
noEscaping: true,
noHelpers: true,
escapeNewlines: true,
})
} else {
enrichedQuery[key] = fields[key]
}
}
if (
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
) {
try {
enrichedQuery.json = JSON.parse(
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
)
} catch (err) {
// no json found, ignore
}
delete enrichedQuery.customData
}
return enrichedQuery
}
export function interpolateSQL(
fields: { [key: string]: any }, fields: { [key: string]: any },
parameters: { [key: string]: any }, parameters: { [key: string]: any },
integration: DatasourcePlus integration: DatasourcePlus
@ -90,7 +45,7 @@ export function interpolateSQL(
else if (listRegexMatch) { else if (listRegexMatch) {
arrays.push(binding) arrays.push(binding)
// determine the length of the array // determine the length of the array
const value = enrichQueryFields([binding], parameters)[0] const value = (await sdk.queries.enrichContext([binding], parameters))[0]
.split(",") .split(",")
.map((val: string) => val.trim()) .map((val: string) => val.trim())
// build a string like ($1, $2, $3) // build a string like ($1, $2, $3)
@ -109,7 +64,7 @@ export function interpolateSQL(
} }
// replicate the knex structure // replicate the knex structure
fields.sql = sql fields.sql = sql
fields.bindings = enrichQueryFields(variables, parameters) fields.bindings = await sdk.queries.enrichContext(variables, parameters)
// check for arrays in the data // check for arrays in the data
let updated: string[] = [] let updated: string[] = []
for (let i = 0; i < variables.length; i++) { for (let i = 0; i < variables.length; i++) {

5
packages/server/src/middleware/currentapp.ts
View File

@ -25,6 +25,7 @@ export default async (ctx: BBContext, next: any) => {
if (!appCookie && !requestAppId) { if (!appCookie && !requestAppId) {
return next() return next()
} }
// check the app exists referenced in cookie // check the app exists referenced in cookie
if (appCookie) { if (appCookie) {
const appId = appCookie.appId const appId = appCookie.appId
@ -51,7 +52,7 @@ export default async (ctx: BBContext, next: any) => {
let appId: string | undefined, let appId: string | undefined,
roleId = roles.BUILTIN_ROLE_IDS.PUBLIC roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
if (!ctx.user) { if (!ctx.user?._id) {
// not logged in, try to set a cookie for public apps // not logged in, try to set a cookie for public apps
appId = requestAppId appId = requestAppId
} else if (requestAppId != null) { } else if (requestAppId != null) {
@ -96,7 +97,7 @@ export default async (ctx: BBContext, next: any) => {
// need to judge this only based on the request app ID, // need to judge this only based on the request app ID,
if ( if (
env.MULTI_TENANCY && env.MULTI_TENANCY &&
ctx.user && ctx.user?._id &&
requestAppId && requestAppId &&
!tenancy.isUserInAppTenant(requestAppId, ctx.user) !tenancy.isUserInAppTenant(requestAppId, ctx.user)
) { ) {

38
packages/server/src/sdk/app/datasources/datasources.ts
View File

@ -1,29 +1,39 @@
import { environmentVariables } from "@budibase/pro" import { context } from "@budibase/backend-core"
import { context, db as dbCore } from "@budibase/backend-core"
import { processObjectSync } from "@budibase/string-templates" import { processObjectSync } from "@budibase/string-templates"
import { AppEnvironment, Datasource } from "@budibase/types" import { Datasource } from "@budibase/types"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import { getEnvironmentVariables } from "../../utils"
export async function enrichDatasourceWithValues(datasource: Datasource) { async function enrichDatasourceWithValues(datasource: Datasource) {
const appId = context.getAppId()
const appEnv = dbCore.isDevAppID(appId)
? AppEnvironment.DEVELOPMENT
: AppEnvironment.PRODUCTION
const cloned = cloneDeep(datasource) const cloned = cloneDeep(datasource)
const envVars = await environmentVariables.fetchValues(appEnv) const env = await getEnvironmentVariables()
const processed = processObjectSync(cloned, { env: envVars }) const processed = processObjectSync(cloned, env)
return processed as Datasource return {
datasource: processed as Datasource,
envVars: env as Record<string, string>,
}
}
export async function enrich(datasource: Datasource) {
const { datasource: response } = await enrichDatasourceWithValues(datasource)
return response
} }
export async function get( export async function get(
datasourceId: string, datasourceId: string,
opts?: { withEnvVars: boolean } opts?: { enriched: boolean }
): Promise<Datasource> { ): Promise<Datasource> {
const appDb = context.getAppDB() const appDb = context.getAppDB()
const datasource = await appDb.get(datasourceId) const datasource = await appDb.get(datasourceId)
if (opts?.withEnvVars) { if (opts?.enriched) {
return await enrichDatasourceWithValues(datasource) return (await enrichDatasourceWithValues(datasource)).datasource
} else { } else {
return datasource return datasource
} }
} }
export async function getWithEnvVars(datasourceId: string) {
const appDb = context.getAppDB()
const datasource = await appDb.get(datasourceId)
return enrichDatasourceWithValues(datasource)
}

5
packages/server/src/sdk/app/queries/index.ts Normal file
View File

@ -0,0 +1,5 @@
import * as queries from "./queries"
export default {
...queries,
}

50
packages/server/src/sdk/app/queries/queries.ts Normal file
View File

@ -0,0 +1,50 @@
import { getEnvironmentVariables } from "../../utils"
import { processStringSync } from "@budibase/string-templates"
export async function enrichContext(
fields: Record<string, any>,
inputs = {}
): Promise<Record<string, any>> {
const enrichedQuery: Record<string, any> = Array.isArray(fields) ? [] : {}
if (!fields || !inputs) {
return enrichedQuery
}
const env = await getEnvironmentVariables()
const parameters = { ...inputs, env }
// enrich the fields with dynamic parameters
for (let key of Object.keys(fields)) {
if (fields[key] == null) {
continue
}
if (typeof fields[key] === "object") {
// enrich nested fields object
enrichedQuery[key] = await enrichContext(fields[key], parameters)
} else if (typeof fields[key] === "string") {
// enrich string value as normal
enrichedQuery[key] = processStringSync(fields[key], parameters, {
noEscaping: true,
noHelpers: true,
escapeNewlines: true,
})
} else {
enrichedQuery[key] = fields[key]
}
}
if (
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
) {
try {
enrichedQuery.json = JSON.parse(
enrichedQuery.json ||
enrichedQuery.customData ||
enrichedQuery.requestBody
)
} catch (err) {
// no json found, ignore
}
delete enrichedQuery.customData
}
return enrichedQuery
}

2
packages/server/src/sdk/app/tables/index.ts
View File

@ -28,7 +28,7 @@ async function getAllExternalTables(
datasourceId: any datasourceId: any
): Promise<Record<string, Table>> { ): Promise<Record<string, Table>> {
const db = context.getAppDB() const db = context.getAppDB()
const datasource = await datasources.get(datasourceId, { withEnvVars: true }) const datasource = await datasources.get(datasourceId, { enriched: true })
if (!datasource || !datasource.entities) { if (!datasource || !datasource.entities) {
throw "Datasource is not configured fully." throw "Datasource is not configured fully."
} }

2
packages/server/src/sdk/index.ts
View File

@ -3,6 +3,7 @@ import { default as tables } from "./app/tables"
import { default as automations } from "./app/automations" import { default as automations } from "./app/automations"
import { default as applications } from "./app/applications" import { default as applications } from "./app/applications"
import { default as datasources } from "./app/datasources" import { default as datasources } from "./app/datasources"
import { default as queries } from "./app/queries"
import { default as rows } from "./app/rows" import { default as rows } from "./app/rows"
import { default as users } from "./users" import { default as users } from "./users"
@ -14,6 +15,7 @@ const sdk = {
rows, rows,
users, users,
datasources, datasources,
queries,
} }
// default export for TS // default export for TS

16
packages/server/src/sdk/utils/index.ts Normal file
View File

@ -0,0 +1,16 @@
import { environmentVariables } from "@budibase/pro"
import { context, db as dbCore } from "@budibase/backend-core"
import { AppEnvironment } from "@budibase/types"
export async function getEnvironmentVariables() {
let envVars = context.getEnvironmentVariables()
if (!envVars) {
const appId = context.getAppId()
const appEnv = dbCore.isDevAppID(appId)
? AppEnvironment.DEVELOPMENT
: AppEnvironment.PRODUCTION
envVars = await environmentVariables.fetchValues(appEnv)
}
return envVars
}

27
packages/server/src/tests/utilities/TestConfiguration.ts
View File

@ -364,20 +364,23 @@ class TestConfiguration {
// create dev app // create dev app
// clear any old app // clear any old app
this.appId = null this.appId = null
// @ts-ignore await context.doInAppContext(null, async () => {
await context.updateAppId(null) this.app = await this._req(
this.app = await this._req({ name: appName }, null, controllers.app.create) { name: appName },
this.appId = this.app.appId null,
// @ts-ignore controllers.app.create
await context.updateAppId(this.appId) )
this.appId = this.app.appId
})
return await context.doInAppContext(this.appId, async () => {
// create production app
this.prodApp = await this.publish()
// create production app this.allApps.push(this.prodApp)
this.prodApp = await this.publish() this.allApps.push(this.app)
this.allApps.push(this.prodApp) return this.app
this.allApps.push(this.app) })
return this.app
} }
async publish() { async publish() {

12
packages/server/src/threads/automation.ts
View File

@ -16,7 +16,6 @@ import { storeLog } from "../automations/logging"
import { Automation, AutomationStep, AutomationStatus } from "@budibase/types" import { Automation, AutomationStep, AutomationStatus } from "@budibase/types"
import { import {
LoopStep, LoopStep,
LoopStepType,
LoopInput, LoopInput,
TriggerOutput, TriggerOutput,
AutomationContext, AutomationContext,
@ -26,6 +25,7 @@ import { WorkerCallback } from "./definitions"
import { context, logging } from "@budibase/backend-core" import { context, logging } from "@budibase/backend-core"
import { processObject } from "@budibase/string-templates" import { processObject } from "@budibase/string-templates"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import * as sdkUtils from "../sdk/utils"
import env from "../environment" import env from "../environment"
const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
@ -221,6 +221,8 @@ class Orchestrator {
} }
async execute() { async execute() {
// this will retrieve from context created at start of thread
this._context.env = await sdkUtils.getEnvironmentVariables()
let automation = this._automation let automation = this._automation
let stopped = false let stopped = false
let loopStep: AutomationStep | undefined = undefined let loopStep: AutomationStep | undefined = undefined
@ -474,7 +476,11 @@ export const removeStalled = async (job: Job) => {
throw new Error("Unable to execute, event doesn't contain app ID.") throw new Error("Unable to execute, event doesn't contain app ID.")
} }
await context.doInAppContext(appId, async () => { await context.doInAppContext(appId, async () => {
const automationOrchestrator = new Orchestrator(job) const envVars = await sdkUtils.getEnvironmentVariables()
await automationOrchestrator.stopCron("stalled") // put into automation thread for whole context
await context.doInEnvironmentContext(envVars, async () => {
const automationOrchestrator = new Orchestrator(job)
await automationOrchestrator.stopCron("stalled")
})
}) })
} }

3
packages/server/src/threads/definitions.ts
View File

@ -1,3 +1,5 @@
import { EnvironmentVariablesDecrypted } from "@budibase/types"
export type WorkerCallback = (error: any, response?: any) => void export type WorkerCallback = (error: any, response?: any) => void
export interface QueryEvent { export interface QueryEvent {
@ -9,6 +11,7 @@ export interface QueryEvent {
pagination?: any pagination?: any
transformer: any transformer: any
queryId: string queryId: string
environmentVariables?: Record<string, string>
ctx?: any ctx?: any
} }

35
packages/server/src/threads/query.ts
View File

@ -10,7 +10,7 @@ import sdk from "../sdk"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import { isSQL } from "../integrations/utils" import { isSQL } from "../integrations/utils"
import { enrichQueryFields, interpolateSQL } from "../integrations/queries/sql" import { interpolateSQL } from "../integrations/queries/sql"
class QueryRunner { class QueryRunner {
datasource: any datasource: any
@ -60,10 +60,11 @@ class QueryRunner {
} }
if (datasourceClone.config.authConfigs) { if (datasourceClone.config.authConfigs) {
datasourceClone.config.authConfigs = const updatedConfigs = []
datasourceClone.config.authConfigs.map((config: any) => { for (let config of datasourceClone.config.authConfigs) {
return enrichQueryFields(config, this.ctx) updatedConfigs.push(await sdk.queries.enrichContext(config, this.ctx))
}) }
datasourceClone.config.authConfigs = updatedConfigs
} }
const integration = new Integration(datasourceClone.config) const integration = new Integration(datasourceClone.config)
@ -73,12 +74,15 @@ class QueryRunner {
// Enrich the parameters with the addition context items. // Enrich the parameters with the addition context items.
// 'user' is now a reserved variable key in mapping parameters // 'user' is now a reserved variable key in mapping parameters
const enrichedParameters = enrichQueryFields(parameters, this.ctx) const enrichedParameters = await sdk.queries.enrichContext(
parameters,
this.ctx
)
const enrichedContext = { ...enrichedParameters, ...this.ctx } const enrichedContext = { ...enrichedParameters, ...this.ctx }
// Parse global headers // Parse global headers
if (datasourceClone.config.defaultHeaders) { if (datasourceClone.config.defaultHeaders) {
datasourceClone.config.defaultHeaders = enrichQueryFields( datasourceClone.config.defaultHeaders = await sdk.queries.enrichContext(
datasourceClone.config.defaultHeaders, datasourceClone.config.defaultHeaders,
enrichedContext enrichedContext
) )
@ -87,9 +91,9 @@ class QueryRunner {
let query let query
// handle SQL injections by interpolating the variables // handle SQL injections by interpolating the variables
if (isSQL(datasourceClone)) { if (isSQL(datasourceClone)) {
query = interpolateSQL(fieldsClone, enrichedParameters, integration) query = await interpolateSQL(fieldsClone, enrichedParameters, integration)
} else { } else {
query = enrichQueryFields(fieldsClone, enrichedContext) query = await sdk.queries.enrichContext(fieldsClone, enrichedContext)
} }
// Add pagination values for REST queries // Add pagination values for REST queries
@ -165,7 +169,7 @@ class QueryRunner {
const db = context.getAppDB() const db = context.getAppDB()
const query = await db.get(queryId) const query = await db.get(queryId)
const datasource = await sdk.datasources.get(query.datasourceId, { const datasource = await sdk.datasources.get(query.datasourceId, {
withEnvVars: true, enriched: true,
}) })
return new QueryRunner( return new QueryRunner(
{ {
@ -280,7 +284,7 @@ class QueryRunner {
} }
export function execute(input: QueryEvent, callback: WorkerCallback) { export function execute(input: QueryEvent, callback: WorkerCallback) {
context.doInAppContext(input.appId!, async () => { const run = async () => {
const Runner = new QueryRunner(input) const Runner = new QueryRunner(input)
try { try {
const response = await Runner.execute() const response = await Runner.execute()
@ -288,5 +292,14 @@ export function execute(input: QueryEvent, callback: WorkerCallback) {
} catch (err) { } catch (err) {
callback(err) callback(err)
} }
}
context.doInAppContext(input.appId!, async () => {
if (input.environmentVariables) {
return context.doInEnvironmentContext(input.environmentVariables, () => {
return run()
})
} else {
return run()
}
}) })
} }

30
packages/server/yarn.lock
View File

@ -1273,13 +1273,13 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.2.12-alpha.16": "@budibase/backend-core@2.2.12-alpha.20":
version "2.2.12-alpha.16" version "2.2.12-alpha.20"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.16.tgz#9ebfa7308fc97b34d6a076e4300fbcf996160d66" resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.20.tgz#aac6b995d0c53c599eff1f069bbde6426f383a36"
integrity sha512-rHMryIOb71U7W5jZtn39vuBI7xSZ6XA4l6P7lc2bBT1lI10G/zQRoQWjsWaUWo+RVBQ5zki3Ok05tFS9Yx/7fA== integrity sha512-ta9gEa3fklCQHUybJk8wxQM0VawV+IN7P4yVH4kZU3mr9nAS42rhvb+xoqx4vB8HIa6312uj0UgUaYWacK15Rw==
dependencies: dependencies:
"@budibase/nano" "10.1.1" "@budibase/nano" "10.1.1"
"@budibase/types" "2.2.12-alpha.16" "@budibase/types" "2.2.12-alpha.20"
"@shopify/jest-koa-mocks" "5.0.1" "@shopify/jest-koa-mocks" "5.0.1"
"@techpass/passport-openidconnect" "0.3.2" "@techpass/passport-openidconnect" "0.3.2"
aws-cloudfront-sign "2.2.0" aws-cloudfront-sign "2.2.0"
@ -1374,13 +1374,13 @@
qs "^6.11.0" qs "^6.11.0"
tough-cookie "^4.1.2" tough-cookie "^4.1.2"
"@budibase/pro@2.2.12-alpha.16": "@budibase/pro@2.2.12-alpha.20":
version "2.2.12-alpha.16" version "2.2.12-alpha.20"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.16.tgz#db5a345b072e725765cd01edcad4a930ae875eba" resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.20.tgz#3eb45b7ddef7e5c2190851a8f021b6638d01ef90"
integrity sha512-GBXdOQMIbxU0TGgGQ4+npNGtuFvanNVFrZBqwB7+3x6rIku313WkbgJJji5uemtU6B8XFh/QqS6AA0R0PS2Kmg== integrity sha512-j169PLqdCxR6oPR6nevdVrwHDTH1SilKhtA7GSrP3ANPGTK3G9fxVBvx+KH4IaGikTSZnCaGUP6p376EcOMMCg==
dependencies: dependencies:
"@budibase/backend-core" "2.2.12-alpha.16" "@budibase/backend-core" "2.2.12-alpha.20"
"@budibase/types" "2.2.12-alpha.16" "@budibase/types" "2.2.12-alpha.20"
"@koa/router" "8.0.8" "@koa/router" "8.0.8"
bull "4.10.1" bull "4.10.1"
joi "17.6.0" joi "17.6.0"
@ -1405,10 +1405,10 @@
svelte-apexcharts "^1.0.2" svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0" svelte-flatpickr "^3.1.0"
"@budibase/types@2.2.12-alpha.16": "@budibase/types@2.2.12-alpha.20":
version "2.2.12-alpha.16" version "2.2.12-alpha.20"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.16.tgz#6fb42d4be88fbd8054a0a3264cf9c4b4a7248893" resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.20.tgz#d708eb74df555a7649873247662b6d6be1054bfe"
integrity sha512-pXn/r3tA0A30f2dJVJfzldMGXAEhpObBfqbONn8AStiD6Qm8Hu9H6aFaCPqS8DDaWBuwY/tMqSry2E0saRaSwg== integrity sha512-HQjCgYjgd8NLkykMy5oQA48VS03Py/GbDqnIuUT70fhmX8mgORctvvK7AgNsSLTrv2uRFuvZOcL8VE8MyJBhKA==
"@bull-board/api@3.7.0": "@bull-board/api@3.7.0":
version "3.7.0" version "3.7.0"

2
packages/string-templates/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@budibase/string-templates", "name": "@budibase/string-templates",
"version": "2.2.12-alpha.16", "version": "2.2.12-alpha.20",
"description": "Handlebars wrapper for Budibase templating.", "description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs", "main": "src/index.cjs",
"module": "dist/bundle.mjs", "module": "dist/bundle.mjs",

2
packages/types/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@budibase/types", "name": "@budibase/types",
"version": "2.2.12-alpha.16", "version": "2.2.12-alpha.20",
"description": "Budibase types", "description": "Budibase types",
"main": "dist/index.js", "main": "dist/index.js",
"types": "dist/index.d.ts", "types": "dist/index.d.ts",

4
packages/types/src/api/web/user.ts
View File

@ -57,3 +57,7 @@ export interface CreateAdminUserRequest {
password: string password: string
tenantId: string tenantId: string
} }
export interface SyncUserRequest {
previousUser?: User
}

4
packages/types/src/documents/global/environmentVariables.ts
View File

@ -14,3 +14,7 @@ export type EnvironmentVariablesDecrypted = Record<
string, string,
EnvironmentVariableValue EnvironmentVariableValue
> >
export interface EnvironmentVariablesDocDecrypted extends Document {
variables: EnvironmentVariablesDecrypted
}

4
packages/types/src/documents/global/user.ts
View File

@ -69,3 +69,7 @@ export interface AdminUser extends User {
global: boolean global: boolean
} }
} }
export function isUser(user: object): user is User {
return !!(user as User).roles
}

2
packages/types/src/sdk/koa.ts
View File

@ -41,7 +41,7 @@ export interface UserCtx<RequestBody = any, ResponseBody = any>
} }
/** /**
* Deprecated: Use UserCtx / Ctx appropriately * @deprecated: Use UserCtx / Ctx appropriately
* Authenticated context. * Authenticated context.
*/ */
export interface BBContext extends Ctx { export interface BBContext extends Ctx {

10
packages/worker/package.json
View File

@ -1,7 +1,7 @@
{ {
"name": "@budibase/worker", "name": "@budibase/worker",
"email": "hi@budibase.com", "email": "hi@budibase.com",
"version": "2.2.12-alpha.16", "version": "2.2.12-alpha.20",
"description": "Budibase background service", "description": "Budibase background service",
"main": "src/index.ts", "main": "src/index.ts",
"repository": { "repository": {
@ -36,10 +36,10 @@
"author": "Budibase", "author": "Budibase",
"license": "GPL-3.0", "license": "GPL-3.0",
"dependencies": { "dependencies": {
"@budibase/backend-core": "2.2.12-alpha.16", "@budibase/backend-core": "2.2.12-alpha.20",
"@budibase/pro": "2.2.12-alpha.16", "@budibase/pro": "2.2.12-alpha.20",
"@budibase/string-templates": "2.2.12-alpha.16", "@budibase/string-templates": "2.2.12-alpha.20",
"@budibase/types": "2.2.12-alpha.16", "@budibase/types": "2.2.12-alpha.20",
"@koa/router": "8.0.8", "@koa/router": "8.0.8",
"@sentry/node": "6.17.7", "@sentry/node": "6.17.7",
"@techpass/passport-openidconnect": "0.3.2", "@techpass/passport-openidconnect": "0.3.2",

8
packages/worker/src/sdk/users/users.ts
View File

@ -31,6 +31,7 @@ import {
SearchUsersRequest, SearchUsersRequest,
User, User,
ThirdPartyUser, ThirdPartyUser,
isUser,
} from "@budibase/types" } from "@budibase/types"
import { sendEmail } from "../../utilities/email" import { sendEmail } from "../../utilities/email"
import { EmailTemplatePurpose } from "../../constants" import { EmailTemplatePurpose } from "../../constants"
@ -265,8 +266,9 @@ export const save = async (
await eventHelpers.handleSaveEvents(builtUser, dbUser) await eventHelpers.handleSaveEvents(builtUser, dbUser)
await addTenant(tenantId, _id, email) await addTenant(tenantId, _id, email)
await cache.user.invalidateUser(response.id) await cache.user.invalidateUser(response.id)
// let server know to sync user // let server know to sync user
await apps.syncUserInApps(_id) await apps.syncUserInApps(_id, dbUser)
await Promise.all(groupPromises) await Promise.all(groupPromises)
@ -572,7 +574,7 @@ export const destroy = async (id: string, currentUser: any) => {
await cache.user.invalidateUser(userId) await cache.user.invalidateUser(userId)
await sessions.invalidateSessions(userId, { reason: "deletion" }) await sessions.invalidateSessions(userId, { reason: "deletion" })
// let server know to sync user // let server know to sync user
await apps.syncUserInApps(userId) await apps.syncUserInApps(userId, dbUser)
} }
const bulkDeleteProcessing = async (dbUser: User) => { const bulkDeleteProcessing = async (dbUser: User) => {
@ -582,7 +584,7 @@ const bulkDeleteProcessing = async (dbUser: User) => {
await cache.user.invalidateUser(userId) await cache.user.invalidateUser(userId)
await sessions.invalidateSessions(userId, { reason: "bulk-deletion" }) await sessions.invalidateSessions(userId, { reason: "bulk-deletion" })
// let server know to sync user // let server know to sync user
await apps.syncUserInApps(userId) await apps.syncUserInApps(userId, dbUser)
} }
export const invite = async ( export const invite = async (

9
packages/worker/src/utilities/appService.ts
View File

@ -2,6 +2,7 @@ import fetch from "node-fetch"
import { constants, tenancy, logging } from "@budibase/backend-core" import { constants, tenancy, logging } from "@budibase/backend-core"
import { checkSlashesInUrl } from "../utilities" import { checkSlashesInUrl } from "../utilities"
import env from "../environment" import env from "../environment"
import { SyncUserRequest, User } from "@budibase/types"
async function makeAppRequest(url: string, method: string, body: any) { async function makeAppRequest(url: string, method: string, body: any) {
if (env.isTest()) { if (env.isTest()) {
@ -24,11 +25,15 @@ async function makeAppRequest(url: string, method: string, body: any) {
return fetch(checkSlashesInUrl(env.APPS_URL + url), request) return fetch(checkSlashesInUrl(env.APPS_URL + url), request)
} }
export async function syncUserInApps(userId: string) { export async function syncUserInApps(userId: string, previousUser?: User) {
const body: SyncUserRequest = {
previousUser,
}
const response = await makeAppRequest( const response = await makeAppRequest(
`/api/users/metadata/sync/${userId}`, `/api/users/metadata/sync/${userId}`,
"POST", "POST",
{} body
) )
if (response && response.status !== 200) { if (response && response.status !== 200) {
throw "Unable to sync user." throw "Unable to sync user."

30
packages/worker/yarn.lock
View File

@ -470,13 +470,13 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.2.12-alpha.16": "@budibase/backend-core@2.2.12-alpha.20":
version "2.2.12-alpha.16" version "2.2.12-alpha.20"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.16.tgz#9ebfa7308fc97b34d6a076e4300fbcf996160d66" resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.20.tgz#aac6b995d0c53c599eff1f069bbde6426f383a36"
integrity sha512-rHMryIOb71U7W5jZtn39vuBI7xSZ6XA4l6P7lc2bBT1lI10G/zQRoQWjsWaUWo+RVBQ5zki3Ok05tFS9Yx/7fA== integrity sha512-ta9gEa3fklCQHUybJk8wxQM0VawV+IN7P4yVH4kZU3mr9nAS42rhvb+xoqx4vB8HIa6312uj0UgUaYWacK15Rw==
dependencies: dependencies:
"@budibase/nano" "10.1.1" "@budibase/nano" "10.1.1"
"@budibase/types" "2.2.12-alpha.16" "@budibase/types" "2.2.12-alpha.20"
"@shopify/jest-koa-mocks" "5.0.1" "@shopify/jest-koa-mocks" "5.0.1"
"@techpass/passport-openidconnect" "0.3.2" "@techpass/passport-openidconnect" "0.3.2"
aws-cloudfront-sign "2.2.0" aws-cloudfront-sign "2.2.0"
@ -521,23 +521,23 @@
qs "^6.11.0" qs "^6.11.0"
tough-cookie "^4.1.2" tough-cookie "^4.1.2"
"@budibase/pro@2.2.12-alpha.16": "@budibase/pro@2.2.12-alpha.20":
version "2.2.12-alpha.16" version "2.2.12-alpha.20"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.16.tgz#db5a345b072e725765cd01edcad4a930ae875eba" resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.20.tgz#3eb45b7ddef7e5c2190851a8f021b6638d01ef90"
integrity sha512-GBXdOQMIbxU0TGgGQ4+npNGtuFvanNVFrZBqwB7+3x6rIku313WkbgJJji5uemtU6B8XFh/QqS6AA0R0PS2Kmg== integrity sha512-j169PLqdCxR6oPR6nevdVrwHDTH1SilKhtA7GSrP3ANPGTK3G9fxVBvx+KH4IaGikTSZnCaGUP6p376EcOMMCg==
dependencies: dependencies:
"@budibase/backend-core" "2.2.12-alpha.16" "@budibase/backend-core" "2.2.12-alpha.20"
"@budibase/types" "2.2.12-alpha.16" "@budibase/types" "2.2.12-alpha.20"
"@koa/router" "8.0.8" "@koa/router" "8.0.8"
bull "4.10.1" bull "4.10.1"
joi "17.6.0" joi "17.6.0"
jsonwebtoken "8.5.1" jsonwebtoken "8.5.1"
node-fetch "^2.6.1" node-fetch "^2.6.1"
"@budibase/types@2.2.12-alpha.16": "@budibase/types@2.2.12-alpha.20":
version "2.2.12-alpha.16" version "2.2.12-alpha.20"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.16.tgz#6fb42d4be88fbd8054a0a3264cf9c4b4a7248893" resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.20.tgz#d708eb74df555a7649873247662b6d6be1054bfe"
integrity sha512-pXn/r3tA0A30f2dJVJfzldMGXAEhpObBfqbONn8AStiD6Qm8Hu9H6aFaCPqS8DDaWBuwY/tMqSry2E0saRaSwg== integrity sha512-HQjCgYjgd8NLkykMy5oQA48VS03Py/GbDqnIuUT70fhmX8mgORctvvK7AgNsSLTrv2uRFuvZOcL8VE8MyJBhKA==
"@cspotcode/source-map-support@^0.8.0": "@cspotcode/source-map-support@^0.8.0":
version "0.8.1" version "0.8.1"