diff --git a/lerna.json b/lerna.json index 86de1e4387..2bded7151f 100644 --- a/lerna.json +++ b/lerna.json @@ -16,7 +16,6 @@ "packages/worker", "packages/pro/packages/pro" ], - "useWorkspaces": true, "command": { "publish": { "ignoreChanges": [ diff --git a/package.json b/package.json index 56f015f8c0..49ffc5fef7 100644 --- a/package.json +++ b/package.json @@ -15,10 +15,10 @@ "husky": "^8.0.3", "js-yaml": "^4.1.0", "kill-port": "^1.6.1", - "lerna": "7.0.0-alpha.0", + "lerna": "^7.0.1", "madge": "^6.0.0", "minimist": "^1.2.8", - "nx": "^16.2.1", + "nx": "^16.3.2", "prettier": "^2.3.1", "prettier-plugin-svelte": "^2.3.0", "rimraf": "^3.0.2", diff --git a/packages/backend-core/src/middleware/passport/datasource/google.ts b/packages/backend-core/src/middleware/passport/datasource/google.ts index 6fd4e9ff32..ae6b3b4913 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.ts +++ b/packages/backend-core/src/middleware/passport/datasource/google.ts @@ -1,10 +1,11 @@ import * as google from "../sso/google" import { Cookie } from "../../../constants" -import { clearCookie, getCookie } from "../../../utils" -import { doWithDB } from "../../../db" import * as configs from "../../../configs" -import { BBContext, Database, SSOProfile } from "@budibase/types" +import * as cache from "../../../cache" +import * as utils from "../../../utils" +import { UserCtx, SSOProfile } from "@budibase/types" import { ssoSaveUserNoOp } from "../sso/sso" + const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy type Passport = { @@ -22,7 +23,7 @@ async function fetchGoogleCreds() { export async function preAuth( passport: Passport, - ctx: BBContext, + ctx: UserCtx, next: Function ) { // get the relevant config @@ -36,8 +37,8 @@ export async function preAuth( ssoSaveUserNoOp ) - if (!ctx.query.appId || !ctx.query.datasourceId) { - ctx.throw(400, "appId and datasourceId query params not present.") + if (!ctx.query.appId) { + ctx.throw(400, "appId query param not present.") } return passport.authenticate(strategy, { @@ -49,7 +50,7 @@ export async function preAuth( export async function postAuth( passport: Passport, - ctx: BBContext, + ctx: UserCtx, next: Function ) { // get the relevant config @@ -57,7 +58,7 @@ export async function postAuth( const platformUrl = await configs.getPlatformUrl({ tenantAware: false }) let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback` - const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth) + const authStateCookie = utils.getCookie(ctx, Cookie.DatasourceAuth) return passport.authenticate( new GoogleStrategy( @@ -69,33 +70,26 @@ export async function postAuth( ( accessToken: string, refreshToken: string, - profile: SSOProfile, + _profile: SSOProfile, done: Function ) => { - clearCookie(ctx, Cookie.DatasourceAuth) + utils.clearCookie(ctx, Cookie.DatasourceAuth) done(null, { accessToken, refreshToken }) } ), { successRedirect: "/", failureRedirect: "/error" }, async (err: any, tokens: string[]) => { const baseUrl = `/builder/app/${authStateCookie.appId}/data` - // update the DB for the datasource with all the user info - await doWithDB(authStateCookie.appId, async (db: Database) => { - let datasource - try { - datasource = await db.get(authStateCookie.datasourceId) - } catch (err: any) { - if (err.status === 404) { - ctx.redirect(baseUrl) - } + + const id = utils.newid() + await cache.store( + `datasource:creation:${authStateCookie.appId}:google:${id}`, + { + tokens, } - if (!datasource.config) { - datasource.config = {} - } - datasource.config.auth = { type: "google", ...tokens } - await db.put(datasource) - ctx.redirect(`${baseUrl}/datasource/${authStateCookie.datasourceId}`) - }) + ) + + ctx.redirect(`${baseUrl}/new?continue_google_setup=${id}`) } )(ctx, next) } diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index f9adb68955..7a8cfaf04a 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -1,12 +1,17 @@ import crypto from "crypto" +import fs from "fs" +import zlib from "zlib" import env from "../environment" +import { join } from "path" const ALGO = "aes-256-ctr" const SEPARATOR = "-" const ITERATIONS = 10000 -const RANDOM_BYTES = 16 const STRETCH_LENGTH = 32 +const SALT_LENGTH = 16 +const IV_LENGTH = 16 + export enum SecretOption { API = "api", ENCRYPTION = "encryption", @@ -31,15 +36,15 @@ export function getSecret(secretOption: SecretOption): string { return secret } -function stretchString(string: string, salt: Buffer) { - return crypto.pbkdf2Sync(string, salt, ITERATIONS, STRETCH_LENGTH, "sha512") +function stretchString(secret: string, salt: Buffer) { + return crypto.pbkdf2Sync(secret, salt, ITERATIONS, STRETCH_LENGTH, "sha512") } export function encrypt( input: string, secretOption: SecretOption = SecretOption.API ) { - const salt = crypto.randomBytes(RANDOM_BYTES) + const salt = crypto.randomBytes(SALT_LENGTH) const stretched = stretchString(getSecret(secretOption), salt) const cipher = crypto.createCipheriv(ALGO, stretched, salt) const base = cipher.update(input) @@ -60,3 +65,115 @@ export function decrypt( const final = decipher.final() return Buffer.concat([base, final]).toString() } + +export async function encryptFile( + { dir, filename }: { dir: string; filename: string }, + secret: string +) { + const outputFileName = `${filename}.enc` + + const filePath = join(dir, filename) + const inputFile = fs.createReadStream(filePath) + const outputFile = fs.createWriteStream(join(dir, outputFileName)) + + const salt = crypto.randomBytes(SALT_LENGTH) + const iv = crypto.randomBytes(IV_LENGTH) + const stretched = stretchString(secret, salt) + const cipher = crypto.createCipheriv(ALGO, stretched, iv) + + outputFile.write(salt) + outputFile.write(iv) + + inputFile.pipe(zlib.createGzip()).pipe(cipher).pipe(outputFile) + + return new Promise<{ filename: string; dir: string }>(r => { + outputFile.on("finish", () => { + r({ + filename: outputFileName, + dir, + }) + }) + }) +} + +async function getSaltAndIV(path: string) { + const fileStream = fs.createReadStream(path) + + const salt = await readBytes(fileStream, SALT_LENGTH) + const iv = await readBytes(fileStream, IV_LENGTH) + fileStream.close() + return { salt, iv } +} + +export async function decryptFile( + inputPath: string, + outputPath: string, + secret: string +) { + const { salt, iv } = await getSaltAndIV(inputPath) + const inputFile = fs.createReadStream(inputPath, { + start: SALT_LENGTH + IV_LENGTH, + }) + + const outputFile = fs.createWriteStream(outputPath) + + const stretched = stretchString(secret, salt) + const decipher = crypto.createDecipheriv(ALGO, stretched, iv) + + const unzip = zlib.createGunzip() + + inputFile.pipe(decipher).pipe(unzip).pipe(outputFile) + + return new Promise((res, rej) => { + outputFile.on("finish", () => { + outputFile.close() + res() + }) + + inputFile.on("error", e => { + outputFile.close() + rej(e) + }) + + decipher.on("error", e => { + outputFile.close() + rej(e) + }) + + unzip.on("error", e => { + outputFile.close() + rej(e) + }) + + outputFile.on("error", e => { + outputFile.close() + rej(e) + }) + }) +} + +function readBytes(stream: fs.ReadStream, length: number) { + return new Promise((resolve, reject) => { + let bytesRead = 0 + const data: Buffer[] = [] + + stream.on("readable", () => { + let chunk + + while ((chunk = stream.read(length - bytesRead)) !== null) { + data.push(chunk) + bytesRead += chunk.length + } + + resolve(Buffer.concat(data)) + }) + + stream.on("end", () => { + reject(new Error("Insufficient data in the stream.")) + }) + + stream.on("error", error => { + reject(error) + }) + }) +} diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index bdf7a38726..e8a3c76c0a 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -140,9 +140,13 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string { * Gets the role object, this is mainly useful for two purposes, to check if the level exists and * to check if the role inherits any others. * @param {string|null} roleId The level ID to lookup. + * @param {object|null} opts options for the function, like whether to halt errors, instead return public. * @returns {Promise} The role object, which may contain an "inherits" property. */ -export async function getRole(roleId?: string): Promise { +export async function getRole( + roleId?: string, + opts?: { defaultPublic?: boolean } +): Promise { if (!roleId) { return undefined } @@ -161,6 +165,9 @@ export async function getRole(roleId?: string): Promise { // finalise the ID role._id = getExternalRoleID(role._id) } catch (err) { + if (!isBuiltin(roleId) && opts?.defaultPublic) { + return cloneDeep(BUILTIN_ROLES.PUBLIC) + } // only throw an error if there is no role at all if (Object.keys(role).length === 0) { throw err diff --git a/packages/bbui/src/FancyForm/FancyCheckbox.svelte b/packages/bbui/src/FancyForm/FancyCheckbox.svelte index 191cc79485..0a2e5ac159 100644 --- a/packages/bbui/src/FancyForm/FancyCheckbox.svelte +++ b/packages/bbui/src/FancyForm/FancyCheckbox.svelte @@ -8,6 +8,8 @@ export let disabled = false export let error = null export let validate = null + export let indeterminate = false + export let compact = false const dispatch = createEventDispatcher() @@ -21,11 +23,19 @@ } - + - + -
+
{#if text} {text} {/if} @@ -47,6 +57,10 @@ line-clamp: 2; -webkit-box-orient: vertical; } + .text.compact { + font-size: 13px; + line-height: 15px; + } .text > :global(*) { font-size: inherit !important; } diff --git a/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte b/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte new file mode 100644 index 0000000000..aaea388c36 --- /dev/null +++ b/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte @@ -0,0 +1,68 @@ + + +{#if options && Array.isArray(options)} +
+ + {#if showSelectAll} + + {/if} + {#each options as option, i} + + {/each} + +
+{/if} + + diff --git a/packages/bbui/src/FancyForm/FancyField.svelte b/packages/bbui/src/FancyForm/FancyField.svelte index 0c99394599..455f4b38fb 100644 --- a/packages/bbui/src/FancyForm/FancyField.svelte +++ b/packages/bbui/src/FancyForm/FancyField.svelte @@ -11,6 +11,7 @@ export let value export let ref export let autoHeight + export let compact = false const formContext = getContext("fancy-form") const id = Math.random() @@ -42,6 +43,7 @@ class:disabled class:focused class:clickable + class:compact class:auto-height={autoHeight} >
@@ -61,7 +63,6 @@ diff --git a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte index b7d70d88b7..ceb8fd7f4b 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte @@ -3,8 +3,6 @@ import { store } from "builderStore" import { auth } from "stores/portal" - export let preAuthStep - export let datasource export let disabled export let samePage @@ -15,18 +13,8 @@ class:disabled {disabled} on:click={async () => { - let ds = datasource let appId = $store.appId - if (!ds) { - const resp = await preAuthStep() - if (resp.datasource && resp.appId) { - ds = resp.datasource - appId = resp.appId - } else { - ds = resp - } - } - const url = `/api/global/auth/${tenantId}/datasource/google?datasourceId=${ds._id}&appId=${appId}` + const url = `/api/global/auth/${tenantId}/datasource/google?appId=${appId}` if (samePage) { window.location = url } else { diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js index 18aa361570..2486942dea 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js @@ -44,6 +44,9 @@ export default ICONS export function getIcon(integrationType, schema) { const integrationList = get(integrations) + if (!integrationList) { + return + } if (integrationList[integrationType]?.iconUrl) { return { url: integrationList[integrationType].iconUrl } } else if (schema?.custom || !ICONS[integrationType]) { diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 31a0d21cd8..1d84dbbe39 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -1,12 +1,19 @@ saveDatasource()} - confirmText={datasource.plus ? "Connect" : "Save and continue to query"} - cancelText="Back" - showSecondaryButton={datasource.plus} + {title} + onConfirm={() => nextStep()} + {confirmText} + cancelText={fetchTableStep ? "Cancel" : "Back"} + showSecondaryButton={datasourcePlus} size="L" disabled={!isValid} > - Connect your database to Budibase using the config below. + + {#if !fetchTableStep} + Connect your database to Budibase using the config below + {:else} + Choose what tables you want to sync with Budibase + {/if} - (isValid = e.detail)} - /> + {#if !fetchTableStep} + (isValid = e.detail)} + /> + {:else} +
+ +
+ {/if}
+ + diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index 0783a9fe53..14f81f915c 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -1,43 +1,207 @@ - - {#if isGoogleConfigured === true} - + {#if step === GoogleDatasouceConfigStep.AUTH} + + {#if isGoogleConfigured === true} + + Authenticate with your google account to use the {integrationName} integration. + + + {:else if isGoogleConfigured === false} Authenticate with your google account to use the {IntegrationNames[ - datasource.type - ]} integration.Google authentication is not enabled, please complete Google SSO + configuration. + Configure Google SSO + {/if} + {/if} + {#if step === GoogleDatasouceConfigStep.SET_URL} + + Add the URL of the sheet you want to connect. + + (isValid = e.detail)} + /> + + {/if} + {#if step === GoogleDatasouceConfigStep.SET_SHEETS} + + Select which spreadsheets you want to connect. + + + + {#if setSheetsErrorTitle || setSheetsErrorMessage} + + {/if} - save(datasource, true)} /> - {:else if isGoogleConfigured === false} - Google authentication is not enabled, please complete Google SSO - configuration. - Configure Google SSO {/if} diff --git a/packages/builder/src/components/commandPalette/CommandPalette.svelte b/packages/builder/src/components/commandPalette/CommandPalette.svelte index ae946dc10c..3a369446a3 100644 --- a/packages/builder/src/components/commandPalette/CommandPalette.svelte +++ b/packages/builder/src/components/commandPalette/CommandPalette.svelte @@ -69,7 +69,7 @@ name: "App", description: "", icon: "Play", - action: () => window.open(`/${$store.appId}`), + action: () => store.update(state => ({ ...state, showPreview: true })), }, { type: "Preview", diff --git a/packages/builder/src/components/deploy/AppActions.svelte b/packages/builder/src/components/deploy/AppActions.svelte index 9813237317..a85eb5a154 100644 --- a/packages/builder/src/components/deploy/AppActions.svelte +++ b/packages/builder/src/components/deploy/AppActions.svelte @@ -62,7 +62,10 @@ } const previewApp = () => { - window.open(`/${application}`) + store.update(state => ({ + ...state, + showPreview: true, + })) } const viewApp = () => { diff --git a/packages/builder/src/components/start/ExportAppModal.svelte b/packages/builder/src/components/start/ExportAppModal.svelte index 948416b192..4a69aaef74 100644 --- a/packages/builder/src/components/start/ExportAppModal.svelte +++ b/packages/builder/src/components/start/ExportAppModal.svelte @@ -1,27 +1,128 @@ - - - Apps can be exported with or without data that is within internal tables - - select this below. - + + {#if currentStep === Step.CONFIG} + + + + + {#if !encypt} + + {/if} + {/if} + {#if currentStep === Step.SET_PASSWORD} + + {/if} diff --git a/packages/builder/src/helpers/validation/validation.js b/packages/builder/src/helpers/validation/validation.js index db5dfe4430..f64bf56835 100644 --- a/packages/builder/src/helpers/validation/validation.js +++ b/packages/builder/src/helpers/validation/validation.js @@ -6,7 +6,6 @@ export function createValidationStore(initialValue, ...validators) { let touched = false const value = writable(initialValue || "") - const error = derived(value, $v => validate($v, validators)) const touchedStore = derived(value, () => { if (!touched) { touched = true @@ -14,6 +13,10 @@ export function createValidationStore(initialValue, ...validators) { } return touched }) + const error = derived( + [value, touchedStore], + ([$v, $t]) => $t && validate($v, validators) + ) return [value, error, touchedStore] } diff --git a/packages/builder/src/helpers/validation/yup/index.js b/packages/builder/src/helpers/validation/yup/index.js index 20ddaebb1a..b5bdf030a5 100644 --- a/packages/builder/src/helpers/validation/yup/index.js +++ b/packages/builder/src/helpers/validation/yup/index.js @@ -5,6 +5,7 @@ import { notifications } from "@budibase/bbui" export const createValidationStore = () => { const DEFAULT = { + values: {}, errors: {}, touched: {}, valid: false, @@ -33,6 +34,9 @@ export const createValidationStore = () => { case "email": propertyValidator = string().email().nullable() break + case "password": + propertyValidator = string().nullable() + break default: propertyValidator = string().nullable() } @@ -41,9 +45,68 @@ export const createValidationStore = () => { propertyValidator = propertyValidator.required() } + // We want to do this after the possible required validation, to prioritise the required error + switch (type) { + case "password": + propertyValidator = propertyValidator.min(8) + break + } + validator[propertyName] = propertyValidator } + const observe = async (propertyName, value) => { + const values = get(validation).values + let fieldIsValid + if (!values.hasOwnProperty(propertyName)) { + // Initial setup + values[propertyName] = value + return + } + + if (value === values[propertyName]) { + return + } + + const obj = object().shape(validator) + try { + validation.update(store => { + store.errors[propertyName] = null + return store + }) + await obj.validateAt(propertyName, { [propertyName]: value }) + fieldIsValid = true + } catch (error) { + const [fieldError] = error.errors + if (fieldError) { + validation.update(store => { + store.errors[propertyName] = capitalise(fieldError) + store.valid = false + return store + }) + } + } + + if (fieldIsValid) { + // Validate the rest of the fields + try { + await obj.validate( + { ...values, [propertyName]: value }, + { abortEarly: false } + ) + validation.update(store => { + store.valid = true + return store + }) + } catch { + validation.update(store => { + store.valid = false + return store + }) + } + } + } + const check = async values => { const obj = object().shape(validator) // clear the previous errors @@ -87,5 +150,6 @@ export const createValidationStore = () => { check, addValidator, addValidatorType, + observe, } } diff --git a/packages/builder/src/pages/builder/app/[application]/_components/PreviewOverlay.svelte b/packages/builder/src/pages/builder/app/[application]/_components/PreviewOverlay.svelte new file mode 100644 index 0000000000..d069d1b4c7 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/_components/PreviewOverlay.svelte @@ -0,0 +1,91 @@ + + +
+
+
+
+ +
+