diff --git a/.eslintignore b/.eslintignore index 1dac74b117..0d81de0ef9 100644 --- a/.eslintignore +++ b/.eslintignore @@ -10,4 +10,4 @@ packages/builder/.routify packages/builder/cypress/support/queryLevelTransformerFunction.js packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js packages/builder/cypress/reports -packages/sdk/sdk \ No newline at end of file +packages/sdk/sdk diff --git a/.eslintrc.json b/.eslintrc.json index 79e0c00abd..d94c749042 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -16,8 +16,7 @@ "dist", "public", "*.spec.js", - "bundle.js", - "packages/pro" + "bundle.js" ], "plugins": ["svelte3"], "extends": ["eslint:recommended"], @@ -30,9 +29,7 @@ "files": ["**/*.ts"], "parser": "@typescript-eslint/parser", "plugins": [], - "extends": [ - "eslint:recommended" - ], + "extends": ["eslint:recommended"], "rules": { "no-unused-vars": "off", "no-inner-declarations": "off", diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 6c875f2dfe..f2e7f2eda9 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -1,5 +1,9 @@ name: Budibase CI +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + on: # Trigger the workflow on push or pull request, # but only for the master branch @@ -22,7 +26,16 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - name: Checkout repo and submodules + uses: actions/checkout@v3 + if: github.repository == github.event.pull_request.head.repo.full_name + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} + - name: Checkout repo only + uses: actions/checkout@v3 + if: github.repository != github.event.pull_request.head.repo.full_name + - name: Use Node.js 14.x uses: actions/setup-node@v3 with: @@ -34,10 +47,16 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - name: Checkout repo and submodules + uses: actions/checkout@v3 + if: github.repository == github.event.pull_request.head.repo.full_name with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} + - name: Checkout repo only + uses: actions/checkout@v3 + if: github.repository != github.event.pull_request.head.repo.full_name + - name: Use Node.js 14.x uses: actions/setup-node@v3 with: @@ -52,10 +71,16 @@ jobs: test-libraries: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - name: Checkout repo and submodules + uses: actions/checkout@v3 + if: github.repository == github.event.pull_request.head.repo.full_name with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} + - name: Checkout repo only + uses: actions/checkout@v3 + if: github.repository != github.event.pull_request.head.repo.full_name + - name: Use Node.js 14.x uses: actions/setup-node@v3 with: @@ -72,10 +97,16 @@ jobs: test-services: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - name: Checkout repo and submodules + uses: actions/checkout@v3 + if: github.repository == github.event.pull_request.head.repo.full_name with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} + - name: Checkout repo only + uses: actions/checkout@v3 + if: github.repository != github.event.pull_request.head.repo.full_name + - name: Use Node.js 14.x uses: actions/setup-node@v3 with: @@ -91,11 +122,14 @@ jobs: test-pro: runs-on: ubuntu-latest + if: github.repository == github.event.pull_request.head.repo.full_name steps: - - uses: actions/checkout@v3 + - name: Checkout repo and submodules + uses: actions/checkout@v3 with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} + - name: Use Node.js 14.x uses: actions/setup-node@v3 with: @@ -107,10 +141,16 @@ jobs: integration-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - name: Checkout repo and submodules + uses: actions/checkout@v3 + if: github.repository == github.event.pull_request.head.repo.full_name with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} + - name: Checkout repo only + uses: actions/checkout@v3 + if: github.repository != github.event.pull_request.head.repo.full_name + - name: Use Node.js 14.x uses: actions/setup-node@v3 with: @@ -129,21 +169,47 @@ jobs: check-pro-submodule: runs-on: ubuntu-latest + if: github.repository == github.event.pull_request.head.repo.full_name steps: - - name: Checkout code + - name: Checkout repo and submodules uses: actions/checkout@v3 with: submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Check submodule + token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} + + - name: Check pro commit + id: get_pro_commits run: | cd packages/pro - git fetch - if ! git merge-base --is-ancestor $(git log -n 1 --pretty=format:%H) origin/develop; then - echo "Current commit has not been merged to develop" - echo "Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md" - exit 1 + pro_commit=$(git rev-parse HEAD) + + branch="${{ github.base_ref || github.ref_name }}" + echo "Running on branch `$branch` (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})" + + if [[ $branch == "master" ]]; then + base_commit=$(git rev-parse origin/master) else - echo "All good, the submodule had been merged!" + base_commit=$(git rev-parse origin/develop) fi + + echo "pro_commit=$pro_commit" + echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT" + echo "base_commit=$base_commit" + echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT" + + - name: Check submodule merged to develop + uses: actions/github-script@v4 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const submoduleCommit = '${{ steps.get_pro_commits.outputs.pro_commit }}'; + const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}'; + + if (submoduleCommit !== baseCommit) { + console.error('Submodule commit does not match the latest commit on the develop branch.'); + console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md') + process.exit(1); + } else { + console.log('All good, the submodule had been merged and setup correctly!') + } diff --git a/.github/workflows/tag-prerelease.yml b/.github/workflows/tag-prerelease.yml index 83660e409d..f6446c55f5 100644 --- a/.github/workflows/tag-prerelease.yml +++ b/.github/workflows/tag-prerelease.yml @@ -32,10 +32,11 @@ jobs: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - run: yarn + - run: cd scripts && yarn - name: Tag prerelease run: | + cd scripts # setup the username and email. git config --global user.name "Budibase Staging Release Bot" git config --global user.email "<>" - ./scripts/versionCommit.sh prerelease + ./versionCommit.sh prerelease diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml index f361c200a0..191c3ad9ef 100644 --- a/.github/workflows/tag-release.yml +++ b/.github/workflows/tag-release.yml @@ -42,12 +42,13 @@ jobs: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - run: yarn + - run: cd scripts && yarn - name: Tag release run: | + cd scripts # setup the username and email. git config --global user.name "Budibase Staging Release Bot" git config --global user.email "<>" BUMP_TYPE_INPUT=${{ github.event.inputs.versioning }} BUMP_TYPE=${BUMP_TYPE_INPUT:-"patch"} - ./scripts/versionCommit.sh $BUMP_TYPE + ./versionCommit.sh $BUMP_TYPE diff --git a/hosting/nginx.dev.conf b/hosting/nginx.dev.conf index 1ecee422cd..915125cbce 100644 --- a/hosting/nginx.dev.conf +++ b/hosting/nginx.dev.conf @@ -126,6 +126,16 @@ http { proxy_pass http://app-service; } + location /embed { + rewrite /embed/(.*) /app/$1 break; + proxy_pass http://app-service; + proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header x-budibase-embed "true"; + add_header x-budibase-embed "true"; + add_header Content-Security-Policy "frame-ancestors *"; + } + location /builder { proxy_read_timeout 120s; proxy_connect_timeout 120s; diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 001a08a9a6..9ce6b54053 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -92,6 +92,16 @@ http { proxy_pass $apps; } + location /embed { + rewrite /embed/(.*) /app/$1 break; + proxy_pass $apps; + proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header x-budibase-embed "true"; + add_header x-budibase-embed "true"; + add_header Content-Security-Policy "frame-ancestors *"; + } + location = / { proxy_pass $apps; } diff --git a/hosting/scripts/airgapped/airgappedDockerBuild.js b/hosting/scripts/airgapped/airgappedDockerBuild.js index cc0ea48eb3..58bc7c09a9 100755 --- a/hosting/scripts/airgapped/airgappedDockerBuild.js +++ b/hosting/scripts/airgapped/airgappedDockerBuild.js @@ -2,7 +2,9 @@ const fs = require("fs") const { execSync } = require("child_process") const path = require("path") -const IMAGES = { +const IS_SINGLE_IMAGE = process.env.SINGLE_IMAGE + +let IMAGES = { worker: "budibase/worker", apps: "budibase/apps", proxy: "budibase/proxy", @@ -10,7 +12,13 @@ const IMAGES = { couch: "ibmcom/couchdb3", curl: "curlimages/curl", redis: "redis", - watchtower: "containrrr/watchtower" + watchtower: "containrrr/watchtower", +} + +if (IS_SINGLE_IMAGE) { + IMAGES = { + budibase: "budibase/budibase" + } } const FILES = { @@ -39,11 +47,10 @@ for (let image in IMAGES) { } // copy config files -copyFile(FILES.COMPOSE) +if (!IS_SINGLE_IMAGE) { + copyFile(FILES.COMPOSE) +} copyFile(FILES.ENV) // compress -execSync(`tar -czf bb-airgapped.tar.gz hosting/scripts/bb-airgapped`) - -// clean up -fs.rmdirSync(OUTPUT_DIR, { recursive: true }) \ No newline at end of file +execSync(`tar -czf bb-airgapped.tar.gz hosting/scripts/bb-airgapped`) \ No newline at end of file diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 56df8185a9..e43e5ad10c 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -37,6 +37,14 @@ COPY --from=build /worker /worker RUN apt-get update && \ apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server +# Install postgres client for pg_dump utils +RUN apt install software-properties-common apt-transport-https gpg -y \ + && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ + && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ + && apt update -y \ + && apt install postgresql-client-15 -y \ + && apt remove software-properties-common apt-transport-https gpg -y + # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx WORKDIR /nodejs RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh && \ diff --git a/lerna.json b/lerna.json index a59a94209d..c7289c8010 100644 --- a/lerna.json +++ b/lerna.json @@ -1,22 +1,10 @@ { - "version": "2.7.36", + "version": "2.7.37-alpha.14", "npmClient": "yarn", "packages": [ - "packages/backend-core", - "packages/bbui", - "packages/builder", - "packages/cli", - "packages/client", - "packages/frontend-core", - "packages/sdk", - "packages/server", - "packages/shared-core", - "packages/string-templates", - "packages/types", - "packages/worker", - "packages/pro/packages/pro" + "packages/*" ], - "useWorkspaces": true, + "useNx": true, "command": { "publish": { "ignoreChanges": [ diff --git a/package.json b/package.json index 56f015f8c0..42e528dfce 100644 --- a/package.json +++ b/package.json @@ -2,28 +2,26 @@ "name": "root", "private": true, "devDependencies": { - "@esbuild-plugins/node-resolve": "^0.2.2", "@esbuild-plugins/tsconfig-paths": "^0.1.2", "@nx/js": "16.2.1", "@rollup/plugin-json": "^4.0.2", "@typescript-eslint/parser": "5.45.0", "babel-eslint": "^10.0.3", "esbuild": "^0.17.18", + "esbuild-node-externals": "^1.7.0", "eslint": "^7.28.0", "eslint-plugin-cypress": "^2.11.3", "eslint-plugin-svelte3": "^3.2.0", "husky": "^8.0.3", "js-yaml": "^4.1.0", "kill-port": "^1.6.1", - "lerna": "7.0.0-alpha.0", + "lerna": "7.0.2", "madge": "^6.0.0", "minimist": "^1.2.8", - "nx": "^16.2.1", - "prettier": "^2.3.1", + "prettier": "2.8.8", "prettier-plugin-svelte": "^2.3.0", "rimraf": "^3.0.2", "rollup-plugin-replace": "^2.2.0", - "semver": "^7.5.0", "svelte": "^3.38.2", "typescript": "4.7.3" }, @@ -48,9 +46,9 @@ "kill-builder": "kill-port 3000", "kill-server": "kill-port 4001 4002", "kill-all": "yarn run kill-builder && yarn run kill-server", - "dev": "yarn run kill-all && lerna run --stream --parallel dev:builder --stream", - "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", - "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server", + "dev": "yarn run kill-all && lerna run --stream --parallel dev:builder --stream", + "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", + "dev:server": "yarn run kill-server && yarn build --projects=@budibase/client && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built", "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", "test": "lerna run --stream test --stream", @@ -67,6 +65,7 @@ "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -", "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", + "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .", "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .", @@ -95,19 +94,7 @@ }, "workspaces": { "packages": [ - "packages/backend-core", - "packages/bbui", - "packages/builder", - "packages/cli", - "packages/client", - "packages/frontend-core", - "packages/sdk", - "packages/server", - "packages/shared-core", - "packages/string-templates", - "packages/types", - "packages/worker", - "packages/pro/packages/pro" + "packages/*" ] }, "resolutions": { diff --git a/packages/backend-core/jest.config.ts b/packages/backend-core/jest.config.ts index 1e69797e71..8d64b24a2f 100644 --- a/packages/backend-core/jest.config.ts +++ b/packages/backend-core/jest.config.ts @@ -31,4 +31,6 @@ const config: Config.InitialOptions = { coverageReporters: ["lcov", "json", "clover"], } +process.env.DISABLE_PINO_LOGGER = "1" + export default config diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index f85687b007..4a1ed5c373 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -27,7 +27,7 @@ "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", "aws-sdk": "2.1030.0", - "bcrypt": "5.0.1", + "bcrypt": "5.1.0", "bcryptjs": "2.4.3", "bull": "4.10.1", "correlation-id": "4.0.0", diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index eb9d613a58..e813722d98 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -57,6 +57,9 @@ class Replication { appReplicateOpts() { return { filter: (doc: any) => { + if (doc._id && doc._id.startsWith(DocumentType.AUTOMATION_LOG)) { + return false + } return doc._id !== DocumentType.APP_METADATA }, } diff --git a/packages/backend-core/src/db/errors.ts b/packages/backend-core/src/db/errors.ts new file mode 100644 index 0000000000..17f76c667b --- /dev/null +++ b/packages/backend-core/src/db/errors.ts @@ -0,0 +1,14 @@ +export function checkErrorCode(error: any, code: number) { + const stringCode = code.toString() + if (typeof error === "object") { + return error.status === code || error.message?.includes(stringCode) + } else if (typeof error === "number") { + return error === code + } else if (typeof error === "string") { + return error.includes(stringCode) + } +} + +export function isDocumentConflictError(error: any) { + return checkErrorCode(error, 409) +} diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index cf71199ecf..c47d1793dc 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -9,3 +9,4 @@ export * from "../constants/db" export { getGlobalDBName, baseGlobalDBName } from "../context" export * from "./lucene" export * as searchIndexes from "./searchIndexes" +export * from "./errors" diff --git a/packages/backend-core/src/docIds/ids.ts b/packages/backend-core/src/docIds/ids.ts index 152977b3af..e0ac85b3df 100644 --- a/packages/backend-core/src/docIds/ids.ts +++ b/packages/backend-core/src/docIds/ids.ts @@ -81,8 +81,19 @@ export function generateAppUserID(prodAppId: string, userId: string) { * Generates a new role ID. * @returns {string} The new role ID which the role doc can be stored under. */ -export function generateRoleID(id?: any) { - return `${DocumentType.ROLE}${SEPARATOR}${id || newid()}` +export function generateRoleID(name: string) { + const prefix = `${DocumentType.ROLE}${SEPARATOR}` + if (name.startsWith(prefix)) { + return name + } + return `${prefix}${name}` +} + +/** + * Utility function to be more verbose. + */ +export function prefixRoleID(name: string) { + return generateRoleID(name) } /** diff --git a/packages/backend-core/src/events/publishers/serve.ts b/packages/backend-core/src/events/publishers/serve.ts index 64e24e20a7..ac6a23dfdb 100644 --- a/packages/backend-core/src/events/publishers/serve.ts +++ b/packages/backend-core/src/events/publishers/serve.ts @@ -14,10 +14,15 @@ async function servedBuilder(timezone: string) { await publishEvent(Event.SERVED_BUILDER, properties) } -async function servedApp(app: App, timezone: string) { +async function servedApp( + app: App, + timezone: string, + embed?: boolean | undefined +) { const properties: AppServedEvent = { appVersion: app.version, timezone, + embed: embed === true, } await publishEvent(Event.SERVED_APP, properties) } diff --git a/packages/backend-core/src/logging/alerts.ts b/packages/backend-core/src/logging/alerts.ts index 625ecfcf15..b7173bc436 100644 --- a/packages/backend-core/src/logging/alerts.ts +++ b/packages/backend-core/src/logging/alerts.ts @@ -21,6 +21,6 @@ export function logAlertWithInfo( logAlert(message, error) } -export function logWarn(message: string) { - console.warn(`bb-warn: ${message}`) +export function logWarn(message: string, e?: any) { + console.warn(`bb-warn: ${message}`, e) } diff --git a/packages/backend-core/src/middleware/passport/datasource/google.ts b/packages/backend-core/src/middleware/passport/datasource/google.ts index 6fd4e9ff32..ae6b3b4913 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.ts +++ b/packages/backend-core/src/middleware/passport/datasource/google.ts @@ -1,10 +1,11 @@ import * as google from "../sso/google" import { Cookie } from "../../../constants" -import { clearCookie, getCookie } from "../../../utils" -import { doWithDB } from "../../../db" import * as configs from "../../../configs" -import { BBContext, Database, SSOProfile } from "@budibase/types" +import * as cache from "../../../cache" +import * as utils from "../../../utils" +import { UserCtx, SSOProfile } from "@budibase/types" import { ssoSaveUserNoOp } from "../sso/sso" + const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy type Passport = { @@ -22,7 +23,7 @@ async function fetchGoogleCreds() { export async function preAuth( passport: Passport, - ctx: BBContext, + ctx: UserCtx, next: Function ) { // get the relevant config @@ -36,8 +37,8 @@ export async function preAuth( ssoSaveUserNoOp ) - if (!ctx.query.appId || !ctx.query.datasourceId) { - ctx.throw(400, "appId and datasourceId query params not present.") + if (!ctx.query.appId) { + ctx.throw(400, "appId query param not present.") } return passport.authenticate(strategy, { @@ -49,7 +50,7 @@ export async function preAuth( export async function postAuth( passport: Passport, - ctx: BBContext, + ctx: UserCtx, next: Function ) { // get the relevant config @@ -57,7 +58,7 @@ export async function postAuth( const platformUrl = await configs.getPlatformUrl({ tenantAware: false }) let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback` - const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth) + const authStateCookie = utils.getCookie(ctx, Cookie.DatasourceAuth) return passport.authenticate( new GoogleStrategy( @@ -69,33 +70,26 @@ export async function postAuth( ( accessToken: string, refreshToken: string, - profile: SSOProfile, + _profile: SSOProfile, done: Function ) => { - clearCookie(ctx, Cookie.DatasourceAuth) + utils.clearCookie(ctx, Cookie.DatasourceAuth) done(null, { accessToken, refreshToken }) } ), { successRedirect: "/", failureRedirect: "/error" }, async (err: any, tokens: string[]) => { const baseUrl = `/builder/app/${authStateCookie.appId}/data` - // update the DB for the datasource with all the user info - await doWithDB(authStateCookie.appId, async (db: Database) => { - let datasource - try { - datasource = await db.get(authStateCookie.datasourceId) - } catch (err: any) { - if (err.status === 404) { - ctx.redirect(baseUrl) - } + + const id = utils.newid() + await cache.store( + `datasource:creation:${authStateCookie.appId}:google:${id}`, + { + tokens, } - if (!datasource.config) { - datasource.config = {} - } - datasource.config.auth = { type: "google", ...tokens } - await db.put(datasource) - ctx.redirect(`${baseUrl}/datasource/${authStateCookie.datasourceId}`) - }) + ) + + ctx.redirect(`${baseUrl}/new?continue_google_setup=${id}`) } )(ctx, next) } diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index f9adb68955..7a8cfaf04a 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -1,12 +1,17 @@ import crypto from "crypto" +import fs from "fs" +import zlib from "zlib" import env from "../environment" +import { join } from "path" const ALGO = "aes-256-ctr" const SEPARATOR = "-" const ITERATIONS = 10000 -const RANDOM_BYTES = 16 const STRETCH_LENGTH = 32 +const SALT_LENGTH = 16 +const IV_LENGTH = 16 + export enum SecretOption { API = "api", ENCRYPTION = "encryption", @@ -31,15 +36,15 @@ export function getSecret(secretOption: SecretOption): string { return secret } -function stretchString(string: string, salt: Buffer) { - return crypto.pbkdf2Sync(string, salt, ITERATIONS, STRETCH_LENGTH, "sha512") +function stretchString(secret: string, salt: Buffer) { + return crypto.pbkdf2Sync(secret, salt, ITERATIONS, STRETCH_LENGTH, "sha512") } export function encrypt( input: string, secretOption: SecretOption = SecretOption.API ) { - const salt = crypto.randomBytes(RANDOM_BYTES) + const salt = crypto.randomBytes(SALT_LENGTH) const stretched = stretchString(getSecret(secretOption), salt) const cipher = crypto.createCipheriv(ALGO, stretched, salt) const base = cipher.update(input) @@ -60,3 +65,115 @@ export function decrypt( const final = decipher.final() return Buffer.concat([base, final]).toString() } + +export async function encryptFile( + { dir, filename }: { dir: string; filename: string }, + secret: string +) { + const outputFileName = `${filename}.enc` + + const filePath = join(dir, filename) + const inputFile = fs.createReadStream(filePath) + const outputFile = fs.createWriteStream(join(dir, outputFileName)) + + const salt = crypto.randomBytes(SALT_LENGTH) + const iv = crypto.randomBytes(IV_LENGTH) + const stretched = stretchString(secret, salt) + const cipher = crypto.createCipheriv(ALGO, stretched, iv) + + outputFile.write(salt) + outputFile.write(iv) + + inputFile.pipe(zlib.createGzip()).pipe(cipher).pipe(outputFile) + + return new Promise<{ filename: string; dir: string }>(r => { + outputFile.on("finish", () => { + r({ + filename: outputFileName, + dir, + }) + }) + }) +} + +async function getSaltAndIV(path: string) { + const fileStream = fs.createReadStream(path) + + const salt = await readBytes(fileStream, SALT_LENGTH) + const iv = await readBytes(fileStream, IV_LENGTH) + fileStream.close() + return { salt, iv } +} + +export async function decryptFile( + inputPath: string, + outputPath: string, + secret: string +) { + const { salt, iv } = await getSaltAndIV(inputPath) + const inputFile = fs.createReadStream(inputPath, { + start: SALT_LENGTH + IV_LENGTH, + }) + + const outputFile = fs.createWriteStream(outputPath) + + const stretched = stretchString(secret, salt) + const decipher = crypto.createDecipheriv(ALGO, stretched, iv) + + const unzip = zlib.createGunzip() + + inputFile.pipe(decipher).pipe(unzip).pipe(outputFile) + + return new Promise((res, rej) => { + outputFile.on("finish", () => { + outputFile.close() + res() + }) + + inputFile.on("error", e => { + outputFile.close() + rej(e) + }) + + decipher.on("error", e => { + outputFile.close() + rej(e) + }) + + unzip.on("error", e => { + outputFile.close() + rej(e) + }) + + outputFile.on("error", e => { + outputFile.close() + rej(e) + }) + }) +} + +function readBytes(stream: fs.ReadStream, length: number) { + return new Promise((resolve, reject) => { + let bytesRead = 0 + const data: Buffer[] = [] + + stream.on("readable", () => { + let chunk + + while ((chunk = stream.read(length - bytesRead)) !== null) { + data.push(chunk) + bytesRead += chunk.length + } + + resolve(Buffer.concat(data)) + }) + + stream.on("end", () => { + reject(new Error("Insufficient data in the stream.")) + }) + + stream.on("error", error => { + reject(error) + }) + }) +} diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index bdf7a38726..cf5c6bc406 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -1,5 +1,5 @@ import { BuiltinPermissionID, PermissionLevel } from "./permissions" -import { generateRoleID, getRoleParams, DocumentType, SEPARATOR } from "../db" +import { prefixRoleID, getRoleParams, DocumentType, SEPARATOR } from "../db" import { getAppDB } from "../context" import { doWithDB } from "../db" import { Screen, Role as RoleDoc } from "@budibase/types" @@ -25,18 +25,28 @@ const EXTERNAL_BUILTIN_ROLE_IDS = [ BUILTIN_IDS.PUBLIC, ] +export const RoleIDVersion = { + // original version, with a UUID based ID + UUID: undefined, + // new version - with name based ID + NAME: "name", +} + export class Role implements RoleDoc { _id: string _rev?: string name: string permissionId: string inherits?: string + version?: string permissions = {} constructor(id: string, name: string, permissionId: string) { this._id = id this.name = name this.permissionId = permissionId + // version for managing the ID - removing the role_ when responding + this.version = RoleIDVersion.NAME } addInheritance(inherits: string) { @@ -140,9 +150,13 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string { * Gets the role object, this is mainly useful for two purposes, to check if the level exists and * to check if the role inherits any others. * @param {string|null} roleId The level ID to lookup. + * @param {object|null} opts options for the function, like whether to halt errors, instead return public. * @returns {Promise} The role object, which may contain an "inherits" property. */ -export async function getRole(roleId?: string): Promise { +export async function getRole( + roleId?: string, + opts?: { defaultPublic?: boolean } +): Promise { if (!roleId) { return undefined } @@ -153,14 +167,20 @@ export async function getRole(roleId?: string): Promise { role = cloneDeep( Object.values(BUILTIN_ROLES).find(role => role._id === roleId) ) + } else { + // make sure has the prefix (if it has it then it won't be added) + roleId = prefixRoleID(roleId) } try { const db = getAppDB() const dbRole = await db.get(getDBRoleID(roleId)) role = Object.assign(role, dbRole) // finalise the ID - role._id = getExternalRoleID(role._id) + role._id = getExternalRoleID(role._id, role.version) } catch (err) { + if (!isBuiltin(roleId) && opts?.defaultPublic) { + return cloneDeep(BUILTIN_ROLES.PUBLIC) + } // only throw an error if there is no role at all if (Object.keys(role).length === 0) { throw err @@ -254,6 +274,9 @@ export async function getAllRoles(appId?: string) { }) ) roles = body.rows.map((row: any) => row.doc) + roles.forEach( + role => (role._id = getExternalRoleID(role._id!, role.version)) + ) } const builtinRoles = getBuiltinRoles() @@ -261,14 +284,15 @@ export async function getAllRoles(appId?: string) { for (let builtinRoleId of EXTERNAL_BUILTIN_ROLE_IDS) { const builtinRole = builtinRoles[builtinRoleId] const dbBuiltin = roles.filter( - dbRole => getExternalRoleID(dbRole._id) === builtinRoleId + dbRole => + getExternalRoleID(dbRole._id!, dbRole.version) === builtinRoleId )[0] if (dbBuiltin == null) { roles.push(builtinRole || builtinRoles.BASIC) } else { // remove role and all back after combining with the builtin roles = roles.filter(role => role._id !== dbBuiltin._id) - dbBuiltin._id = getExternalRoleID(dbBuiltin._id) + dbBuiltin._id = getExternalRoleID(dbBuiltin._id!, dbBuiltin.version) roles.push(Object.assign(builtinRole, dbBuiltin)) } } @@ -374,19 +398,22 @@ export class AccessController { /** * Adds the "role_" for builtin role IDs which are to be written to the DB (for permissions). */ -export function getDBRoleID(roleId?: string) { - if (roleId?.startsWith(DocumentType.ROLE)) { - return roleId +export function getDBRoleID(roleName: string) { + if (roleName?.startsWith(DocumentType.ROLE)) { + return roleName } - return generateRoleID(roleId) + return prefixRoleID(roleName) } /** * Remove the "role_" from builtin role IDs that have been written to the DB (for permissions). */ -export function getExternalRoleID(roleId?: string) { +export function getExternalRoleID(roleId: string, version?: string) { // for built-in roles we want to remove the DB role ID element (role_) - if (roleId?.startsWith(DocumentType.ROLE) && isBuiltin(roleId)) { + if ( + (roleId.startsWith(DocumentType.ROLE) && isBuiltin(roleId)) || + version === RoleIDVersion.NAME + ) { return roleId.split(`${DocumentType.ROLE}${SEPARATOR}`)[1] } return roleId diff --git a/packages/bbui/src/FancyForm/FancyCheckbox.svelte b/packages/bbui/src/FancyForm/FancyCheckbox.svelte index 191cc79485..0a2e5ac159 100644 --- a/packages/bbui/src/FancyForm/FancyCheckbox.svelte +++ b/packages/bbui/src/FancyForm/FancyCheckbox.svelte @@ -8,6 +8,8 @@ export let disabled = false export let error = null export let validate = null + export let indeterminate = false + export let compact = false const dispatch = createEventDispatcher() @@ -21,11 +23,19 @@ } - + - + -
+
{#if text} {text} {/if} @@ -47,6 +57,10 @@ line-clamp: 2; -webkit-box-orient: vertical; } + .text.compact { + font-size: 13px; + line-height: 15px; + } .text > :global(*) { font-size: inherit !important; } diff --git a/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte b/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte new file mode 100644 index 0000000000..b2e412ac24 --- /dev/null +++ b/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte @@ -0,0 +1,69 @@ + + +{#if options && Array.isArray(options)} +
+ + {#if showSelectAll} + + {/if} + {#each options as option, i} + + {/each} + +
+{/if} + + diff --git a/packages/bbui/src/FancyForm/FancyField.svelte b/packages/bbui/src/FancyForm/FancyField.svelte index 0c99394599..455f4b38fb 100644 --- a/packages/bbui/src/FancyForm/FancyField.svelte +++ b/packages/bbui/src/FancyForm/FancyField.svelte @@ -11,6 +11,7 @@ export let value export let ref export let autoHeight + export let compact = false const formContext = getContext("fancy-form") const id = Math.random() @@ -42,6 +43,7 @@ class:disabled class:focused class:clickable + class:compact class:auto-height={autoHeight} >
@@ -61,7 +63,6 @@ diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte deleted file mode 100644 index ab5b3ccee0..0000000000 --- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte +++ /dev/null @@ -1,249 +0,0 @@ - - - - - - - - - - - confirmDialog.hide()} - warning={false} - title="Confirm table fetch" -> - { - requireSpecificTables = e.detail - specificTables = null - }} - thin - text="Fetch listed tables only (one per line)" - /> - {#if requireSpecificTables} - - {/if} -
- - If you have fetched tables from this database before, this action may - overwrite any changes you made after your initial fetch. - - - - -
- Tables -
- - -
-
- - This datasource can determine tables automatically. Budibase can fetch your - tables directly from the database and you can use them without having to write - any queries at all. - -{#if schemaError} - -{/if} -{#if plusTables && Object.values(plusTables).length > 0} - onClickTable(detail)} - schema={tableSchema} - data={Object.values(plusTables)} - allowEditColumns={false} - allowEditRows={false} - allowSelectRows={false} - customRenderers={[{ column: "primary", component: ArrayRenderer }]} - /> -{:else} - No tables found. -{/if} -{#if integration.relationships !== false} - -
- Relationships - -
- - Tell budibase how your tables are related to get even more smart features. - - {#if relationshipInfo && relationshipInfo.length > 0} -
openRelationshipModal(detail.from, detail.to)} - schema={relationshipSchema} - data={relationshipInfo} - allowEditColumns={false} - allowEditRows={false} - allowSelectRows={false} - /> - {:else} - No relationships configured. - {/if} -{/if} - - diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/index.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/index.svelte deleted file mode 100644 index 9277207e37..0000000000 --- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/index.svelte +++ /dev/null @@ -1,86 +0,0 @@ - - -
-
- {#each Object.entries(integrations) as [integrationType, schema]} -
selectIntegration(integrationType)} - > - - {schema.name || integrationType} -
- {/each} -
-
- - diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte deleted file mode 100644 index 60a7a9f30b..0000000000 --- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte +++ /dev/null @@ -1,123 +0,0 @@ - - - -
-
- Headers - Optional -
-
- - Headers enable you to provide additional information about the request, such - as format. - - onDefaultHeaderUpdate(evt.detail)} - noAddButton - bindings={getRestBindings()} -/> -
- addHeader.addEntry()}> - Add header - -
- - -
-
- Authentication - Optional -
-
- - Create an authentication config that can be shared with queries. - - - - -
-
- Variables - Optional -
-
-Variables enable you to store and re-use values in queries, with the choice - of a static value such as a token using static variables, or a value from a - query response using dynamic variables. -Static - - - -
-Dynamic - - Dynamic variables are evaluated when a dependant query is executed. The value - is cached for a period of time and will be refreshed if a query fails. - - - - diff --git a/packages/builder/src/components/backend/DatasourceNavigator/_components/DatasourceCard.svelte b/packages/builder/src/components/backend/DatasourceNavigator/_components/DatasourceCard.svelte deleted file mode 100644 index 67ecf1c56c..0000000000 --- a/packages/builder/src/components/backend/DatasourceNavigator/_components/DatasourceCard.svelte +++ /dev/null @@ -1,60 +0,0 @@ - - -
dispatcher("selected", integrationType)} - class="item hoverable" -> -
- -
- {schema.friendlyName} - {#if schema.type} - {schema.type || ""} - {/if} -
-
-
- - diff --git a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleSignIn.svelte b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleSignIn.svelte deleted file mode 100644 index c30e8fc2ee..0000000000 --- a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleSignIn.svelte +++ /dev/null @@ -1,145 +0,0 @@ - - - - - btn_google_dark_normal_ios - Created with Sketch. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js index 18aa361570..2486942dea 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js @@ -44,6 +44,9 @@ export default ICONS export function getIcon(integrationType, schema) { const integrationList = get(integrations) + if (!integrationList) { + return + } if (integrationList[integrationType]?.iconUrl) { return { url: integrationList[integrationType].iconUrl } } else if (schema?.custom || !ICONS[integrationType]) { diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte deleted file mode 100644 index 31a0d21cd8..0000000000 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ /dev/null @@ -1,81 +0,0 @@ - - - saveDatasource()} - confirmText={datasource.plus ? "Connect" : "Save and continue to query"} - cancelText="Back" - showSecondaryButton={datasource.plus} - size="L" - disabled={!isValid} -> - - Connect your database to Budibase using the config below. - - - (isValid = e.detail)} - /> - diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte deleted file mode 100644 index 0783a9fe53..0000000000 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ /dev/null @@ -1,43 +0,0 @@ - - - - - {#if isGoogleConfigured === true} - - Authenticate with your google account to use the {IntegrationNames[ - datasource.type - ]} integration. - - save(datasource, true)} /> - {:else if isGoogleConfigured === false} - Google authentication is not enabled, please complete Google SSO - configuration. - Configure Google SSO - {/if} - diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/UpdateDatasourceModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/UpdateDatasourceModal.svelte index 89721773cf..991e170a2e 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/UpdateDatasourceModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/UpdateDatasourceModal.svelte @@ -1,7 +1,9 @@ + + diff --git a/packages/builder/src/components/backend/Datasources/ConfigEditor/fields/Boolean.svelte b/packages/builder/src/components/backend/Datasources/ConfigEditor/fields/Boolean.svelte new file mode 100644 index 0000000000..daa4b7a2f9 --- /dev/null +++ b/packages/builder/src/components/backend/Datasources/ConfigEditor/fields/Boolean.svelte @@ -0,0 +1,20 @@ + + +
+ + +
+ + diff --git a/packages/builder/src/components/backend/Datasources/ConfigEditor/fields/FieldGroup.svelte b/packages/builder/src/components/backend/Datasources/ConfigEditor/fields/FieldGroup.svelte new file mode 100644 index 0000000000..eaab7d1508 --- /dev/null +++ b/packages/builder/src/components/backend/Datasources/ConfigEditor/fields/FieldGroup.svelte @@ -0,0 +1,37 @@ + + + !!properties.value)} + header={name} +> + + {#each value as field} + handleChange(field.key, e.detail)} + /> + {/each} + + diff --git a/packages/builder/src/components/backend/Datasources/ConfigEditor/fields/LongForm.svelte b/packages/builder/src/components/backend/Datasources/ConfigEditor/fields/LongForm.svelte new file mode 100644 index 0000000000..efbeb007e8 --- /dev/null +++ b/packages/builder/src/components/backend/Datasources/ConfigEditor/fields/LongForm.svelte @@ -0,0 +1,22 @@ + + +
+ +