diff --git a/.eslintrc.json b/.eslintrc.json index 2a40c6cc29..525072dc6c 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -34,7 +34,6 @@ }, { "files": ["**/*.ts"], - "excludedFiles": ["qa-core/**"], "parser": "@typescript-eslint/parser", "plugins": ["@typescript-eslint"], "extends": ["eslint:recommended"], @@ -49,7 +48,6 @@ }, { "files": ["**/*.spec.ts"], - "excludedFiles": ["qa-core/**"], "parser": "@typescript-eslint/parser", "plugins": ["jest", "@typescript-eslint"], "extends": ["eslint:recommended", "plugin:jest/recommended"], diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 36ea2f5448..42d73ba8bb 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -66,7 +66,8 @@ jobs: # Run build all the projects - name: Build run: | - yarn build + yarn build:oss + yarn build:account-portal # Check the types of the projects built via esbuild - name: Check types run: | @@ -90,6 +91,9 @@ jobs: test-libraries: runs-on: ubuntu-latest + env: + DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull + REUSE_CONTAINERS: true steps: - name: Checkout repo uses: actions/checkout@v4 @@ -103,6 +107,14 @@ jobs: with: node-version: 20.x cache: yarn + - name: Pull testcontainers images + run: | + docker pull testcontainers/ryuk:0.5.1 & + docker pull budibase/couchdb & + docker pull redis & + + wait $(jobs -p) + - run: yarn --frozen-lockfile - name: Test run: | @@ -137,9 +149,10 @@ jobs: fi test-server: - runs-on: ubuntu-latest + runs-on: budi-tubby-tornado-quad-core-150gb env: DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull + REUSE_CONTAINERS: true steps: - name: Checkout repo uses: actions/checkout@v4 @@ -156,13 +169,16 @@ jobs: - name: Pull testcontainers images run: | - docker pull mcr.microsoft.com/mssql/server:2022-latest - docker pull mysql:8.3 - docker pull postgres:16.1-bullseye - docker pull mongo:7.0-jammy - docker pull mariadb:lts - docker pull testcontainers/ryuk:0.5.1 - docker pull budibase/couchdb + docker pull mcr.microsoft.com/mssql/server:2022-latest & + docker pull mysql:8.3 & + docker pull postgres:16.1-bullseye & + docker pull mongo:7.0-jammy & + docker pull mariadb:lts & + docker pull testcontainers/ryuk:0.5.1 & + docker pull budibase/couchdb & + docker pull redis & + + wait $(jobs -p) - run: yarn --frozen-lockfile @@ -174,35 +190,6 @@ jobs: yarn test --scope=@budibase/server fi - integration-test: - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} - token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - - - name: Use Node.js 20.x - uses: actions/setup-node@v4 - with: - node-version: 20.x - cache: yarn - - run: yarn --frozen-lockfile - - name: Build packages - run: yarn build --scope @budibase/server --scope @budibase/worker - - name: Build backend-core for OSS contributor (required for pro) - if: ${{ env.IS_OSS_CONTRIBUTOR == 'true' }} - run: yarn build --scope @budibase/backend-core - - name: Run tests - run: | - cd qa-core - yarn setup - yarn serve:test:self:ci - env: - BB_ADMIN_USER_EMAIL: admin - BB_ADMIN_USER_PASSWORD: admin - check-pro-submodule: runs-on: ubuntu-latest if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') @@ -231,27 +218,34 @@ jobs: echo "pro_commit=$pro_commit" echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT" echo "base_commit=$base_commit" - echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT" + + base_commit_excluding_merges=$(git log --no-merges -n 1 --format=format:%H $base_commit) + echo "base_commit_excluding_merges=$base_commit_excluding_merges" + echo "base_commit_excluding_merges=$base_commit_excluding_merges" >> "$GITHUB_OUTPUT" else echo "Nothing to do - branch to branch merge." fi - - name: Check submodule merged to base branch - if: ${{ steps.get_pro_commits.outputs.base_commit != '' }} - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const submoduleCommit = '${{ steps.get_pro_commits.outputs.pro_commit }}'; - const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}'; + - name: Check submodule merged and latest on base branch + if: ${{ steps.get_pro_commits.outputs.base_commit_excluding_merges != '' }} + run: | + cd packages/pro + base_commit_excluding_merges='${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}' + pro_commit='${{ steps.get_pro_commits.outputs.pro_commit }}' - if (submoduleCommit !== baseCommit) { - console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.'); - console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md') - process.exit(1); - } else { - console.log('All good, the submodule had been merged and setup correctly!') - } + any_commit=$(git log --no-merges $base_commit_excluding_merges...$pro_commit) + + if [ -n "$any_commit" ]; then + echo $any_commit + + echo "An error occurred: " + echo 'Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.' + echo 'Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md' + + exit 1 + else + echo 'All good, the submodule had been merged and setup correctly!' + fi check-accountportal-submodule: runs-on: ubuntu-latest @@ -264,7 +258,15 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Check account portal commit + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + src: + - packages/account-portal/** + + - if: steps.changes.outputs.src == 'true' + name: Check account portal commit id: get_accountportal_commits run: | cd packages/account-portal diff --git a/.gitignore b/.gitignore index 661c60e95e..b68ddd975f 100644 --- a/.gitignore +++ b/.gitignore @@ -69,7 +69,6 @@ typings/ # dotenv environment variables file .env -!qa-core/.env !hosting/.env # parcel-bundler cache (https://parceljs.org/) diff --git a/globalSetup.ts b/globalSetup.ts index 4cb542a3c3..7bf5e2152c 100644 --- a/globalSetup.ts +++ b/globalSetup.ts @@ -1,25 +1,47 @@ import { GenericContainer, Wait } from "testcontainers" +import path from "path" +import lockfile from "proper-lockfile" export default async function setup() { - await new GenericContainer("budibase/couchdb") - .withExposedPorts(5984) - .withEnvironment({ - COUCHDB_PASSWORD: "budibase", - COUCHDB_USER: "budibase", - }) - .withCopyContentToContainer([ - { - content: ` + const lockPath = path.resolve(__dirname, "globalSetup.ts") + if (process.env.REUSE_CONTAINERS) { + // If you run multiple tests at the same time, it's possible for the CouchDB + // shared container to get started multiple times despite having an + // identical reuse hash. To avoid that, we do a filesystem-based lock so + // that only one globalSetup.ts is running at a time. + lockfile.lockSync(lockPath) + } + + try { + let couchdb = new GenericContainer("budibase/couchdb") + .withExposedPorts(5984) + .withEnvironment({ + COUCHDB_PASSWORD: "budibase", + COUCHDB_USER: "budibase", + }) + .withCopyContentToContainer([ + { + content: ` [log] level = warn `, - target: "/opt/couchdb/etc/local.d/test-couchdb.ini", - }, - ]) - .withWaitStrategy( - Wait.forSuccessfulCommand( - "curl http://budibase:budibase@localhost:5984/_up" - ).withStartupTimeout(20000) - ) - .start() + target: "/opt/couchdb/etc/local.d/test-couchdb.ini", + }, + ]) + .withWaitStrategy( + Wait.forSuccessfulCommand( + "curl http://budibase:budibase@localhost:5984/_up" + ).withStartupTimeout(20000) + ) + + if (process.env.REUSE_CONTAINERS) { + couchdb = couchdb.withReuse() + } + + await couchdb.start() + } finally { + if (process.env.REUSE_CONTAINERS) { + lockfile.unlockSync(lockPath) + } + } } diff --git a/lerna.json b/lerna.json index 9cffdba08a..93b103ee00 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.22.12", + "version": "2.22.15", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/package.json b/package.json index 79a7b06eff..4b6716f7e7 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "@babel/preset-env": "^7.22.5", "@esbuild-plugins/tsconfig-paths": "^0.1.2", "@types/node": "20.10.0", + "@types/proper-lockfile": "^4.1.4", "@typescript-eslint/parser": "6.9.0", "esbuild": "^0.18.17", "esbuild-node-externals": "^1.8.0", @@ -23,6 +24,7 @@ "nx-cloud": "16.0.5", "prettier": "2.8.8", "prettier-plugin-svelte": "^2.3.0", + "proper-lockfile": "^4.1.2", "svelte": "^4.2.10", "svelte-eslint-parser": "^0.33.1", "typescript": "5.2.2", @@ -34,6 +36,8 @@ "get-past-client-version": "node scripts/getPastClientVersion.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", + "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui", + "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal --scope @budibase/account-portal-server --scope @budibase/account-portal-ui", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "check:types": "lerna run check:types", "build:sdk": "lerna run --stream build:sdk", @@ -56,11 +60,11 @@ "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", "test": "lerna run --stream test --stream", - "lint:eslint": "eslint packages qa-core --max-warnings=0", - "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", + "lint:eslint": "eslint packages --max-warnings=0", + "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", "lint": "yarn run lint:eslint && yarn run lint:prettier", - "lint:fix:eslint": "eslint --fix --max-warnings=0 packages qa-core", - "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"", + "lint:fix:eslint": "eslint --fix --max-warnings=0 packages", + "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", "lint:fix": "yarn run lint:fix:eslint && yarn run lint:fix:prettier", "build:specs": "lerna run --stream specs", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", diff --git a/packages/backend-core/src/platform/users.ts b/packages/backend-core/src/platform/users.ts index 6f030afb7c..ccaad76b19 100644 --- a/packages/backend-core/src/platform/users.ts +++ b/packages/backend-core/src/platform/users.ts @@ -20,7 +20,7 @@ export async function lookupTenantId(userId: string) { return user.tenantId } -async function getUserDoc(emailOrId: string): Promise { +export async function getUserDoc(emailOrId: string): Promise { const db = getPlatformDB() return db.get(emailOrId) } @@ -79,6 +79,17 @@ async function addUserDoc(emailOrId: string, newDocFn: () => PlatformUser) { } } +export async function addSsoUser( + ssoId: string, + email: string, + userId: string, + tenantId: string +) { + return addUserDoc(ssoId, () => + newUserSsoIdDoc(ssoId, email, userId, tenantId) + ) +} + export async function addUser( tenantId: string, userId: string, @@ -91,9 +102,7 @@ export async function addUser( ] if (ssoId) { - promises.push( - addUserDoc(ssoId, () => newUserSsoIdDoc(ssoId, email, userId, tenantId)) - ) + promises.push(addSsoUser(ssoId, email, userId, tenantId)) } await Promise.all(promises) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 638da4a5b1..48920a3771 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -14,16 +14,16 @@ import { } from "../db" import { BulkDocsResponse, + ContextUser, + CouchFindOptions, + DatabaseQueryOpts, SearchQuery, SearchQueryOperators, SearchUsersRequest, User, - ContextUser, - DatabaseQueryOpts, - CouchFindOptions, } from "@budibase/types" -import { getGlobalDB } from "../context" import * as context from "../context" +import { getGlobalDB } from "../context" import { isCreator } from "./utils" import { UserDB } from "./db" @@ -48,6 +48,7 @@ export function isSupportedUserSearch(query: SearchQuery) { const allowed = [ { op: SearchQueryOperators.STRING, key: "email" }, { op: SearchQueryOperators.EQUAL, key: "_id" }, + { op: SearchQueryOperators.ONE_OF, key: "_id" }, ] for (let [key, operation] of Object.entries(query)) { if (typeof operation !== "object") { @@ -285,6 +286,10 @@ export async function paginatedUsers({ } else if (query?.string?.email) { userList = await searchGlobalUsersByEmail(query?.string?.email, opts) property = "email" + } else if (query?.oneOf?._id) { + userList = await bulkGetGlobalUsersById(query?.oneOf?._id, { + cleanup: true, + }) } else { // no search, query allDocs const response = await db.allDocs(getGlobalUserParams(null, opts)) diff --git a/packages/backend-core/tests/core/utilities/testContainerUtils.ts b/packages/backend-core/tests/core/utilities/testContainerUtils.ts index 5d4f5a3c11..951a6f0517 100644 --- a/packages/backend-core/tests/core/utilities/testContainerUtils.ts +++ b/packages/backend-core/tests/core/utilities/testContainerUtils.ts @@ -1,6 +1,7 @@ -import { DatabaseImpl } from "../../../src/db" import { execSync } from "child_process" +const IPV4_PORT_REGEX = new RegExp(`0\\.0\\.0\\.0:(\\d+)->(\\d+)/tcp`, "g") + interface ContainerInfo { Command: string CreatedAt: string @@ -19,7 +20,10 @@ interface ContainerInfo { } function getTestcontainers(): ContainerInfo[] { - return execSync("docker ps --format json") + // We use --format json to make sure the output is nice and machine-readable, + // and we use --no-trunc so that the command returns full container IDs so we + // can filter on them correctly. + return execSync("docker ps --format json --no-trunc") .toString() .split("\n") .filter(x => x.length > 0) @@ -27,32 +31,55 @@ function getTestcontainers(): ContainerInfo[] { .filter(x => x.Labels.includes("org.testcontainers=true")) } -function getContainerByImage(image: string) { - return getTestcontainers().find(x => x.Image.startsWith(image)) +export function getContainerByImage(image: string) { + const containers = getTestcontainers().filter(x => x.Image.startsWith(image)) + if (containers.length > 1) { + let errorMessage = `Multiple containers found starting with image: "${image}"\n\n` + for (const container of containers) { + errorMessage += JSON.stringify(container, null, 2) + } + throw new Error(errorMessage) + } + return containers[0] } -function getExposedPort(container: ContainerInfo, port: number) { - const match = container.Ports.match(new RegExp(`0.0.0.0:(\\d+)->${port}/tcp`)) - if (!match) { - return undefined +export function getContainerById(id: string) { + return getTestcontainers().find(x => x.ID === id) +} + +export interface Port { + host: number + container: number +} + +export function getExposedV4Ports(container: ContainerInfo): Port[] { + let ports: Port[] = [] + for (const match of container.Ports.matchAll(IPV4_PORT_REGEX)) { + ports.push({ host: parseInt(match[1]), container: parseInt(match[2]) }) } - return parseInt(match[1]) + return ports +} + +export function getExposedV4Port(container: ContainerInfo, port: number) { + return getExposedV4Ports(container).find(x => x.container === port)?.host } export function setupEnv(...envs: any[]) { + // We start couchdb in globalSetup.ts, in the root of the monorepo, so it + // should be relatively safe to look for it by its image name. const couch = getContainerByImage("budibase/couchdb") if (!couch) { throw new Error("CouchDB container not found") } - const couchPort = getExposedPort(couch, 5984) + const couchPort = getExposedV4Port(couch, 5984) if (!couchPort) { throw new Error("CouchDB port not found") } const configs = [ { key: "COUCH_DB_PORT", value: `${couchPort}` }, - { key: "COUCH_DB_URL", value: `http://localhost:${couchPort}` }, + { key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` }, ] for (const config of configs.filter(x => !!x.value)) { @@ -60,7 +87,4 @@ export function setupEnv(...envs: any[]) { env._set(config.key, config.value) } } - - // @ts-expect-error - DatabaseImpl.nano = undefined } diff --git a/packages/bbui/rollup.config.js b/packages/bbui/rollup.config.js index e285d548d6..da274e0ba5 100644 --- a/packages/bbui/rollup.config.js +++ b/packages/bbui/rollup.config.js @@ -12,6 +12,13 @@ export default { format: "esm", file: "dist/bbui.es.js", }, + onwarn(warning, warn) { + // suppress eval warnings + if (warning.code === "EVAL") { + return + } + warn(warning) + }, plugins: [ resolve(), commonjs(), diff --git a/packages/builder/package.json b/packages/builder/package.json index f61ac4fe26..253f5a0c14 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -72,7 +72,7 @@ "fast-json-patch": "^3.1.1", "json-format-highlight": "^1.0.4", "lodash": "4.17.21", - "posthog-js": "^1.36.0", + "posthog-js": "^1.116.6", "remixicon": "2.5.0", "sanitize-html": "^2.7.0", "shortid": "2.2.15", diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 7fa2401c88..0632993cf0 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -31,7 +31,7 @@ import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte" import BindingSidePanel from "components/common/bindings/BindingSidePanel.svelte" - import { BindingHelpers } from "components/common/bindings/utils" + import { BindingHelpers, BindingType } from "components/common/bindings/utils" import { bindingsToCompletions, hbAutocomplete, @@ -576,6 +576,7 @@ { js: true, dontDecode: true, + type: BindingType.RUNTIME, } )} mode="javascript" diff --git a/packages/builder/src/components/backend/DataTable/formula.js b/packages/builder/src/components/backend/DataTable/formula.js index f5ff3caec4..e3da4249bc 100644 --- a/packages/builder/src/components/backend/DataTable/formula.js +++ b/packages/builder/src/components/backend/DataTable/formula.js @@ -1,3 +1,4 @@ +import { FieldType } from "@budibase/types" import { FIELDS } from "constants/backend" import { tables } from "stores/builder" import { get as svelteGet } from "svelte/store" @@ -5,14 +6,12 @@ import { get as svelteGet } from "svelte/store" // currently supported level of relationship depth (server side) const MAX_DEPTH = 1 -//https://github.com/Budibase/budibase/issues/3030 -const internalType = "internal" - const TYPES_TO_SKIP = [ - FIELDS.FORMULA.type, - FIELDS.LONGFORM.type, - FIELDS.ATTACHMENT.type, - internalType, + FieldType.FORMULA, + FieldType.LONGFORM, + FieldType.ATTACHMENT, + //https://github.com/Budibase/budibase/issues/3030 + FieldType.INTERNAL, ] export function getBindings({ @@ -26,7 +25,7 @@ export function getBindings({ return bindings } for (let [column, schema] of Object.entries(table.schema)) { - const isRelationship = schema.type === FIELDS.LINK.type + const isRelationship = schema.type === FieldType.LINK // skip relationships after a certain depth and types which // can't bind to if ( diff --git a/packages/builder/src/components/backend/Datasources/relationshipErrors.js b/packages/builder/src/components/backend/Datasources/relationshipErrors.js index 259484e9a9..610ff9f1fe 100644 --- a/packages/builder/src/components/backend/Datasources/relationshipErrors.js +++ b/packages/builder/src/components/backend/Datasources/relationshipErrors.js @@ -1,4 +1,4 @@ -import { RelationshipType } from "constants/backend" +import { RelationshipType } from "@budibase/types" const typeMismatch = "Column type of the foreign key must match the primary key" const columnBeingUsed = "Column name cannot be an existing column" diff --git a/packages/builder/src/components/backend/TableNavigator/utils.js b/packages/builder/src/components/backend/TableNavigator/utils.js index b7e46042be..ae7aaa0f0a 100644 --- a/packages/builder/src/components/backend/TableNavigator/utils.js +++ b/packages/builder/src/components/backend/TableNavigator/utils.js @@ -12,7 +12,7 @@ const getDefaultSchema = rows => { newSchema[column] = { name: column, type: "string", - constraints: FIELDS["STRING"].constraints, + constraints: FIELDS.STRING.constraints, } }) }) diff --git a/packages/builder/src/components/common/HelpMenu.svelte b/packages/builder/src/components/common/HelpMenu.svelte index baff9a5a27..63156676d2 100644 --- a/packages/builder/src/components/common/HelpMenu.svelte +++ b/packages/builder/src/components/common/HelpMenu.svelte @@ -5,7 +5,7 @@ import { licensing } from "stores/portal" import { isPremiumOrAbove } from "helpers/planTitle" - $: premiumOrAboveLicense = isPremiumOrAbove($licensing?.license.plan.type) + $: premiumOrAboveLicense = isPremiumOrAbove($licensing?.license?.plan?.type) let show let hide diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte index 21d389357f..10d95a3e7e 100644 --- a/packages/builder/src/components/common/bindings/BindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte @@ -371,6 +371,7 @@