diff --git a/.eslintignore b/.eslintignore index f2c53c2fdc..94984a446f 100644 --- a/.eslintignore +++ b/.eslintignore @@ -12,4 +12,5 @@ packages/sdk/sdk packages/account-portal/packages/server/build packages/account-portal/packages/ui/.routify packages/account-portal/packages/ui/build -**/*.ivm.bundle.js \ No newline at end of file +**/*.ivm.bundle.js +packages/server/build/oldClientVersions/**/** diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 5c474aa826..36ea2f5448 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -138,6 +138,8 @@ jobs: test-server: runs-on: ubuntu-latest + env: + DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull steps: - name: Checkout repo uses: actions/checkout@v4 @@ -151,7 +153,19 @@ jobs: with: node-version: 20.x cache: yarn + + - name: Pull testcontainers images + run: | + docker pull mcr.microsoft.com/mssql/server:2022-latest + docker pull mysql:8.3 + docker pull postgres:16.1-bullseye + docker pull mongo:7.0-jammy + docker pull mariadb:lts + docker pull testcontainers/ryuk:0.5.1 + docker pull budibase/couchdb + - run: yarn --frozen-lockfile + - name: Test server run: | if ${{ env.USE_NX_AFFECTED }}; then diff --git a/.gitignore b/.gitignore index 8861a14d20..661c60e95e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ packages/server/runtime_apps/ bb-airgapped.tar.gz *.iml +packages/server/build/oldClientVersions/**/* +packages/builder/src/components/deploy/clientVersions.json + # Logs logs *.log diff --git a/.vscode/launch.json b/.vscode/launch.json index cfd8d7b155..2fda61345b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,4 +1,3 @@ - { // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. @@ -20,6 +19,13 @@ "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], "args": ["${workspaceFolder}/packages/worker/src/index.ts"], "cwd": "${workspaceFolder}/packages/worker" + }, + { + "type": "chrome", + "request": "launch", + "name": "Launch Chrome against localhost", + "url": "http://localhost:10000", + "webRoot": "${workspaceFolder}" } ], "compounds": [ diff --git a/globalSetup.ts b/globalSetup.ts new file mode 100644 index 0000000000..4cb542a3c3 --- /dev/null +++ b/globalSetup.ts @@ -0,0 +1,25 @@ +import { GenericContainer, Wait } from "testcontainers" + +export default async function setup() { + await new GenericContainer("budibase/couchdb") + .withExposedPorts(5984) + .withEnvironment({ + COUCHDB_PASSWORD: "budibase", + COUCHDB_USER: "budibase", + }) + .withCopyContentToContainer([ + { + content: ` + [log] + level = warn + `, + target: "/opt/couchdb/etc/local.d/test-couchdb.ini", + }, + ]) + .withWaitStrategy( + Wait.forSuccessfulCommand( + "curl http://budibase:budibase@localhost:5984/_up" + ).withStartupTimeout(20000) + ) + .start() +} diff --git a/jestTestcontainersConfigGenerator.js b/jestTestcontainersConfigGenerator.js deleted file mode 100644 index 1e39ed771f..0000000000 --- a/jestTestcontainersConfigGenerator.js +++ /dev/null @@ -1,16 +0,0 @@ -module.exports = () => { - return { - couchdb: { - image: "budibase/couchdb", - ports: [5984], - env: { - COUCHDB_PASSWORD: "budibase", - COUCHDB_USER: "budibase", - }, - wait: { - type: "ports", - timeout: 20000, - } - } - } -} diff --git a/package.json b/package.json index 7de22ab456..79a7b06eff 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ }, "scripts": { "preinstall": "node scripts/syncProPackage.js", + "get-past-client-version": "node scripts/getPastClientVersion.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", diff --git a/packages/backend-core/jest-testcontainers-config.js b/packages/backend-core/jest-testcontainers-config.js deleted file mode 100644 index 8ac0f0cd9d..0000000000 --- a/packages/backend-core/jest-testcontainers-config.js +++ /dev/null @@ -1,8 +0,0 @@ -const { join } = require("path") -require("dotenv").config({ - path: join(__dirname, "..", "..", "hosting", ".env"), -}) - -const jestTestcontainersConfigGenerator = require("../../jestTestcontainersConfigGenerator") - -module.exports = jestTestcontainersConfigGenerator() diff --git a/packages/backend-core/jest.config.ts b/packages/backend-core/jest.config.ts index 3f1065ead2..c944b0d7e1 100644 --- a/packages/backend-core/jest.config.ts +++ b/packages/backend-core/jest.config.ts @@ -1,8 +1,8 @@ import { Config } from "@jest/types" const baseConfig: Config.InitialProjectOptions = { - preset: "@trendyol/jest-testcontainers", setupFiles: ["./tests/jestEnv.ts"], + globalSetup: "./../../globalSetup.ts", setupFilesAfterEnv: ["./tests/jestSetup.ts"], transform: { "^.+\\.ts?$": "@swc/jest", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index fe56780982..030fec8728 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -60,7 +60,6 @@ "@shopify/jest-koa-mocks": "5.1.1", "@swc/core": "1.3.71", "@swc/jest": "0.2.27", - "@trendyol/jest-testcontainers": "^2.1.1", "@types/chance": "1.1.3", "@types/cookies": "0.7.8", "@types/jest": "29.5.5", diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index 9c960d76dd..735c2fa86e 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -27,7 +27,7 @@ class Replication { return resolve(info) }) .on("error", function (err) { - throw new Error(`Replication Error: ${err}`) + throw err }) }) } diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index f02b9fdf32..92b81553b0 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -1,4 +1,4 @@ -import { Event } from "@budibase/types" +import { Event, Identity } from "@budibase/types" import { processors } from "./processors" import identification from "./identification" import * as backfill from "./backfill" @@ -7,12 +7,19 @@ import { publishAsyncEvent } from "./asyncEvents" export const publishEvent = async ( event: Event, properties: any, - timestamp?: string | number + timestamp?: string | number, + identityOverride?: Identity ) => { // in future this should use async events via a distributed queue. - const identity = await identification.getCurrentIdentity() + const identity = + identityOverride || (await identification.getCurrentIdentity()) + + // Backfilling is get from the user cache, but when we override the identity cache is not available. Overrides are + // normally performed in automatic actions or operations in async flows (BPM) where the user session is not available. + const backfilling = identityOverride + ? false + : await backfill.isBackfillingEvent(event) - const backfilling = await backfill.isBackfillingEvent(event) // no backfill - send the event and exit if (!backfilling) { // send off async events if required diff --git a/packages/backend-core/src/events/publishers/account.ts b/packages/backend-core/src/events/publishers/account.ts index d337e404ef..99767962dd 100644 --- a/packages/backend-core/src/events/publishers/account.ts +++ b/packages/backend-core/src/events/publishers/account.ts @@ -5,13 +5,19 @@ import { AccountCreatedEvent, AccountDeletedEvent, AccountVerifiedEvent, + Identity, } from "@budibase/types" -async function created(account: Account) { +async function created(account: Account, identityOverride?: Identity) { const properties: AccountCreatedEvent = { tenantId: account.tenantId, } - await publishEvent(Event.ACCOUNT_CREATED, properties) + await publishEvent( + Event.ACCOUNT_CREATED, + properties, + undefined, + identityOverride + ) } async function deleted(account: Account) { diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 136cb4b8ad..04d3264e6f 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -500,13 +500,13 @@ export class UserDB { static async createAdminUser( email: string, - password: string, tenantId: string, + password?: string, opts?: CreateAdminUserOpts ) { const user: User = { email: email, - password: password, + password, createdAt: Date.now(), roles: {}, builder: { diff --git a/packages/backend-core/tests/core/utilities/testContainerUtils.ts b/packages/backend-core/tests/core/utilities/testContainerUtils.ts index 7da6cbc777..5d4f5a3c11 100644 --- a/packages/backend-core/tests/core/utilities/testContainerUtils.ts +++ b/packages/backend-core/tests/core/utilities/testContainerUtils.ts @@ -1,80 +1,58 @@ +import { DatabaseImpl } from "../../../src/db" import { execSync } from "child_process" -let dockerPsResult: string | undefined - -function formatDockerPsResult(serverName: string, port: number) { - const lines = dockerPsResult?.split("\n") - let first = true - if (!lines) { - return null - } - for (let line of lines) { - if (first) { - first = false - continue - } - let toLookFor = serverName.split("-service")[0] - if (!line.includes(toLookFor)) { - continue - } - const regex = new RegExp(`0.0.0.0:([0-9]*)->${port}`, "g") - const found = line.match(regex) - if (found) { - return found[0].split(":")[1].split("->")[0] - } - } - return null +interface ContainerInfo { + Command: string + CreatedAt: string + ID: string + Image: string + Labels: string + LocalVolumes: string + Mounts: string + Names: string + Networks: string + Ports: string + RunningFor: string + Size: string + State: string + Status: string } -function getTestContainerSettings( - serverName: string, - key: string -): string | null { - const entry = Object.entries(global).find( - ([k]) => - k.includes(`${serverName.toUpperCase()}`) && - k.includes(`${key.toUpperCase()}`) - ) - if (!entry) { - return null - } - return entry[1] +function getTestcontainers(): ContainerInfo[] { + return execSync("docker ps --format json") + .toString() + .split("\n") + .filter(x => x.length > 0) + .map(x => JSON.parse(x) as ContainerInfo) + .filter(x => x.Labels.includes("org.testcontainers=true")) } -function getContainerInfo(containerName: string, port: number) { - let assignedPort = getTestContainerSettings( - containerName.toUpperCase(), - `PORT_${port}` - ) - if (!dockerPsResult) { - try { - const outputBuffer = execSync("docker ps") - dockerPsResult = outputBuffer.toString("utf8") - } catch (err) { - //no-op - } - } - const possiblePort = formatDockerPsResult(containerName, port) - if (possiblePort) { - assignedPort = possiblePort - } - const host = getTestContainerSettings(containerName.toUpperCase(), "IP") - return { - port: assignedPort, - host, - url: host && assignedPort && `http://${host}:${assignedPort}`, - } +function getContainerByImage(image: string) { + return getTestcontainers().find(x => x.Image.startsWith(image)) } -function getCouchConfig() { - return getContainerInfo("couchdb", 5984) +function getExposedPort(container: ContainerInfo, port: number) { + const match = container.Ports.match(new RegExp(`0.0.0.0:(\\d+)->${port}/tcp`)) + if (!match) { + return undefined + } + return parseInt(match[1]) } export function setupEnv(...envs: any[]) { - const couch = getCouchConfig() + const couch = getContainerByImage("budibase/couchdb") + if (!couch) { + throw new Error("CouchDB container not found") + } + + const couchPort = getExposedPort(couch, 5984) + if (!couchPort) { + throw new Error("CouchDB port not found") + } + const configs = [ - { key: "COUCH_DB_PORT", value: couch.port }, - { key: "COUCH_DB_URL", value: couch.url }, + { key: "COUCH_DB_PORT", value: `${couchPort}` }, + { key: "COUCH_DB_URL", value: `http://localhost:${couchPort}` }, ] for (const config of configs.filter(x => !!x.value)) { @@ -82,4 +60,7 @@ export function setupEnv(...envs: any[]) { env._set(config.key, config.value) } } + + // @ts-expect-error + DatabaseImpl.nano = undefined } diff --git a/packages/backend-core/tests/jestEnv.ts b/packages/backend-core/tests/jestEnv.ts index c2047118ec..2c797c9fff 100644 --- a/packages/backend-core/tests/jestEnv.ts +++ b/packages/backend-core/tests/jestEnv.ts @@ -4,3 +4,7 @@ process.env.NODE_ENV = "jest" process.env.MOCK_REDIS = "1" process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error" process.env.REDIS_PASSWORD = "budibase" +process.env.COUCH_DB_PASSWORD = "budibase" +process.env.COUCH_DB_USER = "budibase" +process.env.API_ENCRYPTION_KEY = "testsecret" +process.env.JWT_SECRET = "testsecret" diff --git a/packages/builder/.vscode/launch.json b/packages/builder/.vscode/launch.json deleted file mode 100644 index f6b35a0b63..0000000000 --- a/packages/builder/.vscode/launch.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "type": "chrome", - "request": "launch", - "name": "Launch Chrome against localhost", - "url": "http://localhost:3000", - "webRoot": "${workspaceFolder}" - } - ] -} \ No newline at end of file diff --git a/packages/builder/.vscode/settings.json b/packages/builder/.vscode/settings.json deleted file mode 100644 index 7148fd3b98..0000000000 --- a/packages/builder/.vscode/settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "javascript.format.enable": false, - "svelte.plugin.svelte.format.enable": false, - "html.format.enable": false, - "json.format.enable": false, - "editor.trimAutoWhitespace": false, - "sass.format.deleteWhitespace": false -} \ No newline at end of file diff --git a/packages/builder/src/components/deploy/RevertModalVersionSelect.svelte b/packages/builder/src/components/deploy/RevertModalVersionSelect.svelte new file mode 100644 index 0000000000..ed40a101d0 --- /dev/null +++ b/packages/builder/src/components/deploy/RevertModalVersionSelect.svelte @@ -0,0 +1,33 @@ + + +
+