From 9b4c22727f7b422dd4528ff4ddd58151bbb1bf06 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 25 Mar 2024 12:36:56 +0000 Subject: [PATCH 01/93] Fix to ensure runtimeTime binding is used is automation js modal --- .../SetupPanel/AutomationBlockSetup.svelte | 3 ++- .../src/components/common/bindings/utils.js | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 7fa2401c88..0632993cf0 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -31,7 +31,7 @@ import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte" import BindingSidePanel from "components/common/bindings/BindingSidePanel.svelte" - import { BindingHelpers } from "components/common/bindings/utils" + import { BindingHelpers, BindingType } from "components/common/bindings/utils" import { bindingsToCompletions, hbAutocomplete, @@ -576,6 +576,7 @@ { js: true, dontDecode: true, + type: BindingType.RUNTIME, } )} mode="javascript" diff --git a/packages/builder/src/components/common/bindings/utils.js b/packages/builder/src/components/common/bindings/utils.js index c60374f0f7..77e4a1dfb1 100644 --- a/packages/builder/src/components/common/bindings/utils.js +++ b/packages/builder/src/components/common/bindings/utils.js @@ -1,6 +1,11 @@ import { decodeJSBinding } from "@budibase/string-templates" import { hbInsert, jsInsert } from "components/common/CodeEditor" +export const BindingType = { + READABLE: "readableBinding", + RUNTIME: "runtimeBinding", +} + export class BindingHelpers { constructor(getCaretPosition, insertAtPos, { disableWrapping } = {}) { this.getCaretPosition = getCaretPosition @@ -25,16 +30,20 @@ export class BindingHelpers { } // Adds a data binding to the expression - onSelectBinding(value, binding, { js, dontDecode }) { + onSelectBinding( + value, + binding, + { js, dontDecode, type = BindingType.READABLE } + ) { const { start, end } = this.getCaretPosition() if (js) { const jsVal = dontDecode ? value : decodeJSBinding(value) - const insertVal = jsInsert(jsVal, start, end, binding.readableBinding, { + const insertVal = jsInsert(jsVal, start, end, binding[type], { disableWrapping: this.disableWrapping, }) this.insertAtPos({ start, end, value: insertVal }) } else { - const insertVal = hbInsert(value, start, end, binding.readableBinding) + const insertVal = hbInsert(value, start, end, binding[type]) this.insertAtPos({ start, end, value: insertVal }) } } From 6c9127427c6c915044352bef10f158727c412fcd Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 26 Mar 2024 10:43:56 +0000 Subject: [PATCH 02/93] Fix issues with colours in app skeletons --- .../[screenId]/_components/AppPreview.svelte | 4 ++-- packages/builder/src/stores/builder/theme.js | 15 ++++++++++----- packages/client/src/stores/theme.js | 11 ++--------- packages/frontend-core/src/utils/index.js | 1 + packages/frontend-core/src/utils/theme.js | 12 ++++++++++++ 5 files changed, 27 insertions(+), 16 deletions(-) create mode 100644 packages/frontend-core/src/utils/theme.js diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index 12d572ecc4..44394fa7ee 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -247,9 +247,9 @@
- {#if loading} + {#if loading || true}
diff --git a/packages/builder/src/stores/builder/theme.js b/packages/builder/src/stores/builder/theme.js index afdf3e9805..efa2609b97 100644 --- a/packages/builder/src/stores/builder/theme.js +++ b/packages/builder/src/stores/builder/theme.js @@ -1,5 +1,6 @@ import { writable, get } from "svelte/store" import { API } from "api" +import { getBaseTheme } from "@budibase/frontend-core" const INITIAL_THEMES_STATE = { theme: "", @@ -12,11 +13,15 @@ export const themes = () => { }) const syncAppTheme = app => { - store.update(state => ({ - ...state, - theme: app.theme || "spectrum--light", - customTheme: app.customTheme, - })) + store.update(state => { + const theme = app.theme || "spectrum--light" + return { + ...state, + theme, + baseTheme: getBaseTheme(theme), + customTheme: app.customTheme, + } + }) } const save = async (theme, appId) => { diff --git a/packages/client/src/stores/theme.js b/packages/client/src/stores/theme.js index 8877556f0c..50dad12fa4 100644 --- a/packages/client/src/stores/theme.js +++ b/packages/client/src/stores/theme.js @@ -1,7 +1,7 @@ import { derived } from "svelte/store" import { appStore } from "./app" import { builderStore } from "./builder" -import { Constants } from "@budibase/frontend-core" +import { getBaseTheme } from "@budibase/frontend-core" // This is the good old acorn bug where having the word "g l o b a l" makes it // think that this is not ES6 compatible and starts throwing errors when using @@ -29,13 +29,6 @@ const createThemeStore = () => { // Ensure theme is set theme = theme || defaultTheme - // Get base theme - let base = - Constants.Themes.find(x => `spectrum--${x.class}` === theme)?.base || "" - if (base) { - base = `spectrum--${base}` - } - // Delete and nullish keys from the custom theme if (customTheme) { Object.entries(customTheme).forEach(([key, value]) => { @@ -59,7 +52,7 @@ const createThemeStore = () => { return { theme, - baseTheme: base, + baseTheme: getBaseTheme(theme), customTheme, customThemeCss, } diff --git a/packages/frontend-core/src/utils/index.js b/packages/frontend-core/src/utils/index.js index 98998b7f0e..6b79e1d040 100644 --- a/packages/frontend-core/src/utils/index.js +++ b/packages/frontend-core/src/utils/index.js @@ -7,3 +7,4 @@ export * as RowUtils from "./rows" export { memo, derivedMemo } from "./memo" export { createWebsocket } from "./websocket" export * from "./download" +export * from "./theme" diff --git a/packages/frontend-core/src/utils/theme.js b/packages/frontend-core/src/utils/theme.js new file mode 100644 index 0000000000..165f7c9782 --- /dev/null +++ b/packages/frontend-core/src/utils/theme.js @@ -0,0 +1,12 @@ +import { Themes } from "../constants.js" + +export const getBaseTheme = theme => { + if (!theme) { + return "" + } + let base = Themes.find(x => `spectrum--${x.class}` === theme)?.base || "" + if (base) { + base = `spectrum--${base}` + } + return base +} From 884d9ecf6c9e6e6c98380140a55fbaa7a4e83245 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 26 Mar 2024 10:45:23 +0000 Subject: [PATCH 03/93] Remove testing code --- .../design/[screenId]/_components/AppPreview.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index 44394fa7ee..298f044086 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -247,7 +247,7 @@
- {#if loading || true} + {#if loading}
Date: Tue, 26 Mar 2024 10:46:58 +0000 Subject: [PATCH 04/93] Remove redundant classes --- .../design/[screenId]/_components/AppPreview.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index 298f044086..0ce9e096f2 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -249,7 +249,7 @@
{#if loading}
From 66009305598b142833a78cee77699cffba18f446 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 26 Mar 2024 10:51:53 +0000 Subject: [PATCH 05/93] Fix annoying account-portal submodule check --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index f5b467b6b1..13b3bc93e1 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit f5b467b6b1c55c48847545db41be7b1c035e167a +Subproject commit 13b3bc93e115a2ec6c2cde5cfa199773e31a308d From 00d57c7f0fc6e28ed0000321c954112fa0711f0d Mon Sep 17 00:00:00 2001 From: Gerard Burns Date: Tue, 26 Mar 2024 11:06:09 +0000 Subject: [PATCH 06/93] add base theme to portal app preview --- .../src/pages/builder/portal/apps/[appId]/index.svelte | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte b/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte index a08189a400..d7e0e35289 100644 --- a/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte +++ b/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte @@ -20,7 +20,7 @@ import { sdk } from "@budibase/shared-core" import { API } from "api" import ErrorSVG from "./ErrorSVG.svelte" - import { ClientAppSkeleton } from "@budibase/frontend-core" + import { getBaseTheme, ClientAppSkeleton } from "@budibase/frontend-core" $: app = $enrichedApps.find(app => app.appId === $params.appId) $: iframeUrl = getIframeURL(app) @@ -137,7 +137,9 @@ class:hide={!loading || !app?.features?.skeletonLoader} class="loading" > -
+
Date: Tue, 26 Mar 2024 11:22:40 +0000 Subject: [PATCH 07/93] Got container reuse strategy in place, need to convert tests now. --- .../src/integration-test/postgres.spec.ts | 55 ++++++++++--------- .../src/integrations/tests/utils/index.ts | 53 ++++++++++++++---- .../src/integrations/tests/utils/mariadb.ts | 22 ++------ .../src/integrations/tests/utils/mongodb.ts | 23 ++------ .../src/integrations/tests/utils/mssql.ts | 22 ++------ .../src/integrations/tests/utils/mysql.ts | 23 ++------ .../src/integrations/tests/utils/postgres.ts | 23 ++------ 7 files changed, 98 insertions(+), 123 deletions(-) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 107c4ade1e..57da02b507 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -16,7 +16,7 @@ import { import _ from "lodash" import { generator } from "@budibase/backend-core/tests" import { utils } from "@budibase/backend-core" -import { databaseTestProviders } from "../integrations/tests/utils" +import { DatabaseName, getDatasource } from "../integrations/tests/utils" import { Client } from "pg" // @ts-ignore fetch.mockSearch() @@ -41,18 +41,17 @@ describe("postgres integrations", () => { makeRequest = generateMakeRequest(apiKey, true) postgresDatasource = await config.api.datasource.create( - await databaseTestProviders.postgres.datasource() + await getDatasource(DatabaseName.POSTGRES) ) }) - afterAll(async () => { - await databaseTestProviders.postgres.stop() - }) - beforeEach(async () => { async function createAuxTable(prefix: string) { return await config.createTable({ - name: `${prefix}_${generator.word({ length: 6 })}`, + name: `${prefix}_${generator + .guid() + .replaceAll("-", "") + .substring(0, 6)}`, type: "table", primary: ["id"], primaryDisplay: "title", @@ -89,7 +88,7 @@ describe("postgres integrations", () => { } primaryPostgresTable = await config.createTable({ - name: `p_${generator.word({ length: 6 })}`, + name: `p_${generator.guid().replaceAll("-", "").substring(0, 6)}`, type: "table", primary: ["id"], schema: { @@ -251,7 +250,7 @@ describe("postgres integrations", () => { async function createDefaultPgTable() { return await config.createTable({ - name: generator.word({ length: 10 }), + name: generator.guid().replaceAll("-", "").substring(0, 10), type: "table", primary: ["id"], schema: { @@ -1043,7 +1042,7 @@ describe("postgres integrations", () => { it("should be able to verify the connection", async () => { await config.api.datasource.verify( { - datasource: await databaseTestProviders.postgres.datasource(), + datasource: await getDatasource(DatabaseName.POSTGRES), }, { body: { @@ -1054,7 +1053,7 @@ describe("postgres integrations", () => { }) it("should state an invalid datasource cannot connect", async () => { - const dbConfig = await databaseTestProviders.postgres.datasource() + const dbConfig = await getDatasource(DatabaseName.POSTGRES) await config.api.datasource.verify( { datasource: { @@ -1089,21 +1088,21 @@ describe("postgres integrations", () => { describe("POST /api/datasources/:datasourceId/schema", () => { let client: Client + let tableName: string beforeEach(async () => { - client = new Client( - (await databaseTestProviders.postgres.datasource()).config! - ) + tableName = generator.guid().replaceAll("-", "").substring(0, 10) + client = new Client((await getDatasource(DatabaseName.POSTGRES)).config!) await client.connect() }) afterEach(async () => { - await client.query(`DROP TABLE IF EXISTS "table"`) + await client.query(`DROP TABLE IF EXISTS "${tableName}"`) await client.end() }) it("recognises when a table has no primary key", async () => { - await client.query(`CREATE TABLE "table" (id SERIAL)`) + await client.query(`CREATE TABLE "${tableName}" (id SERIAL)`) const response = await makeRequest( "post", @@ -1111,12 +1110,14 @@ describe("postgres integrations", () => { ) expect(response.body.errors).toEqual({ - table: "Table must have a primary key.", + [tableName]: "Table must have a primary key.", }) }) it("recognises when a table is using a reserved column name", async () => { - await client.query(`CREATE TABLE "table" (_id SERIAL PRIMARY KEY) `) + await client.query( + `CREATE TABLE "${tableName}" (_id SERIAL PRIMARY KEY) ` + ) const response = await makeRequest( "post", @@ -1124,18 +1125,22 @@ describe("postgres integrations", () => { ) expect(response.body.errors).toEqual({ - table: "Table contains invalid columns.", + [tableName]: "Table contains invalid columns.", }) }) }) describe("Integration compatibility with postgres search_path", () => { - let client: Client, pathDatasource: Datasource - const schema1 = "test1", - schema2 = "test-2" + let client: Client, + pathDatasource: Datasource, + schema1: string, + schema2: string - beforeAll(async () => { - const dsConfig = await databaseTestProviders.postgres.datasource() + beforeEach(async () => { + schema1 = generator.guid().replaceAll("-", "") + schema2 = generator.guid().replaceAll("-", "") + + const dsConfig = await getDatasource(DatabaseName.POSTGRES) const dbConfig = dsConfig.config! client = new Client(dbConfig) @@ -1153,7 +1158,7 @@ describe("postgres integrations", () => { pathDatasource = await config.api.datasource.create(pathConfig) }) - afterAll(async () => { + afterEach(async () => { await client.query(`DROP SCHEMA "${schema1}" CASCADE;`) await client.query(`DROP SCHEMA "${schema2}" CASCADE;`) await client.end() diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index b2be3df4e0..650a1b414d 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -1,19 +1,50 @@ jest.unmock("pg") import { Datasource } from "@budibase/types" -import * as postgres from "./postgres" -import * as mongodb from "./mongodb" -import * as mysql from "./mysql" -import * as mssql from "./mssql" -import * as mariadb from "./mariadb" -import { StartedTestContainer } from "testcontainers" +import { postgres } from "./postgres" +import { mongodb } from "./mongodb" +import { mysql } from "./mysql" +import { mssql } from "./mssql" +import { mariadb } from "./mariadb" -jest.setTimeout(30000) +export type DatasourceProvider = () => Promise -export interface DatabaseProvider { - start(): Promise - stop(): Promise - datasource(): Promise +export enum DatabaseName { + POSTGRES = "postgres", + MONGODB = "mongodb", + MYSQL = "mysql", + SQL_SERVER = "mssql", + MARIADB = "mariadb", +} + +const providers: Record = { + [DatabaseName.POSTGRES]: postgres, + [DatabaseName.MONGODB]: mongodb, + [DatabaseName.MYSQL]: mysql, + [DatabaseName.SQL_SERVER]: mssql, + [DatabaseName.MARIADB]: mariadb, +} + +export function getDatasourceProviders( + ...sourceNames: DatabaseName[] +): Promise[] { + return sourceNames.map(sourceName => providers[sourceName]()) +} + +export function getDatasourceProvider( + sourceName: DatabaseName +): DatasourceProvider { + return providers[sourceName] +} + +export function getDatasource(sourceName: DatabaseName): Promise { + return providers[sourceName]() +} + +export async function getDatasources( + ...sourceNames: DatabaseName[] +): Promise { + return Promise.all(sourceNames.map(sourceName => providers[sourceName]())) } export const databaseTestProviders = { diff --git a/packages/server/src/integrations/tests/utils/mariadb.ts b/packages/server/src/integrations/tests/utils/mariadb.ts index a097e0aaa1..a10c36f9ff 100644 --- a/packages/server/src/integrations/tests/utils/mariadb.ts +++ b/packages/server/src/integrations/tests/utils/mariadb.ts @@ -1,9 +1,7 @@ import { Datasource, SourceName } from "@budibase/types" -import { GenericContainer, Wait, StartedTestContainer } from "testcontainers" +import { GenericContainer, Wait } from "testcontainers" import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy" -let container: StartedTestContainer | undefined - class MariaDBWaitStrategy extends AbstractWaitStrategy { async waitUntilReady(container: any, boundPorts: any, startTime?: Date) { // Because MariaDB first starts itself up, runs an init script, then restarts, @@ -21,18 +19,15 @@ class MariaDBWaitStrategy extends AbstractWaitStrategy { } } -export async function start(): Promise { - return await new GenericContainer("mariadb:lts") +export async function mariadb(): Promise { + const container = await new GenericContainer("mariadb:lts") + .withName("budibase-test-mariadb") + .withReuse() .withExposedPorts(3306) .withEnvironment({ MARIADB_ROOT_PASSWORD: "password" }) .withWaitStrategy(new MariaDBWaitStrategy()) .start() -} -export async function datasource(): Promise { - if (!container) { - container = await start() - } const host = container.getHost() const port = container.getMappedPort(3306) @@ -49,10 +44,3 @@ export async function datasource(): Promise { }, } } - -export async function stop() { - if (container) { - await container.stop() - container = undefined - } -} diff --git a/packages/server/src/integrations/tests/utils/mongodb.ts b/packages/server/src/integrations/tests/utils/mongodb.ts index 0baafc6276..ff24bbc62e 100644 --- a/packages/server/src/integrations/tests/utils/mongodb.ts +++ b/packages/server/src/integrations/tests/utils/mongodb.ts @@ -1,10 +1,10 @@ import { Datasource, SourceName } from "@budibase/types" -import { GenericContainer, Wait, StartedTestContainer } from "testcontainers" +import { GenericContainer, Wait } from "testcontainers" -let container: StartedTestContainer | undefined - -export async function start(): Promise { - return await new GenericContainer("mongo:7.0-jammy") +export async function mongodb(): Promise { + const container = await new GenericContainer("mongo:7.0-jammy") + .withName("budibase-test-mongodb") + .withReuse() .withExposedPorts(27017) .withEnvironment({ MONGO_INITDB_ROOT_USERNAME: "mongo", @@ -16,14 +16,10 @@ export async function start(): Promise { ).withStartupTimeout(10000) ) .start() -} -export async function datasource(): Promise { - if (!container) { - container = await start() - } const host = container.getHost() const port = container.getMappedPort(27017) + return { type: "datasource", source: SourceName.MONGODB, @@ -34,10 +30,3 @@ export async function datasource(): Promise { }, } } - -export async function stop() { - if (container) { - await container.stop() - container = undefined - } -} diff --git a/packages/server/src/integrations/tests/utils/mssql.ts b/packages/server/src/integrations/tests/utils/mssql.ts index 6bd4290a90..0f4e290526 100644 --- a/packages/server/src/integrations/tests/utils/mssql.ts +++ b/packages/server/src/integrations/tests/utils/mssql.ts @@ -1,12 +1,12 @@ import { Datasource, SourceName } from "@budibase/types" -import { GenericContainer, Wait, StartedTestContainer } from "testcontainers" +import { GenericContainer, Wait } from "testcontainers" -let container: StartedTestContainer | undefined - -export async function start(): Promise { - return await new GenericContainer( +export async function mssql(): Promise { + const container = await new GenericContainer( "mcr.microsoft.com/mssql/server:2022-latest" ) + .withName("budibase-test-mssql") + .withReuse() .withExposedPorts(1433) .withEnvironment({ ACCEPT_EULA: "Y", @@ -23,12 +23,7 @@ export async function start(): Promise { ) ) .start() -} -export async function datasource(): Promise { - if (!container) { - container = await start() - } const host = container.getHost() const port = container.getMappedPort(1433) @@ -47,10 +42,3 @@ export async function datasource(): Promise { }, } } - -export async function stop() { - if (container) { - await container.stop() - container = undefined - } -} diff --git a/packages/server/src/integrations/tests/utils/mysql.ts b/packages/server/src/integrations/tests/utils/mysql.ts index 5e51478998..665d6f0ecf 100644 --- a/packages/server/src/integrations/tests/utils/mysql.ts +++ b/packages/server/src/integrations/tests/utils/mysql.ts @@ -1,9 +1,7 @@ import { Datasource, SourceName } from "@budibase/types" -import { GenericContainer, Wait, StartedTestContainer } from "testcontainers" +import { GenericContainer, Wait } from "testcontainers" import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy" -let container: StartedTestContainer | undefined - class MySQLWaitStrategy extends AbstractWaitStrategy { async waitUntilReady(container: any, boundPorts: any, startTime?: Date) { // Because MySQL first starts itself up, runs an init script, then restarts, @@ -24,18 +22,14 @@ class MySQLWaitStrategy extends AbstractWaitStrategy { } } -export async function start(): Promise { - return await new GenericContainer("mysql:8.3") +export async function mysql(): Promise { + const container = await new GenericContainer("mysql:8.3") + .withName("budibase-test-mysql") + .withReuse() .withExposedPorts(3306) .withEnvironment({ MYSQL_ROOT_PASSWORD: "password" }) .withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000)) .start() -} - -export async function datasource(): Promise { - if (!container) { - container = await start() - } const host = container.getHost() const port = container.getMappedPort(3306) @@ -52,10 +46,3 @@ export async function datasource(): Promise { }, } } - -export async function stop() { - if (container) { - await container.stop() - container = undefined - } -} diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts index 82a62e3916..896c7ea3e0 100644 --- a/packages/server/src/integrations/tests/utils/postgres.ts +++ b/packages/server/src/integrations/tests/utils/postgres.ts @@ -1,10 +1,10 @@ import { Datasource, SourceName } from "@budibase/types" -import { GenericContainer, Wait, StartedTestContainer } from "testcontainers" +import { GenericContainer, Wait } from "testcontainers" -let container: StartedTestContainer | undefined - -export async function start(): Promise { - return await new GenericContainer("postgres:16.1-bullseye") +export async function postgres(): Promise { + const container = await new GenericContainer("postgres:16.1-bullseye") + .withName("budibase-test-postgres") + .withReuse() .withExposedPorts(5432) .withEnvironment({ POSTGRES_PASSWORD: "password" }) .withWaitStrategy( @@ -13,12 +13,6 @@ export async function start(): Promise { ).withStartupTimeout(10000) ) .start() -} - -export async function datasource(): Promise { - if (!container) { - container = await start() - } const host = container.getHost() const port = container.getMappedPort(5432) @@ -39,10 +33,3 @@ export async function datasource(): Promise { }, } } - -export async function stop() { - if (container) { - await container.stop() - container = undefined - } -} From 5530d7f4b63b717c7879a6bfaf70387d35844aa8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Mar 2024 14:05:58 +0000 Subject: [PATCH 08/93] Migrate mongodb.spec.ts to new datasource providers. --- .../routes/tests/queries/generic-sql.spec.ts | 28 ++++++++++--------- .../api/routes/tests/queries/mongodb.spec.ts | 17 ++++++----- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index f9a3ac6e03..d393430060 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -1,6 +1,9 @@ import { Datasource, Query, SourceName } from "@budibase/types" import * as setup from "../utilities" -import { databaseTestProviders } from "../../../../integrations/tests/utils" +import { + DatabaseName, + getDatasource, +} from "../../../../integrations/tests/utils" import pg from "pg" import mysql from "mysql2/promise" import mssql from "mssql" @@ -34,12 +37,14 @@ const createTableSQL: Record = { const insertSQL = `INSERT INTO test_table (name) VALUES ('one'), ('two'), ('three'), ('four'), ('five')` const dropTableSQL = `DROP TABLE test_table;` -describe.each([ - ["postgres", databaseTestProviders.postgres], - ["mysql", databaseTestProviders.mysql], - ["mssql", databaseTestProviders.mssql], - ["mariadb", databaseTestProviders.mariadb], -])("queries (%s)", (dbName, dsProvider) => { +describe.each( + [ + DatabaseName.POSTGRES, + DatabaseName.MYSQL, + DatabaseName.SQL_SERVER, + DatabaseName.MARIADB, + ].map(name => [name, getDatasource(name)]) +)("queries (%s)", (dbName, dsProvider) => { const config = setup.getConfig() let datasource: Datasource @@ -61,7 +66,7 @@ describe.each([ // We re-fetch the datasource here because the one returned by // config.api.datasource.create has the password field blanked out, and we // need the password to connect to the database. - const ds = await dsProvider.datasource() + const ds = await dsProvider switch (ds.source) { case SourceName.POSTGRES: { const client = new pg.Client(ds.config!) @@ -97,9 +102,7 @@ describe.each([ beforeAll(async () => { await config.init() - datasource = await config.api.datasource.create( - await dsProvider.datasource() - ) + datasource = await config.api.datasource.create(await dsProvider) }) beforeEach(async () => { @@ -112,7 +115,6 @@ describe.each([ }) afterAll(async () => { - await dsProvider.stop() setup.afterAll() }) @@ -443,7 +445,7 @@ describe.each([ } catch (err: any) { error = err.message } - if (dbName === "mssql") { + if (dbName === DatabaseName.SQL_SERVER) { expect(error).toBeUndefined() } else { expect(error).toBeDefined() diff --git a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts index 492f24abf9..148f2c15ec 100644 --- a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts +++ b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts @@ -1,14 +1,17 @@ import { Datasource, Query } from "@budibase/types" import * as setup from "../utilities" -import { databaseTestProviders } from "../../../../integrations/tests/utils" +import { + DatabaseName, + getDatasource, +} from "../../../../integrations/tests/utils" import { MongoClient, type Collection, BSON } from "mongodb" - -const collection = "test_collection" +import { generator } from "@budibase/backend-core/tests" const expectValidId = expect.stringMatching(/^\w{24}$/) const expectValidBsonObjectId = expect.any(BSON.ObjectId) describe("/queries", () => { + let collection: string let config = setup.getConfig() let datasource: Datasource @@ -37,7 +40,7 @@ describe("/queries", () => { async function withClient( callback: (client: MongoClient) => Promise ): Promise { - const ds = await databaseTestProviders.mongodb.datasource() + const ds = await getDatasource(DatabaseName.MONGODB) const client = new MongoClient(ds.config!.connectionString) await client.connect() try { @@ -52,25 +55,25 @@ describe("/queries", () => { ): Promise { return await withClient(async client => { const db = client.db( - (await databaseTestProviders.mongodb.datasource()).config!.db + (await getDatasource(DatabaseName.MONGODB)).config!.db ) return await callback(db.collection(collection)) }) } afterAll(async () => { - await databaseTestProviders.mongodb.stop() setup.afterAll() }) beforeAll(async () => { await config.init() datasource = await config.api.datasource.create( - await databaseTestProviders.mongodb.datasource() + await getDatasource(DatabaseName.MONGODB) ) }) beforeEach(async () => { + collection = generator.guid() await withCollection(async collection => { await collection.insertMany([ { name: "one" }, From 2304aeaa719937130fbdd07cee6580414b2f72af Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Mar 2024 14:36:18 +0000 Subject: [PATCH 09/93] Migrate mysql.spec.ts to new datasource providers. --- .../server/src/api/routes/tests/row.spec.ts | 39 +++++++++---------- .../server/src/integration-test/mysql.spec.ts | 30 +++++++------- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index f638f2c4bf..8910522565 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1,4 +1,4 @@ -import { databaseTestProviders } from "../../../integrations/tests/utils" +import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" import tk from "timekeeper" import { outputProcessing } from "../../../utilities/rowProcessor" @@ -34,10 +34,10 @@ jest.unmock("pg") describe.each([ ["internal", undefined], - ["postgres", databaseTestProviders.postgres], - ["mysql", databaseTestProviders.mysql], - ["mssql", databaseTestProviders.mssql], - ["mariadb", databaseTestProviders.mariadb], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/rows (%s)", (__, dsProvider) => { const isInternal = dsProvider === undefined const config = setup.getConfig() @@ -49,23 +49,23 @@ describe.each([ await config.init() if (dsProvider) { datasource = await config.createDatasource({ - datasource: await dsProvider.datasource(), + datasource: await dsProvider, }) } }) afterAll(async () => { - if (dsProvider) { - await dsProvider.stop() - } setup.afterAll() }) function saveTableRequest( - ...overrides: Partial[] + // We omit the name field here because it's generated in the function with a + // high likelihood to be unique. Tests should not have any reason to control + // the table name they're writing to. + ...overrides: Partial>[] ): SaveTableRequest { const req: SaveTableRequest = { - name: uuid.v4().substring(0, 16), + name: uuid.v4().substring(0, 10), type: "table", sourceType: datasource ? TableSourceType.EXTERNAL @@ -87,7 +87,10 @@ describe.each([ } function defaultTable( - ...overrides: Partial[] + // We omit the name field here because it's generated in the function with a + // high likelihood to be unique. Tests should not have any reason to control + // the table name they're writing to. + ...overrides: Partial>[] ): SaveTableRequest { return saveTableRequest( { @@ -194,7 +197,6 @@ describe.each([ const newTable = await config.api.table.save( saveTableRequest({ - name: "TestTableAuto", schema: { "Row ID": { name: "Row ID", @@ -383,11 +385,9 @@ describe.each([ isInternal && it("doesn't allow creating in user table", async () => { - const userTableId = InternalTable.USER_METADATA const response = await config.api.row.save( - userTableId, + InternalTable.USER_METADATA, { - tableId: userTableId, firstName: "Joe", lastName: "Joe", email: "joe@joe.com", @@ -462,7 +462,6 @@ describe.each([ table = await config.api.table.save(defaultTable()) otherTable = await config.api.table.save( defaultTable({ - name: "a", schema: { relationship: { name: "relationship", @@ -898,8 +897,8 @@ describe.each([ let o2mTable: Table let m2mTable: Table beforeAll(async () => { - o2mTable = await config.api.table.save(defaultTable({ name: "o2m" })) - m2mTable = await config.api.table.save(defaultTable({ name: "m2m" })) + o2mTable = await config.api.table.save(defaultTable()) + m2mTable = await config.api.table.save(defaultTable()) }) describe.each([ @@ -1256,7 +1255,6 @@ describe.each([ otherTable = await config.api.table.save(defaultTable()) table = await config.api.table.save( saveTableRequest({ - name: "b", schema: { links: { name: "links", @@ -1354,7 +1352,6 @@ describe.each([ const table = await config.api.table.save( saveTableRequest({ - name: "table", schema: { text: { name: "text", diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index 92420fb336..65fbe2949d 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -3,7 +3,6 @@ import { generateMakeRequest, MakeRequestResponse, } from "../api/routes/public/tests/utils" -import { v4 as uuidv4 } from "uuid" import * as setup from "../api/routes/tests/utilities" import { Datasource, @@ -12,9 +11,10 @@ import { TableRequest, TableSourceType, } from "@budibase/types" -import { databaseTestProviders } from "../integrations/tests/utils" +import { DatabaseName, getDatasource } from "../integrations/tests/utils" import mysql from "mysql2/promise" import { builderSocket } from "../websockets" +import { generator } from "@budibase/backend-core/tests" // @ts-ignore fetch.mockSearch() @@ -47,17 +47,13 @@ describe("mysql integrations", () => { makeRequest = generateMakeRequest(apiKey, true) mysqlDatasource = await config.api.datasource.create( - await databaseTestProviders.mysql.datasource() + await getDatasource(DatabaseName.MYSQL) ) }) - afterAll(async () => { - await databaseTestProviders.mysql.stop() - }) - beforeEach(async () => { primaryMySqlTable = await config.createTable({ - name: uuidv4(), + name: generator.guid().replaceAll("-", "_").substring(0, 10), type: "table", primary: ["id"], schema: { @@ -117,7 +113,7 @@ describe("mysql integrations", () => { it("should be able to verify the connection", async () => { await config.api.datasource.verify( { - datasource: await databaseTestProviders.mysql.datasource(), + datasource: await getDatasource(DatabaseName.MYSQL), }, { body: { @@ -128,7 +124,7 @@ describe("mysql integrations", () => { }) it("should state an invalid datasource cannot connect", async () => { - const dbConfig = await databaseTestProviders.mysql.datasource() + const dbConfig = await getDatasource(DatabaseName.MYSQL) await config.api.datasource.verify( { datasource: { @@ -168,7 +164,7 @@ describe("mysql integrations", () => { const database2 = "test-2" beforeAll(async () => { - const dsConfig = await databaseTestProviders.mysql.datasource() + const dsConfig = await getDatasource(DatabaseName.MYSQL) const dbConfig = dsConfig.config! client = await mysql.createConnection(dbConfig) @@ -237,11 +233,11 @@ describe("mysql integrations", () => { beforeEach(async () => { client = await mysql.createConnection( ( - await databaseTestProviders.mysql.datasource() + await getDatasource(DatabaseName.MYSQL) ).config! ) mysqlDatasource = await config.api.datasource.create( - await databaseTestProviders.mysql.datasource() + await getDatasource(DatabaseName.MYSQL) ) }) @@ -253,7 +249,7 @@ describe("mysql integrations", () => { const addColumnToTable: TableRequest = { type: "table", sourceType: TableSourceType.EXTERNAL, - name: "table", + name: generator.guid().replaceAll("-", "_").substring(0, 10), sourceId: mysqlDatasource._id!, primary: ["id"], schema: { @@ -301,14 +297,16 @@ describe("mysql integrations", () => { }, }, created: true, - _id: `${mysqlDatasource._id}__table`, + _id: `${mysqlDatasource._id}__${addColumnToTable.name}`, } delete expectedTable._add expect(emitDatasourceUpdateMock).toHaveBeenCalledTimes(1) const emittedDatasource: Datasource = emitDatasourceUpdateMock.mock.calls[0][1] - expect(emittedDatasource.entities!["table"]).toEqual(expectedTable) + expect(emittedDatasource.entities![expectedTable.name]).toEqual( + expectedTable + ) }) it("will rename a column", async () => { From b84bbd6003bf1317a5fe4e6430c56c30ad4dfead Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Mar 2024 14:43:17 +0000 Subject: [PATCH 10/93] Migrate viewV2.spec.ts to new datasource providers. --- .../src/api/routes/tests/viewV2.spec.ts | 24 +++++++------------ .../src/integrations/tests/utils/index.ts | 8 ------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index f9d213a26b..ce959ac429 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -19,8 +19,7 @@ import { ViewV2, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" -import * as uuid from "uuid" -import { databaseTestProviders } from "../../../integrations/tests/utils" +import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" import merge from "lodash/merge" import { quotas } from "@budibase/pro" import { roles } from "@budibase/backend-core" @@ -30,10 +29,10 @@ jest.unmock("pg") describe.each([ ["internal", undefined], - ["postgres", databaseTestProviders.postgres], - ["mysql", databaseTestProviders.mysql], - ["mssql", databaseTestProviders.mssql], - ["mariadb", databaseTestProviders.mariadb], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/v2/views (%s)", (_, dsProvider) => { const config = setup.getConfig() const isInternal = !dsProvider @@ -42,10 +41,10 @@ describe.each([ let datasource: Datasource function saveTableRequest( - ...overrides: Partial[] + ...overrides: Partial>[] ): SaveTableRequest { const req: SaveTableRequest = { - name: uuid.v4().substring(0, 16), + name: generator.guid().replaceAll("-", "").substring(0, 16), type: "table", sourceType: datasource ? TableSourceType.EXTERNAL @@ -90,16 +89,13 @@ describe.each([ if (dsProvider) { datasource = await config.createDatasource({ - datasource: await dsProvider.datasource(), + datasource: await dsProvider, }) } table = await config.api.table.save(priceTable()) }) afterAll(async () => { - if (dsProvider) { - await dsProvider.stop() - } setup.afterAll() }) @@ -507,7 +503,6 @@ describe.each([ it("views have extra data trimmed", async () => { const table = await config.api.table.save( saveTableRequest({ - name: "orders", schema: { Country: { type: FieldType.STRING, @@ -523,7 +518,7 @@ describe.each([ const view = await config.api.viewV2.create({ tableId: table._id!, - name: uuid.v4(), + name: generator.guid(), schema: { Country: { visible: true, @@ -853,7 +848,6 @@ describe.each([ beforeAll(async () => { table = await config.api.table.save( saveTableRequest({ - name: `users_${uuid.v4()}`, type: "table", schema: { name: { diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 650a1b414d..57aae02865 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -46,11 +46,3 @@ export async function getDatasources( ): Promise { return Promise.all(sourceNames.map(sourceName => providers[sourceName]())) } - -export const databaseTestProviders = { - postgres, - mongodb, - mysql, - mssql, - mariadb, -} From 84ffaa7022c3ef7de77a40ca2aff1572e1d05026 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 26 Mar 2024 14:47:14 +0000 Subject: [PATCH 11/93] Fix additional scrollbar appearing when hiding binding panels --- .../builder/src/components/common/bindings/BindingPanel.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte index 21d389357f..10d95a3e7e 100644 --- a/packages/builder/src/components/common/bindings/BindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte @@ -371,6 +371,7 @@