From 66789888ace6c636aa63e1db930d7f72ed83bfe1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Jun 2024 11:23:52 +0100 Subject: [PATCH 1/8] Caching is making testing difficult - want to confirm app migrations are doing what they are expected to do --- packages/server/src/appMigrations/appMigrationMetadata.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts index 613f46cf3d..54edd826c3 100644 --- a/packages/server/src/appMigrations/appMigrationMetadata.ts +++ b/packages/server/src/appMigrations/appMigrationMetadata.ts @@ -26,9 +26,9 @@ export async function getAppMigrationVersion(appId: string): Promise { let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey) // We don't want to cache in dev, in order to be able to tweak it - if (metadata && !env.isDev()) { - return metadata.version - } + // if (metadata && !env.isDev()) { + // return metadata.version + // } let version try { From 334334bbcdd5057fff4edc11bccbebc74dc2cd39 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Jun 2024 12:13:18 +0100 Subject: [PATCH 2/8] Adding a check for QA. --- packages/backend-core/src/environment.ts | 6 ++++++ packages/server/src/appMigrations/appMigrationMetadata.ts | 8 ++++---- packages/server/src/startup/index.ts | 6 +++++- packages/worker/src/environment.ts | 1 + packages/worker/src/index.ts | 6 +++++- 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 1e7da2f9a2..c84a162ab6 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -93,15 +93,21 @@ function isApps() { return environment.SERVICE_TYPE === ServiceType.APPS } +function isQA() { + return environment.BUDIBASE_ENVIRONMENT === "QA" +} + const environment = { isTest, isJest, isDev, isWorker, isApps, + isQA, isProd: () => { return !isDev() }, + BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, JS_BCRYPT: process.env.JS_BCRYPT, JWT_SECRET: process.env.JWT_SECRET, JWT_SECRET_FALLBACK: process.env.JWT_SECRET_FALLBACK, diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts index 54edd826c3..a9a75bc104 100644 --- a/packages/server/src/appMigrations/appMigrationMetadata.ts +++ b/packages/server/src/appMigrations/appMigrationMetadata.ts @@ -25,10 +25,10 @@ export async function getAppMigrationVersion(appId: string): Promise { let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey) - // We don't want to cache in dev, in order to be able to tweak it - // if (metadata && !env.isDev()) { - // return metadata.version - // } + // We don't want to cache in dev or QA, in order to be able to tweak it + if (metadata && !env.isDev() && !env.isQA()) { + return metadata.version + } let version try { diff --git a/packages/server/src/startup/index.ts b/packages/server/src/startup/index.ts index c14ec0ca1b..7f244f6573 100644 --- a/packages/server/src/startup/index.ts +++ b/packages/server/src/startup/index.ts @@ -72,7 +72,11 @@ export async function startup( printFeatures() STARTUP_RAN = true if (app && server && !env.CLUSTER_MODE) { - console.log(`Budibase running on ${JSON.stringify(server.address())}`) + let startupLog = `Budibase running on ${JSON.stringify(server.address())}` + if (env.BUDIBASE_ENVIRONMENT) { + startupLog = `${startupLog} - environment: "${env.BUDIBASE_ENVIRONMENT}"` + } + console.log(startupLog) const address = server.address() as AddressInfo env._set("PORT", address.port) } diff --git a/packages/worker/src/environment.ts b/packages/worker/src/environment.ts index 70fb911ee1..d0f5e1fb67 100644 --- a/packages/worker/src/environment.ts +++ b/packages/worker/src/environment.ts @@ -46,6 +46,7 @@ const environment = { SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED, DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE, + BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, // smtp SMTP_USER: process.env.SMTP_USER, SMTP_PASSWORD: process.env.SMTP_PASSWORD, diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 4e770c6ecb..85e5d6ad2e 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -88,7 +88,11 @@ const shutdown = () => { } export default server.listen(parseInt(env.PORT || "4002"), async () => { - console.log(`Worker running on ${JSON.stringify(server.address())}`) + let startupLog = `Worker running on ${JSON.stringify(server.address())}` + if (env.BUDIBASE_ENVIRONMENT) { + startupLog = `${startupLog} - environment: "${env.BUDIBASE_ENVIRONMENT}"` + } + console.log(startupLog) await initPro() await redis.clients.init() cache.docWritethrough.init() From ae74c2498c23a1152b9b76d25717bc74c31ea75f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 11 Jun 2024 14:38:56 +0100 Subject: [PATCH 3/8] small change to ensure that name of datasources is available in dropdown to prevent dupes --- .../controls/DataSourceSelect/DataSourceCategory.svelte | 4 ++-- .../controls/DataSourceSelect/DataSourceSelect.svelte | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte index 71de1716f2..6d0e5ef9e7 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte @@ -28,8 +28,8 @@ tabindex="0" on:click={() => onSelect(data)} > - - {data.label} + + {data.datasource?.name ? `${data.datasource.name} -` : ""} {data.label} ds._id === m.sourceId || m.datasourceId), })) $: viewsV1 = $viewsStore.list.map(view => ({ ...view, From f8f05a59d46869dc22f93a7917d068ab948a7046 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 11 Jun 2024 14:38:56 +0100 Subject: [PATCH 4/8] small change to ensure that name of datasources is available in dropdown to prevent dupes --- .../controls/DataSourceSelect/DataSourceCategory.svelte | 2 +- .../settings/controls/DataSourceSelect/DataSourceSelect.svelte | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte index 71de1716f2..8e4ce3b4c3 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte @@ -29,7 +29,7 @@ on:click={() => onSelect(data)} > - {data.label} + {data.datasource?.name ? `${data.datasource.name} -` : ""} {data.label} ds._id === m.sourceId || m.datasourceId), })) $: viewsV1 = $viewsStore.list.map(view => ({ ...view, From 2b2079b4f1233ca8415849f8a9c1ebf18e2cf0c1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Jun 2024 15:45:07 +0100 Subject: [PATCH 5/8] Moving app migrations back to API service. --- .../src/appMigrations/appMigrationMetadata.ts | 4 ++-- packages/server/src/startup/index.ts | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts index a9a75bc104..176cd0ff84 100644 --- a/packages/server/src/appMigrations/appMigrationMetadata.ts +++ b/packages/server/src/appMigrations/appMigrationMetadata.ts @@ -25,8 +25,8 @@ export async function getAppMigrationVersion(appId: string): Promise { let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey) - // We don't want to cache in dev or QA, in order to be able to tweak it - if (metadata && !env.isDev() && !env.isQA()) { + // We don't want to cache in dev in order to be able to tweak it + if (metadata && !env.isDev()) { return metadata.version } diff --git a/packages/server/src/startup/index.ts b/packages/server/src/startup/index.ts index 7f244f6573..0e08075698 100644 --- a/packages/server/src/startup/index.ts +++ b/packages/server/src/startup/index.ts @@ -20,7 +20,7 @@ import * as pro from "@budibase/pro" import * as api from "../api" import sdk from "../sdk" import { initialise as initialiseWebsockets } from "../websockets" -import { automationsEnabled, printFeatures } from "../features" +import { apiEnabled, automationsEnabled, printFeatures } from "../features" import * as jsRunner from "../jsRunner" import Koa from "koa" import { Server } from "http" @@ -70,13 +70,12 @@ export async function startup( return } printFeatures() + if (env.BUDIBASE_ENVIRONMENT) { + console.log(`service running environment: "${env.BUDIBASE_ENVIRONMENT}"`) + } STARTUP_RAN = true if (app && server && !env.CLUSTER_MODE) { - let startupLog = `Budibase running on ${JSON.stringify(server.address())}` - if (env.BUDIBASE_ENVIRONMENT) { - startupLog = `${startupLog} - environment: "${env.BUDIBASE_ENVIRONMENT}"` - } - console.log(startupLog) + console.log(`Budibase running on ${JSON.stringify(server.address())}`) const address = server.address() as AddressInfo env._set("PORT", address.port) } @@ -121,9 +120,11 @@ export async function startup( queuePromises.push(events.processors.init(pro.sdk.auditLogs.write)) // app migrations and automations on other service if (automationsEnabled()) { - queuePromises.push(appMigrations.init()) queuePromises.push(automations.init()) } + if (apiEnabled()) { + queuePromises.push(appMigrations.init()) + } queuePromises.push(initPro()) if (app) { // bring routes online as final step once everything ready From c2415012c26edf83306cc301e72291ffd67df287 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Jun 2024 16:02:38 +0100 Subject: [PATCH 6/8] Keep QA check for no caching of app migration info. --- packages/server/src/appMigrations/appMigrationMetadata.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts index 176cd0ff84..971dde1e7a 100644 --- a/packages/server/src/appMigrations/appMigrationMetadata.ts +++ b/packages/server/src/appMigrations/appMigrationMetadata.ts @@ -25,8 +25,8 @@ export async function getAppMigrationVersion(appId: string): Promise { let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey) - // We don't want to cache in dev in order to be able to tweak it - if (metadata && !env.isDev()) { + // We don't want to cache in dev or QA in order to be able to tweak it + if (metadata && !env.isDev() && !env.isQA()) { return metadata.version } From 9fcefb18b4adfc9da33d7a79d19ced2761e1f996 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 11 Jun 2024 16:09:13 +0100 Subject: [PATCH 7/8] Update packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte Co-authored-by: Andrew Kingston --- .../controls/DataSourceSelect/DataSourceCategory.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte index 8e4ce3b4c3..fa6f477ed9 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceCategory.svelte @@ -29,7 +29,7 @@ on:click={() => onSelect(data)} > - {data.datasource?.name ? `${data.datasource.name} -` : ""} {data.label} + {data.datasource?.name ? `${data.datasource.name} - ` : ""}{data.label} Date: Tue, 11 Jun 2024 16:56:35 +0100 Subject: [PATCH 8/8] Fix lint on master --- .../controls/DataSourceSelect/DataSourceSelect.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index a3ea677df9..d2b5174139 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -55,7 +55,9 @@ label: m.name, tableId: m._id, type: "table", - datasource: $datasources.list.find(ds => ds._id === m.sourceId || m.datasourceId), + datasource: $datasources.list.find( + ds => ds._id === m.sourceId || m.datasourceId + ), })) $: viewsV1 = $viewsStore.list.map(view => ({ ...view,