From 86717b536b1785b391a42dff390ee29ee417f05a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 14:32:54 +0100 Subject: [PATCH 01/24] Use new feature flag API for SQS. --- .../templates/app-service-deployment.yaml | 4 - .../templates/worker-service-deployment.yaml | 4 - hosting/docker-compose.build.yaml | 4 +- .../backend-core/src/db/couch/DatabaseImpl.ts | 7 +- packages/backend-core/src/db/utils.ts | 33 +---- packages/backend-core/src/environment.ts | 4 - packages/backend-core/src/features/index.ts | 3 +- packages/server/scripts/dev/manage.js | 2 +- .../server/src/api/controllers/datasource.ts | 2 + .../server/src/api/controllers/row/views.ts | 7 +- .../server/src/api/controllers/table/index.ts | 3 + .../server/src/api/controllers/table/utils.ts | 6 +- .../src/api/routes/tests/application.spec.ts | 4 +- .../src/api/routes/tests/search.spec.ts | 5 +- .../src/api/routes/tests/templates.spec.ts | 7 +- .../src/api/routes/tests/viewV2.spec.ts | 5 +- .../server/src/appMigrations/migrations.ts | 2 - .../migrations/20240604153647_initial_sqs.ts | 12 +- .../tests/20240604153647_initial_sqs.spec.ts | 119 +++++++++--------- packages/server/src/sdk/app/rows/search.ts | 4 +- .../sdk/app/rows/search/tests/search.spec.ts | 5 +- packages/server/src/sdk/app/tables/getters.ts | 29 ++--- packages/server/src/websockets/builder.ts | 8 +- packages/worker/scripts/dev/manage.js | 2 +- .../src/api/controllers/system/environment.ts | 4 +- packages/worker/src/api/index.ts | 8 +- packages/worker/src/index.ts | 6 +- 27 files changed, 122 insertions(+), 177 deletions(-) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 6b4fbe5dda..b417fbd765 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -206,10 +206,6 @@ spec: - name: APP_FEATURES value: "api" {{- end }} - {{- if .Values.globals.sqs.enabled }} - - name: SQS_SEARCH_ENABLE - value: "true" - {{- end }} {{- range .Values.services.apps.extraEnv }} - name: {{ .name }} value: {{ .value | quote }} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index ecab8dc645..b07c7f02a7 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -192,10 +192,6 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} - {{- if .Values.globals.sqs.enabled }} - - name: SQS_SEARCH_ENABLE - value: "true" - {{- end }} {{- range .Values.services.worker.extraEnv }} - name: {{ .name }} value: {{ .value | quote }} diff --git a/hosting/docker-compose.build.yaml b/hosting/docker-compose.build.yaml index 253dda0232..1f16baa9e2 100644 --- a/hosting/docker-compose.build.yaml +++ b/hosting/docker-compose.build.yaml @@ -29,7 +29,7 @@ services: BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL} BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} PLUGINS_DIR: ${PLUGINS_DIR} - SQS_SEARCH_ENABLE: 1 + TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - worker-service - redis-service @@ -57,7 +57,7 @@ services: INTERNAL_API_KEY: ${INTERNAL_API_KEY} REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} - SQS_SEARCH_ENABLE: 1 + TENANT_FEATURE_FLAGS: "*:SQS" depends_on: - redis-service - minio-service diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 61fbb3d61e..35181b2ade 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -25,8 +25,8 @@ import { newid } from "../../docIds/newid" import { SQLITE_DESIGN_DOC_ID } from "../../constants" import { DDInstrumentedDatabase } from "../instrumentation" import { checkSlashesInUrl } from "../../helpers" -import env from "../../environment" import { sqlLog } from "../../sql/utils" +import { features } from "../.." const DATABASE_NOT_FOUND = "Database does not exist." @@ -401,7 +401,10 @@ export class DatabaseImpl implements Database { } async destroy() { - if (env.SQS_SEARCH_ENABLE && (await this.exists(SQLITE_DESIGN_DOC_ID))) { + if ( + (await features.flags.isEnabled("SQS")) && + (await this.exists(SQLITE_DESIGN_DOC_ID)) + ) { // delete the design document, then run the cleanup operation const definition = await this.get(SQLITE_DESIGN_DOC_ID) // remove all tables - save the definition then trigger a cleanup diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index a7ad453066..906a95e1db 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -1,6 +1,6 @@ import env from "../environment" import { DEFAULT_TENANT_ID, SEPARATOR, DocumentType } from "../constants" -import { getTenantId, getGlobalDBName, isMultiTenant } from "../context" +import { getTenantId, getGlobalDBName } from "../context" import { doWithDB, directCouchAllDbs } from "./db" import { AppState, DeletedApp, getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions" @@ -206,34 +206,3 @@ export function pagination( nextPage, } } - -export function isSqsEnabledForTenant(): boolean { - const tenantId = getTenantId() - if (!env.SQS_SEARCH_ENABLE) { - return false - } - - // single tenant (self host and dev) always enabled if flag set - if (!isMultiTenant()) { - return true - } - - // This is to guard against the situation in tests where tests pass because - // we're not actually using SQS, we're using Lucene and the tests pass due to - // parity. - if (env.isTest() && env.SQS_SEARCH_ENABLE_TENANTS.length === 0) { - throw new Error( - "to enable SQS you must specify a list of tenants in the SQS_SEARCH_ENABLE_TENANTS env var" - ) - } - - // Special case to enable all tenants, for testing in QA. - if ( - env.SQS_SEARCH_ENABLE_TENANTS.length === 1 && - env.SQS_SEARCH_ENABLE_TENANTS[0] === "*" - ) { - return true - } - - return env.SQS_SEARCH_ENABLE_TENANTS.includes(tenantId) -} diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 6377d682d0..3d931a8c67 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -116,10 +116,6 @@ const environment = { API_ENCRYPTION_KEY: getAPIEncryptionKey(), COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL, - SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE, - SQS_SEARCH_ENABLE_TENANTS: - process.env.SQS_SEARCH_ENABLE_TENANTS?.split(",") || [], - SQS_MIGRATION_ENABLE: process.env.SQS_MIGRATION_ENABLE, COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index 0063830417..d1d757d8a9 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -6,7 +6,7 @@ import tracer from "dd-trace" let posthog: PostHog | undefined export function init(opts?: PostHogOptions) { - if (env.POSTHOG_TOKEN && env.POSTHOG_API_HOST) { + if (env.POSTHOG_TOKEN && env.POSTHOG_API_HOST && !env.SELF_HOSTED) { console.log("initializing posthog client...") posthog = new PostHog(env.POSTHOG_TOKEN, { host: env.POSTHOG_API_HOST, @@ -266,4 +266,5 @@ export class FlagSet, T extends { [key: string]: V }> { // default values set correctly and their types flow through the system. export const flags = new FlagSet({ DEFAULT_VALUES: Flag.boolean(false), + SQS: Flag.boolean(false), }) diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index c36bfcc738..edcc4a5edb 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -47,7 +47,7 @@ async function init() { HTTP_LOGGING: "0", VERSION: "0.0.0+local", PASSWORD_MIN_LENGTH: "1", - SQS_SEARCH_ENABLE: "1", + TENANT_FEATURE_FLAGS: "*:SQS", } config = { ...config, ...existingConfig } diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 243d0a17a0..d5aec678fb 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -27,6 +27,7 @@ import { import sdk from "../../sdk" import { builderSocket } from "../../websockets" import { isEqual } from "lodash" +import { processTable } from "src/sdk/app/tables/getters" export async function fetch(ctx: UserCtx) { ctx.body = await sdk.datasources.fetch() @@ -188,6 +189,7 @@ export async function update( for (let table of Object.values(datasource.entities)) { const oldTable = baseDatasource.entities?.[table.name] if (!oldTable || !isEqual(oldTable, table)) { + table = await processTable(table) builderSocket?.emitTableUpdate(ctx, table, { includeOriginator: true }) } } diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index c38a415aa2..be051677b5 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -10,7 +10,7 @@ import { } from "@budibase/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" -import { db, context } from "@budibase/backend-core" +import { db, context, features } from "@budibase/backend-core" import { enrichSearchContext } from "./utils" import { isExternalTableID } from "../../../integrations/utils" @@ -41,7 +41,10 @@ export async function searchView( delete body.query.allOr delete body.query.onEmptyFilter - if (!isExternalTableID(view.tableId) && !db.isSqsEnabledForTenant()) { + if ( + !isExternalTableID(view.tableId) && + (await features.flags.isEnabled("SQS")) + ) { // Extract existing fields const existingFields = view.query diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index ee6947cec8..6e1b69b7ac 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -39,6 +39,7 @@ import { PROTECTED_EXTERNAL_COLUMNS, PROTECTED_INTERNAL_COLUMNS, } from "@budibase/shared-core" +import { processTable } from "../../../sdk/app/tables/getters" function pickApi({ tableId, table }: { tableId?: string; table?: Table }) { if (table && isExternalTable(table)) { @@ -118,6 +119,8 @@ export async function save(ctx: UserCtx) { ctx.eventEmitter && ctx.eventEmitter.emitTable(`table:save`, appId, { ...savedTable }) ctx.body = savedTable + + savedTable = await processTable(savedTable) builderSocket?.emitTableUpdate(ctx, cloneDeep(savedTable)) } diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 0e0a83e3b3..1b241adbd7 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -15,7 +15,7 @@ import { getViews, saveView } from "../view/utils" import viewTemplate from "../view/viewBuilder" import { cloneDeep } from "lodash/fp" import { quotas } from "@budibase/pro" -import { events, context, db as dbCore } from "@budibase/backend-core" +import { events, context, db as dbCore, features } from "@budibase/backend-core" import { AutoFieldSubType, ContextUser, @@ -332,7 +332,7 @@ class TableSaveFunctions { importRows: this.importRows, user: this.user, }) - if (dbCore.isSqsEnabledForTenant()) { + if (await features.flags.isEnabled("SQS")) { await sdk.tables.sqs.addTable(table) } return table @@ -526,7 +526,7 @@ export async function internalTableCleanup(table: Table, rows?: Row[]) { if (rows) { await AttachmentCleanup.tableDelete(table, rows) } - if (dbCore.isSqsEnabledForTenant()) { + if (await features.flags.isEnabled("SQS")) { await sdk.tables.sqs.removeTable(table) } } diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts index a8e060a2b5..40ed828dce 100644 --- a/packages/server/src/api/routes/tests/application.spec.ts +++ b/packages/server/src/api/routes/tests/application.spec.ts @@ -352,13 +352,13 @@ describe("/applications", () => { expect(events.app.unpublished).toHaveBeenCalledTimes(1) }) - it("should be able to delete an app after SQS_SEARCH_ENABLE has been set but app hasn't been migrated", async () => { + it("should be able to delete an app after SQS has been set but app hasn't been migrated", async () => { const prodAppId = app.appId.replace("_dev", "") nock("http://localhost:10000") .delete(`/api/global/roles/${prodAppId}`) .reply(200, {}) - await withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, async () => { + await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, async () => { await config.api.application.delete(app.appId) }) }) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 9de97747e5..9675cc7271 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -67,11 +67,10 @@ describe.each([ let rows: Row[] beforeAll(async () => { - await withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, () => config.init()) + await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init()) if (isSqs) { envCleanup = setCoreEnv({ - SQS_SEARCH_ENABLE: "true", - SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()], + TENANT_FEATURE_FLAGS: "*:SQS", }) } diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts index 98d375dec6..6f4d468a68 100644 --- a/packages/server/src/api/routes/tests/templates.spec.ts +++ b/packages/server/src/api/routes/tests/templates.spec.ts @@ -2,7 +2,7 @@ import * as setup from "./utilities" import path from "path" import nock from "nock" import { generator } from "@budibase/backend-core/tests" -import { withEnv as withCoreEnv } from "@budibase/backend-core" +import { withEnv as withCoreEnv, env as coreEnv } from "@budibase/backend-core" interface App { background: string @@ -85,9 +85,8 @@ describe("/templates", () => { it.each(["sqs", "lucene"])( `should be able to create an app from a template (%s)`, async source => { - const env = { - SQS_SEARCH_ENABLE: source === "sqs" ? "true" : "false", - SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()], + const env: Partial = { + TENANT_FEATURE_FLAGS: source === "sqs" ? "*:SQS" : "", } await withCoreEnv(env, async () => { diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index e18cffeaa8..4abd0f9b4c 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -94,13 +94,12 @@ describe.each([ } beforeAll(async () => { - await withCoreEnv({ SQS_SEARCH_ENABLE: isSqs ? "true" : "false" }, () => + await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () => config.init() ) if (isSqs) { envCleanup = setCoreEnv({ - SQS_SEARCH_ENABLE: "true", - SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()], + TENANT_FEATURE_FLAGS: "*:SQS", }) } diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts index 247b74ebfc..fc55f87d00 100644 --- a/packages/server/src/appMigrations/migrations.ts +++ b/packages/server/src/appMigrations/migrations.ts @@ -1,6 +1,5 @@ // This file should never be manually modified, use `yarn add-app-migration` in order to add a new one -import { env } from "@budibase/backend-core" import { AppMigration } from "." import m20240604153647_initial_sqs from "./migrations/20240604153647_initial_sqs" @@ -10,6 +9,5 @@ export const MIGRATIONS: AppMigration[] = [ { id: "20240604153647_initial_sqs", func: m20240604153647_initial_sqs, - disabled: !(env.SQS_MIGRATION_ENABLE || env.SQS_SEARCH_ENABLE), }, ] diff --git a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts index ec7fba99e9..a63de1e29e 100644 --- a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts +++ b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts @@ -1,4 +1,4 @@ -import { context, env } from "@budibase/backend-core" +import { context } from "@budibase/backend-core" import { allLinkDocs } from "../../db/utils" import LinkDocumentImpl from "../../db/linkedRows/LinkDocument" import sdk from "../../sdk" @@ -36,16 +36,6 @@ const migration = async () => { // at the end make sure design doc is ready await sdk.tables.sqs.syncDefinition() - // only do initial search if environment is using SQS already - // initial search makes sure that all the indexes have been created - // and are ready to use, avoiding any initial waits for large tables - if (env.SQS_MIGRATION_ENABLE || env.SQS_SEARCH_ENABLE) { - const tables = await sdk.tables.getAllInternalTables() - // do these one by one - running in parallel could cause problems - for (let table of tables) { - await db.sql(`select * from ${table._id} limit 1`) - } - } } export default migration diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts index 5f036bb87d..654b001f61 100644 --- a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts +++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts @@ -70,72 +70,65 @@ function oldLinkDocument(): Omit { } } -type SQSEnvVar = "SQS_MIGRATION_ENABLE" | "SQS_SEARCH_ENABLE" - -async function sqsDisabled(envVar: SQSEnvVar, cb: () => Promise) { - await withCoreEnv({ [envVar]: "", SQS_SEARCH_ENABLE_TENANTS: [] }, cb) +async function sqsDisabled(cb: () => Promise) { + await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" }, cb) } -async function sqsEnabled(envVar: SQSEnvVar, cb: () => Promise) { - await withCoreEnv( - { [envVar]: "1", SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()] }, - cb - ) +async function sqsEnabled(cb: () => Promise) { + await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, cb) } -describe.each(["SQS_MIGRATION_ENABLE", "SQS_SEARCH_ENABLE"] as SQSEnvVar[])( - "SQS migration with (%s)", - envVar => { - beforeAll(async () => { - await sqsDisabled(envVar, async () => { - await config.init() - const table = await config.api.table.save(basicTable()) - tableId = table._id! - const db = dbCore.getDB(config.appId!) - // old link document - await db.put(oldLinkDocument()) - }) - }) - - it("test migration runs as expected against an older DB", async () => { +describe("SQS migration", () => { + beforeAll(async () => { + await sqsDisabled(async () => { + await config.init() + const table = await config.api.table.save(basicTable()) + tableId = table._id! const db = dbCore.getDB(config.appId!) - // confirm nothing exists initially - await sqsDisabled(envVar, async () => { - let error: any | undefined - try { - await db.get(SQLITE_DESIGN_DOC_ID) - } catch (err: any) { - error = err - } - expect(error).toBeDefined() - expect(error.status).toBe(404) - }) - await sqsEnabled(envVar, async () => { - await processMigrations(config.appId!, MIGRATIONS) - const designDoc = await db.get(SQLITE_DESIGN_DOC_ID) - expect(designDoc.sql.tables).toBeDefined() - const mainTableDef = designDoc.sql.tables[tableId] - expect(mainTableDef).toBeDefined() - expect(mainTableDef.fields[prefix("name")]).toEqual({ - field: "name", - type: SQLiteType.TEXT, - }) - expect(mainTableDef.fields[prefix("description")]).toEqual({ - field: "description", - type: SQLiteType.TEXT, - }) - - const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo() - const linkDoc = await db.get(oldLinkDocID()) - expect(linkDoc.tableId).toEqual( - generateJunctionTableID(tableId1, tableId2) - ) - // should have swapped the documents - expect(linkDoc.doc1.tableId).toEqual(tableId2) - expect(linkDoc.doc1.rowId).toEqual(rowId2) - expect(linkDoc.doc2.tableId).toEqual(tableId1) - expect(linkDoc.doc2.rowId).toEqual(rowId1) - }) + // old link document + await db.put(oldLinkDocument()) }) - } -) + }) + + it("test migration runs as expected against an older DB", async () => { + const db = dbCore.getDB(config.appId!) + // confirm nothing exists initially + await sqsDisabled(async () => { + let error: any | undefined + try { + await db.get(SQLITE_DESIGN_DOC_ID) + } catch (err: any) { + error = err + } + expect(error).toBeDefined() + expect(error.status).toBe(404) + }) + + await sqsEnabled(async () => { + await processMigrations(config.appId!, MIGRATIONS) + const designDoc = await db.get(SQLITE_DESIGN_DOC_ID) + expect(designDoc.sql.tables).toBeDefined() + const mainTableDef = designDoc.sql.tables[tableId] + expect(mainTableDef).toBeDefined() + expect(mainTableDef.fields[prefix("name")]).toEqual({ + field: "name", + type: SQLiteType.TEXT, + }) + expect(mainTableDef.fields[prefix("description")]).toEqual({ + field: "description", + type: SQLiteType.TEXT, + }) + + const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo() + const linkDoc = await db.get(oldLinkDocID()) + expect(linkDoc.tableId).toEqual( + generateJunctionTableID(tableId1, tableId2) + ) + // should have swapped the documents + expect(linkDoc.doc1.tableId).toEqual(tableId2) + expect(linkDoc.doc1.rowId).toEqual(rowId2) + expect(linkDoc.doc2.tableId).toEqual(tableId1) + expect(linkDoc.doc2.rowId).toEqual(rowId1) + }) + }) +}) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 1ccd89639b..6c6b03918f 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -12,7 +12,7 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" import { searchInputMapping } from "./search/utils" -import { db as dbCore } from "@budibase/backend-core" +import { db as dbCore, features } from "@budibase/backend-core" import tracer from "dd-trace" export { isValidFilter } from "../../../integrations/utils" @@ -77,7 +77,7 @@ export async function search( if (isExternalTable) { span?.addTags({ searchType: "external" }) result = await external.search(options, table) - } else if (dbCore.isSqsEnabledForTenant()) { + } else if (await features.flags.isEnabled("SQS")) { span?.addTags({ searchType: "sqs" }) result = await internal.sqs.search(options, table) } else { diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 252b9a6556..8603be2fe8 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -35,14 +35,13 @@ describe.each([ let rows: Row[] beforeAll(async () => { - await withCoreEnv({ SQS_SEARCH_ENABLE: isSqs ? "true" : "false" }, () => + await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () => config.init() ) if (isSqs) { envCleanup = setCoreEnv({ - SQS_SEARCH_ENABLE: "true", - SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()], + TENANT_FEATURE_FLAGS: "*:SQS", }) } diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 725c4e5cd2..ddb1bbbf6c 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -1,4 +1,4 @@ -import { context, db as dbCore, env } from "@budibase/backend-core" +import { context, features } from "@budibase/backend-core" import { getTableParams } from "../../../db/utils" import { breakExternalTableId, @@ -16,7 +16,7 @@ import { import datasources from "../datasources" import sdk from "../../../sdk" -export function processTable(table: Table): Table { +export async function processTable(table: Table): Promise { if (!table) { return table } @@ -33,20 +33,21 @@ export function processTable(table: Table): Table { sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID, sourceType: TableSourceType.INTERNAL, } - if (dbCore.isSqsEnabledForTenant()) { - processed.sql = !!env.SQS_SEARCH_ENABLE + const sqsEnabled = await features.flags.isEnabled("SQS") + if (sqsEnabled) { + processed.sql = true } return processed } } -export function processTables(tables: Table[]): Table[] { - return tables.map(table => processTable(table)) +export async function processTables(tables: Table[]): Promise { + return await Promise.all(tables.map(table => processTable(table))) } -function processEntities(tables: Record) { +async function processEntities(tables: Record) { for (let key of Object.keys(tables)) { - tables[key] = processTable(tables[key]) + tables[key] = await processTable(tables[key]) } return tables } @@ -60,7 +61,7 @@ export async function getAllInternalTables(db?: Database): Promise { include_docs: true, }) ) - return processTables(internalTables.rows.map(row => row.doc!)) + return await processTables(internalTables.rows.map(row => row.doc!)) } async function getAllExternalTables(): Promise { @@ -72,7 +73,7 @@ async function getAllExternalTables(): Promise { final = final.concat(Object.values(entities)) } } - return processTables(final) + return await processTables(final) } export async function getExternalTable( @@ -97,7 +98,7 @@ export async function getTable(tableId: string): Promise
{ } else { output = await db.get
(tableId) } - return processTable(output) + return await processTable(output) } export async function getAllTables() { @@ -105,7 +106,7 @@ export async function getAllTables() { getAllInternalTables(), getAllExternalTables(), ]) - return processTables([...internal, ...external]) + return await processTables([...internal, ...external]) } export async function getExternalTablesInDatasource( @@ -115,7 +116,7 @@ export async function getExternalTablesInDatasource( if (!datasource || !datasource.entities) { throw new Error("Datasource is not configured fully.") } - return processEntities(datasource.entities) + return await processEntities(datasource.entities) } export async function getTables(tableIds: string[]): Promise { @@ -139,7 +140,7 @@ export async function getTables(tableIds: string[]): Promise { }) tables = tables.concat(internalTables) } - return processTables(tables) + return await processTables(tables) } export function enrichViewSchemas(table: Table): TableResponse { diff --git a/packages/server/src/websockets/builder.ts b/packages/server/src/websockets/builder.ts index a47d3048d3..702a941095 100644 --- a/packages/server/src/websockets/builder.ts +++ b/packages/server/src/websockets/builder.ts @@ -16,7 +16,6 @@ import { gridSocket } from "./index" import { clearLock, updateLock } from "../utilities/redis" import { Socket } from "socket.io" import { BuilderSocketEvent } from "@budibase/shared-core" -import { processTable } from "../sdk/app/tables/getters" export default class BuilderSocket extends BaseSocket { constructor(app: Koa, server: http.Server) { @@ -102,10 +101,9 @@ export default class BuilderSocket extends BaseSocket { } emitTableUpdate(ctx: any, table: Table, options?: EmitOptions) { - // This was added to make sure that sourceId is always present when - // sending this message to clients. Without this, tables without a - // sourceId (e.g. ta_users) won't get correctly updated client-side. - table = processTable(table) + if (table.sourceId == null || table.sourceId === "") { + throw new Error("Table sourceId is not set") + } this.emitToRoom( ctx, diff --git a/packages/worker/scripts/dev/manage.js b/packages/worker/scripts/dev/manage.js index e34f1e7f89..cec8c68d26 100644 --- a/packages/worker/scripts/dev/manage.js +++ b/packages/worker/scripts/dev/manage.js @@ -30,7 +30,7 @@ async function init() { HTTP_LOGGING: "0", VERSION: "0.0.0+local", PASSWORD_MIN_LENGTH: "1", - SQS_SEARCH_ENABLE: "1", + TENANT_FEATURE_FLAGS: "*:SQS", } config = { ...config, ...existingConfig } diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index ae426ad8b1..90f303d3a8 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -1,6 +1,6 @@ import { Ctx, MaintenanceType } from "@budibase/types" import env from "../../../environment" -import { env as coreEnv, db as dbCore } from "@budibase/backend-core" +import { env as coreEnv, db as dbCore, features } from "@budibase/backend-core" import nodeFetch from "node-fetch" let sqsAvailable: boolean @@ -29,7 +29,7 @@ async function isSqsAvailable() { } async function isSqsMissing() { - return coreEnv.SQS_SEARCH_ENABLE && !(await isSqsAvailable()) + return (await features.flags.isEnabled("SQS")) && !(await isSqsAvailable()) } export const fetch = async (ctx: Ctx) => { diff --git a/packages/worker/src/api/index.ts b/packages/worker/src/api/index.ts index 8ab093a359..db0a80acfd 100644 --- a/packages/worker/src/api/index.ts +++ b/packages/worker/src/api/index.ts @@ -4,12 +4,8 @@ const compress = require("koa-compress") import zlib from "zlib" import { routes } from "./routes" -import { middleware as pro, sdk } from "@budibase/pro" -import { auth, middleware, env } from "@budibase/backend-core" - -if (env.SQS_SEARCH_ENABLE) { - sdk.auditLogs.useSQLSearch() -} +import { middleware as pro } from "@budibase/pro" +import { auth, middleware } from "@budibase/backend-core" const PUBLIC_ENDPOINTS = [ // deprecated single tenant sso callback diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 79e6f4493d..bb8a398ed6 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -7,7 +7,7 @@ import env from "./environment" import Application from "koa" import { bootstrap } from "global-agent" import * as db from "./db" -import { sdk as proSdk } from "@budibase/pro" +import { sdk as proSdk, sdk } from "@budibase/pro" import { auth, logging, @@ -101,6 +101,10 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => { // configure events to use the pro audit log write // can't integrate directly into backend-core due to cyclic issues await events.processors.init(proSdk.auditLogs.write) + + if (await features.flags.isEnabled("SQS")) { + sdk.auditLogs.useSQLSearch() + } }) process.on("uncaughtException", err => { From 4e24035812493f9d42e08660f55e6122e7bc778a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 14:40:38 +0100 Subject: [PATCH 02/24] Update account-portal reference. --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index ae7f9632fb..998eb8f8b5 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit ae7f9632fbe8c0de2d4f82a7f96fef67c921ff54 +Subproject commit 998eb8f8b50aecd9375e6f0119b8e1f67eb9e319 From ea032a0e7eda331774f129a30530957a51bff3fb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 14:59:31 +0100 Subject: [PATCH 03/24] Fix lint. --- packages/account-portal | 2 +- packages/server/src/api/controllers/table/utils.ts | 2 +- packages/server/src/sdk/app/rows/search.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index 998eb8f8b5..516b27b74c 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 998eb8f8b50aecd9375e6f0119b8e1f67eb9e319 +Subproject commit 516b27b74cbcb7069a25f5e738dc91c22d7c4538 diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 1b241adbd7..269f079ae8 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -15,7 +15,7 @@ import { getViews, saveView } from "../view/utils" import viewTemplate from "../view/viewBuilder" import { cloneDeep } from "lodash/fp" import { quotas } from "@budibase/pro" -import { events, context, db as dbCore, features } from "@budibase/backend-core" +import { events, context, features } from "@budibase/backend-core" import { AutoFieldSubType, ContextUser, diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 6c6b03918f..f199a4f6e1 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -12,7 +12,7 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" import { searchInputMapping } from "./search/utils" -import { db as dbCore, features } from "@budibase/backend-core" +import { features } from "@budibase/backend-core" import tracer from "dd-trace" export { isValidFilter } from "../../../integrations/utils" From 354e5dd34cd47f4379f7cde35585096d5447c420 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 15:17:37 +0100 Subject: [PATCH 04/24] More specifically import feature flags. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 35181b2ade..aa4656bf64 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -26,7 +26,7 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants" import { DDInstrumentedDatabase } from "../instrumentation" import { checkSlashesInUrl } from "../../helpers" import { sqlLog } from "../../sql/utils" -import { features } from "../.." +import { flags } from "../../features" const DATABASE_NOT_FOUND = "Database does not exist." @@ -402,7 +402,7 @@ export class DatabaseImpl implements Database { async destroy() { if ( - (await features.flags.isEnabled("SQS")) && + (await flags.isEnabled("SQS")) && (await this.exists(SQLITE_DESIGN_DOC_ID)) ) { // delete the design document, then run the cleanup operation From dc28b7e133416577774732d0455b2078a545f6d4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 15:21:58 +0100 Subject: [PATCH 05/24] Don't error if you can't get the current tenant ID. --- packages/backend-core/src/features/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index d1d757d8a9..42cd5a79b4 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -136,9 +136,10 @@ export class FlagSet, T extends { [key: string]: V }> { return cachedFlags } + const currentContext = context.getCurrentContext() const tags: Record = {} const flagValues = this.defaults() - const currentTenantId = context.getTenantId() + const currentTenantId = currentContext?.tenantId const specificallySetFalse = new Set() const split = (env.TENANT_FEATURE_FLAGS || "") From 06a7801d8ceba625733809a2b245ad830e02ee4b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 15:37:51 +0100 Subject: [PATCH 06/24] Remove global audit logs SQS flag. --- packages/backend-core/src/features/index.ts | 3 +-- packages/pro | 2 +- .../worker/src/api/routes/global/tests/auditLogs.spec.ts | 7 ++++--- packages/worker/src/index.ts | 4 ---- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index 42cd5a79b4..d1d757d8a9 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -136,10 +136,9 @@ export class FlagSet, T extends { [key: string]: V }> { return cachedFlags } - const currentContext = context.getCurrentContext() const tags: Record = {} const flagValues = this.defaults() - const currentTenantId = currentContext?.tenantId + const currentTenantId = context.getTenantId() const specificallySetFalse = new Set() const split = (env.TENANT_FEATURE_FLAGS || "") diff --git a/packages/pro b/packages/pro index 5678433b7c..d4f83bf74a 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 5678433b7cfb3f121fbf837a0d6ddeeee054a7ed +Subproject commit d4f83bf74a80a3e8a1e0c29aa1adc7231fe76d49 diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts index 92a366bf4e..350c9a7edc 100644 --- a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -1,7 +1,6 @@ import { mocks, structures } from "@budibase/backend-core/tests" -import { context, events } from "@budibase/backend-core" +import { context, events, setEnv as setCoreEnv } from "@budibase/backend-core" import { Event, IdentityType } from "@budibase/types" -import { auditLogs } from "@budibase/pro" import { TestConfiguration } from "../../../../tests" mocks.licenses.useAuditLogs() @@ -15,15 +14,17 @@ const APP_ID = "app_1" describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => { const config = new TestConfiguration() + let envCleanup: (() => void) | undefined beforeAll(async () => { if (method === "sql") { - auditLogs.useSQLSearch() + envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }) } await config.beforeAll() }) afterAll(async () => { + envCleanup?.() await config.afterAll() }) diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index bb8a398ed6..3c047e1a44 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -101,10 +101,6 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => { // configure events to use the pro audit log write // can't integrate directly into backend-core due to cyclic issues await events.processors.init(proSdk.auditLogs.write) - - if (await features.flags.isEnabled("SQS")) { - sdk.auditLogs.useSQLSearch() - } }) process.on("uncaughtException", err => { From 987304494f6cca73d30233702aabaf6baaa838fb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 15:46:12 +0100 Subject: [PATCH 07/24] Update pro reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index d4f83bf74a..9d3d02352e 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit d4f83bf74a80a3e8a1e0c29aa1adc7231fe76d49 +Subproject commit 9d3d02352ecd56ad5b3ac3c545f5978d74eafe6e From 00ba9268501cbd7aea55f1c5126829079fba8cfd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 16:01:52 +0100 Subject: [PATCH 08/24] Update pro reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 9d3d02352e..fb7a832295 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 9d3d02352ecd56ad5b3ac3c545f5978d74eafe6e +Subproject commit fb7a83229504b4a593b28d40f9849cff42a685fb From 71000ea967651320d97f64adb984b22e0cf2b8e5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 16:11:06 +0100 Subject: [PATCH 09/24] Fix lint. --- packages/worker/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 3c047e1a44..79e6f4493d 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -7,7 +7,7 @@ import env from "./environment" import Application from "koa" import { bootstrap } from "global-agent" import * as db from "./db" -import { sdk as proSdk, sdk } from "@budibase/pro" +import { sdk as proSdk } from "@budibase/pro" import { auth, logging, From c12a96b4d875f7180d0f9e12399155e4587fed12 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 16:21:12 +0100 Subject: [PATCH 10/24] Fix backend-core tests. --- .../backend-core/src/db/tests/index.spec.js | 24 -------------- .../backend-core/src/db/tests/index.spec.ts | 32 +++++++++++++++++++ .../src/features/tests/features.spec.ts | 2 +- 3 files changed, 33 insertions(+), 25 deletions(-) delete mode 100644 packages/backend-core/src/db/tests/index.spec.js create mode 100644 packages/backend-core/src/db/tests/index.spec.ts diff --git a/packages/backend-core/src/db/tests/index.spec.js b/packages/backend-core/src/db/tests/index.spec.js deleted file mode 100644 index e03c9a5b0e..0000000000 --- a/packages/backend-core/src/db/tests/index.spec.js +++ /dev/null @@ -1,24 +0,0 @@ -require("../../../tests") -const { structures } = require("../../../tests") -const { getDB } = require("../db") - -describe("db", () => { - describe("getDB", () => { - it("returns a db", async () => { - const dbName = structures.db.id() - const db = getDB(dbName) - expect(db).toBeDefined() - expect(db.name).toBe(dbName) - }) - - it("uses the custom put function", async () => { - const db = getDB(structures.db.id()) - let doc = { _id: "test" } - await db.put(doc) - doc = await db.get(doc._id) - expect(doc.createdAt).toBe(new Date().toISOString()) - expect(doc.updatedAt).toBe(new Date().toISOString()) - await db.destroy() - }) - }) -}) diff --git a/packages/backend-core/src/db/tests/index.spec.ts b/packages/backend-core/src/db/tests/index.spec.ts new file mode 100644 index 0000000000..d61b0b7655 --- /dev/null +++ b/packages/backend-core/src/db/tests/index.spec.ts @@ -0,0 +1,32 @@ +import { context } from "../.." +import { structures } from "../../../tests" +import { getDB } from "../db" + +interface Doc { + _id: string + createdAt?: string + updatedAt?: string +} + +describe("db", () => { + describe("getDB", () => { + it("returns a db", async () => { + const dbName = structures.db.id() + const db = getDB(dbName) + expect(db).toBeDefined() + expect(db.name).toBe(dbName) + }) + + it("uses the custom put function", async () => { + await context.doInTenant("foo", async () => { + const db = getDB(structures.db.id()) + let doc: Doc = { _id: "test" } + await db.put(doc) + doc = await db.get(doc._id) + expect(doc.createdAt).toBe(new Date().toISOString()) + expect(doc.updatedAt).toBe(new Date().toISOString()) + await db.destroy() + }) + }) + }) +}) diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts index 1e8e25654a..c8469561c6 100644 --- a/packages/backend-core/src/features/tests/features.spec.ts +++ b/packages/backend-core/src/features/tests/features.spec.ts @@ -147,13 +147,13 @@ describe("feature flags", () => { }) => { const env: Partial = { TENANT_FEATURE_FLAGS: environmentFlags, + SELF_HOSTED: false, } if (posthogFlags) { mockPosthogFlags(posthogFlags) env.POSTHOG_TOKEN = "test" env.POSTHOG_API_HOST = "https://us.i.posthog.com" - env.POSTHOG_PERSONAL_TOKEN = "test" } const ctx = { user: { license: { features: licenseFlags || [] } } } From 061b61dd64f3ed3922ecc2f2fd00124163b6b6e8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 16:27:40 +0100 Subject: [PATCH 11/24] Fix pro test failures. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index fb7a832295..4889395fa3 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit fb7a83229504b4a593b28d40f9849cff42a685fb +Subproject commit 4889395fa35035a74ed098e323c15a4057b31b64 From 278ae65eac8011a90181a69083c556fbaa8e33c1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 16:28:54 +0100 Subject: [PATCH 12/24] Fix broken import. --- packages/server/src/api/controllers/datasource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index d5aec678fb..baab008da8 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -27,7 +27,7 @@ import { import sdk from "../../sdk" import { builderSocket } from "../../websockets" import { isEqual } from "lodash" -import { processTable } from "src/sdk/app/tables/getters" +import { processTable } from "../../sdk/app/tables/getters" export async function fetch(ctx: UserCtx) { ctx.body = await sdk.datasources.fetch() From 43560b3269211fbd16b7b40ef6e496b16648117b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 27 Aug 2024 14:40:15 +0100 Subject: [PATCH 13/24] Fix environment test. --- packages/worker/src/tests/api/environment.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/worker/src/tests/api/environment.ts b/packages/worker/src/tests/api/environment.ts index d9f82c5f0d..d6a76f3b06 100644 --- a/packages/worker/src/tests/api/environment.ts +++ b/packages/worker/src/tests/api/environment.ts @@ -9,6 +9,7 @@ export class EnvironmentAPI extends TestAPI { getEnvironment = () => { return this.request .get(`/api/system/environment`) + .set(this.config.defaultHeaders()) .expect("Content-Type", /json/) .expect(200) } From be0a14489b153eec67fd825e2d1221e7b9d9918e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 27 Aug 2024 15:22:53 +0100 Subject: [PATCH 14/24] Debugging view test failures. --- packages/server/src/api/controllers/row/views.ts | 3 ++- .../server/src/api/routes/tests/viewV2.spec.ts | 14 +++++++------- packages/server/src/tests/jestEnv.ts | 1 + 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index e9b6b0d8e5..1afe56a322 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -59,12 +59,13 @@ export async function searchView( } }) }) - } else + } else { query = { $and: { conditions: [query, body.query], }, } + } } await context.ensureSnippetContext(true) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 079d85c684..c7b224770d 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -34,12 +34,12 @@ import sdk from "../../../sdk" describe.each([ ["lucene", undefined], - ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // ["sqs", undefined], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -1491,7 +1491,7 @@ describe.each([ ) }) - it("can query on top of the view filters", async () => { + it.only("can query on top of the view filters", async () => { await config.api.row.save(table._id!, { one: "foo", two: "bar", diff --git a/packages/server/src/tests/jestEnv.ts b/packages/server/src/tests/jestEnv.ts index a44ce58f81..0071ec52c2 100644 --- a/packages/server/src/tests/jestEnv.ts +++ b/packages/server/src/tests/jestEnv.ts @@ -3,6 +3,7 @@ import { tmpdir } from "os" process.env.SELF_HOSTED = "1" process.env.NODE_ENV = "jest" process.env.MULTI_TENANCY = "1" +process.env.APP_PORT = "0" // @ts-ignore process.env.BUDIBASE_DIR = tmpdir("budibase-unittests") process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error" From 8c890af814626b835182046cbdd6755056df1f98 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 27 Aug 2024 15:29:37 +0100 Subject: [PATCH 15/24] Fix view tests. --- packages/server/src/api/controllers/row/views.ts | 2 +- .../server/src/api/routes/tests/viewV2.spec.ts | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 1afe56a322..d2541dfa25 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -42,7 +42,7 @@ export async function searchView( if ( !isExternalTableID(view.tableId) && - (await features.flags.isEnabled("SQS")) + !(await features.flags.isEnabled("SQS")) ) { // Extract existing fields const existingFields = diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index c7b224770d..079d85c684 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -34,12 +34,12 @@ import sdk from "../../../sdk" describe.each([ ["lucene", undefined], - // ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + ["sqs", undefined], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -1491,7 +1491,7 @@ describe.each([ ) }) - it.only("can query on top of the view filters", async () => { + it("can query on top of the view filters", async () => { await config.api.row.save(table._id!, { one: "foo", two: "bar", From 707da3864b3e412c39606cbb497135bd9a83413c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 27 Aug 2024 15:46:16 +0100 Subject: [PATCH 16/24] Fix migration test. --- .../tests/20240604153647_initial_sqs.spec.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts index 654b001f61..b4f708edb5 100644 --- a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts +++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts @@ -18,7 +18,7 @@ import { } from "../../../db/utils" import { processMigrations } from "../../migrationsProcessor" import migration from "../20240604153647_initial_sqs" -import { AppMigration } from "src/appMigrations" +import { AppMigration, updateAppMigrationMetadata } from "../../" import sdk from "../../../sdk" const MIGRATIONS: AppMigration[] = [ @@ -90,6 +90,15 @@ describe("SQS migration", () => { }) }) + beforeEach(async () => { + await config.doInTenant(async () => { + await updateAppMigrationMetadata({ + appId: config.getAppId(), + version: "", + }) + }) + }) + it("test migration runs as expected against an older DB", async () => { const db = dbCore.getDB(config.appId!) // confirm nothing exists initially From d2b6959b4427daef28d04b26e9ac6d6a3d71b82c Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 29 Aug 2024 12:58:12 +0100 Subject: [PATCH 17/24] Reduced the z-index to just 1 above the binding drawer. The updated value caused index issues in other areas of budibase. --- packages/bbui/src/Modal/Modal.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bbui/src/Modal/Modal.svelte b/packages/bbui/src/Modal/Modal.svelte index 4d55a8a266..c5a6ffc17b 100644 --- a/packages/bbui/src/Modal/Modal.svelte +++ b/packages/bbui/src/Modal/Modal.svelte @@ -10,7 +10,7 @@ export let inline = false export let disableCancel = false export let autoFocus = true - export let zIndex = 99999 + export let zIndex = 1001 const dispatch = createEventDispatcher() let visible = fixed || inline From bced81d2410e1f8b4d14fdbdcf6af4e00befded3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 Aug 2024 15:45:19 +0100 Subject: [PATCH 18/24] Fix tests. --- .../src/utilities/rowProcessor/index.ts | 3 +- .../tests/outputProcessing.spec.ts | 462 +++++++++--------- 2 files changed, 243 insertions(+), 222 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 795f6970ab..d2fb1cc5a8 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -4,6 +4,7 @@ import { cache, context, db, + features, HTTPError, objectStore, utils, @@ -350,7 +351,7 @@ export async function outputProcessing( } // remove null properties to match internal API const isExternal = isExternalTableID(table._id!) - if (isExternal || db.isSqsEnabledForTenant()) { + if (isExternal || (await features.flags.isEnabled("SQS"))) { for (const row of enriched) { for (const key of Object.keys(row)) { if (row[key] === null) { diff --git a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts index 0d0049a36e..b6c2cdb11c 100644 --- a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts @@ -8,15 +8,9 @@ import { } from "@budibase/types" import { outputProcessing } from ".." import { generator, structures } from "@budibase/backend-core/tests" +import { setEnv as setCoreEnv } from "@budibase/backend-core" import * as bbReferenceProcessor from "../bbReferenceProcessor" - -jest.mock("@budibase/backend-core", () => ({ - ...jest.requireActual("@budibase/backend-core"), - db: { - ...jest.requireActual("@budibase/backend-core").db, - isSqsEnabledForTenant: () => true, - }, -})) +import TestConfiguration from "../../../tests/utilities/TestConfiguration" jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({ processInputBBReference: jest.fn(), @@ -26,8 +20,24 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({ })) describe("rowProcessor - outputProcessing", () => { + const config = new TestConfiguration() + let cleanupEnv: () => void = () => {} + + beforeAll(async () => { + await config.init() + }) + + afterAll(async () => { + config.end() + }) + beforeEach(() => { jest.resetAllMocks() + cleanupEnv = setCoreEnv({ TENANT_FEATURE_FLAGS: "*SQS" }) + }) + + afterEach(() => { + cleanupEnv() }) const processOutputBBReferenceMock = @@ -36,266 +46,276 @@ describe("rowProcessor - outputProcessing", () => { bbReferenceProcessor.processOutputBBReferences as jest.Mock it("fetches single user references given a populated field", async () => { - const table: Table = { - _id: generator.guid(), - name: "TestTable", - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - name: { - type: FieldType.STRING, - name: "name", - constraints: { - presence: true, - type: "string", + await config.doInContext(config.getAppId(), async () => { + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + name: { + type: FieldType.STRING, + name: "name", + constraints: { + presence: true, + type: "string", + }, + }, + user: { + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + name: "user", + constraints: { + presence: false, + type: "string", + }, }, }, - user: { - type: FieldType.BB_REFERENCE_SINGLE, - subtype: BBReferenceFieldSubType.USER, - name: "user", - constraints: { - presence: false, - type: "string", - }, - }, - }, - } + } - const row = { - name: "Jack", - user: "123", - } + const row = { + name: "Jack", + user: "123", + } - const user = structures.users.user() - processOutputBBReferenceMock.mockResolvedValue(user) + const user = structures.users.user() + processOutputBBReferenceMock.mockResolvedValue(user) - const result = await outputProcessing(table, row, { squash: false }) + const result = await outputProcessing(table, row, { squash: false }) - expect(result).toEqual({ name: "Jack", user }) + expect(result).toEqual({ name: "Jack", user }) - expect(bbReferenceProcessor.processOutputBBReference).toHaveBeenCalledTimes( - 1 - ) - expect(bbReferenceProcessor.processOutputBBReference).toHaveBeenCalledWith( - "123", - BBReferenceFieldSubType.USER - ) + expect( + bbReferenceProcessor.processOutputBBReference + ).toHaveBeenCalledTimes(1) + expect( + bbReferenceProcessor.processOutputBBReference + ).toHaveBeenCalledWith("123", BBReferenceFieldSubType.USER) + }) }) it("fetches users references given a populated field", async () => { - const table: Table = { - _id: generator.guid(), - name: "TestTable", - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - name: { - type: FieldType.STRING, - name: "name", - constraints: { - presence: true, - type: "string", + await config.doInContext(config.getAppId(), async () => { + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + name: { + type: FieldType.STRING, + name: "name", + constraints: { + presence: true, + type: "string", + }, + }, + users: { + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + name: "users", + constraints: { + presence: false, + type: "string", + }, }, }, - users: { - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - name: "users", - constraints: { - presence: false, - type: "string", - }, - }, - }, - } + } - const row = { - name: "Jack", - users: "123", - } + const row = { + name: "Jack", + users: "123", + } - const users = [structures.users.user()] - processOutputBBReferencesMock.mockResolvedValue(users) + const users = [structures.users.user()] + processOutputBBReferencesMock.mockResolvedValue(users) - const result = await outputProcessing(table, row, { squash: false }) + const result = await outputProcessing(table, row, { squash: false }) - expect(result).toEqual({ name: "Jack", users }) + expect(result).toEqual({ name: "Jack", users }) - expect( - bbReferenceProcessor.processOutputBBReferences - ).toHaveBeenCalledTimes(1) - expect(bbReferenceProcessor.processOutputBBReferences).toHaveBeenCalledWith( - "123", - BBReferenceFieldSubType.USER - ) + expect( + bbReferenceProcessor.processOutputBBReferences + ).toHaveBeenCalledTimes(1) + expect( + bbReferenceProcessor.processOutputBBReferences + ).toHaveBeenCalledWith("123", BBReferenceFieldSubType.USER) + }) }) it("should handle attachment list correctly", async () => { - const table: Table = { - _id: generator.guid(), - name: "TestTable", - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - attach: { - type: FieldType.ATTACHMENTS, - name: "attach", - constraints: {}, + await config.doInContext(config.getAppId(), async () => { + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + attach: { + type: FieldType.ATTACHMENTS, + name: "attach", + constraints: {}, + }, }, - }, - } + } - const row: { attach: RowAttachment[] } = { - attach: [ - { + const row: { attach: RowAttachment[] } = { + attach: [ + { + size: 10, + name: "test", + extension: "jpg", + key: "test.jpg", + }, + ], + } + + const output = await outputProcessing(table, row, { squash: false }) + expect(output.attach[0].url?.split("?")[0]).toBe( + "/files/signed/prod-budi-app-assets/test.jpg" + ) + + row.attach[0].url = "" + const output2 = await outputProcessing(table, row, { squash: false }) + expect(output2.attach[0].url?.split("?")[0]).toBe( + "/files/signed/prod-budi-app-assets/test.jpg" + ) + + row.attach[0].url = "aaaa" + const output3 = await outputProcessing(table, row, { squash: false }) + expect(output3.attach[0].url).toBe("aaaa") + }) + }) + + it("should handle single attachment correctly", async () => { + await config.doInContext(config.getAppId(), async () => { + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + attach: { + type: FieldType.ATTACHMENT_SINGLE, + name: "attach", + constraints: {}, + }, + }, + } + + const row: { attach: RowAttachment } = { + attach: { size: 10, name: "test", extension: "jpg", key: "test.jpg", }, - ], - } + } - const output = await outputProcessing(table, row, { squash: false }) - expect(output.attach[0].url?.split("?")[0]).toBe( - "/files/signed/prod-budi-app-assets/test.jpg" - ) + const output = await outputProcessing(table, row, { squash: false }) + expect(output.attach.url?.split("?")[0]).toBe( + "/files/signed/prod-budi-app-assets/test.jpg" + ) - row.attach[0].url = "" - const output2 = await outputProcessing(table, row, { squash: false }) - expect(output2.attach[0].url?.split("?")[0]).toBe( - "/files/signed/prod-budi-app-assets/test.jpg" - ) + row.attach.url = "" + const output2 = await outputProcessing(table, row, { squash: false }) + expect(output2.attach?.url?.split("?")[0]).toBe( + "/files/signed/prod-budi-app-assets/test.jpg" + ) - row.attach[0].url = "aaaa" - const output3 = await outputProcessing(table, row, { squash: false }) - expect(output3.attach[0].url).toBe("aaaa") - }) - - it("should handle single attachment correctly", async () => { - const table: Table = { - _id: generator.guid(), - name: "TestTable", - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - attach: { - type: FieldType.ATTACHMENT_SINGLE, - name: "attach", - constraints: {}, - }, - }, - } - - const row: { attach: RowAttachment } = { - attach: { - size: 10, - name: "test", - extension: "jpg", - key: "test.jpg", - }, - } - - const output = await outputProcessing(table, row, { squash: false }) - expect(output.attach.url?.split("?")[0]).toBe( - "/files/signed/prod-budi-app-assets/test.jpg" - ) - - row.attach.url = "" - const output2 = await outputProcessing(table, row, { squash: false }) - expect(output2.attach?.url?.split("?")[0]).toBe( - "/files/signed/prod-budi-app-assets/test.jpg" - ) - - row.attach.url = "aaaa" - const output3 = await outputProcessing(table, row, { squash: false }) - expect(output3.attach.url).toBe("aaaa") + row.attach.url = "aaaa" + const output3 = await outputProcessing(table, row, { squash: false }) + expect(output3.attach.url).toBe("aaaa") + }) }) it("process output even when the field is not empty", async () => { - const table: Table = { - _id: generator.guid(), - name: "TestTable", - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - name: { - type: FieldType.STRING, - name: "name", - constraints: { - presence: true, - type: "string", + await config.doInContext(config.getAppId(), async () => { + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + name: { + type: FieldType.STRING, + name: "name", + constraints: { + presence: true, + type: "string", + }, + }, + user: { + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + name: "user", + constraints: { + presence: false, + type: "string", + }, }, }, - user: { - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - name: "user", - constraints: { - presence: false, - type: "string", - }, - }, - }, - } + } - const row = { - name: "Jack", - } + const row = { + name: "Jack", + } - const result = await outputProcessing(table, row, { squash: false }) + const result = await outputProcessing(table, row, { squash: false }) - expect(result).toEqual({ name: "Jack" }) + expect(result).toEqual({ name: "Jack" }) - expect( - bbReferenceProcessor.processOutputBBReferences - ).toHaveBeenCalledTimes(1) + expect( + bbReferenceProcessor.processOutputBBReferences + ).toHaveBeenCalledTimes(1) + }) }) it("does not fetch bb references when not in the schema", async () => { - const table: Table = { - _id: generator.guid(), - name: "TestTable", - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - name: { - type: FieldType.STRING, - name: "name", - constraints: { - presence: true, - type: "string", + await config.doInContext(config.getAppId(), async () => { + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + name: { + type: FieldType.STRING, + name: "name", + constraints: { + presence: true, + type: "string", + }, + }, + user: { + type: FieldType.NUMBER, + name: "user", + constraints: { + presence: false, + type: "string", + }, }, }, - user: { - type: FieldType.NUMBER, - name: "user", - constraints: { - presence: false, - type: "string", - }, - }, - }, - } + } - const row = { - name: "Jack", - user: "123", - } + const row = { + name: "Jack", + user: "123", + } - const result = await outputProcessing(table, row, { squash: false }) + const result = await outputProcessing(table, row, { squash: false }) - expect(result).toEqual({ name: "Jack", user: "123" }) + expect(result).toEqual({ name: "Jack", user: "123" }) - expect( - bbReferenceProcessor.processOutputBBReferences - ).not.toHaveBeenCalled() + expect( + bbReferenceProcessor.processOutputBBReferences + ).not.toHaveBeenCalled() + }) }) }) From edfedec930f6967965e59205041698a08b40615c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 Aug 2024 15:50:46 +0100 Subject: [PATCH 19/24] Fix lint. --- packages/server/src/utilities/rowProcessor/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index d2fb1cc5a8..2940f2118e 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -3,7 +3,6 @@ import { fixAutoColumnSubType, processFormulas } from "./utils" import { cache, context, - db, features, HTTPError, objectStore, From 82396c339d985ce6b011eb0c76d4baa6e86a322f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 Aug 2024 16:30:38 +0100 Subject: [PATCH 20/24] Update account-portal submodule to latest master. --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index c403315c5f..c24374879d 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit c403315c5fa09a05dfd8fa4cd1890acfd8de0430 +Subproject commit c24374879d2b61516fabc24d7404e7da235be05e From 5a5c0d74dd6d1ac0041e9b745b152a5df827229e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 Aug 2024 16:44:15 +0100 Subject: [PATCH 21/24] Update pro reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 0d2728a881..d926494cc5 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 0d2728a88123138ea109e0827a4fdf849015e2d5 +Subproject commit d926494cc54e31441637180a745d0df7156a5120 From d134c21a45f2f0b0e72400e8edfb6727c9d65a93 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 Aug 2024 17:45:47 +0100 Subject: [PATCH 22/24] Fix build. --- packages/server/src/api/routes/tests/row.spec.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index b060b2f0c1..e941c0b267 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -95,12 +95,9 @@ describe.each([ let envCleanup: (() => void) | undefined beforeAll(async () => { - await withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, () => config.init()) + await withCoreEnv({ TENANT_FEATURE_FLAGS: "*SQS" }, () => config.init()) if (isSqs) { - envCleanup = setCoreEnv({ - SQS_SEARCH_ENABLE: "true", - SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()], - }) + envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*SQS" }) } if (dsProvider) { From 2a2dbfb74537b13b1c57f5829fb7dda5ff7c80dc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 Aug 2024 17:59:17 +0100 Subject: [PATCH 23/24] Respond to last PR comment. --- packages/server/src/tests/jestEnv.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/tests/jestEnv.ts b/packages/server/src/tests/jestEnv.ts index 0071ec52c2..0001cec759 100644 --- a/packages/server/src/tests/jestEnv.ts +++ b/packages/server/src/tests/jestEnv.ts @@ -4,6 +4,7 @@ process.env.SELF_HOSTED = "1" process.env.NODE_ENV = "jest" process.env.MULTI_TENANCY = "1" process.env.APP_PORT = "0" +process.env.WORKER_PORT = "0" // @ts-ignore process.env.BUDIBASE_DIR = tmpdir("budibase-unittests") process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error" From 4d9f87e4c48e682d00e31c83bd5f86515601efcd Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Fri, 30 Aug 2024 10:16:16 +0100 Subject: [PATCH 24/24] Set password to 12 characters (#14478) --- .../users/_components/ForceResetPasswordModal.svelte | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/portal/users/users/_components/ForceResetPasswordModal.svelte b/packages/builder/src/pages/builder/portal/users/users/_components/ForceResetPasswordModal.svelte index 8a7a3940bf..36fdcccbce 100644 --- a/packages/builder/src/pages/builder/portal/users/users/_components/ForceResetPasswordModal.svelte +++ b/packages/builder/src/pages/builder/portal/users/users/_components/ForceResetPasswordModal.svelte @@ -7,7 +7,15 @@ export let user - const password = Math.random().toString(36).slice(2, 20) + const generatePassword = length => { + const array = new Uint8Array(length) + crypto.getRandomValues(array) + return Array.from(array, byte => byte.toString(36).padStart(2, "0")) + .join("") + .slice(0, length) + } + + const password = generatePassword(12) async function resetPassword() { try {