From 594551e1b550368609f1e998f356e2dd28b354f3 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 12 Sep 2024 14:05:32 +0100 Subject: [PATCH 01/90] tests for filter steps --- .../tests/scenarios/scenarios.spec.ts | 167 +++++++++++++++++- .../tests/utilities/AutomationTestBuilder.ts | 9 + 2 files changed, 175 insertions(+), 1 deletion(-) diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts index 40d6094525..62a1b9db8f 100644 --- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -1,8 +1,9 @@ import * as automation from "../../index" import * as setup from "../utilities" -import { LoopStepType, FieldType } from "@budibase/types" +import { LoopStepType, FieldType, Table } from "@budibase/types" import { createAutomationBuilder } from "../utilities/AutomationTestBuilder" import { DatabaseName } from "../../../integrations/tests/utils" +import { FilterConditions } from "../../../automations/steps/filter" describe("Automation Scenarios", () => { let config = setup.getConfig() @@ -196,4 +197,168 @@ describe("Automation Scenarios", () => { ) }) }) + describe.only("Automations with filter", () => { + let table: Table + + beforeEach(async () => { + table = await config.createTable({ + name: "TestTable", + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + value: { + name: "value", + type: FieldType.NUMBER, + constraints: { + presence: true, + }, + }, + }, + }) + }) + + it("should stop an automation if the condition is not met", async () => { + const builder = createAutomationBuilder({ + name: "Test Equal", + }) + + const results = await builder + .appAction({ fields: {} }) + .createRow({ + row: { + name: "Equal Test", + value: 10, + tableId: table._id, + }, + }) + .queryRows({ + tableId: table._id!, + }) + .filter({ + field: "{{ steps.2.rows.0.value }}", + condition: FilterConditions.EQUAL, + value: 20, + }) + .serverLog({ text: "Equal condition met" }) + .run() + + expect(results.steps[2].outputs.success).toBeTrue() + expect(results.steps[2].outputs.result).toBeFalse() + expect(results.steps[3]).toBeUndefined() + }) + + it("should continue the automation if the condition is met", async () => { + const builder = createAutomationBuilder({ + name: "Test Not Equal", + }) + + const results = await builder + .appAction({ fields: {} }) + .createRow({ + row: { + name: "Not Equal Test", + value: 10, + tableId: table._id, + }, + }) + .queryRows({ + tableId: table._id!, + }) + .filter({ + field: "{{ steps.2.rows.0.value }}", + condition: FilterConditions.NOT_EQUAL, + value: 20, + }) + .serverLog({ text: "Not Equal condition met" }) + .run() + + expect(results.steps[2].outputs.success).toBeTrue() + expect(results.steps[2].outputs.result).toBeTrue() + expect(results.steps[3].outputs.success).toBeTrue() + }) + + const testCases = [ + { + condition: FilterConditions.EQUAL, + value: 10, + rowValue: 10, + expectPass: true, + }, + { + condition: FilterConditions.NOT_EQUAL, + value: 10, + rowValue: 20, + expectPass: true, + }, + { + condition: FilterConditions.GREATER_THAN, + value: 10, + rowValue: 15, + expectPass: true, + }, + { + condition: FilterConditions.LESS_THAN, + value: 10, + rowValue: 5, + expectPass: true, + }, + { + condition: FilterConditions.GREATER_THAN, + value: 10, + rowValue: 5, + expectPass: false, + }, + { + condition: FilterConditions.LESS_THAN, + value: 10, + rowValue: 15, + expectPass: false, + }, + ] + + testCases.forEach(({ condition, value, rowValue, expectPass }) => { + it(`should ${ + expectPass ? "pass" : "fail" + } the filter when condition is "${condition}" and value is ${value}`, async () => { + const builder = createAutomationBuilder({ + name: `Test ${condition}`, + }) + + const results = await builder + .appAction({ fields: {} }) + .createRow({ + row: { + name: `${condition} Test`, + value: rowValue, + tableId: table._id, + }, + }) + .queryRows({ + tableId: table._id!, + }) + .filter({ + field: "{{ steps.2.rows.0.value }}", + condition, + value, + }) + .serverLog({ + text: `${condition} condition ${expectPass ? "passed" : "failed"}`, + }) + .run() + + expect(results.steps[2].outputs.result).toBe(expectPass) + if (expectPass) { + expect(results.steps[3].outputs.success).toBeTrue() + } else { + expect(results.steps[3]).toBeUndefined() + } + }) + }) + }) }) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index f477efabe4..6c89bd1f60 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -33,6 +33,7 @@ import { BranchStepInputs, SearchFilters, Branch, + FilterStepInputs, } from "@budibase/types" import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as setup from "../utilities" @@ -161,6 +162,14 @@ class BaseStepBuilder { input ) } + + filter(input: FilterStepInputs): this { + return this.step( + AutomationActionStepId.FILTER, + BUILTIN_ACTION_DEFINITIONS.FILTER, + input + ) + } } class StepBuilder extends BaseStepBuilder { build(): AutomationStep[] { From d4db493519bfe45e662ad5c3c4bdfd5e3b4be6bc Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 18 Sep 2024 11:50:15 +0100 Subject: [PATCH 02/90] Set view permissions to explicit roles from the parent table --- .../server/src/api/controllers/permission.ts | 4 +-- packages/server/src/sdk/app/views/index.ts | 33 ++++++++++++++++--- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index 66a3254348..b75af88067 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -20,7 +20,7 @@ import { import { removeFromArray } from "../../utilities" import sdk from "../../sdk" -const enum PermissionUpdateType { +export const enum PermissionUpdateType { REMOVE = "remove", ADD = "add", } @@ -37,7 +37,7 @@ async function getAllDBRoles(db: Database) { return body.rows.map(row => row.doc!) } -async function updatePermissionOnRole( +export async function updatePermissionOnRole( { roleId, resourceId, diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index d7e05abf2f..c580bfde50 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -1,5 +1,6 @@ import { FieldType, + PermissionLevel, RelationSchemaField, RenameColumn, Table, @@ -10,20 +11,22 @@ import { ViewV2ColumnEnriched, ViewV2Enriched, } from "@budibase/types" -import { HTTPError } from "@budibase/backend-core" +import { HTTPError, roles } from "@budibase/backend-core" import { features } from "@budibase/pro" import { helpers, PROTECTED_EXTERNAL_COLUMNS, PROTECTED_INTERNAL_COLUMNS, } from "@budibase/shared-core" - import * as utils from "../../../db/utils" import { isExternalTableID } from "../../../integrations/utils" - import * as internal from "./internal" import * as external from "./external" import sdk from "../../../sdk" +import { + updatePermissionOnRole, + PermissionUpdateType, +} from "src/api/controllers/permission" function pickApi(tableId: any) { if (isExternalTableID(tableId)) { @@ -123,8 +126,30 @@ export async function create( viewRequest: Omit ): Promise { await guardViewSchema(tableId, viewRequest) + const view = await pickApi(tableId).create(tableId, viewRequest) - return pickApi(tableId).create(tableId, viewRequest) + // Set permissions to be the same as the table + const tablePerms = await sdk.permissions.getResourcePerms(tableId) + const readRole = tablePerms[PermissionLevel.READ]?.role + const writeRole = tablePerms[PermissionLevel.WRITE]?.role + await updatePermissionOnRole( + { + roleId: readRole || roles.BUILTIN_ROLE_IDS.BASIC, + resourceId: view.id, + level: PermissionLevel.READ, + }, + PermissionUpdateType.ADD + ) + await updatePermissionOnRole( + { + roleId: writeRole || roles.BUILTIN_ROLE_IDS.BASIC, + resourceId: view.id, + level: PermissionLevel.WRITE, + }, + PermissionUpdateType.ADD + ) + + return view } export async function update(tableId: string, view: ViewV2): Promise { From 55c7751dbbc6f96f66cc1b8d35d7561f797b528c Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 19 Sep 2024 15:12:03 +0100 Subject: [PATCH 03/90] Move permission updates into SDK --- .../server/src/api/controllers/permission.ts | 107 ++---------------- .../server/src/sdk/app/permissions/index.ts | 102 ++++++++++++++++- packages/server/src/sdk/app/views/index.ts | 5 +- 3 files changed, 111 insertions(+), 103 deletions(-) diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index b75af88067..55b942686f 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -3,7 +3,6 @@ import { UserCtx, Database, Role, - PermissionLevel, GetResourcePermsResponse, ResourcePermissionInfo, GetDependantResourcesResponse, @@ -12,107 +11,15 @@ import { RemovePermissionRequest, RemovePermissionResponse, } from "@budibase/types" -import { getRoleParams } from "../../db/utils" import { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, } from "../../utilities/security" -import { removeFromArray } from "../../utilities" import sdk from "../../sdk" - -export const enum PermissionUpdateType { - REMOVE = "remove", - ADD = "add", -} +import { PermissionUpdateType } from "../../sdk/app/permissions" const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS -// utility function to stop this repetition - permissions always stored under roles -async function getAllDBRoles(db: Database) { - const body = await db.allDocs( - getRoleParams(null, { - include_docs: true, - }) - ) - return body.rows.map(row => row.doc!) -} - -export async function updatePermissionOnRole( - { - roleId, - resourceId, - level, - }: { roleId: string; resourceId: string; level: PermissionLevel }, - updateType: PermissionUpdateType -) { - const db = context.getAppDB() - const remove = updateType === PermissionUpdateType.REMOVE - const isABuiltin = roles.isBuiltin(roleId) - const dbRoleId = roles.getDBRoleID(roleId) - const dbRoles = await getAllDBRoles(db) - const docUpdates: Role[] = [] - - // the permission is for a built in, make sure it exists - if (isABuiltin && !dbRoles.some(role => role._id === dbRoleId)) { - const builtin = roles.getBuiltinRoles()[roleId] - builtin._id = roles.getDBRoleID(builtin._id!) - dbRoles.push(builtin) - } - - // now try to find any roles which need updated, e.g. removing the - // resource from another role and then adding to the new role - for (let role of dbRoles) { - let updated = false - const rolePermissions: Record = role.permissions - ? role.permissions - : {} - // make sure its an array, also handle migrating - if ( - !rolePermissions[resourceId] || - !Array.isArray(rolePermissions[resourceId]) - ) { - rolePermissions[resourceId] = - typeof rolePermissions[resourceId] === "string" - ? [rolePermissions[resourceId] as unknown as PermissionLevel] - : [] - } - // handle the removal/updating the role which has this permission first - // the updating (role._id !== dbRoleId) is required because a resource/level can - // only be permitted in a single role (this reduces hierarchy confusion and simplifies - // the general UI for this, rather than needing to show everywhere it is used) - if ( - (role._id !== dbRoleId || remove) && - rolePermissions[resourceId].indexOf(level) !== -1 - ) { - removeFromArray(rolePermissions[resourceId], level) - updated = true - } - // handle the adding, we're on the correct role, at it to this - if (!remove && role._id === dbRoleId) { - const set = new Set(rolePermissions[resourceId]) - rolePermissions[resourceId] = [...set.add(level)] - updated = true - } - // handle the update, add it to bulk docs to perform at end - if (updated) { - role.permissions = rolePermissions - docUpdates.push(role) - } - } - - const response = await db.bulkDocs(docUpdates) - return response.map(resp => { - const version = docUpdates.find(role => role._id === resp.id)?.version - const _id = roles.getExternalRoleID(resp.id, version) - return { - _id, - rev: resp.rev, - error: resp.error, - reason: resp.reason, - } - }) -} - export function fetchBuiltin(ctx: UserCtx) { ctx.body = Object.values(permissions.getBuiltinPermissions()) } @@ -124,7 +31,7 @@ export function fetchLevels(ctx: UserCtx) { export async function fetch(ctx: UserCtx) { const db = context.getAppDB() - const dbRoles: Role[] = await getAllDBRoles(db) + const dbRoles: Role[] = await sdk.permissions.getAllDBRoles(db) let permissions: any = {} // create an object with structure role ID -> resource ID -> level for (let role of dbRoles) { @@ -186,12 +93,18 @@ export async function getDependantResources( export async function addPermission(ctx: UserCtx) { const params: AddPermissionRequest = ctx.params - ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.ADD) + ctx.body = await sdk.permissions.updatePermissionOnRole( + params, + PermissionUpdateType.ADD + ) } export async function removePermission( ctx: UserCtx ) { const params: RemovePermissionRequest = ctx.params - ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.REMOVE) + ctx.body = await sdk.permissions.updatePermissionOnRole( + params, + PermissionUpdateType.REMOVE + ) } diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts index a6e81652ee..5f8882399b 100644 --- a/packages/server/src/sdk/app/permissions/index.ts +++ b/packages/server/src/sdk/app/permissions/index.ts @@ -1,22 +1,34 @@ -import { db, roles } from "@budibase/backend-core" +import { db, roles, context } from "@budibase/backend-core" import { PermissionLevel, PermissionSource, VirtualDocumentType, + Role, + Database, } from "@budibase/types" -import { extractViewInfoFromID, isViewID } from "../../../db/utils" +import { + extractViewInfoFromID, + isViewID, + getRoleParams, +} from "../../../db/utils" import { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, } from "../../../utilities/security" import sdk from "../../../sdk" import { isV2 } from "../views" +import { removeFromArray } from "../../../utilities" type ResourcePermissions = Record< string, { role: string; type: PermissionSource } > +export const enum PermissionUpdateType { + REMOVE = "remove", + ADD = "add", +} + export async function getInheritablePermissions( resourceId: string ): Promise { @@ -100,3 +112,89 @@ export async function getDependantResources( return } + +export async function updatePermissionOnRole( + { + roleId, + resourceId, + level, + }: { roleId: string; resourceId: string; level: PermissionLevel }, + updateType: PermissionUpdateType +) { + const db = context.getAppDB() + const remove = updateType === PermissionUpdateType.REMOVE + const isABuiltin = roles.isBuiltin(roleId) + const dbRoleId = roles.getDBRoleID(roleId) + const dbRoles = await getAllDBRoles(db) + const docUpdates: Role[] = [] + + // the permission is for a built in, make sure it exists + if (isABuiltin && !dbRoles.some(role => role._id === dbRoleId)) { + const builtin = roles.getBuiltinRoles()[roleId] + builtin._id = roles.getDBRoleID(builtin._id!) + dbRoles.push(builtin) + } + + // now try to find any roles which need updated, e.g. removing the + // resource from another role and then adding to the new role + for (let role of dbRoles) { + let updated = false + const rolePermissions: Record = role.permissions + ? role.permissions + : {} + // make sure its an array, also handle migrating + if ( + !rolePermissions[resourceId] || + !Array.isArray(rolePermissions[resourceId]) + ) { + rolePermissions[resourceId] = + typeof rolePermissions[resourceId] === "string" + ? [rolePermissions[resourceId] as unknown as PermissionLevel] + : [] + } + // handle the removal/updating the role which has this permission first + // the updating (role._id !== dbRoleId) is required because a resource/level can + // only be permitted in a single role (this reduces hierarchy confusion and simplifies + // the general UI for this, rather than needing to show everywhere it is used) + if ( + (role._id !== dbRoleId || remove) && + rolePermissions[resourceId].indexOf(level) !== -1 + ) { + removeFromArray(rolePermissions[resourceId], level) + updated = true + } + // handle the adding, we're on the correct role, at it to this + if (!remove && role._id === dbRoleId) { + const set = new Set(rolePermissions[resourceId]) + rolePermissions[resourceId] = [...set.add(level)] + updated = true + } + // handle the update, add it to bulk docs to perform at end + if (updated) { + role.permissions = rolePermissions + docUpdates.push(role) + } + } + + const response = await db.bulkDocs(docUpdates) + return response.map(resp => { + const version = docUpdates.find(role => role._id === resp.id)?.version + const _id = roles.getExternalRoleID(resp.id, version) + return { + _id, + rev: resp.rev, + error: resp.error, + reason: resp.reason, + } + }) +} + +// utility function to stop this repetition - permissions always stored under roles +export async function getAllDBRoles(db: Database) { + const body = await db.allDocs( + getRoleParams(null, { + include_docs: true, + }) + ) + return body.rows.map(row => row.doc!) +} diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index c580bfde50..47af484ebc 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -23,10 +23,7 @@ import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./internal" import * as external from "./external" import sdk from "../../../sdk" -import { - updatePermissionOnRole, - PermissionUpdateType, -} from "src/api/controllers/permission" +import { updatePermissionOnRole, PermissionUpdateType } from "../permissions" function pickApi(tableId: any) { if (isExternalTableID(tableId)) { From 418bbff2f57e573ce9f36796f5fb77014aaa4870 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 19 Sep 2024 15:15:19 +0100 Subject: [PATCH 04/90] Lint --- packages/server/src/api/controllers/permission.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index 55b942686f..c7afb6a351 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -1,7 +1,6 @@ import { permissions, roles, context } from "@budibase/backend-core" import { UserCtx, - Database, Role, GetResourcePermsResponse, ResourcePermissionInfo, From ddc360a3da3a1c3c31a5e7b6cb7daebdbd7caaba Mon Sep 17 00:00:00 2001 From: mikesealey Date: Fri, 20 Sep 2024 11:54:26 +0100 Subject: [PATCH 05/90] tweaks sizing of multi-attachment preview in compact setting --- packages/bbui/src/Form/Core/Dropzone.svelte | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte index c69bf0d6bb..e223d789a1 100644 --- a/packages/bbui/src/Form/Core/Dropzone.svelte +++ b/packages/bbui/src/Form/Core/Dropzone.svelte @@ -396,6 +396,10 @@ padding: 6px 10px; margin-bottom: 8px; } + + .compact .gallery > * { + max-height: 25px; + } .title { display: flex; flex-direction: row; From 4975ae7fef1b2bcac717a3706194743d51f0cf8b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Sep 2024 16:58:06 +0100 Subject: [PATCH 06/90] Extra test logging. --- .github/workflows/budibase_ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 4b9ebf1e5d..388f2000ed 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -186,6 +186,8 @@ jobs: - run: yarn --frozen-lockfile - name: Test server + env: + DEBUG: "testcontainers*" run: | if ${{ env.ONLY_AFFECTED_TASKS }}; then node scripts/run-affected.js --task=test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }} From da805b10ae576f23fd6d8d823d655a3d20382a49 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Sep 2024 17:31:44 +0100 Subject: [PATCH 07/90] Set mssql SHA back to 2019. --- packages/server/datasource-sha.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 61249d530c..9b935ed8eb 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -1,4 +1,4 @@ -MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545 +MSSQL_SHA=sha256:c4369c38385eba011c10906dc8892425831275bb035d5ce69656da8e29de50d8 MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d From 51e09ddf7b2c2d0274cd2b4a503a375993505f1d Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 23 Sep 2024 10:08:21 +0100 Subject: [PATCH 08/90] Update row action tests to revoke explicit view permissions when testing triggering against views --- packages/server/src/api/routes/tests/rowAction.spec.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts index ef7d2afbba..efd28eb92f 100644 --- a/packages/server/src/api/routes/tests/rowAction.spec.ts +++ b/packages/server/src/api/routes/tests/rowAction.spec.ts @@ -826,11 +826,20 @@ describe("/rowsActions", () => { ) ).id + // Allow row action on view await config.api.rowAction.setViewPermission( tableId, viewId, rowAction.id ) + + // Delete explicit view permissions so they inherit table permissions + await config.api.permission.revoke({ + level: PermissionLevel.READ, + resourceId: viewId, + roleId: "inherited", + }) + return { permissionResource: tableId, triggerResouce: viewId } }, ], From 028d15911ee1d02d17382c7b127d2bbf61935114 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Sep 2024 11:47:08 +0100 Subject: [PATCH 09/90] Revert "Set mssql SHA back to 2019." This reverts commit da805b10ae576f23fd6d8d823d655a3d20382a49. --- packages/server/datasource-sha.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 9b935ed8eb..61249d530c 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -1,4 +1,4 @@ -MSSQL_SHA=sha256:c4369c38385eba011c10906dc8892425831275bb035d5ce69656da8e29de50d8 +MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545 MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d From 8a5a94338ad7b63d83288b73b24358f2a2a420de Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Sep 2024 14:18:34 +0100 Subject: [PATCH 10/90] Remove extra logging. --- .github/workflows/budibase_ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 388f2000ed..4b9ebf1e5d 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -186,8 +186,6 @@ jobs: - run: yarn --frozen-lockfile - name: Test server - env: - DEBUG: "testcontainers*" run: | if ${{ env.ONLY_AFFECTED_TASKS }}; then node scripts/run-affected.js --task=test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }} From 980615b37b5b83b109278a012093ee06e9bc0e19 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 24 Sep 2024 11:08:44 +0100 Subject: [PATCH 11/90] looping query rows --- .../tests/scenarios/looping.spec.ts | 146 ++++++++++++++++++ .../tests/scenarios/scenarios.spec.ts | 2 +- 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/packages/server/src/automations/tests/scenarios/looping.spec.ts b/packages/server/src/automations/tests/scenarios/looping.spec.ts index 9bc382a187..9f7ff04156 100644 --- a/packages/server/src/automations/tests/scenarios/looping.spec.ts +++ b/packages/server/src/automations/tests/scenarios/looping.spec.ts @@ -5,6 +5,7 @@ import { LoopStepType, CreateRowStepOutputs, ServerLogStepOutputs, + FieldType, } from "@budibase/types" import { createAutomationBuilder } from "../utilities/AutomationTestBuilder" @@ -242,4 +243,149 @@ describe("Loop automations", () => { expect(results.steps[1].outputs.message).toContain("- 3") expect(results.steps[3].outputs.message).toContain("- 3") }) + + it("should run an automation with a loop and update row step", async () => { + const table = await config.createTable({ + name: "TestTable", + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + value: { + name: "value", + type: FieldType.NUMBER, + constraints: { + presence: true, + }, + }, + }, + }) + + const rows = [ + { name: "Row 1", value: 1, tableId: table._id }, + { name: "Row 2", value: 2, tableId: table._id }, + { name: "Row 3", value: 3, tableId: table._id }, + ] + + for (const row of rows) { + await config.createRow(row) + } + + const builder = createAutomationBuilder({ + name: "Test Loop and Update Row", + }) + + const results = await builder + .appAction({ fields: {} }) + .queryRows({ + tableId: table._id!, + }) + .loop({ + option: LoopStepType.ARRAY, + binding: "{{ steps.1.rows }}", + }) + .updateRow({ + rowId: "{{ loop.currentItem._id }}", + row: { + name: "Updated {{ loop.currentItem.name }}", + value: "{{ loop.currentItem.value }}", + tableId: table._id, + }, + meta: {}, + }) + .queryRows({ + tableId: table._id!, + }) + .run() + + const expectedRows = [ + { name: "Updated Row 1", value: 1 }, + { name: "Updated Row 2", value: 2 }, + { name: "Updated Row 3", value: 3 }, + ] + + expect(results.steps[1].outputs.items).toEqual( + expect.arrayContaining( + expectedRows.map(row => + expect.objectContaining({ + success: true, + row: expect.objectContaining(row), + }) + ) + ) + ) + + expect(results.steps[2].outputs.rows).toEqual( + expect.arrayContaining( + expectedRows.map(row => expect.objectContaining(row)) + ) + ) + + expect(results.steps[1].outputs.items).toHaveLength(expectedRows.length) + expect(results.steps[2].outputs.rows).toHaveLength(expectedRows.length) + }) + + it("should run an automation with a loop and delete row step", async () => { + const table = await config.createTable({ + name: "TestTable", + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + value: { + name: "value", + type: FieldType.NUMBER, + constraints: { + presence: true, + }, + }, + }, + }) + + const rows = [ + { name: "Row 1", value: 1, tableId: table._id }, + { name: "Row 2", value: 2, tableId: table._id }, + { name: "Row 3", value: 3, tableId: table._id }, + ] + + for (const row of rows) { + await config.createRow(row) + } + + const builder = createAutomationBuilder({ + name: "Test Loop and Delete Row", + }) + + const results = await builder + .appAction({ fields: {} }) + .queryRows({ + tableId: table._id!, + }) + .loop({ + option: LoopStepType.ARRAY, + binding: "{{ steps.1.rows }}", + }) + .deleteRow({ + tableId: table._id!, + id: "{{ loop.currentItem._id }}", + }) + .queryRows({ + tableId: table._id!, + }) + .run() + + expect(results.steps).toHaveLength(3) + + expect(results.steps[2].outputs.rows).toHaveLength(0) + }) }) diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts index 62a1b9db8f..12b8392ffc 100644 --- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -197,7 +197,7 @@ describe("Automation Scenarios", () => { ) }) }) - describe.only("Automations with filter", () => { + describe("Automations with filter", () => { let table: Table beforeEach(async () => { From 3e9ca562c598a7099ba9c18669b8d6cda8f37013 Mon Sep 17 00:00:00 2001 From: mikesealey Date: Tue, 24 Sep 2024 12:12:46 +0100 Subject: [PATCH 12/90] reworks changes based on feedback --- packages/bbui/src/Form/Core/Dropzone.svelte | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte index e223d789a1..2922d88e7a 100644 --- a/packages/bbui/src/Form/Core/Dropzone.svelte +++ b/packages/bbui/src/Form/Core/Dropzone.svelte @@ -397,9 +397,10 @@ margin-bottom: 8px; } - .compact .gallery > * { - max-height: 25px; + .compact .placeholder { + height: fit-content; } + .title { display: flex; flex-direction: row; From 0c6946af62fae3a1f17926fd7f4c2cbed72486c4 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 24 Sep 2024 15:01:05 +0100 Subject: [PATCH 13/90] more automation tests --- .../tests/scenarios/scenarios.spec.ts | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts index b54b7c0e0e..4e9340992d 100644 --- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -196,6 +196,91 @@ describe("Automation Scenarios", () => { ) }) }) + + it("should trigger an automation which creates and then updates a row", async () => { + const table = await config.createTable({ + name: "TestTable", + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + value: { + name: "value", + type: FieldType.NUMBER, + constraints: { + presence: true, + }, + }, + }, + }) + + const builder = createAutomationBuilder({ + name: "Test Create and Update Row", + }) + + const results = await builder + .appAction({ fields: {} }) + .createRow( + { + row: { + name: "Initial Row", + value: 1, + tableId: table._id, + }, + }, + { stepName: "CreateRowStep" } + ) + .updateRow( + { + rowId: "{{ steps.CreateRowStep.row._id }}", + row: { + name: "Updated Row", + value: 2, + tableId: table._id, + }, + meta: {}, + }, + { stepName: "UpdateRowStep" } + ) + .queryRows( + { + tableId: table._id!, + }, + { stepName: "QueryRowsStep" } + ) + .run() + + expect(results.steps).toHaveLength(3) + + expect(results.steps[0].outputs).toMatchObject({ + success: true, + row: { + name: "Initial Row", + value: 1, + }, + }) + + expect(results.steps[1].outputs).toMatchObject({ + success: true, + row: { + name: "Updated Row", + value: 2, + }, + }) + + const expectedRows = [{ name: "Updated Row", value: 2 }] + + expect(results.steps[2].outputs.rows).toEqual( + expect.arrayContaining( + expectedRows.map(row => expect.objectContaining(row)) + ) + ) + }) }) describe("Name Based Automations", () => { From 4a55021844c163c82d98c087a4ddf1fa2dea33db Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 24 Sep 2024 16:15:07 +0100 Subject: [PATCH 14/90] refs --- packages/account-portal | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index 7899d07904..26903524ff 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 7899d07904d89d48954dd500da7b5dec32b781dd +Subproject commit 26903524ffb6e8f2d99b667ce7a84f09ea238073 diff --git a/packages/pro b/packages/pro index ec1d2bda75..e2fe0f9cc8 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit ec1d2bda756f02c6b4efdee086e4c59b0c2a1b0c +Subproject commit e2fe0f9cc856b4ee1a97df96d623b2d87d4e8733 From d7d8284caf06b512b915c58c1e5d52cd56ebf6bc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 24 Sep 2024 17:42:25 +0200 Subject: [PATCH 15/90] Add test --- .../src/api/routes/tests/viewV2.spec.ts | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index c4a39ae8a9..c34f4fb3ac 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -22,6 +22,7 @@ import { TableSchema, ViewFieldMetadata, RenameColumn, + FeatureFlag, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -2213,6 +2214,100 @@ describe.each([ }) ) }) + + describe("foreign relationship columns", () => { + const createMainTable = async ( + links: { + name: string + tableId: string + fk: string + }[] + ) => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { title: { name: "title", type: FieldType.STRING } }, + }) + ) + await config.api.table.save({ + ...table, + schema: { + ...table.schema, + ...links.reduce((acc, c) => { + acc[c.name] = { + name: c.name, + relationshipType: RelationshipType.ONE_TO_MANY, + type: FieldType.LINK, + tableId: c.tableId, + fieldName: c.fk, + constraints: { type: "array" }, + } + return acc + }, {}), + }, + }) + return table + } + const createAuxTable = (schema: TableSchema) => + config.api.table.save( + saveTableRequest({ + primaryDisplay: "name", + schema: { + ...schema, + name: { name: "name", type: FieldType.STRING }, + }, + }) + ) + + it("returns squashed fields respecting the view config", async () => { + const auxTable = await createAuxTable({ + age: { name: "age", type: FieldType.NUMBER }, + }) + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + ]) + + const auxRow = await config.api.row.save(auxTable._id!, { + name: generator.name(), + age: generator.age(), + }) + const row = await config.api.row.save(table._id!, { + title: generator.word(), + aux: [auxRow], + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + title: { visible: true }, + aux: { + visible: true, + columns: { + name: { visible: false, readonly: false }, + age: { visible: true, readonly: true }, + }, + }, + }, + }) + + const response = await withCoreEnv( + { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}` }, + () => config.api.viewV2.search(view.id) + ) + + expect(response.rows).toEqual([ + expect.objectContaining({ + aux: [ + { + _id: auxRow._id, + primaryDisplay: auxRow.name, + age: auxRow.age, + }, + ], + }), + ]) + }) + }) }) describe("permissions", () => { From 53b4634cffb48d254c7d4e1b5a91f4a61f017880 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 24 Sep 2024 17:51:05 +0200 Subject: [PATCH 16/90] Add enrichment tests --- .../src/api/routes/tests/viewV2.spec.ts | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index c34f4fb3ac..a2af95bee3 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -23,6 +23,7 @@ import { ViewFieldMetadata, RenameColumn, FeatureFlag, + BBReferenceFieldSubType, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -2270,7 +2271,7 @@ describe.each([ name: generator.name(), age: generator.age(), }) - const row = await config.api.row.save(table._id!, { + await config.api.row.save(table._id!, { title: generator.word(), aux: [auxRow], }) @@ -2307,6 +2308,69 @@ describe.each([ }), ]) }) + + it("enriches squashed fields", async () => { + const auxTable = await createAuxTable({ + user: { + name: "user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + constraints: { presence: true }, + }, + }) + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + ]) + + const user = config.getUser() + const auxRow = await config.api.row.save(auxTable._id!, { + name: generator.name(), + user: user._id, + }) + await config.api.row.save(table._id!, { + title: generator.word(), + aux: [auxRow], + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + title: { visible: true }, + aux: { + visible: true, + columns: { + name: { visible: true, readonly: true }, + user: { visible: true, readonly: true }, + }, + }, + }, + }) + + const response = await withCoreEnv( + { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}` }, + () => config.api.viewV2.search(view.id) + ) + + expect(response.rows).toEqual([ + expect.objectContaining({ + aux: [ + { + _id: auxRow._id, + primaryDisplay: auxRow.name, + name: auxRow.name, + user: { + _id: user._id, + email: user.email, + firstName: user.firstName, + lastName: user.lastName, + primaryDisplay: user.email, + }, + }, + ], + }), + ]) + }) }) }) From 7a7ce3dc629ed365a67cbdbae4d812d80ffc7440 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 24 Sep 2024 17:56:07 +0200 Subject: [PATCH 17/90] Fix --- packages/server/src/db/linkedRows/index.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 2c8d1f77ac..c2b043785f 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -10,7 +10,7 @@ import flatten from "lodash/flatten" import { USER_METDATA_PREFIX } from "../utils" import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" -import { processFormulas } from "../../utilities/rowProcessor" +import { outputProcessing, processFormulas } from "../../utilities/rowProcessor" import { context, features } from "@budibase/backend-core" import { ContextUser, @@ -275,7 +275,7 @@ export async function squashLinks( // will populate this as we find them const linkedTables = [table] const isArray = Array.isArray(enriched) - const enrichedArray = !isArray ? [enriched] : enriched + const enrichedArray = !isArray ? [enriched as Row] : (enriched as Row[]) for (const row of enrichedArray) { // this only fetches the table if its not already in array const rowTable = await getLinkedTable(row.tableId!, linkedTables) @@ -292,6 +292,9 @@ export async function squashLinks( obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable) if (viewSchema[column]?.columns) { + const enrichedLink = await outputProcessing(linkedTable, link, { + squash: false, + }) const squashFields = Object.entries(viewSchema[column].columns) .filter(([columnName, viewColumnConfig]) => { const tableColumn = linkedTable.schema[columnName] @@ -312,7 +315,7 @@ export async function squashLinks( .map(([columnName]) => columnName) for (const relField of squashFields) { - obj[relField] = link[relField] + obj[relField] = enrichedLink[relField] } } @@ -321,5 +324,5 @@ export async function squashLinks( row[column] = newLinks } } - return isArray ? enrichedArray : enrichedArray[0] + return (isArray ? enrichedArray : enrichedArray[0]) as T } From 9ecb64a99229abc626012be597af55f54eba2e0a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 25 Sep 2024 10:32:58 +0200 Subject: [PATCH 18/90] Use sqs flag correctly on test --- packages/server/src/api/routes/tests/viewV2.spec.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index a2af95bee3..a47d6dd828 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -2291,8 +2291,17 @@ describe.each([ }, }) + const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`] + if (isLucene) { + flags.push("*:!SQS") + } else if (isSqs) { + flags.push("*:SQS") + } + const response = await withCoreEnv( - { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}` }, + { + TENANT_FEATURE_FLAGS: flags.join(","), + }, () => config.api.viewV2.search(view.id) ) From 7072244f31a165a0b3af80bae55922b725e6d13a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 25 Sep 2024 12:44:30 +0200 Subject: [PATCH 19/90] Fix --- .../src/api/routes/tests/viewV2.spec.ts | 71 ++++++++++--------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index a47d6dd828..e38e4c2ed5 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -34,6 +34,7 @@ import { roles, withEnv as withCoreEnv, setEnv as setCoreEnv, + env, } from "@budibase/backend-core" import sdk from "../../../sdk" @@ -696,22 +697,23 @@ describe.each([ ) }) - it("cannot update views v1", async () => { - const viewV1 = await config.api.legacyView.save({ - tableId: table._id!, - name: generator.guid(), - filters: [], - schema: {}, - }) + isInternal && + it("cannot update views v1", async () => { + const viewV1 = await config.api.legacyView.save({ + tableId: table._id!, + name: generator.guid(), + filters: [], + schema: {}, + }) - await config.api.viewV2.update(viewV1 as unknown as ViewV2, { - status: 400, - body: { - message: "Only views V2 can be updated", + await config.api.viewV2.update(viewV1 as unknown as ViewV2, { status: 400, - }, + body: { + message: "Only views V2 can be updated", + status: 400, + }, + }) }) - }) it("cannot update the a view with unmatching ids between url and body", async () => { const anotherView = await config.api.viewV2.create({ @@ -2217,6 +2219,21 @@ describe.each([ }) describe("foreign relationship columns", () => { + let envCleanup: () => void + beforeAll(() => { + const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`] + if (env.TENANT_FEATURE_FLAGS) { + flags.push(...env.TENANT_FEATURE_FLAGS.split(",")) + } + envCleanup = setCoreEnv({ + TENANT_FEATURE_FLAGS: flags.join(","), + }) + }) + + afterAll(() => { + envCleanup?.() + }) + const createMainTable = async ( links: { name: string @@ -2263,14 +2280,14 @@ describe.each([ const auxTable = await createAuxTable({ age: { name: "age", type: FieldType.NUMBER }, }) - const table = await createMainTable([ - { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, - ]) - const auxRow = await config.api.row.save(auxTable._id!, { name: generator.name(), age: generator.age(), }) + + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + ]) await config.api.row.save(table._id!, { title: generator.word(), aux: [auxRow], @@ -2291,20 +2308,7 @@ describe.each([ }, }) - const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`] - if (isLucene) { - flags.push("*:!SQS") - } else if (isSqs) { - flags.push("*:SQS") - } - - const response = await withCoreEnv( - { - TENANT_FEATURE_FLAGS: flags.join(","), - }, - () => config.api.viewV2.search(view.id) - ) - + const response = await config.api.viewV2.search(view.id) expect(response.rows).toEqual([ expect.objectContaining({ aux: [ @@ -2356,10 +2360,7 @@ describe.each([ }, }) - const response = await withCoreEnv( - { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}` }, - () => config.api.viewV2.search(view.id) - ) + const response = await config.api.viewV2.search(view.id) expect(response.rows).toEqual([ expect.objectContaining({ From 77587c86868b9970daa9664db3a38d3befe6b847 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 12:23:32 +0200 Subject: [PATCH 20/90] Add related columns --- .../GridColumnConfiguration/getColumns.js | 44 ++++++++++++++++--- .../src/components/app/GridBlock.svelte | 1 + .../src/components/grid/stores/datasource.js | 14 ++++++ 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js index 148055d727..384f20cf65 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js @@ -1,3 +1,5 @@ +import { FieldType } from "@budibase/types" + const modernize = columns => { if (!columns) { return [] @@ -8,6 +10,7 @@ const modernize = columns => { label: column.displayName, field: column.name, active: true, + related: column.related, })) } @@ -50,12 +53,35 @@ const removeInvalidAddMissing = ( const getDefault = (schema = {}) => { const defaultValues = Object.values(schema) .filter(column => !column.nestedJSON) - .map(column => ({ - label: column.name, - field: column.name, - active: column.visible ?? true, - order: column.visible ? column.order ?? -1 : Number.MAX_SAFE_INTEGER, - })) + .flatMap(column => { + const order = column.visible + ? column.order ?? -1 + : Number.MAX_SAFE_INTEGER + const columns = [ + { + label: column.name, + field: column.name, + active: column.visible ?? true, + order, + }, + ] + + if (column.columns) { + for (const relColumn of Object.keys(column.columns).filter( + relColumn => column.columns[relColumn].visible !== false + )) { + columns.push({ + label: `${relColumn} (${column.name})`, + field: `${column.name}.${relColumn}`, + active: column.visible ?? true, + order, + related: true, + }) + } + } + + return columns + }) defaultValues.sort((a, b) => a.order - b.order) @@ -69,6 +95,7 @@ const toGridFormat = draggableListColumns => { active: entry.active, width: entry.width, conditions: entry.conditions, + related: entry.related, })) } @@ -82,9 +109,12 @@ const toDraggableListFormat = (gridFormatColumns, createComponent, schema) => { active: column.active, field: column.field, label: column.label, - columnType: schema[column.field].type, + columnType: column.related + ? FieldType.FORMULA + : schema[column.field]?.type, width: column.width, conditions: column.conditions, + related: column.related, }, {} ) diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte index 30a35b0713..45aba66bab 100644 --- a/packages/client/src/components/app/GridBlock.svelte +++ b/packages/client/src/components/app/GridBlock.svelte @@ -98,6 +98,7 @@ order: idx, conditions: column.conditions, visible: !!column.active, + related: column.related, } if (column.width) { overrides[column.field].width = column.width diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index 68053f38ae..3af6948650 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -1,6 +1,7 @@ import { derived, get } from "svelte/store" import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch" import { memo } from "../../../utils" +import { FieldType } from "@budibase/types" export const createStores = () => { const definition = memo(null) @@ -73,6 +74,19 @@ export const deriveStores = context => { } } }) + if ($schemaOverrides) { + Object.keys($schemaOverrides).forEach(field => { + if (!$schemaOverrides[field].related) { + return + } + enrichedSchema[field] = { + ...$schemaOverrides[field], + name: field, + type: FieldType.FORMULA, + related: true, + } + }) + } return enrichedSchema } ) From 55190883763eb2b58f64a10d79fe557c8ee7872a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 12:36:04 +0200 Subject: [PATCH 21/90] Display related --- .../GridColumnConfiguration/getColumns.js | 2 +- .../src/components/grid/stores/datasource.js | 1 - .../src/components/grid/stores/rows.js | 32 +++++++++++++++---- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js index 384f20cf65..419628ead2 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js @@ -75,7 +75,7 @@ const getDefault = (schema = {}) => { field: `${column.name}.${relColumn}`, active: column.visible ?? true, order, - related: true, + related: { field: column.name, subField: relColumn }, }) } } diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index 3af6948650..6e0b31de4b 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -83,7 +83,6 @@ export const deriveStores = context => { ...$schemaOverrides[field], name: field, type: FieldType.FORMULA, - related: true, } }) } diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index fb7a487c8e..e13767cbca 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -6,6 +6,7 @@ import { tick } from "svelte" import { Helpers } from "@budibase/bbui" import { sleep } from "../../../utils/utils" import { FieldType } from "@budibase/types" +import { processStringSync } from "@budibase/string-templates" export const createStores = () => { const rows = writable([]) @@ -42,15 +43,32 @@ export const createStores = () => { } export const deriveStores = context => { - const { rows } = context + const { rows, enrichedSchema } = context // Enrich rows with an index property and any pending changes - const enrichedRows = derived(rows, $rows => { - return $rows.map((row, idx) => ({ - ...row, - __idx: idx, - })) - }) + const enrichedRows = derived( + [rows, enrichedSchema], + ([$rows, $enrichedSchema]) => { + const customColumns = Object.values($enrichedSchema || {}).filter( + f => f.related + ) + return $rows.map((row, idx) => ({ + ...row, + __idx: idx, + ...customColumns.reduce((acc, c) => { + try { + acc[c.name] = processStringSync( + `{{ join (pluck ${c.related.field} '${c.related.subField}') ', ' }}`, + row + ) + } catch { + // It might be some formula not set, or anything being incorrect + } + return acc + }, {}), + })) + } + ) // Generate a lookup map to quick find a row by ID const rowLookupMap = derived(enrichedRows, $enrichedRows => { From 49502afbeed80088fd5e161e4ab6bd886b3cbe9e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 13:04:57 +0200 Subject: [PATCH 22/90] Default off --- .../settings/controls/GridColumnConfiguration/getColumns.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js index 419628ead2..22b42dd427 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js @@ -73,7 +73,7 @@ const getDefault = (schema = {}) => { columns.push({ label: `${relColumn} (${column.name})`, field: `${column.name}.${relColumn}`, - active: column.visible ?? true, + active: false, order, related: { field: column.name, subField: relColumn }, }) From 8ee6e52f08c133231e94dd6b607830421be4de6a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 13:24:10 +0200 Subject: [PATCH 23/90] Use types --- .../frontend-core/src/components/grid/stores/datasource.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index 6e0b31de4b..ed84dc478a 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -79,10 +79,14 @@ export const deriveStores = context => { if (!$schemaOverrides[field].related) { return } + + const { field: relField, subField: relSubField } = + $schemaOverrides[field].related + enrichedSchema[field] = { ...$schemaOverrides[field], name: field, - type: FieldType.FORMULA, + type: $schema[relField]?.columns?.[relSubField]?.type, } }) } From 574b2e5a3c16668a710df57d2625e30906a6329b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 13:34:23 +0200 Subject: [PATCH 24/90] Map types --- .../GridColumnConfiguration/getColumns.js | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js index 22b42dd427..7380e40de0 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js @@ -1,5 +1,3 @@ -import { FieldType } from "@budibase/types" - const modernize = columns => { if (!columns) { return [] @@ -20,7 +18,8 @@ const modernize = columns => { const removeInvalidAddMissing = ( columns = [], defaultColumns, - primaryDisplayColumnName + primaryDisplayColumnName, + schema ) => { const defaultColumnNames = defaultColumns.map(column => column.field) const columnNames = columns.map(column => column.field) @@ -47,6 +46,16 @@ const removeInvalidAddMissing = ( combinedColumns[primaryDisplayIndex].active = true } + for (const column of combinedColumns) { + if (!column.related) { + column.columnType = schema[column.field]?.type + continue + } + + const { field: relField, subField: relSubField } = column.related + column.columnType = schema[relField]?.columns?.[relSubField]?.type + } + return combinedColumns } @@ -99,7 +108,7 @@ const toGridFormat = draggableListColumns => { })) } -const toDraggableListFormat = (gridFormatColumns, createComponent, schema) => { +const toDraggableListFormat = (gridFormatColumns, createComponent) => { return gridFormatColumns.map(column => { return createComponent( "@budibase/standard-components/labelfield", @@ -109,9 +118,7 @@ const toDraggableListFormat = (gridFormatColumns, createComponent, schema) => { active: column.active, field: column.field, label: column.label, - columnType: column.related - ? FieldType.FORMULA - : schema[column.field]?.type, + columnType: column.columnType, width: column.width, conditions: column.conditions, related: column.related, @@ -131,13 +138,10 @@ const getColumns = ({ const validatedColumns = removeInvalidAddMissing( modernize(columns), getDefault(schema), - primaryDisplayColumnName - ) - const draggableList = toDraggableListFormat( - validatedColumns, - createComponent, + primaryDisplayColumnName, schema ) + const draggableList = toDraggableListFormat(validatedColumns, createComponent) const primary = draggableList .filter(entry => entry.field === primaryDisplayColumnName) .map(instance => ({ ...instance, schema }))[0] From d799bfacb60022c44c95ace62ca24f08fd9dfd04 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 13:35:11 +0200 Subject: [PATCH 25/90] Readonly --- packages/frontend-core/src/components/grid/stores/datasource.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index ed84dc478a..399ec341ee 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -87,6 +87,7 @@ export const deriveStores = context => { ...$schemaOverrides[field], name: field, type: $schema[relField]?.columns?.[relSubField]?.type, + readonly: true, } }) } From 4aa9e08ec564083ca6ff0e38f0feb45767631fa3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 13:39:04 +0200 Subject: [PATCH 26/90] Don't show non visible even if previously configured --- .../frontend-core/src/components/grid/stores/datasource.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index 399ec341ee..65e925fdf2 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -83,6 +83,10 @@ export const deriveStores = context => { const { field: relField, subField: relSubField } = $schemaOverrides[field].related + if (!$schema[relField]?.columns?.[relSubField]?.visible) { + return + } + enrichedSchema[field] = { ...$schemaOverrides[field], name: field, From e9db3d64e7a8914cb154c654417e74ddd642e799 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 13:50:11 +0200 Subject: [PATCH 27/90] Extra enrichement --- .../frontend-core/src/components/grid/stores/datasource.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index 65e925fdf2..a7a3083cde 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -83,7 +83,10 @@ export const deriveStores = context => { const { field: relField, subField: relSubField } = $schemaOverrides[field].related - if (!$schema[relField]?.columns?.[relSubField]?.visible) { + if ( + !$schema[relField].visible || + !$schema[relField]?.columns?.[relSubField]?.visible + ) { return } From 488165d85985056ad3b6034bb10aa87ec47347d3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 14:00:53 +0200 Subject: [PATCH 28/90] Lint --- packages/frontend-core/src/components/grid/stores/datasource.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index a7a3083cde..bdd338618c 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -1,7 +1,6 @@ import { derived, get } from "svelte/store" import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch" import { memo } from "../../../utils" -import { FieldType } from "@budibase/types" export const createStores = () => { const definition = memo(null) From eed82075fd6c678a4b509a76a9e3e7ef1060603f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 9 Sep 2024 09:52:09 +0200 Subject: [PATCH 29/90] Enrich visible columns --- .../src/components/grid/stores/columns.js | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js index 0073754a5d..9648b367f9 100644 --- a/packages/frontend-core/src/components/grid/stores/columns.js +++ b/packages/frontend-core/src/components/grid/stores/columns.js @@ -31,7 +31,7 @@ export const createStores = () => { } export const deriveStores = context => { - const { columns } = context + const { columns, enrichedSchema } = context // Derive a lookup map for all columns by name const columnLookupMap = derived(columns, $columns => { @@ -43,9 +43,38 @@ export const deriveStores = context => { }) // Derived list of columns which have not been explicitly hidden - const visibleColumns = derived(columns, $columns => { - return $columns.filter(col => col.visible) - }) + const visibleColumns = derived( + [columns, enrichedSchema], + ([$columns, $enrichedSchema]) => { + return $columns + .filter(col => col.visible) + .flatMap(c => { + const relatedColumns = [] + + const schemaColumns = $enrichedSchema?.[c.name]?.columns + if (schemaColumns) { + for (const relColumn of Object.keys(schemaColumns)) { + const relFieldSchema = schemaColumns[relColumn] + if (!relFieldSchema.visible) { + continue + } + relatedColumns.push({ + name: `${c.name}.${relColumn}`, + label: `${relColumn} (${c.name})`, + schema: relFieldSchema, + width: relFieldSchema.width || DefaultColumnWidth, + visible: relFieldSchema.visible ?? true, + readonly: relFieldSchema.readonly, + order: relFieldSchema.order, + conditions: relFieldSchema.conditions, + }) + } + } + + return [c, ...relatedColumns] + }) + } + ) // Split visible columns into their discrete types const displayColumn = derived(visibleColumns, $visibleColumns => { @@ -136,7 +165,7 @@ export const initialise = context => { .map(field => { const fieldSchema = $enrichedSchema[field] const oldColumn = $columns?.find(col => col.name === field) - let column = { + const column = { name: field, label: fieldSchema.displayName || field, schema: fieldSchema, From 4272b614e254927ddbea2bce2762ca4128854522 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 9 Sep 2024 09:58:43 +0200 Subject: [PATCH 30/90] Cleanup --- .../GridColumnConfiguration/getColumns.js | 17 ----------------- .../client/src/components/app/GridBlock.svelte | 1 - 2 files changed, 18 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js index 7380e40de0..f5ac106162 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js @@ -8,7 +8,6 @@ const modernize = columns => { label: column.displayName, field: column.name, active: true, - related: column.related, })) } @@ -75,20 +74,6 @@ const getDefault = (schema = {}) => { }, ] - if (column.columns) { - for (const relColumn of Object.keys(column.columns).filter( - relColumn => column.columns[relColumn].visible !== false - )) { - columns.push({ - label: `${relColumn} (${column.name})`, - field: `${column.name}.${relColumn}`, - active: false, - order, - related: { field: column.name, subField: relColumn }, - }) - } - } - return columns }) @@ -104,7 +89,6 @@ const toGridFormat = draggableListColumns => { active: entry.active, width: entry.width, conditions: entry.conditions, - related: entry.related, })) } @@ -121,7 +105,6 @@ const toDraggableListFormat = (gridFormatColumns, createComponent) => { columnType: column.columnType, width: column.width, conditions: column.conditions, - related: column.related, }, {} ) diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte index 45aba66bab..30a35b0713 100644 --- a/packages/client/src/components/app/GridBlock.svelte +++ b/packages/client/src/components/app/GridBlock.svelte @@ -98,7 +98,6 @@ order: idx, conditions: column.conditions, visible: !!column.active, - related: column.related, } if (column.width) { overrides[column.field].width = column.width From 6d8505cee617cd3ebdc50d5edaf4d6733ba97688 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 9 Sep 2024 10:18:43 +0200 Subject: [PATCH 31/90] Add tablecolumns --- .../grid/controls/ColumnsSettingButton.svelte | 12 ++-- .../src/components/grid/stores/columns.js | 69 ++++++++++--------- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte index 2a3ca139fc..75108870bd 100644 --- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte +++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte @@ -4,13 +4,15 @@ import ColumnsSettingContent from "./ColumnsSettingContent.svelte" import { FieldPermissions } from "../../../constants" - const { columns, datasource } = getContext("grid") + const { tableColumns, datasource } = getContext("grid") let open = false let anchor - $: anyRestricted = $columns.filter(col => !col.visible || col.readonly).length - $: text = anyRestricted ? `Columns: (${anyRestricted} restricted)` : "Columns" + $: anyRestricted = $tableColumns.filter( + col => !col.visible || col.readonly + ).length + $: text = anyRestricted ? `Columns (${anyRestricted} restricted)` : "Columns" $: permissions = $datasource.type === "viewV2" ? [ @@ -28,12 +30,12 @@ size="M" on:click={() => (open = !open)} selected={open || anyRestricted} - disabled={!$columns.length} + disabled={!$tableColumns.length} > {text} - + diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js index 9648b367f9..b20f17fd4d 100644 --- a/packages/frontend-core/src/components/grid/stores/columns.js +++ b/packages/frontend-core/src/components/grid/stores/columns.js @@ -31,7 +31,7 @@ export const createStores = () => { } export const deriveStores = context => { - const { columns, enrichedSchema } = context + const { columns } = context // Derive a lookup map for all columns by name const columnLookupMap = derived(columns, $columns => { @@ -42,39 +42,15 @@ export const deriveStores = context => { return map }) + // Derived list of columns which are direct part of the table + const tableColumns = derived(columns, $columns => { + return $columns.filter(col => !col.related) + }) + // Derived list of columns which have not been explicitly hidden - const visibleColumns = derived( - [columns, enrichedSchema], - ([$columns, $enrichedSchema]) => { - return $columns - .filter(col => col.visible) - .flatMap(c => { - const relatedColumns = [] - - const schemaColumns = $enrichedSchema?.[c.name]?.columns - if (schemaColumns) { - for (const relColumn of Object.keys(schemaColumns)) { - const relFieldSchema = schemaColumns[relColumn] - if (!relFieldSchema.visible) { - continue - } - relatedColumns.push({ - name: `${c.name}.${relColumn}`, - label: `${relColumn} (${c.name})`, - schema: relFieldSchema, - width: relFieldSchema.width || DefaultColumnWidth, - visible: relFieldSchema.visible ?? true, - readonly: relFieldSchema.readonly, - order: relFieldSchema.order, - conditions: relFieldSchema.conditions, - }) - } - } - - return [c, ...relatedColumns] - }) - } - ) + const visibleColumns = derived(columns, $columns => { + return $columns.filter(col => col.visible) + }) // Split visible columns into their discrete types const displayColumn = derived(visibleColumns, $visibleColumns => { @@ -93,6 +69,7 @@ export const deriveStores = context => { }) return { + tableColumns, displayColumn, columnLookupMap, visibleColumns, @@ -183,6 +160,32 @@ export const initialise = context => { } return column }) + .flatMap(field => { + const relatedColumns = [] + + const schemaColumns = $enrichedSchema?.[field.name]?.columns + if (field.visible && schemaColumns) { + for (const relColumn of Object.keys(schemaColumns)) { + const relFieldSchema = schemaColumns[relColumn] + if (!relFieldSchema.visible) { + continue + } + relatedColumns.push({ + name: `${field.name}.${relColumn}`, + label: `${relColumn} (${field.name})`, + schema: relFieldSchema, + width: relFieldSchema.width || DefaultColumnWidth, + visible: relFieldSchema.visible ?? true, + readonly: relFieldSchema.readonly, + order: relFieldSchema.order, + conditions: relFieldSchema.conditions, + related: true, + }) + } + } + + return [field, ...relatedColumns] + }) .sort((a, b) => { // Display column should always come first if (a.name === primaryDisplay) { From 083e6ae15bb1a3d87b1cf48ff5a07dfd4241f918 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 9 Sep 2024 12:31:00 +0200 Subject: [PATCH 32/90] Enrich at schema level --- .../src/components/grid/stores/columns.js | 27 +--------- .../src/components/grid/stores/datasource.js | 49 +++++++++---------- 2 files changed, 24 insertions(+), 52 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js index b20f17fd4d..b4eba300ad 100644 --- a/packages/frontend-core/src/components/grid/stores/columns.js +++ b/packages/frontend-core/src/components/grid/stores/columns.js @@ -151,6 +151,7 @@ export const initialise = context => { readonly: fieldSchema.readonly, order: fieldSchema.order ?? oldColumn?.order, conditions: fieldSchema.conditions, + related: fieldSchema.related, } // Override a few properties for primary display if (field === primaryDisplay) { @@ -160,32 +161,6 @@ export const initialise = context => { } return column }) - .flatMap(field => { - const relatedColumns = [] - - const schemaColumns = $enrichedSchema?.[field.name]?.columns - if (field.visible && schemaColumns) { - for (const relColumn of Object.keys(schemaColumns)) { - const relFieldSchema = schemaColumns[relColumn] - if (!relFieldSchema.visible) { - continue - } - relatedColumns.push({ - name: `${field.name}.${relColumn}`, - label: `${relColumn} (${field.name})`, - schema: relFieldSchema, - width: relFieldSchema.width || DefaultColumnWidth, - visible: relFieldSchema.visible ?? true, - readonly: relFieldSchema.readonly, - order: relFieldSchema.order, - conditions: relFieldSchema.conditions, - related: true, - }) - } - } - - return [field, ...relatedColumns] - }) .sort((a, b) => { // Display column should always come first if (a.name === primaryDisplay) { diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index bdd338618c..5ec94874dd 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -53,10 +53,31 @@ export const deriveStores = context => { if (!$schema) { return null } + + const schemaWithRelatedColumns = Object.keys($schema || {}).reduce( + (acc, c) => { + const field = $schema[c] + acc[c] = field + + if (field.columns) { + for (const relColumn of Object.keys(field.columns)) { + const name = `${field.name}.${relColumn}` + acc[name] = { + ...field.columns[relColumn], + name, + related: { field: c, subField: relColumn }, + } + } + } + return acc + }, + {} + ) + let enrichedSchema = {} - Object.keys($schema).forEach(field => { + Object.keys(schemaWithRelatedColumns).forEach(field => { enrichedSchema[field] = { - ...$schema[field], + ...schemaWithRelatedColumns[field], ...$schemaOverrides?.[field], ...$schemaMutations[field], } @@ -73,30 +94,6 @@ export const deriveStores = context => { } } }) - if ($schemaOverrides) { - Object.keys($schemaOverrides).forEach(field => { - if (!$schemaOverrides[field].related) { - return - } - - const { field: relField, subField: relSubField } = - $schemaOverrides[field].related - - if ( - !$schema[relField].visible || - !$schema[relField]?.columns?.[relSubField]?.visible - ) { - return - } - - enrichedSchema[field] = { - ...$schemaOverrides[field], - name: field, - type: $schema[relField]?.columns?.[relSubField]?.type, - readonly: true, - } - }) - } return enrichedSchema } ) From 4e83daf5d96fc734f74fe62150545a1130de729d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 9 Sep 2024 12:41:59 +0200 Subject: [PATCH 33/90] Allow selecting columns in design --- .../GridColumnConfiguration.svelte | 4 ++- .../src/components/grid/stores/datasource.js | 24 +++-------------- packages/frontend-core/src/utils/index.js | 1 + packages/frontend-core/src/utils/schema.js | 27 +++++++++++++++++++ 4 files changed, 34 insertions(+), 22 deletions(-) create mode 100644 packages/frontend-core/src/utils/schema.js diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte index 17cb171da5..b89c45046e 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte @@ -1,4 +1,5 @@ \ No newline at end of file diff --git a/packages/builder/src/pages/builder/portal/settings/index.svelte b/packages/builder/src/pages/builder/portal/settings/index.svelte index 09ead3e410..1448b43ec4 100644 --- a/packages/builder/src/pages/builder/portal/settings/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/index.svelte @@ -1,5 +1,10 @@ From ee962380b3d72c40b4f337ebd573e7ed22ab8f30 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 26 Sep 2024 16:18:19 +0200 Subject: [PATCH 68/90] Handle undefined properly --- .../frontend-core/src/utils/relatedColumns.js | 50 +++++++++---------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/packages/frontend-core/src/utils/relatedColumns.js b/packages/frontend-core/src/utils/relatedColumns.js index 55f1bbbb00..9d389524fd 100644 --- a/packages/frontend-core/src/utils/relatedColumns.js +++ b/packages/frontend-core/src/utils/relatedColumns.js @@ -75,36 +75,32 @@ export function getRelatedTableValues(row, field, fromField) { fromField?.relationshipType === RelationshipType.ONE_TO_MANY let result = "" - try { - if (fromSingle) { - result = row[field.related.field]?.[0]?.[field.related.subField] - } else { - const parser = columnTypeManyParser[field.type] || (value => value) - result = parser( - row[field.related.field] - .flatMap(r => r[field.related.subField]) - .filter(i => i !== undefined && i !== null), - field - ) + if (fromSingle) { + result = row[field.related.field]?.[0]?.[field.related.subField] + } else { + const parser = columnTypeManyParser[field.type] || (value => value) - if ( - [ - FieldType.STRING, - FieldType.NUMBER, - FieldType.BIGINT, - FieldType.BOOLEAN, - FieldType.DATETIME, - FieldType.LONGFORM, - FieldType.BARCODEQR, - ].includes(field.type) - ) { - result = result.join(", ") - } + result = parser( + row[field.related.field] + ?.flatMap(r => r[field.related.subField]) + ?.filter(i => i !== undefined && i !== null), + field + ) + + if ( + [ + FieldType.STRING, + FieldType.NUMBER, + FieldType.BIGINT, + FieldType.BOOLEAN, + FieldType.DATETIME, + FieldType.LONGFORM, + FieldType.BARCODEQR, + ].includes(field.type) + ) { + result = result?.join(", ") } - } catch (e) { - result = "Not rendable" - console.error(e.message) } return result From d7715e4f5c84ec0ccdde40a20cc93836c96c3660 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 26 Sep 2024 17:00:44 +0200 Subject: [PATCH 69/90] Don't allow related columns as display or sorting --- .../DataTable/modals/CreateEditColumn.svelte | 5 ++--- .../backend/TableNavigator/TableDataImport.svelte | 7 ++++--- .../src/components/grid/cells/HeaderCell.svelte | 6 +++--- packages/frontend-core/src/utils/index.js | 1 + packages/frontend-core/src/utils/table.js | 14 ++++++++++++++ 5 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 packages/frontend-core/src/utils/table.js diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index a956d09ee6..0130c39715 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -19,7 +19,6 @@ helpers, PROTECTED_INTERNAL_COLUMNS, PROTECTED_EXTERNAL_COLUMNS, - canBeDisplayColumn, canHaveDefaultColumn, } from "@budibase/shared-core" import { createEventDispatcher, getContext, onMount } from "svelte" @@ -43,7 +42,7 @@ SourceName, } from "@budibase/types" import RelationshipSelector from "components/common/RelationshipSelector.svelte" - import { RowUtils } from "@budibase/frontend-core" + import { RowUtils, canBeDisplayColumn } from "@budibase/frontend-core" import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" import OptionsEditor from "./OptionsEditor.svelte" import { isEnabled } from "helpers/featureFlags" @@ -166,7 +165,7 @@ : availableAutoColumns // used to select what different options can be displayed for column type $: canBeDisplay = - canBeDisplayColumn(editableColumn.type) && !editableColumn.autocolumn + canBeDisplayColumn(editableColumn) && !editableColumn.autocolumn $: canHaveDefault = isEnabled("DEFAULT_VALUES") && canHaveDefaultColumn(editableColumn.type) $: canBeRequired = diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte index 4c9f4dd10f..5804dc3172 100644 --- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte @@ -1,7 +1,8 @@ \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + From 133ce9e784a3187fde6a80ce5098674c3b2fe54d Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 26 Sep 2024 17:21:18 +0100 Subject: [PATCH 71/90] update colour on tags --- .../src/pages/builder/portal/settings/ai/AIConfigTile.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte index a95d14b273..c63f894d56 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte @@ -112,7 +112,7 @@ .tag { display: flex; - color: var(--spectrum-body-m-text-color); + color: #FFFFFF; padding: 4px 8px; justify-content: center; align-items: center; From 98a2da20b4be8623c524aac3221677c45f7b3aaf Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 26 Sep 2024 17:25:13 +0100 Subject: [PATCH 72/90] lint --- .../src/pages/builder/portal/settings/ai/AIConfigTile.svelte | 2 +- .../builder/src/pages/builder/portal/settings/ai/index.svelte | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte index c63f894d56..fb040204c2 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte @@ -112,7 +112,7 @@ .tag { display: flex; - color: #FFFFFF; + color: #ffffff; padding: 4px 8px; justify-content: center; align-items: center; diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index 54b1bd60da..42de9f19ae 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -27,7 +27,6 @@ let editingUuid $: isCloud = $admin.cloud - $: budibaseAIEnabled = $licensing.budibaseAIEnabled $: customAIConfigsEnabled = $licensing.customAIConfigsEnabled async function fetchAIConfig() { From aa044b94cf4615f778d27b1cd6f422ff65bca68b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 26 Sep 2024 21:09:47 +0200 Subject: [PATCH 73/90] Don't allow sort columns as display or sorting --- .../src/components/backend/DataTable/Table.svelte | 4 ++-- .../settings/controls/SortableFieldSelect.svelte | 4 ++-- .../components/app/deprecated/table/Table.svelte | 4 ++-- .../src/components/grid/cells/HeaderCell.svelte | 7 +++---- .../components/grid/controls/SortButton.svelte | 4 ++-- packages/frontend-core/src/utils/table.js | 15 ++++++++++++++- 6 files changed, 25 insertions(+), 13 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/Table.svelte b/packages/builder/src/components/backend/DataTable/Table.svelte index f7eccd5242..e09dd4bd39 100644 --- a/packages/builder/src/components/backend/DataTable/Table.svelte +++ b/packages/builder/src/components/backend/DataTable/Table.svelte @@ -6,7 +6,7 @@ import { TableNames, UNEDITABLE_USER_FIELDS } from "constants" import RoleCell from "./cells/RoleCell.svelte" import { createEventDispatcher } from "svelte" - import { canBeSortColumn } from "@budibase/shared-core" + import { canBeSortColumn } from "@budibase/frontend-core" export let schema = {} export let data = [] @@ -31,7 +31,7 @@ acc[key] = typeof schema[key] === "string" ? { type: schema[key] } : schema[key] - if (!canBeSortColumn(acc[key].type)) { + if (!canBeSortColumn(acc[key])) { acc[key].sortable = false } return acc diff --git a/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte b/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte index d51493616e..0f9290ec8d 100644 --- a/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte @@ -3,7 +3,7 @@ import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding" import { selectedScreen } from "stores/builder" import { createEventDispatcher } from "svelte" - import { canBeSortColumn } from "@budibase/shared-core" + import { canBeSortColumn } from "@budibase/frontend-core" export let componentInstance = {} export let value = "" @@ -17,7 +17,7 @@ const getSortableFields = schema => { return Object.entries(schema || {}) - .filter(entry => canBeSortColumn(entry[1].type)) + .filter(entry => canBeSortColumn(entry[1])) .map(entry => entry[0]) } diff --git a/packages/client/src/components/app/deprecated/table/Table.svelte b/packages/client/src/components/app/deprecated/table/Table.svelte index ac3d88d29c..ca97a012bc 100644 --- a/packages/client/src/components/app/deprecated/table/Table.svelte +++ b/packages/client/src/components/app/deprecated/table/Table.svelte @@ -2,7 +2,7 @@ import { getContext, onDestroy } from "svelte" import { Table } from "@budibase/bbui" import SlotRenderer from "./SlotRenderer.svelte" - import { canBeSortColumn } from "@budibase/shared-core" + import { canBeSortColumn } from "@budibase/frontend-core" import Provider from "components/context/Provider.svelte" export let dataProvider @@ -146,7 +146,7 @@ return } newSchema[columnName] = schema[columnName] - if (!canBeSortColumn(schema[columnName].type)) { + if (!canBeSortColumn(schema[columnName])) { newSchema[columnName].sortable = false } diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte index 23e19af96b..00b2cbabf3 100644 --- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte @@ -1,7 +1,6 @@