diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 9d1131ed7f..14809c1118 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -20,6 +20,7 @@ env: PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} NX_BASE_BRANCH: origin/${{ github.base_ref }} USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}} + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} jobs: lint: diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index ddf185a1d9..2c6302b56a 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - develop + - master jobs: release: diff --git a/lerna.json b/lerna.json index 688e02532f..e5ad209fd4 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.5-alpha.4", + "version": "2.11.15-alpha.0", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/bbui/src/OptionSelectDnD/OptionSelectDnD.svelte b/packages/bbui/src/OptionSelectDnD/OptionSelectDnD.svelte index f64a51ade4..8b13135b33 100644 --- a/packages/bbui/src/OptionSelectDnD/OptionSelectDnD.svelte +++ b/packages/bbui/src/OptionSelectDnD/OptionSelectDnD.svelte @@ -21,14 +21,6 @@ "hsla(240, 90%, 75%, 0.3)", "hsla(320, 90%, 75%, 0.3)", ] - $: { - if (constraints.inclusion.length) { - options = constraints.inclusion.map(value => ({ - name: value, - id: Math.random(), - })) - } - } const removeInput = idx => { delete optionColors[options[idx].name] constraints.inclusion = constraints.inclusion.filter((e, i) => i !== idx) @@ -80,6 +72,11 @@ // Initialize anchor arrays on mount, assuming 'options' is already populated colorPopovers = constraints.inclusion.map(() => undefined) anchors = constraints.inclusion.map(() => undefined) + + options = constraints.inclusion.map(value => ({ + name: value, + id: Math.random(), + })) }) diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte index c3097f3072..289f2e20be 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte @@ -110,20 +110,7 @@
{#each schemaFields as [field, schema]} {#if !schema.autocolumn && schema.type !== "attachment"} - onChange(e, field)} - {bindings} - allowJS={true} - updateOnChange={false} - drawerLeft="260px" - > + {#if isTestModal} - + {:else} + onChange(e, field)} + {bindings} + allowJS={true} + updateOnChange={false} + drawerLeft="260px" + > + + + {/if} {/if} {#if isUpdateRow && schema.type === "link"}
diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte index 23f6d1dea1..91456da655 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte @@ -13,7 +13,13 @@ let modal $: tempValue = filters || [] - $: schemaFields = Object.values(schema || {}) + $: schemaFields = Object.entries(schema || {}).map( + ([fieldName, fieldSchema]) => ({ + name: fieldName, // Using the key as name if not defined in the schema, for example in some autogenerated columns + ...fieldSchema, + }) + ) + $: text = getText(filters) $: selected = tempValue.filter(x => !x.onEmptyFilter)?.length > 0 diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index c67ce67d57..8233278e58 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -660,7 +660,8 @@ >Open schema editor {:else if editableColumn.type === USER_REFRENCE_TYPE} - + {/if} {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn} onOperatorChange(filter)} placeholder={null} @@ -285,6 +298,14 @@ timeOnly={getSchema(filter)?.timeOnly} bind:value={filter.value} /> + {:else if filter.type === FieldType.BB_REFERENCE} + {:else} {/if} diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte new file mode 100644 index 0000000000..88383ba170 --- /dev/null +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte @@ -0,0 +1,34 @@ + + + option.email} + getOptionValue={option => option._id} + {disabled} +/> diff --git a/packages/cli/src/hosting/utils.ts b/packages/cli/src/hosting/utils.ts index 93e31b8aea..a4b28539e9 100644 --- a/packages/cli/src/hosting/utils.ts +++ b/packages/cli/src/hosting/utils.ts @@ -57,7 +57,8 @@ export async function checkDockerConfigured() { "docker/docker-compose has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose" const docker = await lookpath("docker") const compose = await lookpath("docker-compose") - if (!docker || !compose) { + const composeV2 = await lookpath("docker compose") + if (!docker || (!compose && !composeV2)) { throw error } } diff --git a/packages/cli/src/prebuilds.ts b/packages/cli/src/prebuilds.ts index 21f3042274..561f9be474 100644 --- a/packages/cli/src/prebuilds.ts +++ b/packages/cli/src/prebuilds.ts @@ -12,6 +12,10 @@ if (!process.argv[0].includes("node")) { checkForBinaries() } +function localPrebuildPath() { + return join(process.execPath, "..", PREBUILDS) +} + function checkForBinaries() { const readDir = join(__filename, "..", "..", "..", "cli", PREBUILDS, ARCH) if (fs.existsSync(PREBUILD_DIR) || !fs.existsSync(readDir)) { @@ -19,17 +23,21 @@ function checkForBinaries() { } const natives = fs.readdirSync(readDir) if (fs.existsSync(readDir)) { - const writePath = join(process.execPath, PREBUILDS, ARCH) + const writePath = join(localPrebuildPath(), ARCH) fs.mkdirSync(writePath, { recursive: true }) for (let native of natives) { const filename = `${native.split(".fake")[0]}.node` fs.cpSync(join(readDir, native), join(writePath, filename)) } - console.log("copied something") } } function cleanup(evt?: number) { + // cleanup prebuilds first + const path = localPrebuildPath() + if (fs.existsSync(path)) { + fs.rmSync(path, { recursive: true }) + } if (evt && !isNaN(evt)) { return } @@ -41,10 +49,6 @@ function cleanup(evt?: number) { ) console.error(error(evt)) } - const path = join(process.execPath, PREBUILDS) - if (fs.existsSync(path)) { - fs.rmSync(path, { recursive: true }) - } } const events = ["exit", "SIGINT", "SIGUSR1", "SIGUSR2", "uncaughtException"] diff --git a/packages/server/scripts/integrations/postgres/docker-compose.yml b/packages/server/scripts/integrations/postgres/docker-compose.yml index d682ad7361..88efd0301d 100644 --- a/packages/server/scripts/integrations/postgres/docker-compose.yml +++ b/packages/server/scripts/integrations/postgres/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.8" services: db: container_name: postgres - image: postgres + image: postgres:15 restart: unless-stopped environment: POSTGRES_USER: root @@ -25,4 +25,4 @@ services: - "5050:80" volumes: - pg_data: + pg_data: diff --git a/packages/server/src/db/linkedRows/LinkController.ts b/packages/server/src/db/linkedRows/LinkController.ts index 5bfae49e8b..457819251a 100644 --- a/packages/server/src/db/linkedRows/LinkController.ts +++ b/packages/server/src/db/linkedRows/LinkController.ts @@ -308,12 +308,19 @@ class LinkController { } }) ) - // remove schema from other table - let linkedTable = await this._db.get(field.tableId) - if (field.fieldName) { - delete linkedTable.schema[field.fieldName] + try { + // remove schema from other table, if it exists + let linkedTable = await this._db.get
(field.tableId) + if (field.fieldName) { + delete linkedTable.schema[field.fieldName] + } + await this._db.put(linkedTable) + } catch (error: any) { + // ignore missing to ensure broken relationship columns can be deleted + if (error.statusCode !== 404) { + throw error + } } - await this._db.put(linkedTable) } /** diff --git a/packages/server/src/db/tests/linkController.spec.js b/packages/server/src/db/tests/linkController.spec.js index 59d0f3f983..5caf35f61a 100644 --- a/packages/server/src/db/tests/linkController.spec.js +++ b/packages/server/src/db/tests/linkController.spec.js @@ -233,4 +233,19 @@ describe("test the link controller", () => { } await config.updateTable(table) }) + + it("should be able to remove a linked field from a table, even if the linked table does not exist", async () => { + await createLinkedRow() + await createLinkedRow("link2") + table1.schema["link"].tableId = "not_found" + const controller = await createLinkController(table1, null, table1) + await context.doInAppContext(appId, async () => { + let before = await controller.getTableLinkDocs() + await controller.removeFieldFromTable("link") + let after = await controller.getTableLinkDocs() + expect(before.length).toEqual(2) + // shouldn't delete the other field + expect(after.length).toEqual(1) + }) + }) }) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index f908be0b3c..8dd141f8ef 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -16,6 +16,7 @@ import { cleanExportRows } from "../utils" import { utils } from "@budibase/shared-core" import { ExportRowsParams, ExportRowsResult } from "../search" import { HTTPError, db } from "@budibase/backend-core" +import { searchInputMapping } from "./utils" import pick from "lodash/pick" import { outputProcessing } from "../../../../utilities/rowProcessor" @@ -50,7 +51,10 @@ export async function search(options: SearchParams) { [params.sort]: { direction }, } } + try { + const table = await sdk.tables.getTable(tableId) + options = searchInputMapping(table, options) let rows = (await handleRequest(Operation.READ, tableId, { filters: query, sort, @@ -76,7 +80,6 @@ export async function search(options: SearchParams) { rows = rows.map((r: any) => pick(r, fields)) } - const table = await sdk.tables.getTable(tableId) rows = await outputProcessing(table, rows, { preserveLinks: true }) // need wrapper object for bookmarks etc when paginating diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 4cdeca87f6..d78c0213b3 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -29,6 +29,7 @@ import { } from "../../../../api/controllers/view/utils" import sdk from "../../../../sdk" import { ExportRowsParams, ExportRowsResult } from "../search" +import { searchInputMapping } from "./utils" import pick from "lodash/pick" export async function search(options: SearchParams) { @@ -47,9 +48,9 @@ export async function search(options: SearchParams) { disableEscaping: options.disableEscaping, } - let table + let table = await sdk.tables.getTable(tableId) + options = searchInputMapping(table, options) if (params.sort && !params.sortType) { - table = await sdk.tables.getTable(tableId) const schema = table.schema const sortField = schema[params.sort] params.sortType = sortField.type === "number" ? "number" : "string" @@ -68,7 +69,6 @@ export async function search(options: SearchParams) { if (tableId === InternalTables.USER_METADATA) { response.rows = await getGlobalUsersFromMetadata(response.rows) } - table = table || (await sdk.tables.getTable(tableId)) if (options.fields) { const fields = [...options.fields, ...db.CONSTANT_INTERNAL_ROW_COLS] diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts new file mode 100644 index 0000000000..08d1f1b1cb --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts @@ -0,0 +1,77 @@ +import { searchInputMapping } from "../utils" +import { db as dbCore } from "@budibase/backend-core" +import { + FieldType, + FieldTypeSubtypes, + Table, + SearchParams, +} from "@budibase/types" + +const tableId = "ta_a" +const tableWithUserCol: Table = { + _id: tableId, + name: "table", + schema: { + user: { + name: "user", + type: FieldType.BB_REFERENCE, + subtype: FieldTypeSubtypes.BB_REFERENCE.USER, + }, + }, +} + +describe("searchInputMapping", () => { + const globalUserId = dbCore.generateGlobalUserID() + const userMedataId = dbCore.generateUserMetadataID(globalUserId) + + it("should be able to map ro_ to global user IDs", () => { + const params: SearchParams = { + tableId, + query: { + equal: { + "1:user": userMedataId, + }, + }, + } + const output = searchInputMapping(tableWithUserCol, params) + expect(output.query.equal!["1:user"]).toBe(globalUserId) + }) + + it("should handle array of user IDs", () => { + const params: SearchParams = { + tableId, + query: { + oneOf: { + "1:user": [userMedataId, globalUserId], + }, + }, + } + const output = searchInputMapping(tableWithUserCol, params) + expect(output.query.oneOf!["1:user"]).toStrictEqual([ + globalUserId, + globalUserId, + ]) + }) + + it("shouldn't change any other input", () => { + const email = "test@test.com" + const params: SearchParams = { + tableId, + query: { + equal: { + "1:user": email, + }, + }, + } + const output = searchInputMapping(tableWithUserCol, params) + expect(output.query.equal!["1:user"]).toBe(email) + }) + + it("shouldn't error if no query supplied", () => { + const params: any = { + tableId, + } + const output = searchInputMapping(tableWithUserCol, params) + expect(output.query).toBeUndefined() + }) +}) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts new file mode 100644 index 0000000000..14f7907e4f --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -0,0 +1,76 @@ +import { + FieldType, + FieldTypeSubtypes, + SearchParams, + Table, + DocumentType, + SEPARATOR, +} from "@budibase/types" +import { db as dbCore } from "@budibase/backend-core" + +function findColumnInQueries( + column: string, + options: SearchParams, + callback: (filter: any) => any +) { + if (!options.query) { + return + } + for (let filterBlock of Object.values(options.query)) { + if (typeof filterBlock !== "object") { + continue + } + for (let [key, filter] of Object.entries(filterBlock)) { + if (key.endsWith(column)) { + filterBlock[key] = callback(filter) + } + } + } +} + +function userColumnMapping(column: string, options: SearchParams) { + findColumnInQueries(column, options, (filterValue: any): any => { + const isArray = Array.isArray(filterValue), + isString = typeof filterValue === "string" + if (!isString && !isArray) { + return filterValue + } + const processString = (input: string) => { + const rowPrefix = DocumentType.ROW + SEPARATOR + if (input.startsWith(rowPrefix)) { + return dbCore.getGlobalIDFromUserMetadataID(input) + } else { + return input + } + } + if (isArray) { + return filterValue.map(el => { + if (typeof el === "string") { + return processString(el) + } else { + return el + } + }) + } else { + return processString(filterValue) + } + }) +} + +// maps through the search parameters to check if any of the inputs are invalid +// based on the table schema, converts them to something that is valid. +export function searchInputMapping(table: Table, options: SearchParams) { + if (!table?.schema) { + return options + } + for (let [key, column] of Object.entries(table.schema)) { + switch (column.type) { + case FieldType.BB_REFERENCE: + if (column.subtype === FieldTypeSubtypes.BB_REFERENCE.USER) { + userColumnMapping(key, options) + } + break + } + } + return options +} diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 2cd6fa8c13..e443f35dbe 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -14,7 +14,6 @@ const HBS_REGEX = /{{([^{].*?)}}/g /** * Returns the valid operator options for a certain data type - * @param type the data type */ export const getValidOperatorsForType = ( type: FieldType, @@ -44,22 +43,24 @@ export const getValidOperatorsForType = ( value: string label: string }[] = [] - if (type === "string") { + if (type === FieldType.STRING) { ops = stringOps - } else if (type === "number" || type === "bigint") { + } else if (type === FieldType.NUMBER || type === FieldType.BIGINT) { ops = numOps - } else if (type === "options") { + } else if (type === FieldType.OPTIONS) { ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In] - } else if (type === "array") { + } else if (type === FieldType.ARRAY) { ops = [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny] - } else if (type === "boolean") { + } else if (type === FieldType.BOOLEAN) { ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty] - } else if (type === "longform") { + } else if (type === FieldType.LONGFORM) { ops = stringOps - } else if (type === "datetime") { + } else if (type === FieldType.DATETIME) { ops = numOps - } else if (type === "formula") { + } else if (type === FieldType.FORMULA) { ops = stringOps.concat([Op.MoreThan, Op.LessThan]) + } else if (type === FieldType.BB_REFERENCE) { + ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In] } // Only allow equal/not equal for _id in SQL tables