diff --git a/.github/workflows/close-featurebranch.yml b/.github/workflows/close-featurebranch.yml index 0ec3b43598..5f232b2f26 100644 --- a/.github/workflows/close-featurebranch.yml +++ b/.github/workflows/close-featurebranch.yml @@ -14,7 +14,7 @@ jobs: - uses: passeidireto/trigger-external-workflow-action@main env: PAYLOAD_BRANCH: ${{ github.head_ref }} - PAYLOAD_PR_NUMBER: ${{ github.ref }} + PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} with: repository: budibase/budibase-deploys event: featurebranch-qa-close diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index f06707ab2b..ddf185a1d9 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -13,7 +13,7 @@ jobs: - uses: passeidireto/trigger-external-workflow-action@main env: PAYLOAD_BRANCH: ${{ github.head_ref }} - PAYLOAD_PR_NUMBER: ${{ github.ref }} + PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} with: repository: budibase/budibase-deploys event: featurebranch-qa-deploy diff --git a/lerna.json b/lerna.json index d99355a157..7ee8f6695e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.16-alpha.11", + "version": "2.11.5-alpha.3", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/src/plugin/utils.ts b/packages/backend-core/src/plugin/utils.ts index f73ded0659..8974a9f5a2 100644 --- a/packages/backend-core/src/plugin/utils.ts +++ b/packages/backend-core/src/plugin/utils.ts @@ -6,6 +6,7 @@ import { AutomationStepIdArray, AutomationIOType, AutomationCustomIOType, + DatasourceFeature, } from "@budibase/types" import joi from "joi" @@ -67,9 +68,27 @@ function validateDatasource(schema: any) { version: joi.string().optional(), schema: joi.object({ docs: joi.string(), + plus: joi.boolean().optional(), + isSQL: joi.boolean().optional(), + auth: joi + .object({ + type: joi.string().required(), + }) + .optional(), + features: joi + .object( + Object.fromEntries( + Object.values(DatasourceFeature).map(key => [ + key, + joi.boolean().optional(), + ]) + ) + ) + .optional(), + relationships: joi.boolean().optional(), + description: joi.string().required(), friendlyName: joi.string().required(), type: joi.string().allow(...DATASOURCE_TYPES), - description: joi.string().required(), datasource: joi.object().pattern(joi.string(), fieldValidator).required(), query: joi .object() diff --git a/packages/builder/src/builderStore/store/automation/index.js b/packages/builder/src/builderStore/store/automation/index.js index 4ebf0515d6..c22240370b 100644 --- a/packages/builder/src/builderStore/store/automation/index.js +++ b/packages/builder/src/builderStore/store/automation/index.js @@ -221,18 +221,6 @@ const automationActions = store => ({ newAutomation.definition.steps.splice(blockIdx, 0, block) await store.actions.save(newAutomation) }, - /** - * "rowControl" appears to be the name of the flag used to determine whether - * a certain automation block uses values or bindings as inputs - */ - toggleRowControl: async (block, rowControl) => { - const newBlock = { ...block, rowControl } - const newAutomation = store.actions.getUpdatedDefinition( - get(selectedAutomation), - newBlock - ) - await store.actions.save(newAutomation) - }, deleteAutomationBlock: async block => { const automation = get(selectedAutomation) let newAutomation = cloneDeep(automation) diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte index 85c3776fdb..6c964c84a9 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte @@ -7,7 +7,6 @@ Detail, Modal, Button, - Select, ActionButton, notifications, Label, @@ -39,9 +38,6 @@ step => step.stepId === ActionStepID.COLLECT ) $: automationId = $selectedAutomation?._id - $: showBindingPicker = - block.stepId === ActionStepID.CREATE_ROW || - block.stepId === ActionStepID.UPDATE_ROW $: isTrigger = block.type === "TRIGGER" $: steps = $selectedAutomation?.definition?.steps ?? [] $: blockIdx = steps.findIndex(step => step.id === block.id) @@ -96,15 +92,6 @@ } } - /** - * "rowControl" appears to be the name of the flag used to determine whether - * a certain automation block uses values or bindings as inputs - */ - function toggleRowControl(evt) { - const rowControl = evt.detail !== "Use values" - automationStore.actions.toggleRowControl(block, rowControl) - } - async function addLooping() { const loopDefinition = $automationStore.blockDefinitions.ACTION.LOOP const loopBlock = automationStore.actions.constructBlock( @@ -189,16 +176,6 @@ Add Looping {/if} - {#if showBindingPicker} - onChange(e, key)} @@ -469,7 +420,6 @@ /> {:else if value.customType === "row"} { diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte index 5039c37ece..c3097f3072 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte @@ -1,18 +1,16 @@ + +
+ {#if !isValid(value)} + onChange(event.detail)} + on:blur={onBlur} + {placeholder} + {updateOnChange} + /> +
{ + if (!isJS) { + dispatch("change", "") + } + }} + > + +
+ {:else} + + {/if} + {#if !disabled && type !== "formula"} +
{ + bindingDrawer.show() + }} + > + +
+ {/if} +
+ + + Add the objects on the left to enrich your text. + + + (tempValue = event.detail)} + {bindings} + {allowJS} + {allowHelpers} + /> + + + diff --git a/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte b/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte index 21ed68ce68..74b044e75e 100644 --- a/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte @@ -20,7 +20,9 @@ const getSortableFields = schema => { return Object.entries(schema || {}) - .filter(entry => !UNSORTABLE_TYPES.includes(entry[1].type)) + .filter( + entry => !UNSORTABLE_TYPES.includes(entry[1].type) && entry[1].sortable + ) .map(entry => entry[0]) } diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/EditDatasourceConfigButton.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/EditDatasourceConfigButton.svelte index 9654b27b50..06b739e858 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/EditDatasourceConfigButton.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/EditDatasourceConfigButton.svelte @@ -62,7 +62,14 @@ {/if}
- {getSubtitle(datasource)} + + {@const subtitle = getSubtitle(datasource)} + {#if subtitle} + {subtitle} + {:else} + {Object.values(datasource.config).join(" / ")} + {/if} +
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte index 384b87e11d..1a46ecb540 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte @@ -21,15 +21,22 @@ function getRelationships(tables) { const relatedColumns = {} - tables.forEach(({ name: tableName, schema }) => { + tables.forEach(({ name: tableName, schema, _id: tableId }) => { Object.values(schema).forEach(column => { if (column.type !== "link") return - relatedColumns[column._id] ??= {} - relatedColumns[column._id].through = - relatedColumns[column._id].through || column.through + const columnId = + column.through || + column._id || + (column.main + ? `${tableId}_${column.fieldName}__${column.tableId}_${column.foreignKey}` + : `${column.tableId}_${column.foreignKey}__${tableId}_${column.fieldName}`) - relatedColumns[column._id][column.main ? "from" : "to"] = { + relatedColumns[columnId] ??= {} + relatedColumns[columnId].through = + relatedColumns[columnId].through || column.through + + relatedColumns[columnId][column.main ? "from" : "to"] = { ...column, tableName, } diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index 7d2db44d6a..00384a6b1c 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -136,6 +136,7 @@ export function createDatasourcesStore() { config, name: `${integration.friendlyName}${nameModifier}`, plus: integration.plus && integration.name !== IntegrationTypes.REST, + isSQL: integration.isSQL, } if (await checkDatasourceValidity(integration, datasource)) { diff --git a/packages/client/src/components/app/Layout.svelte b/packages/client/src/components/app/Layout.svelte index e482e6b336..bdab0dd9ab 100644 --- a/packages/client/src/components/app/Layout.svelte +++ b/packages/client/src/components/app/Layout.svelte @@ -3,6 +3,7 @@ import { writable } from "svelte/store" import { Heading, Icon, clickOutside } from "@budibase/bbui" import { FieldTypes } from "constants" + import { Constants } from "@budibase/frontend-core" import active from "svelte-spa-router/active" const sdk = getContext("sdk") @@ -103,7 +104,8 @@ let validLinks = (allLinks || []).filter(link => link.text && link.url) // Filter to only links allowed by the current role return validLinks.filter(link => { - return userRoleHierarchy?.find(roleId => roleId === link.roleId) + const role = link.roleId || Constants.Roles.BASIC + return userRoleHierarchy?.find(roleId => roleId === role) }) } diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index bfa7c6cbd2..bb003730b3 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -60,6 +60,12 @@ // even if they are not in the inital fetch results initialValuesProcessed = true optionsObj = (fieldState?.value || []).reduce((accumulator, value) => { + // fieldState has to be an array of strings to be valid for an update + // therefore we cannot guarantee value will be an object + // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for + if (!value._id) { + return accumulator + } accumulator[value._id] = { _id: value._id, [primaryDisplay]: value.primaryDisplay, diff --git a/packages/client/src/components/devtools/DevToolsHeader.svelte b/packages/client/src/components/devtools/DevToolsHeader.svelte index a15e8351a5..55b705e717 100644 --- a/packages/client/src/components/devtools/DevToolsHeader.svelte +++ b/packages/client/src/components/devtools/DevToolsHeader.svelte @@ -25,7 +25,6 @@ value: roleId, }) } - devToolsStore.actions.changeRole(SELF_ROLE) return list } diff --git a/packages/client/src/stores/devTools.js b/packages/client/src/stores/devTools.js index 32f3c8e617..db9b9e10b4 100644 --- a/packages/client/src/stores/devTools.js +++ b/packages/client/src/stores/devTools.js @@ -2,6 +2,7 @@ import { createLocalStorageStore } from "@budibase/frontend-core" import { initialise } from "./initialise" import { authStore } from "./auth" import { API } from "../api" +import { get } from "svelte/store" const initialState = { visible: false, @@ -27,9 +28,15 @@ const createDevToolStore = () => { } const changeRole = async role => { + if (role === "self") { + role = null + } + if (role === get(store).role) { + return + } store.update(state => ({ ...state, - role: role === "self" ? null : role, + role, })) API.invalidateCache() await authStore.actions.fetchUser() diff --git a/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte b/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte index be99c9f633..b4168474b0 100644 --- a/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte @@ -3,6 +3,8 @@ import RelationshipCell from "./RelationshipCell.svelte" import { FieldSubtype } from "@budibase/types" + export let api + const { API } = getContext("grid") const { subtype } = $$props.schema @@ -17,8 +19,11 @@ throw `Search for '${subtype}' not implemented` } + // As we are overriding the search function from RelationshipCell, we want to map one shape to the expected one for the specific API + const email = Object.values(searchParams.query.string)[0] + const results = await API.searchUsers({ - ...searchParams, + email, }) // Mapping to the expected data within RelationshipCell @@ -31,6 +36,7 @@ { +const _import = async (ctx: UserCtx) => { const body = ctx.request.body const data = body.data @@ -73,7 +73,7 @@ const _import = async (ctx: any) => { } export { _import as import } -export async function save(ctx: any) { +export async function save(ctx: UserCtx) { const db = context.getAppDB() const query = ctx.request.body @@ -100,19 +100,19 @@ export async function save(ctx: any) { ctx.message = `Query ${query.name} saved successfully.` } -export async function find(ctx: any) { +export async function find(ctx: UserCtx) { const queryId = ctx.params.queryId ctx.body = await sdk.queries.find(queryId) } //Required to discern between OIDC OAuth config entries -function getOAuthConfigCookieId(ctx: any) { - if (ctx.user.providerType === constants.Config.OIDC) { +function getOAuthConfigCookieId(ctx: UserCtx) { + if (ctx.user.providerType === ConfigType.OIDC) { return utils.getCookie(ctx, constants.Cookie.OIDC_CONFIG) } } -function getAuthConfig(ctx: any) { +function getAuthConfig(ctx: UserCtx) { const authCookie = utils.getCookie(ctx, constants.Cookie.Auth) let authConfigCtx: any = {} authConfigCtx["configId"] = getOAuthConfigCookieId(ctx) @@ -120,7 +120,7 @@ function getAuthConfig(ctx: any) { return authConfigCtx } -export async function preview(ctx: any) { +export async function preview(ctx: UserCtx) { const { datasource, envVars } = await sdk.datasources.getWithEnvVars( ctx.request.body.datasourceId ) @@ -129,6 +129,19 @@ export async function preview(ctx: any) { // this stops dynamic variables from calling the same query const { fields, parameters, queryVerb, transformer, queryId, schema } = query + let existingSchema = schema + if (queryId && !existingSchema) { + try { + const db = context.getAppDB() + const existing = (await db.get(queryId)) as Query + existingSchema = existing.schema + } catch (err: any) { + if (err.status !== 404) { + ctx.throw(500, "Unable to retrieve existing query") + } + } + } + const authConfigCtx: any = getAuthConfig(ctx) try { @@ -180,6 +193,14 @@ export async function preview(ctx: any) { schemaFields[key] = fieldType } } + // if existing schema, update to include any previous schema keys + if (existingSchema) { + for (let key of Object.keys(schemaFields)) { + if (existingSchema[key]?.type) { + schemaFields[key] = existingSchema[key].type + } + } + } // remove configuration before sending event delete datasource.config await events.query.previewed(datasource, query) @@ -189,13 +210,13 @@ export async function preview(ctx: any) { info, extra, } - } catch (err) { + } catch (err: any) { ctx.throw(400, err) } } async function execute( - ctx: any, + ctx: UserCtx, opts: any = { rowsOnly: false, isAutomation: false } ) { const db = context.getAppDB() @@ -255,17 +276,17 @@ async function execute( } else { ctx.body = { data: rows, pagination, ...extra, ...info } } - } catch (err) { + } catch (err: any) { ctx.throw(400, err) } } -export async function executeV1(ctx: any) { +export async function executeV1(ctx: UserCtx) { return execute(ctx, { rowsOnly: true, isAutomation: false }) } export async function executeV2( - ctx: any, + ctx: UserCtx, { isAutomation }: { isAutomation?: boolean } = {} ) { return execute(ctx, { rowsOnly: false, isAutomation }) @@ -292,7 +313,7 @@ const removeDynamicVariables = async (queryId: any) => { } } -export async function destroy(ctx: any) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() const queryId = ctx.params.queryId await removeDynamicVariables(queryId) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 9ab96fba69..2ad1afe202 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -340,10 +340,16 @@ export class ExternalRequest { // one to many if (isOneSide(field)) { let id = row[key][0] - if (typeof row[key] === "string") { - id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1] + if (id) { + if (typeof row[key] === "string") { + id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1] + } + newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0] + } else { + // Removing from both new and row, as we don't know if it has already been processed + row[field.foreignKey || linkTablePrimary] = null + newRow[field.foreignKey || linkTablePrimary] = null } - newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0] } // many to many else if (field.through) { @@ -830,10 +836,7 @@ export class ExternalRequest { // can't really use response right now const response = await getDatasourceAndQuery(json) // handle many to many relationships now if we know the ID (could be auto increment) - if ( - operation !== Operation.READ && - processed.manyRelationships?.length > 0 - ) { + if (operation !== Operation.READ) { await this.handleManyRelationships( table._id || "", response[0], diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 899849e3a7..ddc63e5790 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -108,13 +108,11 @@ export async function save(ctx: UserCtx) { row, }) - const responseRow = response as { row: Row } - if (!isEqual(table, updatedTable)) { await sdk.tables.saveTable(updatedTable) } - const rowId = responseRow.row._id + const rowId = response.row._id if (rowId) { const row = await sdk.rows.external.getRow(tableId, rowId, { relationships: true, diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 5029856cf4..e7c6ae57b0 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -14,7 +14,6 @@ import { Table, TableResponse, UserCtx, - Datasource, } from "@budibase/types" import sdk from "../../../sdk" import { jsonFromCsvString } from "../../../utilities/csv" diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index b4a33efdde..fa329dbb4b 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -18,7 +18,6 @@ import { SortType, StaticQuotaName, Table, - User, } from "@budibase/types" import { expectAnyExternalColsAttributes, @@ -1515,9 +1514,82 @@ describe.each([ }) }) - describe("bb reference fields", () => { + let o2mTable: Table + let m2mTable: Table + beforeAll(async () => { + o2mTable = await config.createTable( + { ...generateTableConfig(), name: "o2m" }, + { + skipReassigning: true, + } + ) + m2mTable = await config.createTable( + { ...generateTableConfig(), name: "m2m" }, + { + skipReassigning: true, + } + ) + }) + + describe.each([ + [ + "relationship fields", + () => ({ + user: { + name: "user", + relationshipType: RelationshipType.ONE_TO_MANY, + type: FieldType.LINK, + tableId: o2mTable._id!, + fieldName: "fk_o2m", + }, + users: { + name: "users", + relationshipType: RelationshipType.MANY_TO_MANY, + type: FieldType.LINK, + tableId: m2mTable._id!, + fieldName: "fk_m2m", + }, + }), + (tableId: string) => + config.api.row.save(tableId, { + name: generator.word(), + description: generator.paragraph(), + tableId, + }), + (row: Row) => ({ + _id: row._id, + primaryDisplay: row.name, + }), + ], + [ + "bb reference fields", + () => ({ + user: { + name: "user", + relationshipType: RelationshipType.ONE_TO_MANY, + type: FieldType.BB_REFERENCE, + subtype: FieldTypeSubtypes.BB_REFERENCE.USER, + }, + users: { + name: "users", + type: FieldType.BB_REFERENCE, + subtype: FieldTypeSubtypes.BB_REFERENCE.USER, + relationshipType: RelationshipType.MANY_TO_MANY, + }, + }), + () => config.createUser(), + (row: Row) => ({ + _id: row._id, + email: row.email, + firstName: row.firstName, + lastName: row.lastName, + primaryDisplay: row.email, + }), + ], + ])("links - %s", (__, relSchema, dataGenerator, resultMapper) => { let tableId: string - let users: User[] + let o2mData: Row[] + let m2mData: Row[] beforeAll(async () => { const tableConfig = generateTableConfig() @@ -1532,31 +1604,27 @@ describe.each([ ...tableConfig, schema: { ...tableConfig.schema, - user: { - name: "user", - type: FieldType.BB_REFERENCE, - subtype: FieldTypeSubtypes.BB_REFERENCE.USER, - relationshipType: RelationshipType.ONE_TO_MANY, - }, - users: { - name: "users", - type: FieldType.BB_REFERENCE, - subtype: FieldTypeSubtypes.BB_REFERENCE.USER, - relationshipType: RelationshipType.MANY_TO_MANY, - }, + ...relSchema(), }, }) tableId = table._id! - users = [ - await config.createUser(), - await config.createUser(), - await config.createUser(), - await config.createUser(), + o2mData = [ + await dataGenerator(o2mTable._id!), + await dataGenerator(o2mTable._id!), + await dataGenerator(o2mTable._id!), + await dataGenerator(o2mTable._id!), + ] + + m2mData = [ + await dataGenerator(m2mTable._id!), + await dataGenerator(m2mTable._id!), + await dataGenerator(m2mTable._id!), + await dataGenerator(m2mTable._id!), ] }) - it("can save a row when BB reference fields are empty", async () => { + it("can save a row when relationship fields are empty", async () => { const rowData = { ...basicRow(tableId), name: generator.name(), @@ -1575,13 +1643,13 @@ describe.each([ }) }) - it("can save a row with a single BB reference field", async () => { - const user = _.sample(users)! + it("can save a row with a single relationship field", async () => { + const user = _.sample(o2mData)! const rowData = { ...basicRow(tableId), name: generator.name(), description: generator.name(), - user: user, + user: [user], } const row = await config.api.row.save(tableId, rowData) @@ -1589,24 +1657,17 @@ describe.each([ name: rowData.name, description: rowData.description, tableId, - user: [ - { - _id: user._id, - email: user.email, - firstName: user.firstName, - lastName: user.lastName, - primaryDisplay: user.email, - }, - ], + user: [user].map(u => resultMapper(u)), _id: expect.any(String), _rev: expect.any(String), id: isInternal ? undefined : expect.any(Number), type: isInternal ? "row" : undefined, + [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id, }) }) - it("can save a row with a multiple BB reference field", async () => { - const selectedUsers = _.sampleSize(users, 2) + it("can save a row with a multiple relationship field", async () => { + const selectedUsers = _.sampleSize(m2mData, 2) const rowData = { ...basicRow(tableId), name: generator.name(), @@ -1619,13 +1680,7 @@ describe.each([ name: rowData.name, description: rowData.description, tableId, - users: selectedUsers.map(u => ({ - _id: u._id, - email: u.email, - firstName: u.firstName, - lastName: u.lastName, - primaryDisplay: u.email, - })), + users: expect.arrayContaining(selectedUsers.map(u => resultMapper(u))), _id: expect.any(String), _rev: expect.any(String), id: isInternal ? undefined : expect.any(Number), @@ -1633,7 +1688,7 @@ describe.each([ }) }) - it("can retrieve rows with no populated BB references", async () => { + it("can retrieve rows with no populated relationships", async () => { const rowData = { ...basicRow(tableId), name: generator.name(), @@ -1655,14 +1710,15 @@ describe.each([ }) }) - it("can retrieve rows with populated BB references", async () => { - const [user1, user2] = _.sampleSize(users, 2) + it("can retrieve rows with populated relationships", async () => { + const user1 = _.sample(o2mData)! + const [user2, user3] = _.sampleSize(m2mData, 2) const rowData = { ...basicRow(tableId), name: generator.name(), description: generator.name(), - users: [user1, user2], + users: [user2, user3], user: [user1], } const row = await config.api.row.save(tableId, rowData) @@ -1672,72 +1728,51 @@ describe.each([ name: rowData.name, description: rowData.description, tableId, - user: [user1].map(u => ({ - _id: u._id, - email: u.email, - firstName: u.firstName, - lastName: u.lastName, - primaryDisplay: u.email, - })), - users: [user1, user2].map(u => ({ - _id: u._id, - email: u.email, - firstName: u.firstName, - lastName: u.lastName, - primaryDisplay: u.email, - })), + user: expect.arrayContaining([user1].map(u => resultMapper(u))), + users: expect.arrayContaining([user2, user3].map(u => resultMapper(u))), _id: row._id, _rev: expect.any(String), id: isInternal ? undefined : expect.any(Number), + [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user1.id, ...defaultRowFields, }) }) it("can update an existing populated row", async () => { - const [user1, user2, user3] = _.sampleSize(users, 3) + const user = _.sample(o2mData)! + const [users1, users2, users3] = _.sampleSize(m2mData, 3) const rowData = { ...basicRow(tableId), name: generator.name(), description: generator.name(), - users: [user1, user2], + users: [users1, users2], } const row = await config.api.row.save(tableId, rowData) const updatedRow = await config.api.row.save(tableId, { ...row, - user: [user3], - users: [user3, user2], + user: [user], + users: [users3, users1], }) expect(updatedRow).toEqual({ name: rowData.name, description: rowData.description, tableId, - user: [ - { - _id: user3._id, - email: user3.email, - firstName: user3.firstName, - lastName: user3.lastName, - primaryDisplay: user3.email, - }, - ], - users: [user3, user2].map(u => ({ - _id: u._id, - email: u.email, - firstName: u.firstName, - lastName: u.lastName, - primaryDisplay: u.email, - })), + user: expect.arrayContaining([user].map(u => resultMapper(u))), + users: expect.arrayContaining( + [users3, users1].map(u => resultMapper(u)) + ), _id: row._id, _rev: expect.any(String), id: isInternal ? undefined : expect.any(Number), type: isInternal ? "row" : undefined, + [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id, }) }) - it("can wipe an existing populated BB references in row", async () => { - const [user1, user2] = _.sampleSize(users, 2) + it("can wipe an existing populated relationships in row", async () => { + const [user1, user2] = _.sampleSize(m2mData, 2) const rowData = { ...basicRow(tableId), @@ -1756,8 +1791,6 @@ describe.each([ name: rowData.name, description: rowData.description, tableId, - user: isInternal ? null : undefined, - users: isInternal ? null : undefined, _id: row._id, _rev: expect.any(String), id: isInternal ? undefined : expect.any(Number), @@ -1765,34 +1798,35 @@ describe.each([ }) }) - it("fetch all will populate the BB references", async () => { - const [user1, user2, user3] = _.sampleSize(users, 3) + it("fetch all will populate the relationships", async () => { + const [user1] = _.sampleSize(o2mData, 1) + const [users1, users2, users3] = _.sampleSize(m2mData, 3) const rows: { name: string description: string - user?: User[] - users?: User[] + user?: Row[] + users?: Row[] tableId: string }[] = [ { ...basicRow(tableId), name: generator.name(), description: generator.name(), - users: [user1, user2], + users: [users1, users2], }, { ...basicRow(tableId), name: generator.name(), description: generator.name(), user: [user1], - users: [user1, user3], + users: [users1, users3], }, { ...basicRow(tableId), name: generator.name(), description: generator.name(), - users: [user3], + users: [users3], }, ] @@ -1808,57 +1842,50 @@ describe.each([ name: r.name, description: r.description, tableId, - user: r.user?.map(u => ({ - _id: u._id, - email: u.email, - firstName: u.firstName, - lastName: u.lastName, - primaryDisplay: u.email, - })), - users: r.users?.map(u => ({ - _id: u._id, - email: u.email, - firstName: u.firstName, - lastName: u.lastName, - primaryDisplay: u.email, - })), + user: r.user?.map(u => resultMapper(u)), + users: r.users?.length + ? expect.arrayContaining(r.users?.map(u => resultMapper(u))) + : undefined, _id: expect.any(String), _rev: expect.any(String), id: isInternal ? undefined : expect.any(Number), + [`fk_${o2mTable.name}_fk_o2m`]: + isInternal || !r.user?.length ? undefined : r.user[0].id, ...defaultRowFields, })) ) ) }) - it("search all will populate the BB references", async () => { - const [user1, user2, user3] = _.sampleSize(users, 3) + it("search all will populate the relationships", async () => { + const [user1] = _.sampleSize(o2mData, 1) + const [users1, users2, users3] = _.sampleSize(m2mData, 3) const rows: { name: string description: string - user?: User[] - users?: User[] + user?: Row[] + users?: Row[] tableId: string }[] = [ { ...basicRow(tableId), name: generator.name(), description: generator.name(), - users: [user1, user2], + users: [users1, users2], }, { ...basicRow(tableId), name: generator.name(), description: generator.name(), user: [user1], - users: [user1, user3], + users: [users1, users3], }, { ...basicRow(tableId), name: generator.name(), description: generator.name(), - users: [user3], + users: [users3], }, ] @@ -1874,23 +1901,15 @@ describe.each([ name: r.name, description: r.description, tableId, - user: r.user?.map(u => ({ - _id: u._id, - email: u.email, - firstName: u.firstName, - lastName: u.lastName, - primaryDisplay: u.email, - })), - users: r.users?.map(u => ({ - _id: u._id, - email: u.email, - firstName: u.firstName, - lastName: u.lastName, - primaryDisplay: u.email, - })), + user: r.user?.map(u => resultMapper(u)), + users: r.users?.length + ? expect.arrayContaining(r.users?.map(u => resultMapper(u))) + : undefined, _id: expect.any(String), _rev: expect.any(String), id: isInternal ? undefined : expect.any(Number), + [`fk_${o2mTable.name}_fk_o2m`]: + isInternal || !r.user?.length ? undefined : r.user[0].id, ...defaultRowFields, })) ), diff --git a/packages/server/src/automations/steps/make.ts b/packages/server/src/automations/steps/make.ts index e90648d573..06e96907d9 100644 --- a/packages/server/src/automations/steps/make.ts +++ b/packages/server/src/automations/steps/make.ts @@ -78,8 +78,7 @@ export const definition: AutomationStepSchema = { } export async function run({ inputs }: AutomationStepInput) { - //TODO - Remove deprecated values 1,2,3,4,5 after November 2023 - const { url, value1, value2, value3, value4, value5, body } = inputs + const { url, body } = inputs let payload = {} try { @@ -104,11 +103,6 @@ export async function run({ inputs }: AutomationStepInput) { response = await fetch(url, { method: "post", body: JSON.stringify({ - value1, - value2, - value3, - value4, - value5, ...payload, }), headers: { diff --git a/packages/server/src/automations/steps/zapier.ts b/packages/server/src/automations/steps/zapier.ts index d69ac39ba7..eeff0c2c7d 100644 --- a/packages/server/src/automations/steps/zapier.ts +++ b/packages/server/src/automations/steps/zapier.ts @@ -71,8 +71,7 @@ export const definition: AutomationStepSchema = { } export async function run({ inputs }: AutomationStepInput) { - //TODO - Remove deprecated values 1,2,3,4,5 after November 2023 - const { url, value1, value2, value3, value4, value5, body } = inputs + const { url, body } = inputs let payload = {} try { @@ -100,11 +99,6 @@ export async function run({ inputs }: AutomationStepInput) { method: "post", body: JSON.stringify({ platform: "budibase", - value1, - value2, - value3, - value4, - value5, ...payload, }), headers: { diff --git a/packages/server/src/threads/definitions.ts b/packages/server/src/threads/definitions.ts index dd0891d34a..8915642949 100644 --- a/packages/server/src/threads/definitions.ts +++ b/packages/server/src/threads/definitions.ts @@ -11,12 +11,7 @@ export interface QueryEvent { queryId: string environmentVariables?: Record ctx?: any - schema?: { - [key: string]: { - name: string - type: string - } - } + schema?: Record } export interface QueryVariable { diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index 5409ed925c..6f41d3d55f 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -48,7 +48,7 @@ export async function processOutputBBReferences( ) { if (typeof value !== "string") { // Already processed or nothing to process - return value + return value || undefined } const ids = value.split(",").filter(id => !!id) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 773b54dd6a..0bdaaa393e 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -11,6 +11,7 @@ import { processInputBBReferences, processOutputBBReferences, } from "./bbReferenceProcessor" +import { isExternalTable } from "../../integrations/utils" export * from "./utils" type AutoColumnProcessingOpts = { @@ -234,9 +235,6 @@ export async function outputProcessing( } } else if (column.type == FieldTypes.BB_REFERENCE) { for (let row of enriched) { - if (row[property] == null) { - continue - } row[property] = await processOutputBBReferences( row[property], column.subtype as FieldSubtype @@ -250,6 +248,16 @@ export async function outputProcessing( enriched )) as Row[] } + // remove null properties to match internal API + if (isExternalTable(table._id!)) { + for (let row of enriched) { + for (let key of Object.keys(row)) { + if (row[key] === null) { + delete row[key] + } + } + } + } return (wasArray ? enriched : enriched[0]) as T } diff --git a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts index 1b780bed54..ecb8856c88 100644 --- a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts @@ -66,7 +66,7 @@ describe("rowProcessor - outputProcessing", () => { ) }) - it("does not fetch bb references when fields are empty", async () => { + it("process output even when the field is not empty", async () => { const table: Table = { _id: generator.guid(), name: "TestTable", @@ -100,7 +100,7 @@ describe("rowProcessor - outputProcessing", () => { expect(result).toEqual({ name: "Jack" }) - expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled() + expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledTimes(1) }) it("does not fetch bb references when not in the schema", async () => { diff --git a/packages/shared-core/src/helpers/integrations.ts b/packages/shared-core/src/helpers/integrations.ts index b8c220c6a5..5cc8de880f 100644 --- a/packages/shared-core/src/helpers/integrations.ts +++ b/packages/shared-core/src/helpers/integrations.ts @@ -14,5 +14,5 @@ export function isSQL(datasource: Datasource): boolean { SourceName.MYSQL, SourceName.ORACLE, ] - return SQL.indexOf(datasource.source) !== -1 + return SQL.indexOf(datasource.source) !== -1 || datasource.isSQL === true } diff --git a/packages/types/src/documents/app/datasource.ts b/packages/types/src/documents/app/datasource.ts index 855006ea4c..67035a2e72 100644 --- a/packages/types/src/documents/app/datasource.ts +++ b/packages/types/src/documents/app/datasource.ts @@ -9,6 +9,7 @@ export interface Datasource extends Document { // the config is defined by the schema config?: Record plus?: boolean + isSQL?: boolean entities?: { [key: string]: Table } diff --git a/packages/types/src/documents/app/query.ts b/packages/types/src/documents/app/query.ts index 31a3a3ba09..c288ed9980 100644 --- a/packages/types/src/documents/app/query.ts +++ b/packages/types/src/documents/app/query.ts @@ -6,7 +6,7 @@ export interface Query extends Document { parameters: QueryParameter[] fields: RestQueryFields | any transformer: string | null - schema: any + schema: Record readable: boolean queryVerb: string } diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index d6a0d4a7c8..0e06b8fae0 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -140,6 +140,7 @@ export interface DatasourceConfig { export interface Integration { docs: string plus?: boolean + isSQL?: boolean auth?: { type: string } features?: Partial> relationships?: boolean diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts index 861f5e9329..a7df701171 100644 --- a/packages/types/src/sdk/koa.ts +++ b/packages/types/src/sdk/koa.ts @@ -1,5 +1,5 @@ import { Context, Request } from "koa" -import { User, Role, UserRoles, Account } from "../documents" +import { User, Role, UserRoles, Account, ConfigType } from "../documents" import { FeatureFlag, License } from "../sdk" import { Files } from "formidable" @@ -13,6 +13,7 @@ export interface ContextUser extends Omit { csrfToken?: string featureFlags?: FeatureFlag[] accountPortalAccess?: boolean + providerType?: ConfigType account?: Account } diff --git a/packages/worker/src/api/routes/global/tests/scim.spec.ts b/packages/worker/src/api/routes/global/tests/scim.spec.ts index fba1523cd4..884625805c 100644 --- a/packages/worker/src/api/routes/global/tests/scim.spec.ts +++ b/packages/worker/src/api/routes/global/tests/scim.spec.ts @@ -11,7 +11,7 @@ import { TestConfiguration } from "../../../../tests" import { events } from "@budibase/backend-core" // this test can 409 - retries reduce issues with this -jest.retryTimes(2) +jest.retryTimes(2, { logErrorsBeforeRetry: true }) jest.setTimeout(30000) mocks.licenses.useScimIntegration()