diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 9b0c49d9f8..5a6907faa0 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -272,17 +272,6 @@ class InternalBuilder { return parts.join(".") } - private isFullSelectStatementRequired(): boolean { - for (let column of Object.values(this.table.schema)) { - if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) { - return true - } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) { - return true - } - } - return false - } - private generateSelectStatement(): (string | Knex.Raw)[] | "*" { const { table, resource } = this.query @@ -292,11 +281,9 @@ class InternalBuilder { const alias = this.getTableName(table) const schema = this.table.schema - if (!this.isFullSelectStatementRequired()) { - return [this.knex.raw("??", [`${alias}.*`])] - } + // get just the fields for this table - return resource.fields + const tableFields = resource.fields .map(field => { const parts = field.split(/\./g) let table: string | undefined = undefined @@ -311,34 +298,33 @@ class InternalBuilder { return { table, column, field } }) .filter(({ table }) => !table || table === alias) - .map(({ table, column, field }) => { - const columnSchema = schema[column] - if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) { - return this.knex.raw(`??::money::numeric as ??`, [ - this.rawQuotedIdentifier([table, column].join(".")), - this.knex.raw(this.quote(field)), - ]) - } + return tableFields.map(({ table, column, field }) => { + const columnSchema = schema[column] - if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) { - // Time gets returned as timestamp from mssql, not matching the expected - // HH:mm format + if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) { + return this.knex.raw(`??::money::numeric as ??`, [ + this.rawQuotedIdentifier([table, column].join(".")), + this.knex.raw(this.quote(field)), + ]) + } - // TODO: figure out how to express this safely without string - // interpolation. - return this.knex.raw(`CONVERT(varchar, ??, 108) as ??`, [ - this.rawQuotedIdentifier(field), - this.knex.raw(this.quote(field)), - ]) - } + if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) { + // Time gets returned as timestamp from mssql, not matching the expected + // HH:mm format - if (table) { - return this.rawQuotedIdentifier(`${table}.${column}`) - } else { - return this.rawQuotedIdentifier(field) - } - }) + return this.knex.raw(`CONVERT(varchar, ??, 108) as ??`, [ + this.rawQuotedIdentifier(field), + this.knex.raw(this.quote(field)), + ]) + } + + if (table) { + return this.rawQuotedIdentifier(`${table}.${column}`) + } else { + return this.rawQuotedIdentifier(field) + } + }) } // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses, @@ -1291,6 +1277,7 @@ class InternalBuilder { if (!toTable || !fromTable) { continue } + const relatedTable = tables[toTable] if (!relatedTable) { throw new Error(`related table "${toTable}" not found in datasource`) @@ -1319,6 +1306,10 @@ class InternalBuilder { const fieldList = relationshipFields.map(field => this.buildJsonField(relatedTable, field) ) + if (!fieldList.length) { + continue + } + const fieldListFormatted = fieldList .map(f => { const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," @@ -1359,7 +1350,9 @@ class InternalBuilder { ) const standardWrap = (select: Knex.Raw): Knex.QueryBuilder => { - subQuery = subQuery.select(`${toAlias}.*`).limit(getRelationshipLimit()) + subQuery = subQuery + .select(relationshipFields) + .limit(getRelationshipLimit()) // @ts-ignore - the from alias syntax isn't in Knex typing return knex.select(select).from({ [toAlias]: subQuery, @@ -1589,11 +1582,12 @@ class InternalBuilder { limits?: { base: number; query: number } } = {} ): Knex.QueryBuilder { - let { operation, filters, paginate, relationships, table } = this.query + const { operation, filters, paginate, relationships, table } = this.query const { limits } = opts // start building the query let query = this.qualifiedKnex() + // handle pagination let foundOffset: number | null = null let foundLimit = limits?.query || limits?.base @@ -1642,7 +1636,7 @@ class InternalBuilder { const mainTable = this.query.tableAliases?.[table.name] || table.name const cte = this.addSorting( this.knex - .with("paginated", query) + .with("paginated", query.clone().clearSelect().select("*")) .select(this.generateSelectStatement()) .from({ [mainTable]: "paginated", diff --git a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte index 37abd7f1eb..2260892913 100644 --- a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte @@ -442,13 +442,11 @@ const onUpdateUserInvite = async (invite, role) => { let updateBody = { - code: invite.code, apps: { ...invite.apps, [prodAppId]: role, }, } - if (role === Constants.Roles.CREATOR) { updateBody.builder = updateBody.builder || {} updateBody.builder.apps = [...(updateBody.builder.apps ?? []), prodAppId] @@ -456,7 +454,7 @@ } else if (role !== Constants.Roles.CREATOR && invite?.builder?.apps) { invite.builder.apps = [] } - await users.updateInvite(updateBody) + await users.updateInvite(invite.code, updateBody) await filterInvites(query) } @@ -470,8 +468,7 @@ let updated = { ...invite } delete updated.info.apps[prodAppId] - return await users.updateInvite({ - code: updated.code, + return await users.updateInvite(updated.code, { apps: updated.apps, }) } diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte index f94bad2147..bcd59cd948 100644 --- a/packages/builder/src/pages/builder/portal/apps/index.svelte +++ b/packages/builder/src/pages/builder/portal/apps/index.svelte @@ -191,8 +191,14 @@ ? "View errors" : "View error"} on:dismiss={async () => { - await automationStore.actions.clearLogErrors({ appId }) - await appsStore.load() + const automationId = Object.keys(automationErrors[appId] || {})[0] + if (automationId) { + await automationStore.actions.clearLogErrors({ + appId, + automationId, + }) + await appsStore.load() + } }} message={automationErrorMessage(appId)} /> diff --git a/packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte b/packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte index 8d99d406fd..71fd4c0be3 100644 --- a/packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte +++ b/packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte @@ -52,7 +52,7 @@ ] const removeUser = async id => { - await groups.actions.removeUser(groupId, id) + await groups.removeUser(groupId, id) fetchGroupUsers.refresh() } diff --git a/packages/builder/src/pages/builder/portal/users/users/index.svelte b/packages/builder/src/pages/builder/portal/users/users/index.svelte index 97120c55d4..c77e40c964 100644 --- a/packages/builder/src/pages/builder/portal/users/users/index.svelte +++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte @@ -251,6 +251,7 @@ passwordModal.show() await fetch.refresh() } catch (error) { + console.error(error) notifications.error("Error creating user") } } diff --git a/packages/builder/src/stores/portal/users.js b/packages/builder/src/stores/portal/users.ts similarity index 52% rename from packages/builder/src/stores/portal/users.js rename to packages/builder/src/stores/portal/users.ts index 99ead22317..605f8612aa 100644 --- a/packages/builder/src/stores/portal/users.js +++ b/packages/builder/src/stores/portal/users.ts @@ -1,41 +1,71 @@ -import { writable } from "svelte/store" import { API } from "@/api" -import { update } from "lodash" import { licensing } from "." import { sdk } from "@budibase/shared-core" import { Constants } from "@budibase/frontend-core" +import { + DeleteInviteUsersRequest, + InviteUsersRequest, + SearchUsersRequest, + SearchUsersResponse, + UpdateInviteRequest, + User, + UserIdentifier, + UnsavedUser, +} from "@budibase/types" +import { BudiStore } from "../BudiStore" -export function createUsersStore() { - const { subscribe, set } = writable({}) +interface UserInfo { + email: string + password: string + forceResetPassword?: boolean + role: keyof typeof Constants.BudibaseRoles +} - // opts can contain page and search params - async function search(opts = {}) { +type UserState = SearchUsersResponse & SearchUsersRequest + +class UserStore extends BudiStore { + constructor() { + super({ + data: [], + }) + } + + async search(opts: SearchUsersRequest = {}) { const paged = await API.searchUsers(opts) - set({ + this.set({ ...paged, ...opts, }) return paged } - async function get(userId) { + async get(userId: string) { try { return await API.getUser(userId) } catch (err) { return null } } - const fetch = async () => { + + async fetch() { return await API.getUsers() } - // One or more users. - async function onboard(payload) { + async onboard(payload: InviteUsersRequest) { return await API.onboardUsers(payload) } - async function invite(payload) { - const users = payload.map(user => { + async invite( + payload: { + admin?: boolean + builder?: boolean + creator?: boolean + email: string + apps?: any[] + groups?: any[] + }[] + ) { + const users: InviteUsersRequest = payload.map(user => { let builder = undefined if (user.admin || user.builder) { builder = { global: true } @@ -55,11 +85,16 @@ export function createUsersStore() { return API.inviteUsers(users) } - async function removeInvites(payload) { + async removeInvites(payload: DeleteInviteUsersRequest) { return API.removeUserInvites(payload) } - async function acceptInvite(inviteCode, password, firstName, lastName) { + async acceptInvite( + inviteCode: string, + password: string, + firstName: string, + lastName?: string + ) { return API.acceptInvite({ inviteCode, password, @@ -68,21 +103,25 @@ export function createUsersStore() { }) } - async function fetchInvite(inviteCode) { + async fetchInvite(inviteCode: string) { return API.getUserInvite(inviteCode) } - async function getInvites() { + async getInvites() { return API.getUserInvites() } - async function updateInvite(invite) { - return API.updateUserInvite(invite.code, invite) + async updateInvite(code: string, invite: UpdateInviteRequest) { + return API.updateUserInvite(code, invite) } - async function create(data) { - let mappedUsers = data.users.map(user => { - const body = { + async getUserCountByApp(appId: string) { + return await API.getUserCountByApp(appId) + } + + async create(data: { users: UserInfo[]; groups: any[] }) { + let mappedUsers: UnsavedUser[] = data.users.map((user: any) => { + const body: UnsavedUser = { email: user.email, password: user.password, roles: {}, @@ -92,17 +131,17 @@ export function createUsersStore() { } switch (user.role) { - case "appUser": + case Constants.BudibaseRoles.AppUser: body.builder = { global: false } body.admin = { global: false } break - case "developer": + case Constants.BudibaseRoles.Developer: body.builder = { global: true } break - case "creator": + case Constants.BudibaseRoles.Creator: body.builder = { creator: true, global: false } break - case "admin": + case Constants.BudibaseRoles.Admin: body.admin = { global: true } body.builder = { global: true } break @@ -111,43 +150,47 @@ export function createUsersStore() { return body }) const response = await API.createUsers(mappedUsers, data.groups) + licensing.setQuotaUsage() // re-search from first page - await search() + await this.search() return response } - async function del(id) { + async delete(id: string) { await API.deleteUser(id) - update(users => users.filter(user => user._id !== id)) + licensing.setQuotaUsage() } - async function getUserCountByApp(appId) { - return await API.getUserCountByApp(appId) + async bulkDelete(users: UserIdentifier[]) { + const res = API.deleteUsers(users) + licensing.setQuotaUsage() + return res } - async function bulkDelete(users) { - return API.deleteUsers(users) + async save(user: User) { + const res = await API.saveUser(user) + licensing.setQuotaUsage() + return res } - async function save(user) { - return await API.saveUser(user) - } - - async function addAppBuilder(userId, appId) { + async addAppBuilder(userId: string, appId: string) { return await API.addAppBuilder(userId, appId) } - async function removeAppBuilder(userId, appId) { + async removeAppBuilder(userId: string, appId: string) { return await API.removeAppBuilder(userId, appId) } - async function getAccountHolder() { + async getAccountHolder() { return await API.getAccountHolder() } - const getUserRole = user => { - if (user && user.email === user.tenantOwnerEmail) { + getUserRole(user?: User & { tenantOwnerEmail?: string }) { + if (!user) { + return Constants.BudibaseRoles.AppUser + } + if (user.email === user.tenantOwnerEmail) { return Constants.BudibaseRoles.Owner } else if (sdk.users.isAdmin(user)) { return Constants.BudibaseRoles.Admin @@ -159,38 +202,6 @@ export function createUsersStore() { return Constants.BudibaseRoles.AppUser } } - - const refreshUsage = - fn => - async (...args) => { - const response = await fn(...args) - await licensing.setQuotaUsage() - return response - } - - return { - subscribe, - search, - get, - getUserRole, - fetch, - invite, - onboard, - fetchInvite, - getInvites, - removeInvites, - updateInvite, - getUserCountByApp, - addAppBuilder, - removeAppBuilder, - // any operation that adds or deletes users - acceptInvite, - create: refreshUsage(create), - save: refreshUsage(save), - bulkDelete: refreshUsage(bulkDelete), - delete: refreshUsage(del), - getAccountHolder, - } } -export const users = createUsersStore() +export const users = new UserStore() diff --git a/packages/frontend-core/src/api/user.ts b/packages/frontend-core/src/api/user.ts index 84ec68644d..cf66751078 100644 --- a/packages/frontend-core/src/api/user.ts +++ b/packages/frontend-core/src/api/user.ts @@ -21,11 +21,12 @@ import { SaveUserResponse, SearchUsersRequest, SearchUsersResponse, + UnsavedUser, UpdateInviteRequest, UpdateInviteResponse, UpdateSelfMetadataRequest, UpdateSelfMetadataResponse, - User, + UserIdentifier, } from "@budibase/types" import { BaseAPIClient } from "./types" @@ -38,14 +39,9 @@ export interface UserEndpoints { createAdminUser: ( user: CreateAdminUserRequest ) => Promise - saveUser: (user: User) => Promise + saveUser: (user: UnsavedUser) => Promise deleteUser: (userId: string) => Promise - deleteUsers: ( - users: Array<{ - userId: string - email: string - }> - ) => Promise + deleteUsers: (users: UserIdentifier[]) => Promise onboardUsers: (data: InviteUsersRequest) => Promise getUserInvite: (code: string) => Promise getUserInvites: () => Promise @@ -60,7 +56,7 @@ export interface UserEndpoints { getAccountHolder: () => Promise searchUsers: (data: SearchUsersRequest) => Promise createUsers: ( - users: User[], + users: UnsavedUser[], groups: any[] ) => Promise updateUserInvite: ( diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index d56fb1a344..1c021baab5 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -45,6 +45,9 @@ export async function handleRequest( export async function patch(ctx: UserCtx) { const source = await utils.getSource(ctx) + const { viewId, tableId } = utils.getSourceId(ctx) + const sourceId = viewId || tableId + if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) { ctx.throw(400, "Cannot update rows through a calculation view") } @@ -86,7 +89,7 @@ export async function patch(ctx: UserCtx) { // The id might have been changed, so the refetching would fail. Recalculating the id just in case const updatedId = generateIdForRow({ ...beforeRow, ...dataToUpdate }, table) || _id - const row = await sdk.rows.external.getRow(table._id!, updatedId, { + const row = await sdk.rows.external.getRow(sourceId, updatedId, { relationships: true, }) diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index af696c0758..1793d64b89 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -14,7 +14,8 @@ import { import { breakExternalTableId } from "../../../../integrations/utils" import { generateJunctionTableID } from "../../../../db/utils" import sdk from "../../../../sdk" -import { helpers } from "@budibase/shared-core" +import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core" +import { sql } from "@budibase/backend-core" type TableMap = Record @@ -118,45 +119,131 @@ export async function buildSqlFieldList( opts?: { relationships: boolean } ) { const { relationships } = opts || {} + + const nonMappedColumns = [FieldType.LINK, FieldType.FORMULA, FieldType.AI] + function extractRealFields(table: Table, existing: string[] = []) { return Object.entries(table.schema) .filter( ([columnName, column]) => - column.type !== FieldType.LINK && - column.type !== FieldType.FORMULA && - column.type !== FieldType.AI && - !existing.find( - (field: string) => field === `${table.name}.${columnName}` - ) + !nonMappedColumns.includes(column.type) && + !existing.find((field: string) => field === columnName) ) - .map(([columnName]) => `${table.name}.${columnName}`) + .map(([columnName]) => columnName) + } + + function getRequiredFields(table: Table, existing: string[] = []) { + const requiredFields: string[] = [] + if (table.primary) { + requiredFields.push(...table.primary) + } + if (table.primaryDisplay) { + requiredFields.push(table.primaryDisplay) + } + + if (!sql.utils.isExternalTable(table)) { + requiredFields.push(...PROTECTED_INTERNAL_COLUMNS) + } + + return requiredFields.filter( + column => + !existing.find((field: string) => field === column) && + table.schema[column] && + !nonMappedColumns.includes(table.schema[column].type) + ) } let fields: string[] = [] - if (sdk.views.isView(source)) { - fields = Object.keys(helpers.views.basicFields(source)) - } else { - fields = extractRealFields(source) - } + + const isView = sdk.views.isView(source) let table: Table - if (sdk.views.isView(source)) { + if (isView) { table = await sdk.views.getTable(source.id) + + fields = Object.keys(helpers.views.basicFields(source)).filter( + f => table.schema[f].type !== FieldType.LINK + ) } else { table = source + fields = extractRealFields(source).filter( + f => table.schema[f].visible !== false + ) } - for (let field of Object.values(table.schema)) { + const containsFormula = (isView ? fields : Object.keys(table.schema)).some( + f => table.schema[f]?.type === FieldType.FORMULA + ) + // If are requesting for a formula field, we need to retrieve all fields + if (containsFormula) { + fields = extractRealFields(table) + } + + if (!isView || !helpers.views.isCalculationView(source)) { + fields.push( + ...getRequiredFields( + { + ...table, + primaryDisplay: source.primaryDisplay || table.primaryDisplay, + }, + fields + ) + ) + } + + fields = fields.map(c => `${table.name}.${c}`) + + for (const field of Object.values(table.schema)) { if (field.type !== FieldType.LINK || !relationships || !field.tableId) { continue } - const { tableName } = breakExternalTableId(field.tableId) - if (tables[tableName]) { - fields = fields.concat(extractRealFields(tables[tableName], fields)) + + if ( + isView && + (!source.schema?.[field.name] || + !helpers.views.isVisible(source.schema[field.name])) && + !containsFormula + ) { + continue } + + const { tableName } = breakExternalTableId(field.tableId) + const relatedTable = tables[tableName] + if (!relatedTable) { + continue + } + + const viewFields = new Set() + if (containsFormula) { + extractRealFields(relatedTable).forEach(f => viewFields.add(f)) + } else { + relatedTable.primary?.forEach(f => viewFields.add(f)) + if (relatedTable.primaryDisplay) { + viewFields.add(relatedTable.primaryDisplay) + } + + if (isView) { + Object.entries(source.schema?.[field.name]?.columns || {}) + .filter( + ([columnName, columnConfig]) => + relatedTable.schema[columnName] && + helpers.views.isVisible(columnConfig) && + ![FieldType.LINK, FieldType.FORMULA].includes( + relatedTable.schema[columnName].type + ) + ) + .forEach(([field]) => viewFields.add(field)) + } + } + + const fieldsToAdd = Array.from(viewFields) + .filter(f => !nonMappedColumns.includes(relatedTable.schema[f].type)) + .map(f => `${relatedTable.name}.${f}`) + .filter(f => !fields.includes(f)) + fields.push(...fieldsToAdd) } - return fields + return [...new Set(fields)] } export function isKnexEmptyReadResponse(resp: DatasourcePlusQueryResponse) { diff --git a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts new file mode 100644 index 0000000000..365f571fcf --- /dev/null +++ b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts @@ -0,0 +1,511 @@ +import { + AIOperationEnum, + CalculationType, + FieldType, + RelationshipType, + SourceName, + Table, + ViewV2, + ViewV2Type, +} from "@budibase/types" +import { buildSqlFieldList } from "../sqlUtils" +import { structures } from "../../../../routes/tests/utilities" +import { sql } from "@budibase/backend-core" +import { generator } from "@budibase/backend-core/tests" +import { generateViewID } from "../../../../../db/utils" + +import sdk from "../../../../../sdk" +import { cloneDeep } from "lodash" +import { utils } from "@budibase/shared-core" + +jest.mock("../../../../../sdk/app/views", () => ({ + ...jest.requireActual("../../../../../sdk/app/views"), + getTable: jest.fn(), +})) +const getTableMock = sdk.views.getTable as jest.MockedFunction< + typeof sdk.views.getTable +> + +describe("buildSqlFieldList", () => { + let allTables: Record + + class TableConfig { + private _table: Table & { _id: string } + + constructor(name: string) { + this._table = { + ...structures.tableForDatasource({ + type: "datasource", + source: SourceName.POSTGRES, + }), + name, + _id: sql.utils.buildExternalTableId("ds_id", name), + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + description: { + name: "description", + type: FieldType.STRING, + }, + amount: { + name: "amount", + type: FieldType.NUMBER, + }, + }, + } + + allTables[name] = this._table + } + + withHiddenField(field: string) { + this._table.schema[field].visible = false + return this + } + + withField( + name: string, + type: + | FieldType.STRING + | FieldType.NUMBER + | FieldType.FORMULA + | FieldType.AI, + options?: { visible: boolean } + ) { + switch (type) { + case FieldType.NUMBER: + case FieldType.STRING: + this._table.schema[name] = { + name, + type, + ...options, + } + break + case FieldType.FORMULA: + this._table.schema[name] = { + name, + type, + formula: "any", + ...options, + } + break + case FieldType.AI: + this._table.schema[name] = { + name, + type, + operation: AIOperationEnum.PROMPT, + ...options, + } + break + default: + utils.unreachable(type) + } + return this + } + + withRelation(name: string, toTableId: string) { + this._table.schema[name] = { + name, + type: FieldType.LINK, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "link", + tableId: toTableId, + } + return this + } + + withPrimary(field: string) { + this._table.primary = [field] + return this + } + + withDisplay(field: string) { + this._table.primaryDisplay = field + return this + } + + create() { + return cloneDeep(this._table) + } + } + + class ViewConfig { + private _table: Table + private _view: ViewV2 + + constructor(table: Table) { + this._table = table + this._view = { + version: 2, + id: generateViewID(table._id!), + name: generator.word(), + tableId: table._id!, + } + } + + withVisible(field: string) { + this._view.schema ??= {} + this._view.schema[field] ??= {} + this._view.schema[field].visible = true + return this + } + + withHidden(field: string) { + this._view.schema ??= {} + this._view.schema[field] ??= {} + this._view.schema[field].visible = false + return this + } + + withRelationshipColumns( + field: string, + columns: Record + ) { + this._view.schema ??= {} + this._view.schema[field] ??= {} + this._view.schema[field].columns = columns + return this + } + + withCalculation( + name: string, + field: string, + calculationType: CalculationType + ) { + this._view.type = ViewV2Type.CALCULATION + this._view.schema ??= {} + this._view.schema[name] = { + field, + calculationType, + visible: true, + } + return this + } + + create() { + getTableMock.mockResolvedValueOnce(this._table) + return cloneDeep(this._view) + } + } + + beforeEach(() => { + jest.clearAllMocks() + allTables = {} + }) + + describe("table", () => { + it("extracts fields from table schema", async () => { + const table = new TableConfig("table").create() + const result = await buildSqlFieldList(table, {}) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + ]) + }) + + it("excludes hidden fields", async () => { + const table = new TableConfig("table") + .withHiddenField("description") + .create() + const result = await buildSqlFieldList(table, {}) + expect(result).toEqual(["table.name", "table.amount"]) + }) + + it("excludes non-sql fields fields", async () => { + const table = new TableConfig("table") + .withField("formula", FieldType.FORMULA) + .withField("ai", FieldType.AI) + .withRelation("link", "otherTableId") + .create() + + const result = await buildSqlFieldList(table, {}) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + ]) + }) + + it("includes hidden fields if there is a formula column", async () => { + const table = new TableConfig("table") + .withHiddenField("description") + .withField("formula", FieldType.FORMULA) + .create() + + const result = await buildSqlFieldList(table, {}) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + ]) + }) + + it("includes relationships fields when flagged", async () => { + const otherTable = new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withPrimary("id") + .withDisplay("name") + .create() + + const table = new TableConfig("table") + .withRelation("link", otherTable._id) + .create() + + const result = await buildSqlFieldList(table, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "linkedTable.id", + "linkedTable.name", + ]) + }) + + it("includes all relationship fields if there is a formula column", async () => { + const otherTable = new TableConfig("linkedTable") + .withField("hidden", FieldType.STRING, { visible: false }) + .create() + + const table = new TableConfig("table") + .withRelation("link", otherTable._id) + .withField("formula", FieldType.FORMULA) + .create() + + const result = await buildSqlFieldList(table, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "linkedTable.name", + "linkedTable.description", + "linkedTable.amount", + "linkedTable.hidden", + ]) + }) + + it("never includes non-sql columns from relationships", async () => { + const otherTable = new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withField("hidden", FieldType.STRING, { visible: false }) + .withField("formula", FieldType.FORMULA) + .withField("ai", FieldType.AI) + .withRelation("link", "otherTableId") + .create() + + const table = new TableConfig("table") + .withRelation("link", otherTable._id) + .withField("formula", FieldType.FORMULA) + .create() + + const result = await buildSqlFieldList(table, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "linkedTable.name", + "linkedTable.description", + "linkedTable.amount", + "linkedTable.id", + "linkedTable.hidden", + ]) + }) + }) + + describe("view", () => { + it("extracts fields from table schema", async () => { + const view = new ViewConfig(new TableConfig("table").create()) + .withVisible("amount") + .withHidden("name") + .create() + + const result = await buildSqlFieldList(view, {}) + expect(result).toEqual(["table.amount"]) + }) + + it("includes all fields if there is a formula column", async () => { + const table = new TableConfig("table") + .withField("formula", FieldType.FORMULA) + .create() + const view = new ViewConfig(table) + .withHidden("name") + .withVisible("amount") + .withVisible("formula") + .create() + + const result = await buildSqlFieldList(view, {}) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + ]) + }) + + it("does not includes all fields if the formula column is not included", async () => { + const table = new TableConfig("table") + .withField("formula", FieldType.FORMULA) + .create() + const view = new ViewConfig(table) + .withHidden("name") + .withVisible("amount") + .withHidden("formula") + .create() + + const result = await buildSqlFieldList(view, {}) + expect(result).toEqual(["table.amount"]) + }) + + it("includes relationships columns", async () => { + const otherTable = new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withField("formula", FieldType.FORMULA) + .withPrimary("id") + .create() + + const table = new TableConfig("table") + .withRelation("link", otherTable._id) + .create() + + const view = new ViewConfig(table) + .withVisible("name") + .withVisible("link") + .withRelationshipColumns("link", { + name: { visible: false }, + amount: { visible: true }, + formula: { visible: false }, + }) + .create() + + const result = await buildSqlFieldList(view, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "linkedTable.id", + "linkedTable.amount", + ]) + }) + + it("excludes relationships fields when view is not included in the view", async () => { + const otherTable = new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withPrimary("id") + .withDisplay("name") + .create() + + const table = new TableConfig("table") + .withRelation("link", otherTable._id) + .withField("formula", FieldType.FORMULA) + .create() + + const view = new ViewConfig(table) + .withVisible("name") + .withHidden("amount") + .create() + + const result = await buildSqlFieldList(view, allTables, { + relationships: true, + }) + expect(result).toEqual(["table.name"]) + }) + + it("does not include relationships columns for hidden links", async () => { + const otherTable = new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withField("formula", FieldType.FORMULA) + .withPrimary("id") + .create() + + const table = new TableConfig("table") + .withRelation("link", otherTable._id) + .create() + + const view = new ViewConfig(table) + .withVisible("name") + .withHidden("link") + .withRelationshipColumns("link", { + name: { visible: false }, + amount: { visible: true }, + formula: { visible: false }, + }) + .create() + + const result = await buildSqlFieldList(view, allTables, { + relationships: true, + }) + expect(result).toEqual(["table.name"]) + }) + + it("includes all relationship fields if there is a formula column", async () => { + const otherTable = new TableConfig("linkedTable") + .withField("id", FieldType.NUMBER) + .withField("hidden", FieldType.STRING, { visible: false }) + .withField("formula", FieldType.FORMULA) + .withField("ai", FieldType.AI) + .withRelation("link", "otherTableId") + .withPrimary("id") + .create() + + const table = new TableConfig("table") + .withRelation("link", otherTable._id) + .withField("formula", FieldType.FORMULA) + .create() + + const view = new ViewConfig(table) + .withVisible("name") + .withVisible("formula") + .withHidden("link") + .withRelationshipColumns("link", { + name: { visible: false }, + amount: { visible: true }, + formula: { visible: false }, + }) + .create() + + const result = await buildSqlFieldList(view, allTables, { + relationships: true, + }) + expect(result).toEqual([ + "table.name", + "table.description", + "table.amount", + "linkedTable.name", + "linkedTable.description", + "linkedTable.amount", + "linkedTable.id", + "linkedTable.hidden", + ]) + }) + }) + + describe("calculation view", () => { + it("does not include calculation fields", async () => { + const view = new ViewConfig(new TableConfig("table").create()) + .withCalculation("average", "amount", CalculationType.AVG) + + .create() + + const result = await buildSqlFieldList(view, {}) + expect(result).toEqual([]) + }) + + it("includes visible fields calculation fields", async () => { + const view = new ViewConfig(new TableConfig("table").create()) + .withCalculation("average", "amount", CalculationType.AVG) + .withHidden("name") + .withVisible("amount") + + .create() + + const result = await buildSqlFieldList(view, {}) + expect(result).toEqual(["table.amount"]) + }) + }) +}) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 6e674aa58e..713605eadb 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -13,7 +13,7 @@ const mainDescriptions = datasourceDescribe({ if (mainDescriptions.length) { describe.each(mainDescriptions)( - "/postgres integrations", + "/postgres integrations ($dbName)", ({ config, dsProvider }) => { let datasource: Datasource let client: Knex diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 7c6f583762..898ab9314a 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -73,6 +73,27 @@ describe("Captures of real examples", () => { }) describe("read", () => { + it("should retrieve all fields if non are specified", () => { + const queryJson = getJson("basicFetch.json") + delete queryJson.resource + + let query = new Sql(SqlClient.POSTGRES)._query(queryJson) + expect(query).toEqual({ + bindings: [primaryLimit], + sql: `select * from "persons" as "a" order by "a"."firstname" asc nulls first, "a"."personid" asc limit $1`, + }) + }) + + it("should retrieve only requested fields", () => { + const queryJson = getJson("basicFetch.json") + + let query = new Sql(SqlClient.POSTGRES)._query(queryJson) + expect(query).toEqual({ + bindings: [primaryLimit], + sql: `select "a"."year", "a"."firstname", "a"."personid", "a"."age", "a"."type", "a"."lastname" from "persons" as "a" order by "a"."firstname" asc nulls first, "a"."personid" asc limit $1`, + }) + }) + it("should handle basic retrieval with relationships", () => { const queryJson = getJson("basicFetchWithRelationships.json") let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( @@ -112,9 +133,9 @@ describe("Captures of real examples", () => { bindings: [primaryLimit, relationshipLimit], sql: expect.stringContaining( multiline( - `with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) - select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) - from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" + `with "paginated" as (select * from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) + select "a"."productname", "a"."productid", (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) + from (select "b"."executorid", "b"."qaid", "b"."taskid", "b"."completed", "b"."taskname" from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc` ) ), @@ -130,9 +151,9 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [...filters, relationshipLimit, relationshipLimit], sql: multiline( - `with "paginated" as (select "a".* from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) - select "a".*, (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) - from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" + `with "paginated" as (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) + select "a"."executorid", "a"."taskname", "a"."taskid", "a"."completed", "a"."qaid", (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) + from (select "b"."productid", "b"."productname" from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc` ), }) @@ -209,7 +230,7 @@ describe("Captures of real examples", () => { bindings: ["ddd", ""], sql: multiline(`delete from "compositetable" as "a" where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE) - returning "a".*`), + returning "a"."keyparttwo", "a"."keypartone", "a"."name"`), }) }) }) diff --git a/packages/server/src/integrations/tests/sqlQueryJson/basicFetch.json b/packages/server/src/integrations/tests/sqlQueryJson/basicFetch.json new file mode 100644 index 0000000000..9d9026c922 --- /dev/null +++ b/packages/server/src/integrations/tests/sqlQueryJson/basicFetch.json @@ -0,0 +1,137 @@ +{ + "operation": "READ", + "resource": { + "fields": [ + "a.year", + "a.firstname", + "a.personid", + "a.age", + "a.type", + "a.lastname" + ] + }, + "filters": {}, + "sort": { + "firstname": { + "direction": "ascending" + } + }, + "paginate": { + "limit": 100, + "page": 1 + }, + "relationships": [], + "extra": { + "idFilter": {} + }, + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", + "primary": ["personid"], + "name": "persons", + "schema": { + "year": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "year", + "constraints": { + "presence": false + } + }, + "firstname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "firstname", + "constraints": { + "presence": false + } + }, + "personid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "personid", + "constraints": { + "presence": false + } + }, + "address": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "address", + "constraints": { + "presence": false + } + }, + "age": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + }, + "type": { + "type": "options", + "externalType": "USER-DEFINED", + "autocolumn": false, + "name": "type", + "constraints": { + "presence": false, + "inclusion": ["support", "designer", "programmer", "qa"] + } + }, + "city": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "city", + "constraints": { + "presence": false + } + }, + "lastname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "lastname", + "constraints": { + "presence": false + } + }, + "QA": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "QA", + "relationshipType": "many-to-one", + "fieldName": "qaid", + "type": "link", + "main": true, + "_id": "ccb68481c80c34217a4540a2c6c27fe46", + "foreignKey": "personid" + }, + "executor": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "executor", + "relationshipType": "many-to-one", + "fieldName": "executorid", + "type": "link", + "main": true, + "_id": "c89530b9770d94bec851e062b5cff3001", + "foreignKey": "personid", + "tableName": "persons" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "firstname", + "views": {} + }, + "tableAliases": { + "persons": "a", + "tasks": "b" + } +} diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index aa799390b8..84162a67af 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -55,32 +55,44 @@ const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`) const MISSING_TABLE_REGX = new RegExp(`no such table: .+`) const DUPLICATE_COLUMN_REGEX = new RegExp(`duplicate column name: .+`) -async function buildInternalFieldList( +export async function buildInternalFieldList( source: Table | ViewV2, tables: Table[], - opts?: { relationships?: RelationshipsJson[]; allowedFields?: string[] } + opts?: { + relationships?: RelationshipsJson[] + allowedFields?: string[] + includeHiddenFields?: boolean + } ) { - const { relationships, allowedFields } = opts || {} + const { relationships, allowedFields, includeHiddenFields } = opts || {} let schemaFields: string[] = [] - if (sdk.views.isView(source)) { - schemaFields = Object.keys(helpers.views.basicFields(source)) - } else { - schemaFields = Object.keys(source.schema).filter( - key => source.schema[key].visible !== false - ) - } - - if (allowedFields) { - schemaFields = schemaFields.filter(field => allowedFields.includes(field)) - } + const isView = sdk.views.isView(source) let table: Table - if (sdk.views.isView(source)) { + if (isView) { table = await sdk.views.getTable(source.id) } else { table = source } + if (isView) { + schemaFields = Object.keys(helpers.views.basicFields(source)) + } else { + schemaFields = Object.keys(source.schema).filter( + key => includeHiddenFields || source.schema[key].visible !== false + ) + } + + const containsFormula = schemaFields.some( + f => table.schema[f]?.type === FieldType.FORMULA + ) + // If are requesting for a formula field, we need to retrieve all fields + if (containsFormula) { + schemaFields = Object.keys(table.schema) + } else if (allowedFields) { + schemaFields = schemaFields.filter(field => allowedFields.includes(field)) + } + let fieldList: string[] = [] const getJunctionFields = (relatedTable: Table, fields: string[]) => { const junctionFields: string[] = [] @@ -101,10 +113,12 @@ async function buildInternalFieldList( } for (let key of schemaFields) { const col = table.schema[key] + const isRelationship = col.type === FieldType.LINK if (!relationships && isRelationship) { continue } + if (!isRelationship) { fieldList.push(`${table._id}.${mapToUserColumn(key)}`) } else { @@ -113,8 +127,17 @@ async function buildInternalFieldList( if (!relatedTable) { continue } + + // a quirk of how junction documents work in Budibase, refer to the "LinkDocument" type to see the full + // structure - essentially all relationships between two tables will be inserted into a single "table" + // we don't use an independent junction table ID for each separate relationship between two tables. For + // example if we have table A and B, with two relationships between them, all the junction documents will + // end up in the same junction table ID. We need to retrieve the field name property of the junction documents + // as part of the relationship to tell us which relationship column the junction is related to. const relatedFields = ( - await buildInternalFieldList(relatedTable, tables) + await buildInternalFieldList(relatedTable, tables, { + includeHiddenFields: containsFormula, + }) ).concat( getJunctionFields(relatedTable, ["doc1.fieldName", "doc2.fieldName"]) ) @@ -125,6 +148,13 @@ async function buildInternalFieldList( fieldList = fieldList.concat(relatedFields) } } + + if (!isView || !helpers.views.isCalculationView(source)) { + for (const field of PROTECTED_INTERNAL_COLUMNS) { + fieldList.push(`${table._id}.${field}`) + } + } + return [...new Set(fieldList)] } @@ -323,8 +353,9 @@ export async function search( } let aggregations: Aggregation[] = [] - if (sdk.views.isView(source)) { + if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) { const calculationFields = helpers.views.calculationFields(source) + for (const [key, field] of Object.entries(calculationFields)) { if (options.fields && !options.fields.includes(key)) { continue diff --git a/packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts b/packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts new file mode 100644 index 0000000000..91674089dc --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts @@ -0,0 +1,618 @@ +import { + AIOperationEnum, + CalculationType, + FieldType, + RelationshipType, + SourceName, + Table, + ViewV2, + ViewV2Type, +} from "@budibase/types" +import { buildInternalFieldList } from "../sqs" +import { structures } from "../../../../../../api/routes/tests/utilities" +import { sql } from "@budibase/backend-core" +import { generator } from "@budibase/backend-core/tests" +import { + generateJunctionTableID, + generateViewID, +} from "../../../../../../db/utils" + +import sdk from "../../../../../../sdk" +import { cloneDeep } from "lodash" +import { utils } from "@budibase/shared-core" + +jest.mock("../../../../../../sdk/app/views", () => ({ + ...jest.requireActual("../../../../../../sdk/app/views"), + getTable: jest.fn(), +})) +const getTableMock = sdk.views.getTable as jest.MockedFunction< + typeof sdk.views.getTable +> + +describe("buildInternalFieldList", () => { + let allTables: Table[] + + class TableConfig { + private _table: Table & { _id: string } + + constructor() { + const name = generator.word() + this._table = { + ...structures.tableForDatasource({ + type: "datasource", + source: SourceName.POSTGRES, + }), + name, + _id: sql.utils.buildExternalTableId("ds_id", name), + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + description: { + name: "description", + type: FieldType.STRING, + }, + amount: { + name: "amount", + type: FieldType.NUMBER, + }, + }, + } + + allTables.push(this._table) + } + + withHiddenField(field: string) { + this._table.schema[field].visible = false + return this + } + + withField( + name: string, + type: + | FieldType.STRING + | FieldType.NUMBER + | FieldType.FORMULA + | FieldType.AI, + options?: { visible: boolean } + ) { + switch (type) { + case FieldType.NUMBER: + case FieldType.STRING: + this._table.schema[name] = { + name, + type, + ...options, + } + break + case FieldType.FORMULA: + this._table.schema[name] = { + name, + type, + formula: "any", + ...options, + } + break + case FieldType.AI: + this._table.schema[name] = { + name, + type, + operation: AIOperationEnum.PROMPT, + ...options, + } + break + default: + utils.unreachable(type) + } + return this + } + + withRelation(name: string, toTableId: string) { + this._table.schema[name] = { + name, + type: FieldType.LINK, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "link", + tableId: toTableId, + } + return this + } + + withEmptySchema() { + this._table.schema = {} + return this + } + + create() { + return cloneDeep(this._table) + } + } + + class ViewConfig { + private _table: Table + private _view: ViewV2 + + constructor(table: Table) { + this._table = table + this._view = { + version: 2, + id: generateViewID(table._id!), + name: generator.word(), + tableId: table._id!, + } + } + + withVisible(field: string) { + this._view.schema ??= {} + this._view.schema[field] ??= {} + this._view.schema[field].visible = true + return this + } + + withHidden(field: string) { + this._view.schema ??= {} + this._view.schema[field] ??= {} + this._view.schema[field].visible = false + return this + } + + withRelationshipColumns( + field: string, + columns: Record + ) { + this._view.schema ??= {} + this._view.schema[field] ??= {} + this._view.schema[field].columns = columns + return this + } + + withCalculation( + name: string, + field: string, + calculationType: CalculationType + ) { + this._view.type = ViewV2Type.CALCULATION + this._view.schema ??= {} + this._view.schema[name] = { + field, + calculationType, + visible: true, + } + return this + } + + create() { + getTableMock.mockResolvedValueOnce(this._table) + return cloneDeep(this._view) + } + } + + beforeEach(() => { + jest.clearAllMocks() + allTables = [] + }) + + describe("table", () => { + it("includes internal columns by default", async () => { + const table = new TableConfig().withEmptySchema().create() + const result = await buildInternalFieldList(table, []) + expect(result).toEqual([ + `${table._id}._id`, + `${table._id}._rev`, + `${table._id}.type`, + `${table._id}.createdAt`, + `${table._id}.updatedAt`, + `${table._id}.tableId`, + ]) + }) + + it("extracts fields from table schema", async () => { + const table = new TableConfig().create() + const result = await buildInternalFieldList(table, []) + expect(result).toEqual([ + `${table._id}.data_name`, + `${table._id}.data_description`, + `${table._id}.data_amount`, + `${table._id}._id`, + `${table._id}._rev`, + `${table._id}.type`, + `${table._id}.createdAt`, + `${table._id}.updatedAt`, + `${table._id}.tableId`, + ]) + }) + + it("excludes hidden fields", async () => { + const table = new TableConfig().withHiddenField("description").create() + const result = await buildInternalFieldList(table, []) + expect(result).toEqual([ + `${table._id}.data_name`, + `${table._id}.data_amount`, + `${table._id}._id`, + `${table._id}._rev`, + `${table._id}.type`, + `${table._id}.createdAt`, + `${table._id}.updatedAt`, + `${table._id}.tableId`, + ]) + }) + + it("includes hidden fields if there is a formula column", async () => { + const table = new TableConfig() + .withHiddenField("description") + .withField("formula", FieldType.FORMULA) + .create() + + const result = await buildInternalFieldList(table, []) + expect(result).toEqual([ + `${table._id}.data_name`, + `${table._id}.data_description`, + `${table._id}.data_amount`, + `${table._id}.data_formula`, + `${table._id}._id`, + `${table._id}._rev`, + `${table._id}.type`, + `${table._id}.createdAt`, + `${table._id}.updatedAt`, + `${table._id}.tableId`, + ]) + }) + + it("includes relationships fields when flagged", async () => { + const otherTable = new TableConfig() + .withHiddenField("description") + .create() + + const table = new TableConfig() + .withHiddenField("amount") + .withRelation("link", otherTable._id) + .create() + + const relationships = [{ tableName: otherTable.name, column: "link" }] + + const result = await buildInternalFieldList(table, allTables, { + relationships, + }) + expect(result).toEqual([ + `${table._id}.data_name`, + `${table._id}.data_description`, + `${otherTable._id}.data_name`, + `${otherTable._id}.data_amount`, + `${otherTable._id}._id`, + `${otherTable._id}._rev`, + `${otherTable._id}.type`, + `${otherTable._id}.createdAt`, + `${otherTable._id}.updatedAt`, + `${otherTable._id}.tableId`, + `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`, + `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`, + `${table._id}._id`, + `${table._id}._rev`, + `${table._id}.type`, + `${table._id}.createdAt`, + `${table._id}.updatedAt`, + `${table._id}.tableId`, + ]) + }) + + it("includes all relationship fields if there is a formula column", async () => { + const otherTable = new TableConfig() + .withField("hidden", FieldType.STRING, { visible: false }) + .create() + + const table = new TableConfig() + .withRelation("link", otherTable._id) + .withHiddenField("description") + .withField("formula", FieldType.FORMULA) + .create() + + const relationships = [{ tableName: otherTable.name, column: "link" }] + const result = await buildInternalFieldList(table, allTables, { + relationships, + }) + expect(result).toEqual([ + `${table._id}.data_name`, + `${table._id}.data_description`, + `${table._id}.data_amount`, + `${otherTable._id}.data_name`, + `${otherTable._id}.data_description`, + `${otherTable._id}.data_amount`, + `${otherTable._id}.data_hidden`, + `${otherTable._id}._id`, + `${otherTable._id}._rev`, + `${otherTable._id}.type`, + `${otherTable._id}.createdAt`, + `${otherTable._id}.updatedAt`, + `${otherTable._id}.tableId`, + `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`, + `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`, + `${table._id}.data_formula`, + `${table._id}._id`, + `${table._id}._rev`, + `${table._id}.type`, + `${table._id}.createdAt`, + `${table._id}.updatedAt`, + `${table._id}.tableId`, + ]) + }) + }) + + describe("view", () => { + it("includes internal columns by default", async () => { + const view = new ViewConfig(new TableConfig().create()).create() + const result = await buildInternalFieldList(view, []) + expect(result).toEqual([ + `${view.tableId}._id`, + `${view.tableId}._rev`, + `${view.tableId}.type`, + `${view.tableId}.createdAt`, + `${view.tableId}.updatedAt`, + `${view.tableId}.tableId`, + ]) + }) + + it("extracts fields from table schema", async () => { + const view = new ViewConfig(new TableConfig().create()) + .withVisible("amount") + .withHidden("name") + .create() + + const result = await buildInternalFieldList(view, []) + expect(result).toEqual([ + `${view.tableId}.data_amount`, + `${view.tableId}._id`, + `${view.tableId}._rev`, + `${view.tableId}.type`, + `${view.tableId}.createdAt`, + `${view.tableId}.updatedAt`, + `${view.tableId}.tableId`, + ]) + }) + + it("includes all fields if there is a formula column", async () => { + const table = new TableConfig() + .withField("formula", FieldType.FORMULA) + .create() + const view = new ViewConfig(table) + .withHidden("name") + .withVisible("amount") + .withVisible("formula") + .create() + + const result = await buildInternalFieldList(view, []) + expect(result).toEqual([ + `${view.tableId}.data_name`, + `${view.tableId}.data_description`, + `${view.tableId}.data_amount`, + `${view.tableId}.data_formula`, + `${view.tableId}._id`, + `${view.tableId}._rev`, + `${view.tableId}.type`, + `${view.tableId}.createdAt`, + `${view.tableId}.updatedAt`, + `${view.tableId}.tableId`, + ]) + }) + + it("does not includes all fields if the formula column is not included", async () => { + const table = new TableConfig() + .withField("formula", FieldType.FORMULA) + .create() + const view = new ViewConfig(table) + .withHidden("name") + .withVisible("amount") + .withHidden("formula") + .create() + + const result = await buildInternalFieldList(view, []) + expect(result).toEqual([ + `${view.tableId}.data_amount`, + `${view.tableId}._id`, + `${view.tableId}._rev`, + `${view.tableId}.type`, + `${view.tableId}.createdAt`, + `${view.tableId}.updatedAt`, + `${view.tableId}.tableId`, + ]) + }) + + it("includes relationships fields", async () => { + const otherTable = new TableConfig().create() + + const table = new TableConfig() + .withRelation("link", otherTable._id) + .withField("formula", FieldType.FORMULA) + .create() + + const view = new ViewConfig(table) + .withVisible("name") + .withVisible("link") + .withHidden("amount") + .create() + + const relationships = [{ tableName: otherTable.name, column: "link" }] + const result = await buildInternalFieldList(view, allTables, { + relationships, + }) + expect(result).toEqual([ + `${table._id}.data_name`, + `${otherTable._id}.data_name`, + `${otherTable._id}.data_description`, + `${otherTable._id}.data_amount`, + `${otherTable._id}._id`, + `${otherTable._id}._rev`, + `${otherTable._id}.type`, + `${otherTable._id}.createdAt`, + `${otherTable._id}.updatedAt`, + `${otherTable._id}.tableId`, + `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`, + `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`, + `${table._id}._id`, + `${table._id}._rev`, + `${table._id}.type`, + `${table._id}.createdAt`, + `${table._id}.updatedAt`, + `${table._id}.tableId`, + ]) + }) + + it("includes relationships columns", async () => { + const otherTable = new TableConfig() + .withField("formula", FieldType.FORMULA) + .create() + + const table = new TableConfig() + .withRelation("link", otherTable._id) + .create() + + const view = new ViewConfig(table) + .withVisible("name") + .withVisible("link") + .withRelationshipColumns("link", { + name: { visible: false }, + amount: { visible: true }, + formula: { visible: false }, + }) + .create() + + const relationships = [{ tableName: otherTable.name, column: "link" }] + const result = await buildInternalFieldList(view, allTables, { + relationships, + }) + expect(result).toEqual([ + `${table._id}.data_name`, + `${otherTable._id}.data_name`, + `${otherTable._id}.data_description`, + `${otherTable._id}.data_amount`, + `${otherTable._id}.data_formula`, + `${otherTable._id}._id`, + `${otherTable._id}._rev`, + `${otherTable._id}.type`, + `${otherTable._id}.createdAt`, + `${otherTable._id}.updatedAt`, + `${otherTable._id}.tableId`, + `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`, + `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`, + `${table._id}._id`, + `${table._id}._rev`, + `${table._id}.type`, + `${table._id}.createdAt`, + `${table._id}.updatedAt`, + `${table._id}.tableId`, + ]) + }) + + it("does not include relationships columns for hidden links", async () => { + const otherTable = new TableConfig() + .withField("formula", FieldType.FORMULA) + .create() + + const table = new TableConfig() + .withRelation("link", otherTable._id) + .create() + + const view = new ViewConfig(table) + .withVisible("name") + .withHidden("link") + .withRelationshipColumns("link", { + name: { visible: false }, + amount: { visible: true }, + formula: { visible: false }, + }) + .create() + + const relationships = [{ tableName: otherTable.name, column: "link" }] + const result = await buildInternalFieldList(view, allTables, { + relationships, + }) + expect(result).toEqual([ + `${table._id}.data_name`, + `${table._id}._id`, + `${table._id}._rev`, + `${table._id}.type`, + `${table._id}.createdAt`, + `${table._id}.updatedAt`, + `${table._id}.tableId`, + ]) + }) + + it("includes all relationship fields if there is a formula column", async () => { + const otherTable = new TableConfig() + .withField("hidden", FieldType.STRING, { visible: false }) + .withField("formula", FieldType.FORMULA) + .withField("ai", FieldType.AI) + .withRelation("link", "otherTableId") + .create() + + const table = new TableConfig() + .withRelation("link", otherTable._id) + .withField("formula", FieldType.FORMULA) + .create() + + const view = new ViewConfig(table) + .withVisible("name") + .withVisible("formula") + .withHidden("link") + .withRelationshipColumns("link", { + name: { visible: false }, + amount: { visible: true }, + formula: { visible: false }, + }) + .create() + + const relationships = [{ tableName: otherTable.name, column: "link" }] + const result = await buildInternalFieldList(view, allTables, { + relationships, + }) + expect(result).toEqual([ + `${table._id}.data_name`, + `${table._id}.data_description`, + `${table._id}.data_amount`, + `${otherTable._id}.data_name`, + `${otherTable._id}.data_description`, + `${otherTable._id}.data_amount`, + `${otherTable._id}.data_hidden`, + `${otherTable._id}.data_formula`, + `${otherTable._id}.data_ai`, + `${otherTable._id}._id`, + `${otherTable._id}._rev`, + `${otherTable._id}.type`, + `${otherTable._id}.createdAt`, + `${otherTable._id}.updatedAt`, + `${otherTable._id}.tableId`, + `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`, + `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`, + `${table._id}.data_formula`, + `${table._id}._id`, + `${table._id}._rev`, + `${table._id}.type`, + `${table._id}.createdAt`, + `${table._id}.updatedAt`, + `${table._id}.tableId`, + ]) + }) + }) + + describe("calculation view", () => { + it("does not include calculation fields", async () => { + const view = new ViewConfig(new TableConfig().create()) + .withCalculation("average", "amount", CalculationType.AVG) + .create() + + const result = await buildInternalFieldList(view, []) + expect(result).toEqual([]) + }) + + it("includes visible fields calculation fields", async () => { + const view = new ViewConfig(new TableConfig().create()) + .withCalculation("average", "amount", CalculationType.AVG) + .withHidden("name") + .withVisible("amount") + .create() + + const result = await buildInternalFieldList(view, []) + expect(result).toEqual([`${view.tableId}.data_amount`]) + }) + }) +}) diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts index a42449d550..c1f37fd3f0 100644 --- a/packages/types/src/api/web/user.ts +++ b/packages/types/src/api/web/user.ts @@ -22,6 +22,8 @@ export interface UserDetails { password?: string } +export type UnsavedUser = Omit + export interface BulkUserRequest { delete?: { users: Array<{ @@ -31,7 +33,7 @@ export interface BulkUserRequest { } create?: { roles?: any[] - users: User[] + users: UnsavedUser[] groups: any[] } } @@ -124,7 +126,7 @@ export interface AcceptUserInviteRequest { inviteCode: string password: string firstName: string - lastName: string + lastName?: string } export interface AcceptUserInviteResponse { diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 83f2f41b0e..0bcdadfefc 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -33,6 +33,7 @@ import { SaveUserResponse, SearchUsersRequest, SearchUsersResponse, + UnsavedUser, UpdateInviteRequest, UpdateInviteResponse, User, @@ -49,6 +50,7 @@ import { tenancy, db, locks, + context, } from "@budibase/backend-core" import { checkAnyUserExists } from "../../../utilities/users" import { isEmailConfigured } from "../../../utilities/email" @@ -66,10 +68,11 @@ const generatePassword = (length: number) => { .slice(0, length) } -export const save = async (ctx: UserCtx) => { +export const save = async (ctx: UserCtx) => { try { const currentUserId = ctx.user?._id - const requestUser = ctx.request.body + const tenantId = context.getTenantId() + const requestUser: User = { ...ctx.request.body, tenantId } // Do not allow the account holder role to be changed if ( @@ -151,7 +154,12 @@ export const bulkUpdate = async ( let created, deleted try { if (input.create) { - created = await bulkCreate(input.create.users, input.create.groups) + const tenantId = context.getTenantId() + const users: User[] = input.create.users.map(user => ({ + ...user, + tenantId, + })) + created = await bulkCreate(users, input.create.groups) } if (input.delete) { deleted = await bulkDelete(input.delete.users, currentUserId)