diff --git a/lerna.json b/lerna.json index 13530e9aee..df50828b30 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.17", + "version": "2.33.2", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 2ab8c550cc..4e93e8d9ee 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -54,30 +54,46 @@ function getPackageJsonFields(): { VERSION: string SERVICE_NAME: string } { - function findFileInAncestors( - fileName: string, - currentDir: string - ): string | null { - const filePath = `${currentDir}/${fileName}` - if (existsSync(filePath)) { - return filePath + function getParentFile(file: string) { + function findFileInAncestors( + fileName: string, + currentDir: string + ): string | null { + const filePath = `${currentDir}/${fileName}` + if (existsSync(filePath)) { + return filePath + } + + const parentDir = `${currentDir}/..` + if (parentDir === currentDir) { + // reached root directory + return null + } + + return findFileInAncestors(fileName, parentDir) } - const parentDir = `${currentDir}/..` - if (parentDir === currentDir) { - // reached root directory - return null - } + const packageJsonFile = findFileInAncestors(file, process.cwd()) + const content = readFileSync(packageJsonFile!, "utf-8") + const parsedContent = JSON.parse(content) + return parsedContent + } - return findFileInAncestors(fileName, parentDir) + let localVersion: string | undefined + if (isDev() && !isTest()) { + try { + const lerna = getParentFile("lerna.json") + localVersion = lerna.version + } catch { + // + } } try { - const packageJsonFile = findFileInAncestors("package.json", process.cwd()) - const content = readFileSync(packageJsonFile!, "utf-8") - const parsedContent = JSON.parse(content) + const parsedContent = getParentFile("package.json") return { - VERSION: process.env.BUDIBASE_VERSION || parsedContent.version, + VERSION: + localVersion || process.env.BUDIBASE_VERSION || parsedContent.version, SERVICE_NAME: parsedContent.name, } } catch { diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index fdea09944a..3f26368efe 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -1,3 +1,4 @@ +import semver from "semver" import { BuiltinPermissionID, PermissionLevel } from "./permissions" import { prefixRoleID, @@ -7,7 +8,13 @@ import { doWithDB, } from "../db" import { getAppDB } from "../context" -import { Screen, Role as RoleDoc, RoleUIMetadata } from "@budibase/types" +import { + Screen, + Role as RoleDoc, + RoleUIMetadata, + Database, + App, +} from "@budibase/types" import cloneDeep from "lodash/fp/cloneDeep" import { RoleColor, helpers } from "@budibase/shared-core" @@ -23,14 +30,6 @@ const BUILTIN_IDS = { BUILDER: "BUILDER", } -// exclude internal roles like builder -const EXTERNAL_BUILTIN_ROLE_IDS = [ - BUILTIN_IDS.ADMIN, - BUILTIN_IDS.POWER, - BUILTIN_IDS.BASIC, - BUILTIN_IDS.PUBLIC, -] - export const RoleIDVersion = { // original version, with a UUID based ID UUID: undefined, @@ -425,7 +424,7 @@ export async function getAllRoles(appId?: string): Promise { } return internal(appDB) } - async function internal(db: any) { + async function internal(db: Database | undefined) { let roles: RoleDoc[] = [] if (db) { const body = await db.allDocs( @@ -440,8 +439,26 @@ export async function getAllRoles(appId?: string): Promise { } const builtinRoles = getBuiltinRoles() + // exclude internal roles like builder + let externalBuiltinRoles = [] + + if (!db || (await shouldIncludePowerRole(db))) { + externalBuiltinRoles = [ + BUILTIN_IDS.ADMIN, + BUILTIN_IDS.POWER, + BUILTIN_IDS.BASIC, + BUILTIN_IDS.PUBLIC, + ] + } else { + externalBuiltinRoles = [ + BUILTIN_IDS.ADMIN, + BUILTIN_IDS.BASIC, + BUILTIN_IDS.PUBLIC, + ] + } + // need to combine builtin with any DB record of them (for sake of permissions) - for (let builtinRoleId of EXTERNAL_BUILTIN_ROLE_IDS) { + for (let builtinRoleId of externalBuiltinRoles) { const builtinRole = builtinRoles[builtinRoleId] const dbBuiltin = roles.filter( dbRole => @@ -472,6 +489,18 @@ export async function getAllRoles(appId?: string): Promise { } } +async function shouldIncludePowerRole(db: Database) { + const app = await db.tryGet(DocumentType.APP_METADATA) + const creationVersion = app?.creationVersion + if (!creationVersion || !semver.valid(creationVersion)) { + // Old apps don't have creationVersion, so we should include it for backward compatibility + return true + } + + const isGreaterThan3x = semver.gte(creationVersion, "3.0.0") + return !isGreaterThan3x +} + export class AccessController { userHierarchies: { [key: string]: string[] } constructor() { diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 8ff3fb1606..2b75752aec 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -23,12 +23,14 @@ import { InternalSearchFilterOperator, JsonFieldMetadata, JsonTypes, + LogicalOperator, Operation, prefixed, QueryJson, QueryOptions, RangeOperator, RelationshipsJson, + SearchFilterKey, SearchFilters, SortOrder, SqlClient, @@ -96,6 +98,22 @@ function isSqs(table: Table): boolean { ) } +const allowEmptyRelationships: Record = { + [BasicOperator.EQUAL]: false, + [BasicOperator.NOT_EQUAL]: true, + [BasicOperator.EMPTY]: false, + [BasicOperator.NOT_EMPTY]: true, + [BasicOperator.FUZZY]: false, + [BasicOperator.STRING]: false, + [RangeOperator.RANGE]: false, + [ArrayOperator.CONTAINS]: false, + [ArrayOperator.NOT_CONTAINS]: true, + [ArrayOperator.CONTAINS_ANY]: false, + [ArrayOperator.ONE_OF]: false, + [LogicalOperator.AND]: false, + [LogicalOperator.OR]: false, +} + class InternalBuilder { private readonly client: SqlClient private readonly query: QueryJson @@ -405,31 +423,48 @@ class InternalBuilder { addRelationshipForFilter( query: Knex.QueryBuilder, + allowEmptyRelationships: boolean, filterKey: string, - whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder + whereCb: (filterKey: string, query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { const mainKnex = this.knex const { relationships, endpoint, tableAliases: aliases } = this.query const tableName = endpoint.entityId const fromAlias = aliases?.[tableName] || tableName - const matches = (possibleTable: string) => - filterKey.startsWith(`${possibleTable}`) + const matches = (value: string) => + filterKey.match(new RegExp(`^${value}\\.`)) if (!relationships) { return query } for (const relationship of relationships) { const relatedTableName = relationship.tableName const toAlias = aliases?.[relatedTableName] || relatedTableName + + const matchesTableName = matches(relatedTableName) || matches(toAlias) + const matchesRelationName = matches(relationship.column) + // this is the relationship which is being filtered if ( - (matches(relatedTableName) || matches(toAlias)) && + (matchesTableName || matchesRelationName) && relationship.to && relationship.tableName ) { - let subQuery = mainKnex + const joinTable = mainKnex .select(mainKnex.raw(1)) .from({ [toAlias]: relatedTableName }) + let subQuery = joinTable.clone() const manyToMany = validateManyToMany(relationship) + let updatedKey + + if (!matchesTableName) { + updatedKey = filterKey.replace( + new RegExp(`^${relationship.column}.`), + `${aliases![relationship.tableName]}.` + ) + } else { + updatedKey = filterKey + } + if (manyToMany) { const throughAlias = aliases?.[manyToMany.through] || relationship.through @@ -440,7 +475,6 @@ class InternalBuilder { subQuery = subQuery // add a join through the junction table .innerJoin(throughTable, function () { - // @ts-ignore this.on( `${toAlias}.${manyToMany.toPrimary}`, "=", @@ -460,18 +494,38 @@ class InternalBuilder { if (this.client === SqlClient.SQL_LITE) { subQuery = this.addJoinFieldCheck(subQuery, manyToMany) } + + query = query.where(q => { + q.whereExists(whereCb(updatedKey, subQuery)) + if (allowEmptyRelationships) { + q.orWhereNotExists( + joinTable.clone().innerJoin(throughTable, function () { + this.on( + `${fromAlias}.${manyToMany.fromPrimary}`, + "=", + `${throughAlias}.${manyToMany.from}` + ) + }) + ) + } + }) } else { + const toKey = `${toAlias}.${relationship.to}` + const foreignKey = `${fromAlias}.${relationship.from}` // "join" to the main table, making sure the ID matches that of the main subQuery = subQuery.where( - `${toAlias}.${relationship.to}`, + toKey, "=", - mainKnex.raw( - this.quotedIdentifier(`${fromAlias}.${relationship.from}`) - ) + mainKnex.raw(this.quotedIdentifier(foreignKey)) ) + + query = query.where(q => { + q.whereExists(whereCb(updatedKey, subQuery.clone())) + if (allowEmptyRelationships) { + q.orWhereNotExists(subQuery) + } + }) } - query = query.whereExists(whereCb(subQuery)) - break } } return query @@ -502,6 +556,7 @@ class InternalBuilder { } function iterate( structure: AnySearchFilter, + operation: SearchFilterKey, fn: ( query: Knex.QueryBuilder, key: string, @@ -558,9 +613,14 @@ class InternalBuilder { if (allOr) { query = query.or } - query = builder.addRelationshipForFilter(query, updatedKey, q => { - return handleRelationship(q, updatedKey, value) - }) + query = builder.addRelationshipForFilter( + query, + allowEmptyRelationships[operation], + updatedKey, + (updatedKey, q) => { + return handleRelationship(q, updatedKey, value) + } + ) } } } @@ -592,7 +652,7 @@ class InternalBuilder { return `[${value.join(",")}]` } if (this.client === SqlClient.POSTGRES) { - iterate(mode, (q, key, value) => { + iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { const wrap = any ? "" : "'" const op = any ? "\\?| array" : "@>" const fieldNames = key.split(/\./g) @@ -610,7 +670,7 @@ class InternalBuilder { this.client === SqlClient.MARIADB ) { const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" - iterate(mode, (q, key, value) => { + iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { return q[rawFnc]( `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray( value @@ -619,7 +679,7 @@ class InternalBuilder { }) } else { const andOr = mode === filters?.containsAny ? " OR " : " AND " - iterate(mode, (q, key, value) => { + iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { let statement = "" const identifier = this.quotedIdentifier(key) for (let i in value) { @@ -673,6 +733,7 @@ class InternalBuilder { const fnc = allOr ? "orWhereIn" : "whereIn" iterate( filters.oneOf, + ArrayOperator.ONE_OF, (q, key: string, array) => { if (this.client === SqlClient.ORACLE) { key = this.convertClobs(key) @@ -697,7 +758,7 @@ class InternalBuilder { ) } if (filters.string) { - iterate(filters.string, (q, key, value) => { + iterate(filters.string, BasicOperator.STRING, (q, key, value) => { const fnc = allOr ? "orWhere" : "where" // postgres supports ilike, nothing else does if (this.client === SqlClient.POSTGRES) { @@ -712,10 +773,10 @@ class InternalBuilder { }) } if (filters.fuzzy) { - iterate(filters.fuzzy, like) + iterate(filters.fuzzy, BasicOperator.FUZZY, like) } if (filters.range) { - iterate(filters.range, (q, key, value) => { + iterate(filters.range, RangeOperator.RANGE, (q, key, value) => { const isEmptyObject = (val: any) => { return ( val && @@ -781,7 +842,7 @@ class InternalBuilder { }) } if (filters.equal) { - iterate(filters.equal, (q, key, value) => { + iterate(filters.equal, BasicOperator.EQUAL, (q, key, value) => { const fnc = allOr ? "orWhereRaw" : "whereRaw" if (this.client === SqlClient.MS_SQL) { return q[fnc]( @@ -801,7 +862,7 @@ class InternalBuilder { }) } if (filters.notEqual) { - iterate(filters.notEqual, (q, key, value) => { + iterate(filters.notEqual, BasicOperator.NOT_EQUAL, (q, key, value) => { const fnc = allOr ? "orWhereRaw" : "whereRaw" if (this.client === SqlClient.MS_SQL) { return q[fnc]( @@ -822,13 +883,13 @@ class InternalBuilder { }) } if (filters.empty) { - iterate(filters.empty, (q, key) => { + iterate(filters.empty, BasicOperator.EMPTY, (q, key) => { const fnc = allOr ? "orWhereNull" : "whereNull" return q[fnc](key) }) } if (filters.notEmpty) { - iterate(filters.notEmpty, (q, key) => { + iterate(filters.notEmpty, BasicOperator.NOT_EMPTY, (q, key) => { const fnc = allOr ? "orWhereNotNull" : "whereNotNull" return q[fnc](key) }) @@ -1224,12 +1285,10 @@ class InternalBuilder { }) : undefined if (!throughTable) { - // @ts-ignore query = query.leftJoin(toTableWithSchema, function () { for (let relationship of columns) { const from = relationship.from, to = relationship.to - // @ts-ignore this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`) } }) @@ -1240,7 +1299,6 @@ class InternalBuilder { for (let relationship of columns) { const fromPrimary = relationship.fromPrimary const from = relationship.from - // @ts-ignore this.orOn( `${fromAlias}.${fromPrimary}`, "=", @@ -1252,7 +1310,6 @@ class InternalBuilder { for (let relationship of columns) { const toPrimary = relationship.toPrimary const to = relationship.to - // @ts-ignore this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`) } }) diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte index d6f1732e64..34c317e865 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte @@ -59,6 +59,7 @@ bind:this={drawer} title="Filtering" on:drawerHide + on:drawerShow on:drawerShow={() => { // Reset to the currently available value. localFilters = Helpers.cloneDeep(value) diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte index 7b5187ffb6..e6629aa3f3 100644 --- a/packages/client/src/components/app/DataProvider.svelte +++ b/packages/client/src/components/app/DataProvider.svelte @@ -126,6 +126,9 @@ } const extendQuery = (defaultQuery, extensions) => { + if (!Object.keys(extensions).length) { + return defaultQuery + } const extended = { [LogicalOperator.AND]: { conditions: [ diff --git a/packages/pro b/packages/pro index 6139154961..1a749caba9 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 61391549614b5ac153f267633d0aaea9b07f05c5 +Subproject commit 1a749caba9c85aab2645e5d00db479eb53d3f80f diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 830acc55bf..59f67540fe 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -208,9 +208,8 @@ export async function fetchAppDefinition( export async function fetchAppPackage( ctx: UserCtx ) { - const db = context.getAppDB() const appId = context.getAppId() - let application = await db.get(DocumentType.APP_METADATA) + const application = await sdk.applications.metadata.get() const layouts = await getLayouts() let screens = await getScreens() const license = await licensing.cache.getCachedLicense() @@ -272,6 +271,7 @@ async function performAppCreate(ctx: UserCtx) { path: ctx.request.body.file?.path, } } + const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null const appId = generateDevAppID(generateAppID(tenantId)) @@ -279,7 +279,7 @@ async function performAppCreate(ctx: UserCtx) { const instance = await createInstance(appId, instanceConfig) const db = context.getAppDB() - let newApplication: App = { + const newApplication: App = { _id: DocumentType.APP_METADATA, _rev: undefined, appId, @@ -310,12 +310,18 @@ async function performAppCreate(ctx: UserCtx) { disableUserMetadata: true, skeletonLoader: true, }, + creationVersion: undefined, } + const isImport = !!instanceConfig.file + if (!isImport) { + newApplication.creationVersion = envCore.VERSION + } + + const existing = await sdk.applications.metadata.tryGet() // If we used a template or imported an app there will be an existing doc. // Fetch and migrate some metadata from the existing app. - try { - const existing: App = await db.get(DocumentType.APP_METADATA) + if (existing) { const keys: (keyof App)[] = [ "_rev", "navigation", @@ -323,6 +329,7 @@ async function performAppCreate(ctx: UserCtx) { "customTheme", "icon", "snippets", + "creationVersion", ] keys.forEach(key => { if (existing[key]) { @@ -340,14 +347,10 @@ async function performAppCreate(ctx: UserCtx) { } // Migrate navigation settings and screens if required - if (existing) { - const navigation = await migrateAppNavigation() - if (navigation) { - newApplication.navigation = navigation - } + const navigation = await migrateAppNavigation() + if (navigation) { + newApplication.navigation = navigation } - } catch (err) { - // Nothing to do } const response = await db.put(newApplication, { force: true }) @@ -489,8 +492,7 @@ export async function update( export async function updateClient(ctx: UserCtx) { // Get current app version - const db = context.getAppDB() - const application = await db.get(DocumentType.APP_METADATA) + const application = await sdk.applications.metadata.get() const currentVersion = application.version let manifest @@ -518,8 +520,7 @@ export async function updateClient(ctx: UserCtx) { export async function revertClient(ctx: UserCtx) { // Check app can be reverted - const db = context.getAppDB() - const application = await db.get(DocumentType.APP_METADATA) + const application = await sdk.applications.metadata.get() if (!application.revertableVersion) { ctx.throw(400, "There is no version to revert to") } @@ -577,7 +578,7 @@ async function destroyApp(ctx: UserCtx) { const db = dbCore.getDB(devAppId) // standard app deletion flow - const app = await db.get(DocumentType.APP_METADATA) + const app = await sdk.applications.metadata.get() const result = await db.destroy() await quotas.removeApp() await events.app.deleted(app) @@ -728,7 +729,7 @@ export async function updateAppPackage( ) { return context.doInAppContext(appId, async () => { const db = context.getAppDB() - const application = await db.get(DocumentType.APP_METADATA) + const application = await sdk.applications.metadata.get() const newAppPackage: App = { ...application, ...appPackage } if (appPackage._rev !== application._rev) { @@ -754,7 +755,7 @@ export async function setRevertableVersion( return } const db = context.getAppDB() - const app = await db.get(DocumentType.APP_METADATA) + const app = await sdk.applications.metadata.get() app.revertableVersion = ctx.request.body.revertableVersion await db.put(app) @@ -763,7 +764,7 @@ export async function setRevertableVersion( async function migrateAppNavigation() { const db = context.getAppDB() - const existing: App = await db.get(DocumentType.APP_METADATA) + const existing = await sdk.applications.metadata.get() const layouts: Layout[] = await getLayouts() const screens: Screen[] = await getScreens() diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 1969bc6537..a8229463ea 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -174,9 +174,9 @@ export class ExternalRequest { if (!opts.datasource) { if (sdk.views.isView(source)) { const table = await sdk.views.getTable(source.id) - opts.datasource = await sdk.datasources.get(table.sourceId!) + opts.datasource = await sdk.datasources.get(table.sourceId) } else { - opts.datasource = await sdk.datasources.get(source.sourceId!) + opts.datasource = await sdk.datasources.get(source.sourceId) } } @@ -205,18 +205,6 @@ export class ExternalRequest { filters: SearchFilters, table: Table ): SearchFilters { - // replace any relationship columns initially, table names and relationship column names are acceptable - const relationshipColumns = sdk.rows.filters.getRelationshipColumns(table) - filters = sdk.rows.filters.updateFilterKeys( - filters, - relationshipColumns.map(({ name, definition }) => { - const { tableName } = breakExternalTableId(definition.tableId) - return { - original: name, - updated: tableName, - } - }) - ) const primary = table.primary // if passed in array need to copy for shifting etc let idCopy: undefined | string | any[] = cloneDeep(id) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 680a7671d5..60775ce628 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -15,13 +15,16 @@ import { ExportRowsResponse, FieldType, GetRowResponse, + isRelationshipField, PatchRowRequest, PatchRowResponse, Row, RowAttachment, RowSearchParams, + SearchFilters, SearchRowRequest, SearchRowResponse, + Table, UserCtx, ValidateResponse, } from "@budibase/types" @@ -33,6 +36,7 @@ import sdk from "../../../sdk" import * as exporters from "../view/exporters" import { Format } from "../view/exporters" import { apiFileReturn } from "../../../utilities/fileSystem" +import { dataFilters } from "@budibase/shared-core" export * as views from "./views" @@ -211,12 +215,15 @@ export async function search(ctx: Ctx) { await context.ensureSnippetContext(true) - const enrichedQuery = await utils.enrichSearchContext( - { ...ctx.request.body.query }, - { - user: sdk.users.getUserContextBindings(ctx.user), - } - ) + let { query } = ctx.request.body + if (query) { + const allTables = await sdk.tables.getAllTables() + query = replaceTableNamesInFilters(tableId, query, allTables) + } + + let enrichedQuery: SearchFilters = await utils.enrichSearchContext(query, { + user: sdk.users.getUserContextBindings(ctx.user), + }) const searchParams: RowSearchParams = { ...ctx.request.body, @@ -229,6 +236,47 @@ export async function search(ctx: Ctx) { ctx.body = await sdk.rows.search(searchParams) } +function replaceTableNamesInFilters( + tableId: string, + filters: SearchFilters, + allTables: Table[] +): SearchFilters { + for (const filter of Object.values(filters)) { + for (const key of Object.keys(filter)) { + const matches = key.match(`^(?.+)\\.(?.+)`) + + const relation = matches?.groups?.["relation"] + const field = matches?.groups?.["field"] + + if (!relation || !field) { + continue + } + + const table = allTables.find(r => r._id === tableId)! + if (Object.values(table.schema).some(f => f.name === relation)) { + continue + } + + const matchedTable = allTables.find(t => t.name === relation) + const relationship = Object.values(table.schema).find( + f => isRelationshipField(f) && f.tableId === matchedTable?._id + ) + if (!relationship) { + continue + } + + const updatedField = `${relationship.name}.${field}` + if (updatedField && updatedField !== key) { + filter[updatedField] = filter[key] + delete filter[key] + } + } + } + return dataFilters.recurseLogicalOperators(filters, (f: SearchFilters) => { + return replaceTableNamesInFilters(tableId, f, allTables) + }) +} + export async function validate(ctx: Ctx) { const source = await utils.getSource(ctx) const table = await utils.getTableFromSource(source) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 180940751b..f0b27e232f 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2364,12 +2364,16 @@ describe.each([ // It also can't work for in-memory searching because the related table name // isn't available. !isInMemory && - describe("relations", () => { + describe.each([ + RelationshipType.ONE_TO_MANY, + RelationshipType.MANY_TO_ONE, + RelationshipType.MANY_TO_MANY, + ])("relations (%s)", relationshipType => { let productCategoryTable: Table, productCatRows: Row[] beforeAll(async () => { const { relatedTable, tableId } = await basicRelationshipTables( - RelationshipType.ONE_TO_MANY + relationshipType ) tableOrViewId = tableId productCategoryTable = relatedTable @@ -2466,7 +2470,10 @@ describe.each([ ], }, }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { + name: "foo", + productCat: [{ _id: productCatRows[0]._id }], + }, ]) } ) @@ -2544,7 +2551,7 @@ describe.each([ }).toContainExactly([ { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - // { name: "baz", productCat: undefined }, // TODO + { name: "baz", productCat: undefined }, ]) }) @@ -2566,7 +2573,10 @@ describe.each([ ], }, }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { + name: "foo", + productCat: [{ _id: productCatRows[0]._id }], + }, ]) } ) @@ -2590,9 +2600,15 @@ describe.each([ ], }, }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - // { name: "baz", productCat: undefined }, // TODO + { + name: "foo", + productCat: [{ _id: productCatRows[0]._id }], + }, + { + name: "bar", + productCat: [{ _id: productCatRows[1]._id }], + }, + { name: "baz", productCat: undefined }, ]) } ) @@ -2616,7 +2632,7 @@ describe.each([ }).toContainExactly([ { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - // { name: "baz", productCat: undefined }, // TODO + { name: "baz", productCat: undefined }, ]) }) }) @@ -2624,10 +2640,13 @@ describe.each([ }) isSql && - describe("big relations", () => { + describe.each([ + RelationshipType.MANY_TO_ONE, + RelationshipType.MANY_TO_MANY, + ])("big relations (%s)", relationshipType => { beforeAll(async () => { const { relatedTable, tableId } = await basicRelationshipTables( - RelationshipType.MANY_TO_ONE + relationshipType ) tableOrViewId = tableId const mainRow = await config.api.row.save(tableOrViewId, { @@ -2653,7 +2672,8 @@ describe.each([ expect(response.rows[0].productCat).toBeArrayOfSize(11) }) }) - ;(isSqs || isLucene) && + + isSql && describe("relations to same table", () => { let relatedTable: string, relatedRows: Row[] @@ -2695,6 +2715,11 @@ describe.each([ related1: [relatedRows[2]._id!], related2: [relatedRows[3]._id!], }), + config.api.row.save(tableOrViewId, { + name: "test3", + related1: [relatedRows[1]._id], + related2: [relatedRows[2]._id!], + }), ]) }) @@ -2712,42 +2737,59 @@ describe.each([ related1: [{ _id: relatedRows[2]._id }], related2: [{ _id: relatedRows[3]._id }], }, + { + name: "test3", + related1: [{ _id: relatedRows[1]._id }], + related2: [{ _id: relatedRows[2]._id }], + }, ]) }) - isSqs && - it("should be able to filter down to second row with equal", async () => { - await expectSearch({ - query: { - equal: { - ["related1.name"]: "baz", - }, + it("should be able to filter via the first relation field with equal", async () => { + await expectSearch({ + query: { + equal: { + ["related1.name"]: "baz", }, - }).toContainExactly([ - { - name: "test2", - related1: [{ _id: relatedRows[2]._id }], - }, - ]) - }) + }, + }).toContainExactly([ + { + name: "test2", + related1: [{ _id: relatedRows[2]._id }], + }, + ]) + }) - isSqs && - it("should be able to filter down to first row with not equal", async () => { - await expectSearch({ - query: { - notEqual: { - ["1:related2.name"]: "bar", - ["2:related2.name"]: "baz", - ["3:related2.name"]: "boo", - }, + it("should be able to filter via the second relation field with not equal", async () => { + await expectSearch({ + query: { + notEqual: { + ["1:related2.name"]: "foo", + ["2:related2.name"]: "baz", + ["3:related2.name"]: "boo", }, - }).toContainExactly([ - { - name: "test", - related1: [{ _id: relatedRows[0]._id }], + }, + }).toContainExactly([ + { + name: "test", + }, + ]) + }) + + it("should be able to filter on both fields", async () => { + await expectSearch({ + query: { + notEqual: { + ["related1.name"]: "foo", + ["related2.name"]: "baz", }, - ]) - }) + }, + }).toContainExactly([ + { + name: "test2", + }, + ]) + }) }) isInternal && diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 890c8c4663..739d3a4aee 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -78,8 +78,7 @@ describe("Captures of real examples", () => { bindings: ["assembling", primaryLimit, relationshipLimit], sql: expect.stringContaining( multiline( - `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" - and (COALESCE("b"."taskname" = $1, FALSE))` + `where (exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" and (COALESCE("b"."taskname" = $1, FALSE)))` ) ), }) @@ -133,6 +132,8 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [ + rangeValue.low, + rangeValue.high, rangeValue.low, rangeValue.high, equalValue, @@ -144,7 +145,7 @@ describe("Captures of real examples", () => { ], sql: expect.stringContaining( multiline( - `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))` + `where (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))) and (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."qaid" and ("c"."year" between $3 and $4))) and (exists (select 1 from "products" as "b" inner join "products_tasks" as "d" on "b"."productid" = "d"."productid" where "d"."taskid" = "a"."taskid" and (COALESCE("b"."productname" = $5, FALSE))))` ) ), }) diff --git a/packages/server/src/sdk/app/applications/index.ts b/packages/server/src/sdk/app/applications/index.ts index 04ed3b2919..f315e84896 100644 --- a/packages/server/src/sdk/app/applications/index.ts +++ b/packages/server/src/sdk/app/applications/index.ts @@ -2,10 +2,12 @@ import * as sync from "./sync" import * as utils from "./utils" import * as applications from "./applications" import * as imports from "./import" +import * as metadata from "./metadata" export default { ...sync, ...utils, ...applications, ...imports, + metadata, } diff --git a/packages/server/src/sdk/app/applications/metadata.ts b/packages/server/src/sdk/app/applications/metadata.ts new file mode 100644 index 0000000000..bbda55c085 --- /dev/null +++ b/packages/server/src/sdk/app/applications/metadata.ts @@ -0,0 +1,18 @@ +import { context, DocumentType } from "@budibase/backend-core" +import { App } from "@budibase/types" + +/** + * @deprecated the plan is to get everything using `tryGet` instead, then rename + * `tryGet` to `get`. + */ +export async function get() { + const db = context.getAppDB() + const application = await db.get(DocumentType.APP_METADATA) + return application +} + +export async function tryGet() { + const db = context.getAppDB() + const application = await db.tryGet(DocumentType.APP_METADATA) + return application +} diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts index fb077509a9..c117941419 100644 --- a/packages/server/src/sdk/app/rows/index.ts +++ b/packages/server/src/sdk/app/rows/index.ts @@ -3,14 +3,12 @@ import * as rows from "./rows" import * as search from "./search" import * as utils from "./utils" import * as external from "./external" -import * as filters from "./search/filters" import AliasTables from "./sqlAlias" export default { ...attachments, ...rows, ...search, - filters, utils, external, AliasTables, diff --git a/packages/server/src/sdk/app/rows/search/filters.ts b/packages/server/src/sdk/app/rows/search/filters.ts deleted file mode 100644 index 4049fc5352..0000000000 --- a/packages/server/src/sdk/app/rows/search/filters.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { - FieldType, - RelationshipFieldMetadata, - SearchFilters, - Table, -} from "@budibase/types" -import { isPlainObject } from "lodash" -import { dataFilters } from "@budibase/shared-core" - -export function getRelationshipColumns(table: Table): { - name: string - definition: RelationshipFieldMetadata -}[] { - // performing this with a for loop rather than an array filter improves - // type guarding, as no casts are required - const linkEntries: [string, RelationshipFieldMetadata][] = [] - for (let entry of Object.entries(table.schema)) { - if (entry[1].type === FieldType.LINK) { - const linkColumn: RelationshipFieldMetadata = entry[1] - linkEntries.push([entry[0], linkColumn]) - } - } - return linkEntries.map(entry => ({ - name: entry[0], - definition: entry[1], - })) -} - -export function getTableIDList( - tables: Table[] -): { name: string; id: string }[] { - return tables - .filter(table => table.originalName && table._id) - .map(table => ({ id: table._id!, name: table.originalName! })) -} - -export function updateFilterKeys( - filters: SearchFilters, - updates: { original: string; updated: string }[] -): SearchFilters { - const makeFilterKeyRegex = (str: string) => - new RegExp(`^${str}\\.|:${str}\\.`) - for (let filter of Object.values(filters)) { - if (!isPlainObject(filter)) { - continue - } - for (let [key, keyFilter] of Object.entries(filter)) { - if (keyFilter === "") { - delete filter[key] - } - const possibleKey = updates.find(({ original }) => - key.match(makeFilterKeyRegex(original)) - ) - if (possibleKey && possibleKey.original !== possibleKey.updated) { - // only replace the first, not replaceAll - filter[key.replace(possibleKey.original, possibleKey.updated)] = - filter[key] - delete filter[key] - } - } - } - return dataFilters.recurseLogicalOperators(filters, (f: SearchFilters) => { - return updateFilterKeys(f, updates) - }) -} diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 916e20957b..e3bbca271e 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -39,11 +39,6 @@ import AliasTables from "../../sqlAlias" import { outputProcessing } from "../../../../../utilities/rowProcessor" import pick from "lodash/pick" import { processRowCountResponse } from "../../utils" -import { - getRelationshipColumns, - getTableIDList, - updateFilterKeys, -} from "../filters" import { dataFilters, helpers, @@ -133,31 +128,7 @@ async function buildInternalFieldList( return [...new Set(fieldList)] } -function cleanupFilters( - filters: SearchFilters, - table: Table, - allTables: Table[] -) { - // get a list of all relationship columns in the table for updating - const relationshipColumns = getRelationshipColumns(table) - // get table names to ID map for relationships - const tableNameToID = getTableIDList(allTables) - // all should be applied at once - filters = updateFilterKeys( - filters, - relationshipColumns - .map(({ name, definition }) => ({ - original: name, - updated: definition.tableId, - })) - .concat( - tableNameToID.map(({ name, id }) => ({ - original: name, - updated: id, - })) - ) - ) - +function cleanupFilters(filters: SearchFilters, allTables: Table[]) { // generate a map of all possible column names (these can be duplicated across tables // the map of them will always be the same const userColumnMap: Record = {} @@ -356,7 +327,7 @@ export async function search( const relationships = buildInternalRelationships(table, allTables) const searchFilters: SearchFilters = { - ...cleanupFilters(query, table, allTables), + ...cleanupFilters(query, allTables), documentType: DocumentType.ROW, } diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 280fb5378a..49944bce85 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -90,7 +90,11 @@ export async function getExternalTable( if (!entities[tableName]) { throw new Error(`Unable to find table named "${tableName}"`) } - return processTable(entities[tableName]) + const table = await processTable(entities[tableName]) + if (!table.sourceId) { + table.sourceId = datasourceId + } + return table } export async function getTable(tableId: string): Promise { diff --git a/packages/types/src/documents/app/app.ts b/packages/types/src/documents/app/app.ts index 69bf3489e1..06fca8307c 100644 --- a/packages/types/src/documents/app/app.ts +++ b/packages/types/src/documents/app/app.ts @@ -27,6 +27,7 @@ export interface App extends Document { usedPlugins?: Plugin[] upgradableVersion?: string snippets?: Snippet[] + creationVersion?: string } export interface AppInstance { diff --git a/packages/worker/src/api/routes/global/tests/roles.spec.ts b/packages/worker/src/api/routes/global/tests/roles.spec.ts index 11de06328e..28977d8521 100644 --- a/packages/worker/src/api/routes/global/tests/roles.spec.ts +++ b/packages/worker/src/api/routes/global/tests/roles.spec.ts @@ -1,6 +1,6 @@ import { structures, TestConfiguration } from "../../../../tests" import { context, db, permissions, roles } from "@budibase/backend-core" -import { Database } from "@budibase/types" +import { App, Database } from "@budibase/types" jest.mock("@budibase/backend-core", () => { const core = jest.requireActual("@budibase/backend-core") @@ -30,6 +30,14 @@ async function addAppMetadata() { }) } +async function updateAppMetadata(update: Partial>) { + const app = await appDb.get("app_metadata") + await appDb.put({ + ...app, + ...update, + }) +} + describe("/api/global/roles", () => { const config = new TestConfiguration() @@ -69,6 +77,53 @@ describe("/api/global/roles", () => { expect(res.body[appId].roles.length).toEqual(5) expect(res.body[appId].roles.map((r: any) => r._id)).toContain(ROLE_NAME) }) + + it.each(["3.0.0", "3.0.1", "3.1.0", "3.0.0+2146.b125a7c"])( + "exclude POWER roles after v3 (%s)", + async creationVersion => { + await updateAppMetadata({ creationVersion }) + const res = await config.api.roles.get() + expect(res.body).toBeDefined() + expect(res.body[appId].roles.map((r: any) => r._id)).toEqual([ + ROLE_NAME, + roles.BUILTIN_ROLE_IDS.ADMIN, + roles.BUILTIN_ROLE_IDS.BASIC, + roles.BUILTIN_ROLE_IDS.PUBLIC, + ]) + } + ) + + it.each(["2.9.0", "1.0.0", "0.0.0", "2.32.17+2146.b125a7c"])( + "include POWER roles before v3 (%s)", + async creationVersion => { + await updateAppMetadata({ creationVersion }) + const res = await config.api.roles.get() + expect(res.body).toBeDefined() + expect(res.body[appId].roles.map((r: any) => r._id)).toEqual([ + ROLE_NAME, + roles.BUILTIN_ROLE_IDS.ADMIN, + roles.BUILTIN_ROLE_IDS.POWER, + roles.BUILTIN_ROLE_IDS.BASIC, + roles.BUILTIN_ROLE_IDS.PUBLIC, + ]) + } + ) + + it.each(["invalid", ""])( + "include POWER roles when the version is corrupted (%s)", + async creationVersion => { + await updateAppMetadata({ creationVersion }) + const res = await config.api.roles.get() + + expect(res.body[appId].roles.map((r: any) => r._id)).toEqual([ + ROLE_NAME, + roles.BUILTIN_ROLE_IDS.ADMIN, + roles.BUILTIN_ROLE_IDS.POWER, + roles.BUILTIN_ROLE_IDS.BASIC, + roles.BUILTIN_ROLE_IDS.PUBLIC, + ]) + } + ) }) describe("GET api/global/roles/:appId", () => {