diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 7d3a9f18f5..4cb0a9c731 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -223,6 +223,8 @@ const environment = { BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL, BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD, OPENAI_API_KEY: process.env.OPENAI_API_KEY, + MIN_VERSION_WITHOUT_POWER_ROLE: + process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0", } export function setEnv(newEnvVars: Partial): () => void { diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index df7f41e6ce..76c4f08fbe 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -19,6 +19,7 @@ import { import cloneDeep from "lodash/fp/cloneDeep" import { RoleColor, helpers } from "@budibase/shared-core" import { uniqBy } from "lodash" +import { default as env } from "../environment" export const BUILTIN_ROLE_IDS = { ADMIN: "ADMIN", @@ -545,7 +546,10 @@ async function shouldIncludePowerRole(db: Database) { return true } - const isGreaterThan3x = semver.gte(creationVersion, "3.0.0") + const isGreaterThan3x = semver.gte( + creationVersion, + env.MIN_VERSION_WITHOUT_POWER_ROLE + ) return !isGreaterThan3x } diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 2b75752aec..88953bbf99 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -13,6 +13,7 @@ import SqlTableQueryBuilder from "./sqlTable" import { Aggregation, AnySearchFilter, + ArrayFilter, ArrayOperator, BasicOperator, BBReferenceFieldMetadata, @@ -98,6 +99,23 @@ function isSqs(table: Table): boolean { ) } +function escapeQuotes(value: string, quoteChar = '"'): string { + return value.replace(new RegExp(quoteChar, "g"), `${quoteChar}${quoteChar}`) +} + +function wrap(value: string, quoteChar = '"'): string { + return `${quoteChar}${escapeQuotes(value, quoteChar)}${quoteChar}` +} + +function stringifyArray(value: any[], quoteStyle = '"'): string { + for (let i in value) { + if (typeof value[i] === "string") { + value[i] = wrap(value[i], quoteStyle) + } + } + return `[${value.join(",")}]` +} + const allowEmptyRelationships: Record = { [BasicOperator.EQUAL]: false, [BasicOperator.NOT_EQUAL]: true, @@ -152,30 +170,30 @@ class InternalBuilder { return this.query.meta.table } + get knexClient(): Knex.Client { + return this.knex.client as Knex.Client + } + getFieldSchema(key: string): FieldSchema | undefined { const { column } = this.splitter.run(key) return this.table.schema[column] } - private quoteChars(): [string, string] { - switch (this.client) { - case SqlClient.ORACLE: - case SqlClient.POSTGRES: - return ['"', '"'] - case SqlClient.MS_SQL: - return ["[", "]"] - case SqlClient.MARIADB: - case SqlClient.MY_SQL: - case SqlClient.SQL_LITE: - return ["`", "`"] - } + private supportsILike(): boolean { + return !( + this.client === SqlClient.ORACLE || this.client === SqlClient.SQL_LITE + ) } - // Takes a string like foo and returns a quoted string like [foo] for SQL Server - // and "foo" for Postgres. + private quoteChars(): [string, string] { + const wrapped = this.knexClient.wrapIdentifier("foo", {}) + return [wrapped[0], wrapped[wrapped.length - 1]] + } + + // Takes a string like foo and returns a quoted string like [foo] for SQL + // Server and "foo" for Postgres. private quote(str: string): string { - const [start, end] = this.quoteChars() - return `${start}${str}${end}` + return this.knexClient.wrapIdentifier(str, {}) } private isQuoted(key: string): boolean { @@ -193,6 +211,30 @@ class InternalBuilder { return key.map(part => this.quote(part)).join(".") } + private quotedValue(value: string): string { + const formatter = this.knexClient.formatter(this.knexClient.queryBuilder()) + return formatter.wrap(value, false) + } + + private rawQuotedValue(value: string): Knex.Raw { + return this.knex.raw(this.quotedValue(value)) + } + + // Unfortuantely we cannot rely on knex's identifier escaping because it trims + // the identifier string before escaping it, which breaks cases for us where + // columns that start or end with a space aren't referenced correctly anymore. + // + // So whenever you're using an identifier binding in knex, e.g. knex.raw("?? + // as ?", ["foo", "bar"]), you need to make sure you call this: + // + // knex.raw("?? as ?", [this.quotedIdentifier("foo"), "bar"]) + // + // Issue we filed against knex about this: + // https://github.com/knex/knex/issues/6143 + private rawQuotedIdentifier(key: string): Knex.Raw { + return this.knex.raw(this.quotedIdentifier(key)) + } + // Turns an identifier like a.b.c or `a`.`b`.`c` into ["a", "b", "c"] private splitIdentifier(key: string): string[] { const [start, end] = this.quoteChars() @@ -236,7 +278,7 @@ class InternalBuilder { const alias = this.getTableName(endpoint.entityId) const schema = meta.table.schema if (!this.isFullSelectStatementRequired()) { - return [this.knex.raw(`${this.quote(alias)}.*`)] + return [this.knex.raw("??", [`${alias}.*`])] } // get just the fields for this table return resource.fields @@ -258,30 +300,40 @@ class InternalBuilder { const columnSchema = schema[column] if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) { - return this.knex.raw( - `${this.quotedIdentifier( - [table, column].join(".") - )}::money::numeric as ${this.quote(field)}` - ) + // TODO: figure out how to express this safely without string + // interpolation. + return this.knex.raw(`??::money::numeric as "${field}"`, [ + this.rawQuotedIdentifier([table, column].join(".")), + field, + ]) } if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) { // Time gets returned as timestamp from mssql, not matching the expected // HH:mm format - return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) + + // TODO: figure out how to express this safely without string + // interpolation. + return this.knex.raw(`CONVERT(varchar, ??, 108) as "${field}"`, [ + this.rawQuotedIdentifier(field), + ]) } - const quoted = table - ? `${this.quote(table)}.${this.quote(column)}` - : this.quote(field) - return this.knex.raw(quoted) + if (table) { + return this.rawQuotedIdentifier(`${table}.${column}`) + } else { + return this.rawQuotedIdentifier(field) + } }) } // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses, // so when we use them we need to wrap them in to_char(). This function // converts a field name to the appropriate identifier. - private convertClobs(field: string, opts?: { forSelect?: boolean }): string { + private convertClobs( + field: string, + opts?: { forSelect?: boolean } + ): Knex.Raw { if (this.client !== SqlClient.ORACLE) { throw new Error( "you've called convertClobs on a DB that's not Oracle, this is a mistake" @@ -290,7 +342,7 @@ class InternalBuilder { const parts = this.splitIdentifier(field) const col = parts.pop()! const schema = this.table.schema[col] - let identifier = this.quotedIdentifier(field) + let identifier = this.rawQuotedIdentifier(field) if ( schema.type === FieldType.STRING || @@ -301,9 +353,12 @@ class InternalBuilder { schema.type === FieldType.BARCODEQR ) { if (opts?.forSelect) { - identifier = `to_char(${identifier}) as ${this.quotedIdentifier(col)}` + identifier = this.knex.raw("to_char(??) as ??", [ + identifier, + this.rawQuotedIdentifier(col), + ]) } else { - identifier = `to_char(${identifier})` + identifier = this.knex.raw("to_char(??)", [identifier]) } } return identifier @@ -427,7 +482,6 @@ class InternalBuilder { filterKey: string, whereCb: (filterKey: string, query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { - const mainKnex = this.knex const { relationships, endpoint, tableAliases: aliases } = this.query const tableName = endpoint.entityId const fromAlias = aliases?.[tableName] || tableName @@ -449,8 +503,8 @@ class InternalBuilder { relationship.to && relationship.tableName ) { - const joinTable = mainKnex - .select(mainKnex.raw(1)) + const joinTable = this.knex + .select(this.knex.raw(1)) .from({ [toAlias]: relatedTableName }) let subQuery = joinTable.clone() const manyToMany = validateManyToMany(relationship) @@ -485,9 +539,7 @@ class InternalBuilder { .where( `${throughAlias}.${manyToMany.from}`, "=", - mainKnex.raw( - this.quotedIdentifier(`${fromAlias}.${manyToMany.fromPrimary}`) - ) + this.rawQuotedIdentifier(`${fromAlias}.${manyToMany.fromPrimary}`) ) // in SQS the same junction table is used for different many-to-many relationships between the // two same tables, this is needed to avoid rows ending up in all columns @@ -516,7 +568,7 @@ class InternalBuilder { subQuery = subQuery.where( toKey, "=", - mainKnex.raw(this.quotedIdentifier(foreignKey)) + this.rawQuotedIdentifier(foreignKey) ) query = query.where(q => { @@ -546,7 +598,7 @@ class InternalBuilder { filters = this.parseFilters({ ...filters }) const aliases = this.query.tableAliases // if all or specified in filters, then everything is an or - const allOr = filters.allOr + const shouldOr = filters.allOr const isSqlite = this.client === SqlClient.SQL_LITE const tableName = isSqlite ? this.table._id! : this.table.name @@ -610,7 +662,7 @@ class InternalBuilder { value ) } else if (shouldProcessRelationship) { - if (allOr) { + if (shouldOr) { query = query.or } query = builder.addRelationshipForFilter( @@ -626,85 +678,102 @@ class InternalBuilder { } const like = (q: Knex.QueryBuilder, key: string, value: any) => { - const fuzzyOr = filters?.fuzzyOr - const fnc = fuzzyOr || allOr ? "orWhere" : "where" - // postgres supports ilike, nothing else does - if (this.client === SqlClient.POSTGRES) { - return q[fnc](key, "ilike", `%${value}%`) - } else { - const rawFnc = `${fnc}Raw` - // @ts-ignore - return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ + if (filters?.fuzzyOr || shouldOr) { + q = q.or + } + if ( + this.client === SqlClient.ORACLE || + this.client === SqlClient.SQL_LITE + ) { + return q.whereRaw(`LOWER(??) LIKE ?`, [ + this.rawQuotedIdentifier(key), `%${value.toLowerCase()}%`, ]) } + return q.whereILike( + // @ts-expect-error knex types are wrong, raw is fine here + this.rawQuotedIdentifier(key), + this.knex.raw("?", [`%${value}%`]) + ) } - const contains = (mode: AnySearchFilter, any: boolean = false) => { - const rawFnc = allOr ? "orWhereRaw" : "whereRaw" - const not = mode === filters?.notContains ? "NOT " : "" - function stringifyArray(value: Array, quoteStyle = '"'): string { - for (let i in value) { - if (typeof value[i] === "string") { - value[i] = `${quoteStyle}${value[i]}${quoteStyle}` - } + const contains = (mode: ArrayFilter, any = false) => { + function addModifiers(q: Knex.QueryBuilder) { + if (shouldOr || mode === filters?.containsAny) { + q = q.or } - return `[${value.join(",")}]` + if (mode === filters?.notContains) { + q = q.not + } + return q } + if (this.client === SqlClient.POSTGRES) { iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { - const wrap = any ? "" : "'" - const op = any ? "\\?| array" : "@>" - const fieldNames = key.split(/\./g) - const table = fieldNames[0] - const col = fieldNames[1] - return q[rawFnc]( - `${not}COALESCE("${table}"."${col}"::jsonb ${op} ${wrap}${stringifyArray( - value, - any ? "'" : '"' - )}${wrap}, FALSE)` - ) + q = addModifiers(q) + if (any) { + return q.whereRaw(`COALESCE(??::jsonb \\?| array??, FALSE)`, [ + this.rawQuotedIdentifier(key), + this.knex.raw(stringifyArray(value, "'")), + ]) + } else { + return q.whereRaw(`COALESCE(??::jsonb @> '??', FALSE)`, [ + this.rawQuotedIdentifier(key), + this.knex.raw(stringifyArray(value)), + ]) + } }) } else if ( this.client === SqlClient.MY_SQL || this.client === SqlClient.MARIADB ) { - const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { - return q[rawFnc]( - `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray( - value - )}'), FALSE)` - ) + return addModifiers(q).whereRaw(`COALESCE(?(??, ?), FALSE)`, [ + this.knex.raw(any ? "JSON_OVERLAPS" : "JSON_CONTAINS"), + this.rawQuotedIdentifier(key), + this.knex.raw(wrap(stringifyArray(value))), + ]) }) } else { - const andOr = mode === filters?.containsAny ? " OR " : " AND " iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { - let statement = "" - const identifier = this.quotedIdentifier(key) - for (let i in value) { - if (typeof value[i] === "string") { - value[i] = `%"${value[i].toLowerCase()}"%` - } else { - value[i] = `%${value[i]}%` - } - statement += `${ - statement ? andOr : "" - }COALESCE(LOWER(${identifier}), '') LIKE ?` - } - - if (statement === "") { + if (value.length === 0) { return q } - if (not) { - return q[rawFnc]( - `(NOT (${statement}) OR ${identifier} IS NULL)`, - value - ) - } else { - return q[rawFnc](statement, value) - } + q = q.where(subQuery => { + if (mode === filters?.notContains) { + subQuery = subQuery.not + } + + subQuery = subQuery.where(subSubQuery => { + for (const elem of value) { + if (mode === filters?.containsAny) { + subSubQuery = subSubQuery.or + } else { + subSubQuery = subSubQuery.and + } + + const lower = + typeof elem === "string" ? `"${elem.toLowerCase()}"` : elem + + subSubQuery = subSubQuery.whereLike( + // @ts-expect-error knex types are wrong, raw is fine here + this.knex.raw(`COALESCE(LOWER(??), '')`, [ + this.rawQuotedIdentifier(key), + ]), + `%${lower}%` + ) + } + }) + if (mode === filters?.notContains) { + subQuery = subQuery.or.whereNull( + // @ts-expect-error knex types are wrong, raw is fine here + this.rawQuotedIdentifier(key) + ) + } + return subQuery + }) + return q }) } } @@ -730,45 +799,46 @@ class InternalBuilder { } if (filters.oneOf) { - const fnc = allOr ? "orWhereIn" : "whereIn" iterate( filters.oneOf, ArrayOperator.ONE_OF, (q, key: string, array) => { - if (this.client === SqlClient.ORACLE) { - key = this.convertClobs(key) - array = Array.isArray(array) ? array : [array] - const binding = new Array(array.length).fill("?").join(",") - return q.whereRaw(`${key} IN (${binding})`, array) - } else { - return q[fnc](key, Array.isArray(array) ? array : [array]) + if (shouldOr) { + q = q.or } + if (this.client === SqlClient.ORACLE) { + // @ts-ignore + key = this.convertClobs(key) + } + return q.whereIn(key, Array.isArray(array) ? array : [array]) }, (q, key: string[], array) => { - if (this.client === SqlClient.ORACLE) { - const keyStr = `(${key.map(k => this.convertClobs(k)).join(",")})` - const binding = `(${array - .map((a: any) => `(${new Array(a.length).fill("?").join(",")})`) - .join(",")})` - return q.whereRaw(`${keyStr} IN ${binding}`, array.flat()) - } else { - return q[fnc](key, Array.isArray(array) ? array : [array]) + if (shouldOr) { + q = q.or } + if (this.client === SqlClient.ORACLE) { + // @ts-ignore + key = key.map(k => this.convertClobs(k)) + } + return q.whereIn(key, Array.isArray(array) ? array : [array]) } ) } if (filters.string) { iterate(filters.string, BasicOperator.STRING, (q, key, value) => { - const fnc = allOr ? "orWhere" : "where" - // postgres supports ilike, nothing else does - if (this.client === SqlClient.POSTGRES) { - return q[fnc](key, "ilike", `${value}%`) - } else { - const rawFnc = `${fnc}Raw` - // @ts-ignore - return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ + if (shouldOr) { + q = q.or + } + if ( + this.client === SqlClient.ORACLE || + this.client === SqlClient.SQL_LITE + ) { + return q.whereRaw(`LOWER(??) LIKE ?`, [ + this.rawQuotedIdentifier(key), `${value.toLowerCase()}%`, ]) + } else { + return q.whereILike(key, `${value}%`) } }) } @@ -795,67 +865,59 @@ class InternalBuilder { const schema = this.getFieldSchema(key) + let rawKey: string | Knex.Raw = key + let high = value.high + let low = value.low + if (this.client === SqlClient.ORACLE) { - // @ts-ignore - key = this.knex.raw(this.convertClobs(key)) + rawKey = this.convertClobs(key) + } else if ( + this.client === SqlClient.SQL_LITE && + schema?.type === FieldType.BIGINT + ) { + rawKey = this.knex.raw("CAST(?? AS INTEGER)", [ + this.rawQuotedIdentifier(key), + ]) + high = this.knex.raw("CAST(? AS INTEGER)", [value.high]) + low = this.knex.raw("CAST(? AS INTEGER)", [value.low]) + } + + if (shouldOr) { + q = q.or } if (lowValid && highValid) { - if ( - schema?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - return q.whereRaw( - `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, - [value.low, value.high] - ) - } else { - const fnc = allOr ? "orWhereBetween" : "whereBetween" - return q[fnc](key, [value.low, value.high]) - } + // @ts-expect-error knex types are wrong, raw is fine here + return q.whereBetween(rawKey, [low, high]) } else if (lowValid) { - if ( - schema?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - return q.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [ - value.low, - ]) - } else { - const fnc = allOr ? "orWhere" : "where" - return q[fnc](key, ">=", value.low) - } + // @ts-expect-error knex types are wrong, raw is fine here + return q.where(rawKey, ">=", low) } else if (highValid) { - if ( - schema?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - return q.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [ - value.high, - ]) - } else { - const fnc = allOr ? "orWhere" : "where" - return q[fnc](key, "<=", value.high) - } + // @ts-expect-error knex types are wrong, raw is fine here + return q.where(rawKey, "<=", high) } return q }) } if (filters.equal) { iterate(filters.equal, BasicOperator.EQUAL, (q, key, value) => { - const fnc = allOr ? "orWhereRaw" : "whereRaw" + if (shouldOr) { + q = q.or + } if (this.client === SqlClient.MS_SQL) { - return q[fnc]( - `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 1`, - [value] - ) - } else if (this.client === SqlClient.ORACLE) { - const identifier = this.convertClobs(key) - return q[fnc](`(${identifier} IS NOT NULL AND ${identifier} = ?)`, [ + return q.whereRaw(`CASE WHEN ?? = ? THEN 1 ELSE 0 END = 1`, [ + this.rawQuotedIdentifier(key), value, ]) + } else if (this.client === SqlClient.ORACLE) { + const identifier = this.convertClobs(key) + return q.where(subq => + // @ts-expect-error knex types are wrong, raw is fine here + subq.whereNotNull(identifier).andWhere(identifier, value) + ) } else { - return q[fnc](`COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`, [ + return q.whereRaw(`COALESCE(?? = ?, FALSE)`, [ + this.rawQuotedIdentifier(key), value, ]) } @@ -863,20 +925,30 @@ class InternalBuilder { } if (filters.notEqual) { iterate(filters.notEqual, BasicOperator.NOT_EQUAL, (q, key, value) => { - const fnc = allOr ? "orWhereRaw" : "whereRaw" + if (shouldOr) { + q = q.or + } if (this.client === SqlClient.MS_SQL) { - return q[fnc]( - `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 0`, - [value] - ) + return q.whereRaw(`CASE WHEN ?? = ? THEN 1 ELSE 0 END = 0`, [ + this.rawQuotedIdentifier(key), + value, + ]) } else if (this.client === SqlClient.ORACLE) { const identifier = this.convertClobs(key) - return q[fnc]( - `(${identifier} IS NOT NULL AND ${identifier} != ?) OR ${identifier} IS NULL`, - [value] + return ( + q + .where(subq => + subq.not + // @ts-expect-error knex types are wrong, raw is fine here + .whereNull(identifier) + .and.where(identifier, "!=", value) + ) + // @ts-expect-error knex types are wrong, raw is fine here + .or.whereNull(identifier) ) } else { - return q[fnc](`COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`, [ + return q.whereRaw(`COALESCE(?? != ?, TRUE)`, [ + this.rawQuotedIdentifier(key), value, ]) } @@ -884,14 +956,18 @@ class InternalBuilder { } if (filters.empty) { iterate(filters.empty, BasicOperator.EMPTY, (q, key) => { - const fnc = allOr ? "orWhereNull" : "whereNull" - return q[fnc](key) + if (shouldOr) { + q = q.or + } + return q.whereNull(key) }) } if (filters.notEmpty) { iterate(filters.notEmpty, BasicOperator.NOT_EMPTY, (q, key) => { - const fnc = allOr ? "orWhereNotNull" : "whereNotNull" - return q[fnc](key) + if (shouldOr) { + q = q.or + } + return q.whereNotNull(key) }) } if (filters.contains) { @@ -976,9 +1052,7 @@ class InternalBuilder { const selectFields = qualifiedFields.map(field => this.convertClobs(field, { forSelect: true }) ) - query = query - .groupByRaw(groupByFields.join(", ")) - .select(this.knex.raw(selectFields.join(", "))) + query = query.groupBy(groupByFields).select(selectFields) } else { query = query.groupBy(qualifiedFields).select(qualifiedFields) } @@ -990,11 +1064,10 @@ class InternalBuilder { if (this.client === SqlClient.ORACLE) { const field = this.convertClobs(`${tableName}.${aggregation.field}`) query = query.select( - this.knex.raw( - `COUNT(DISTINCT ${field}) as ${this.quotedIdentifier( - aggregation.name - )}` - ) + this.knex.raw(`COUNT(DISTINCT ??) as ??`, [ + field, + aggregation.name, + ]) ) } else { query = query.countDistinct( @@ -1059,9 +1132,11 @@ class InternalBuilder { } else { let composite = `${aliased}.${key}` if (this.client === SqlClient.ORACLE) { - query = query.orderByRaw( - `${this.convertClobs(composite)} ${direction} nulls ${nulls}` - ) + query = query.orderByRaw(`?? ?? nulls ??`, [ + this.convertClobs(composite), + this.knex.raw(direction), + this.knex.raw(nulls as string), + ]) } else { query = query.orderBy(composite, direction, nulls) } @@ -1091,17 +1166,22 @@ class InternalBuilder { private buildJsonField(field: string): string { const parts = field.split(".") - let tableField: string, unaliased: string + let unaliased: string + + let tableField: string if (parts.length > 1) { const alias = parts.shift()! unaliased = parts.join(".") - tableField = `${this.quote(alias)}.${this.quote(unaliased)}` + tableField = `${alias}.${unaliased}` } else { unaliased = parts.join(".") - tableField = this.quote(unaliased) + tableField = unaliased } + const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," - return `'${unaliased}'${separator}${tableField}` + return this.knex + .raw(`?${separator}??`, [unaliased, this.rawQuotedIdentifier(tableField)]) + .toString() } maxFunctionParameters() { @@ -1197,13 +1277,13 @@ class InternalBuilder { subQuery = subQuery.where( correlatedTo, "=", - knex.raw(this.quotedIdentifier(correlatedFrom)) + this.rawQuotedIdentifier(correlatedFrom) ) - const standardWrap = (select: string): Knex.QueryBuilder => { + const standardWrap = (select: Knex.Raw): Knex.QueryBuilder => { subQuery = subQuery.select(`${toAlias}.*`).limit(getRelationshipLimit()) // @ts-ignore - the from alias syntax isn't in Knex typing - return knex.select(knex.raw(select)).from({ + return knex.select(select).from({ [toAlias]: subQuery, }) } @@ -1213,12 +1293,12 @@ class InternalBuilder { // need to check the junction table document is to the right column, this is just for SQS subQuery = this.addJoinFieldCheck(subQuery, relationship) wrapperQuery = standardWrap( - `json_group_array(json_object(${fieldList}))` + this.knex.raw(`json_group_array(json_object(${fieldList}))`) ) break case SqlClient.POSTGRES: wrapperQuery = standardWrap( - `json_agg(json_build_object(${fieldList}))` + this.knex.raw(`json_agg(json_build_object(${fieldList}))`) ) break case SqlClient.MARIADB: @@ -1232,21 +1312,25 @@ class InternalBuilder { case SqlClient.MY_SQL: case SqlClient.ORACLE: wrapperQuery = standardWrap( - `json_arrayagg(json_object(${fieldList}))` + this.knex.raw(`json_arrayagg(json_object(${fieldList}))`) ) break - case SqlClient.MS_SQL: + case SqlClient.MS_SQL: { + const comparatorQuery = knex + .select(`${fromAlias}.*`) + // @ts-ignore - from alias syntax not TS supported + .from({ + [fromAlias]: subQuery + .select(`${toAlias}.*`) + .limit(getRelationshipLimit()), + }) + wrapperQuery = knex.raw( - `(SELECT ${this.quote(toAlias)} = (${knex - .select(`${fromAlias}.*`) - // @ts-ignore - from alias syntax not TS supported - .from({ - [fromAlias]: subQuery - .select(`${toAlias}.*`) - .limit(getRelationshipLimit()), - })} FOR JSON PATH))` + `(SELECT ?? = (${comparatorQuery} FOR JSON PATH))`, + [this.rawQuotedIdentifier(toAlias)] ) break + } default: throw new Error(`JSON relationships not implement for ${sqlClient}`) } diff --git a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte index 2826d8d986..85227b9140 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/ManageAccessButton.svelte @@ -34,7 +34,9 @@ $: buttonLabel = readableRole ? `Access: ${readableRole}` : "Access" $: highlight = roleMismatch || selectedRole === Roles.PUBLIC - $: builtInRoles = builtins.map(roleId => $roles.find(x => x._id === roleId)) + $: builtInRoles = builtins + .map(roleId => $roles.find(x => x._id === roleId)) + .filter(r => !!r) $: customRoles = $roles .filter(x => !builtins.includes(x._id)) .slice() @@ -100,6 +102,9 @@ } const changePermission = async role => { + if (role === selectedRole) { + return + } try { await permissionsStore.save({ level: "read", diff --git a/packages/builder/src/components/deploy/AppActions.svelte b/packages/builder/src/components/deploy/AppActions.svelte index bb950983a6..9ba46832f4 100644 --- a/packages/builder/src/components/deploy/AppActions.svelte +++ b/packages/builder/src/components/deploy/AppActions.svelte @@ -25,6 +25,7 @@ appStore, deploymentStore, sortedScreens, + appPublished, } from "stores/builder" import TourWrap from "components/portal/onboarding/TourWrap.svelte" import { TOUR_STEP_KEYS } from "components/portal/onboarding/tours.js" @@ -45,11 +46,6 @@ $: filteredApps = $appsStore.apps.filter(app => app.devId === application) $: selectedApp = filteredApps?.length ? filteredApps[0] : null - $: latestDeployments = $deploymentStore - .filter(deployment => deployment.status === "SUCCESS") - .sort((a, b) => a.updatedAt > b.updatedAt) - $: isPublished = - selectedApp?.status === "published" && latestDeployments?.length > 0 $: updateAvailable = $appStore.upgradableVersion && $appStore.version && @@ -117,7 +113,7 @@ } const confirmUnpublishApp = async () => { - if (!application || !isPublished) { + if (!application || !$appPublished) { //confirm the app has loaded. return } @@ -204,7 +200,7 @@ >
- + Publish @@ -219,7 +215,7 @@ { @@ -236,13 +232,13 @@ class="app-link" on:click={() => { appActionPopover.hide() - if (isPublished) { + if ($appPublished) { viewApp() } }} > {$appStore.url} - {#if isPublished} + {#if $appPublished} {/if} @@ -250,7 +246,7 @@ - {#if isPublished} + {#if $appPublished} {lastDeployed} @@ -279,7 +275,7 @@
- {#if isPublished} + {#if $appPublished}
+ {#if !$appPublished} +
+ +
+ {/if} + {#if promptInvite && !userOnboardResponse}
@@ -623,7 +635,7 @@ {/if} {#if !promptInvite} - + {#if filteredInvites?.length}
@@ -926,7 +938,7 @@ .auth-entity, .auth-entity-header { display: grid; - grid-template-columns: 1fr 180px; + grid-template-columns: 1fr 220px; align-items: center; gap: var(--spacing-xl); } @@ -957,7 +969,7 @@ overflow-y: auto; overflow-x: hidden; position: absolute; - width: 440px; + width: 480px; right: 0; height: 100%; box-shadow: 0 0 40px 10px rgba(0, 0, 0, 0.1); @@ -1034,4 +1046,7 @@ gap: var(--spacing-xl); padding: var(--spacing-xl) 0; } + .alert { + padding: 0 var(--spacing-xl); + } diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte index 88ddf6f9a5..93044cdb9a 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte @@ -57,13 +57,7 @@ } .title, .icon { - color: var(--spectrum-global-color-gray-600); - } - .info { - background-color: var(--background-alt); - padding: var(--spacing-m) var(--spacing-l) var(--spacing-m) var(--spacing-l); - border-radius: var(--border-radius-s); - font-size: 13px; + color: var(--spectrum-global-color-gray-700); } .quiet { background: none; diff --git a/packages/builder/src/stores/builder/index.js b/packages/builder/src/stores/builder/index.js index 158cd29973..08d87bebf5 100644 --- a/packages/builder/src/stores/builder/index.js +++ b/packages/builder/src/stores/builder/index.js @@ -30,6 +30,7 @@ import { queries } from "./queries" import { flags } from "./flags" import { rowActions } from "./rowActions" import componentTreeNodesStore from "./componentTreeNodes" +import { appPublished } from "./published" export { componentTreeNodesStore, @@ -65,6 +66,7 @@ export { hoverStore, snippets, rowActions, + appPublished, } export const reset = () => { diff --git a/packages/builder/src/stores/builder/published.js b/packages/builder/src/stores/builder/published.js new file mode 100644 index 0000000000..6ae9392f40 --- /dev/null +++ b/packages/builder/src/stores/builder/published.js @@ -0,0 +1,13 @@ +import { appStore } from "./app" +import { appsStore } from "stores/portal/apps" +import { deploymentStore } from "./deployments" +import { derived } from "svelte/store" + +export const appPublished = derived( + [appStore, appsStore, deploymentStore], + ([$appStore, $appsStore, $deploymentStore]) => { + const app = $appsStore.apps.find(app => app.devId === $appStore.appId) + const deployments = $deploymentStore.filter(x => x.status === "SUCCESS") + return app?.status === "published" && deployments.length > 0 + } +) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index baab008da8..794e2dfddd 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -267,6 +267,8 @@ export async function destroy(ctx: UserCtx) { const datasource = await sdk.datasources.get(datasourceId) // Delete all queries for the datasource + await sdk.rowActions.deleteAllForDatasource(datasourceId) + if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) { await destroyInternalTablesBySourceId(datasourceId) } else { diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 4d40476b7c..fe29d46700 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -11,6 +11,7 @@ import { DeleteRow, DeleteRowRequest, DeleteRows, + EventType, ExportRowsRequest, ExportRowsResponse, FieldType, @@ -64,15 +65,15 @@ export async function patch( ctx.throw(404, "Row not found") } ctx.status = 200 - ctx.eventEmitter && - ctx.eventEmitter.emitRow({ - eventName: `row:update`, - appId, - row, - table, - oldRow, - user: sdk.users.getUserContextBindings(ctx.user), - }) + + ctx.eventEmitter?.emitRow({ + eventName: EventType.ROW_UPDATE, + appId, + row, + table, + oldRow, + user: sdk.users.getUserContextBindings(ctx.user), + }) ctx.message = `${table.name} updated successfully.` ctx.body = row gridSocket?.emitRowUpdate(ctx, row) @@ -103,14 +104,14 @@ export const save = async (ctx: UserCtx) => { sdk.rows.save(sourceId, ctx.request.body, ctx.user?._id) ) ctx.status = 200 - ctx.eventEmitter && - ctx.eventEmitter.emitRow({ - eventName: `row:save`, - appId, - row, - table, - user: sdk.users.getUserContextBindings(ctx.user), - }) + + ctx.eventEmitter?.emitRow({ + eventName: EventType.ROW_SAVE, + appId, + row, + table, + user: sdk.users.getUserContextBindings(ctx.user), + }) ctx.message = `${table.name} saved successfully` // prefer squashed for response ctx.body = row || squashed @@ -182,13 +183,12 @@ async function deleteRows(ctx: UserCtx) { } for (let row of rows) { - ctx.eventEmitter && - ctx.eventEmitter.emitRow({ - eventName: `row:delete`, - appId, - row, - user: sdk.users.getUserContextBindings(ctx.user), - }) + ctx.eventEmitter?.emitRow({ + eventName: EventType.ROW_DELETE, + appId, + row, + user: sdk.users.getUserContextBindings(ctx.user), + }) gridSocket?.emitRowDeletion(ctx, row) } return rows @@ -203,13 +203,12 @@ async function deleteRow(ctx: UserCtx) { await quotas.removeRow() } - ctx.eventEmitter && - ctx.eventEmitter.emitRow({ - eventName: `row:delete`, - appId, - row: resp.row, - user: sdk.users.getUserContextBindings(ctx.user), - }) + ctx.eventEmitter?.emitRow({ + eventName: EventType.ROW_DELETE, + appId, + row: resp.row, + user: sdk.users.getUserContextBindings(ctx.user), + }) gridSocket?.emitRowDeletion(ctx, resp.row) return resp diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 2f2f93bffe..77c1f3923a 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -16,6 +16,7 @@ import { BulkImportResponse, CsvToJsonRequest, CsvToJsonResponse, + EventType, FetchTablesResponse, FieldType, MigrateRequest, @@ -129,8 +130,7 @@ export async function save(ctx: UserCtx) { } ctx.status = 200 ctx.message = `Table ${table.name} saved successfully.` - ctx.eventEmitter && - ctx.eventEmitter.emitTable(`table:save`, appId, { ...savedTable }) + ctx.eventEmitter?.emitTable(EventType.TABLE_SAVE, appId, { ...savedTable }) ctx.body = savedTable savedTable = await processTable(savedTable) @@ -143,8 +143,8 @@ export async function destroy(ctx: UserCtx) { await sdk.rowActions.deleteAll(tableId) const deletedTable = await pickApi({ tableId }).destroy(ctx) await events.table.deleted(deletedTable) - ctx.eventEmitter && - ctx.eventEmitter.emitTable(`table:delete`, appId, deletedTable) + + ctx.eventEmitter?.emitTable(EventType.TABLE_DELETE, appId, deletedTable) ctx.status = 200 ctx.table = deletedTable ctx.body = { message: `Table ${tableId} deleted.` } diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts index 5cec83fe23..14a1812195 100644 --- a/packages/server/src/api/routes/tests/rowAction.spec.ts +++ b/packages/server/src/api/routes/tests/rowAction.spec.ts @@ -1,17 +1,24 @@ import _ from "lodash" import tk from "timekeeper" +import { + context, + DEFAULT_BB_DATASOURCE_ID, + roles, +} from "@budibase/backend-core" +import { automations } from "@budibase/pro" import { CreateRowActionRequest, DocumentType, PermissionLevel, RowActionResponse, + TableRowActions, } from "@budibase/types" import * as setup from "./utilities" import { generator, mocks } from "@budibase/backend-core/tests" import { Expectations } from "../../../tests/utilities/api/base" -import { roles } from "@budibase/backend-core" -import { automations } from "@budibase/pro" +import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" +import { generateRowActionsID } from "../../../db/utils" const expectAutomationId = () => expect.stringMatching(`^${DocumentType.AUTOMATION}_.+`) @@ -958,9 +965,74 @@ describe("/rowsActions", () => { // document was not being cleaned up. This meant there existed code paths // that would find it and try to reference the tables within it, resulting // in errors. - await config.api.automation.fetchEnriched({ + await config.api.automation.fetch({ status: 200, }) }) + + it.each([ + [ + "internal", + async () => { + await config.newTenant() + await config.api.application.addSampleData(config.getAppId()) + const tables = await config.api.table.fetch() + const table = tables.find( + t => t.sourceId === DEFAULT_BB_DATASOURCE_ID + )! + return table + }, + ], + [ + "external", + async () => { + await config.newTenant() + const ds = await config.createDatasource({ + datasource: await getDatasource(DatabaseName.POSTGRES), + }) + const table = await config.api.table.save( + setup.structures.tableForDatasource(ds) + ) + return table + }, + ], + ])( + "should delete all the row actions (and automations) for its tables when a datasource is deleted", + async (_, getTable) => { + async function getRowActionsFromDb(tableId: string) { + return await context.doInAppContext(config.getAppId(), async () => { + const db = context.getAppDB() + const tableDoc = await db.tryGet( + generateRowActionsID(tableId) + ) + return tableDoc + }) + } + + const table = await getTable() + const tableId = table._id! + + await config.api.rowAction.save(tableId, { + name: generator.guid(), + }) + await config.api.rowAction.save(tableId, { + name: generator.guid(), + }) + + const { actions } = (await getRowActionsFromDb(tableId))! + expect(Object.entries(actions)).toHaveLength(2) + + const { automations } = await config.api.automation.fetch() + expect(automations).toHaveLength(2) + + const datasource = await config.api.datasource.get(table.sourceId) + await config.api.datasource.delete(datasource) + + const automationsResp = await config.api.automation.fetch() + expect(automationsResp.automations).toHaveLength(0) + + expect(await getRowActionsFromDb(tableId)).toBeUndefined() + } + ) }) }) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index d915776ad1..cbf91266dc 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -7,6 +7,7 @@ import { import { context, db as dbCore, + docIds, features, MAX_VALID_DATE, MIN_VALID_DATE, @@ -74,6 +75,7 @@ describe.each([ const isLucene = name === "lucene" const isInMemory = name === "in-memory" const isInternal = isSqs || isLucene || isInMemory + const isOracle = name === DatabaseName.ORACLE const isSql = !isInMemory && !isLucene const config = setup.getConfig() @@ -142,14 +144,14 @@ describe.each([ } }) - async function createTable(schema: TableSchema) { + async function createTable(schema?: TableSchema) { const table = await config.api.table.save( tableForDatasource(datasource, { schema }) ) return table._id! } - async function createView(tableId: string, schema: ViewV2Schema) { + async function createView(tableId: string, schema?: ViewV2Schema) { const view = await config.api.viewV2.create({ tableId: tableId, name: generator.guid(), @@ -166,22 +168,51 @@ describe.each([ rows = await config.api.row.fetch(tableOrViewId) } + async function getTable(tableOrViewId: string): Promise { + if (docIds.isViewId(tableOrViewId)) { + const view = await config.api.viewV2.get(tableOrViewId) + return await config.api.table.get(view.tableId) + } else { + return await config.api.table.get(tableOrViewId) + } + } + + async function assertTableExists(nameOrTable: string | Table) { + const name = + typeof nameOrTable === "string" ? nameOrTable : nameOrTable.name + expect(await client!.schema.hasTable(name)).toBeTrue() + } + + async function assertTableNumRows( + nameOrTable: string | Table, + numRows: number + ) { + const name = + typeof nameOrTable === "string" ? nameOrTable : nameOrTable.name + const row = await client!.from(name).count() + const count = parseInt(Object.values(row[0])[0] as string) + expect(count).toEqual(numRows) + } + describe.each([ ["table", createTable], [ "view", - async (schema: TableSchema) => { + async (schema?: TableSchema) => { const tableId = await createTable(schema) const viewId = await createView( tableId, - Object.keys(schema).reduce((viewSchema, fieldName) => { - const field = schema[fieldName] - viewSchema[fieldName] = { - visible: field.visible ?? true, - readonly: false, - } - return viewSchema - }, {}) + Object.keys(schema || {}).reduce( + (viewSchema, fieldName) => { + const field = schema![fieldName] + viewSchema[fieldName] = { + visible: field.visible ?? true, + readonly: false, + } + return viewSchema + }, + {} + ) ) return viewId }, @@ -805,10 +836,11 @@ describe.each([ }) }) - describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => { + const stringTypes = [FieldType.STRING, FieldType.LONGFORM] as const + describe.each(stringTypes)("%s", type => { beforeAll(async () => { tableOrViewId = await createTableOrView({ - name: { name: "name", type: FieldType.STRING }, + name: { name: "name", type }, }) await createRows([{ name: "foo" }, { name: "bar" }]) }) @@ -3556,5 +3588,105 @@ describe.each([ ]) }) }) + + isSql && + !isSqs && + describe("SQL injection", () => { + const badStrings = [ + "1; DROP TABLE %table_name%;", + "1; DELETE FROM %table_name%;", + "1; UPDATE %table_name% SET name = 'foo';", + "1; INSERT INTO %table_name% (name) VALUES ('foo');", + "' OR '1'='1' --", + "'; DROP TABLE %table_name%; --", + "' OR 1=1 --", + "' UNION SELECT null, null, null; --", + "' AND (SELECT COUNT(*) FROM %table_name%) > 0 --", + "\"; EXEC xp_cmdshell('dir'); --", + "\"' OR 'a'='a", + "OR 1=1;", + "'; SHUTDOWN --", + ] + + describe.each(badStrings)("bad string: %s", badStringTemplate => { + // The SQL that knex generates when you try to use a double quote in a + // field name is always invalid and never works, so we skip it for these + // tests. + const skipFieldNameCheck = isOracle && badStringTemplate.includes('"') + + !skipFieldNameCheck && + it("should not allow SQL injection as a field name", async () => { + const tableOrViewId = await createTableOrView() + const table = await getTable(tableOrViewId) + const badString = badStringTemplate.replace( + /%table_name%/g, + table.name + ) + + await config.api.table.save({ + ...table, + schema: { + ...table.schema, + [badString]: { name: badString, type: FieldType.STRING }, + }, + }) + + if (docIds.isViewId(tableOrViewId)) { + const view = await config.api.viewV2.get(tableOrViewId) + await config.api.viewV2.update({ + ...view, + schema: { + [badString]: { visible: true }, + }, + }) + } + + await config.api.row.save(tableOrViewId, { [badString]: "foo" }) + + await assertTableExists(table) + await assertTableNumRows(table, 1) + + const { rows } = await config.api.row.search( + tableOrViewId, + { query: {} }, + { status: 200 } + ) + + expect(rows).toHaveLength(1) + + await assertTableExists(table) + await assertTableNumRows(table, 1) + }) + + it("should not allow SQL injection as a field value", async () => { + const tableOrViewId = await createTableOrView({ + foo: { + name: "foo", + type: FieldType.STRING, + }, + }) + const table = await getTable(tableOrViewId) + const badString = badStringTemplate.replace( + /%table_name%/g, + table.name + ) + + await config.api.row.save(tableOrViewId, { foo: "foo" }) + + await assertTableExists(table) + await assertTableNumRows(table, 1) + + const { rows } = await config.api.row.search( + tableOrViewId, + { query: { equal: { foo: badString } } }, + { status: 200 } + ) + + expect(rows).toBeEmpty() + await assertTableExists(table) + await assertTableNumRows(table, 1) + }) + }) + }) }) }) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index fbb83b49a5..73ac695878 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -17,6 +17,7 @@ import { import { context, features } from "@budibase/backend-core" import { ContextUser, + EventType, FeatureFlag, FieldType, LinkDocumentValue, @@ -44,15 +45,7 @@ const INVALID_DISPLAY_COLUMN_TYPE = [ * This functionality makes sure that when rows with links are created, updated or deleted they are processed * correctly - making sure that no stale links are left around and that all links have been made successfully. */ - -export const EventType = { - ROW_SAVE: "row:save", - ROW_UPDATE: "row:update", - ROW_DELETE: "row:delete", - TABLE_SAVE: "table:save", - TABLE_UPDATED: "table:updated", - TABLE_DELETE: "table:delete", -} +export { EventType } from "@budibase/types" function clearRelationshipFields(schema: TableSchema, rows: Row[]) { for (let [key, field] of Object.entries(schema)) { diff --git a/packages/server/src/events/AutomationEmitter.ts b/packages/server/src/events/AutomationEmitter.ts index a63273bdc0..a95acd0877 100644 --- a/packages/server/src/events/AutomationEmitter.ts +++ b/packages/server/src/events/AutomationEmitter.ts @@ -1,12 +1,20 @@ import { rowEmission, tableEmission } from "./utils" import mainEmitter from "./index" import env from "../environment" -import { Table, Row, DocumentType, App } from "@budibase/types" +import { + Table, + Row, + DocumentType, + App, + ContextEmitter, + EventType, + UserBindings, +} from "@budibase/types" import { context } from "@budibase/backend-core" const MAX_AUTOMATIONS_ALLOWED = 5 -class AutomationEmitter { +class AutomationEmitter implements ContextEmitter { chainCount: number metadata: { automationChainCount: number } @@ -36,11 +44,15 @@ class AutomationEmitter { appId, row, table, + oldRow, + user, }: { - eventName: string + eventName: EventType.ROW_SAVE | EventType.ROW_DELETE | EventType.ROW_UPDATE appId: string row: Row table?: Table + oldRow?: Row + user: UserBindings }) { let MAX_AUTOMATION_CHAIN = await this.getMaxAutomationChain() @@ -54,7 +66,9 @@ class AutomationEmitter { appId, row, table, + oldRow, metadata: this.metadata, + user, }) } diff --git a/packages/server/src/events/BudibaseEmitter.ts b/packages/server/src/events/BudibaseEmitter.ts index c8983096d0..9fc5c2f906 100644 --- a/packages/server/src/events/BudibaseEmitter.ts +++ b/packages/server/src/events/BudibaseEmitter.ts @@ -1,6 +1,12 @@ import { EventEmitter } from "events" import { rowEmission, tableEmission } from "./utils" -import { Table, Row, User } from "@budibase/types" +import { + Table, + Row, + UserBindings, + EventType, + ContextEmitter, +} from "@budibase/types" /** * keeping event emitter in one central location as it might be used for things other than @@ -12,7 +18,7 @@ import { Table, Row, User } from "@budibase/types" * Extending the standard emitter to some syntactic sugar and standardisation to the emitted event. * This is specifically quite important for template strings used in automations. */ -class BudibaseEmitter extends EventEmitter { +class BudibaseEmitter extends EventEmitter implements ContextEmitter { emitRow({ eventName, appId, @@ -21,17 +27,17 @@ class BudibaseEmitter extends EventEmitter { oldRow, user, }: { - eventName: string + eventName: EventType.ROW_SAVE | EventType.ROW_DELETE | EventType.ROW_UPDATE appId: string row: Row table?: Table oldRow?: Row - user: User + user: UserBindings }) { rowEmission({ emitter: this, eventName, appId, row, table, oldRow, user }) } - emitTable(eventName: string, appId: string, table?: Table) { + emitTable(eventName: EventType, appId: string, table?: Table) { tableEmission({ emitter: this, eventName, appId, table }) } diff --git a/packages/server/src/events/utils.ts b/packages/server/src/events/utils.ts index 5e4a1bebbf..9cb5eef187 100644 --- a/packages/server/src/events/utils.ts +++ b/packages/server/src/events/utils.ts @@ -1,4 +1,4 @@ -import { Table, Row, User } from "@budibase/types" +import { Table, Row, UserBindings } from "@budibase/types" import BudibaseEmitter from "./BudibaseEmitter" type BBEventOpts = { @@ -9,7 +9,7 @@ type BBEventOpts = { row?: Row oldRow?: Row metadata?: any - user?: User + user?: UserBindings } interface BBEventTable extends Table { @@ -25,7 +25,7 @@ type BBEvent = { id?: string revision?: string metadata?: any - user?: User + user?: UserBindings } export function rowEmission({ diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index 5a505fbb40..2b3649161c 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -212,7 +212,7 @@ describe("SQL query builder", () => { const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`] expect(query).toEqual({ bindings: [...filterSet, limit], - sql: `select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5`, + sql: `select * from (select * from "test" where ((COALESCE(LOWER("test"."age"), '') like :1 and COALESCE(LOWER("test"."age"), '') like :2)) and ((COALESCE(LOWER("test"."name"), '') like :3 and COALESCE(LOWER("test"."name"), '') like :4)) order by "test"."id" asc) where rownum <= :5`, }) query = new Sql(SqlClient.ORACLE, limit)._query( @@ -244,7 +244,7 @@ describe("SQL query builder", () => { expect(query).toEqual({ bindings: ["John", limit], - sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2`, + sql: `select * from (select * from "test" where (to_char("test"."name") is not null and to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2`, }) }) @@ -262,7 +262,7 @@ describe("SQL query builder", () => { expect(query).toEqual({ bindings: ["John", limit], - sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2`, + sql: `select * from (select * from "test" where (to_char("test"."name") is not null and to_char("test"."name") != :1) or to_char("test"."name") is null order by "test"."id" asc) where rownum <= :2`, }) }) }) diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts index 31ec51c728..a8a21e6906 100644 --- a/packages/server/src/sdk/app/datasources/plus.ts +++ b/packages/server/src/sdk/app/datasources/plus.ts @@ -9,6 +9,7 @@ import * as datasources from "./datasources" import tableSdk from "../tables" import { getIntegration } from "../../../integrations" import { context } from "@budibase/backend-core" +import sdk from "../.." function checkForSchemaErrors(schema: Record) { const errors: Record = {} @@ -96,6 +97,15 @@ export async function buildSchemaFromSource( const datasource = await datasources.get(datasourceId) const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter) + + const oldTables = datasource.entities || {} + const tablesToRemove = Object.keys(oldTables).filter( + t => !Object.keys(tables).includes(t) + ) + for (const table of tablesToRemove) { + await sdk.rowActions.deleteAll(oldTables[table]._id!) + } + datasource.entities = tables datasources.setDefaultDisplayColumns(datasource) diff --git a/packages/server/src/sdk/app/rowActions.ts b/packages/server/src/sdk/app/rowActions/crud.ts similarity index 96% rename from packages/server/src/sdk/app/rowActions.ts rename to packages/server/src/sdk/app/rowActions/crud.ts index de73969696..1153677685 100644 --- a/packages/server/src/sdk/app/rowActions.ts +++ b/packages/server/src/sdk/app/rowActions/crud.ts @@ -7,11 +7,11 @@ import { User, VirtualDocumentType, } from "@budibase/types" -import { generateRowActionsID } from "../../db/utils" -import automations from "./automations" -import { definitions as TRIGGER_DEFINITIONS } from "../../automations/triggerInfo" -import * as triggers from "../../automations/triggers" -import sdk from ".." +import { generateRowActionsID } from "../../../db/utils" +import automations from "../automations" +import { definitions as TRIGGER_DEFINITIONS } from "../../../automations/triggerInfo" +import * as triggers from "../../../automations/triggers" +import sdk from "../.." async function ensureUniqueAndThrow( doc: TableRowActions, diff --git a/packages/server/src/sdk/app/rowActions/index.ts b/packages/server/src/sdk/app/rowActions/index.ts new file mode 100644 index 0000000000..b0dbd426df --- /dev/null +++ b/packages/server/src/sdk/app/rowActions/index.ts @@ -0,0 +1,2 @@ +export * from "./crud" +export * from "./utils" diff --git a/packages/server/src/sdk/app/rowActions/utils.ts b/packages/server/src/sdk/app/rowActions/utils.ts new file mode 100644 index 0000000000..d4e5a4610d --- /dev/null +++ b/packages/server/src/sdk/app/rowActions/utils.ts @@ -0,0 +1,9 @@ +import sdk from "../../../sdk" + +export async function deleteAllForDatasource(datasourceId: string) { + const allTables = await sdk.tables.getAllTables() + const tables = allTables.filter(t => t.sourceId === datasourceId) + for (const table of Object.values(tables)) { + await sdk.rowActions.deleteAll(table._id!) + } +} diff --git a/packages/server/src/tests/utilities/api/automation.ts b/packages/server/src/tests/utilities/api/automation.ts index 11c041d52b..9d9a27e891 100644 --- a/packages/server/src/tests/utilities/api/automation.ts +++ b/packages/server/src/tests/utilities/api/automation.ts @@ -23,17 +23,6 @@ export class AutomationAPI extends TestAPI { }) } - fetchEnriched = async ( - expectations?: Expectations - ): Promise => { - return await this._get( - `/api/automations?enrich=true`, - { - expectations, - } - ) - } - post = async ( body: Automation, expectations?: Expectations diff --git a/packages/types/src/core/events.ts b/packages/types/src/core/events.ts new file mode 100644 index 0000000000..3ce948b3b5 --- /dev/null +++ b/packages/types/src/core/events.ts @@ -0,0 +1,8 @@ +export const enum EventType { + ROW_SAVE = "row:save", + ROW_UPDATE = "row:update", + ROW_DELETE = "row:delete", + TABLE_SAVE = "table:save", + TABLE_UPDATED = "table:updated", + TABLE_DELETE = "table:delete", +} diff --git a/packages/types/src/core/index.ts b/packages/types/src/core/index.ts index b5cbd7affa..73cc7d35e0 100644 --- a/packages/types/src/core/index.ts +++ b/packages/types/src/core/index.ts @@ -1 +1,2 @@ export * from "./installation" +export * from "./events" diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts index a7df701171..95ea2b652f 100644 --- a/packages/types/src/sdk/koa.ts +++ b/packages/types/src/sdk/koa.ts @@ -1,7 +1,17 @@ import { Context, Request } from "koa" -import { User, Role, UserRoles, Account, ConfigType } from "../documents" +import { + User, + Role, + UserRoles, + Account, + ConfigType, + Row, + Table, + UserBindings, +} from "../documents" import { FeatureFlag, License } from "../sdk" import { Files } from "formidable" +import { EventType } from "../core" export interface ContextUser extends Omit { globalId?: string @@ -40,6 +50,7 @@ export interface UserCtx extends Ctx { user: ContextUser roleId?: string + eventEmitter?: ContextEmitter } /** @@ -49,3 +60,32 @@ export interface UserCtx export interface BBContext extends Ctx { user?: ContextUser } + +export interface ContextEmitter { + emitRow(values: { + eventName: EventType.ROW_SAVE + appId: string + row: Row + table: Table + user: UserBindings + }): void + emitRow(values: { + eventName: EventType.ROW_UPDATE + appId: string + row: Row + table: Table + oldRow: Row + user: UserBindings + }): void + emitRow(values: { + eventName: EventType.ROW_DELETE + appId: string + row: Row + user: UserBindings + }): void + emitTable( + eventName: EventType.TABLE_SAVE | EventType.TABLE_DELETE, + appId: string, + table?: Table + ): void +} diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index a2d4b4760f..d6dcb0a2fa 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -61,7 +61,7 @@ type BasicFilter = Record & { [InternalSearchFilterOperator.COMPLEX_ID_OPERATOR]?: never } -type ArrayFilter = Record & { +export type ArrayFilter = Record & { [InternalSearchFilterOperator.COMPLEX_ID_OPERATOR]?: { id: string[] values: string[]