diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 1bc1915a71..a17ca352cc 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -108,7 +108,7 @@ jobs: - name: Pull testcontainers images run: | docker pull testcontainers/ryuk:0.5.1 & - docker pull budibase/couchdb:v3.2.1-sqs & + docker pull budibase/couchdb:v3.3.3 & docker pull redis & wait $(jobs -p) @@ -162,17 +162,24 @@ jobs: node-version: 20.x cache: yarn + - name: Load dotenv + id: dotenv + uses: falti/dotenv-action@v1.1.3 + with: + path: ./packages/server/datasource-sha.env + - name: Pull testcontainers images run: | - docker pull mcr.microsoft.com/mssql/server:2022-CU13-ubuntu-22.04 & - docker pull mysql:8.3 & - docker pull postgres:16.1-bullseye & - docker pull mongo:7.0-jammy & - docker pull mariadb:lts & - docker pull testcontainers/ryuk:0.5.1 & - docker pull budibase/couchdb:v3.2.1-sqs & + docker pull mcr.microsoft.com/mssql/server@${{ steps.dotenv.outputs.MSSQL_SHA }} & + docker pull mysql@${{ steps.dotenv.outputs.MYSQL_SHA }} & + docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }} & + docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }} & + docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }} & + docker pull budibase/oracle-database:23.2-slim-faststart & docker pull minio/minio & docker pull redis & + docker pull testcontainers/ryuk:0.5.1 & + docker pull budibase/couchdb:v3.3.3 & wait $(jobs -p) diff --git a/globalSetup.ts b/globalSetup.ts index dd1454b6e1..aa1cb00fe1 100644 --- a/globalSetup.ts +++ b/globalSetup.ts @@ -46,7 +46,7 @@ export default async function setup() { await killContainers(containers) try { - const couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs") + const couchdb = new GenericContainer("budibase/couchdb:v3.3.3") .withExposedPorts(5984, 4984) .withEnvironment({ COUCHDB_PASSWORD: "budibase", diff --git a/lerna.json b/lerna.json index fb239ee35d..19303c8580 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.29.24", + "version": "2.29.29", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/package.json b/package.json index 29b87898ac..f3cbd75836 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "@types/proper-lockfile": "^4.1.4", "@typescript-eslint/parser": "6.9.0", "esbuild": "^0.18.17", - "esbuild-node-externals": "^1.8.0", + "esbuild-node-externals": "^1.14.0", "eslint": "^8.52.0", "eslint-plugin-import": "^2.29.0", "eslint-plugin-jest": "^27.9.0", diff --git a/packages/account-portal b/packages/account-portal index b03e584e46..32b8fa4643 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit b03e584e465f620b49a1b688ff4afc973e6c0758 +Subproject commit 32b8fa4643b4f0f74ee89760deffe431ab347ad9 diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index f3c3beeaab..61fbb3d61e 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -56,24 +56,24 @@ class CouchDBError extends Error implements DBError { constructor( message: string, info: { - status: number | undefined - statusCode: number | undefined + status?: number + statusCode?: number name: string - errid: string - description: string - reason: string - error: string + errid?: string + description?: string + reason?: string + error?: string } ) { super(message) const statusCode = info.status || info.statusCode || 500 this.status = statusCode this.statusCode = statusCode - this.reason = info.reason + this.reason = info.reason || "Unknown" this.name = info.name - this.errid = info.errid - this.description = info.description - this.error = info.error + this.errid = info.errid || "Unknown" + this.description = info.description || "Unknown" + this.error = info.error || "Not found" } } @@ -246,6 +246,35 @@ export class DatabaseImpl implements Database { }) } + async bulkRemove(documents: Document[], opts?: { silenceErrors?: boolean }) { + const response: Nano.DocumentBulkResponse[] = await this.performCall(db => { + return () => + db.bulk({ + docs: documents.map(doc => ({ + ...doc, + _deleted: true, + })), + }) + }) + if (opts?.silenceErrors) { + return + } + let errorFound = false + let errorMessage: string = "Unable to bulk remove documents: " + for (let res of response) { + if (res.error) { + errorFound = true + errorMessage += res.error + } + } + if (errorFound) { + throw new CouchDBError(errorMessage, { + name: this.name, + status: 400, + }) + } + } + async post(document: AnyDocument, opts?: DatabasePutOpts) { if (!document._id) { document._id = newid() @@ -279,8 +308,12 @@ export class DatabaseImpl implements Database { } async bulkDocs(documents: AnyDocument[]) { + const now = new Date().toISOString() return this.performCall(db => { - return () => db.bulk({ docs: documents }) + return () => + db.bulk({ + docs: documents.map(d => ({ createdAt: now, ...d, updatedAt: now })), + }) }) } diff --git a/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts b/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts new file mode 100644 index 0000000000..89eecc3785 --- /dev/null +++ b/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts @@ -0,0 +1,118 @@ +import tk from "timekeeper" + +import { DatabaseImpl } from ".." + +import { generator, structures } from "../../../../tests" + +const initialTime = new Date() +tk.freeze(initialTime) + +describe("DatabaseImpl", () => { + const db = new DatabaseImpl(structures.db.id()) + + beforeEach(() => { + tk.freeze(initialTime) + }) + + describe("put", () => { + it("persists createdAt and updatedAt fields", async () => { + const id = generator.guid() + await db.put({ _id: id }) + + expect(await db.get(id)).toEqual({ + _id: id, + _rev: expect.any(String), + createdAt: initialTime.toISOString(), + updatedAt: initialTime.toISOString(), + }) + }) + + it("updates updated at fields", async () => { + const id = generator.guid() + + await db.put({ _id: id }) + tk.travel(100) + + await db.put({ ...(await db.get(id)), newValue: 123 }) + + expect(await db.get(id)).toEqual({ + _id: id, + _rev: expect.any(String), + newValue: 123, + createdAt: initialTime.toISOString(), + updatedAt: new Date().toISOString(), + }) + }) + }) + + describe("bulkDocs", () => { + it("persists createdAt and updatedAt fields", async () => { + const ids = generator.unique(() => generator.guid(), 5) + await db.bulkDocs(ids.map(id => ({ _id: id }))) + + for (const id of ids) { + expect(await db.get(id)).toEqual({ + _id: id, + _rev: expect.any(String), + createdAt: initialTime.toISOString(), + updatedAt: initialTime.toISOString(), + }) + } + }) + + it("updates updated at fields", async () => { + const ids = generator.unique(() => generator.guid(), 5) + + await db.bulkDocs(ids.map(id => ({ _id: id }))) + tk.travel(100) + + const docsToUpdate = await Promise.all( + ids.map(async id => ({ ...(await db.get(id)), newValue: 123 })) + ) + await db.bulkDocs(docsToUpdate) + + for (const id of ids) { + expect(await db.get(id)).toEqual({ + _id: id, + _rev: expect.any(String), + newValue: 123, + createdAt: initialTime.toISOString(), + updatedAt: new Date().toISOString(), + }) + } + }) + + it("keeps existing createdAt", async () => { + const ids = generator.unique(() => generator.guid(), 2) + + await db.bulkDocs(ids.map(id => ({ _id: id }))) + tk.travel(100) + + const newDocs = generator + .unique(() => generator.guid(), 3) + .map(id => ({ _id: id })) + const docsToUpdate = await Promise.all( + ids.map(async id => ({ ...(await db.get(id)), newValue: 123 })) + ) + await db.bulkDocs([...newDocs, ...docsToUpdate]) + + for (const { _id } of docsToUpdate) { + expect(await db.get(_id)).toEqual({ + _id, + _rev: expect.any(String), + newValue: 123, + createdAt: initialTime.toISOString(), + updatedAt: new Date().toISOString(), + }) + } + for (const { _id } of newDocs) { + expect(await db.get(_id)).toEqual({ + _id, + _rev: expect.any(String), + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + } + }) + }) +}) diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts index 4e2b147ef3..7026224564 100644 --- a/packages/backend-core/src/db/instrumentation.ts +++ b/packages/backend-core/src/db/instrumentation.ts @@ -71,6 +71,16 @@ export class DDInstrumentedDatabase implements Database { }) } + bulkRemove( + documents: Document[], + opts?: { silenceErrors?: boolean } + ): Promise { + return tracer.trace("db.bulkRemove", span => { + span?.addTags({ db_name: this.name, num_docs: documents.length }) + return this.db.bulkRemove(documents, opts) + }) + } + put( document: AnyDocument, opts?: DatabasePutOpts | undefined diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index 5d9c5b74d3..6ee06d12ef 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -199,9 +199,8 @@ export const createPlatformUserView = async () => { export const queryPlatformView = async ( viewName: ViewName, - params: DatabaseQueryOpts, - opts?: QueryViewOptions -): Promise => { + params: DatabaseQueryOpts +): Promise => { const CreateFuncByName: any = { [ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView, [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView, @@ -209,7 +208,9 @@ export const queryPlatformView = async ( return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => { const createFn = CreateFuncByName[viewName] - return queryView(viewName, params, db, createFn, opts) + return queryView(viewName, params, db, createFn, { + arrayResponse: true, + }) as Promise }) } diff --git a/packages/backend-core/src/platform/users.ts b/packages/backend-core/src/platform/users.ts index ccaad76b19..9378e23724 100644 --- a/packages/backend-core/src/platform/users.ts +++ b/packages/backend-core/src/platform/users.ts @@ -25,6 +25,11 @@ export async function getUserDoc(emailOrId: string): Promise { return db.get(emailOrId) } +export async function updateUserDoc(platformUser: PlatformUserById) { + const db = getPlatformDB() + await db.put(platformUser) +} + // CREATE function newUserIdDoc(id: string, tenantId: string): PlatformUserById { @@ -113,15 +118,12 @@ export async function addUser( export async function removeUser(user: User) { const db = getPlatformDB() const keys = [user._id!, user.email] - const userDocs = await db.allDocs({ + const userDocs = await db.allDocs({ keys, include_docs: true, }) - const toDelete = userDocs.rows.map((row: any) => { - return { - ...row.doc, - _deleted: true, - } - }) - await db.bulkDocs(toDelete) + await db.bulkRemove( + userDocs.rows.map(row => row.doc!), + { silenceErrors: true } + ) } diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index a67da7bc10..bafaef40e4 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -3,15 +3,16 @@ import * as dbCore from "../db" import { getNativeSql, isExternalTable, - isValidISODateString, - isValidFilter, - sqlLog, isInvalidISODateString, + isValidFilter, + isValidISODateString, + sqlLog, } from "./utils" -import { SqlStatements } from "./sqlStatements" import SqlTableQueryBuilder from "./sqlTable" import { AnySearchFilter, + ArrayOperator, + BasicOperator, BBReferenceFieldMetadata, FieldSchema, FieldType, @@ -23,6 +24,7 @@ import { prefixed, QueryJson, QueryOptions, + RangeOperator, RelationshipsJson, SearchFilters, SortOrder, @@ -33,163 +35,15 @@ import { TableSourceType, } from "@budibase/types" import environment from "../environment" -import { helpers } from "@budibase/shared-core" +import { dataFilters, helpers } from "@budibase/shared-core" type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any -const envLimit = environment.SQL_MAX_ROWS - ? parseInt(environment.SQL_MAX_ROWS) - : null -const BASE_LIMIT = envLimit || 5000 - -// Takes a string like foo and returns a quoted string like [foo] for SQL Server -// and "foo" for Postgres. -function quote(client: SqlClient, str: string): string { - switch (client) { - case SqlClient.SQL_LITE: - case SqlClient.ORACLE: - case SqlClient.POSTGRES: - return `"${str}"` - case SqlClient.MS_SQL: - return `[${str}]` - case SqlClient.MY_SQL: - return `\`${str}\`` - } -} - -// Takes a string like a.b.c and returns a quoted identifier like [a].[b].[c] -// for SQL Server and `a`.`b`.`c` for MySQL. -function quotedIdentifier(client: SqlClient, key: string): string { - return key - .split(".") - .map(part => quote(client, part)) - .join(".") -} - -function parse(input: any) { - if (Array.isArray(input)) { - return JSON.stringify(input) - } - if (input == undefined) { - return null - } - if (typeof input !== "string") { - return input - } - if (isInvalidISODateString(input)) { - return null - } - if (isValidISODateString(input)) { - return new Date(input.trim()) - } - return input -} - -function parseBody(body: any) { - for (let [key, value] of Object.entries(body)) { - body[key] = parse(value) - } - return body -} - -function parseFilters(filters: SearchFilters | undefined): SearchFilters { - if (!filters) { - return {} - } - for (let [key, value] of Object.entries(filters)) { - let parsed - if (typeof value === "object") { - parsed = parseFilters(value) - } else { - parsed = parse(value) - } - // @ts-ignore - filters[key] = parsed - } - return filters -} - -function generateSelectStatement( - json: QueryJson, - knex: Knex -): (string | Knex.Raw)[] | "*" { - const { resource, meta } = json - const client = knex.client.config.client as SqlClient - - if (!resource || !resource.fields || resource.fields.length === 0) { - return "*" - } - - const schema = meta.table.schema - return resource.fields.map(field => { - const parts = field.split(/\./g) - let table: string | undefined = undefined - let column: string | undefined = undefined - - // Just a column name, e.g.: "column" - if (parts.length === 1) { - column = parts[0] - } - - // A table name and a column name, e.g.: "table.column" - if (parts.length === 2) { - table = parts[0] - column = parts[1] - } - - // A link doc, e.g.: "table.doc1.fieldName" - if (parts.length > 2) { - table = parts[0] - column = parts.slice(1).join(".") - } - - if (!column) { - throw new Error(`Invalid field name: ${field}`) - } - - const columnSchema = schema[column] - - if ( - client === SqlClient.POSTGRES && - columnSchema?.externalType?.includes("money") - ) { - return knex.raw( - `${quotedIdentifier( - client, - [table, column].join(".") - )}::money::numeric as ${quote(client, field)}` - ) - } - - if ( - client === SqlClient.MS_SQL && - columnSchema?.type === FieldType.DATETIME && - columnSchema.timeOnly - ) { - // Time gets returned as timestamp from mssql, not matching the expected - // HH:mm format - return knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) - } - - // There's at least two edge cases being handled in the expression below. - // 1. The column name could start/end with a space, and in that case we - // want to preseve that space. - // 2. Almost all column names are specified in the form table.column, except - // in the case of relationships, where it's table.doc1.column. In that - // case, we want to split it into `table`.`doc1.column` for reasons that - // aren't actually clear to me, but `table`.`doc1` breaks things with the - // sample data tests. - if (table) { - return knex.raw( - `${quote(client, table)}.${quote(client, column)} as ${quote( - client, - field - )}` - ) - } else { - return knex.raw(`${quote(client, field)} as ${quote(client, field)}`) - } - }) +function getBaseLimit() { + const envLimit = environment.SQL_MAX_ROWS + ? parseInt(environment.SQL_MAX_ROWS) + : null + return envLimit || 5000 } function getTableName(table?: Table): string | undefined { @@ -222,37 +76,276 @@ function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { class InternalBuilder { private readonly client: SqlClient + private readonly query: QueryJson + private readonly splitter: dataFilters.ColumnSplitter + private readonly knex: Knex - constructor(client: SqlClient) { + constructor(client: SqlClient, knex: Knex, query: QueryJson) { this.client = client + this.query = query + this.knex = knex + + this.splitter = new dataFilters.ColumnSplitter([this.table], { + aliases: this.query.tableAliases, + columnPrefix: this.query.meta.columnPrefix, + }) + } + + get table(): Table { + return this.query.meta.table + } + + getFieldSchema(key: string): FieldSchema | undefined { + const { column } = this.splitter.run(key) + return this.table.schema[column] + } + + // Takes a string like foo and returns a quoted string like [foo] for SQL Server + // and "foo" for Postgres. + private quote(str: string): string { + switch (this.client) { + case SqlClient.SQL_LITE: + case SqlClient.ORACLE: + case SqlClient.POSTGRES: + return `"${str}"` + case SqlClient.MS_SQL: + return `[${str}]` + case SqlClient.MY_SQL: + return `\`${str}\`` + } + } + + // Takes a string like a.b.c and returns a quoted identifier like [a].[b].[c] + // for SQL Server and `a`.`b`.`c` for MySQL. + private quotedIdentifier(key: string): string { + return key + .split(".") + .map(part => this.quote(part)) + .join(".") + } + + private generateSelectStatement(): (string | Knex.Raw)[] | "*" { + const { resource, meta } = this.query + + if (!resource || !resource.fields || resource.fields.length === 0) { + return "*" + } + + const schema = meta.table.schema + return resource.fields.map(field => { + const parts = field.split(/\./g) + let table: string | undefined = undefined + let column: string | undefined = undefined + + // Just a column name, e.g.: "column" + if (parts.length === 1) { + column = parts[0] + } + + // A table name and a column name, e.g.: "table.column" + if (parts.length === 2) { + table = parts[0] + column = parts[1] + } + + // A link doc, e.g.: "table.doc1.fieldName" + if (parts.length > 2) { + table = parts[0] + column = parts.slice(1).join(".") + } + + if (!column) { + throw new Error(`Invalid field name: ${field}`) + } + + const columnSchema = schema[column] + + if ( + this.client === SqlClient.POSTGRES && + columnSchema?.externalType?.includes("money") + ) { + return this.knex.raw( + `${this.quotedIdentifier( + [table, column].join(".") + )}::money::numeric as ${this.quote(field)}` + ) + } + + if ( + this.client === SqlClient.MS_SQL && + columnSchema?.type === FieldType.DATETIME && + columnSchema.timeOnly + ) { + // Time gets returned as timestamp from mssql, not matching the expected + // HH:mm format + return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) + } + + // There's at least two edge cases being handled in the expression below. + // 1. The column name could start/end with a space, and in that case we + // want to preseve that space. + // 2. Almost all column names are specified in the form table.column, except + // in the case of relationships, where it's table.doc1.column. In that + // case, we want to split it into `table`.`doc1.column` for reasons that + // aren't actually clear to me, but `table`.`doc1` breaks things with the + // sample data tests. + if (table) { + return this.knex.raw( + `${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}` + ) + } else { + return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`) + } + }) + } + + // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses, + // so when we use them we need to wrap them in to_char(). This function + // converts a field name to the appropriate identifier. + private convertClobs(field: string): string { + const parts = field.split(".") + const col = parts.pop()! + const schema = this.table.schema[col] + let identifier = this.quotedIdentifier(field) + if ( + schema.type === FieldType.STRING || + schema.type === FieldType.LONGFORM || + schema.type === FieldType.BB_REFERENCE_SINGLE || + schema.type === FieldType.BB_REFERENCE || + schema.type === FieldType.OPTIONS || + schema.type === FieldType.BARCODEQR + ) { + identifier = `to_char(${identifier})` + } + return identifier + } + + private parse(input: any, schema: FieldSchema) { + if (Array.isArray(input)) { + return JSON.stringify(input) + } + if (input == undefined) { + return null + } + + if ( + this.client === SqlClient.ORACLE && + schema.type === FieldType.DATETIME && + schema.timeOnly + ) { + if (input instanceof Date) { + const hours = input.getHours().toString().padStart(2, "0") + const minutes = input.getMinutes().toString().padStart(2, "0") + const seconds = input.getSeconds().toString().padStart(2, "0") + return `${hours}:${minutes}:${seconds}` + } + if (typeof input === "string") { + return new Date(`1970-01-01T${input}Z`) + } + } + + if (typeof input === "string") { + if (isInvalidISODateString(input)) { + return null + } + if (isValidISODateString(input)) { + return new Date(input.trim()) + } + } + return input + } + + private parseBody(body: any) { + for (let [key, value] of Object.entries(body)) { + const { column } = this.splitter.run(key) + const schema = this.table.schema[column] + if (!schema) { + continue + } + body[key] = this.parse(value, schema) + } + return body + } + + private parseFilters(filters: SearchFilters): SearchFilters { + for (const op of Object.values(BasicOperator)) { + const filter = filters[op] + if (!filter) { + continue + } + for (const key of Object.keys(filter)) { + if (Array.isArray(filter[key])) { + filter[key] = JSON.stringify(filter[key]) + continue + } + const { column } = this.splitter.run(key) + const schema = this.table.schema[column] + if (!schema) { + continue + } + filter[key] = this.parse(filter[key], schema) + } + } + + for (const op of Object.values(ArrayOperator)) { + const filter = filters[op] + if (!filter) { + continue + } + for (const key of Object.keys(filter)) { + const { column } = this.splitter.run(key) + const schema = this.table.schema[column] + if (!schema) { + continue + } + filter[key] = filter[key].map(v => this.parse(v, schema)) + } + } + + for (const op of Object.values(RangeOperator)) { + const filter = filters[op] + if (!filter) { + continue + } + for (const key of Object.keys(filter)) { + const { column } = this.splitter.run(key) + const schema = this.table.schema[column] + if (!schema) { + continue + } + const value = filter[key] + if ("low" in value) { + value.low = this.parse(value.low, schema) + } + if ("high" in value) { + value.high = this.parse(value.high, schema) + } + } + } + + return filters } // right now we only do filters on the specific table being queried addFilters( query: Knex.QueryBuilder, filters: SearchFilters | undefined, - table: Table, - opts: { - aliases?: Record + opts?: { relationship?: boolean - columnPrefix?: string } ): Knex.QueryBuilder { if (!filters) { return query } - filters = parseFilters(filters) + filters = this.parseFilters(filters) + const aliases = this.query.tableAliases // if all or specified in filters, then everything is an or const allOr = filters.allOr - const sqlStatements = new SqlStatements(this.client, table, { - allOr, - columnPrefix: opts.columnPrefix, - }) const tableName = - this.client === SqlClient.SQL_LITE ? table._id! : table.name + this.client === SqlClient.SQL_LITE ? this.table._id! : this.table.name function getTableAlias(name: string) { - const alias = opts.aliases?.[name] + const alias = aliases?.[name] return alias || name } function iterate( @@ -278,10 +371,10 @@ class InternalBuilder { ), castedTypeValue.values ) - } else if (!opts.relationship && !isRelationshipField) { + } else if (!opts?.relationship && !isRelationshipField) { const alias = getTableAlias(tableName) fn(alias ? `${alias}.${updatedKey}` : updatedKey, value) - } else if (opts.relationship && isRelationshipField) { + } else if (opts?.relationship && isRelationshipField) { const [filterTableName, property] = updatedKey.split(".") const alias = getTableAlias(filterTableName) fn(alias ? `${alias}.${property}` : property, value) @@ -298,10 +391,9 @@ class InternalBuilder { } else { const rawFnc = `${fnc}Raw` // @ts-ignore - query = query[rawFnc]( - `LOWER(${quotedIdentifier(this.client, key)}) LIKE ?`, - [`%${value.toLowerCase()}%`] - ) + query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ + `%${value.toLowerCase()}%`, + ]) } } @@ -343,26 +435,30 @@ class InternalBuilder { const andOr = mode === filters?.containsAny ? " OR " : " AND " iterate(mode, (key, value) => { let statement = "" + const identifier = this.quotedIdentifier(key) for (let i in value) { if (typeof value[i] === "string") { value[i] = `%"${value[i].toLowerCase()}"%` } else { value[i] = `%${value[i]}%` } - statement += - (statement ? andOr : "") + - `COALESCE(LOWER(${quotedIdentifier( - this.client, - key - )}), '') LIKE ?` + statement += `${ + statement ? andOr : "" + }COALESCE(LOWER(${identifier}), '') LIKE ?` } if (statement === "") { return } - // @ts-ignore - query = query[rawFnc](`${not}(${statement})`, value) + if (not) { + query = query[rawFnc]( + `(NOT (${statement}) OR ${identifier} IS NULL)`, + value + ) + } else { + query = query[rawFnc](statement, value) + } }) } } @@ -372,10 +468,25 @@ class InternalBuilder { iterate( filters.oneOf, (key: string, array) => { - query = query[fnc](key, Array.isArray(array) ? array : [array]) + if (this.client === SqlClient.ORACLE) { + key = this.convertClobs(key) + array = Array.isArray(array) ? array : [array] + const binding = new Array(array.length).fill("?").join(",") + query = query.whereRaw(`${key} IN (${binding})`, array) + } else { + query = query[fnc](key, Array.isArray(array) ? array : [array]) + } }, (key: string[], array) => { - query = query[fnc](key, Array.isArray(array) ? array : [array]) + if (this.client === SqlClient.ORACLE) { + const keyStr = `(${key.map(k => this.convertClobs(k)).join(",")})` + const binding = `(${array + .map((a: any) => `(${new Array(a.length).fill("?").join(",")})`) + .join(",")})` + query = query.whereRaw(`${keyStr} IN ${binding}`, array.flat()) + } else { + query = query[fnc](key, Array.isArray(array) ? array : [array]) + } } ) } @@ -388,10 +499,9 @@ class InternalBuilder { } else { const rawFnc = `${fnc}Raw` // @ts-ignore - query = query[rawFnc]( - `LOWER(${quotedIdentifier(this.client, key)}) LIKE ?`, - [`${value.toLowerCase()}%`] - ) + query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ + `${value.toLowerCase()}%`, + ]) } }) } @@ -415,12 +525,53 @@ class InternalBuilder { } const lowValid = isValidFilter(value.low), highValid = isValidFilter(value.high) + + const schema = this.getFieldSchema(key) + + if (this.client === SqlClient.ORACLE) { + // @ts-ignore + key = this.knex.raw(this.convertClobs(key)) + } + if (lowValid && highValid) { - query = sqlStatements.between(query, key, value.low, value.high) + if ( + schema?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, + [value.low, value.high] + ) + } else { + const fnc = allOr ? "orWhereBetween" : "whereBetween" + query = query[fnc](key, [value.low, value.high]) + } } else if (lowValid) { - query = sqlStatements.lte(query, key, value.low) + if ( + schema?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, + [value.low] + ) + } else { + const fnc = allOr ? "orWhere" : "where" + query = query[fnc](key, ">=", value.low) + } } else if (highValid) { - query = sqlStatements.gte(query, key, value.high) + if ( + schema?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, + [value.high] + ) + } else { + const fnc = allOr ? "orWhere" : "where" + query = query[fnc](key, "<=", value.high) + } } }) } @@ -429,20 +580,18 @@ class InternalBuilder { const fnc = allOr ? "orWhereRaw" : "whereRaw" if (this.client === SqlClient.MS_SQL) { query = query[fnc]( - `CASE WHEN ${quotedIdentifier( - this.client, - key - )} = ? THEN 1 ELSE 0 END = 1`, + `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 1`, [value] ) } else if (this.client === SqlClient.ORACLE) { + const identifier = this.convertClobs(key) query = query[fnc]( - `COALESCE(${quotedIdentifier(this.client, key)}, -1) = ?`, + `(${identifier} IS NOT NULL AND ${identifier} = ?)`, [value] ) } else { query = query[fnc]( - `COALESCE(${quotedIdentifier(this.client, key)} = ?, FALSE)`, + `COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`, [value] ) } @@ -453,20 +602,18 @@ class InternalBuilder { const fnc = allOr ? "orWhereRaw" : "whereRaw" if (this.client === SqlClient.MS_SQL) { query = query[fnc]( - `CASE WHEN ${quotedIdentifier( - this.client, - key - )} = ? THEN 1 ELSE 0 END = 0`, + `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 0`, [value] ) } else if (this.client === SqlClient.ORACLE) { + const identifier = this.convertClobs(key) query = query[fnc]( - `COALESCE(${quotedIdentifier(this.client, key)}, -1) != ?`, + `(${identifier} IS NOT NULL AND ${identifier} != ?) OR ${identifier} IS NULL`, [value] ) } else { query = query[fnc]( - `COALESCE(${quotedIdentifier(this.client, key)} != ?, TRUE)`, + `COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`, [value] ) } @@ -494,9 +641,9 @@ class InternalBuilder { contains(filters.containsAny, true) } - const tableRef = opts?.aliases?.[table._id!] || table._id + const tableRef = aliases?.[this.table._id!] || this.table._id // when searching internal tables make sure long looking for rows - if (filters.documentType && !isExternalTable(table) && tableRef) { + if (filters.documentType && !isExternalTable(this.table) && tableRef) { // has to be its own option, must always be AND onto the search query.andWhereLike( `${tableRef}._id`, @@ -507,29 +654,26 @@ class InternalBuilder { return query } - addDistinctCount( - query: Knex.QueryBuilder, - json: QueryJson - ): Knex.QueryBuilder { - const table = json.meta.table - const primary = table.primary - const aliases = json.tableAliases + addDistinctCount(query: Knex.QueryBuilder): Knex.QueryBuilder { + const primary = this.table.primary + const aliases = this.query.tableAliases const aliased = - table.name && aliases?.[table.name] ? aliases[table.name] : table.name + this.table.name && aliases?.[this.table.name] + ? aliases[this.table.name] + : this.table.name if (!primary) { throw new Error("SQL counting requires primary key to be supplied") } return query.countDistinct(`${aliased}.${primary[0]} as total`) } - addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder { - let { sort } = json - const table = json.meta.table - const primaryKey = table.primary - const tableName = getTableName(table) - const aliases = json.tableAliases + addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder { + let { sort } = this.query + const primaryKey = this.table.primary + const tableName = getTableName(this.table) + const aliases = this.query.tableAliases const aliased = - tableName && aliases?.[tableName] ? aliases[tableName] : table?.name + tableName && aliases?.[tableName] ? aliases[tableName] : this.table?.name if (!Array.isArray(primaryKey)) { throw new Error("Sorting requires primary key to be specified for table") } @@ -537,13 +681,23 @@ class InternalBuilder { for (let [key, value] of Object.entries(sort)) { const direction = value.direction === SortOrder.ASCENDING ? "asc" : "desc" - let nulls - if (this.client === SqlClient.POSTGRES) { - // All other clients already sort this as expected by default, and adding this to the rest of the clients is causing issues + + let nulls: "first" | "last" | undefined = undefined + if ( + this.client === SqlClient.POSTGRES || + this.client === SqlClient.ORACLE + ) { nulls = value.direction === SortOrder.ASCENDING ? "first" : "last" } - query = query.orderBy(`${aliased}.${key}`, direction, nulls) + let composite = `${aliased}.${key}` + if (this.client === SqlClient.ORACLE) { + query = query.orderByRaw( + `${this.convertClobs(composite)} ${direction} nulls ${nulls}` + ) + } else { + query = query.orderBy(composite, direction, nulls) + } } } @@ -644,30 +798,52 @@ class InternalBuilder { return query } - knexWithAlias( - knex: Knex, - endpoint: QueryJson["endpoint"], - aliases?: QueryJson["tableAliases"] - ): Knex.QueryBuilder { - const tableName = endpoint.entityId - const tableAlias = aliases?.[tableName] - - return knex( - this.tableNameWithSchema(tableName, { - alias: tableAlias, - schema: endpoint.schema, + qualifiedKnex(opts?: { alias?: string | boolean }): Knex.QueryBuilder { + let alias = this.query.tableAliases?.[this.query.endpoint.entityId] + if (opts?.alias === false) { + alias = undefined + } else if (typeof opts?.alias === "string") { + alias = opts.alias + } + return this.knex( + this.tableNameWithSchema(this.query.endpoint.entityId, { + alias, + schema: this.query.endpoint.schema, }) ) } - create(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { - const { endpoint, body } = json - let query = this.knexWithAlias(knex, endpoint) - const parsedBody = parseBody(body) - // make sure no null values in body for creation - for (let [key, value] of Object.entries(parsedBody)) { - if (value == null) { - delete parsedBody[key] + create(opts: QueryOptions): Knex.QueryBuilder { + const { body } = this.query + let query = this.qualifiedKnex({ alias: false }) + const parsedBody = this.parseBody(body) + + if (this.client === SqlClient.ORACLE) { + // Oracle doesn't seem to automatically insert nulls + // if we don't specify them, so we need to do that here + for (const [column, schema] of Object.entries( + this.query.meta.table.schema + )) { + if ( + schema.constraints?.presence === true || + schema.type === FieldType.FORMULA || + schema.type === FieldType.AUTO || + schema.type === FieldType.LINK + ) { + continue + } + + const value = parsedBody[column] + if (value == null) { + parsedBody[column] = null + } + } + } else { + // make sure no null values in body for creation + for (let [key, value] of Object.entries(parsedBody)) { + if (value == null) { + delete parsedBody[key] + } } } @@ -679,36 +855,39 @@ class InternalBuilder { } } - bulkCreate(knex: Knex, json: QueryJson): Knex.QueryBuilder { - const { endpoint, body } = json - let query = this.knexWithAlias(knex, endpoint) + bulkCreate(): Knex.QueryBuilder { + const { body } = this.query + let query = this.qualifiedKnex({ alias: false }) if (!Array.isArray(body)) { return query } - const parsedBody = body.map(row => parseBody(row)) + const parsedBody = body.map(row => this.parseBody(row)) return query.insert(parsedBody) } - bulkUpsert(knex: Knex, json: QueryJson): Knex.QueryBuilder { - const { endpoint, body } = json - let query = this.knexWithAlias(knex, endpoint) + bulkUpsert(): Knex.QueryBuilder { + const { body } = this.query + let query = this.qualifiedKnex({ alias: false }) if (!Array.isArray(body)) { return query } - const parsedBody = body.map(row => parseBody(row)) + const parsedBody = body.map(row => this.parseBody(row)) if ( this.client === SqlClient.POSTGRES || this.client === SqlClient.SQL_LITE || this.client === SqlClient.MY_SQL ) { - const primary = json.meta.table.primary + const primary = this.table.primary if (!primary) { throw new Error("Primary key is required for upsert") } const ret = query.insert(parsedBody).onConflict(primary).merge() return ret - } else if (this.client === SqlClient.MS_SQL) { - // No upsert or onConflict support in MSSQL yet, see: + } else if ( + this.client === SqlClient.MS_SQL || + this.client === SqlClient.ORACLE + ) { + // No upsert or onConflict support in MSSQL/Oracle yet, see: // https://github.com/knex/knex/pull/6050 return query.insert(parsedBody) } @@ -716,19 +895,18 @@ class InternalBuilder { } read( - knex: Knex, - json: QueryJson, opts: { limits?: { base: number; query: number } } = {} ): Knex.QueryBuilder { - let { endpoint, filters, paginate, relationships, tableAliases } = json + let { endpoint, filters, paginate, relationships, tableAliases } = + this.query const { limits } = opts const counting = endpoint.operation === Operation.COUNT const tableName = endpoint.entityId // start building the query - let query = this.knexWithAlias(knex, endpoint, tableAliases) + let query = this.qualifiedKnex() // handle pagination let foundOffset: number | null = null let foundLimit = limits?.query || limits?.base @@ -756,16 +934,13 @@ class InternalBuilder { } // add sorting to pre-query // no point in sorting when counting - query = this.addSorting(query, json) + query = this.addSorting(query) } // add filters to the query (where) - query = this.addFilters(query, filters, json.meta.table, { - columnPrefix: json.meta.columnPrefix, - aliases: tableAliases, - }) + query = this.addFilters(query, filters) const alias = tableAliases?.[tableName] || tableName - let preQuery: Knex.QueryBuilder = knex({ + let preQuery: Knex.QueryBuilder = this.knex({ // the typescript definition for the knex constructor doesn't support this // syntax, but it is the only way to alias a pre-query result as part of // a query - there is an alias dictionary type, but it assumes it can only @@ -774,11 +949,11 @@ class InternalBuilder { }) // if counting, use distinct count, else select preQuery = !counting - ? preQuery.select(generateSelectStatement(json, knex)) - : this.addDistinctCount(preQuery, json) + ? preQuery.select(this.generateSelectStatement()) + : this.addDistinctCount(preQuery) // have to add after as well (this breaks MS-SQL) if (this.client !== SqlClient.MS_SQL && !counting) { - preQuery = this.addSorting(preQuery, json) + preQuery = this.addSorting(preQuery) } // handle joins query = this.addRelationships( @@ -795,21 +970,14 @@ class InternalBuilder { query = query.limit(limits.base) } - return this.addFilters(query, filters, json.meta.table, { - columnPrefix: json.meta.columnPrefix, - relationship: true, - aliases: tableAliases, - }) + return this.addFilters(query, filters, { relationship: true }) } - update(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { - const { endpoint, body, filters, tableAliases } = json - let query = this.knexWithAlias(knex, endpoint, tableAliases) - const parsedBody = parseBody(body) - query = this.addFilters(query, filters, json.meta.table, { - columnPrefix: json.meta.columnPrefix, - aliases: tableAliases, - }) + update(opts: QueryOptions): Knex.QueryBuilder { + const { body, filters } = this.query + let query = this.qualifiedKnex() + const parsedBody = this.parseBody(body) + query = this.addFilters(query, filters) // mysql can't use returning if (opts.disableReturning) { return query.update(parsedBody) @@ -818,18 +986,15 @@ class InternalBuilder { } } - delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { - const { endpoint, filters, tableAliases } = json - let query = this.knexWithAlias(knex, endpoint, tableAliases) - query = this.addFilters(query, filters, json.meta.table, { - columnPrefix: json.meta.columnPrefix, - aliases: tableAliases, - }) + delete(opts: QueryOptions): Knex.QueryBuilder { + const { filters } = this.query + let query = this.qualifiedKnex() + query = this.addFilters(query, filters) // mysql can't use returning if (opts.disableReturning) { return query.delete() } else { - return query.delete().returning(generateSelectStatement(json, knex)) + return query.delete().returning(this.generateSelectStatement()) } } } @@ -838,7 +1003,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { private readonly limit: number // pass through client to get flavour of SQL - constructor(client: SqlClient, limit: number = BASE_LIMIT) { + constructor(client: SqlClient, limit: number = getBaseLimit()) { super(client) this.limit = limit } @@ -867,40 +1032,40 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { const config: Knex.Config = { client: sqlClient, } - if (sqlClient === SqlClient.SQL_LITE) { + if (sqlClient === SqlClient.SQL_LITE || sqlClient === SqlClient.ORACLE) { config.useNullAsDefault = true } const client = knex(config) let query: Knex.QueryBuilder - const builder = new InternalBuilder(sqlClient) + const builder = new InternalBuilder(sqlClient, client, json) switch (this._operation(json)) { case Operation.CREATE: - query = builder.create(client, json, opts) + query = builder.create(opts) break case Operation.READ: - query = builder.read(client, json, { + query = builder.read({ limits: { query: this.limit, - base: BASE_LIMIT, + base: getBaseLimit(), }, }) break case Operation.COUNT: // read without any limits to count - query = builder.read(client, json) + query = builder.read() break case Operation.UPDATE: - query = builder.update(client, json, opts) + query = builder.update(opts) break case Operation.DELETE: - query = builder.delete(client, json, opts) + query = builder.delete(opts) break case Operation.BULK_CREATE: - query = builder.bulkCreate(client, json) + query = builder.bulkCreate() break case Operation.BULK_UPSERT: - query = builder.bulkUpsert(client, json) + query = builder.bulkUpsert() break case Operation.CREATE_TABLE: case Operation.UPDATE_TABLE: diff --git a/packages/backend-core/src/sql/sqlStatements.ts b/packages/backend-core/src/sql/sqlStatements.ts deleted file mode 100644 index 311f7c7d49..0000000000 --- a/packages/backend-core/src/sql/sqlStatements.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { FieldType, Table, FieldSchema, SqlClient } from "@budibase/types" -import { Knex } from "knex" - -export class SqlStatements { - client: string - table: Table - allOr: boolean | undefined - columnPrefix: string | undefined - - constructor( - client: string, - table: Table, - { allOr, columnPrefix }: { allOr?: boolean; columnPrefix?: string } = {} - ) { - this.client = client - this.table = table - this.allOr = allOr - this.columnPrefix = columnPrefix - } - - getField(key: string): FieldSchema | undefined { - const fieldName = key.split(".")[1] - let found = this.table.schema[fieldName] - if (!found && this.columnPrefix) { - const prefixRemovedFieldName = fieldName.replace(this.columnPrefix, "") - found = this.table.schema[prefixRemovedFieldName] - } - return found - } - - between( - query: Knex.QueryBuilder, - key: string, - low: number | string, - high: number | string - ) { - // Use a between operator if we have 2 valid range values - const field = this.getField(key) - if ( - field?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw( - `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, - [low, high] - ) - } else { - const fnc = this.allOr ? "orWhereBetween" : "whereBetween" - query = query[fnc](key, [low, high]) - } - return query - } - - lte(query: Knex.QueryBuilder, key: string, low: number | string) { - // Use just a single greater than operator if we only have a low - const field = this.getField(key) - if ( - field?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [ - low, - ]) - } else { - const fnc = this.allOr ? "orWhere" : "where" - query = query[fnc](key, ">=", low) - } - return query - } - - gte(query: Knex.QueryBuilder, key: string, high: number | string) { - const field = this.getField(key) - // Use just a single less than operator if we only have a high - if ( - field?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [ - high, - ]) - } else { - const fnc = this.allOr ? "orWhere" : "where" - query = query[fnc](key, "<=", high) - } - return query - } -} diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts index 67b5d2081b..1b32cc6da7 100644 --- a/packages/backend-core/src/sql/utils.ts +++ b/packages/backend-core/src/sql/utils.ts @@ -22,6 +22,7 @@ export function getNativeSql( query: Knex.SchemaBuilder | Knex.QueryBuilder ): SqlQuery | SqlQuery[] { let sql = query.toSQL() + if (Array.isArray(sql)) { return sql as SqlQuery[] } diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 4865ebb5bc..c96c615f4b 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -18,6 +18,9 @@ import { User, UserStatus, UserGroup, + PlatformUserBySsoId, + PlatformUserById, + AnyDocument, } from "@budibase/types" import { getAccountHolderFromUserIds, @@ -25,7 +28,11 @@ import { isCreator, validateUniqueUser, } from "./utils" -import { searchExistingEmails } from "./lookup" +import { + getFirstPlatformUser, + getPlatformUsers, + searchExistingEmails, +} from "./lookup" import { hash } from "../utils" import { validatePassword } from "../security" @@ -446,9 +453,32 @@ export class UserDB { creator => !!creator ).length + const ssoUsersToDelete: AnyDocument[] = [] for (let user of usersToDelete) { + const platformUser = (await getFirstPlatformUser( + user._id! + )) as PlatformUserById + const ssoId = platformUser.ssoId + if (ssoId) { + // Need to get the _rev of the SSO user doc to delete it. The view also returns docs that have the ssoId property, so we need to ignore those. + const ssoUsers = (await getPlatformUsers( + ssoId + )) as PlatformUserBySsoId[] + ssoUsers + .filter(user => user.ssoId == null) + .forEach(user => { + ssoUsersToDelete.push({ + ...user, + _deleted: true, + }) + }) + } await bulkDeleteProcessing(user) } + + // Delete any associated SSO user docs + await platform.getPlatformDB().bulkDocs(ssoUsersToDelete) + await UserDB.quotas.removeUsers(toDelete.length, creatorsToDeleteCount) // Build Response diff --git a/packages/backend-core/src/users/lookup.ts b/packages/backend-core/src/users/lookup.ts index 355be74dab..5324ba950f 100644 --- a/packages/backend-core/src/users/lookup.ts +++ b/packages/backend-core/src/users/lookup.ts @@ -34,15 +34,22 @@ export async function searchExistingEmails(emails: string[]) { } // lookup, could be email or userId, either will return a doc -export async function getPlatformUser( +export async function getPlatformUsers( identifier: string -): Promise { +): Promise { // use the view here and allow to find anyone regardless of casing // Use lowercase to ensure email login is case insensitive - return (await dbUtils.queryPlatformView(ViewName.PLATFORM_USERS_LOWERCASE, { + return await dbUtils.queryPlatformView(ViewName.PLATFORM_USERS_LOWERCASE, { keys: [identifier.toLowerCase()], include_docs: true, - })) as PlatformUser + }) +} + +export async function getFirstPlatformUser( + identifier: string +): Promise { + const platformUserDocs = await getPlatformUsers(identifier) + return platformUserDocs[0] ?? null } export async function getExistingTenantUsers( @@ -74,15 +81,10 @@ export async function getExistingPlatformUsers( keys: lcEmails, include_docs: true, } - - const opts = { - arrayResponse: true, - } - return (await dbUtils.queryPlatformView( + return await dbUtils.queryPlatformView( ViewName.PLATFORM_USERS_LOWERCASE, - params, - opts - )) as PlatformUserByEmail[] + params + ) } export async function getExistingAccounts( @@ -93,14 +95,5 @@ export async function getExistingAccounts( keys: lcEmails, include_docs: true, } - - const opts = { - arrayResponse: true, - } - - return (await dbUtils.queryPlatformView( - ViewName.ACCOUNT_BY_EMAIL, - params, - opts - )) as AccountMetadata[] + return await dbUtils.queryPlatformView(ViewName.ACCOUNT_BY_EMAIL, params) } diff --git a/packages/backend-core/src/users/utils.ts b/packages/backend-core/src/users/utils.ts index 348ad1532f..e1e3da181d 100644 --- a/packages/backend-core/src/users/utils.ts +++ b/packages/backend-core/src/users/utils.ts @@ -1,7 +1,7 @@ import { CloudAccount, ContextUser, User, UserGroup } from "@budibase/types" import * as accountSdk from "../accounts" import env from "../environment" -import { getPlatformUser } from "./lookup" +import { getFirstPlatformUser } from "./lookup" import { EmailUnavailableError } from "../errors" import { getTenantId } from "../context" import { sdk } from "@budibase/shared-core" @@ -51,7 +51,7 @@ async function isCreatorByGroupMembership(user?: User | ContextUser) { export async function validateUniqueUser(email: string, tenantId: string) { // check budibase users in other tenants if (env.MULTI_TENANCY) { - const tenantUser = await getPlatformUser(email) + const tenantUser = await getFirstPlatformUser(email) if (tenantUser != null && tenantUser.tenantId !== tenantId) { throw new EmailUnavailableError(email) } diff --git a/packages/backend-core/tests/core/utilities/jestUtils.ts b/packages/backend-core/tests/core/utilities/jestUtils.ts index a49c2a795e..683a4e025b 100644 --- a/packages/backend-core/tests/core/utilities/jestUtils.ts +++ b/packages/backend-core/tests/core/utilities/jestUtils.ts @@ -1,6 +1,6 @@ import { - CONSTANT_EXTERNAL_ROW_COLS, - CONSTANT_INTERNAL_ROW_COLS, + PROTECTED_EXTERNAL_COLUMNS, + PROTECTED_INTERNAL_COLUMNS, } from "@budibase/shared-core" export function expectFunctionWasCalledTimesWith( @@ -14,7 +14,7 @@ export function expectFunctionWasCalledTimesWith( } export const expectAnyInternalColsAttributes: { - [K in (typeof CONSTANT_INTERNAL_ROW_COLS)[number]]: any + [K in (typeof PROTECTED_INTERNAL_COLUMNS)[number]]: any } = { tableId: expect.anything(), type: expect.anything(), @@ -25,7 +25,7 @@ export const expectAnyInternalColsAttributes: { } export const expectAnyExternalColsAttributes: { - [K in (typeof CONSTANT_EXTERNAL_ROW_COLS)[number]]: any + [K in (typeof PROTECTED_EXTERNAL_COLUMNS)[number]]: any } = { tableId: expect.anything(), _id: expect.anything(), diff --git a/packages/bbui/src/InlineAlert/InlineAlert.svelte b/packages/bbui/src/InlineAlert/InlineAlert.svelte index bfc56818cb..3b98936f62 100644 --- a/packages/bbui/src/InlineAlert/InlineAlert.svelte +++ b/packages/bbui/src/InlineAlert/InlineAlert.svelte @@ -36,9 +36,11 @@
{header}
- {#each split as splitMsg} -
{splitMsg}
- {/each} + + {#each split as splitMsg} +
{splitMsg}
+ {/each} + {#if onConfirm}
{ testDataModal.show() }} @@ -80,6 +81,7 @@ automation._id, automation.disabled )} + disabled={!$selectedAutomation?.definition?.trigger} value={!automation.disabled} />
diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte index 811909845a..c88317c79f 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte @@ -3,6 +3,7 @@ automationStore, selectedAutomation, permissions, + selectedAutomationDisplayData, } from "stores/builder" import { Icon, @@ -14,6 +15,7 @@ notifications, Label, AbsTooltip, + InlineAlert, } from "@budibase/bbui" import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte" import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte" @@ -49,6 +51,8 @@ $: isAppAction && setPermissions(role) $: isAppAction && getPermissions(automationId) + $: triggerInfo = $selectedAutomationDisplayData?.triggerInfo + async function setPermissions(role) { if (!role || !automationId) { return @@ -183,6 +187,12 @@ {block} {webhookModal} /> + {#if isTrigger && triggerInfo} + + {/if} {#if lastStep}
- {:else if value.customType === "filters" || value.customType === "trigger_filter"} + {:else if value.customType === AutomationCustomIOType.FILTERS || value.customType === AutomationCustomIOType.TRIGGER_FILTER} {filters.length > 0 ? "Update Filter" @@ -1021,6 +1033,7 @@ {isTrigger} value={inputData[key]} on:change={e => onChange({ [key]: e.detail })} + disabled={value.readonly} /> {:else if value.customType === "webhookUrl"} diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte index 8e3d90be41..148db7554c 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte @@ -13,7 +13,7 @@ const { datasource } = getContext("grid") - $: triggers = $automationStore.blockDefinitions.TRIGGER + $: triggers = $automationStore.blockDefinitions.CREATABLE_TRIGGER $: table = $tables.list.find(table => table._id === $datasource.tableId) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index f982ef5333..a9ea90242a 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -17,8 +17,8 @@ SWITCHABLE_TYPES, ValidColumnNameRegex, helpers, - CONSTANT_INTERNAL_ROW_COLS, - CONSTANT_EXTERNAL_ROW_COLS, + PROTECTED_INTERNAL_COLUMNS, + PROTECTED_EXTERNAL_COLUMNS, } from "@budibase/shared-core" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" @@ -489,8 +489,8 @@ } const newError = {} const prohibited = externalTable - ? CONSTANT_EXTERNAL_ROW_COLS - : CONSTANT_INTERNAL_ROW_COLS + ? PROTECTED_EXTERNAL_COLUMNS + : PROTECTED_INTERNAL_COLUMNS if (!externalTable && fieldInfo.name?.startsWith("_")) { newError.name = `Column name cannot start with an underscore.` } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) { diff --git a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavItem/DeleteConfirmationModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavItem/DeleteConfirmationModal.svelte index 13380c2700..3d36d1e00e 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavItem/DeleteConfirmationModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavItem/DeleteConfirmationModal.svelte @@ -33,6 +33,5 @@ title="Confirm Deletion" > Are you sure you wish to delete the datasource - {datasource.name}? - This action cannot be undone. + {datasource.name}? This action cannot be undone. diff --git a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte index 80655d1099..f1d85a6a30 100644 --- a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte @@ -100,51 +100,43 @@ async function handleFile(e) { loading = true error = null + const previousValidation = validation validation = {} try { const response = await parseFile(e) rows = response.rows fileName = response.fileName + + const newValidateHash = JSON.stringify(rows) + if (newValidateHash === validateHash) { + validation = previousValidation + } else { + await validate(rows) + validateHash = newValidateHash + } } catch (e) { + error = e.message || e + } finally { loading = false - error = e } } async function validate(rows) { - loading = true error = null validation = {} allValid = false - try { - if (rows.length > 0) { - const response = await API.validateExistingTableImport({ - rows, - tableId, - }) + if (rows.length > 0) { + const response = await API.validateExistingTableImport({ + rows, + tableId, + }) - validation = response.schemaValidation - invalidColumns = response.invalidColumns - allValid = response.allValid - } - } catch (e) { - error = e.message + validation = response.schemaValidation + invalidColumns = response.invalidColumns + allValid = response.allValid } - - loading = false - } - - $: { - // binding in consumer is causing double renders here - const newValidateHash = JSON.stringify(rows) - - if (newValidateHash !== validateHash) { - validate(rows) - } - - validateHash = newValidateHash } diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte index 5bc3b9e728..4c9f4dd10f 100644 --- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte @@ -1,9 +1,9 @@ -

- Are you sure you wish to delete the table - {table.name}? - The following will also be deleted: -

- -
- {#each willBeDeleted as item} -
{item}
- {/each} -
- -

- This action cannot be undone - to continue please enter the table name below - to confirm. -

- +
+

+ Are you sure you wish to delete the table + + + + {table.name} + ? + +

+ +

All table data will be deleted{viewsMessage}.

+

This action cannot be undone.

+ + {#if screensPossiblyAffected.length > 0} +
+ +
    + {#each screensPossiblyAffected as item} +
  • + {item.text} +
    • + {/each} +
+ +
+ {/if} +

Please enter the app name below to confirm.

+ +
diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte index 098369d3b4..b62c8af03d 100644 --- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte +++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte @@ -78,7 +78,7 @@ await datasources.fetch() await afterSave(table) } catch (e) { - notifications.error(e) + notifications.error(e.message || e) // reload in case the table was created await tables.fetch() } diff --git a/packages/builder/src/components/common/RoleIcon.svelte b/packages/builder/src/components/common/RoleIcon.svelte new file mode 100644 index 0000000000..1bd6ba49bc --- /dev/null +++ b/packages/builder/src/components/common/RoleIcon.svelte @@ -0,0 +1,12 @@ + + + diff --git a/packages/builder/src/components/common/users/PasswordRepeatInput.svelte b/packages/builder/src/components/common/users/PasswordRepeatInput.svelte index 4a453ef049..9aa9000720 100644 --- a/packages/builder/src/components/common/users/PasswordRepeatInput.svelte +++ b/packages/builder/src/components/common/users/PasswordRepeatInput.svelte @@ -1,20 +1,32 @@ - - + - - + diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index d2b5174139..99b5ace847 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -115,6 +115,7 @@ }) $: fields = bindings .filter(x => arrayTypes.includes(x.fieldSchema?.type)) + .filter(x => x.fieldSchema?.tableId != null) .map(binding => { const { providerId, readableBinding, runtimeBinding } = binding const { name, type, tableId } = binding.fieldSchema diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index ea8d35704f..6fbc36afe2 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -9,7 +9,10 @@ import { Constants } from "@budibase/frontend-core" const { TypeIconMap } = Constants -export { RelationshipType } from "@budibase/types" +export { + RelationshipType, + RowExportFormat as ROW_EXPORT_FORMATS, +} from "@budibase/types" export const AUTO_COLUMN_SUB_TYPES = AutoFieldSubType @@ -307,9 +310,3 @@ export const DatasourceTypes = { GRAPH: "Graph", API: "API", } - -export const ROW_EXPORT_FORMATS = { - CSV: "csv", - JSON: "json", - JSON_WITH_SCHEMA: "jsonWithSchema", -} diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte index 68d74218c8..7c2775e054 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte @@ -1,108 +1,88 @@ - - - - - { - confirmScreenCreation() - }} - bind:screenAccessRole - onCancel={roleSelectBack} - screenUrl={blankScreenUrl} - confirmText={screenMode === "form" ? "Confirm" : "Done"} + - + { formTypeModal.hide() datasourceModal.show() }} - on:select={e => { - formType = e.detail - }} - type={formType} /> diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte index 8e2428d832..a6bfbfd3d3 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte @@ -1,42 +1,95 @@ - - - -
- - {datasource.label} - {#if selected} - - - - {/if} -
- - diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/FormTypeModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/FormTypeModal.svelte index 6183f682b0..b7108fd9aa 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/FormTypeModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/FormTypeModal.svelte @@ -1,12 +1,10 @@ @@ -14,7 +12,7 @@ title="Select form type" confirmText="Done" cancelText="Back" - {onConfirm} + onConfirm={() => onConfirm(type)} {onCancel} disabled={!type} size="L" @@ -25,9 +23,7 @@
{ - dispatch("select", "Create") - }} + on:click={() => (type = "Create")} >
@@ -46,9 +42,7 @@
{ - dispatch("select", "Update") - }} + on:click={() => (type = "Update")} >
@@ -65,9 +59,7 @@
{ - dispatch("select", "View") - }} + on:click={() => (type = "View")} >
diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte deleted file mode 100644 index 8605412c1c..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte +++ /dev/null @@ -1,62 +0,0 @@ - - - - Select the level of access required to see these screens -