diff --git a/.eslintrc.json b/.eslintrc.json index 3de9d13046..ae9512152f 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -44,7 +44,8 @@ "no-undef": "off", "no-prototype-builtins": "off", "local-rules/no-budibase-imports": "error", - "local-rules/no-test-com": "error" + "local-rules/no-test-com": "error", + "local-rules/email-domain-example-com": "error" } }, { diff --git a/eslint-local-rules/index.js b/eslint-local-rules/index.js index 71bb5068da..177b0a129c 100644 --- a/eslint-local-rules/index.js +++ b/eslint-local-rules/index.js @@ -51,4 +51,41 @@ module.exports = { } }, }, + "email-domain-example-com": { + meta: { + type: "problem", + docs: { + description: + "enforce using the example.com domain for generator.email calls", + category: "Possible Errors", + recommended: false, + }, + fixable: "code", + schema: [], + }, + create: function (context) { + return { + CallExpression(node) { + if ( + node.callee.type === "MemberExpression" && + node.callee.object.name === "generator" && + node.callee.property.name === "email" && + node.arguments.length === 0 + ) { + context.report({ + node, + message: + "Prefer using generator.email with the domain \"{ domain: 'example.com' }\".", + fix: function (fixer) { + return fixer.replaceText( + node, + 'generator.email({ domain: "example.com" })' + ) + }, + }) + } + }, + } + }, + }, } diff --git a/lerna.json b/lerna.json index 6fb032ac77..57e3a7b34e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.21.2", + "version": "2.21.6", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 3f8c34f823..fe56780982 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -67,7 +67,7 @@ "@types/lodash": "4.14.200", "@types/node-fetch": "2.6.4", "@types/pouchdb": "6.4.0", - "@types/redlock": "4.0.3", + "@types/redlock": "4.0.7", "@types/semver": "7.3.7", "@types/tar-fs": "2.0.1", "@types/uuid": "8.3.4", @@ -78,6 +78,7 @@ "jest-serial-runner": "1.2.1", "pino-pretty": "10.0.0", "pouchdb-adapter-memory": "7.2.2", + "testcontainers": "^10.7.2", "timekeeper": "2.2.0", "typescript": "5.2.2" }, diff --git a/packages/backend-core/src/cache/base/index.ts b/packages/backend-core/src/cache/base/index.ts index 264984c6a5..74da4fe0d2 100644 --- a/packages/backend-core/src/cache/base/index.ts +++ b/packages/backend-core/src/cache/base/index.ts @@ -23,6 +23,18 @@ export default class BaseCache { return client.keys(pattern) } + async exists(key: string, opts = { useTenancy: true }) { + key = opts.useTenancy ? generateTenantKey(key) : key + const client = await this.getClient() + return client.exists(key) + } + + async scan(key: string, opts = { useTenancy: true }) { + key = opts.useTenancy ? generateTenantKey(key) : key + const client = await this.getClient() + return client.scan(key) + } + /** * Read only from the cache. */ @@ -32,6 +44,15 @@ export default class BaseCache { return client.get(key) } + /** + * Read only from the cache. + */ + async bulkGet(keys: string[], opts = { useTenancy: true }) { + keys = opts.useTenancy ? keys.map(key => generateTenantKey(key)) : keys + const client = await this.getClient() + return client.bulkGet(keys) + } + /** * Write to the cache. */ @@ -46,6 +67,25 @@ export default class BaseCache { await client.store(key, value, ttl) } + /** + * Bulk write to the cache. + */ + async bulkStore( + data: Record, + ttl: number | null = null, + opts = { useTenancy: true } + ) { + if (opts.useTenancy) { + data = Object.entries(data).reduce((acc, [key, value]) => { + acc[generateTenantKey(key)] = value + return acc + }, {} as Record) + } + + const client = await this.getClient() + await client.bulkStore(data, ttl) + } + /** * Remove from cache. */ @@ -55,15 +95,24 @@ export default class BaseCache { return client.delete(key) } + /** + * Remove from cache. + */ + async bulkDelete(keys: string[], opts = { useTenancy: true }) { + keys = opts.useTenancy ? keys.map(key => generateTenantKey(key)) : keys + const client = await this.getClient() + return client.bulkDelete(keys) + } + /** * Read from the cache. Write to the cache if not exists. */ - async withCache( + async withCache( key: string, - ttl: number, - fetchFn: any, + ttl: number | null = null, + fetchFn: () => Promise | T, opts = { useTenancy: true } - ) { + ): Promise { const cachedValue = await this.get(key, opts) if (cachedValue) { return cachedValue @@ -89,4 +138,13 @@ export default class BaseCache { throw err } } + + /** + * Delete the entry if the provided value matches the stored one. + */ + async deleteIfValue(key: string, value: any, opts = { useTenancy: true }) { + key = opts.useTenancy ? generateTenantKey(key) : key + const client = await this.getClient() + await client.deleteIfValue(key, value) + } } diff --git a/packages/backend-core/src/cache/docWritethrough.ts b/packages/backend-core/src/cache/docWritethrough.ts new file mode 100644 index 0000000000..1b129bb26a --- /dev/null +++ b/packages/backend-core/src/cache/docWritethrough.ts @@ -0,0 +1,97 @@ +import { AnyDocument, Database } from "@budibase/types" + +import { JobQueue, createQueue } from "../queue" +import * as dbUtils from "../db" + +interface ProcessDocMessage { + dbName: string + docId: string + data: Record +} + +const PERSIST_MAX_ATTEMPTS = 100 +let processor: DocWritethroughProcessor | undefined + +export const docWritethroughProcessorQueue = createQueue( + JobQueue.DOC_WRITETHROUGH_QUEUE, + { + jobOptions: { + attempts: PERSIST_MAX_ATTEMPTS, + }, + } +) + +class DocWritethroughProcessor { + init() { + docWritethroughProcessorQueue.process(async message => { + try { + await this.persistToDb(message.data) + } catch (err: any) { + if (err.status === 409) { + // If we get a 409, it means that another job updated it meanwhile. We want to retry it to persist it again. + throw new Error( + `Conflict persisting message ${message.id}. Attempt ${message.attemptsMade}` + ) + } + + throw err + } + }) + return this + } + + private async persistToDb({ + dbName, + docId, + data, + }: { + dbName: string + docId: string + data: Record + }) { + const db = dbUtils.getDB(dbName) + let doc: AnyDocument | undefined + try { + doc = await db.get(docId) + } catch { + doc = { _id: docId } + } + + doc = { ...doc, ...data } + await db.put(doc) + } +} + +export class DocWritethrough { + private db: Database + private _docId: string + + constructor(db: Database, docId: string) { + this.db = db + this._docId = docId + } + + get docId() { + return this._docId + } + + async patch(data: Record) { + await docWritethroughProcessorQueue.add({ + dbName: this.db.name, + docId: this.docId, + data, + }) + } +} + +export function init(): DocWritethroughProcessor { + processor = new DocWritethroughProcessor().init() + return processor +} + +export function getProcessor(): DocWritethroughProcessor { + if (!processor) { + return init() + } + return processor +} diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts index 3ac323a8d4..2d6d8b9472 100644 --- a/packages/backend-core/src/cache/generic.ts +++ b/packages/backend-core/src/cache/generic.ts @@ -26,7 +26,8 @@ export const store = (...args: Parameters) => GENERIC.store(...args) export const destroy = (...args: Parameters) => GENERIC.delete(...args) -export const withCache = (...args: Parameters) => - GENERIC.withCache(...args) +export const withCache = ( + ...args: Parameters> +) => GENERIC.withCache(...args) export const bustCache = (...args: Parameters) => GENERIC.bustCache(...args) diff --git a/packages/backend-core/src/cache/index.ts b/packages/backend-core/src/cache/index.ts index 4fa986e4e2..3b25108634 100644 --- a/packages/backend-core/src/cache/index.ts +++ b/packages/backend-core/src/cache/index.ts @@ -5,3 +5,4 @@ export * as writethrough from "./writethrough" export * as invite from "./invite" export * as passwordReset from "./passwordReset" export * from "./generic" +export * as docWritethrough from "./docWritethrough" diff --git a/packages/backend-core/src/cache/tests/docWritethrough.spec.ts b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts new file mode 100644 index 0000000000..5fe09b95ff --- /dev/null +++ b/packages/backend-core/src/cache/tests/docWritethrough.spec.ts @@ -0,0 +1,293 @@ +import tk from "timekeeper" + +import _ from "lodash" +import { DBTestConfiguration, generator, structures } from "../../../tests" +import { getDB } from "../../db" + +import { + DocWritethrough, + docWritethroughProcessorQueue, + init, +} from "../docWritethrough" + +import InMemoryQueue from "../../queue/inMemoryQueue" + +const initialTime = Date.now() + +async function waitForQueueCompletion() { + const queue: InMemoryQueue = docWritethroughProcessorQueue as never + await queue.waitForCompletion() +} + +describe("docWritethrough", () => { + beforeAll(() => { + init() + }) + + const config = new DBTestConfiguration() + + const db = getDB(structures.db.id()) + let documentId: string + let docWritethrough: DocWritethrough + + describe("patch", () => { + function generatePatchObject(fieldCount: number) { + const keys = generator.unique(() => generator.word(), fieldCount) + return keys.reduce((acc, c) => { + acc[c] = generator.word() + return acc + }, {} as Record) + } + + beforeEach(async () => { + jest.clearAllMocks() + documentId = structures.uuid() + docWritethrough = new DocWritethrough(db, documentId) + }) + + it("patching will not persist until the messages are persisted", async () => { + await config.doInTenant(async () => { + await docWritethrough.patch(generatePatchObject(2)) + await docWritethrough.patch(generatePatchObject(2)) + + expect(await db.exists(documentId)).toBe(false) + }) + }) + + it("patching will persist when the messages are persisted", async () => { + await config.doInTenant(async () => { + const patch1 = generatePatchObject(2) + const patch2 = generatePatchObject(2) + await docWritethrough.patch(patch1) + await docWritethrough.patch(patch2) + + await waitForQueueCompletion() + + // This will not be persisted + const patch3 = generatePatchObject(3) + await docWritethrough.patch(patch3) + + expect(await db.get(documentId)).toEqual({ + _id: documentId, + ...patch1, + ...patch2, + _rev: expect.stringMatching(/2-.+/), + createdAt: new Date(initialTime).toISOString(), + updatedAt: new Date(initialTime).toISOString(), + }) + }) + }) + + it("patching will persist keeping the previous data", async () => { + await config.doInTenant(async () => { + const patch1 = generatePatchObject(2) + const patch2 = generatePatchObject(2) + await docWritethrough.patch(patch1) + await docWritethrough.patch(patch2) + + await waitForQueueCompletion() + + const patch3 = generatePatchObject(3) + await docWritethrough.patch(patch3) + + await waitForQueueCompletion() + + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ + _id: documentId, + ...patch1, + ...patch2, + ...patch3, + }) + ) + }) + }) + + it("date audit fields are set correctly when persisting", async () => { + await config.doInTenant(async () => { + const patch1 = generatePatchObject(2) + const patch2 = generatePatchObject(2) + await docWritethrough.patch(patch1) + const date1 = new Date() + await waitForQueueCompletion() + await docWritethrough.patch(patch2) + + tk.travel(Date.now() + 100) + const date2 = new Date() + await waitForQueueCompletion() + + expect(date1).not.toEqual(date2) + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ + createdAt: date1.toISOString(), + updatedAt: date2.toISOString(), + }) + ) + }) + }) + + it("concurrent patches will override keys", async () => { + await config.doInTenant(async () => { + const patch1 = generatePatchObject(2) + await docWritethrough.patch(patch1) + await waitForQueueCompletion() + const patch2 = generatePatchObject(1) + await docWritethrough.patch(patch2) + + const keyToOverride = _.sample(Object.keys(patch1))! + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ + [keyToOverride]: patch1[keyToOverride], + }) + ) + + await waitForQueueCompletion() + + const patch3 = { + ...generatePatchObject(3), + [keyToOverride]: generator.word(), + } + await docWritethrough.patch(patch3) + await waitForQueueCompletion() + + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ + ...patch1, + ...patch2, + ...patch3, + }) + ) + }) + }) + + it("concurrent patches to different docWritethrough will not pollute each other", async () => { + await config.doInTenant(async () => { + const secondDocWritethrough = new DocWritethrough( + db, + structures.db.id() + ) + + const doc1Patch = generatePatchObject(2) + await docWritethrough.patch(doc1Patch) + const doc2Patch = generatePatchObject(1) + await secondDocWritethrough.patch(doc2Patch) + + await waitForQueueCompletion() + + const doc1Patch2 = generatePatchObject(3) + await docWritethrough.patch(doc1Patch2) + const doc2Patch2 = generatePatchObject(3) + await secondDocWritethrough.patch(doc2Patch2) + await waitForQueueCompletion() + + expect(await db.get(docWritethrough.docId)).toEqual( + expect.objectContaining({ + ...doc1Patch, + ...doc1Patch2, + }) + ) + + expect(await db.get(secondDocWritethrough.docId)).toEqual( + expect.objectContaining({ + ...doc2Patch, + ...doc2Patch2, + }) + ) + }) + }) + + it("cached values are persisted only once", async () => { + await config.doInTenant(async () => { + const initialPatch = generatePatchObject(5) + + await docWritethrough.patch(initialPatch) + await waitForQueueCompletion() + + expect(await db.get(documentId)).toEqual( + expect.objectContaining(initialPatch) + ) + + await db.remove(await db.get(documentId)) + + await waitForQueueCompletion() + const extraPatch = generatePatchObject(5) + await docWritethrough.patch(extraPatch) + await waitForQueueCompletion() + + expect(await db.get(documentId)).toEqual( + expect.objectContaining(extraPatch) + ) + expect(await db.get(documentId)).not.toEqual( + expect.objectContaining(initialPatch) + ) + }) + }) + + it("concurrent calls will not cause conflicts", async () => { + async function parallelPatch(count: number) { + const patches = Array.from({ length: count }).map(() => + generatePatchObject(1) + ) + await Promise.all(patches.map(p => docWritethrough.patch(p))) + + return patches.reduce((acc, c) => { + acc = { ...acc, ...c } + return acc + }, {}) + } + const queueMessageSpy = jest.spyOn(docWritethroughProcessorQueue, "add") + + await config.doInTenant(async () => { + let patches = await parallelPatch(5) + expect(queueMessageSpy).toBeCalledTimes(5) + + await waitForQueueCompletion() + expect(await db.get(documentId)).toEqual( + expect.objectContaining(patches) + ) + + patches = { ...patches, ...(await parallelPatch(40)) } + expect(queueMessageSpy).toBeCalledTimes(45) + + await waitForQueueCompletion() + expect(await db.get(documentId)).toEqual( + expect.objectContaining(patches) + ) + + patches = { ...patches, ...(await parallelPatch(10)) } + expect(queueMessageSpy).toBeCalledTimes(55) + + await waitForQueueCompletion() + expect(await db.get(documentId)).toEqual( + expect.objectContaining(patches) + ) + }) + }) + + // This is not yet supported + it.skip("patches will execute in order", async () => { + let incrementalValue = 0 + const keyToOverride = generator.word() + async function incrementalPatches(count: number) { + for (let i = 0; i < count; i++) { + await docWritethrough.patch({ [keyToOverride]: incrementalValue++ }) + } + } + + await config.doInTenant(async () => { + await incrementalPatches(5) + + await waitForQueueCompletion() + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ [keyToOverride]: 5 }) + ) + + await incrementalPatches(40) + await waitForQueueCompletion() + expect(await db.get(documentId)).toEqual( + expect.objectContaining({ [keyToOverride]: 45 }) + ) + }) + }) + }) +}) diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index 313b9a4d4a..ecfa20f99e 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -6,7 +6,7 @@ import env from "../environment" import * as accounts from "../accounts" import { UserDB } from "../users" import { sdk } from "@budibase/shared-core" -import { User } from "@budibase/types" +import { User, UserMetadata } from "@budibase/types" const EXPIRY_SECONDS = 3600 @@ -15,7 +15,7 @@ const EXPIRY_SECONDS = 3600 */ async function populateFromDB(userId: string, tenantId: string) { const db = tenancy.getTenantDB(tenantId) - const user = await db.get(userId) + const user = await db.get(userId) user.budibaseAccess = true if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { const account = await accounts.getAccount(user.email) diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index ac00483021..f4caac502e 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -57,6 +57,9 @@ export const StaticDatabases = { AUDIT_LOGS: { name: "audit-logs", }, + SCIM_LOGS: { + name: "scim-logs", + }, } export const APP_PREFIX = prefixed(DocumentType.APP) diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 36fd5dcb48..ae86695168 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -35,6 +35,17 @@ export function getAuditLogDBName(tenantId?: string) { } } +export function getScimDBName(tenantId?: string) { + if (!tenantId) { + tenantId = getTenantId() + } + if (tenantId === DEFAULT_TENANT_ID) { + return StaticDatabases.SCIM_LOGS.name + } else { + return `${tenantId}${SEPARATOR}${StaticDatabases.SCIM_LOGS.name}` + } +} + export function baseGlobalDBName(tenantId: string | undefined | null) { if (!tenantId || tenantId === DEFAULT_TENANT_ID) { return StaticDatabases.GLOBAL.name diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index f91a37ce8f..9c960d76dd 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,66 +1,57 @@ +import PouchDB from "pouchdb" import { getPouchDB, closePouchDB } from "./couch" import { DocumentType } from "../constants" class Replication { - source: any - target: any - replication: any + source: PouchDB.Database + target: PouchDB.Database - /** - * - * @param source - the DB you want to replicate or rollback to - * @param target - the DB you want to replicate to, or rollback from - */ - constructor({ source, target }: any) { + constructor({ source, target }: { source: string; target: string }) { this.source = getPouchDB(source) this.target = getPouchDB(target) } - close() { - return Promise.all([closePouchDB(this.source), closePouchDB(this.target)]) + async close() { + await Promise.all([closePouchDB(this.source), closePouchDB(this.target)]) } - promisify(operation: any, opts = {}) { - return new Promise(resolve => { - operation(this.target, opts) - .on("denied", function (err: any) { + replicate(opts: PouchDB.Replication.ReplicateOptions = {}) { + return new Promise>(resolve => { + this.source.replicate + .to(this.target, opts) + .on("denied", function (err) { // a document failed to replicate (e.g. due to permissions) throw new Error(`Denied: Document failed to replicate ${err}`) }) - .on("complete", function (info: any) { + .on("complete", function (info) { return resolve(info) }) - .on("error", function (err: any) { + .on("error", function (err) { throw new Error(`Replication Error: ${err}`) }) }) } - /** - * Two way replication operation, intended to be promise based. - * @param opts - PouchDB replication options - */ - sync(opts = {}) { - this.replication = this.promisify(this.source.sync, opts) - return this.replication - } + appReplicateOpts( + opts: PouchDB.Replication.ReplicateOptions = {} + ): PouchDB.Replication.ReplicateOptions { + if (typeof opts.filter === "string") { + return opts + } - /** - * One way replication operation, intended to be promise based. - * @param opts - PouchDB replication options - */ - replicate(opts = {}) { - this.replication = this.promisify(this.source.replicate.to, opts) - return this.replication - } + const filter = opts.filter + delete opts.filter - appReplicateOpts() { return { - filter: (doc: any) => { + ...opts, + filter: (doc: any, params: any) => { if (doc._id && doc._id.startsWith(DocumentType.AUTOMATION_LOG)) { return false } - return doc._id !== DocumentType.APP_METADATA + if (doc._id === DocumentType.APP_METADATA) { + return false + } + return filter ? filter(doc, params) : true }, } } @@ -75,10 +66,6 @@ class Replication { // take the opportunity to remove deleted tombstones await this.replicate() } - - cancel() { - this.replication.cancel() - } } export default Replication diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 7e7c997cbe..416313f520 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -70,7 +70,15 @@ export class DatabaseImpl implements Database { DatabaseImpl.nano = buildNano(couchInfo) } - async exists() { + exists(docId?: string) { + if (docId === undefined) { + return this.dbExists() + } + + return this.docExists(docId) + } + + private async dbExists() { const response = await directCouchUrlCall({ url: `${this.couchInfo.url}/${this.name}`, method: "HEAD", @@ -79,6 +87,15 @@ export class DatabaseImpl implements Database { return response.status === 200 } + private async docExists(id: string): Promise { + try { + await this.performCall(db => () => db.head(id)) + return true + } catch { + return false + } + } + private nano() { return this.instanceNano || DatabaseImpl.nano } diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts index 03010d4c92..795f30d7cd 100644 --- a/packages/backend-core/src/db/instrumentation.ts +++ b/packages/backend-core/src/db/instrumentation.ts @@ -24,9 +24,12 @@ export class DDInstrumentedDatabase implements Database { return this.db.name } - exists(): Promise { + exists(docId?: string): Promise { return tracer.trace("db.exists", span => { - span?.addTags({ db_name: this.name }) + span?.addTags({ db_name: this.name, doc_id: docId }) + if (docId) { + return this.db.exists(docId) + } return this.db.exists() }) } diff --git a/packages/backend-core/src/db/tests/DatabaseImpl.spec.ts b/packages/backend-core/src/db/tests/DatabaseImpl.spec.ts new file mode 100644 index 0000000000..586f13f417 --- /dev/null +++ b/packages/backend-core/src/db/tests/DatabaseImpl.spec.ts @@ -0,0 +1,55 @@ +import _ from "lodash" +import { AnyDocument } from "@budibase/types" +import { generator } from "../../../tests" +import { DatabaseImpl } from "../couch" +import { newid } from "../../utils" + +describe("DatabaseImpl", () => { + const database = new DatabaseImpl(generator.word()) + const documents: AnyDocument[] = [] + + beforeAll(async () => { + const docsToCreate = Array.from({ length: 10 }).map(() => ({ + _id: newid(), + })) + const createdDocs = await database.bulkDocs(docsToCreate) + + documents.push(...createdDocs.map((x: any) => ({ _id: x.id, _rev: x.rev }))) + }) + + describe("document exists", () => { + it("can check existing docs by id", async () => { + const existingDoc = _.sample(documents) + const result = await database.exists(existingDoc!._id!) + + expect(result).toBe(true) + }) + + it("can check non existing docs by id", async () => { + const result = await database.exists(newid()) + + expect(result).toBe(false) + }) + + it("can check an existing doc by id multiple times", async () => { + const existingDoc = _.sample(documents) + const id = existingDoc!._id! + + const results = [] + results.push(await database.exists(id)) + results.push(await database.exists(id)) + results.push(await database.exists(id)) + + expect(results).toEqual([true, true, true]) + }) + + it("returns false after the doc is deleted", async () => { + const existingDoc = _.sample(documents) + const id = existingDoc!._id! + expect(await database.exists(id)).toBe(true) + + await database.remove(existingDoc!) + expect(await database.exists(id)).toBe(false) + }) + }) +}) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index b3179cbeea..2da2a77d67 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -186,6 +186,7 @@ const environment = { environment[key] = value }, ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M", + DISABLE_SCIM_CALLS: process.env.DISABLE_SCIM_CALLS, } // clean up any environment variable edge cases diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts index eb4f21aced..a095c6c769 100644 --- a/packages/backend-core/src/queue/constants.ts +++ b/packages/backend-core/src/queue/constants.ts @@ -4,4 +4,5 @@ export enum JobQueue { AUDIT_LOG = "auditLogQueue", SYSTEM_EVENT_QUEUE = "systemEventQueue", APP_MIGRATION = "appMigration", + DOC_WRITETHROUGH_QUEUE = "docWritethroughQueue", } diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index c05bbffbe9..afb5592562 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -1,5 +1,14 @@ import events from "events" -import { timeout } from "../utils" +import { newid, timeout } from "../utils" +import { Queue, QueueOptions, JobOptions } from "./queue" + +interface JobMessage { + id: string + timestamp: number + queue: string + data: any + opts?: JobOptions +} /** * Bull works with a Job wrapper around all messages that contains a lot more information about @@ -10,12 +19,13 @@ import { timeout } from "../utils" * @returns A new job which can now be put onto the queue, this is mostly an * internal structure so that an in memory queue can be easily swapped for a Bull queue. */ -function newJob(queue: string, message: any) { +function newJob(queue: string, message: any, opts?: JobOptions): JobMessage { return { + id: newid(), timestamp: Date.now(), queue: queue, data: message, - opts: {}, + opts, } } @@ -24,26 +34,29 @@ function newJob(queue: string, message: any) { * It is relatively simple, using an event emitter internally to register when messages are available * to the consumers - in can support many inputs and many consumers. */ -class InMemoryQueue { +class InMemoryQueue implements Partial { _name: string - _opts?: any - _messages: any[] + _opts?: QueueOptions + _messages: JobMessage[] + _queuedJobIds: Set _emitter: EventEmitter _runCount: number _addCount: number + /** * The constructor the queue, exactly the same as that of Bulls. * @param name The name of the queue which is being configured. * @param opts This is not used by the in memory queue as there is no real use * case when in memory, but is the same API as Bull */ - constructor(name: string, opts?: any) { + constructor(name: string, opts?: QueueOptions) { this._name = name this._opts = opts this._messages = [] this._emitter = new events.EventEmitter() this._runCount = 0 this._addCount = 0 + this._queuedJobIds = new Set() } /** @@ -55,22 +68,42 @@ class InMemoryQueue { * note this is incredibly limited compared to Bull as in reality the Job would contain * a lot more information about the queue and current status of Bull cluster. */ - process(func: any) { + async process(func: any) { this._emitter.on("message", async () => { if (this._messages.length <= 0) { return } let msg = this._messages.shift() + let resp = func(msg) + + async function retryFunc(fnc: any) { + try { + await fnc + } catch (e: any) { + await new Promise(r => setTimeout(() => r(), 50)) + + await retryFunc(func(msg)) + } + } + if (resp.then != null) { - await resp + try { + await retryFunc(resp) + } catch (e: any) { + console.error(e) + } } this._runCount++ + const jobId = msg?.opts?.jobId?.toString() + if (jobId && msg?.opts?.removeOnComplete) { + this._queuedJobIds.delete(jobId) + } }) } async isReady() { - return true + return this as any } // simply puts a message to the queue and emits to the queue for processing @@ -83,27 +116,45 @@ class InMemoryQueue { * @param repeat serves no purpose for the import queue. */ // eslint-disable-next-line no-unused-vars - add(msg: any, repeat: boolean) { - if (typeof msg !== "object") { + async add(data: any, opts?: JobOptions) { + const jobId = opts?.jobId?.toString() + if (jobId && this._queuedJobIds.has(jobId)) { + console.log(`Ignoring already queued job ${jobId}`) + return + } + + if (typeof data !== "object") { throw "Queue only supports carrying JSON." } - this._messages.push(newJob(this._name, msg)) - this._addCount++ - this._emitter.emit("message") + if (jobId) { + this._queuedJobIds.add(jobId) + } + + const pushMessage = () => { + this._messages.push(newJob(this._name, data, opts)) + this._addCount++ + this._emitter.emit("message") + } + + const delay = opts?.delay + if (delay) { + setTimeout(pushMessage, delay) + } else { + pushMessage() + } + return {} as any } /** * replicating the close function from bull, which waits for jobs to finish. */ - async close() { - return [] - } + async close() {} /** * This removes a cron which has been implemented, this is part of Bull API. * @param cronJobId The cron which is to be removed. */ - removeRepeatableByKey(cronJobId: string) { + async removeRepeatableByKey(cronJobId: string) { // TODO: implement for testing console.log(cronJobId) } @@ -111,12 +162,12 @@ class InMemoryQueue { /** * Implemented for tests */ - getRepeatableJobs() { + async getRepeatableJobs() { return [] } // eslint-disable-next-line no-unused-vars - removeJobs(pattern: string) { + async removeJobs(pattern: string) { // no-op } @@ -128,18 +179,22 @@ class InMemoryQueue { } async getJob() { - return {} + return null } on() { // do nothing - return this + return this as any } async waitForCompletion() { do { await timeout(50) - } while (this._addCount < this._runCount) + } while (this.hasRunningJobs()) + } + + hasRunningJobs() { + return this._addCount > this._runCount } } diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts index 063a01bd2f..14dce5fe8d 100644 --- a/packages/backend-core/src/queue/listeners.ts +++ b/packages/backend-core/src/queue/listeners.ts @@ -88,6 +88,7 @@ enum QueueEventType { AUDIT_LOG_EVENT = "audit-log-event", SYSTEM_EVENT = "system-event", APP_MIGRATION = "app-migration", + DOC_WRITETHROUGH = "doc-writethrough", } const EventTypeMap: { [key in JobQueue]: QueueEventType } = { @@ -96,6 +97,7 @@ const EventTypeMap: { [key in JobQueue]: QueueEventType } = { [JobQueue.AUDIT_LOG]: QueueEventType.AUDIT_LOG_EVENT, [JobQueue.SYSTEM_EVENT_QUEUE]: QueueEventType.SYSTEM_EVENT, [JobQueue.APP_MIGRATION]: QueueEventType.APP_MIGRATION, + [JobQueue.DOC_WRITETHROUGH_QUEUE]: QueueEventType.DOC_WRITETHROUGH, } function logging(queue: Queue, jobQueue: JobQueue) { diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 0bcb25a35f..1838eed92f 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -7,6 +7,8 @@ import { addListeners, StalledFn } from "./listeners" import { Duration } from "../utils" import * as timers from "../timers" +export { QueueOptions, Queue, JobOptions } from "bull" + // the queue lock is held for 5 minutes const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs() // queue lock is refreshed every 30 seconds diff --git a/packages/backend-core/src/redis/init.ts b/packages/backend-core/src/redis/init.ts index f3bcee3209..7920dfed2d 100644 --- a/packages/backend-core/src/redis/init.ts +++ b/packages/backend-core/src/redis/init.ts @@ -9,7 +9,8 @@ let userClient: Client, lockClient: Client, socketClient: Client, inviteClient: Client, - passwordResetClient: Client + passwordResetClient: Client, + docWritethroughClient: Client export async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() @@ -24,6 +25,9 @@ export async function init() { utils.Databases.SOCKET_IO, utils.SelectableDatabase.SOCKET_IO ).init() + docWritethroughClient = await new Client( + utils.Databases.DOC_WRITE_THROUGH + ).init() } export async function shutdown() { @@ -104,3 +108,10 @@ export async function getPasswordResetClient() { } return passwordResetClient } + +export async function getDocWritethroughClient() { + if (!writethroughClient) { + await init() + } + return writethroughClient +} diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index d15453ba62..79f75421d3 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -1,5 +1,5 @@ import env from "../environment" -import Redis from "ioredis" +import Redis, { Cluster } from "ioredis" // mock-redis doesn't have any typing let MockRedis: any | undefined if (env.MOCK_REDIS) { @@ -28,7 +28,7 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT // for testing just generate the client once let CLOSED = false -let CLIENTS: { [key: number]: any } = {} +const CLIENTS: Record = {} let CONNECTED = false // mock redis always connected @@ -36,7 +36,7 @@ if (env.MOCK_REDIS) { CONNECTED = true } -function pickClient(selectDb: number): any { +function pickClient(selectDb: number) { return CLIENTS[selectDb] } @@ -201,12 +201,15 @@ class RedisWrapper { key = `${db}${SEPARATOR}${key}` let stream if (CLUSTERED) { - let node = this.getClient().nodes("master") + let node = (this.getClient() as never as Cluster).nodes("master") stream = node[0].scanStream({ match: key + "*", count: 100 }) } else { - stream = this.getClient().scanStream({ match: key + "*", count: 100 }) + stream = (this.getClient() as Redis).scanStream({ + match: key + "*", + count: 100, + }) } - return promisifyStream(stream, this.getClient()) + return promisifyStream(stream, this.getClient() as any) } async keys(pattern: string) { @@ -221,14 +224,16 @@ class RedisWrapper { async get(key: string) { const db = this._db - let response = await this.getClient().get(addDbPrefix(db, key)) + const response = await this.getClient().get(addDbPrefix(db, key)) // overwrite the prefixed key + // @ts-ignore if (response != null && response.key) { + // @ts-ignore response.key = key } // if its not an object just return the response try { - return JSON.parse(response) + return JSON.parse(response!) } catch (err) { return response } @@ -274,13 +279,37 @@ class RedisWrapper { } } + async bulkStore( + data: Record, + expirySeconds: number | null = null + ) { + const client = this.getClient() + + const dataToStore = Object.entries(data).reduce((acc, [key, value]) => { + acc[addDbPrefix(this._db, key)] = + typeof value === "object" ? JSON.stringify(value) : value + return acc + }, {} as Record) + + const pipeline = client.pipeline() + pipeline.mset(dataToStore) + + if (expirySeconds !== null) { + for (const key of Object.keys(dataToStore)) { + pipeline.expire(key, expirySeconds) + } + } + + await pipeline.exec() + } + async getTTL(key: string) { const db = this._db const prefixedKey = addDbPrefix(db, key) return this.getClient().ttl(prefixedKey) } - async setExpiry(key: string, expirySeconds: number | null) { + async setExpiry(key: string, expirySeconds: number) { const db = this._db const prefixedKey = addDbPrefix(db, key) await this.getClient().expire(prefixedKey, expirySeconds) @@ -291,10 +320,35 @@ class RedisWrapper { await this.getClient().del(addDbPrefix(db, key)) } + async bulkDelete(keys: string[]) { + const db = this._db + await this.getClient().del(keys.map(key => addDbPrefix(db, key))) + } + async clear() { let items = await this.scan() await Promise.all(items.map((obj: any) => this.delete(obj.key))) } + + async increment(key: string) { + const result = await this.getClient().incr(addDbPrefix(this._db, key)) + if (isNaN(result)) { + throw new Error(`Redis ${key} does not contain a number`) + } + return result + } + + async deleteIfValue(key: string, value: any) { + const client = this.getClient() + + const luaScript = ` + if redis.call('GET', KEYS[1]) == ARGV[1] then + redis.call('DEL', KEYS[1]) + end + ` + + await client.eval(luaScript, 1, addDbPrefix(this._db, key), value) + } } export default RedisWrapper diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 7009dc6f55..adeb5b12ec 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -72,7 +72,7 @@ const OPTIONS: Record = { export async function newRedlock(opts: Redlock.Options = {}) { const options = { ...OPTIONS.DEFAULT, ...opts } const redisWrapper = await getLockClient() - const client = redisWrapper.getClient() + const client = redisWrapper.getClient() as any return new Redlock([client], options) } diff --git a/packages/backend-core/src/redis/tests/redis.spec.ts b/packages/backend-core/src/redis/tests/redis.spec.ts new file mode 100644 index 0000000000..c2c9e4a14e --- /dev/null +++ b/packages/backend-core/src/redis/tests/redis.spec.ts @@ -0,0 +1,214 @@ +import { GenericContainer, StartedTestContainer } from "testcontainers" +import { generator, structures } from "../../../tests" +import RedisWrapper from "../redis" +import { env } from "../.." + +jest.setTimeout(30000) + +describe("redis", () => { + let redis: RedisWrapper + let container: StartedTestContainer + + beforeAll(async () => { + const container = await new GenericContainer("redis") + .withExposedPorts(6379) + .start() + + env._set( + "REDIS_URL", + `${container.getHost()}:${container.getMappedPort(6379)}` + ) + env._set("MOCK_REDIS", 0) + env._set("REDIS_PASSWORD", 0) + }) + + afterAll(() => container?.stop()) + + beforeEach(async () => { + redis = new RedisWrapper(structures.db.id()) + await redis.init() + }) + + describe("store", () => { + it("a basic value can be persisted", async () => { + const key = structures.uuid() + const value = generator.word() + + await redis.store(key, value) + + expect(await redis.get(key)).toEqual(value) + }) + + it("objects can be persisted", async () => { + const key = structures.uuid() + const value = { [generator.word()]: generator.word() } + + await redis.store(key, value) + + expect(await redis.get(key)).toEqual(value) + }) + }) + + describe("bulkStore", () => { + function createRandomObject( + keyLength: number, + valueGenerator: () => any = () => generator.word() + ) { + return generator + .unique(() => generator.word(), keyLength) + .reduce((acc, key) => { + acc[key] = valueGenerator() + return acc + }, {} as Record) + } + + it("a basic object can be persisted", async () => { + const data = createRandomObject(10) + + await redis.bulkStore(data) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toEqual(value) + } + + expect(await redis.keys("*")).toHaveLength(10) + }) + + it("a complex object can be persisted", async () => { + const data = { + ...createRandomObject(10, () => createRandomObject(5)), + ...createRandomObject(5), + } + + await redis.bulkStore(data) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toEqual(value) + } + + expect(await redis.keys("*")).toHaveLength(15) + }) + + it("no TTL is set by default", async () => { + const data = createRandomObject(10) + + await redis.bulkStore(data) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toEqual(value) + expect(await redis.getTTL(key)).toEqual(-1) + } + }) + + it("a bulk store can be persisted with TTL", async () => { + const ttl = 500 + const data = createRandomObject(8) + + await redis.bulkStore(data, ttl) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toEqual(value) + expect(await redis.getTTL(key)).toEqual(ttl) + } + + expect(await redis.keys("*")).toHaveLength(8) + }) + + it("setting a TTL of -1 will not persist the key", async () => { + const ttl = -1 + const data = createRandomObject(5) + + await redis.bulkStore(data, ttl) + + for (const [key, value] of Object.entries(data)) { + expect(await redis.get(key)).toBe(null) + } + + expect(await redis.keys("*")).toHaveLength(0) + }) + }) + + describe("increment", () => { + it("can increment on a new key", async () => { + const key = structures.uuid() + const result = await redis.increment(key) + expect(result).toBe(1) + }) + + it("can increment multiple times", async () => { + const key = structures.uuid() + const results = [ + await redis.increment(key), + await redis.increment(key), + await redis.increment(key), + await redis.increment(key), + await redis.increment(key), + ] + expect(results).toEqual([1, 2, 3, 4, 5]) + }) + + it("can increment on a new key", async () => { + const key1 = structures.uuid() + const key2 = structures.uuid() + + const result1 = await redis.increment(key1) + expect(result1).toBe(1) + + const result2 = await redis.increment(key2) + expect(result2).toBe(1) + }) + + it("can increment multiple times in parallel", async () => { + const key = structures.uuid() + const results = await Promise.all( + Array.from({ length: 100 }).map(() => redis.increment(key)) + ) + expect(results).toHaveLength(100) + expect(results).toEqual(Array.from({ length: 100 }).map((_, i) => i + 1)) + }) + + it("can increment existing set keys", async () => { + const key = structures.uuid() + await redis.store(key, 70) + await redis.increment(key) + + const result = await redis.increment(key) + expect(result).toBe(72) + }) + + it.each([ + generator.word(), + generator.bool(), + { [generator.word()]: generator.word() }, + ])("cannot increment if the store value is not a number", async value => { + const key = structures.uuid() + await redis.store(key, value) + + await expect(redis.increment(key)).rejects.toThrowError( + "ERR value is not an integer or out of range" + ) + }) + }) + + describe("deleteIfValue", () => { + it("can delete if the value matches", async () => { + const key = structures.uuid() + const value = generator.word() + await redis.store(key, value) + + await redis.deleteIfValue(key, value) + + expect(await redis.get(key)).toBeNull() + }) + + it("will not delete if the value does not matches", async () => { + const key = structures.uuid() + const value = generator.word() + await redis.store(key, value) + + await redis.deleteIfValue(key, generator.word()) + + expect(await redis.get(key)).toEqual(value) + }) + }) +}) diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts index 7b93458b52..7f84f11467 100644 --- a/packages/backend-core/src/redis/utils.ts +++ b/packages/backend-core/src/redis/utils.ts @@ -30,6 +30,7 @@ export enum Databases { LOCKS = "locks", SOCKET_IO = "socket_io", BPM_EVENTS = "bpmEvents", + DOC_WRITE_THROUGH = "docWriteThrough", } /** diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 4f048c0a11..a64be6b319 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -84,25 +84,24 @@ export function getBuiltinRoles(): { [key: string]: RoleDoc } { return cloneDeep(BUILTIN_ROLES) } -export const BUILTIN_ROLE_ID_ARRAY = Object.values(BUILTIN_ROLES).map( - role => role._id -) +export function isBuiltin(role: string) { + return getBuiltinRole(role) !== undefined +} -export const BUILTIN_ROLE_NAME_ARRAY = Object.values(BUILTIN_ROLES).map( - role => role.name -) - -export function isBuiltin(role?: string) { - return BUILTIN_ROLE_ID_ARRAY.some(builtin => role?.includes(builtin)) +export function getBuiltinRole(roleId: string): Role | undefined { + const role = Object.values(BUILTIN_ROLES).find(role => + roleId.includes(role._id) + ) + if (!role) { + return undefined + } + return cloneDeep(role) } /** * Works through the inheritance ranks to see how far up the builtin stack this ID is. */ -export function builtinRoleToNumber(id?: string) { - if (!id) { - return 0 - } +export function builtinRoleToNumber(id: string) { const builtins = getBuiltinRoles() const MAX = Object.values(builtins).length + 1 if (id === BUILTIN_IDS.ADMIN || id === BUILTIN_IDS.BUILDER) { @@ -123,7 +122,7 @@ export function builtinRoleToNumber(id?: string) { /** * Converts any role to a number, but has to be async to get the roles from db. */ -export async function roleToNumber(id?: string) { +export async function roleToNumber(id: string) { if (isBuiltin(id)) { return builtinRoleToNumber(id) } @@ -131,7 +130,7 @@ export async function roleToNumber(id?: string) { defaultPublic: true, })) as RoleDoc[] for (let role of hierarchy) { - if (isBuiltin(role?.inherits)) { + if (role?.inherits && isBuiltin(role.inherits)) { return builtinRoleToNumber(role.inherits) + 1 } } @@ -161,35 +160,28 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string { * @returns The role object, which may contain an "inherits" property. */ export async function getRole( - roleId?: string, + roleId: string, opts?: { defaultPublic?: boolean } -): Promise { - if (!roleId) { - return undefined - } - let role: any = {} +): Promise { // built in roles mostly come from the in-code implementation, // but can be extended by a doc stored about them (e.g. permissions) - if (isBuiltin(roleId)) { - role = cloneDeep( - Object.values(BUILTIN_ROLES).find(role => role._id === roleId) - ) - } else { + let role: RoleDoc | undefined = getBuiltinRole(roleId) + if (!role) { // make sure has the prefix (if it has it then it won't be added) roleId = prefixRoleID(roleId) } try { const db = getAppDB() - const dbRole = await db.get(getDBRoleID(roleId)) - role = Object.assign(role, dbRole) + const dbRole = await db.get(getDBRoleID(roleId)) + role = Object.assign(role || {}, dbRole) // finalise the ID - role._id = getExternalRoleID(role._id, role.version) + role._id = getExternalRoleID(role._id!, role.version) } catch (err) { if (!isBuiltin(roleId) && opts?.defaultPublic) { return cloneDeep(BUILTIN_ROLES.PUBLIC) } // only throw an error if there is no role at all - if (Object.keys(role).length === 0) { + if (!role || Object.keys(role).length === 0) { throw err } } @@ -200,7 +192,7 @@ export async function getRole( * Simple function to get all the roles based on the top level user role ID. */ async function getAllUserRoles( - userRoleId?: string, + userRoleId: string, opts?: { defaultPublic?: boolean } ): Promise { // admins have access to all roles @@ -226,7 +218,7 @@ async function getAllUserRoles( } export async function getUserRoleIdHierarchy( - userRoleId?: string + userRoleId: string ): Promise { const roles = await getUserRoleHierarchy(userRoleId) return roles.map(role => role._id!) @@ -241,7 +233,7 @@ export async function getUserRoleIdHierarchy( * highest level of access and the last being the lowest level. */ export async function getUserRoleHierarchy( - userRoleId?: string, + userRoleId: string, opts?: { defaultPublic?: boolean } ) { // special case, if they don't have a role then they are a public user @@ -265,9 +257,9 @@ export function checkForRoleResourceArray( return rolePerms } -export async function getAllRoleIds(appId?: string) { +export async function getAllRoleIds(appId: string): Promise { const roles = await getAllRoles(appId) - return roles.map(role => role._id) + return roles.map(role => role._id!) } /** diff --git a/packages/backend-core/tests/core/utilities/structures/accounts.ts b/packages/backend-core/tests/core/utilities/structures/accounts.ts index 515f94db1e..7dcc2de116 100644 --- a/packages/backend-core/tests/core/utilities/structures/accounts.ts +++ b/packages/backend-core/tests/core/utilities/structures/accounts.ts @@ -18,7 +18,7 @@ export const account = (partial: Partial = {}): Account => { return { accountId: uuid(), tenantId: generator.word(), - email: generator.email(), + email: generator.email({ domain: "example.com" }), tenantName: generator.word(), hosting: Hosting.SELF, createdAt: Date.now(), diff --git a/packages/backend-core/tests/core/utilities/structures/scim.ts b/packages/backend-core/tests/core/utilities/structures/scim.ts index 80f41c605d..f424b2881a 100644 --- a/packages/backend-core/tests/core/utilities/structures/scim.ts +++ b/packages/backend-core/tests/core/utilities/structures/scim.ts @@ -13,7 +13,7 @@ interface CreateUserRequestFields { export function createUserRequest(userData?: Partial) { const defaultValues = { externalId: uuid(), - email: generator.email(), + email: `${uuid()}@example.com`, firstName: generator.first(), lastName: generator.last(), username: generator.name(), diff --git a/packages/bbui/src/Actions/position_dropdown.js b/packages/bbui/src/Actions/position_dropdown.js index b0dd94ad46..770d1bd507 100644 --- a/packages/bbui/src/Actions/position_dropdown.js +++ b/packages/bbui/src/Actions/position_dropdown.js @@ -36,7 +36,10 @@ export default function positionDropdown(element, opts) { } if (typeof customUpdate === "function") { - styles = customUpdate(anchorBounds, elementBounds, styles) + styles = customUpdate(anchorBounds, elementBounds, { + ...styles, + offset: opts.offset, + }) } else { // Determine vertical styles if (align === "right-outside" || align === "left-outside") { diff --git a/packages/bbui/src/Icon/IconAvatar.svelte b/packages/bbui/src/Icon/IconAvatar.svelte index add51f5bdc..b046e4dfd5 100644 --- a/packages/bbui/src/Icon/IconAvatar.svelte +++ b/packages/bbui/src/Icon/IconAvatar.svelte @@ -1,22 +1,41 @@ + +
(showTooltip = true)} + on:mouseleave={() => (showTooltip = false)} + on:focus={() => (showTooltip = true)} + on:blur={() => (showTooltip = false)} + on:click={() => (showTooltip = false)} > + {#if tooltip && showTooltip} +
+ +
+ {/if}
diff --git a/packages/bbui/src/Modal/Modal.svelte b/packages/bbui/src/Modal/Modal.svelte index f891d0584d..be9c338892 100644 --- a/packages/bbui/src/Modal/Modal.svelte +++ b/packages/bbui/src/Modal/Modal.svelte @@ -10,6 +10,7 @@ export let inline = false export let disableCancel = false export let autoFocus = true + export let zIndex = 999 const dispatch = createEventDispatcher() let visible = fixed || inline @@ -101,7 +102,11 @@ {#if visible} -
+
client.init()) + if (analyticsStatus.enabled && !this.initialised) { + this.clients.forEach(client => { + client.init() + }) + this.initialised = true } } diff --git a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte index 6b9524776c..b54ecbf9fd 100644 --- a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte +++ b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte @@ -40,8 +40,15 @@ part2: PrettyRelationshipDefinitions.MANY, }, } - let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions) - let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions) + $: relationshipOpts1 = + relationshipPart2 === PrettyRelationshipDefinitions.ONE + ? [PrettyRelationshipDefinitions.MANY] + : Object.values(PrettyRelationshipDefinitions) + + $: relationshipOpts2 = + relationshipPart1 === PrettyRelationshipDefinitions.ONE + ? [PrettyRelationshipDefinitions.MANY] + : Object.values(PrettyRelationshipDefinitions) let relationshipPart1 = PrettyRelationshipDefinitions.ONE let relationshipPart2 = PrettyRelationshipDefinitions.MANY diff --git a/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte b/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte index 90188d3094..b63c646ed4 100644 --- a/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte +++ b/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte @@ -55,6 +55,8 @@ export let jsBindingWrapping = true export let readonly = false + const dispatch = createEventDispatcher() + // Export a function to expose caret position export const getCaretPosition = () => { const selection_range = editor.state.selection.ranges[0] @@ -106,8 +108,6 @@ } ) - const dispatch = createEventDispatcher() - // Theming! let currentTheme = $themeStore?.theme let isDark = !currentTheme.includes("light") diff --git a/packages/builder/src/components/common/CodeEditor/index.js b/packages/builder/src/components/common/CodeEditor/index.js index bd55c371e6..c104267aa4 100644 --- a/packages/builder/src/components/common/CodeEditor/index.js +++ b/packages/builder/src/components/common/CodeEditor/index.js @@ -163,26 +163,12 @@ export const jsAutocomplete = baseCompletions => { } export const buildBindingInfoNode = (completion, binding) => { + if (!binding.valueHTML || binding.value == null) { + return null + } const ele = document.createElement("div") ele.classList.add("info-bubble") - - if (binding.value != null && binding.valueHTML) { - ele.innerHTML = `
${binding.valueHTML}
` - return ele - } - - const exampleNodeHtml = binding.readableBinding - ? `
{{ ${binding.readableBinding} }}
` - : "" - - const descriptionNodeHtml = binding.description - ? `
${binding.description}
` - : "" - - ele.innerHTML = ` - ${exampleNodeHtml} - ${descriptionNodeHtml} - ` + ele.innerHTML = `
${binding.valueHTML}
` return ele } diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte index f03c18fde3..0e9a6d4b4a 100644 --- a/packages/builder/src/components/common/bindings/BindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte @@ -89,6 +89,18 @@ sidePanel = sidePanelOptions[0] } } + $: hbsCompletions = [ + hbAutocomplete([ + ...bindingCompletions, + ...getHelperCompletions(EditorModes.Handlebars), + ]), + ] + $: jsCompletions = [ + jsAutocomplete([ + ...bindingCompletions, + ...getHelperCompletions(EditorModes.JS), + ]), + ] const getModeOptions = (allowHBS, allowJS) => { let options = [] @@ -148,6 +160,9 @@ const enrichBindings = (bindings, context, snippets) => { return bindings.map(binding => { + if (!context) { + return binding + } const value = getBindingValue(binding, context, snippets) return { ...binding, @@ -265,38 +280,34 @@ {/if}
{#if mode === Modes.Text} - + {#key hbsCompletions} + + {/key} {:else if mode === Modes.JavaScript} - + {#key jsCompletions} + + {/key} {/if} {#if targetMode}
@@ -332,7 +343,7 @@ {context} addHelper={onSelectHelper} addBinding={onSelectBinding} - mode={editorMode} + {mode} /> {:else if sidePanel === SidePanels.Evaluation} - + (tempValue = event.detail)} {bindings} {allowJS} diff --git a/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte b/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte index 12aa4edf2e..d11ebcf87a 100644 --- a/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte +++ b/packages/builder/src/components/common/bindings/DrawerBindableInput.svelte @@ -28,7 +28,6 @@ const dispatch = createEventDispatcher() let bindingDrawer - let valid = true let currentVal = value $: readableValue = runtimeToReadableBinding(bindings, value) @@ -93,13 +92,10 @@ title={title ?? placeholder ?? "Bindings"} {forceModal} > - + (tempValue = event.detail)} {bindings} diff --git a/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte b/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte index 84045e786a..8ce9dda209 100644 --- a/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte +++ b/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte @@ -25,7 +25,6 @@ const dispatch = createEventDispatcher() let bindingDrawer - let valid = true let currentVal = value $: readableValue = runtimeToReadableBinding(bindings, value) @@ -176,13 +175,10 @@ title={title ?? placeholder ?? "Bindings"} left={drawerLeft} > - + (tempValue = event.detail)} {bindings} diff --git a/packages/builder/src/components/common/bindings/ServerBindingPanel.svelte b/packages/builder/src/components/common/bindings/ServerBindingPanel.svelte index 8546758a04..df0da768e0 100644 --- a/packages/builder/src/components/common/bindings/ServerBindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/ServerBindingPanel.svelte @@ -3,7 +3,6 @@ import { snippets } from "stores/builder" export let bindings = [] - export let valid export let value = "" export let allowJS = false export let context = null @@ -21,7 +20,6 @@ {/if}
@@ -209,7 +210,7 @@
- + Publish {#if parameters.confirm} - + + + + {#if parameters.confirm} - + + + + + + diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte index a1fe773455..d834e9aac9 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte @@ -72,7 +72,10 @@ {#if parameters.confirm} - + + + + - import EditComponentPopover from "../EditComponentPopover.svelte" + import EditComponentPopover from "../EditComponentPopover/EditComponentPopover.svelte" import { Icon } from "@budibase/bbui" import { runtimeToReadableBinding } from "dataBinding" import { isJSBinding } from "@budibase/string-templates" diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index 0e01c264fc..bf0aa75bed 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -48,7 +48,6 @@ let drawer let tmpQueryParams let tmpCustomData - let customDataValid = true let modal $: text = value?.label ?? "Choose an option" @@ -267,14 +266,11 @@
- +
Provide a JSON array to use as data
(tmpCustomData = event.detail)} {bindings} diff --git a/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte b/packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte similarity index 83% rename from packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte rename to packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte index 4e645fe343..39e4bc2ada 100644 --- a/packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte +++ b/packages/builder/src/components/design/settings/controls/EditComponentPopover/EditComponentPopover.svelte @@ -3,6 +3,7 @@ import { componentStore } from "stores/builder" import { cloneDeep } from "lodash/fp" import { createEventDispatcher, getContext } from "svelte" + import { customPositionHandler } from "." import ComponentSettingsSection from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte" export let anchor @@ -54,25 +55,6 @@ dispatch("change", nestedComponentInstance) } - - const customPositionHandler = (anchorBounds, eleBounds, cfg) => { - let { left, top } = cfg - let percentageOffset = 30 - // left-outside - left = anchorBounds.left - eleBounds.width - 18 - - // shift up from the anchor, if space allows - let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset - let defaultTop = anchorBounds.top - offsetPos - - if (window.innerHeight - defaultTop < eleBounds.height) { - top = window.innerHeight - eleBounds.height - 5 - } else { - top = anchorBounds.top - offsetPos - } - - return { ...cfg, left, top } - } 0} maxHeight={600} + offset={18} handlePostionUpdate={customPositionHandler} > diff --git a/packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js b/packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js new file mode 100644 index 0000000000..2dc3f60185 --- /dev/null +++ b/packages/builder/src/components/design/settings/controls/EditComponentPopover/index.js @@ -0,0 +1,18 @@ +export const customPositionHandler = (anchorBounds, eleBounds, cfg) => { + let { left, top, offset } = cfg + let percentageOffset = 30 + // left-outside + left = anchorBounds.left - eleBounds.width - (offset || 5) + + // shift up from the anchor, if space allows + let offsetPos = Math.floor(eleBounds.height / 100) * percentageOffset + let defaultTop = anchorBounds.top - offsetPos + + if (window.innerHeight - defaultTop < eleBounds.height) { + top = window.innerHeight - eleBounds.height - 5 + } else { + top = anchorBounds.top - offsetPos + } + + return { ...cfg, left, top } +} diff --git a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte index 27590a9858..771bcf20e0 100644 --- a/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte +++ b/packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte @@ -1,5 +1,5 @@
{#if label && !labelHidden} @@ -116,6 +123,16 @@
diff --git a/packages/builder/src/components/portal/onboarding/steps/index.js b/packages/builder/src/components/portal/onboarding/steps/index.js index 8e27748f36..e15d191652 100644 --- a/packages/builder/src/components/portal/onboarding/steps/index.js +++ b/packages/builder/src/components/portal/onboarding/steps/index.js @@ -1,3 +1,5 @@ export { default as OnboardingData } from "./OnboardingData.svelte" export { default as OnboardingDesign } from "./OnboardingDesign.svelte" export { default as OnboardingPublish } from "./OnboardingPublish.svelte" +export { default as NewViewUpdateFormRowId } from "./NewViewUpdateFormRowId.svelte" +export { default as NewFormSteps } from "./NewFormSteps.svelte" diff --git a/packages/builder/src/components/portal/onboarding/tours.js b/packages/builder/src/components/portal/onboarding/tours.js index 623a00fcc2..fab97cdd03 100644 --- a/packages/builder/src/components/portal/onboarding/tours.js +++ b/packages/builder/src/components/portal/onboarding/tours.js @@ -2,8 +2,15 @@ import { get } from "svelte/store" import { builderStore } from "stores/builder" import { auth } from "stores/portal" import analytics from "analytics" -import { OnboardingData, OnboardingDesign, OnboardingPublish } from "./steps" +import { + OnboardingData, + OnboardingDesign, + OnboardingPublish, + NewViewUpdateFormRowId, + NewFormSteps, +} from "./steps" import { API } from "api" +import { customPositionHandler } from "components/design/settings/controls/EditComponentPopover" const ONBOARDING_EVENT_PREFIX = "onboarding" @@ -14,11 +21,26 @@ export const TOUR_STEP_KEYS = { BUILDER_USER_MANAGEMENT: "builder-user-management", BUILDER_AUTOMATION_SECTION: "builder-automation-section", FEATURE_USER_MANAGEMENT: "feature-user-management", + BUILDER_FORM_CREATE_STEPS: "builder-form-create-steps", + BUILDER_FORM_VIEW_UPDATE_STEPS: "builder-form-view-update-steps", + BUILDER_FORM_ROW_ID: "builder-form-row-id", } export const TOUR_KEYS = { TOUR_BUILDER_ONBOARDING: "builder-onboarding", FEATURE_ONBOARDING: "feature-onboarding", + BUILDER_FORM_CREATE: "builder-form-create", + BUILDER_FORM_VIEW_UPDATE: "builder-form-view-update", +} + +export const getCurrentStepIdx = (steps, tourStepKey) => { + if (!steps?.length) { + return + } + if (steps?.length && !tourStepKey) { + return 0 + } + return steps.findIndex(step => step.id === tourStepKey) } const endUserOnboarding = async ({ skipped = false } = {}) => { @@ -37,13 +59,8 @@ const endUserOnboarding = async ({ skipped = false } = {}) => { // Update the cached user await auth.getSelf() - builderStore.update(state => ({ - ...state, - tourNodes: null, - tourKey: null, - tourStepKey: null, - onboarding: false, - })) + builderStore.endBuilderOnboarding() + builderStore.setTour() } catch (e) { console.error("Onboarding failed", e) return false @@ -52,9 +69,29 @@ const endUserOnboarding = async ({ skipped = false } = {}) => { } } -const tourEvent = eventKey => { +const endTour = async ({ key, skipped = false } = {}) => { + const { tours = {} } = get(auth).user + tours[key] = new Date().toISOString() + + await API.updateSelf({ + tours, + }) + + if (skipped) { + tourEvent(key, skipped) + } + + // Update the cached user + await auth.getSelf() + + // Reset tour state + builderStore.setTour() +} + +const tourEvent = (eventKey, skipped) => { analytics.captureEvent(`${ONBOARDING_EVENT_PREFIX}:${eventKey}`, { eventSource: EventSource.PORTAL, + skipped, }) } @@ -135,7 +172,71 @@ const getTours = () => { }, ], }, + [TOUR_KEYS.BUILDER_FORM_CREATE]: { + steps: [ + { + id: TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS, + title: "Add multiple steps", + layout: NewFormSteps, + query: "#steps-prop-control-wrap", + onComplete: () => { + builderStore.highlightSetting() + endTour({ key: TOUR_KEYS.BUILDER_FORM_CREATE }) + }, + onLoad: () => { + tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_CREATE_STEPS) + builderStore.highlightSetting("steps", "info") + }, + positionHandler: customPositionHandler, + align: "left-outside", + }, + ], + }, + [TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE]: { + steps: [ + { + id: TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID, + title: "Add row ID to update a row", + layout: NewViewUpdateFormRowId, + query: "#rowId-prop-control-wrap", + onLoad: () => { + tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_ROW_ID) + builderStore.highlightSetting("rowId", "info") + }, + positionHandler: customPositionHandler, + align: "left-outside", + }, + { + id: TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS, + title: "Add multiple steps", + layout: NewFormSteps, + query: "#steps-prop-control-wrap", + onComplete: () => { + builderStore.highlightSetting() + endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE }) + }, + onLoad: () => { + tourEvent(TOUR_STEP_KEYS.BUILDER_FORM_VIEW_UPDATE_STEPS) + builderStore.highlightSetting("steps", "info") + }, + positionHandler: customPositionHandler, + align: "left-outside", + scrollIntoView: true, + }, + ], + onSkip: async () => { + builderStore.highlightSetting() + endTour({ key: TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE, skipped: true }) + }, + }, } } export const TOURS = getTours() +export const TOURSBYSTEP = Object.keys(TOURS).reduce((acc, tour) => { + TOURS[tour].steps.forEach(element => { + acc[element.id] = element + acc[element.id]["tour"] = tour + }) + return acc +}, {}) diff --git a/packages/builder/src/dataBinding.js b/packages/builder/src/dataBinding.js index 0442a67da9..8fdf71c156 100644 --- a/packages/builder/src/dataBinding.js +++ b/packages/builder/src/dataBinding.js @@ -1146,7 +1146,7 @@ export const getAllStateVariables = () => { "@budibase/standard-components/multistepformblockstep" ) - steps.forEach(step => { + steps?.forEach(step => { parseComponentSettings(stepDefinition, step) }) }) diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index dd66f5bc34..45b10d3d9e 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -96,7 +96,7 @@ const release_date = new Date("2023-03-01T00:00:00.000Z") const onboarded = new Date($auth.user?.onboardedAt) if (onboarded < release_date) { - builderStore.startTour(TOUR_KEYS.FEATURE_ONBOARDING) + builderStore.setTour(TOUR_KEYS.FEATURE_ONBOARDING) } } } @@ -144,7 +144,7 @@
{#each $layout.children as { path, title }} - + - + diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte index 8b778a2ce7..7a5372e883 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte @@ -16,6 +16,14 @@ } from "dataBinding" import { ActionButton, notifications } from "@budibase/bbui" import { capitalise } from "helpers" + import TourWrap from "components/portal/onboarding/TourWrap.svelte" + import { TOUR_STEP_KEYS } from "components/portal/onboarding/tours.js" + + const { + BUILDER_FORM_CREATE_STEPS, + BUILDER_FORM_VIEW_UPDATE_STEPS, + BUILDER_FORM_ROW_ID, + } = TOUR_STEP_KEYS const onUpdateName = async value => { try { @@ -46,7 +54,6 @@ $: id = $selectedComponent?._id $: id, (section = tabs[0]) - $: componentName = getComponentName(componentInstance) @@ -92,13 +99,21 @@
{#if section == "settings"} - + + + {/if} {#if section == "styles"} import { helpers } from "@budibase/shared-core" import { DetailSummary, notifications } from "@budibase/bbui" - import { componentStore } from "stores/builder" + import { componentStore, builderStore } from "stores/builder" import PropertyControl from "components/design/settings/controls/PropertyControl.svelte" import ResetFieldsButton from "components/design/settings/controls/ResetFieldsButton.svelte" import EjectBlockButton from "components/design/settings/controls/EjectBlockButton.svelte" @@ -177,9 +177,7 @@ defaultValue={setting.defaultValue} nested={setting.nested} onChange={val => updateSetting(setting, val)} - highlighted={$componentStore.highlightedSettingKey === - setting.key} - propertyFocus={$componentStore.propertyFocus === setting.key} + propertyFocus={$builderStore.propertyFocus === setting.key} info={setting.info} disableBindings={setting.disableBindings} props={{ diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index cadc83d16a..95e7a66be9 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -176,7 +176,7 @@ } else if (type === "request-add-component") { toggleAddComponent() } else if (type === "highlight-setting") { - builderStore.highlightSetting(data.setting) + builderStore.highlightSetting(data.setting, "error") // Also scroll setting into view const selector = `#${data.setting}-prop-control` diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte index eadd9d4257..8c1a11289d 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte @@ -3,13 +3,23 @@ import DatasourceModal from "./DatasourceModal.svelte" import ScreenRoleModal from "./ScreenRoleModal.svelte" import sanitizeUrl from "helpers/sanitizeUrl" + import FormTypeModal from "./FormTypeModal.svelte" import { Modal, notifications } from "@budibase/bbui" - import { screenStore, navigationStore, tables } from "stores/builder" + import { + screenStore, + navigationStore, + tables, + builderStore, + } from "stores/builder" + import { auth } from "stores/portal" import { get } from "svelte/store" import getTemplates from "templates" import { Roles } from "constants/backend" import { capitalise } from "helpers" import { goto } from "@roxi/routify" + import { TOUR_KEYS } from "components/portal/onboarding/tours.js" + import formScreen from "templates/formScreen" + import rowListScreen from "templates/rowListScreen" let mode let pendingScreen @@ -18,12 +28,18 @@ let screenDetailsModal let datasourceModal let screenAccessRoleModal + let formTypeModal // Cache variables for workflow let screenAccessRole = Roles.BASIC - let selectedTemplates = null + + let templates = null + let screens = null + + let selectedDatasources = null let blankScreenUrl = null let screenMode = null + let formType = null // Creates an array of screens, checking and sanitising their URLs const createScreens = async ({ screens, screenAccessRole }) => { @@ -32,7 +48,7 @@ } try { - let screenId + let createdScreens = [] for (let screen of screens) { // Check we aren't clashing with an existing URL @@ -56,7 +72,7 @@ // Create the screen const response = await screenStore.save(screen) - screenId = response._id + createdScreens.push(response) // Add link in layout. We only ever actually create 1 screen now, even // for autoscreens, so it's always safe to do this. @@ -66,9 +82,7 @@ ) } - // Go to new screen - $goto(`./${screenId}`) - screenStore.select(screenId) + return createdScreens } catch (error) { console.error(error) notifications.error("Error creating screens") @@ -104,13 +118,16 @@ // Handler for NewScreenModal export const show = newMode => { mode = newMode - selectedTemplates = null + templates = null + screens = null + selectedDatasources = null blankScreenUrl = null screenMode = mode pendingScreen = null screenAccessRole = Roles.BASIC + formType = null - if (mode === "table" || mode === "grid") { + if (mode === "table" || mode === "grid" || mode === "form") { datasourceModal.show() } else if (mode === "blank") { let templates = getTemplates($tables.list) @@ -125,19 +142,26 @@ } // Handler for DatasourceModal confirmation, move to screen access select - const confirmScreenDatasources = async ({ templates }) => { - selectedTemplates = templates - screenAccessRoleModal.show() + const confirmScreenDatasources = async ({ datasources }) => { + selectedDatasources = datasources + if (screenMode === "form") { + formTypeModal.show() + } else { + screenAccessRoleModal.show() + } } // Handler for Datasource Screen Creation const completeDatasourceScreenCreation = async () => { - const screens = selectedTemplates.map(template => { + templates = rowListScreen(selectedDatasources, mode) + + const screens = templates.map(template => { let screenTemplate = template.create() screenTemplate.autoTableId = template.resourceId return screenTemplate }) - await createScreens({ screens, screenAccessRole }) + const createdScreens = await createScreens({ screens, screenAccessRole }) + loadNewScreen(createdScreens) } const confirmScreenBlank = async ({ screenUrl }) => { @@ -154,7 +178,54 @@ return } pendingScreen.routing.route = screenUrl - await createScreens({ screens: [pendingScreen], screenAccessRole }) + const createdScreens = await createScreens({ + screens: [pendingScreen], + screenAccessRole, + }) + loadNewScreen(createdScreens) + } + + const onConfirmFormType = () => { + screenAccessRoleModal.show() + } + + const loadNewScreen = createdScreens => { + const lastScreen = createdScreens.slice(-1)[0] + + // Go to new screen + if (lastScreen?.props?._children.length) { + // Focus on the main component for the streen type + const mainComponent = lastScreen?.props?._children?.[0]._id + $goto(`./${lastScreen._id}/${mainComponent}`) + } else { + $goto(`./${lastScreen._id}`) + } + + screenStore.select(lastScreen._id) + } + + const confirmFormScreenCreation = async () => { + templates = formScreen(selectedDatasources, { actionType: formType }) + screens = templates.map(template => { + let screenTemplate = template.create() + return screenTemplate + }) + const createdScreens = await createScreens({ screens, screenAccessRole }) + + if (formType === "Update" || formType === "Create") { + const associatedTour = + formType === "Update" + ? TOUR_KEYS.BUILDER_FORM_VIEW_UPDATE + : TOUR_KEYS.BUILDER_FORM_CREATE + + const tourRequired = !$auth?.user?.tours?.[associatedTour] + if (tourRequired) { + builderStore.setTour(associatedTour) + } + } + + // Go to new screen + loadNewScreen(createdScreens) } // Submit screen config for creation. @@ -164,6 +235,8 @@ screenUrl: blankScreenUrl, screenAccessRole, }) + } else if (screenMode === "form") { + confirmFormScreenCreation() } else { completeDatasourceScreenCreation() } @@ -179,19 +252,18 @@ - + { + confirmScreenCreation() + }} bind:screenAccessRole + onCancel={roleSelectBack} screenUrl={blankScreenUrl} + confirmText={screenMode === "form" ? "Confirm" : "Done"} /> @@ -201,3 +273,17 @@ initialUrl={blankScreenUrl} /> + + + { + formTypeModal.hide() + datasourceModal.show() + }} + on:select={e => { + formType = e.detail + }} + type={formType} + /> + diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte index bb6c47c4b1..8e2428d832 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/DatasourceModal.svelte @@ -4,37 +4,33 @@ import ICONS from "components/backend/DatasourceNavigator/icons" import { IntegrationNames } from "constants" import { onMount } from "svelte" - import rowListScreen from "templates/rowListScreen" import DatasourceTemplateRow from "./DatasourceTemplateRow.svelte" - export let mode export let onCancel export let onConfirm - export let initialScreens = [] - let selectedScreens = [...initialScreens] + let selectedSources = [] $: filteredSources = $datasources.list?.filter(datasource => { return datasource.source !== IntegrationNames.REST && datasource["entities"] }) const toggleSelection = datasource => { - const { resourceId } = datasource - if (selectedScreens.find(s => s.resourceId === resourceId)) { - selectedScreens = selectedScreens.filter( - screen => screen.resourceId !== resourceId + const exists = selectedSources.find( + d => d.resourceId === datasource.resourceId + ) + if (exists) { + selectedSources = selectedSources.filter( + d => d.resourceId === datasource.resourceId ) } else { - selectedScreens = [ - ...selectedScreens, - rowListScreen([datasource], mode)[0], - ] + selectedSources = [...selectedSources, datasource] } } const confirmDatasourceSelection = async () => { await onConfirm({ - templates: selectedScreens, + datasources: selectedSources, }) } @@ -54,7 +50,7 @@ cancelText="Back" onConfirm={confirmDatasourceSelection} {onCancel} - disabled={!selectedScreens.length} + disabled={!selectedSources.length} size="L" > @@ -85,8 +81,8 @@ resourceId: table._id, type: "table", }} - {@const selected = selectedScreens.find( - screen => screen.resourceId === tableDS.resourceId + {@const selected = selectedSources.find( + datasource => datasource.resourceId === tableDS.resourceId )} toggleSelection(tableDS)} @@ -103,7 +99,7 @@ tableId: view.tableId, type: "viewV2", }} - {@const selected = selectedScreens.find( + {@const selected = selectedSources.find( x => x.resourceId === viewDS.resourceId )} + import { ModalContent, Layout, Body, Icon } from "@budibase/bbui" + import { createEventDispatcher } from "svelte" + + export let onCancel = () => {} + export let onConfirm = () => {} + export let type + + const dispatch = createEventDispatcher() + + + + + + + +
{ + dispatch("select", "Create") + }} + > +
+
+ Create a new row + + For capturing and storing new data from your users + +
+ {#if type === "Create"} + + + + {/if} +
+
+
{ + dispatch("select", "Update") + }} + > +
+
+ Update an existing row + For viewing and updating existing data +
+ {#if type === "Update"} + + + + {/if} +
+
+
{ + dispatch("select", "View") + }} + > +
+
+ View an existing row + For a read only view of your data +
+ {#if type === "View"} + + + + {/if} +
+
+
+
+
+ + diff --git a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte index 5e34d32219..8605412c1c 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/ScreenRoleModal.svelte @@ -9,6 +9,7 @@ export let onCancel export let screenUrl export let screenAccessRole + export let confirmText = "Done" let error @@ -40,7 +41,7 @@ import { Body } from "@budibase/bbui" import CreationPage from "components/common/CreationPage.svelte" - import blankImage from "./blank.png" - import tableImage from "./table.png" - import gridImage from "./grid.png" + import blankImage from "./images/blank.png" + import tableImage from "./images/table.png" + import gridImage from "./images/grid.png" + import formImage from "./images/form.png" import CreateScreenModal from "./CreateScreenModal.svelte" import { screenStore } from "stores/builder" @@ -56,6 +57,16 @@ View and manipulate rows on a grid
+ +
createScreenModal.show("form")}> +
+ +
+
+ Form + Capture data from your users +
+
diff --git a/packages/builder/src/pages/builder/portal/users/_components/ActiveDirectoryInfo.svelte b/packages/builder/src/pages/builder/portal/users/_components/ActiveDirectoryInfo.svelte new file mode 100644 index 0000000000..e060e2ca3f --- /dev/null +++ b/packages/builder/src/pages/builder/portal/users/_components/ActiveDirectoryInfo.svelte @@ -0,0 +1,23 @@ + + +
+ + {text} +
+ + diff --git a/packages/builder/src/pages/builder/portal/users/_components/SCIMBanner.svelte b/packages/builder/src/pages/builder/portal/users/_components/SCIMBanner.svelte deleted file mode 100644 index 2d02214800..0000000000 --- a/packages/builder/src/pages/builder/portal/users/_components/SCIMBanner.svelte +++ /dev/null @@ -1,15 +0,0 @@ - - -
- - Users are synced from your AD -
- - diff --git a/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte b/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte index d03c62d528..d0eb50c8dd 100644 --- a/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte +++ b/packages/builder/src/pages/builder/portal/users/groups/[groupId].svelte @@ -13,7 +13,7 @@ import ConfirmDialog from "components/common/ConfirmDialog.svelte" import { Breadcrumb, Breadcrumbs } from "components/portal/page" import { roles } from "stores/builder" - import { apps, auth, features, groups } from "stores/portal" + import { apps, auth, groups } from "stores/portal" import { onMount, setContext } from "svelte" import AppNameTableRenderer from "../users/_components/AppNameTableRenderer.svelte" import AppRoleTableRenderer from "../users/_components/AppRoleTableRenderer.svelte" @@ -47,9 +47,9 @@ let loaded = false let editModal, deleteModal - $: scimEnabled = $features.isScimEnabled - $: readonly = !sdk.users.isAdmin($auth.user) || scimEnabled $: group = $groups.find(x => x._id === groupId) + $: isScimGroup = group?.scimInfo?.isSync + $: readonly = !sdk.users.isAdmin($auth.user) || isScimGroup $: groupApps = $apps .filter(app => groups.actions @@ -119,23 +119,27 @@
{group?.name} - {#if !readonly} - - - - - editModal.show()}> - Edit - - deleteModal.show()}> + + + + + editModal.show()}> + Edit + +
+ deleteModal.show()} + disabled={isScimGroup} + > Delete - - {/if} +
+
- + diff --git a/packages/builder/src/pages/builder/portal/users/groups/_components/CreateEditGroupModal.svelte b/packages/builder/src/pages/builder/portal/users/groups/_components/CreateEditGroupModal.svelte index 2f0d295052..9a0c3540af 100644 --- a/packages/builder/src/pages/builder/portal/users/groups/_components/CreateEditGroupModal.svelte +++ b/packages/builder/src/pages/builder/portal/users/groups/_components/CreateEditGroupModal.svelte @@ -11,6 +11,8 @@ export let group export let saveGroup + let readonlyTitle = group?.scimInfo?.isSync + let nameError @@ -26,7 +28,12 @@ title={group?._rev ? "Edit group" : "Create group"} confirmText="Save" > - +