diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index a9de0ba342..65e6529678 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -56,6 +56,7 @@ jobs: run: yarn install:pro $BRANCH $BASE_BRANCH - run: yarn - run: yarn bootstrap + - run: yarn build - run: yarn test - uses: codecov/codecov-action@v3 with: @@ -94,4 +95,4 @@ jobs: yarn test:ci env: BB_ADMIN_USER_EMAIL: admin - BB_ADMIN_USER_PASSWORD: admin \ No newline at end of file + BB_ADMIN_USER_PASSWORD: admin diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index ab0771def5..5d51b080f0 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -75,7 +75,6 @@ jobs: - name: Build/release Docker images run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - yarn build yarn build:docker env: DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} diff --git a/README.md b/README.md index bd38610566..9deb16cd4f 100644 --- a/README.md +++ b/README.md @@ -216,35 +216,9 @@ If you are having issues between updates of the builder, please use the guide [h Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): - - - - - - - - - - - - - - - - - - - - - - - - -

Martin McKeaveney

πŸ’» πŸ“– ⚠️ πŸš‡

Michael Drury

πŸ“– πŸ’» ⚠️ πŸš‡

Andrew Kingston

πŸ“– πŸ’» ⚠️ 🎨

Michael Shanks

πŸ“– πŸ’» ⚠️

Kevin Γ…berg Kultalahti

πŸ“– πŸ’» ⚠️

Joe

πŸ“– πŸ’» πŸ–‹ 🎨

Rory Powell

πŸ’» πŸ“– ⚠️

Peter Clement

πŸ’» πŸ“– ⚠️

Conor_Mack

πŸ’» ⚠️

pngwn

πŸ’» ⚠️

HugoLd

πŸ’»

victoriasloan

πŸ’»

yashank09

πŸ’»

SOVLOOKUP

πŸ’»

seoulaja

🌍

Maurits Lourens

⚠️ πŸ’»
- - + + + - - -This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! +Made with [contrib.rocks](https://contrib.rocks). diff --git a/charts/budibase/templates/alb-ingress.yaml b/charts/budibase/templates/alb-ingress.yaml index c128b70843..6cd1cf2cba 100644 --- a/charts/budibase/templates/alb-ingress.yaml +++ b/charts/budibase/templates/alb-ingress.yaml @@ -14,6 +14,9 @@ metadata: alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' alb.ingress.kubernetes.io/certificate-arn: {{ .Values.ingress.certificateArn }} {{- end }} + {{- if .Values.ingress.sslPolicy }} + alb.ingress.kubernetes.io/actions.ssl-policy: {{ .Values.ingress.sslPolicy }} + {{- end }} {{- if .Values.ingress.securityGroups }} alb.ingress.kubernetes.io/security-groups: {{ .Values.ingress.securityGroups }} {{- end }} diff --git a/docs/DEV-SETUP-MACOSX.md b/docs/DEV-SETUP-MACOSX.md index d9e2dcad6a..67eb5506ff 100644 --- a/docs/DEV-SETUP-MACOSX.md +++ b/docs/DEV-SETUP-MACOSX.md @@ -61,5 +61,18 @@ http://127.0.0.1:10000/builder/admin | **NOTE**: If you are working on a M1 Apple Silicon, you will need to uncomment `# platform: linux/amd64` line in [hosting/docker-compose-dev.yaml](../hosting/docker-compose.dev.yaml) -### Troubleshooting -If there are errors with the `yarn setup` command, you can try installing nvm and node 14. This is the same as the instructions for Debian 11. +### Troubleshootings + +#### Yarn setup errors + +If there are errors with the `yarn setup` command, you can try installing nvm and node 14. This is the same as the instructions for Debian 11. + +#### Node 14.20.1 not supported for arm64 + +If you are working with M1 or M2 Mac and trying the Node installation via `nvm`, probably you will find the error `curl: (22) The requested URL returned error: 404`. + +Version `v14.20.1` is not supported for arm64; in order to use it, you can switch the CPU architecture for this by the following command: + +```shell +arch -x86_64 zsh #Run this before nvm install +``` diff --git a/lerna.json b/lerna.json index 78a218fcd4..a542a51851 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.5.5-alpha.0", + "version": "2.5.6-alpha.32", "npmClient": "yarn", "useWorkspaces": true, "packages": ["packages/*"], diff --git a/package.json b/package.json index e944a5bf5e..29b7c7c723 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "dev": "yarn run kill-all && lerna link && lerna run --stream --parallel dev:builder --concurrency 1 --stream", "dev:noserver": "yarn run kill-builder && lerna link && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server", - "dev:built": "cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built", + "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built", "test": "lerna run --stream test --stream", "test:pro": "bash scripts/pro/test.sh", "lint:eslint": "eslint packages && eslint qa-core", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 730acda357..96f16747e0 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.5.5-alpha.0", + "version": "2.5.6-alpha.32", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -24,7 +24,7 @@ "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "2.5.5-alpha.0", + "@budibase/types": "2.5.6-alpha.32", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", @@ -47,6 +47,8 @@ "passport-jwt": "4.0.0", "passport-local": "1.0.0", "passport-oauth2-refresh": "^2.1.0", + "pino": "8.11.0", + "pino-http": "8.3.3", "posthog-node": "1.3.0", "pouchdb": "7.3.0", "pouchdb-find": "7.2.2", @@ -54,8 +56,7 @@ "sanitize-s3-objectkey": "0.0.1", "semver": "7.3.7", "tar-fs": "2.1.1", - "uuid": "8.3.2", - "zlib": "1.0.5" + "uuid": "8.3.2" }, "devDependencies": { "@jest/test-sequencer": "29.5.0", @@ -81,7 +82,6 @@ "jest-serial-runner": "^1.2.1", "koa": "2.13.4", "nodemon": "2.0.16", - "pino": "7.11.0", "pino-pretty": "10.0.0", "pouchdb-adapter-memory": "7.2.2", "timekeeper": "2.2.0", diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index d41098c405..aa40f13775 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -14,6 +14,7 @@ export enum ViewName { USER_BY_APP = "by_app", USER_BY_EMAIL = "by_email2", BY_API_KEY = "by_api_key", + /** @deprecated - could be deleted */ USER_BY_BUILDERS = "by_builders", LINK = "by_link", ROUTING = "screen_routes", diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 2f66c4bb7d..861777b679 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -115,10 +115,10 @@ export async function doInContext(appId: string, task: any): Promise { ) } -export async function doInTenant( +export async function doInTenant( tenantId: string | null, - task: any -): Promise { + task: () => T +): Promise { // make sure default always selected in single tenancy if (!env.MULTI_TENANCY) { tenantId = tenantId || DEFAULT_TENANT_ID diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index 6f2f4fc991..5d21d4f4e4 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -243,7 +243,7 @@ export class QueryBuilder { } // Escape characters if (!this.#noEscaping && escape && originalType === "string") { - value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&") + value = `${value}`.replace(/[ \/#+\-&|!(){}\]^"~*?:\\]/g, "\\$&") } // Wrap in quotes @@ -320,6 +320,18 @@ export class QueryBuilder { return `${key}:(${statement})` } + const fuzzy = (key: string, value: any) => { + if (!value) { + return null + } + value = builder.preprocess(value, { + escape: true, + lowercase: true, + type: "fuzzy", + }) + return `${key}:/.*${value}.*/` + } + const notContains = (key: string, value: any) => { const allPrefix = allOr ? "*:* AND " : "" const mode = allOr ? "AND" : undefined @@ -408,17 +420,7 @@ export class QueryBuilder { }) } if (this.#query.fuzzy) { - build(this.#query.fuzzy, (key: string, value: any) => { - if (!value) { - return null - } - value = builder.preprocess(value, { - escape: true, - lowercase: true, - type: "fuzzy", - }) - return `${key}:${value}~` - }) + build(this.#query.fuzzy, fuzzy) } if (this.#query.equal) { build(this.#query.equal, equal) diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index 8a2c2e7efd..fddb1ab34b 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -7,7 +7,7 @@ import { } from "../constants" import { getGlobalDB } from "../context" import { doWithDB } from "./" -import { Database, DatabaseQueryOpts } from "@budibase/types" +import { AllDocsResponse, Database, DatabaseQueryOpts } from "@budibase/types" import env from "../environment" const DESIGN_DB = "_design/database" @@ -42,7 +42,11 @@ async function removeDeprecated(db: Database, viewName: ViewName) { } } -export async function createView(db: any, viewJs: string, viewName: string) { +export async function createView( + db: any, + viewJs: string, + viewName: string +): Promise { let designDoc try { designDoc = (await db.get(DESIGN_DB)) as DesignDocument @@ -57,7 +61,15 @@ export async function createView(db: any, viewJs: string, viewName: string) { ...designDoc.views, [viewName]: view, } - await db.put(designDoc) + try { + await db.put(designDoc) + } catch (err: any) { + if (err.status === 409) { + return await createView(db, viewJs, viewName) + } else { + throw err + } + } } export const createNewUserEmailView = async () => { @@ -107,6 +119,34 @@ export interface QueryViewOptions { arrayResponse?: boolean } +export async function queryViewRaw( + viewName: ViewName, + params: DatabaseQueryOpts, + db: Database, + createFunc: any, + opts?: QueryViewOptions +): Promise> { + try { + const response = await db.query(`database/${viewName}`, params) + // await to catch error + return response + } catch (err: any) { + const pouchNotFound = err && err.name === "not_found" + const couchNotFound = err && err.status === 404 + if (pouchNotFound || couchNotFound) { + await removeDeprecated(db, viewName) + await createFunc() + return queryViewRaw(viewName, params, db, createFunc, opts) + } else if (err.status === 409) { + // can happen when multiple queries occur at once, view couldn't be created + // other design docs being updated, re-run + return queryViewRaw(viewName, params, db, createFunc, opts) + } else { + throw err + } + } +} + export const queryView = async ( viewName: ViewName, params: DatabaseQueryOpts, @@ -114,30 +154,18 @@ export const queryView = async ( createFunc: any, opts?: QueryViewOptions ): Promise => { - try { - let response = await db.query(`database/${viewName}`, params) - const rows = response.rows - const docs = rows.map((row: any) => - params.include_docs ? row.doc : row.value - ) + const response = await queryViewRaw(viewName, params, db, createFunc, opts) + const rows = response.rows + const docs = rows.map((row: any) => + params.include_docs ? row.doc : row.value + ) - // if arrayResponse has been requested, always return array regardless of length - if (opts?.arrayResponse) { - return docs as T[] - } else { - // return the single document if there is only one - return docs.length <= 1 ? (docs[0] as T) : (docs as T[]) - } - } catch (err: any) { - const pouchNotFound = err && err.name === "not_found" - const couchNotFound = err && err.status === 404 - if (pouchNotFound || couchNotFound) { - await removeDeprecated(db, viewName) - await createFunc() - return queryView(viewName, params, db, createFunc, opts) - } else { - throw err - } + // if arrayResponse has been requested, always return array regardless of length + if (opts?.arrayResponse) { + return docs as T[] + } else { + // return the single document if there is only one + return docs.length <= 1 ? (docs[0] as T) : (docs as T[]) } } @@ -192,18 +220,19 @@ export const queryPlatformView = async ( }) } +const CreateFuncByName: any = { + [ViewName.USER_BY_EMAIL]: createNewUserEmailView, + [ViewName.BY_API_KEY]: createApiKeyView, + [ViewName.USER_BY_BUILDERS]: createUserBuildersView, + [ViewName.USER_BY_APP]: createUserAppView, +} + export const queryGlobalView = async ( viewName: ViewName, params: DatabaseQueryOpts, db?: Database, opts?: QueryViewOptions ): Promise => { - const CreateFuncByName: any = { - [ViewName.USER_BY_EMAIL]: createNewUserEmailView, - [ViewName.BY_API_KEY]: createApiKeyView, - [ViewName.USER_BY_BUILDERS]: createUserBuildersView, - [ViewName.USER_BY_APP]: createUserAppView, - } // can pass DB in if working with something specific if (!db) { db = getGlobalDB() @@ -211,3 +240,13 @@ export const queryGlobalView = async ( const createFn = CreateFuncByName[viewName] return queryView(viewName, params, db!, createFn, opts) } + +export async function queryGlobalViewRaw( + viewName: ViewName, + params: DatabaseQueryOpts, + opts?: QueryViewOptions +) { + const db = getGlobalDB() + const createFn = CreateFuncByName[viewName] + return queryViewRaw(viewName, params, db, createFn, opts) +} diff --git a/packages/backend-core/src/docUpdates/index.ts b/packages/backend-core/src/docUpdates/index.ts new file mode 100644 index 0000000000..3971f8de12 --- /dev/null +++ b/packages/backend-core/src/docUpdates/index.ts @@ -0,0 +1,29 @@ +import { asyncEventQueue, init as initQueue } from "../events/asyncEvents" +import { + ProcessorMap, + default as DocumentUpdateProcessor, +} from "../events/processors/async/DocumentUpdateProcessor" + +let processingPromise: Promise +let documentProcessor: DocumentUpdateProcessor + +export function init(processors: ProcessorMap) { + if (!asyncEventQueue) { + initQueue() + } + if (!documentProcessor) { + documentProcessor = new DocumentUpdateProcessor(processors) + } + // if not processing in this instance, kick it off + if (!processingPromise) { + processingPromise = asyncEventQueue.process(async job => { + const { event, identity, properties, timestamp } = job.data + await documentProcessor.processEvent( + event, + identity, + properties, + timestamp + ) + }) + } +} diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 68f056c80c..0d413b8fa9 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -1,3 +1,5 @@ +import { existsSync, readFileSync } from "fs" + function isTest() { return isCypress() || isJest() } @@ -45,6 +47,35 @@ function httpLogging() { return process.env.HTTP_LOGGING } +function findVersion() { + function findFileInAncestors( + fileName: string, + currentDir: string + ): string | null { + const filePath = `${currentDir}/${fileName}` + if (existsSync(filePath)) { + return filePath + } + + const parentDir = `${currentDir}/..` + if (parentDir === currentDir) { + // reached root directory + return null + } + + return findFileInAncestors(fileName, parentDir) + } + + try { + const packageJsonFile = findFileInAncestors("package.json", process.cwd()) + const content = readFileSync(packageJsonFile!, "utf-8") + const version = JSON.parse(content).version + return version + } catch { + throw new Error("Cannot find a valid version in its package.json") + } +} + const environment = { isTest, isJest, @@ -122,6 +153,7 @@ const environment = { ENABLE_SSO_MAINTENANCE_MODE: selfHosted ? process.env.ENABLE_SSO_MAINTENANCE_MODE : false, + VERSION: findVersion(), _set(key: any, value: any) { process.env[key] = value // @ts-ignore diff --git a/packages/backend-core/src/events/asyncEvents/index.ts b/packages/backend-core/src/events/asyncEvents/index.ts new file mode 100644 index 0000000000..65f7f8f58b --- /dev/null +++ b/packages/backend-core/src/events/asyncEvents/index.ts @@ -0,0 +1,2 @@ +export * from "./queue" +export * from "./publisher" diff --git a/packages/backend-core/src/events/asyncEvents/publisher.ts b/packages/backend-core/src/events/asyncEvents/publisher.ts new file mode 100644 index 0000000000..4e44c4ddc5 --- /dev/null +++ b/packages/backend-core/src/events/asyncEvents/publisher.ts @@ -0,0 +1,12 @@ +import { AsyncEvents } from "@budibase/types" +import { EventPayload, asyncEventQueue, init } from "./queue" + +export async function publishAsyncEvent(payload: EventPayload) { + if (!asyncEventQueue) { + init() + } + const { event, identity } = payload + if (AsyncEvents.indexOf(event) !== -1 && identity.tenantId) { + await asyncEventQueue.add(payload) + } +} diff --git a/packages/backend-core/src/events/asyncEvents/queue.ts b/packages/backend-core/src/events/asyncEvents/queue.ts new file mode 100644 index 0000000000..196fd359b3 --- /dev/null +++ b/packages/backend-core/src/events/asyncEvents/queue.ts @@ -0,0 +1,22 @@ +import BullQueue from "bull" +import { createQueue, JobQueue } from "../../queue" +import { Event, Identity } from "@budibase/types" + +export interface EventPayload { + event: Event + identity: Identity + properties: any + timestamp?: string | number +} + +export let asyncEventQueue: BullQueue.Queue + +export function init() { + asyncEventQueue = createQueue(JobQueue.SYSTEM_EVENT_QUEUE) +} + +export async function shutdown() { + if (asyncEventQueue) { + await asyncEventQueue.close() + } +} diff --git a/packages/backend-core/src/events/documentId.ts b/packages/backend-core/src/events/documentId.ts new file mode 100644 index 0000000000..887e56f07b --- /dev/null +++ b/packages/backend-core/src/events/documentId.ts @@ -0,0 +1,56 @@ +import { + Event, + UserCreatedEvent, + UserUpdatedEvent, + UserDeletedEvent, + UserPermissionAssignedEvent, + UserPermissionRemovedEvent, + GroupCreatedEvent, + GroupUpdatedEvent, + GroupDeletedEvent, + GroupUsersAddedEvent, + GroupUsersDeletedEvent, + GroupPermissionsEditedEvent, +} from "@budibase/types" + +const getEventProperties: Record< + string, + (properties: any) => string | undefined +> = { + [Event.USER_CREATED]: (properties: UserCreatedEvent) => properties.userId, + [Event.USER_UPDATED]: (properties: UserUpdatedEvent) => properties.userId, + [Event.USER_DELETED]: (properties: UserDeletedEvent) => properties.userId, + [Event.USER_PERMISSION_ADMIN_ASSIGNED]: ( + properties: UserPermissionAssignedEvent + ) => properties.userId, + [Event.USER_PERMISSION_ADMIN_REMOVED]: ( + properties: UserPermissionRemovedEvent + ) => properties.userId, + [Event.USER_PERMISSION_BUILDER_ASSIGNED]: ( + properties: UserPermissionAssignedEvent + ) => properties.userId, + [Event.USER_PERMISSION_BUILDER_REMOVED]: ( + properties: UserPermissionRemovedEvent + ) => properties.userId, + [Event.USER_GROUP_CREATED]: (properties: GroupCreatedEvent) => + properties.groupId, + [Event.USER_GROUP_UPDATED]: (properties: GroupUpdatedEvent) => + properties.groupId, + [Event.USER_GROUP_DELETED]: (properties: GroupDeletedEvent) => + properties.groupId, + [Event.USER_GROUP_USERS_ADDED]: (properties: GroupUsersAddedEvent) => + properties.groupId, + [Event.USER_GROUP_USERS_REMOVED]: (properties: GroupUsersDeletedEvent) => + properties.groupId, + [Event.USER_GROUP_PERMISSIONS_EDITED]: ( + properties: GroupPermissionsEditedEvent + ) => properties.groupId, +} + +export function getDocumentId(event: Event, properties: any) { + const extractor = getEventProperties[event] + if (!extractor) { + throw new Error("Event does not have a method of document ID extraction") + } + return extractor(properties) +} diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index c2f7cf66ec..f02b9fdf32 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -1,7 +1,8 @@ -import { Event, AuditedEventFriendlyName } from "@budibase/types" +import { Event } from "@budibase/types" import { processors } from "./processors" import identification from "./identification" import * as backfill from "./backfill" +import { publishAsyncEvent } from "./asyncEvents" export const publishEvent = async ( event: Event, @@ -14,6 +15,14 @@ export const publishEvent = async ( const backfilling = await backfill.isBackfillingEvent(event) // no backfill - send the event and exit if (!backfilling) { + // send off async events if required + await publishAsyncEvent({ + event, + identity, + properties, + timestamp, + }) + // now handle the main sync event processing pipeline await processors.processEvent(event, identity, properties, timestamp) return } diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index 9534fb293d..5c02e5db9e 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -23,8 +23,6 @@ import * as installation from "../installation" import * as configs from "../configs" import { withCache, TTL, CacheKey } from "../cache/generic" -const pkg = require("../../package.json") - /** * An identity can be: * - account user (Self host) @@ -65,6 +63,7 @@ const getCurrentIdentity = async (): Promise => { hosting, installationId, tenantId, + realTenantId: context.getTenantId(), environment, } } else if (identityType === IdentityType.USER) { @@ -101,7 +100,7 @@ const identifyInstallationGroup = async ( const id = installId const type = IdentityType.INSTALLATION const hosting = getHostingFromEnv() - const version = pkg.version + const version = env.VERSION const environment = getDeploymentEnvironment() const group: InstallationGroup = { @@ -305,4 +304,5 @@ export default { identify, identifyGroup, getInstallationId, + getUniqueTenantId, } diff --git a/packages/backend-core/src/events/index.ts b/packages/backend-core/src/events/index.ts index d0d59a5b22..a238d72bac 100644 --- a/packages/backend-core/src/events/index.ts +++ b/packages/backend-core/src/events/index.ts @@ -6,6 +6,8 @@ export * as backfillCache from "./backfill" import { processors } from "./processors" +export function initAsyncEvents() {} + export const shutdown = () => { processors.shutdown() console.log("Events shutdown") diff --git a/packages/backend-core/src/events/processors/Processors.ts b/packages/backend-core/src/events/processors/Processors.ts index 4baedd909f..72de945d44 100644 --- a/packages/backend-core/src/events/processors/Processors.ts +++ b/packages/backend-core/src/events/processors/Processors.ts @@ -25,7 +25,9 @@ export default class Processor implements EventProcessor { timestamp?: string | number ): Promise { for (const eventProcessor of this.processors) { - await eventProcessor.identify(identity, timestamp) + if (eventProcessor.identify) { + await eventProcessor.identify(identity, timestamp) + } } } @@ -34,13 +36,17 @@ export default class Processor implements EventProcessor { timestamp?: string | number ): Promise { for (const eventProcessor of this.processors) { - await eventProcessor.identifyGroup(identity, timestamp) + if (eventProcessor.identifyGroup) { + await eventProcessor.identifyGroup(identity, timestamp) + } } } shutdown() { for (const eventProcessor of this.processors) { - eventProcessor.shutdown() + if (eventProcessor.shutdown) { + eventProcessor.shutdown() + } } } } diff --git a/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts b/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts new file mode 100644 index 0000000000..54304ee21b --- /dev/null +++ b/packages/backend-core/src/events/processors/async/DocumentUpdateProcessor.ts @@ -0,0 +1,43 @@ +import { EventProcessor } from "../types" +import { Event, Identity, DocUpdateEvent } from "@budibase/types" +import { doInTenant } from "../../../context" +import { getDocumentId } from "../../documentId" +import { shutdown } from "../../asyncEvents" + +export type Processor = (update: DocUpdateEvent) => Promise +export type ProcessorMap = { events: Event[]; processor: Processor }[] + +export default class DocumentUpdateProcessor implements EventProcessor { + processors: ProcessorMap = [] + + constructor(processors: ProcessorMap) { + this.processors = processors + } + + async processEvent( + event: Event, + identity: Identity, + properties: any, + timestamp?: string | number + ) { + const tenantId = identity.realTenantId + const docId = getDocumentId(event, properties) + if (!tenantId || !docId) { + return + } + for (let { events, processor } of this.processors) { + if (events.includes(event)) { + await doInTenant(tenantId, async () => { + await processor({ + id: docId, + tenantId, + }) + }) + } + } + } + + shutdown() { + return shutdown() + } +} diff --git a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts index 0dbe70d543..d37b85a9b8 100644 --- a/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts @@ -4,7 +4,6 @@ import { EventProcessor } from "../types" import env from "../../../environment" import * as context from "../../../context" import * as rateLimiting from "./rateLimiting" -const pkg = require("../../../../package.json") const EXCLUDED_EVENTS: Event[] = [ Event.USER_UPDATED, @@ -49,7 +48,7 @@ export default class PosthogProcessor implements EventProcessor { properties = this.clearPIIProperties(properties) - properties.version = pkg.version + properties.version = env.VERSION properties.service = env.SERVICE properties.environment = identity.environment properties.hosting = identity.hosting diff --git a/packages/backend-core/src/events/processors/types.ts b/packages/backend-core/src/events/processors/types.ts index f4066fe248..5256a1bc62 100644 --- a/packages/backend-core/src/events/processors/types.ts +++ b/packages/backend-core/src/events/processors/types.ts @@ -1,18 +1 @@ -import { Event, Identity, Group } from "@budibase/types" - -export enum EventProcessorType { - POSTHOG = "posthog", - LOGGING = "logging", -} - -export interface EventProcessor { - processEvent( - event: Event, - identity: Identity, - properties: any, - timestamp?: string | number - ): Promise - identify(identity: Identity, timestamp?: string | number): Promise - identifyGroup(group: Group, timestamp?: string | number): Promise - shutdown(): void -} +export { EventProcessor } from "@budibase/types" diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 30072196ba..40233b3827 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -27,6 +27,7 @@ export * as errors from "./errors" export * as timers from "./timers" export { default as env } from "./environment" export * as blacklist from "./blacklist" +export * as docUpdates from "./docUpdates" export { SearchParams } from "./db" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal diff --git a/packages/backend-core/src/installation.ts b/packages/backend-core/src/installation.ts index 64be6f3f43..17eda2004d 100644 --- a/packages/backend-core/src/installation.ts +++ b/packages/backend-core/src/installation.ts @@ -6,8 +6,7 @@ import { Installation, IdentityType, Database } from "@budibase/types" import * as context from "./context" import semver from "semver" import { bustCache, withCache, TTL, CacheKey } from "./cache/generic" - -const pkg = require("../package.json") +import environment from "./environment" export const getInstall = async (): Promise => { return withCache(CacheKey.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, { @@ -18,7 +17,7 @@ async function createInstallDoc(platformDb: Database) { const install: Installation = { _id: StaticDatabases.PLATFORM_INFO.docs.install, installId: newid(), - version: pkg.version, + version: environment.VERSION, } try { const resp = await platformDb.put(install) @@ -33,7 +32,7 @@ async function createInstallDoc(platformDb: Database) { } } -const getInstallFromDB = async (): Promise => { +export const getInstallFromDB = async (): Promise => { return doWithDB( StaticDatabases.PLATFORM_INFO.name, async (platformDb: any) => { @@ -80,7 +79,7 @@ export const checkInstallVersion = async (): Promise => { const install = await getInstall() const currentVersion = install.version - const newVersion = pkg.version + const newVersion = environment.VERSION if (currentVersion !== newVersion) { const isUpgrade = semver.gt(newVersion, currentVersion) diff --git a/packages/backend-core/src/logging/index.ts b/packages/backend-core/src/logging/index.ts index 276a8d627c..b229f47dea 100644 --- a/packages/backend-core/src/logging/index.ts +++ b/packages/backend-core/src/logging/index.ts @@ -1,5 +1,5 @@ export * as correlation from "./correlation/correlation" -export { default as logger } from "./pino/logger" +export { logger, disableLogger } from "./pino/logger" export * from "./alerts" // turn off or on context logging i.e. tenantId, appId etc diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index c82876f49a..dd4b505d30 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -5,6 +5,17 @@ import * as correlation from "../correlation" import { IdentityType } from "@budibase/types" import { LOG_CONTEXT } from "../index" +// CORE LOGGERS - for disabling + +const BUILT_INS = { + log: console.log, + error: console.error, + info: console.info, + warn: console.warn, + trace: console.trace, + debug: console.debug, +} + // LOGGER const pinoOptions: LoggerOptions = { @@ -31,6 +42,15 @@ if (env.isDev()) { export const logger = pino(pinoOptions) +export function disableLogger() { + console.log = BUILT_INS.log + console.error = BUILT_INS.error + console.info = BUILT_INS.info + console.warn = BUILT_INS.warn + console.trace = BUILT_INS.trace + console.debug = BUILT_INS.debug +} + // CONSOLE OVERRIDES interface MergingObject { @@ -166,5 +186,3 @@ const getIdentity = () => { } return identity } - -export default logger diff --git a/packages/backend-core/src/logging/pino/middleware.ts b/packages/backend-core/src/logging/pino/middleware.ts index e9d37ab692..569420c5f2 100644 --- a/packages/backend-core/src/logging/pino/middleware.ts +++ b/packages/backend-core/src/logging/pino/middleware.ts @@ -1,5 +1,5 @@ import env from "../../environment" -import logger from "./logger" +import { logger } from "./logger" import { IncomingMessage } from "http" const pino = require("koa-pino-logger") import { Options } from "pino-http" diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index f877985ee0..8bd6591d05 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -44,7 +44,7 @@ async function checkApiKey(apiKey: string, populateUser?: Function) { // check both the primary and the fallback internal api keys // this allows for rotation if (isValidInternalAPIKey(apiKey)) { - return { valid: true } + return { valid: true, user: undefined } } const decrypted = decrypt(apiKey) const tenantId = decrypted.split(SEPARATOR)[0] diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 059e1b228d..4ac3641de1 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -3,7 +3,7 @@ import AWS from "aws-sdk" import stream from "stream" import fetch from "node-fetch" import tar from "tar-fs" -const zlib = require("zlib") +import zlib from "zlib" import { promisify } from "util" import { join } from "path" import fs from "fs" @@ -415,7 +415,7 @@ export const downloadTarballDirect = async ( throw new Error(`unexpected response ${response.statusText}`) } - await streamPipeline(response.body, zlib.Unzip(), tar.extract(path)) + await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path)) } export const downloadTarball = async ( @@ -431,7 +431,7 @@ export const downloadTarball = async ( } const tmpPath = join(budibaseTempDir(), path) - await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath)) + await streamPipeline(response.body, zlib.createUnzip(), tar.extract(tmpPath)) if (!env.isTest() && env.SELF_HOSTED) { await uploadDirectory(bucketName, tmpPath, path) } diff --git a/packages/backend-core/src/plugin/tests/validation.spec.ts b/packages/backend-core/src/plugin/tests/validation.spec.ts new file mode 100644 index 0000000000..0fea009645 --- /dev/null +++ b/packages/backend-core/src/plugin/tests/validation.spec.ts @@ -0,0 +1,83 @@ +import { validate } from "../utils" +import fetch from "node-fetch" +import { PluginType } from "@budibase/types" + +const repoUrl = + "https://raw.githubusercontent.com/Budibase/budibase-skeleton/master" +const automationLink = `${repoUrl}/automation/schema.json.hbs` +const componentLink = `${repoUrl}/component/schema.json.hbs` +const datasourceLink = `${repoUrl}/datasource/schema.json.hbs` + +async function getSchema(link: string) { + const response = await fetch(link) + if (response.status > 300) { + return + } + const text = await response.text() + return JSON.parse(text) +} + +async function runTest(opts: { link?: string; schema?: any }) { + let error + try { + let schema = opts.schema + if (opts.link) { + schema = await getSchema(opts.link) + } + validate(schema) + } catch (err) { + error = err + } + return error +} + +describe("it should be able to validate an automation schema", () => { + it("should return automation skeleton schema is valid", async () => { + const error = await runTest({ link: automationLink }) + expect(error).toBeUndefined() + }) + + it("should fail given invalid automation schema", async () => { + const error = await runTest({ + schema: { + type: PluginType.AUTOMATION, + schema: {}, + }, + }) + expect(error).toBeDefined() + }) +}) + +describe("it should be able to validate a component schema", () => { + it("should return component skeleton schema is valid", async () => { + const error = await runTest({ link: componentLink }) + expect(error).toBeUndefined() + }) + + it("should fail given invalid component schema", async () => { + const error = await runTest({ + schema: { + type: PluginType.COMPONENT, + schema: {}, + }, + }) + expect(error).toBeDefined() + }) +}) + +describe("it should be able to validate a datasource schema", () => { + it("should return datasource skeleton schema is valid", async () => { + const error = await runTest({ link: datasourceLink }) + expect(error).toBeUndefined() + }) + + it("should fail given invalid datasource schema", async () => { + const error = await runTest({ + schema: { + type: PluginType.DATASOURCE, + schema: {}, + }, + }) + expect(error).toBeDefined() + }) +}) diff --git a/packages/backend-core/src/plugin/utils.ts b/packages/backend-core/src/plugin/utils.ts index 7b62248bb5..f73ded0659 100644 --- a/packages/backend-core/src/plugin/utils.ts +++ b/packages/backend-core/src/plugin/utils.ts @@ -1,4 +1,12 @@ -import { DatasourceFieldType, QueryType, PluginType } from "@budibase/types" +import { + DatasourceFieldType, + QueryType, + PluginType, + AutomationStepType, + AutomationStepIdArray, + AutomationIOType, + AutomationCustomIOType, +} from "@budibase/types" import joi from "joi" const DATASOURCE_TYPES = [ @@ -19,7 +27,7 @@ function runJoi(validator: joi.Schema, schema: any) { function validateComponent(schema: any) { const validator = joi.object({ - type: joi.string().allow("component").required(), + type: joi.string().allow(PluginType.COMPONENT).required(), metadata: joi.object().unknown(true).required(), hash: joi.string().optional(), version: joi.string().optional(), @@ -53,7 +61,7 @@ function validateDatasource(schema: any) { .required() const validator = joi.object({ - type: joi.string().allow("datasource").required(), + type: joi.string().allow(PluginType.DATASOURCE).required(), metadata: joi.object().unknown(true).required(), hash: joi.string().optional(), version: joi.string().optional(), @@ -82,6 +90,55 @@ function validateDatasource(schema: any) { runJoi(validator, schema) } +function validateAutomation(schema: any) { + const basePropsValidator = joi.object().pattern(joi.string(), { + type: joi + .string() + .allow(...Object.values(AutomationIOType)) + .required(), + customType: joi.string().allow(...Object.values(AutomationCustomIOType)), + title: joi.string(), + description: joi.string(), + enum: joi.array().items(joi.string()), + pretty: joi.array().items(joi.string()), + }) + const stepSchemaValidator = joi + .object({ + properties: basePropsValidator, + required: joi.array().items(joi.string()), + }) + .concat(basePropsValidator) + .required() + const validator = joi.object({ + type: joi.string().allow(PluginType.AUTOMATION).required(), + metadata: joi.object().unknown(true).required(), + hash: joi.string().optional(), + version: joi.string().optional(), + schema: joi.object({ + name: joi.string().required(), + tagline: joi.string().required(), + icon: joi.string().required(), + description: joi.string().required(), + type: joi + .string() + .allow(AutomationStepType.ACTION, AutomationStepType.LOGIC) + .required(), + stepId: joi + .string() + .disallow(...AutomationStepIdArray) + .required(), + inputs: joi.object().optional(), + schema: joi + .object({ + inputs: stepSchemaValidator, + outputs: stepSchemaValidator, + }) + .required(), + }), + }) + runJoi(validator, schema) +} + export function validate(schema: any) { switch (schema?.type) { case PluginType.COMPONENT: @@ -90,6 +147,9 @@ export function validate(schema: any) { case PluginType.DATASOURCE: validateDatasource(schema) break + case PluginType.AUTOMATION: + validateAutomation(schema) + break default: throw new Error(`Unknown plugin type - check schema.json: ${schema.type}`) } diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts index 9261ed1176..e1ffcfee36 100644 --- a/packages/backend-core/src/queue/constants.ts +++ b/packages/backend-core/src/queue/constants.ts @@ -2,4 +2,5 @@ export enum JobQueue { AUTOMATION = "automationQueue", APP_BACKUP = "appBackupQueue", AUDIT_LOG = "auditLogQueue", + SYSTEM_EVENT_QUEUE = "systemEventQueue", } diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 5e71488689..4e9cd569ed 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -1,10 +1,16 @@ -import Redlock, { Options } from "redlock" +import Redlock from "redlock" import { getLockClient } from "./init" import { LockOptions, LockType } from "@budibase/types" import * as context from "../context" import env from "../environment" -const getClient = async (type: LockType): Promise => { +const getClient = async ( + type: LockType, + opts?: Redlock.Options +): Promise => { + if (type === LockType.CUSTOM) { + return newRedlock(opts) + } if (env.isTest() && type !== LockType.TRY_ONCE) { return newRedlock(OPTIONS.TEST) } @@ -56,7 +62,7 @@ const OPTIONS = { }, } -const newRedlock = async (opts: Options = {}) => { +const newRedlock = async (opts: Redlock.Options = {}) => { let options = { ...OPTIONS.DEFAULT, ...opts } const redisWrapper = await getLockClient() const client = redisWrapper.getClient() diff --git a/packages/backend-core/src/security/permissions.ts b/packages/backend-core/src/security/permissions.ts index 42189bba0c..6cacc12dd6 100644 --- a/packages/backend-core/src/security/permissions.ts +++ b/packages/backend-core/src/security/permissions.ts @@ -24,7 +24,7 @@ export enum PermissionType { QUERY = "query", } -class Permission { +export class Permission { type: PermissionType level: PermissionLevel @@ -34,7 +34,7 @@ class Permission { } } -function levelToNumber(perm: PermissionLevel) { +export function levelToNumber(perm: PermissionLevel) { switch (perm) { // not everything has execute privileges case PermissionLevel.EXECUTE: @@ -55,7 +55,7 @@ function levelToNumber(perm: PermissionLevel) { * @param {string} userPermLevel The permission level of the user. * @return {string[]} All the permission levels this user is allowed to carry out. */ -function getAllowedLevels(userPermLevel: PermissionLevel) { +export function getAllowedLevels(userPermLevel: PermissionLevel): string[] { switch (userPermLevel) { case PermissionLevel.EXECUTE: return [PermissionLevel.EXECUTE] @@ -64,9 +64,9 @@ function getAllowedLevels(userPermLevel: PermissionLevel) { case PermissionLevel.WRITE: case PermissionLevel.ADMIN: return [ + PermissionLevel.EXECUTE, PermissionLevel.READ, PermissionLevel.WRITE, - PermissionLevel.EXECUTE, ] default: return [] @@ -81,7 +81,7 @@ export enum BuiltinPermissionID { POWER = "power", } -const BUILTIN_PERMISSIONS = { +export const BUILTIN_PERMISSIONS = { PUBLIC: { _id: BuiltinPermissionID.PUBLIC, name: "Public", diff --git a/packages/backend-core/src/security/tests/permissions.spec.ts b/packages/backend-core/src/security/tests/permissions.spec.ts new file mode 100644 index 0000000000..caf8bb29a6 --- /dev/null +++ b/packages/backend-core/src/security/tests/permissions.spec.ts @@ -0,0 +1,145 @@ +import { cloneDeep } from "lodash" +import * as permissions from "../permissions" +import { BUILTIN_ROLE_IDS } from "../roles" + +describe("levelToNumber", () => { + it("should return 0 for EXECUTE", () => { + expect(permissions.levelToNumber(permissions.PermissionLevel.EXECUTE)).toBe( + 0 + ) + }) + + it("should return 1 for READ", () => { + expect(permissions.levelToNumber(permissions.PermissionLevel.READ)).toBe(1) + }) + + it("should return 2 for WRITE", () => { + expect(permissions.levelToNumber(permissions.PermissionLevel.WRITE)).toBe(2) + }) + + it("should return 3 for ADMIN", () => { + expect(permissions.levelToNumber(permissions.PermissionLevel.ADMIN)).toBe(3) + }) + + it("should return -1 for an unknown permission level", () => { + expect( + permissions.levelToNumber("unknown" as permissions.PermissionLevel) + ).toBe(-1) + }) +}) +describe("getAllowedLevels", () => { + it('should return ["execute"] for EXECUTE', () => { + expect( + permissions.getAllowedLevels(permissions.PermissionLevel.EXECUTE) + ).toEqual([permissions.PermissionLevel.EXECUTE]) + }) + + it('should return ["execute", "read"] for READ', () => { + expect( + permissions.getAllowedLevels(permissions.PermissionLevel.READ) + ).toEqual([ + permissions.PermissionLevel.EXECUTE, + permissions.PermissionLevel.READ, + ]) + }) + + it('should return ["execute", "read", "write"] for WRITE', () => { + expect( + permissions.getAllowedLevels(permissions.PermissionLevel.WRITE) + ).toEqual([ + permissions.PermissionLevel.EXECUTE, + permissions.PermissionLevel.READ, + permissions.PermissionLevel.WRITE, + ]) + }) + + it('should return ["execute", "read", "write"] for ADMIN', () => { + expect( + permissions.getAllowedLevels(permissions.PermissionLevel.ADMIN) + ).toEqual([ + permissions.PermissionLevel.EXECUTE, + permissions.PermissionLevel.READ, + permissions.PermissionLevel.WRITE, + ]) + }) + + it("should return [] for an unknown permission level", () => { + expect( + permissions.getAllowedLevels("unknown" as permissions.PermissionLevel) + ).toEqual([]) + }) +}) + +describe("doesHaveBasePermission", () => { + it("should return true if base permission has the required level", () => { + const permType = permissions.PermissionType.USER + const permLevel = permissions.PermissionLevel.READ + const rolesHierarchy = [ + { + roleId: BUILTIN_ROLE_IDS.ADMIN, + permissionId: permissions.BuiltinPermissionID.ADMIN, + }, + ] + expect( + permissions.doesHaveBasePermission(permType, permLevel, rolesHierarchy) + ).toBe(true) + }) + + it("should return false if base permission does not have the required level", () => { + const permType = permissions.PermissionType.APP + const permLevel = permissions.PermissionLevel.READ + const rolesHierarchy = [ + { + roleId: BUILTIN_ROLE_IDS.PUBLIC, + permissionId: permissions.BuiltinPermissionID.PUBLIC, + }, + ] + expect( + permissions.doesHaveBasePermission(permType, permLevel, rolesHierarchy) + ).toBe(false) + }) +}) + +describe("isPermissionLevelHigherThanRead", () => { + it("should return true if level is higher than read", () => { + expect( + permissions.isPermissionLevelHigherThanRead( + permissions.PermissionLevel.WRITE + ) + ).toBe(true) + }) + + it("should return false if level is read or lower", () => { + expect( + permissions.isPermissionLevelHigherThanRead( + permissions.PermissionLevel.READ + ) + ).toBe(false) + }) +}) + +describe("getBuiltinPermissions", () => { + it("returns a clone of the builtin permissions", () => { + const builtins = permissions.getBuiltinPermissions() + expect(builtins).toEqual(cloneDeep(permissions.BUILTIN_PERMISSIONS)) + expect(builtins).not.toBe(permissions.BUILTIN_PERMISSIONS) + }) +}) + +describe("getBuiltinPermissionByID", () => { + it("returns correct permission object for valid ID", () => { + const expectedPermission = { + _id: permissions.BuiltinPermissionID.PUBLIC, + name: "Public", + permissions: [ + new permissions.Permission( + permissions.PermissionType.WEBHOOK, + permissions.PermissionLevel.EXECUTE + ), + ], + } + expect(permissions.getBuiltinPermissionByID("public")).toEqual( + expectedPermission + ) + }) +}) diff --git a/packages/backend-core/src/users.ts b/packages/backend-core/src/users.ts index c7d8a94e95..166136df3c 100644 --- a/packages/backend-core/src/users.ts +++ b/packages/backend-core/src/users.ts @@ -1,15 +1,16 @@ import { - ViewName, - getUsersByAppParams, - getProdAppID, - generateAppUserID, - queryGlobalView, - UNICODE_MAX, - DocumentType, - SEPARATOR, directCouchFind, + DocumentType, + generateAppUserID, getGlobalUserParams, + getProdAppID, + getUsersByAppParams, pagination, + queryGlobalView, + queryGlobalViewRaw, + SEPARATOR, + UNICODE_MAX, + ViewName, } from "./db" import { BulkDocsResponse, SearchUsersRequest, User } from "@budibase/types" import { getGlobalDB } from "./context" @@ -239,3 +240,11 @@ export const paginatedUsers = async ({ getKey, }) } + +export async function getUserCount() { + const response = await queryGlobalViewRaw(ViewName.USER_BY_EMAIL, { + limit: 0, // to be as fast as possible - we just want the total rows count + include_docs: false, + }) + return response.total_rows +} diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 7c222a9831..75b098093b 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -46,8 +46,9 @@ export async function resolveAppUrl(ctx: Ctx) { } // search prod apps for a url that matches - const apps: App[] = await context.doInTenant(tenantId, () => - getAllApps({ dev: false }) + const apps: App[] = await context.doInTenant( + tenantId, + () => getAllApps({ dev: false }) as Promise ) const app = apps.filter( a => a.url && a.url.toLowerCase() === possibleAppUrl @@ -221,27 +222,6 @@ export function isClient(ctx: Ctx) { return ctx.headers[Header.TYPE] === "client" } -async function getBuilders() { - const builders = await queryGlobalView(ViewName.USER_BY_BUILDERS, { - include_docs: false, - }) - - if (!builders) { - return [] - } - - if (Array.isArray(builders)) { - return builders - } else { - return [builders] - } -} - -export async function getBuildersCount() { - const builders = await getBuilders() - return builders.length -} - export function timeout(timeMs: number) { return new Promise(resolve => setTimeout(resolve, timeMs)) } diff --git a/packages/backend-core/tests/core/utilities/index.ts b/packages/backend-core/tests/core/utilities/index.ts index eee41cc3d4..787d69be2c 100644 --- a/packages/backend-core/tests/core/utilities/index.ts +++ b/packages/backend-core/tests/core/utilities/index.ts @@ -2,5 +2,5 @@ export * as mocks from "./mocks" export * as structures from "./structures" export { generator } from "./structures" export * as testContainerUtils from "./testContainerUtils" - +export * as utils from "./utils" export * from "./jestUtils" diff --git a/packages/backend-core/tests/core/utilities/mocks/events.ts b/packages/backend-core/tests/core/utilities/mocks/events.ts index 17e35a5d0c..dacf7dcce8 100644 --- a/packages/backend-core/tests/core/utilities/mocks/events.ts +++ b/packages/backend-core/tests/core/utilities/mocks/events.ts @@ -1,3 +1,5 @@ +import * as events from "../../../../src/events" + beforeAll(async () => { const processors = await import("../../../../src/events/processors") const events = await import("../../../../src/events") @@ -120,4 +122,13 @@ beforeAll(async () => { jest.spyOn(events.plugin, "init") jest.spyOn(events.plugin, "imported") jest.spyOn(events.plugin, "deleted") + + jest.spyOn(events.license, "tierChanged") + jest.spyOn(events.license, "planChanged") + jest.spyOn(events.license, "activated") + jest.spyOn(events.license, "checkoutOpened") + jest.spyOn(events.license, "checkoutSuccess") + jest.spyOn(events.license, "portalOpened") + jest.spyOn(events.license, "paymentFailed") + jest.spyOn(events.license, "paymentRecovered") }) diff --git a/packages/backend-core/tests/core/utilities/mocks/fetch.ts b/packages/backend-core/tests/core/utilities/mocks/fetch.ts index 287280067a..f7447d2c47 100644 --- a/packages/backend-core/tests/core/utilities/mocks/fetch.ts +++ b/packages/backend-core/tests/core/utilities/mocks/fetch.ts @@ -1,7 +1,7 @@ const mockFetch = jest.fn((url: any, opts: any) => { const fetch = jest.requireActual("node-fetch") const env = jest.requireActual("../../../../src/environment").default - if (url.includes(env.COUCH_DB_URL)) { + if (url.includes(env.COUCH_DB_URL) || url.includes("raw.github")) { return fetch(url, opts) } return undefined diff --git a/packages/backend-core/tests/core/utilities/structures/Chance.ts b/packages/backend-core/tests/core/utilities/structures/Chance.ts new file mode 100644 index 0000000000..73d7ba102f --- /dev/null +++ b/packages/backend-core/tests/core/utilities/structures/Chance.ts @@ -0,0 +1,20 @@ +import Chance from "chance" + +export default class CustomChance extends Chance { + arrayOf( + generateFn: () => T, + opts: { min?: number; max?: number } = {} + ): T[] { + const itemCount = this.integer({ + min: opts.min != null ? opts.min : 1, + max: opts.max != null ? opts.max : 50, + }) + + const items = [] + for (let i = 0; i < itemCount; i++) { + items.push(generateFn()) + } + + return items + } +} diff --git a/packages/backend-core/tests/core/utilities/structures/accounts.ts b/packages/backend-core/tests/core/utilities/structures/accounts.ts index 30ef6e4192..807153cd09 100644 --- a/packages/backend-core/tests/core/utilities/structures/accounts.ts +++ b/packages/backend-core/tests/core/utilities/structures/accounts.ts @@ -1,4 +1,4 @@ -import { generator, uuid } from "." +import { generator, uuid, quotas } from "." import { generateGlobalUserID } from "../../../../src/docIds" import { Account, @@ -28,6 +28,7 @@ export const account = (): Account => { name: generator.name(), size: "10+", profession: "Software Engineer", + quotaUsage: quotas.usage(), } } diff --git a/packages/backend-core/tests/core/utilities/structures/generator.ts b/packages/backend-core/tests/core/utilities/structures/generator.ts index 51567b152e..ed4dac8255 100644 --- a/packages/backend-core/tests/core/utilities/structures/generator.ts +++ b/packages/backend-core/tests/core/utilities/structures/generator.ts @@ -1,2 +1,2 @@ -import Chance from "chance" +import Chance from "./Chance" export const generator = new Chance() diff --git a/packages/backend-core/tests/core/utilities/structures/index.ts b/packages/backend-core/tests/core/utilities/structures/index.ts index 5592a7e1f9..2c094f43a7 100644 --- a/packages/backend-core/tests/core/utilities/structures/index.ts +++ b/packages/backend-core/tests/core/utilities/structures/index.ts @@ -11,3 +11,4 @@ export * as users from "./users" export * as userGroups from "./userGroups" export { generator } from "./generator" export * as scim from "./scim" +export * as quotas from "./quotas" diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts index a541e91860..24b120451e 100644 --- a/packages/backend-core/tests/core/utilities/structures/licenses.ts +++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts @@ -1,18 +1,132 @@ -import { AccountPlan, License, PlanType, Quotas } from "@budibase/types" +import { + Billing, + Customer, + Feature, + License, + PlanModel, + PlanType, + PriceDuration, + PurchasedPlan, + Quotas, + Subscription, +} from "@budibase/types" -const newPlan = (type: PlanType = PlanType.FREE): AccountPlan => { +export const plan = (type: PlanType = PlanType.FREE): PurchasedPlan => { return { type, + usesInvoicing: false, + minUsers: 1, + model: PlanModel.PER_USER, } } -export const newLicense = (opts: { - quotas: Quotas - planType?: PlanType -}): License => { +export function quotas(): Quotas { return { - features: [], - quotas: opts.quotas, - plan: newPlan(opts.planType), + usage: { + monthly: { + queries: { + name: "Queries", + value: 1, + triggers: [], + }, + automations: { + name: "Queries", + value: 1, + triggers: [], + }, + dayPasses: { + name: "Queries", + value: 1, + triggers: [], + }, + }, + static: { + rows: { + name: "Rows", + value: 1, + triggers: [], + }, + apps: { + name: "Apps", + value: 1, + triggers: [], + }, + users: { + name: "Users", + value: 1, + triggers: [], + }, + userGroups: { + name: "User Groups", + value: 1, + triggers: [], + }, + plugins: { + name: "Plugins", + value: 1, + triggers: [], + }, + }, + }, + constant: { + automationLogRetentionDays: { + name: "Automation Logs", + value: 1, + triggers: [], + }, + appBackupRetentionDays: { + name: "Backups", + value: 1, + triggers: [], + }, + }, + } +} + +export function billing( + opts: { customer?: Customer; subscription?: Subscription } = {} +): Billing { + return { + customer: opts.customer || customer(), + subscription: opts.subscription || subscription(), + } +} + +export function customer(): Customer { + return { + balance: 0, + currency: "usd", + } +} + +export function subscription(): Subscription { + return { + amount: 10000, + cancelAt: undefined, + currency: "usd", + currentPeriodEnd: 0, + currentPeriodStart: 0, + downgradeAt: 0, + duration: PriceDuration.MONTHLY, + pastDueAt: undefined, + quantity: 0, + status: "active", + } +} + +export const license = ( + opts: { + quotas?: Quotas + plan?: PurchasedPlan + planType?: PlanType + features?: Feature[] + billing?: Billing + } = {} +): License => { + return { + features: opts.features || [], + quotas: opts.quotas || quotas(), + plan: opts.plan || plan(opts.planType), + billing: opts.billing || billing(), } } diff --git a/packages/backend-core/tests/core/utilities/structures/quotas.ts b/packages/backend-core/tests/core/utilities/structures/quotas.ts new file mode 100644 index 0000000000..e82117053f --- /dev/null +++ b/packages/backend-core/tests/core/utilities/structures/quotas.ts @@ -0,0 +1,67 @@ +import { MonthlyQuotaName, QuotaUsage } from "@budibase/types" + +export const usage = (): QuotaUsage => { + return { + _id: "usage_quota", + quotaReset: new Date().toISOString(), + apps: { + app_1: { + // @ts-ignore - the apps definition doesn't match up to actual usage + usageQuota: { + rows: 0, + }, + }, + }, + monthly: { + "01-2023": { + automations: 0, + dayPasses: 0, + queries: 0, + triggers: {}, + breakdown: { + rowQueries: { + parent: MonthlyQuotaName.QUERIES, + values: { + row_1: 0, + row_2: 0, + }, + }, + datasourceQueries: { + parent: MonthlyQuotaName.QUERIES, + values: { + ds_1: 0, + ds_2: 0, + }, + }, + automations: { + parent: MonthlyQuotaName.AUTOMATIONS, + values: { + auto_1: 0, + auto_2: 0, + }, + }, + }, + }, + "02-2023": { + automations: 0, + dayPasses: 0, + queries: 0, + triggers: {}, + }, + current: { + automations: 0, + dayPasses: 0, + queries: 0, + triggers: {}, + }, + }, + usageQuota: { + apps: 0, + plugins: 0, + users: 0, + userGroups: 0, + rows: 0, + triggers: {}, + }, + } +} diff --git a/packages/backend-core/tests/core/utilities/utils/index.ts b/packages/backend-core/tests/core/utilities/utils/index.ts new file mode 100644 index 0000000000..41a249c7e6 --- /dev/null +++ b/packages/backend-core/tests/core/utilities/utils/index.ts @@ -0,0 +1 @@ +export * as time from "./time" diff --git a/packages/backend-core/tests/core/utilities/utils/time.ts b/packages/backend-core/tests/core/utilities/utils/time.ts new file mode 100644 index 0000000000..c71167bc72 --- /dev/null +++ b/packages/backend-core/tests/core/utilities/utils/time.ts @@ -0,0 +1,3 @@ +export function addDaysToDate(date: Date, days: number) { + return new Date(date.getTime() + days * 24 * 60 * 60 * 1000) +} diff --git a/packages/backend-core/tsconfig.build.json b/packages/backend-core/tsconfig.build.json index 12f8255a7c..bfbed31e23 100644 --- a/packages/backend-core/tsconfig.build.json +++ b/packages/backend-core/tsconfig.build.json @@ -10,15 +10,11 @@ "incremental": true, "sourceMap": true, "declaration": true, - "types": [ "node", "jest" ], + "types": ["node", "jest"], "outDir": "dist", "skipLibCheck": true }, - "include": [ - "**/*.js", - "**/*.ts", - "package.json" - ], + "include": ["**/*.js", "**/*.ts"], "exclude": [ "node_modules", "dist", @@ -26,4 +22,4 @@ "**/*.spec.js", "__mocks__" ] -} \ No newline at end of file +} diff --git a/packages/bbui/package.json b/packages/bbui/package.json index fd05734559..7664855326 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.5.5-alpha.0", + "version": "2.5.6-alpha.32", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,8 +38,8 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/shared-core": "2.5.5-alpha.0", - "@budibase/string-templates": "2.5.5-alpha.0", + "@budibase/shared-core": "2.5.6-alpha.32", + "@budibase/string-templates": "2.5.6-alpha.32", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", @@ -84,7 +84,7 @@ "@spectrum-css/vars": "3.0.1", "dayjs": "^1.10.4", "easymde": "^2.16.1", - "svelte-flatpickr": "^3.2.3", + "svelte-flatpickr": "^3.3.2", "svelte-portal": "^1.0.0" }, "resolutions": { diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte index 60c8bec80b..e9eb3b3471 100644 --- a/packages/bbui/src/ActionButton/ActionButton.svelte +++ b/packages/bbui/src/ActionButton/ActionButton.svelte @@ -71,7 +71,7 @@ {/if} {#if icon}
+
{#if selectedImage} {#if gallery} @@ -57,7 +59,6 @@ --spectrum-semantic-negative-icon-color: #e34850; min-width: 100px; margin: 0; - border-color: var(--spectrum-global-color-gray-400); border-width: 1px; } diff --git a/packages/bbui/src/Label/Label.svelte b/packages/bbui/src/Label/Label.svelte index 261ca946ea..71b0967d99 100644 --- a/packages/bbui/src/Label/Label.svelte +++ b/packages/bbui/src/Label/Label.svelte @@ -9,6 +9,7 @@
{/if} - - {#if !$builderStore.inBuilder && licensing.logoEnabled() && $environmentStore.cloud} - - {/if} -
diff --git a/packages/client/src/components/app/Link.svelte b/packages/client/src/components/app/Link.svelte index 79bf296208..6cabcec7df 100644 --- a/packages/client/src/components/app/Link.svelte +++ b/packages/client/src/components/app/Link.svelte @@ -1,7 +1,8 @@ diff --git a/packages/client/src/sdk.js b/packages/client/src/sdk.js index 264cc85626..c9ff1eba36 100644 --- a/packages/client/src/sdk.js +++ b/packages/client/src/sdk.js @@ -12,6 +12,7 @@ import { environmentStore, sidePanelStore, dndIsDragging, + confirmationStore, } from "stores" import { styleable } from "utils/styleable" import { linkable } from "utils/linkable" @@ -35,6 +36,7 @@ export default { sidePanelStore, dndIsDragging, currentRole, + confirmationStore, styleable, linkable, getAction, diff --git a/packages/client/src/utils/domDebounce.js b/packages/client/src/utils/domDebounce.js index b7fc017247..b15d2698b4 100644 --- a/packages/client/src/utils/domDebounce.js +++ b/packages/client/src/utils/domDebounce.js @@ -1,12 +1,14 @@ -export const domDebounce = callback => { +export const domDebounce = (callback, extractParams = x => x) => { let active = false - return e => { + let lastParams + return (...params) => { + lastParams = extractParams(...params) if (!active) { - window.requestAnimationFrame(() => { - callback(e) + active = true + requestAnimationFrame(() => { + callback(lastParams) active = false }) - active = true } } } diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 4a6004b0ed..70cec3f9e0 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,14 +1,16 @@ { "name": "@budibase/frontend-core", - "version": "2.5.5-alpha.0", + "version": "2.5.6-alpha.32", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "2.5.5-alpha.0", - "@budibase/shared-core": "2.5.5-alpha.0", + "@budibase/bbui": "2.5.6-alpha.32", + "@budibase/shared-core": "2.5.6-alpha.32", + "dayjs": "^1.11.7", "lodash": "^4.17.21", + "socket.io-client": "^4.6.1", "svelte": "^3.46.2" } } diff --git a/packages/frontend-core/src/api/other.js b/packages/frontend-core/src/api/other.js index ac4b481395..3d171eaab4 100644 --- a/packages/frontend-core/src/api/other.js +++ b/packages/frontend-core/src/api/other.js @@ -1,13 +1,4 @@ export const buildOtherEndpoints = API => ({ - /** - * TODO: find out what this is - */ - checkImportComplete: async () => { - return await API.get({ - url: "/api/cloud/import/complete", - }) - }, - /** * Gets the current environment details. */ diff --git a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte new file mode 100644 index 0000000000..4d830723c2 --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte @@ -0,0 +1,154 @@ + + +
+ {#each value || [] as attachment} + {#if isImage(attachment.extension)} + {attachment.extension} + {:else} +
+ {attachment.extension} +
+ {/if} + {/each} +
+ +{#if isOpen} +
+ onChange(e.detail)} + {processFiles} + {deleteAttachments} + {handleFileTooLarge} + /> +
+{/if} + + diff --git a/packages/frontend-core/src/components/grid/cells/BooleanCell.svelte b/packages/frontend-core/src/components/grid/cells/BooleanCell.svelte new file mode 100644 index 0000000000..52aecb07a7 --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/BooleanCell.svelte @@ -0,0 +1,44 @@ + + +
+ +
+ + diff --git a/packages/frontend-core/src/components/grid/cells/DataCell.svelte b/packages/frontend-core/src/components/grid/cells/DataCell.svelte new file mode 100644 index 0000000000..5a2e02340f --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/DataCell.svelte @@ -0,0 +1,88 @@ + + + focusedCellId.set(cellId)} + on:contextmenu={e => menu.actions.open(cellId, e)} + width={column.width} +> + + diff --git a/packages/frontend-core/src/components/grid/cells/DateCell.svelte b/packages/frontend-core/src/components/grid/cells/DateCell.svelte new file mode 100644 index 0000000000..0112bcda15 --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/DateCell.svelte @@ -0,0 +1,77 @@ + + +
+
+ {#if value} + {dayjs(timeOnly ? time : value).format(format)} + {/if} +
+ {#if editable} + + {/if} +
+ +{#if editable} +
+ onChange(e.detail)} + appendTo={document.documentElement} + enableTime={!dateOnly} + {timeOnly} + time24hr + ignoreTimezones={schema.ignoreTimezones} + /> +
+{/if} + + diff --git a/packages/frontend-core/src/components/grid/cells/FormulaCell.svelte b/packages/frontend-core/src/components/grid/cells/FormulaCell.svelte new file mode 100644 index 0000000000..b4db795e44 --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/FormulaCell.svelte @@ -0,0 +1,5 @@ + + + diff --git a/packages/frontend-core/src/components/grid/cells/GridCell.svelte b/packages/frontend-core/src/components/grid/cells/GridCell.svelte new file mode 100644 index 0000000000..dfc53f6f0c --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/GridCell.svelte @@ -0,0 +1,162 @@ + + +
+ {#if error} +
+ {error} +
+ {/if} + + {#if selectedUser && !focused} +
+ {selectedUser.label} +
+ {/if} +
+ + diff --git a/packages/frontend-core/src/components/grid/cells/GutterCell.svelte b/packages/frontend-core/src/components/grid/cells/GutterCell.svelte new file mode 100644 index 0000000000..d9fd09fb6c --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/GutterCell.svelte @@ -0,0 +1,127 @@ + + + +
+ {#if $$slots.default} + + {:else} +
+ +
+ {#if !disableNumber} +
+ {row.__idx + 1} +
+ {/if} + {/if} + {#if $config.allowExpandRows} +
+ +
+ {/if} +
+
+ + diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte new file mode 100644 index 0000000000..165711c51f --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte @@ -0,0 +1,246 @@ + + +
+ + +
+ {column.label} +
+ {#if sortedBy} +
+ +
+ {/if} +
(open = true)}> + +
+
+
+ + + + + Edit column + + + Use as display column + + + Sort A-Z + + + Sort Z-A + + + Move left + + + Move right + + Hide column + + + + diff --git a/packages/frontend-core/src/components/grid/cells/JSONCell.svelte b/packages/frontend-core/src/components/grid/cells/JSONCell.svelte new file mode 100644 index 0000000000..30803fd862 --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/JSONCell.svelte @@ -0,0 +1,36 @@ + + + diff --git a/packages/frontend-core/src/components/grid/cells/LongFormCell.svelte b/packages/frontend-core/src/components/grid/cells/LongFormCell.svelte new file mode 100644 index 0000000000..00e12dc6a3 --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/LongFormCell.svelte @@ -0,0 +1,120 @@ + + +{#if isOpen} +