diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 6f55269dd6..a50d0ac552 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -26,6 +26,7 @@ "aws-sdk": "2.1030.0", "bcrypt": "5.0.1", "bcryptjs": "2.4.3", + "bull": "4.10.1", "dotenv": "16.0.1", "emitter-listener": "1.1.2", "ioredis": "4.28.0", @@ -63,6 +64,7 @@ }, "devDependencies": { "@types/chance": "1.1.3", + "@types/ioredis": "4.28.0", "@types/jest": "27.5.1", "@types/koa": "2.0.52", "@types/lodash": "4.14.180", diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts index 35eeee608b..7efe0e23f7 100644 --- a/packages/backend-core/src/context/index.ts +++ b/packages/backend-core/src/context/index.ts @@ -53,6 +53,9 @@ export const getTenantIDFromAppID = (appId: string) => { if (!appId) { return null } + if (!isMultiTenant()) { + return DEFAULT_TENANT_ID + } const split = appId.split(SEPARATOR) const hasDev = split[1] === DocumentType.DEV if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts index a61e8a2af2..446f1f7d01 100644 --- a/packages/backend-core/src/db/constants.ts +++ b/packages/backend-core/src/db/constants.ts @@ -21,6 +21,7 @@ export enum ViewName { ACCOUNT_BY_EMAIL = "account_by_email", PLATFORM_USERS_LOWERCASE = "platform_users_lowercase", USER_BY_GROUP = "by_group_user", + APP_BACKUP_BY_TRIGGER = "by_trigger", } export const DeprecatedViews = { @@ -30,6 +31,10 @@ export const DeprecatedViews = { ], } +export enum InternalTable { + USER_METADATA = "ta_users", +} + export enum DocumentType { USER = "us", GROUP = "gr", @@ -46,9 +51,23 @@ export enum DocumentType { AUTOMATION_LOG = "log_au", ACCOUNT_METADATA = "acc_metadata", PLUGIN = "plg", - TABLE = "ta", DATASOURCE = "datasource", DATASOURCE_PLUS = "datasource_plus", + APP_BACKUP = "backup", + TABLE = "ta", + ROW = "ro", + AUTOMATION = "au", + LINK = "li", + WEBHOOK = "wh", + INSTANCE = "inst", + LAYOUT = "layout", + SCREEN = "screen", + QUERY = "query", + DEPLOYMENTS = "deployments", + METADATA = "metadata", + MEM_VIEW = "view", + USER_FLAG = "flag", + AUTOMATION_METADATA = "meta_au", } export const StaticDatabases = { diff --git a/packages/backend-core/src/db/index.js b/packages/backend-core/src/db/index.ts similarity index 52% rename from packages/backend-core/src/db/index.js rename to packages/backend-core/src/db/index.ts index aa6f7ebc2c..8374aecd8d 100644 --- a/packages/backend-core/src/db/index.js +++ b/packages/backend-core/src/db/index.ts @@ -1,8 +1,11 @@ -const pouch = require("./pouch") -const env = require("../environment") +import pouch from "./pouch" +import env from "../environment" +import { checkSlashesInUrl } from "../helpers" +import fetch from "node-fetch" +import { PouchOptions, CouchFindOptions } from "@budibase/types" -const openDbs = [] -let PouchDB +const openDbs: string[] = [] +let PouchDB: any let initialised = false const dbList = new Set() @@ -14,8 +17,8 @@ if (env.MEMORY_LEAK_CHECK) { } const put = - dbPut => - async (doc, options = {}) => { + (dbPut: any) => + async (doc: any, options = {}) => { if (!doc.createdAt) { doc.createdAt = new Date().toISOString() } @@ -29,7 +32,7 @@ const checkInitialised = () => { } } -exports.init = opts => { +export async function init(opts?: PouchOptions) { PouchDB = pouch.getPouch(opts) initialised = true } @@ -37,7 +40,7 @@ exports.init = opts => { // NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION // this function is prone to leaks, should only be used // in situations that using the function doWithDB does not work -exports.dangerousGetDB = (dbName, opts) => { +export function dangerousGetDB(dbName: string, opts?: any) { checkInitialised() if (env.isTest()) { dbList.add(dbName) @@ -53,7 +56,7 @@ exports.dangerousGetDB = (dbName, opts) => { // use this function if you have called dangerousGetDB - close // the databases you've opened once finished -exports.closeDB = async db => { +export async function closeDB(db: PouchDB.Database) { if (!db || env.isTest()) { return } @@ -71,21 +74,59 @@ exports.closeDB = async db => { // we have to use a callback for this so that we can close // the DB when we're done, without this manual requests would // need to close the database when done with it to avoid memory leaks -exports.doWithDB = async (dbName, cb, opts = {}) => { - const db = exports.dangerousGetDB(dbName, opts) +export async function doWithDB(dbName: string, cb: any, opts = {}) { + const db = dangerousGetDB(dbName, opts) // need this to be async so that we can correctly close DB after all // async operations have been completed try { return await cb(db) } finally { - await exports.closeDB(db) + await closeDB(db) } } -exports.allDbs = () => { +export function allDbs() { if (!env.isTest()) { throw new Error("Cannot be used outside test environment.") } checkInitialised() return [...dbList] } + +export async function directCouchQuery( + path: string, + method: string = "GET", + body?: any +) { + let { url, cookie } = pouch.getCouchInfo() + const couchUrl = `${url}/${path}` + const params: any = { + method: method, + headers: { + Authorization: cookie, + }, + } + if (body && method !== "GET") { + params.body = JSON.stringify(body) + params.headers["Content-Type"] = "application/json" + } + const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params) + if (response.status < 300) { + return await response.json() + } else { + throw "Cannot connect to CouchDB instance" + } +} + +export async function directCouchAllDbs(queryString?: string) { + let couchPath = "/_all_dbs" + if (queryString) { + couchPath += `?${queryString}` + } + return await directCouchQuery(couchPath) +} + +export async function directCouchFind(dbName: string, opts: CouchFindOptions) { + const json = await directCouchQuery(`${dbName}/_find`, "POST", opts) + return { rows: json.docs, bookmark: json.bookmark } +} diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 1c4be7e366..c04da5da4f 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -1,14 +1,17 @@ import { newid } from "../hashing" import { DEFAULT_TENANT_ID, Configs } from "../constants" import env from "../environment" -import { SEPARATOR, DocumentType, UNICODE_MAX, ViewName } from "./constants" +import { + SEPARATOR, + DocumentType, + UNICODE_MAX, + ViewName, + InternalTable, +} from "./constants" import { getTenantId, getGlobalDB } from "../context" import { getGlobalDBName } from "./tenancy" -import fetch from "node-fetch" -import { doWithDB, allDbs } from "./index" -import { getCouchInfo } from "./pouch" +import { doWithDB, allDbs, directCouchAllDbs } from "./index" import { getAppMetadata } from "../cache/appMetadata" -import { checkSlashesInUrl } from "../helpers" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" import { APP_PREFIX } from "./constants" import * as events from "../events" @@ -43,8 +46,8 @@ export const generateAppID = (tenantId = null) => { * @returns {object} Parameters which can then be used with an allDocs request. */ export function getDocParams( - docType: any, - docId: any = null, + docType: string, + docId?: string | null, otherProps: any = {} ) { if (docId == null) { @@ -57,6 +60,28 @@ export function getDocParams( } } +/** + * Gets the DB allDocs/query params for retrieving a row. + * @param {string|null} tableId The table in which the rows have been stored. + * @param {string|null} rowId The ID of the row which is being specifically queried for. This can be + * left null to get all the rows in the table. + * @param {object} otherProps Any other properties to add to the request. + * @returns {object} Parameters which can then be used with an allDocs request. + */ +export function getRowParams( + tableId?: string | null, + rowId?: string | null, + otherProps = {} +) { + if (tableId == null) { + return getDocParams(DocumentType.ROW, null, otherProps) + } + + const endOfKey = rowId == null ? `${tableId}${SEPARATOR}` : rowId + + return getDocParams(DocumentType.ROW, endOfKey, otherProps) +} + /** * Retrieve the correct index for a view based on default design DB. */ @@ -64,6 +89,17 @@ export function getQueryIndex(viewName: ViewName) { return `database/${viewName}` } +/** + * Gets a new row ID for the specified table. + * @param {string} tableId The table which the row is being created for. + * @param {string|null} id If an ID is to be used then the UUID can be substituted for this. + * @returns {string} The new ID which a row doc can be stored under. + */ +export function generateRowID(tableId: string, id?: string) { + id = id || newid() + return `${DocumentType.ROW}${SEPARATOR}${tableId}${SEPARATOR}${id}` +} + /** * Check if a given ID is that of a table. * @returns {boolean} @@ -131,6 +167,33 @@ export function getGlobalUserParams(globalId: any, otherProps: any = {}) { } } +/** + * Gets parameters for retrieving users, this is a utility function for the getDocParams function. + */ +export function getUserMetadataParams(userId?: string, otherProps = {}) { + return getRowParams(InternalTable.USER_METADATA, userId, otherProps) +} + +/** + * Generates a new user ID based on the passed in global ID. + * @param {string} globalId The ID of the global user. + * @returns {string} The new user ID which the user doc can be stored under. + */ +export function generateUserMetadataID(globalId: string) { + return generateRowID(InternalTable.USER_METADATA, globalId) +} + +/** + * Breaks up the ID to get the global ID. + */ +export function getGlobalIDFromUserMetadataID(id: string) { + const prefix = `${DocumentType.ROW}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}` + if (!id || !id.includes(prefix)) { + return id + } + return id.split(prefix)[1] +} + export function getUsersByAppParams(appId: any, otherProps: any = {}) { const prodAppId = getProdAppID(appId) return { @@ -191,9 +254,9 @@ export function getRoleParams(roleId = null, otherProps = {}) { return getDocParams(DocumentType.ROLE, roleId, otherProps) } -export function getStartEndKeyURL(base: any, baseKey: any, tenantId = null) { +export function getStartEndKeyURL(baseKey: any, tenantId = null) { const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : "" - return `${base}?startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"` + return `startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"` } /** @@ -209,22 +272,10 @@ export async function getAllDbs(opts = { efficient: false }) { return allDbs() } let dbs: any[] = [] - let { url, cookie } = getCouchInfo() - async function addDbs(couchUrl: string) { - const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), { - method: "GET", - headers: { - Authorization: cookie, - }, - }) - if (response.status === 200) { - let json = await response.json() - dbs = dbs.concat(json) - } else { - throw "Cannot connect to CouchDB instance" - } + async function addDbs(queryString?: string) { + const json = await directCouchAllDbs(queryString) + dbs = dbs.concat(json) } - let couchUrl = `${url}/_all_dbs` let tenantId = getTenantId() if (!env.MULTI_TENANCY || (!efficient && tenantId === DEFAULT_TENANT_ID)) { // just get all DBs when: @@ -232,12 +283,12 @@ export async function getAllDbs(opts = { efficient: false }) { // - default tenant // - apps dbs don't contain tenant id // - non-default tenant dbs are filtered out application side in getAllApps - await addDbs(couchUrl) + await addDbs() } else { // get prod apps - await addDbs(getStartEndKeyURL(couchUrl, DocumentType.APP, tenantId)) + await addDbs(getStartEndKeyURL(DocumentType.APP, tenantId)) // get dev apps - await addDbs(getStartEndKeyURL(couchUrl, DocumentType.APP_DEV, tenantId)) + await addDbs(getStartEndKeyURL(DocumentType.APP_DEV, tenantId)) // add global db name dbs.push(getGlobalDBName(tenantId)) } diff --git a/packages/backend-core/src/events/publishers/backup.ts b/packages/backend-core/src/events/publishers/backup.ts new file mode 100644 index 0000000000..0fc81da259 --- /dev/null +++ b/packages/backend-core/src/events/publishers/backup.ts @@ -0,0 +1,12 @@ +import { AppBackup, AppBackupRestoreEvent, Event } from "@budibase/types" +import { publishEvent } from "../events" + +export async function appBackupRestored(backup: AppBackup) { + const properties: AppBackupRestoreEvent = { + appId: backup.appId, + backupName: backup.name!, + backupCreatedAt: backup.timestamp, + } + + await publishEvent(Event.APP_BACKUP_RESTORED, properties) +} diff --git a/packages/backend-core/src/events/publishers/index.ts b/packages/backend-core/src/events/publishers/index.ts index 6fe42c4bda..7306312a8f 100644 --- a/packages/backend-core/src/events/publishers/index.ts +++ b/packages/backend-core/src/events/publishers/index.ts @@ -19,3 +19,4 @@ export * as installation from "./installation" export * as backfill from "./backfill" export * as group from "./group" export * as plugin from "./plugin" +export * as backup from "./backup" diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 42cad17620..659a56c051 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -19,6 +19,7 @@ import pino from "./pino" import * as middleware from "./middleware" import plugins from "./plugin" import encryption from "./security/encryption" +import * as queue from "./queue" // mimic the outer package exports import * as db from "./pkg/db" @@ -63,6 +64,7 @@ const core = { ...errorClasses, middleware, encryption, + queue, } export = core diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts index 17e002cc49..8453c9aee6 100644 --- a/packages/backend-core/src/objectStore/index.ts +++ b/packages/backend-core/src/objectStore/index.ts @@ -18,11 +18,16 @@ const STATE = { bucketCreationPromises: {}, } +type ListParams = { + ContinuationToken?: string +} + const CONTENT_TYPE_MAP: any = { html: "text/html", css: "text/css", js: "application/javascript", json: "application/json", + gz: "application/gzip", } const STRING_CONTENT_TYPES = [ CONTENT_TYPE_MAP.html, @@ -32,16 +37,16 @@ const STRING_CONTENT_TYPES = [ ] // does normal sanitization and then swaps dev apps to apps -export function sanitizeKey(input: any) { +export function sanitizeKey(input: string) { return sanitize(sanitizeBucket(input)).replace(/\\/g, "/") } // simply handles the dev app to app conversion -export function sanitizeBucket(input: any) { +export function sanitizeBucket(input: string) { return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX) } -function publicPolicy(bucketName: any) { +function publicPolicy(bucketName: string) { return { Version: "2012-10-17", Statement: [ @@ -69,7 +74,7 @@ const PUBLIC_BUCKETS = [ * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage. * @constructor */ -export const ObjectStore = (bucket: any) => { +export const ObjectStore = (bucket: string) => { const config: any = { s3ForcePathStyle: true, signatureVersion: "v4", @@ -93,7 +98,7 @@ export const ObjectStore = (bucket: any) => { * Given an object store and a bucket name this will make sure the bucket exists, * if it does not exist then it will create it. */ -export const makeSureBucketExists = async (client: any, bucketName: any) => { +export const makeSureBucketExists = async (client: any, bucketName: string) => { bucketName = sanitizeBucket(bucketName) try { await client @@ -145,7 +150,7 @@ export const upload = async ({ type, metadata, }: any) => { - const extension = [...filename.split(".")].pop() + const extension = filename.split(".").pop() const fileBytes = fs.readFileSync(path) const objectStore = ObjectStore(bucketName) @@ -168,8 +173,8 @@ export const upload = async ({ * through to the object store. */ export const streamUpload = async ( - bucketName: any, - filename: any, + bucketName: string, + filename: string, stream: any, extra = {} ) => { @@ -202,7 +207,7 @@ export const streamUpload = async ( * retrieves the contents of a file from the object store, if it is a known content type it * will be converted, otherwise it will be returned as a buffer stream. */ -export const retrieve = async (bucketName: any, filepath: any) => { +export const retrieve = async (bucketName: string, filepath: string) => { const objectStore = ObjectStore(bucketName) const params = { Bucket: sanitizeBucket(bucketName), @@ -217,10 +222,38 @@ export const retrieve = async (bucketName: any, filepath: any) => { } } +export const listAllObjects = async (bucketName: string, path: string) => { + const objectStore = ObjectStore(bucketName) + const list = (params: ListParams = {}) => { + return objectStore + .listObjectsV2({ + ...params, + Bucket: sanitizeBucket(bucketName), + Prefix: sanitizeKey(path), + }) + .promise() + } + let isTruncated = false, + token, + objects: AWS.S3.Types.Object[] = [] + do { + let params: ListParams = {} + if (token) { + params.ContinuationToken = token + } + const response = await list(params) + if (response.Contents) { + objects = objects.concat(response.Contents) + } + isTruncated = !!response.IsTruncated + } while (isTruncated) + return objects +} + /** * Same as retrieval function but puts to a temporary file. */ -export const retrieveToTmp = async (bucketName: any, filepath: any) => { +export const retrieveToTmp = async (bucketName: string, filepath: string) => { bucketName = sanitizeBucket(bucketName) filepath = sanitizeKey(filepath) const data = await retrieve(bucketName, filepath) @@ -229,10 +262,31 @@ export const retrieveToTmp = async (bucketName: any, filepath: any) => { return outputPath } +export const retrieveDirectory = async (bucketName: string, path: string) => { + let writePath = join(budibaseTempDir(), v4()) + fs.mkdirSync(writePath) + const objects = await listAllObjects(bucketName, path) + let fullObjects = await Promise.all( + objects.map(obj => retrieve(bucketName, obj.Key!)) + ) + let count = 0 + for (let obj of objects) { + const filename = obj.Key! + const data = fullObjects[count++] + const possiblePath = filename.split("/") + if (possiblePath.length > 1) { + const dirs = possiblePath.slice(0, possiblePath.length - 1) + fs.mkdirSync(join(writePath, ...dirs), { recursive: true }) + } + fs.writeFileSync(join(writePath, ...possiblePath), data) + } + return writePath +} + /** * Delete a single file. */ -export const deleteFile = async (bucketName: any, filepath: any) => { +export const deleteFile = async (bucketName: string, filepath: string) => { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { @@ -242,7 +296,7 @@ export const deleteFile = async (bucketName: any, filepath: any) => { return objectStore.deleteObject(params) } -export const deleteFiles = async (bucketName: any, filepaths: any) => { +export const deleteFiles = async (bucketName: string, filepaths: string[]) => { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { @@ -258,8 +312,8 @@ export const deleteFiles = async (bucketName: any, filepaths: any) => { * Delete a path, including everything within. */ export const deleteFolder = async ( - bucketName: any, - folder: any + bucketName: string, + folder: string ): Promise => { bucketName = sanitizeBucket(bucketName) folder = sanitizeKey(folder) @@ -292,9 +346,9 @@ export const deleteFolder = async ( } export const uploadDirectory = async ( - bucketName: any, - localPath: any, - bucketPath: any + bucketName: string, + localPath: string, + bucketPath: string ) => { bucketName = sanitizeBucket(bucketName) let uploads = [] @@ -326,7 +380,11 @@ exports.downloadTarballDirect = async ( await streamPipeline(response.body, zlib.Unzip(), tar.extract(path)) } -export const downloadTarball = async (url: any, bucketName: any, path: any) => { +export const downloadTarball = async ( + url: string, + bucketName: string, + path: string +) => { bucketName = sanitizeBucket(bucketName) path = sanitizeKey(path) const response = await fetch(url) diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts new file mode 100644 index 0000000000..e8323dacb8 --- /dev/null +++ b/packages/backend-core/src/queue/constants.ts @@ -0,0 +1,4 @@ +export enum JobQueue { + AUTOMATION = "automationQueue", + APP_BACKUP = "appBackupQueue", +} diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts new file mode 100644 index 0000000000..80ee7362e4 --- /dev/null +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -0,0 +1,127 @@ +import events from "events" + +/** + * Bull works with a Job wrapper around all messages that contains a lot more information about + * the state of the message, this object constructor implements the same schema of Bull jobs + * for the sake of maintaining API consistency. + * @param {string} queue The name of the queue which the message will be carried on. + * @param {object} message The JSON message which will be passed back to the consumer. + * @returns {Object} A new job which can now be put onto the queue, this is mostly an + * internal structure so that an in memory queue can be easily swapped for a Bull queue. + */ +function newJob(queue: string, message: any) { + return { + timestamp: Date.now(), + queue: queue, + data: message, + } +} + +/** + * This is designed to replicate Bull (https://github.com/OptimalBits/bull) in memory as a sort of mock. + * It is relatively simple, using an event emitter internally to register when messages are available + * to the consumers - in can support many inputs and many consumers. + */ +class InMemoryQueue { + _name: string + _opts?: any + _messages: any[] + _emitter: EventEmitter + /** + * The constructor the queue, exactly the same as that of Bulls. + * @param {string} name The name of the queue which is being configured. + * @param {object|null} opts This is not used by the in memory queue as there is no real use + * case when in memory, but is the same API as Bull + */ + constructor(name: string, opts = null) { + this._name = name + this._opts = opts + this._messages = [] + this._emitter = new events.EventEmitter() + } + + /** + * Same callback API as Bull, each callback passed to this will consume messages as they are + * available. Please note this is a queue service, not a notification service, so each + * consumer will receive different messages. + * @param {function} func The callback function which will return a "Job", the same + * as the Bull API, within this job the property "data" contains the JSON message. Please + * note this is incredibly limited compared to Bull as in reality the Job would contain + * a lot more information about the queue and current status of Bull cluster. + */ + process(func: any) { + this._emitter.on("message", async () => { + if (this._messages.length <= 0) { + return + } + let msg = this._messages.shift() + let resp = func(msg) + if (resp.then != null) { + await resp + } + }) + } + + // simply puts a message to the queue and emits to the queue for processing + /** + * Simple function to replicate the add message functionality of Bull, putting + * a new message on the queue. This then emits an event which will be used to + * return the message to a consumer (if one is attached). + * @param {object} msg A message to be transported over the queue, this should be + * a JSON message as this is required by Bull. + * @param {boolean} repeat serves no purpose for the import queue. + */ + // eslint-disable-next-line no-unused-vars + add(msg: any, repeat: boolean) { + if (typeof msg !== "object") { + throw "Queue only supports carrying JSON." + } + this._messages.push(newJob(this._name, msg)) + this._emitter.emit("message") + } + + /** + * replicating the close function from bull, which waits for jobs to finish. + */ + async close() { + return [] + } + + /** + * This removes a cron which has been implemented, this is part of Bull API. + * @param {string} cronJobId The cron which is to be removed. + */ + removeRepeatableByKey(cronJobId: string) { + // TODO: implement for testing + console.log(cronJobId) + } + + /** + * Implemented for tests + */ + getRepeatableJobs() { + return [] + } + + // eslint-disable-next-line no-unused-vars + removeJobs(pattern: string) { + // no-op + } + + /** + * Implemented for tests + */ + async clean() { + return [] + } + + async getJob() { + return {} + } + + on() { + // do nothing + } +} + +export = InMemoryQueue diff --git a/packages/backend-core/src/queue/index.ts b/packages/backend-core/src/queue/index.ts new file mode 100644 index 0000000000..b7d565ba13 --- /dev/null +++ b/packages/backend-core/src/queue/index.ts @@ -0,0 +1,2 @@ +export * from "./queue" +export * from "./constants" diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts new file mode 100644 index 0000000000..e1975b5d06 --- /dev/null +++ b/packages/backend-core/src/queue/listeners.ts @@ -0,0 +1,101 @@ +import { Job, JobId, Queue } from "bull" +import { JobQueue } from "./constants" + +export type StalledFn = (job: Job) => Promise + +export function addListeners( + queue: Queue, + jobQueue: JobQueue, + removeStalledCb?: StalledFn +) { + logging(queue, jobQueue) + if (removeStalledCb) { + handleStalled(queue, removeStalledCb) + } +} + +function handleStalled(queue: Queue, removeStalledCb?: StalledFn) { + queue.on("stalled", async (job: Job) => { + if (removeStalledCb) { + await removeStalledCb(job) + } else if (job.opts.repeat) { + const jobId = job.id + const repeatJobs = await queue.getRepeatableJobs() + for (let repeatJob of repeatJobs) { + if (repeatJob.id === jobId) { + await queue.removeRepeatableByKey(repeatJob.key) + } + } + console.log(`jobId=${jobId} disabled`) + } + }) +} + +function logging(queue: Queue, jobQueue: JobQueue) { + let eventType: string + switch (jobQueue) { + case JobQueue.AUTOMATION: + eventType = "automation-event" + break + case JobQueue.APP_BACKUP: + eventType = "app-backup-event" + break + } + if (process.env.NODE_DEBUG?.includes("bull")) { + queue + .on("error", (error: any) => { + // An error occurred. + console.error(`${eventType}=error error=${JSON.stringify(error)}`) + }) + .on("waiting", (jobId: JobId) => { + // A Job is waiting to be processed as soon as a worker is idling. + console.log(`${eventType}=waiting jobId=${jobId}`) + }) + .on("active", (job: Job, jobPromise: any) => { + // A job has started. You can use `jobPromise.cancel()`` to abort it. + console.log(`${eventType}=active jobId=${job.id}`) + }) + .on("stalled", (job: Job) => { + // A job has been marked as stalled. This is useful for debugging job + // workers that crash or pause the event loop. + console.error( + `${eventType}=stalled jobId=${job.id} job=${JSON.stringify(job)}` + ) + }) + .on("progress", (job: Job, progress: any) => { + // A job's progress was updated! + console.log( + `${eventType}=progress jobId=${job.id} progress=${progress}` + ) + }) + .on("completed", (job: Job, result) => { + // A job successfully completed with a `result`. + console.log(`${eventType}=completed jobId=${job.id} result=${result}`) + }) + .on("failed", (job, err: any) => { + // A job failed with reason `err`! + console.log(`${eventType}=failed jobId=${job.id} error=${err}`) + }) + .on("paused", () => { + // The queue has been paused. + console.log(`${eventType}=paused`) + }) + .on("resumed", (job: Job) => { + // The queue has been resumed. + console.log(`${eventType}=paused jobId=${job.id}`) + }) + .on("cleaned", (jobs: Job[], type: string) => { + // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned + // jobs, and `type` is the type of jobs cleaned. + console.log(`${eventType}=cleaned length=${jobs.length} type=${type}`) + }) + .on("drained", () => { + // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed) + console.log(`${eventType}=drained`) + }) + .on("removed", (job: Job) => { + // A job successfully removed. + console.log(`${eventType}=removed jobId=${job.id}`) + }) + } +} diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts new file mode 100644 index 0000000000..b4eeeb31aa --- /dev/null +++ b/packages/backend-core/src/queue/queue.ts @@ -0,0 +1,51 @@ +import env from "../environment" +import { getRedisOptions } from "../redis/utils" +import { JobQueue } from "./constants" +import InMemoryQueue from "./inMemoryQueue" +import BullQueue from "bull" +import { addListeners, StalledFn } from "./listeners" +const { opts: redisOpts, redisProtocolUrl } = getRedisOptions() + +const CLEANUP_PERIOD_MS = 60 * 1000 +let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = [] +let cleanupInterval: NodeJS.Timeout + +async function cleanup() { + for (let queue of QUEUES) { + await queue.clean(CLEANUP_PERIOD_MS, "completed") + } +} + +export function createQueue( + jobQueue: JobQueue, + opts: { removeStalledCb?: StalledFn } = {} +): BullQueue.Queue { + const queueConfig: any = redisProtocolUrl || { redis: redisOpts } + let queue: any + if (!env.isTest()) { + queue = new BullQueue(jobQueue, queueConfig) + } else { + queue = new InMemoryQueue(jobQueue, queueConfig) + } + addListeners(queue, jobQueue, opts?.removeStalledCb) + QUEUES.push(queue) + if (!cleanupInterval) { + cleanupInterval = setInterval(cleanup, CLEANUP_PERIOD_MS) + // fire off an initial cleanup + cleanup().catch(err => { + console.error(`Unable to cleanup automation queue initially - ${err}`) + }) + } + return queue +} + +exports.shutdown = async () => { + if (QUEUES.length) { + clearInterval(cleanupInterval) + for (let queue of QUEUES) { + await queue.close() + } + QUEUES = [] + } + console.log("Queues shutdown") +} diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 6bc9b63728..d301526ba1 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -543,6 +543,36 @@ semver "^7.3.5" tar "^6.1.11" +"@msgpackr-extract/msgpackr-extract-darwin-arm64@2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-2.1.2.tgz#9571b87be3a3f2c46de05585470bc4f3af2f6f00" + integrity sha512-TyVLn3S/+ikMDsh0gbKv2YydKClN8HaJDDpONlaZR+LVJmsxLFUgA+O7zu59h9+f9gX1aj/ahw9wqa6rosmrYQ== + +"@msgpackr-extract/msgpackr-extract-darwin-x64@2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-2.1.2.tgz#bfbc6936ede2955218f5621a675679a5fe8e6f4c" + integrity sha512-YPXtcVkhmVNoMGlqp81ZHW4dMxK09msWgnxtsDpSiZwTzUBG2N+No2bsr7WMtBKCVJMSD6mbAl7YhKUqkp/Few== + +"@msgpackr-extract/msgpackr-extract-linux-arm64@2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-2.1.2.tgz#22555e28382af2922e7450634c8a2f240bb9eb82" + integrity sha512-vHZ2JiOWF2+DN9lzltGbhtQNzDo8fKFGrf37UJrgqxU0yvtERrzUugnfnX1wmVfFhSsF8OxrfqiNOUc5hko1Zg== + +"@msgpackr-extract/msgpackr-extract-linux-arm@2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-2.1.2.tgz#ffb6ae1beea7ac572b6be6bf2a8e8162ebdd8be7" + integrity sha512-42R4MAFeIeNn+L98qwxAt360bwzX2Kf0ZQkBBucJ2Ircza3asoY4CDbgiu9VWklq8gWJVSJSJBwDI+c/THiWkA== + +"@msgpackr-extract/msgpackr-extract-linux-x64@2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-2.1.2.tgz#7caf62eebbfb1345de40f75e89666b3d4194755f" + integrity sha512-RjRoRxg7Q3kPAdUSC5EUUPlwfMkIVhmaRTIe+cqHbKrGZ4M6TyCA/b5qMaukQ/1CHWrqYY2FbKOAU8Hg0pQFzg== + +"@msgpackr-extract/msgpackr-extract-win32-x64@2.1.2": + version "2.1.2" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.1.2.tgz#f2d8b9ddd8d191205ed26ce54aba3dfc5ae3e7c9" + integrity sha512-rIZVR48zA8hGkHIK7ED6+ZiXsjRCcAVBJbm8o89OKAMTmEAQ2QvoOxoiu3w2isAaWwzgtQIOFIqHwvZDyLKCvw== + "@shopify/jest-koa-mocks@5.0.1": version "5.0.1" resolved "https://registry.yarnpkg.com/@shopify/jest-koa-mocks/-/jest-koa-mocks-5.0.1.tgz#fba490b6b7985fbb571eb9974897d396a3642e94" @@ -733,6 +763,13 @@ resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1" integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w== +"@types/ioredis@4.28.0": + version "4.28.0" + resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.0.tgz#609b2ea0d91231df2dd7f67dd77436bc72584911" + integrity sha512-HSA/JQivJgV0e+353gvgu6WVoWvGRe0HyHOnAN2AvbVIhUlJBhNnnkP8gEEokrDWrxywrBkwo8NuDZ6TVPL9XA== + dependencies: + "@types/node" "*" + "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": version "2.0.4" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz#8467d4b3c087805d63580480890791277ce35c44" @@ -1497,6 +1534,21 @@ buffer@^5.5.0, buffer@^5.6.0: base64-js "^1.3.1" ieee754 "^1.1.13" +bull@4.10.1: + version "4.10.1" + resolved "https://registry.yarnpkg.com/bull/-/bull-4.10.1.tgz#f14974b6089358b62b495a2cbf838aadc098e43f" + integrity sha512-Fp21tRPb2EaZPVfmM+ONZKVz2RA+to+zGgaTLyCKt3JMSU8OOBqK8143OQrnGuGpsyE5G+9FevFAGhdZZfQP2g== + dependencies: + cron-parser "^4.2.1" + debuglog "^1.0.0" + get-port "^5.1.1" + ioredis "^4.28.5" + lodash "^4.17.21" + msgpackr "^1.5.2" + p-timeout "^3.2.0" + semver "^7.3.2" + uuid "^8.3.0" + cache-content-type@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/cache-content-type/-/cache-content-type-1.0.1.tgz#035cde2b08ee2129f4a8315ea8f00a00dba1453c" @@ -1764,6 +1816,13 @@ core-util-is@~1.0.0: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== +cron-parser@^4.2.1: + version "4.6.0" + resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.6.0.tgz#404c3fdbff10ae80eef6b709555d577ef2fd2e0d" + integrity sha512-guZNLMGUgg6z4+eGhmHGw7ft+v6OQeuHzd1gcLxCo9Yg/qoxmG3nindp2/uwGCLizEisf2H0ptqeVXeoCpP6FA== + dependencies: + luxon "^3.0.1" + cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" @@ -1837,6 +1896,11 @@ debug@~3.1.0: dependencies: ms "2.0.0" +debuglog@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492" + integrity sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw== + decimal.js@^10.2.1: version "10.3.1" resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.3.1.tgz#d8c3a444a9c6774ba60ca6ad7261c3a94fd5e783" @@ -2318,6 +2382,11 @@ get-package-type@^0.1.0: resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q== +get-port@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/get-port/-/get-port-5.1.1.tgz#0469ed07563479de6efb986baf053dcd7d4e3193" + integrity sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ== + get-stream@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5" @@ -2652,6 +2721,23 @@ ioredis@4.28.0: redis-parser "^3.0.0" standard-as-callback "^2.1.0" +ioredis@^4.28.5: + version "4.28.5" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.5.tgz#5c149e6a8d76a7f8fa8a504ffc85b7d5b6797f9f" + integrity sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A== + dependencies: + cluster-key-slot "^1.1.0" + debug "^4.3.1" + denque "^1.1.0" + lodash.defaults "^4.2.0" + lodash.flatten "^4.4.0" + lodash.isarguments "^3.1.0" + p-map "^2.1.0" + redis-commands "1.7.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" @@ -3725,6 +3811,11 @@ ltgt@2.2.1, ltgt@^2.1.2, ltgt@~2.2.0: resolved "https://registry.yarnpkg.com/ltgt/-/ltgt-2.2.1.tgz#f35ca91c493f7b73da0e07495304f17b31f87ee5" integrity sha512-AI2r85+4MquTw9ZYqabu4nMwy9Oftlfa/e/52t9IjtfG+mGBbTNdAoZ3RQKLHR6r0wQnwZnPIEh/Ya6XTWAKNA== +luxon@^3.0.1: + version "3.0.4" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.0.4.tgz#d179e4e9f05e092241e7044f64aaa54796b03929" + integrity sha512-aV48rGUwP/Vydn8HT+5cdr26YYQiUZ42NM6ToMoaGKwYfWbfLeRkEu1wXWMHBZT6+KyLfcbbtVcoQFCbbPjKlw== + make-dir@^3.0.0, make-dir@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" @@ -3872,6 +3963,27 @@ ms@^2.1.1, ms@^2.1.3: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== +msgpackr-extract@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-2.1.2.tgz#56272030f3e163e1b51964ef8b1cd5e7240c03ed" + integrity sha512-cmrmERQFb19NX2JABOGtrKdHMyI6RUyceaPBQ2iRz9GnDkjBWFjNJC0jyyoOfZl2U/LZE3tQCCQc4dlRyA8mcA== + dependencies: + node-gyp-build-optional-packages "5.0.3" + optionalDependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64" "2.1.2" + "@msgpackr-extract/msgpackr-extract-darwin-x64" "2.1.2" + "@msgpackr-extract/msgpackr-extract-linux-arm" "2.1.2" + "@msgpackr-extract/msgpackr-extract-linux-arm64" "2.1.2" + "@msgpackr-extract/msgpackr-extract-linux-x64" "2.1.2" + "@msgpackr-extract/msgpackr-extract-win32-x64" "2.1.2" + +msgpackr@^1.5.2: + version "1.7.2" + resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.7.2.tgz#68d6debf5999d6b61abb6e7046a689991ebf7261" + integrity sha512-mWScyHTtG6TjivXX9vfIy2nBtRupaiAj0HQ2mtmpmYujAmqZmaaEVPaSZ1NKLMvicaMLFzEaMk0ManxMRg8rMQ== + optionalDependencies: + msgpackr-extract "^2.1.2" + napi-macros@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.0.0.tgz#2b6bae421e7b96eb687aa6c77a7858640670001b" @@ -3919,6 +4031,11 @@ node-forge@^0.7.1: resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.7.6.tgz#fdf3b418aee1f94f0ef642cd63486c77ca9724ac" integrity sha512-sol30LUpz1jQFBjOKwbjxijiE3b6pjd74YwfD0fJOKPjF+fONKb2Yg8rYgS6+bK6VDl+/wfr4IYpC7jDzLUIfw== +node-gyp-build-optional-packages@5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17" + integrity sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA== + node-gyp-build@~4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.1.1.tgz#d7270b5d86717068d114cc57fff352f96d745feb" @@ -4075,6 +4192,11 @@ p-cancelable@^1.0.0: resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc" integrity sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw== +p-finally@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" + integrity sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow== + p-limit@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" @@ -4094,6 +4216,13 @@ p-map@^2.1.0: resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw== +p-timeout@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe" + integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg== + dependencies: + p-finally "^1.0.0" + p-try@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" @@ -5360,7 +5489,7 @@ uuid@8.1.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.1.0.tgz#6f1536eb43249f473abc6bd58ff983da1ca30d8d" integrity sha512-CI18flHDznR0lq54xBycOVmphdCYnQLKn8abKn7PXUiKUGdEd+/l9LWNJmugXel4hXq7S+RMNl34ecyC9TntWg== -uuid@8.3.2, uuid@^8.3.2: +uuid@8.3.2, uuid@^8.3.0, uuid@^8.3.2: version "8.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte index a2fa954389..0b760c4b4a 100644 --- a/packages/builder/src/pages/builder/portal/apps/index.svelte +++ b/packages/builder/src/pages/builder/portal/apps/index.svelte @@ -21,7 +21,6 @@ import { API } from "api" import { onMount } from "svelte" import { apps, auth, admin, templates, licensing } from "stores/portal" - import download from "downloadjs" import { goto } from "@roxi/routify" import AppRow from "components/start/AppRow.svelte" import { AppStatus } from "constants" @@ -140,7 +139,7 @@ const initiateAppsExport = () => { try { - download(`/api/cloud/export`) + window.location = `/api/cloud/export` notifications.success("Apps exported successfully") } catch (err) { notifications.error(`Error exporting apps: ${err}`) diff --git a/packages/builder/src/stores/portal/apps.js b/packages/builder/src/stores/portal/apps.js index a83e35e941..f84e0c973c 100644 --- a/packages/builder/src/stores/portal/apps.js +++ b/packages/builder/src/stores/portal/apps.js @@ -2,6 +2,9 @@ import { writable } from "svelte/store" import { AppStatus } from "../../constants" import { API } from "api" +// properties that should always come from the dev app, not the deployed +const DEV_PROPS = ["updatedBy", "updatedAt"] + const extractAppId = id => { const split = id?.split("_") || [] return split.length ? split[split.length - 1] : null @@ -57,9 +60,19 @@ export function createAppStore() { return } + let devProps = {} + if (appMap[id]) { + const entries = Object.entries(appMap[id]).filter( + ([key]) => DEV_PROPS.indexOf(key) !== -1 + ) + entries.forEach(entry => { + devProps[entry[0]] = entry[1] + }) + } appMap[id] = { ...appMap[id], ...app, + ...devProps, prodId: app.appId, prodRev: app._rev, } diff --git a/packages/client/yarn.lock b/packages/client/yarn.lock index 4ed5b395fe..fbe688c0b7 100644 --- a/packages/client/yarn.lock +++ b/packages/client/yarn.lock @@ -670,6 +670,11 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" +html5-qrcode@^2.2.1: + version "2.2.3" + resolved "https://registry.yarnpkg.com/html5-qrcode/-/html5-qrcode-2.2.3.tgz#5acb826860365e7c7ab91e1e14528ea16a502e8a" + integrity sha512-9CtEz5FVT56T76entiQxyrASzBWl8Rm30NHiQH8T163Eml5LS14BoZlYel9igxbikOt7O8KhvrT3awN1Y2HMqw== + htmlparser2@^6.0.0: version "6.1.0" resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.1.0.tgz#c4d762b6c3371a05dbe65e94ae43a9f845fb8fb7" diff --git a/packages/server/package.json b/packages/server/package.json index 8c429e2bb8..50b6576360 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -86,14 +86,14 @@ "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", "@google-cloud/firestore": "5.0.2", - "@koa/router": "8.0.0", + "@koa/router": "8.0.8", "@sendgrid/mail": "7.1.1", "@sentry/node": "6.17.7", "airtable": "0.10.1", "arangojs": "7.2.0", "aws-sdk": "2.1030.0", "bcryptjs": "2.4.3", - "bull": "4.8.5", + "bull": "4.10.1", "chmodr": "1.2.0", "chokidar": "3.5.3", "csvtojson": "2.0.10", @@ -112,7 +112,7 @@ "js-yaml": "4.1.0", "jsonschema": "1.4.0", "knex": "0.95.15", - "koa": "2.7.0", + "koa": "2.13.4", "koa-body": "4.2.0", "koa-compress": "4.0.1", "koa-connect": "2.1.0", @@ -159,12 +159,12 @@ "@jest/test-sequencer": "24.9.0", "@types/apidoc": "0.50.0", "@types/bson": "4.2.0", - "@types/bull": "3.15.8", "@types/global-agent": "2.1.1", "@types/google-spreadsheet": "3.1.5", + "@types/ioredis": "4.28.10", "@types/jest": "27.5.1", "@types/koa": "2.13.4", - "@types/koa__router": "8.0.0", + "@types/koa__router": "8.0.11", "@types/lodash": "4.14.180", "@types/node": "14.18.20", "@types/node-fetch": "2.6.1", diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index ce410823ec..62f59e9113 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -783,6 +783,7 @@ "type": "string", "enum": [ "string", + "barcodeqr", "longform", "options", "number", @@ -986,6 +987,7 @@ "type": "string", "enum": [ "string", + "barcodeqr", "longform", "options", "number", @@ -1200,6 +1202,7 @@ "type": "string", "enum": [ "string", + "barcodeqr", "longform", "options", "number", diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index ed13ac01f4..25069f40a4 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -579,6 +579,7 @@ components: type: string enum: - string + - barcodeqr - longform - options - number @@ -741,6 +742,7 @@ components: type: string enum: - string + - barcodeqr - longform - options - number @@ -910,6 +912,7 @@ components: type: string enum: - string + - barcodeqr - longform - options - number diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index a7caf85e94..f3dca51f72 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -5,11 +5,7 @@ import { createRoutingView, createAllSearchIndex, } from "../../db/views/staticViews" -import { - getTemplateStream, - createApp, - deleteApp, -} from "../../utilities/fileSystem" +import { createApp, deleteApp } from "../../utilities/fileSystem" import { generateAppID, getLayoutParams, @@ -50,6 +46,7 @@ import { errors, events, migrations } from "@budibase/backend-core" import { App, Layout, Screen, MigrationType } from "@budibase/types" import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts" import { enrichPluginURLs } from "../../utilities/plugins" +import sdk from "../../sdk" const URL_REGEX_SLASH = /\/|\\/g @@ -153,11 +150,7 @@ async function createInstance(template: any) { throw "Error loading database dump from memory." } } else if (template && template.useTemplate === "true") { - /* istanbul ignore next */ - const { ok } = await db.load(await getTemplateStream(template)) - if (!ok) { - throw "Error loading database dump from template." - } + await sdk.backups.importApp(appId, db, template) } else { // create the users table await db.put(USERS_TABLE_SCHEMA) diff --git a/packages/server/src/api/controllers/backup.js b/packages/server/src/api/controllers/backup.ts similarity index 55% rename from packages/server/src/api/controllers/backup.js rename to packages/server/src/api/controllers/backup.ts index 5e08c823ed..0ffda2c733 100644 --- a/packages/server/src/api/controllers/backup.js +++ b/packages/server/src/api/controllers/backup.ts @@ -1,15 +1,15 @@ -const { streamBackup } = require("../../utilities/fileSystem") -const { events, context } = require("@budibase/backend-core") -const { DocumentType } = require("../../db/utils") -const { isQsTrue } = require("../../utilities") +import sdk from "../../sdk" +import { events, context } from "@budibase/backend-core" +import { DocumentType } from "../../db/utils" +import { isQsTrue } from "../../utilities" -exports.exportAppDump = async function (ctx) { +export async function exportAppDump(ctx: any) { let { appId, excludeRows } = ctx.query const appName = decodeURI(ctx.query.appname) excludeRows = isQsTrue(excludeRows) - const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt` + const backupIdentifier = `${appName}-export-${new Date().getTime()}.tar.gz` ctx.attachment(backupIdentifier) - ctx.body = await streamBackup(appId, excludeRows) + ctx.body = await sdk.backups.streamExportApp(appId, excludeRows) await context.doInAppContext(appId, async () => { const appDb = context.getAppDB() diff --git a/packages/server/src/api/controllers/cloud.js b/packages/server/src/api/controllers/cloud.js index 1e6abb1d3b..0a111eae83 100644 --- a/packages/server/src/api/controllers/cloud.js +++ b/packages/server/src/api/controllers/cloud.js @@ -1,51 +1,30 @@ const env = require("../../environment") const { getAllApps, getGlobalDBName } = require("@budibase/backend-core/db") -const { - exportDB, - sendTempFile, - readFileSync, -} = require("../../utilities/fileSystem") -const { stringToReadStream } = require("../../utilities") const { getGlobalDB } = require("@budibase/backend-core/tenancy") -const { create } = require("./application") +const { streamFile } = require("../../utilities/fileSystem") +const { stringToReadStream } = require("../../utilities") const { getDocParams, DocumentType, isDevAppID } = require("../../db/utils") +const { create } = require("./application") +const { join } = require("path") +const sdk = require("../../sdk") -async function createApp(appName, appImport) { +async function createApp(appName, appDirectory) { const ctx = { request: { body: { - templateString: appImport, + useTemplate: true, name: appName, }, + files: { + templateFile: { + path: appDirectory, + }, + }, }, } return create(ctx) } -exports.exportApps = async ctx => { - if (env.SELF_HOSTED || !env.MULTI_TENANCY) { - ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.") - } - const apps = await getAllApps({ all: true }) - const globalDBString = await exportDB(getGlobalDBName(), { - filter: doc => !doc._id.startsWith(DocumentType.USER), - }) - let allDBs = { - global: globalDBString, - } - for (let app of apps) { - const appId = app.appId || app._id - // only export the dev apps as they will be the latest, the user can republish the apps - // in their self hosted environment - if (isDevAppID(appId)) { - allDBs[app.name] = await exportDB(appId) - } - } - const filename = `cloud-export-${new Date().getTime()}.txt` - ctx.attachment(filename) - ctx.body = sendTempFile(JSON.stringify(allDBs)) -} - async function getAllDocType(db, docType) { const response = await db.allDocs( getDocParams(docType, null, { @@ -55,6 +34,28 @@ async function getAllDocType(db, docType) { return response.rows.map(row => row.doc) } +exports.exportApps = async ctx => { + if (env.SELF_HOSTED || !env.MULTI_TENANCY) { + ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.") + } + const apps = await getAllApps({ all: true }) + const globalDBString = await sdk.backups.exportDB(getGlobalDBName(), { + filter: doc => !doc._id.startsWith(DocumentType.USER), + }) + // only export the dev apps as they will be the latest, the user can republish the apps + // in their self-hosted environment + let appMetadata = apps + .filter(app => isDevAppID(app.appId || app._id)) + .map(app => ({ appId: app.appId || app._id, name: app.name })) + const tmpPath = await sdk.backups.exportMultipleApps( + appMetadata, + globalDBString + ) + const filename = `cloud-export-${new Date().getTime()}.tar.gz` + ctx.attachment(filename) + ctx.body = streamFile(tmpPath) +} + async function hasBeenImported() { if (!env.SELF_HOSTED || env.MULTI_TENANCY) { return true @@ -80,17 +81,20 @@ exports.importApps = async ctx => { "Import file is required and environment must be fresh to import apps." ) } - const importFile = ctx.request.files.importFile - const importString = readFileSync(importFile.path) - const dbs = JSON.parse(importString) - const globalDbImport = dbs.global - // remove from the list of apps - delete dbs.global + if (ctx.request.files.importFile.type !== "application/gzip") { + ctx.throw(400, "Import file must be a gzipped tarball.") + } + + // initially get all the app databases out of the tarball + const tmpPath = sdk.backups.untarFile(ctx.request.files.importFile) + const globalDbImport = sdk.backups.getGlobalDBFile(tmpPath) + const appNames = sdk.backups.getListOfAppsInMulti(tmpPath) + const globalDb = getGlobalDB() // load the global db first await globalDb.load(stringToReadStream(globalDbImport)) - for (let [appName, appImport] of Object.entries(dbs)) { - await createApp(appName, appImport) + for (let appName of appNames) { + await createApp(appName, join(tmpPath, appName)) } // if there are any users make sure to remove them diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index a51e7ad6ec..cb4534a1a5 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -1,23 +1,25 @@ import Deployment from "./Deployment" import { - Replication, - getProdAppID, getDevelopmentAppID, + getProdAppID, + Replication, } from "@budibase/backend-core/db" import { DocumentType, getAutomationParams } from "../../../db/utils" import { + clearMetadata, disableAllCrons, enableCronTrigger, - clearMetadata, } from "../../../automations/utils" import { app as appCache } from "@budibase/backend-core/cache" import { - getAppId, getAppDB, - getProdAppDB, + getAppId, getDevAppDB, + getProdAppDB, } from "@budibase/backend-core/context" import { events } from "@budibase/backend-core" +import { backups } from "@budibase/pro" +import { AppBackupTrigger } from "@budibase/types" // the max time we can wait for an invalidation to complete before considering it failed const MAX_PENDING_TIME_MS = 30 * 60000 @@ -98,13 +100,24 @@ async function initDeployedApp(prodAppId: any) { console.log("Enabled cron triggers for deployed app..") } -async function deployApp(deployment: any) { +async function deployApp(deployment: any, userId: string) { let replication try { const appId = getAppId() const devAppId = getDevelopmentAppID(appId) const productionAppId = getProdAppID(appId) + // don't try this if feature isn't allowed, will error + if (await backups.isEnabled()) { + // trigger backup initially + await backups.triggerAppBackup( + productionAppId, + AppBackupTrigger.PUBLISH, + { + createdBy: userId, + } + ) + } const config: any = { source: devAppId, target: productionAppId, @@ -205,7 +218,7 @@ const _deployApp = async function (ctx: any) { console.log("Deploying app...") - let app = await deployApp(deployment) + let app = await deployApp(deployment, ctx.user._id) await events.app.published(app) ctx.body = deployment diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 08213c2cf8..ec2ea70820 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -5,7 +5,7 @@ require("svelte/register") const send = require("koa-send") const { resolve, join } = require("../../../utilities/centralPath") const uuid = require("uuid") -const { ObjectStoreBuckets } = require("../../../constants") +const { ObjectStoreBuckets, ATTACHMENT_DIR } = require("../../../constants") const { processString } = require("@budibase/string-templates") const { loadHandlebarsFile, @@ -90,7 +90,7 @@ export const uploadFile = async function (ctx: any) { return prepareUpload({ file, - s3Key: `${ctx.appId}/attachments/${processedFileName}`, + s3Key: `${ctx.appId}/${ATTACHMENT_DIR}/${processedFileName}`, bucket: ObjectStoreBuckets.APPS, }) }) diff --git a/packages/server/src/api/controllers/table/bulkFormula.js b/packages/server/src/api/controllers/table/bulkFormula.js index d736c126f2..733c16d455 100644 --- a/packages/server/src/api/controllers/table/bulkFormula.js +++ b/packages/server/src/api/controllers/table/bulkFormula.js @@ -1,10 +1,11 @@ const { FieldTypes, FormulaTypes } = require("../../../constants") -const { getAllInternalTables, clearColumns } = require("./utils") +const { clearColumns } = require("./utils") const { doesContainStrings } = require("@budibase/string-templates") const { cloneDeep } = require("lodash/fp") const { isEqual, uniq } = require("lodash") const { updateAllFormulasInTable } = require("../row/staticFormula") const { getAppDB } = require("@budibase/backend-core/context") +const sdk = require("../../../sdk") function isStaticFormula(column) { return ( @@ -39,7 +40,7 @@ function getFormulaThatUseColumn(table, columnNames) { */ async function checkIfFormulaNeedsCleared(table, { oldTable, deletion }) { // start by retrieving all tables, remove the current table from the list - const tables = (await getAllInternalTables()).filter( + const tables = (await sdk.tables.getAllInternalTables()).filter( tbl => tbl._id !== table._id ) const schemaToUse = oldTable ? oldTable.schema : table.schema @@ -99,7 +100,7 @@ async function updateRelatedFormulaLinksOnTables( ) { const db = getAppDB() // start by retrieving all tables, remove the current table from the list - const tables = (await getAllInternalTables()).filter( + const tables = (await sdk.tables.getAllInternalTables()).filter( tbl => tbl._id !== table._id ) // clone the tables, so we can compare at end diff --git a/packages/server/src/api/controllers/table/external.js b/packages/server/src/api/controllers/table/external.js index d919e9dad7..fe9270fe1d 100644 --- a/packages/server/src/api/controllers/table/external.js +++ b/packages/server/src/api/controllers/table/external.js @@ -3,7 +3,6 @@ const { breakExternalTableId, } = require("../../../integrations/utils") const { - getTable, generateForeignKey, generateJunctionTableName, foreignKeyStructure, @@ -20,6 +19,7 @@ const csvParser = require("../../../utilities/csvParser") const { handleRequest } = require("../row/external") const { getAppDB } = require("@budibase/backend-core/context") const { events } = require("@budibase/backend-core") +const sdk = require("../../../sdk") async function makeTableRequest( datasource, @@ -181,7 +181,7 @@ exports.save = async function (ctx) { let oldTable if (ctx.request.body && ctx.request.body._id) { - oldTable = await getTable(ctx.request.body._id) + oldTable = await sdk.tables.getTable(ctx.request.body._id) } if (hasTypeChanged(tableToSave, oldTable)) { @@ -281,7 +281,7 @@ exports.save = async function (ctx) { } exports.destroy = async function (ctx) { - const tableToDelete = await getTable(ctx.params.tableId) + const tableToDelete = await sdk.tables.getTable(ctx.params.tableId) if (!tableToDelete || !tableToDelete.created) { ctx.throw(400, "Cannot delete tables which weren't created in Budibase.") } @@ -303,7 +303,7 @@ exports.destroy = async function (ctx) { } exports.bulkImport = async function (ctx) { - const table = await getTable(ctx.params.tableId) + const table = await sdk.tables.getTable(ctx.params.tableId) const { dataImport } = ctx.request.body if (!dataImport || !dataImport.schema || !dataImport.csvString) { ctx.throw(400, "Provided data import information is invalid.") diff --git a/packages/server/src/api/controllers/table/index.js b/packages/server/src/api/controllers/table/index.js index e6192457af..3a20f4dff6 100644 --- a/packages/server/src/api/controllers/table/index.js +++ b/packages/server/src/api/controllers/table/index.js @@ -4,8 +4,8 @@ const csvParser = require("../../../utilities/csvParser") const { isExternalTable, isSQL } = require("../../../integrations/utils") const { getDatasourceParams } = require("../../../db/utils") const { getAppDB } = require("@budibase/backend-core/context") -const { getTable, getAllInternalTables } = require("./utils") const { events } = require("@budibase/backend-core") +const sdk = require("../../../sdk") function pickApi({ tableId, table }) { if (table && !tableId) { @@ -23,7 +23,7 @@ function pickApi({ tableId, table }) { exports.fetch = async function (ctx) { const db = getAppDB() - const internal = await getAllInternalTables() + const internal = await sdk.tables.getAllInternalTables() const externalTables = await db.allDocs( getDatasourceParams("plus", { @@ -50,7 +50,7 @@ exports.fetch = async function (ctx) { exports.find = async function (ctx) { const tableId = ctx.params.tableId - ctx.body = await getTable(tableId) + ctx.body = await sdk.tables.getTable(tableId) } exports.save = async function (ctx) { @@ -101,7 +101,7 @@ exports.validateCSVSchema = async function (ctx) { const { csvString, schema = {}, tableId } = ctx.request.body let existingTable if (tableId) { - existingTable = await getTable(tableId) + existingTable = await sdk.tables.getTable(tableId) } let result = await csvParser.parse(csvString, schema) if (existingTable) { diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 7e55c71aea..a50009b1f6 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -1,12 +1,7 @@ import { updateLinks, EventType } from "../../../db/linkedRows" import { getRowParams, generateTableID } from "../../../db/utils" import { FieldTypes } from "../../../constants" -import { - TableSaveFunctions, - hasTypeChanged, - getTable, - handleDataImport, -} from "./utils" +import { TableSaveFunctions, hasTypeChanged, handleDataImport } from "./utils" const { getAppDB } = require("@budibase/backend-core/context") import { isTest } from "../../../environment" import { @@ -19,6 +14,7 @@ import { quotas } from "@budibase/pro" import { isEqual } from "lodash" import { cloneDeep } from "lodash/fp" import env from "../../../environment" +import sdk from "../../../sdk" function checkAutoColumns(table: Table, oldTable: Table) { if (!table.schema) { @@ -188,7 +184,7 @@ export async function destroy(ctx: any) { } export async function bulkImport(ctx: any) { - const table = await getTable(ctx.params.tableId) + const table = await sdk.tables.getTable(ctx.params.tableId) const { dataImport } = ctx.request.body await handleDataImport(ctx.user, table, dataImport) return table diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index c2aa2f47c9..6d17d12f3d 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -1,11 +1,5 @@ import { transform } from "../../../utilities/csvParser" -import { - getRowParams, - generateRowID, - InternalTables, - getTableParams, - BudibaseInternalDB, -} from "../../../db/utils" +import { getRowParams, generateRowID, InternalTables } from "../../../db/utils" import { isEqual } from "lodash" import { AutoFieldSubTypes, FieldTypes } from "../../../constants" import { @@ -17,11 +11,6 @@ import { SwitchableTypes, CanSwitchTypes, } from "../../../constants" -import { - isExternalTable, - breakExternalTableId, - isSQL, -} from "../../../integrations/utils" import { getViews, saveView } from "../view/utils" import viewTemplate from "../view/viewBuilder" const { getAppDB } = require("@budibase/backend-core/context") @@ -256,46 +245,6 @@ class TableSaveFunctions { } } -export async function getAllInternalTables() { - const db = getAppDB() - const internalTables = await db.allDocs( - getTableParams(null, { - include_docs: true, - }) - ) - return internalTables.rows.map((tableDoc: any) => ({ - ...tableDoc.doc, - type: "internal", - sourceId: BudibaseInternalDB._id, - })) -} - -export async function getAllExternalTables(datasourceId: any) { - const db = getAppDB() - const datasource = await db.get(datasourceId) - if (!datasource || !datasource.entities) { - throw "Datasource is not configured fully." - } - return datasource.entities -} - -export async function getExternalTable(datasourceId: any, tableName: any) { - const entities = await getAllExternalTables(datasourceId) - return entities[tableName] -} - -export async function getTable(tableId: any) { - const db = getAppDB() - if (isExternalTable(tableId)) { - let { datasourceId, tableName } = breakExternalTableId(tableId) - const datasource = await db.get(datasourceId) - const table = await getExternalTable(datasourceId, tableName) - return { ...table, sql: isSQL(datasource) } - } else { - return db.get(tableId) - } -} - export async function checkForViewUpdates( table: any, rename: any, diff --git a/packages/server/src/api/controllers/view/index.js b/packages/server/src/api/controllers/view/index.js index b2c3a84c59..91657cfc21 100644 --- a/packages/server/src/api/controllers/view/index.js +++ b/packages/server/src/api/controllers/view/index.js @@ -3,12 +3,12 @@ const { apiFileReturn } = require("../../../utilities/fileSystem") const exporters = require("./exporters") const { saveView, getView, getViews, deleteView } = require("./utils") const { fetchView } = require("../row") -const { getTable } = require("../table/utils") const { FieldTypes } = require("../../../constants") const { getAppDB } = require("@budibase/backend-core/context") const { events } = require("@budibase/backend-core") const { DocumentType } = require("../../../db/utils") const { cloneDeep, isEqual } = require("lodash") +const sdk = require("../../../sdk") exports.fetch = async ctx => { ctx.body = await getViews() @@ -144,7 +144,7 @@ exports.exportView = async ctx => { let schema = view && view.meta && view.meta.schema const tableId = ctx.params.tableId || view.meta.tableId - const table = await getTable(tableId) + const table = await sdk.tables.getTable(tableId) if (!schema) { schema = table.schema } diff --git a/packages/server/src/api/routes/backup.js b/packages/server/src/api/routes/backup.js deleted file mode 100644 index 9f3b27e95a..0000000000 --- a/packages/server/src/api/routes/backup.js +++ /dev/null @@ -1,10 +0,0 @@ -const Router = require("@koa/router") -const controller = require("../controllers/backup") -const authorized = require("../../middleware/authorized") -const { BUILDER } = require("@budibase/backend-core/permissions") - -const router = new Router() - -router.get("/api/backups/export", authorized(BUILDER), controller.exportAppDump) - -module.exports = router diff --git a/packages/server/src/api/routes/backup.ts b/packages/server/src/api/routes/backup.ts new file mode 100644 index 0000000000..2473fa9f67 --- /dev/null +++ b/packages/server/src/api/routes/backup.ts @@ -0,0 +1,10 @@ +import Router from "@koa/router" +import * as controller from "../controllers/backup" +import authorized from "../../middleware/authorized" +import { BUILDER } from "@budibase/backend-core/permissions" + +const router = new Router() + +router.get("/api/backups/export", authorized(BUILDER), controller.exportAppDump) + +export default router diff --git a/packages/server/src/api/routes/index.ts b/packages/server/src/api/routes/index.ts index 64f5f9cb89..02a4900077 100644 --- a/packages/server/src/api/routes/index.ts +++ b/packages/server/src/api/routes/index.ts @@ -25,11 +25,17 @@ import devRoutes from "./dev" import cloudRoutes from "./cloud" import migrationRoutes from "./migrations" import pluginRoutes from "./plugin" +import Router from "@koa/router" +import { api } from "@budibase/pro" export { default as staticRoutes } from "./static" export { default as publicRoutes } from "./public" -export const mainRoutes = [ +const appBackupRoutes = api.appBackups +const scheduleRoutes = api.schedules +export const mainRoutes: Router[] = [ + appBackupRoutes, + backupRoutes, authRoutes, deployRoutes, layoutRoutes, @@ -49,14 +55,14 @@ export const mainRoutes = [ permissionRoutes, datasourceRoutes, queryRoutes, - backupRoutes, metadataRoutes, devRoutes, cloudRoutes, - // these need to be handled last as they still use /api/:tableId - // this could be breaking as koa may recognise other routes as this - tableRoutes, rowRoutes, migrationRoutes, pluginRoutes, + scheduleRoutes, + // these need to be handled last as they still use /api/:tableId + // this could be breaking as koa may recognise other routes as this + tableRoutes, ] diff --git a/packages/server/src/api/routes/tests/backup.spec.js b/packages/server/src/api/routes/tests/backup.spec.js index 7f92a4f3d1..7863129c75 100644 --- a/packages/server/src/api/routes/tests/backup.spec.js +++ b/packages/server/src/api/routes/tests/backup.spec.js @@ -21,7 +21,7 @@ describe("/backups", () => { .set(config.defaultHeaders()) .expect(200) expect(res.text).toBeDefined() - expect(res.text.includes(`"db_name":"${config.getAppId()}"`)).toEqual(true) + expect(res.headers["content-type"]).toEqual("application/gzip") expect(events.app.exported.mock.calls.length).toBe(1) }) diff --git a/packages/server/src/api/routes/tests/deployment.spec.js b/packages/server/src/api/routes/tests/deployment.spec.ts similarity index 74% rename from packages/server/src/api/routes/tests/deployment.spec.js rename to packages/server/src/api/routes/tests/deployment.spec.ts index be126fa239..0219e3f2b4 100644 --- a/packages/server/src/api/routes/tests/deployment.spec.js +++ b/packages/server/src/api/routes/tests/deployment.spec.ts @@ -1,5 +1,5 @@ -const setup = require("./utilities") -const { events } = require("@budibase/backend-core") +import setup from "./utilities" +import { events } from "@budibase/backend-core" describe("/deployments", () => { let request = setup.getRequest() @@ -19,7 +19,7 @@ describe("/deployments", () => { .set(config.defaultHeaders()) .expect("Content-Type", /json/) .expect(200) - expect(events.app.published.mock.calls.length).toBe(1) + expect((events.app.published as jest.Mock).mock.calls.length).toBe(1) }) }) -}) \ No newline at end of file +}) diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 3bfe8976f7..330adadd3d 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -37,6 +37,8 @@ import { } from "./utilities/workerRequests" import { watch } from "./watch" import { initialise as initialiseWebsockets } from "./websocket" +import sdk from "./sdk" +import * as pro from "@budibase/pro" const app = new Koa() @@ -102,12 +104,25 @@ server.on("close", async () => { } }) +const initPro = async () => { + await pro.init({ + backups: { + processing: { + exportAppFn: sdk.backups.exportApp, + importAppFn: sdk.backups.importApp, + statsFn: sdk.backups.calculateBackupStats, + }, + }, + }) +} + module.exports = server.listen(env.PORT || 0, async () => { console.log(`Budibase running on ${JSON.stringify(server.address())}`) env._set("PORT", server.address().port) eventEmitter.emitPort(env.PORT) fileSystem.init() await redis.init() + await initPro() // run migrations on startup if not done via http // not recommended in a clustered environment diff --git a/packages/server/src/automations/automationUtils.js b/packages/server/src/automations/automationUtils.js index 0646e453c2..06a79e6ab4 100644 --- a/packages/server/src/automations/automationUtils.js +++ b/packages/server/src/automations/automationUtils.js @@ -1,10 +1,10 @@ -const { getTable } = require("../api/controllers/table/utils") const { findHBSBlocks, decodeJSBinding, isJSBinding, encodeJSBinding, } = require("@budibase/string-templates") +const sdk = require("../sdk") /** * When values are input to the system generally they will be of type string as this is required for template strings. @@ -64,7 +64,7 @@ exports.cleanInputValues = (inputs, schema) => { * @returns {Promise} The cleaned up rows object, will should now have all the required primitive types. */ exports.cleanUpRow = async (tableId, row) => { - let table = await getTable(tableId) + let table = await sdk.tables.getTable(tableId) return exports.cleanInputValues(row, { properties: table.schema }) } diff --git a/packages/server/src/automations/bullboard.js b/packages/server/src/automations/bullboard.js index 3aac6c4fed..c4f33e07a9 100644 --- a/packages/server/src/automations/bullboard.js +++ b/packages/server/src/automations/bullboard.js @@ -1,37 +1,17 @@ const { createBullBoard } = require("@bull-board/api") const { BullAdapter } = require("@bull-board/api/bullAdapter") const { KoaAdapter } = require("@bull-board/koa") -const env = require("../environment") -const Queue = env.isTest() - ? require("../utilities/queue/inMemoryQueue") - : require("bull") -const { JobQueues } = require("../constants") -const { utils } = require("@budibase/backend-core/redis") -const { opts, redisProtocolUrl } = utils.getRedisOptions() -const listeners = require("./listeners") +const { queue } = require("@budibase/backend-core") +const automation = require("../threads/automation") -const CLEANUP_PERIOD_MS = 60 * 1000 -const queueConfig = redisProtocolUrl || { redis: opts } -let cleanupInternal = null - -let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig) -listeners.addListeners(automationQueue) - -async function cleanup() { - await automationQueue.clean(CLEANUP_PERIOD_MS, "completed") -} +let automationQueue = queue.createQueue( + queue.JobQueue.AUTOMATION, + automation.removeStalled +) const PATH_PREFIX = "/bulladmin" exports.init = () => { - // cleanup the events every 5 minutes - if (!cleanupInternal) { - cleanupInternal = setInterval(cleanup, CLEANUP_PERIOD_MS) - // fire off an initial cleanup - cleanup().catch(err => { - console.error(`Unable to cleanup automation queue initially - ${err}`) - }) - } // Set up queues for bull board admin const queues = [automationQueue] const adapters = [] @@ -48,12 +28,7 @@ exports.init = () => { } exports.shutdown = async () => { - if (automationQueue) { - clearInterval(cleanupInternal) - await automationQueue.close() - automationQueue = null - } - console.log("Bull shutdown") + await queue.shutdown() } -exports.queue = automationQueue +exports.automationQueue = automationQueue diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js index 2baa868890..521991dd2c 100644 --- a/packages/server/src/automations/index.js +++ b/packages/server/src/automations/index.js @@ -1,5 +1,5 @@ const { processEvent } = require("./utils") -const { queue, shutdown } = require("./bullboard") +const { automationQueue, shutdown } = require("./bullboard") const { TRIGGER_DEFINITIONS, rebootTrigger } = require("./triggers") const { ACTION_DEFINITIONS } = require("./actions") @@ -8,7 +8,7 @@ const { ACTION_DEFINITIONS } = require("./actions") */ exports.init = async function () { // this promise will not complete - const promise = queue.process(async job => { + const promise = automationQueue.process(async job => { await processEvent(job) }) // on init we need to trigger any reboot automations @@ -17,13 +17,13 @@ exports.init = async function () { } exports.getQueues = () => { - return [queue] + return [automationQueue] } exports.shutdown = () => { return shutdown() } -exports.queue = queue +exports.automationQueue = automationQueue exports.TRIGGER_DEFINITIONS = TRIGGER_DEFINITIONS exports.ACTION_DEFINITIONS = ACTION_DEFINITIONS diff --git a/packages/server/src/automations/listeners.ts b/packages/server/src/automations/listeners.ts deleted file mode 100644 index 9f8667bd29..0000000000 --- a/packages/server/src/automations/listeners.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { Queue, Job, JobId } from "bull" -import { AutomationEvent } from "../definitions/automations" -import * as automation from "../threads/automation" - -export const addListeners = (queue: Queue) => { - logging(queue) - handleStalled(queue) -} - -const handleStalled = (queue: Queue) => { - queue.on("stalled", async (job: Job) => { - await automation.removeStalled(job as AutomationEvent) - }) -} - -const logging = (queue: Queue) => { - if (process.env.NODE_DEBUG?.includes("bull")) { - queue - .on("error", (error: any) => { - // An error occurred. - console.error(`automation-event=error error=${JSON.stringify(error)}`) - }) - .on("waiting", (jobId: JobId) => { - // A Job is waiting to be processed as soon as a worker is idling. - console.log(`automation-event=waiting jobId=${jobId}`) - }) - .on("active", (job: Job, jobPromise: any) => { - // A job has started. You can use `jobPromise.cancel()`` to abort it. - console.log(`automation-event=active jobId=${job.id}`) - }) - .on("stalled", (job: Job) => { - // A job has been marked as stalled. This is useful for debugging job - // workers that crash or pause the event loop. - console.error( - `automation-event=stalled jobId=${job.id} job=${JSON.stringify(job)}` - ) - }) - .on("progress", (job: Job, progress: any) => { - // A job's progress was updated! - console.log( - `automation-event=progress jobId=${job.id} progress=${progress}` - ) - }) - .on("completed", (job: Job, result) => { - // A job successfully completed with a `result`. - console.log( - `automation-event=completed jobId=${job.id} result=${result}` - ) - }) - .on("failed", (job, err: any) => { - // A job failed with reason `err`! - console.log(`automation-event=failed jobId=${job.id} error=${err}`) - }) - .on("paused", () => { - // The queue has been paused. - console.log(`automation-event=paused`) - }) - .on("resumed", (job: Job) => { - // The queue has been resumed. - console.log(`automation-event=paused jobId=${job.id}`) - }) - .on("cleaned", (jobs: Job[], type: string) => { - // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned - // jobs, and `type` is the type of jobs cleaned. - console.log( - `automation-event=cleaned length=${jobs.length} type=${type}` - ) - }) - .on("drained", () => { - // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed) - console.log(`automation-event=drained`) - }) - .on("removed", (job: Job) => { - // A job successfully removed. - console.log(`automation-event=removed jobId=${job.id}`) - }) - } -} diff --git a/packages/server/src/automations/triggers.js b/packages/server/src/automations/triggers.js index 395390113a..6a4bbd8da6 100644 --- a/packages/server/src/automations/triggers.js +++ b/packages/server/src/automations/triggers.js @@ -4,7 +4,7 @@ const { coerce } = require("../utilities/rowProcessor") const { definitions } = require("./triggerInfo") const { isDevAppID } = require("../db/utils") // need this to call directly, so we can get a response -const { queue } = require("./bullboard") +const { automationQueue } = require("./bullboard") const { checkTestFlag } = require("../utilities/redis") const utils = require("./utils") const env = require("../environment") @@ -56,7 +56,7 @@ async function queueRelevantRowAutomations(event, eventType) { automationTrigger.inputs && automationTrigger.inputs.tableId === event.row.tableId ) { - await queue.add({ automation, event }, JOB_OPTS) + await automationQueue.add({ automation, event }, JOB_OPTS) } } }) @@ -110,7 +110,7 @@ exports.externalTrigger = async function ( if (getResponses) { return utils.processEvent({ data }) } else { - return queue.add(data, JOB_OPTS) + return automationQueue.add(data, JOB_OPTS) } } @@ -136,7 +136,7 @@ exports.rebootTrigger = async () => { timestamp: Date.now(), }, } - rebootEvents.push(queue.add(job, JOB_OPTS)) + rebootEvents.push(automationQueue.add(job, JOB_OPTS)) } } await Promise.all(rebootEvents) diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 7e19486798..0eebcb21cf 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -1,7 +1,7 @@ import { Thread, ThreadType } from "../threads" import { definitions } from "./triggerInfo" import * as webhooks from "../api/controllers/webhook" -import { queue } from "./bullboard" +import { automationQueue } from "./bullboard" import newid from "../db/newid" import { updateEntityMetadata } from "../utilities" import { MetadataTypes, WebhookType } from "../constants" @@ -79,21 +79,25 @@ export function removeDeprecated(definitions: any) { // end the repetition and the job itself export async function disableAllCrons(appId: any) { const promises = [] - const jobs = await queue.getRepeatableJobs() + const jobs = await automationQueue.getRepeatableJobs() for (let job of jobs) { if (job.key.includes(`${appId}_cron`)) { - promises.push(queue.removeRepeatableByKey(job.key)) + promises.push(automationQueue.removeRepeatableByKey(job.key)) if (job.id) { - promises.push(queue.removeJobs(job.id)) + promises.push(automationQueue.removeJobs(job.id)) } } } return Promise.all(promises) } -export async function disableCron(jobId: string, jobKey: string) { - await queue.removeRepeatableByKey(jobKey) - await queue.removeJobs(jobId) +export async function disableCronById(jobId: number | string) { + const repeatJobs = await automationQueue.getRepeatableJobs() + for (let repeatJob of repeatJobs) { + if (repeatJob.id === jobId) { + await automationQueue.removeRepeatableByKey(repeatJob.key) + } + } console.log(`jobId=${jobId} disabled`) } @@ -141,7 +145,7 @@ export async function enableCronTrigger(appId: any, automation: Automation) { ) { // make a job id rather than letting Bull decide, makes it easier to handle on way out const jobId = `${appId}_cron_${newid()}` - const job: any = await queue.add( + const job: any = await automationQueue.add( { automation, event: { appId, timestamp: Date.now() }, diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index 2a92e87ff8..a3bccae754 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -1,10 +1,6 @@ const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles") const { UserStatus } = require("@budibase/backend-core/constants") -const { ObjectStoreBuckets } = require("@budibase/backend-core/objectStore") - -exports.JobQueues = { - AUTOMATIONS: "automationQueue", -} +const { objectStore } = require("@budibase/backend-core") const FilterTypes = { STRING: "string", @@ -211,6 +207,6 @@ exports.AutomationErrors = { } // pass through the list from the auth/core lib -exports.ObjectStoreBuckets = ObjectStoreBuckets +exports.ObjectStoreBuckets = objectStore.ObjectStoreBuckets exports.MAX_AUTOMATION_RECURRING_ERRORS = 5 diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js index 046172df73..acd72cbf66 100644 --- a/packages/server/src/db/utils.js +++ b/packages/server/src/db/utils.js @@ -1,6 +1,7 @@ const newid = require("./newid") const { - DocumentType: CoreDocTypes, + DocumentType: CoreDocType, + InternalTable, getRoleParams, generateRoleID, APP_DEV_PREFIX, @@ -13,6 +14,12 @@ const { generateAppID, getQueryIndex, ViewName, + getDocParams, + getRowParams, + generateRowID, + getUserMetadataParams, + generateUserMetadataID, + getGlobalIDFromUserMetadataID, } = require("@budibase/backend-core/db") const UNICODE_MAX = "\ufff0" @@ -23,28 +30,7 @@ const AppStatus = { DEPLOYED: "published", } -const DocumentType = { - ...CoreDocTypes, - TABLE: "ta", - ROW: "ro", - USER: "us", - AUTOMATION: "au", - LINK: "li", - WEBHOOK: "wh", - INSTANCE: "inst", - LAYOUT: "layout", - SCREEN: "screen", - QUERY: "query", - DEPLOYMENTS: "deployments", - METADATA: "metadata", - MEM_VIEW: "view", - USER_FLAG: "flag", - AUTOMATION_METADATA: "meta_au", -} - -const InternalTables = { - USER_METADATA: "ta_users", -} +const DocumentType = CoreDocType const SearchIndexes = { ROWS: "rows", @@ -64,11 +50,11 @@ exports.APP_PREFIX = APP_PREFIX exports.APP_DEV_PREFIX = APP_DEV_PREFIX exports.isDevAppID = isDevAppID exports.isProdAppID = isProdAppID -exports.USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}` -exports.LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}` +exports.USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}` +exports.LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}` exports.TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}` exports.ViewName = ViewName -exports.InternalTables = InternalTables +exports.InternalTables = InternalTable exports.DocumentType = DocumentType exports.SEPARATOR = SEPARATOR exports.UNICODE_MAX = UNICODE_MAX @@ -77,36 +63,15 @@ exports.AppStatus = AppStatus exports.BudibaseInternalDB = BudibaseInternalDB exports.generateAppID = generateAppID exports.generateDevAppID = getDevelopmentAppID - exports.generateRoleID = generateRoleID exports.getRoleParams = getRoleParams - exports.getQueryIndex = getQueryIndex - -/** - * If creating DB allDocs/query params with only a single top level ID this can be used, this - * is usually the case as most of our docs are top level e.g. tables, automations, users and so on. - * More complex cases such as link docs and rows which have multiple levels of IDs that their - * ID consists of need their own functions to build the allDocs parameters. - * @param {string} docType The type of document which input params are being built for, e.g. user, - * link, app, table and so on. - * @param {string|null} docId The ID of the document minus its type - this is only needed if looking - * for a singular document. - * @param {object} otherProps Add any other properties onto the request, e.g. include_docs. - * @returns {object} Parameters which can then be used with an allDocs request. - */ -function getDocParams(docType, docId = null, otherProps = {}) { - if (docId == null) { - docId = "" - } - return { - ...otherProps, - startkey: `${docType}${SEPARATOR}${docId}`, - endkey: `${docType}${SEPARATOR}${docId}${UNICODE_MAX}`, - } -} - exports.getDocParams = getDocParams +exports.getRowParams = getRowParams +exports.generateRowID = generateRowID +exports.getUserMetadataParams = getUserMetadataParams +exports.generateUserMetadataID = generateUserMetadataID +exports.getGlobalIDFromUserMetadataID = getGlobalIDFromUserMetadataID /** * Gets parameters for retrieving tables, this is a utility function for the getDocParams function. @@ -123,24 +88,6 @@ exports.generateTableID = () => { return `${DocumentType.TABLE}${SEPARATOR}${newid()}` } -/** - * Gets the DB allDocs/query params for retrieving a row. - * @param {string|null} tableId The table in which the rows have been stored. - * @param {string|null} rowId The ID of the row which is being specifically queried for. This can be - * left null to get all the rows in the table. - * @param {object} otherProps Any other properties to add to the request. - * @returns {object} Parameters which can then be used with an allDocs request. - */ -exports.getRowParams = (tableId = null, rowId = null, otherProps = {}) => { - if (tableId == null) { - return getDocParams(DocumentType.ROW, null, otherProps) - } - - const endOfKey = rowId == null ? `${tableId}${SEPARATOR}` : rowId - - return getDocParams(DocumentType.ROW, endOfKey, otherProps) -} - /** * Given a row ID this will find the table ID within it (only works for internal tables). * @param {string} rowId The ID of the row. @@ -153,44 +100,6 @@ exports.getTableIDFromRowID = rowId => { return `${DocumentType.TABLE}${SEPARATOR}${components[0]}` } -/** - * Gets a new row ID for the specified table. - * @param {string} tableId The table which the row is being created for. - * @param {string|null} id If an ID is to be used then the UUID can be substituted for this. - * @returns {string} The new ID which a row doc can be stored under. - */ -exports.generateRowID = (tableId, id = null) => { - id = id || newid() - return `${DocumentType.ROW}${SEPARATOR}${tableId}${SEPARATOR}${id}` -} - -/** - * Gets parameters for retrieving users, this is a utility function for the getDocParams function. - */ -exports.getUserMetadataParams = (userId = null, otherProps = {}) => { - return exports.getRowParams(InternalTables.USER_METADATA, userId, otherProps) -} - -/** - * Generates a new user ID based on the passed in global ID. - * @param {string} globalId The ID of the global user. - * @returns {string} The new user ID which the user doc can be stored under. - */ -exports.generateUserMetadataID = globalId => { - return exports.generateRowID(InternalTables.USER_METADATA, globalId) -} - -/** - * Breaks up the ID to get the global ID. - */ -exports.getGlobalIDFromUserMetadataID = id => { - const prefix = `${DocumentType.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}` - if (!id || !id.includes(prefix)) { - return id - } - return id.split(prefix)[1] -} - /** * Gets parameters for retrieving automations, this is a utility function for the getDocParams function. */ diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index ed1455c049..877a1b4579 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -27,18 +27,6 @@ export interface TriggerOutput { timestamp?: number } -export interface AutomationEvent { - data: { - automation: Automation - event: any - } - opts?: { - repeat?: { - jobId: string - } - } -} - export interface AutomationContext extends AutomationResults { steps: any[] trigger: any diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts index 150a3b4d4a..b6d896a1ca 100644 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ b/packages/server/src/migrations/functions/backfill/app/tables.ts @@ -1,18 +1,11 @@ import { events } from "@budibase/backend-core" -import { getTableParams } from "../../../../db/utils" -import { Table } from "@budibase/types" +import sdk from "../../../../sdk" -const getTables = async (appDb: any): Promise => { - const response = await appDb.allDocs( - getTableParams(null, { - include_docs: true, - }) - ) - return response.rows.map((row: any) => row.doc) -} - -export const backfill = async (appDb: any, timestamp: string | number) => { - const tables = await getTables(appDb) +export const backfill = async ( + appDb: PouchDB.Database, + timestamp: string | number +) => { + const tables = await sdk.tables.getAllInternalTables(appDb) for (const table of tables) { await events.table.created(table, timestamp) diff --git a/packages/server/src/sdk/app/backups/constants.ts b/packages/server/src/sdk/app/backups/constants.ts new file mode 100644 index 0000000000..2f011ea2de --- /dev/null +++ b/packages/server/src/sdk/app/backups/constants.ts @@ -0,0 +1,2 @@ +export const DB_EXPORT_FILE = "db.txt" +export const GLOBAL_DB_EXPORT_FILE = "global.txt" diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts new file mode 100644 index 0000000000..8de51ed1e6 --- /dev/null +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -0,0 +1,171 @@ +import { db as dbCore } from "@budibase/backend-core" +import { budibaseTempDir } from "../../../utilities/budibaseDir" +import { retrieveDirectory } from "../../../utilities/fileSystem/utilities" +import { streamFile, createTempFolder } from "../../../utilities/fileSystem" +import { ObjectStoreBuckets } from "../../../constants" +import { + LINK_USER_METADATA_PREFIX, + TABLE_ROW_PREFIX, + USER_METDATA_PREFIX, +} from "../../../db/utils" +import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" +import fs from "fs" +import { join } from "path" +import env from "../../../environment" +const uuid = require("uuid/v4") +const tar = require("tar") +const MemoryStream = require("memorystream") + +type ExportOpts = { + filter?: any + exportPath?: string + tar?: boolean + excludeRows?: boolean +} + +function tarFilesToTmp(tmpDir: string, files: string[]) { + const exportFile = join(budibaseTempDir(), `${uuid()}.tar.gz`) + tar.create( + { + sync: true, + gzip: true, + file: exportFile, + recursive: true, + cwd: tmpDir, + }, + files + ) + return exportFile +} + +/** + * Exports a DB to either file or a variable (memory). + * @param {string} dbName the DB which is to be exported. + * @param {object} opts various options for the export, e.g. whether to stream, + * a filter function or the name of the export. + * @return {*} either a readable stream or a string + */ +export async function exportDB(dbName: string, opts: ExportOpts = {}) { + return dbCore.doWithDB(dbName, async (db: any) => { + // Write the dump to file if required + if (opts?.exportPath) { + const path = opts?.exportPath + const writeStream = fs.createWriteStream(path) + await db.dump(writeStream, { filter: opts?.filter }) + return path + } else { + // Stringify the dump in memory if required + const memStream = new MemoryStream() + let appString = "" + memStream.on("data", (chunk: any) => { + appString += chunk.toString() + }) + await db.dump(memStream, { filter: opts?.filter }) + return appString + } + }) +} + +function defineFilter(excludeRows?: boolean) { + const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX] + if (excludeRows) { + ids.push(TABLE_ROW_PREFIX) + } + return (doc: any) => + !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr) +} + +/** + * Local utility to back up the database state for an app, excluding global user + * data or user relationships. + * @param {string} appId The app to back up + * @param {object} config Config to send to export DB/attachment export + * @returns {*} either a string or a stream of the backup + */ +export async function exportApp(appId: string, config?: ExportOpts) { + const prodAppId = dbCore.getProdAppID(appId) + const appPath = `${prodAppId}/` + // export bucket contents + let tmpPath + if (!env.isTest()) { + tmpPath = await retrieveDirectory(ObjectStoreBuckets.APPS, appPath) + } else { + tmpPath = createTempFolder(uuid()) + } + const downloadedPath = join(tmpPath, appPath) + if (fs.existsSync(downloadedPath)) { + const allFiles = fs.readdirSync(downloadedPath) + for (let file of allFiles) { + const path = join(downloadedPath, file) + // move out of app directory, simplify structure + fs.renameSync(path, join(downloadedPath, "..", file)) + } + // remove the old app directory created by object export + fs.rmdirSync(downloadedPath) + } + // enforce an export of app DB to the tmp path + const dbPath = join(tmpPath, DB_EXPORT_FILE) + await exportDB(appId, { + ...config, + filter: defineFilter(config?.excludeRows), + exportPath: dbPath, + }) + // if tar requested, return where the tarball is + if (config?.tar) { + // now the tmpPath contains both the DB export and attachments, tar this + const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath)) + // cleanup the tmp export files as tarball returned + fs.rmSync(tmpPath, { recursive: true, force: true }) + return tarPath + } + // tar not requested, turn the directory where export is + else { + return tmpPath + } +} + +/** + * Export all apps + global DB (if supplied) to a single tarball, this includes + * the attachments for each app as well. + * @param {object[]} appMetadata The IDs and names of apps to export. + * @param {string} globalDbContents The contents of the global DB to export as well. + * @return {string} The path to the tarball. + */ +export async function exportMultipleApps( + appMetadata: { appId: string; name: string }[], + globalDbContents?: string +) { + const tmpPath = join(budibaseTempDir(), uuid()) + fs.mkdirSync(tmpPath) + let exportPromises: Promise[] = [] + // export each app to a directory, then move it into the complete export + const exportAndMove = async (appId: string, appName: string) => { + const path = await exportApp(appId) + await fs.promises.rename(path, join(tmpPath, appName)) + } + for (let metadata of appMetadata) { + exportPromises.push(exportAndMove(metadata.appId, metadata.name)) + } + // wait for all exports to finish + await Promise.all(exportPromises) + // add the global DB contents + if (globalDbContents) { + fs.writeFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), globalDbContents) + } + const appNames = appMetadata.map(metadata => metadata.name) + const tarPath = tarFilesToTmp(tmpPath, [...appNames, GLOBAL_DB_EXPORT_FILE]) + // clear up the tmp path now tarball generated + fs.rmSync(tmpPath, { recursive: true, force: true }) + return tarPath +} + +/** + * Streams a backup of the database state for an app + * @param {string} appId The ID of the app which is to be backed up. + * @param {boolean} excludeRows Flag to state whether the export should include data. + * @returns {*} a readable stream of the backup which is written in real time + */ +export async function streamExportApp(appId: string, excludeRows: boolean) { + const tmpPath = await exportApp(appId, { excludeRows, tar: true }) + return streamFile(tmpPath) +} diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts new file mode 100644 index 0000000000..aaab85ec92 --- /dev/null +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -0,0 +1,168 @@ +import { db as dbCore } from "@budibase/backend-core" +import { TABLE_ROW_PREFIX } from "../../../db/utils" +import { budibaseTempDir } from "../../../utilities/budibaseDir" +import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" +import { + uploadDirectory, + upload, +} from "../../../utilities/fileSystem/utilities" +import { downloadTemplate } from "../../../utilities/fileSystem" +import { ObjectStoreBuckets, FieldTypes } from "../../../constants" +import { join } from "path" +import fs from "fs" +import sdk from "../../" +import { CouchFindOptions, RowAttachment } from "@budibase/types" +const uuid = require("uuid/v4") +const tar = require("tar") + +type TemplateType = { + file?: { + type: string + path: string + } + key?: string +} + +async function updateAttachmentColumns( + prodAppId: string, + db: PouchDB.Database +) { + // iterate through attachment documents and update them + const tables = await sdk.tables.getAllInternalTables(db) + for (let table of tables) { + const attachmentCols: string[] = [] + for (let [key, column] of Object.entries(table.schema)) { + if (column.type === FieldTypes.ATTACHMENT) { + attachmentCols.push(key) + } + } + // no attachment columns, nothing to do + if (attachmentCols.length === 0) { + continue + } + // use the CouchDB Mango query API to lookup rows that have attachments + const params: CouchFindOptions = { + selector: { + _id: { + $regex: `^${TABLE_ROW_PREFIX}`, + }, + }, + } + attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) + const { rows } = await dbCore.directCouchFind(db.name, params) + for (let row of rows) { + for (let column of attachmentCols) { + if (!Array.isArray(row[column])) { + continue + } + row[column] = row[column].map((attachment: RowAttachment) => { + // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv + const urlParts = attachment.url.split("/") + // drop the first empty element + urlParts.shift() + // get the prefix + const prefix = urlParts.shift() + // remove the app ID + urlParts.shift() + // add new app ID + urlParts.unshift(prodAppId) + const key = urlParts.join("/") + return { + ...attachment, + key, + url: `/${prefix}/${key}`, + } + }) + } + } + // write back the updated attachments + await db.bulkDocs(rows) + } +} + +/** + * This function manages temporary template files which are stored by Koa. + * @param {Object} template The template object retrieved from the Koa context object. + * @returns {Object} Returns a fs read stream which can be loaded into the database. + */ +async function getTemplateStream(template: TemplateType) { + if (template.file) { + return fs.createReadStream(template.file.path) + } else if (template.key) { + const [type, name] = template.key.split("/") + const tmpPath = await downloadTemplate(type, name) + return fs.createReadStream(join(tmpPath, name, "db", "dump.txt")) + } +} + +export function untarFile(file: { path: string }) { + const tmpPath = join(budibaseTempDir(), uuid()) + fs.mkdirSync(tmpPath) + // extract the tarball + tar.extract({ + sync: true, + cwd: tmpPath, + file: file.path, + }) + return tmpPath +} + +export function getGlobalDBFile(tmpPath: string) { + return fs.readFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), "utf8") +} + +export function getListOfAppsInMulti(tmpPath: string) { + return fs.readdirSync(tmpPath).filter(dir => dir !== GLOBAL_DB_EXPORT_FILE) +} + +export async function importApp( + appId: string, + db: PouchDB.Database, + template: TemplateType +) { + let prodAppId = dbCore.getProdAppID(appId) + let dbStream: any + const isTar = template.file && template.file.type === "application/gzip" + const isDirectory = + template.file && fs.lstatSync(template.file.path).isDirectory() + if (template.file && (isTar || isDirectory)) { + const tmpPath = isTar ? untarFile(template.file) : template.file.path + const contents = fs.readdirSync(tmpPath) + // have to handle object import + if (contents.length) { + let promises = [] + let excludedFiles = [GLOBAL_DB_EXPORT_FILE, DB_EXPORT_FILE] + for (let filename of contents) { + const path = join(tmpPath, filename) + if (excludedFiles.includes(filename)) { + continue + } + filename = join(prodAppId, filename) + if (fs.lstatSync(path).isDirectory()) { + promises.push( + uploadDirectory(ObjectStoreBuckets.APPS, path, filename) + ) + } else { + promises.push( + upload({ + bucket: ObjectStoreBuckets.APPS, + path, + filename, + }) + ) + } + } + await Promise.all(promises) + } + dbStream = fs.createReadStream(join(tmpPath, DB_EXPORT_FILE)) + } else { + dbStream = await getTemplateStream(template) + } + // @ts-ignore + const { ok } = await db.load(dbStream) + if (!ok) { + throw "Error loading database dump from template." + } + await updateAttachmentColumns(prodAppId, db) + return ok +} diff --git a/packages/server/src/sdk/app/backups/index.ts b/packages/server/src/sdk/app/backups/index.ts new file mode 100644 index 0000000000..8e5697c53c --- /dev/null +++ b/packages/server/src/sdk/app/backups/index.ts @@ -0,0 +1,9 @@ +import * as exportApps from "./exports" +import * as importApps from "./imports" +import * as statistics from "./statistics" + +export default { + ...exportApps, + ...importApps, + ...statistics, +} diff --git a/packages/server/src/sdk/app/backups/statistics.ts b/packages/server/src/sdk/app/backups/statistics.ts new file mode 100644 index 0000000000..3f03158264 --- /dev/null +++ b/packages/server/src/sdk/app/backups/statistics.ts @@ -0,0 +1,77 @@ +import { context, db as dbCore } from "@budibase/backend-core" +import { + getDatasourceParams, + getTableParams, + getAutomationParams, + getScreenParams, +} from "../../../db/utils" + +async function runInContext(appId: string, cb: any, db?: PouchDB.Database) { + if (db) { + return cb(db) + } else { + const devAppId = dbCore.getDevAppID(appId) + return context.doInAppContext(devAppId, () => { + const db = context.getAppDB() + return cb(db) + }) + } +} + +export async function calculateDatasourceCount( + appId: string, + db?: PouchDB.Database +) { + return runInContext( + appId, + async (db: PouchDB.Database) => { + const datasourceList = await db.allDocs(getDatasourceParams()) + const tableList = await db.allDocs(getTableParams()) + return datasourceList.rows.length + tableList.rows.length + }, + db + ) +} + +export async function calculateAutomationCount( + appId: string, + db?: PouchDB.Database +) { + return runInContext( + appId, + async (db: PouchDB.Database) => { + const automationList = await db.allDocs(getAutomationParams()) + return automationList.rows.length + }, + db + ) +} + +export async function calculateScreenCount( + appId: string, + db?: PouchDB.Database +) { + return runInContext( + appId, + async (db: PouchDB.Database) => { + const screenList = await db.allDocs(getScreenParams()) + return screenList.rows.length + }, + db + ) +} + +export async function calculateBackupStats(appId: string) { + return runInContext(appId, async (db: PouchDB.Database) => { + const promises = [] + promises.push(calculateDatasourceCount(appId, db)) + promises.push(calculateAutomationCount(appId, db)) + promises.push(calculateScreenCount(appId, db)) + const responses = await Promise.all(promises) + return { + datasources: responses[0], + automations: responses[1], + screens: responses[2], + } + }) +} diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts new file mode 100644 index 0000000000..eeaf869055 --- /dev/null +++ b/packages/server/src/sdk/app/tables/index.ts @@ -0,0 +1,60 @@ +import { getAppDB } from "@budibase/backend-core/context" +import { BudibaseInternalDB, getTableParams } from "../../../db/utils" +import { + breakExternalTableId, + isExternalTable, + isSQL, +} from "../../../integrations/utils" +import { Table } from "@budibase/types" + +async function getAllInternalTables(db?: PouchDB.Database): Promise { + if (!db) { + db = getAppDB() as PouchDB.Database + } + const internalTables = await db.allDocs( + getTableParams(null, { + include_docs: true, + }) + ) + return internalTables.rows.map((tableDoc: any) => ({ + ...tableDoc.doc, + type: "internal", + sourceId: BudibaseInternalDB._id, + })) +} + +async function getAllExternalTables(datasourceId: any): Promise { + const db = getAppDB() + const datasource = await db.get(datasourceId) + if (!datasource || !datasource.entities) { + throw "Datasource is not configured fully." + } + return datasource.entities +} + +async function getExternalTable( + datasourceId: any, + tableName: any +): Promise { + const entities = await getAllExternalTables(datasourceId) + return entities[tableName] +} + +async function getTable(tableId: any): Promise
{ + const db = getAppDB() + if (isExternalTable(tableId)) { + let { datasourceId, tableName } = breakExternalTableId(tableId) + const datasource = await db.get(datasourceId) + const table = await getExternalTable(datasourceId, tableName) + return { ...table, sql: isSQL(datasource) } + } else { + return db.get(tableId) + } +} + +export default { + getAllInternalTables, + getAllExternalTables, + getExternalTable, + getTable, +} diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts new file mode 100644 index 0000000000..8bdc4f8e77 --- /dev/null +++ b/packages/server/src/sdk/index.ts @@ -0,0 +1,13 @@ +import { default as backups } from "./app/backups" +import { default as tables } from "./app/tables" + +const sdk = { + backups, + tables, +} + +// default export for TS +export default sdk + +// default export for JS +module.exports = sdk diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js index 38aa84be2d..097b2eabaf 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.js +++ b/packages/server/src/tests/utilities/TestConfiguration.js @@ -25,7 +25,7 @@ const newid = require("../../db/newid") const context = require("@budibase/backend-core/context") const { generateDevInfoID, SEPARATOR } = require("@budibase/backend-core/db") const { encrypt } = require("@budibase/backend-core/encryption") -const { DocumentType } = require("../../db/utils") +const { DocumentType, generateUserMetadataID } = require("../../db/utils") const GLOBAL_USER_ID = "us_uuid1" const EMAIL = "babs@babs.com" @@ -95,7 +95,10 @@ class TestConfiguration { // use a new id as the name to avoid name collisions async init(appName = newid()) { - await this.globalUser() + this.user = await this.globalUser() + this.globalUserId = this.user._id + this.userMetadataId = generateUserMetadataID(this.globalUserId) + return this.createApp(appName) } diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 64ae9439d8..b4b290462e 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -1,6 +1,11 @@ import { default as threadUtils } from "./utils" +import { Job } from "bull" threadUtils.threadSetup() -import { isRecurring, disableCron, isErrorInOutput } from "../automations/utils" +import { + isRecurring, + disableCronById, + isErrorInOutput, +} from "../automations/utils" import { default as actions } from "../automations/actions" import { default as automationUtils } from "../automations/automationUtils" import { default as AutomationEmitter } from "../events/AutomationEmitter" @@ -13,7 +18,6 @@ import { LoopStep, LoopStepType, LoopInput, - AutomationEvent, TriggerOutput, AutomationContext, AutomationMetadata, @@ -73,19 +77,16 @@ class Orchestrator { _automation: Automation _emitter: any _context: AutomationContext - _repeat?: { jobId: string; jobKey: string } + _job: Job executionOutput: AutomationContext - constructor(automation: Automation, triggerOutput: TriggerOutput, opts: any) { + constructor(job: Job) { + let automation = job.data.automation, + triggerOutput = job.data.event const metadata = triggerOutput.metadata this._chainCount = metadata ? metadata.automationChainCount : 0 this._appId = triggerOutput.appId as string - if (opts?.repeat) { - this._repeat = { - jobId: opts.repeat.jobId, - jobKey: opts.repeat.key, - } - } + this._job = job const triggerStepId = automation.definition.trigger.stepId triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput) // remove from context @@ -134,7 +135,7 @@ class Orchestrator { } async stopCron(reason: string) { - if (!this._repeat) { + if (!this._job.opts.repeat) { return } logWarn( @@ -142,7 +143,7 @@ class Orchestrator { ) const automation = this._automation const trigger = automation.definition.trigger - await disableCron(this._repeat?.jobId, this._repeat?.jobKey) + await disableCronById(this._job.id) this.updateExecutionOutput( trigger.id, trigger.stepId, @@ -156,7 +157,7 @@ class Orchestrator { } async checkIfShouldStop(metadata: AutomationMetadata): Promise { - if (!metadata.errorCount || !this._repeat) { + if (!metadata.errorCount || !this._job.opts.repeat) { return false } if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) { @@ -475,17 +476,13 @@ class Orchestrator { } } -export function execute(input: AutomationEvent, callback: WorkerCallback) { - const appId = input.data.event.appId +export function execute(job: Job, callback: WorkerCallback) { + const appId = job.data.event.appId if (!appId) { throw new Error("Unable to execute, event doesn't contain app ID.") } doInAppContext(appId, async () => { - const automationOrchestrator = new Orchestrator( - input.data.automation, - input.data.event, - input.opts - ) + const automationOrchestrator = new Orchestrator(job) try { const response = await automationOrchestrator.execute() callback(null, response) @@ -495,17 +492,13 @@ export function execute(input: AutomationEvent, callback: WorkerCallback) { }) } -export const removeStalled = async (input: AutomationEvent) => { - const appId = input.data.event.appId +export const removeStalled = async (job: Job) => { + const appId = job.data.event.appId if (!appId) { throw new Error("Unable to execute, event doesn't contain app ID.") } await doInAppContext(appId, async () => { - const automationOrchestrator = new Orchestrator( - input.data.automation, - input.data.event, - input.opts - ) + const automationOrchestrator = new Orchestrator(job) await automationOrchestrator.stopCron("stalled") }) } diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 4e9b13cca0..1eb8a481e5 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -2,17 +2,11 @@ const { budibaseTempDir } = require("../budibaseDir") const fs = require("fs") const { join } = require("path") const uuid = require("uuid/v4") -const { - doWithDB, - dangerousGetDB, - closeDB, -} = require("@budibase/backend-core/db") const { ObjectStoreBuckets } = require("../../constants") const { upload, retrieve, retrieveToTmp, - streamUpload, deleteFolder, downloadTarball, downloadTarballDirect, @@ -21,12 +15,6 @@ const { const { updateClientLibrary } = require("./clientLibrary") const { checkSlashesInUrl } = require("../") const env = require("../../environment") -const { - USER_METDATA_PREFIX, - LINK_USER_METADATA_PREFIX, - TABLE_ROW_PREFIX, -} = require("../../db/utils") -const MemoryStream = require("memorystream") const { getAppId } = require("@budibase/backend-core/context") const tar = require("tar") const fetch = require("node-fetch") @@ -86,21 +74,6 @@ exports.checkDevelopmentEnvironment = () => { } } -/** - * This function manages temporary template files which are stored by Koa. - * @param {Object} template The template object retrieved from the Koa context object. - * @returns {Object} Returns an fs read stream which can be loaded into the database. - */ -exports.getTemplateStream = async template => { - if (template.file) { - return fs.createReadStream(template.file.path) - } else { - const [type, name] = template.key.split("/") - const tmpPath = await exports.downloadTemplate(type, name) - return fs.createReadStream(join(tmpPath, name, "db", "dump.txt")) - } -} - /** * Used to retrieve a handlebars file from the system which will be used as a template. * This is allowable as the template handlebars files should be static and identical across @@ -124,98 +97,8 @@ exports.apiFileReturn = contents => { return fs.createReadStream(path) } -exports.defineFilter = excludeRows => { - const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX] - if (excludeRows) { - ids.push(TABLE_ROW_PREFIX) - } - return doc => - !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr) -} - -/** - * Local utility to back up the database state for an app, excluding global user - * data or user relationships. - * @param {string} appId The app to backup - * @param {object} config Config to send to export DB - * @param {boolean} excludeRows Flag to state whether the export should include data. - * @returns {*} either a string or a stream of the backup - */ -const backupAppData = async (appId, config, excludeRows) => { - return await exports.exportDB(appId, { - ...config, - filter: exports.defineFilter(excludeRows), - }) -} - -/** - * Takes a copy of the database state for an app to the object store. - * @param {string} appId The ID of the app which is to be backed up. - * @param {string} backupName The name of the backup located in the object store. - * @return {*} a readable stream to the completed backup file - */ -exports.performBackup = async (appId, backupName) => { - return await backupAppData(appId, { exportName: backupName }) -} - -/** - * Streams a backup of the database state for an app - * @param {string} appId The ID of the app which is to be backed up. - * @param {boolean} excludeRows Flag to state whether the export should include data. - * @returns {*} a readable stream of the backup which is written in real time - */ -exports.streamBackup = async (appId, excludeRows) => { - return await backupAppData(appId, { stream: true }, excludeRows) -} - -/** - * Exports a DB to either file or a variable (memory). - * @param {string} dbName the DB which is to be exported. - * @param {string} exportName optional - provide a filename to write the backup to a file - * @param {boolean} stream optional - whether to perform a full backup - * @param {function} filter optional - a filter function to clear out any un-wanted docs. - * @return {*} either a readable stream or a string - */ -exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => { - // streaming a DB dump is a bit more complicated, can't close DB - if (stream) { - const db = dangerousGetDB(dbName) - const memStream = new MemoryStream() - memStream.on("end", async () => { - await closeDB(db) - }) - db.dump(memStream, { filter }) - return memStream - } - - return doWithDB(dbName, async db => { - // Write the dump to file if required - if (exportName) { - const path = join(budibaseTempDir(), exportName) - const writeStream = fs.createWriteStream(path) - await db.dump(writeStream, { filter }) - - // Upload the dump to the object store if self hosted - if (env.SELF_HOSTED) { - await streamUpload( - ObjectStoreBuckets.BACKUPS, - join(dbName, exportName), - fs.createReadStream(path) - ) - } - - return fs.createReadStream(path) - } - - // Stringify the dump in memory if required - const memStream = new MemoryStream() - let appString = "" - memStream.on("data", chunk => { - appString += chunk.toString() - }) - await db.dump(memStream, { filter }) - return appString - }) +exports.streamFile = path => { + return fs.createReadStream(path) } /** diff --git a/packages/server/src/utilities/fileSystem/utilities.js b/packages/server/src/utilities/fileSystem/utilities.js index 1c804c0142..01ba58f5bc 100644 --- a/packages/server/src/utilities/fileSystem/utilities.js +++ b/packages/server/src/utilities/fileSystem/utilities.js @@ -6,6 +6,7 @@ const { streamUpload, retrieve, retrieveToTmp, + retrieveDirectory, deleteFolder, uploadDirectory, downloadTarball, @@ -27,6 +28,7 @@ exports.upload = upload exports.streamUpload = streamUpload exports.retrieve = retrieve exports.retrieveToTmp = retrieveToTmp +exports.retrieveDirectory = retrieveDirectory exports.deleteFolder = deleteFolder exports.uploadDirectory = uploadDirectory exports.downloadTarball = downloadTarball diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 02fac81e31..33efb13a59 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -2003,18 +2003,6 @@ resolved "https://registry.yarnpkg.com/@jsdevtools/ono/-/ono-7.1.3.tgz#9df03bbd7c696a5c58885c34aa06da41c8543796" integrity sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg== -"@koa/router@8.0.0": - version "8.0.0" - resolved "https://registry.yarnpkg.com/@koa/router/-/router-8.0.0.tgz#fd4ffa6f03d8293a04c023cb4a22b612401fbe70" - integrity sha512-P70CGOGs6JPu/mnrd9lt6ESzlBXLHT/uTK8+5U4M7Oapt8la/tiZv2c7X9jq0ksFsM59RH3AwJYzKOuavDcjIw== - dependencies: - debug "^3.1.0" - http-errors "^1.3.1" - koa-compose "^3.0.0" - methods "^1.0.1" - path-to-regexp "^1.1.1" - urijs "^1.19.0" - "@koa/router@8.0.8": version "8.0.8" resolved "https://registry.yarnpkg.com/@koa/router/-/router-8.0.8.tgz#95f32d11373d03d89dcb63fabe9ac6f471095236" @@ -2668,14 +2656,6 @@ dependencies: bson "*" -"@types/bull@3.15.8": - version "3.15.8" - resolved "https://registry.yarnpkg.com/@types/bull/-/bull-3.15.8.tgz#ae2139f94490d740b37c8da5d828ce75dd82ce7c" - integrity sha512-8DbSPMSsZH5PWPnGEkAZLYgJEH4ghHJNKF7LB6Wr5R0/v6g+Vs+JoaA7kcvLtHE936xg2WpFPkaoaJgExOmKDw== - dependencies: - "@types/ioredis" "*" - "@types/redis" "^2.8.0" - "@types/caseless@*": version "0.12.2" resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.2.tgz#f65d3d6389e01eeb458bd54dc8f52b95a9463bc8" @@ -2792,7 +2772,7 @@ resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1" integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w== -"@types/ioredis@*": +"@types/ioredis@4.28.10": version "4.28.10" resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff" integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ== @@ -2865,10 +2845,10 @@ "@types/koa-compose" "*" "@types/node" "*" -"@types/koa__router@8.0.0": - version "8.0.0" - resolved "https://registry.yarnpkg.com/@types/koa__router/-/koa__router-8.0.0.tgz#057a7254a25df5bc93b42a1acacb2d99cd02d297" - integrity sha512-XaGqudqJyFOmByN+f9BrEIZEgLfBnvVtZlm/beuTxWpbWpMHiA+ZmA+mB5dsrbGemko61wUA+WG0jhUzMSq+JA== +"@types/koa__router@8.0.11": + version "8.0.11" + resolved "https://registry.yarnpkg.com/@types/koa__router/-/koa__router-8.0.11.tgz#d7b37e6db934fc072ea1baa2ab92bc8ac4564f3e" + integrity sha512-WXgKWpBsbS14kzmzD9LeFapOIa678h7zvUHxDwXwSx4ETKXhXLVUAToX6jZ/U7EihM7qwyD9W/BZvB0MRu7MTQ== dependencies: "@types/koa" "*" @@ -2971,13 +2951,6 @@ dependencies: redis "*" -"@types/redis@^2.8.0": - version "2.8.32" - resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.32.tgz#1d3430219afbee10f8cfa389dad2571a05ecfb11" - integrity sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w== - dependencies: - "@types/node" "*" - "@types/request@^2.48.7": version "2.48.8" resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.8.tgz#0b90fde3b655ab50976cb8c5ac00faca22f5a82c" @@ -3528,7 +3501,7 @@ any-base@^1.1.0: resolved "https://registry.yarnpkg.com/any-base/-/any-base-1.1.0.tgz#ae101a62bc08a597b4c9ab5b7089d456630549fe" integrity sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg== -any-promise@^1.0.0, any-promise@^1.1.0: +any-promise@^1.0.0: version "1.3.0" resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f" integrity sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A== @@ -4376,10 +4349,10 @@ buffer@^5.1.0, buffer@^5.2.0, buffer@^5.2.1, buffer@^5.5.0, buffer@^5.6.0: base64-js "^1.3.1" ieee754 "^1.1.13" -bull@4.8.5: - version "4.8.5" - resolved "https://registry.yarnpkg.com/bull/-/bull-4.8.5.tgz#eebafddc3249d6d5e8ced1c42b8bfa8efcc274aa" - integrity sha512-2Z630e4f6VsLJnWMAtfEHwIqJYmND4W3dcG48RIbXeWpvb4UnYtpe/zxEdslJu0PKrltB4IkFj5YtBsdeQRn8w== +bull@4.10.1: + version "4.10.1" + resolved "https://registry.yarnpkg.com/bull/-/bull-4.10.1.tgz#f14974b6089358b62b495a2cbf838aadc098e43f" + integrity sha512-Fp21tRPb2EaZPVfmM+ONZKVz2RA+to+zGgaTLyCKt3JMSU8OOBqK8143OQrnGuGpsyE5G+9FevFAGhdZZfQP2g== dependencies: cron-parser "^4.2.1" debuglog "^1.0.0" @@ -4906,14 +4879,6 @@ cookiejar@^2.1.0: resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.3.tgz#fc7a6216e408e74414b90230050842dacda75acc" integrity sha512-JxbCBUdrfr6AQjOXrxoTvAMJO4HBTUIlBzslcJPAz+/KT8yk53fXun51u+RenNYvad/+Vc2DIz5o9UxlCDymFQ== -cookies@~0.7.1: - version "0.7.3" - resolved "https://registry.yarnpkg.com/cookies/-/cookies-0.7.3.tgz#7912ce21fbf2e8c2da70cf1c3f351aecf59dadfa" - integrity sha512-+gixgxYSgQLTaTIilDHAdlNPZDENDQernEMiIcZpYYP14zgHsCt4Ce1FEjFtcp6GefhozebB6orvhAAWx/IS0A== - dependencies: - depd "~1.1.2" - keygrip "~1.0.3" - cookies@~0.8.0: version "0.8.0" resolved "https://registry.yarnpkg.com/cookies/-/cookies-0.8.0.tgz#1293ce4b391740a8406e3c9870e828c4b54f3f90" @@ -5137,13 +5102,6 @@ debug@^3.1.0, debug@^3.2.6, debug@^3.2.7: dependencies: ms "^2.1.1" -debug@~3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" - integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== - dependencies: - ms "2.0.0" - debuglog@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492" @@ -5714,11 +5672,6 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" -error-inject@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/error-inject/-/error-inject-1.0.0.tgz#e2b3d91b54aed672f309d950d154850fa11d4f37" - integrity sha512-JM8N6PytDbmIYm1IhPWlo8vr3NtfjhDY/1MhD/a5b/aad/USE8a0+NsqE9d5n+GVGmuNkPQWm4bFQWv18d8tMg== - error-stack-parser@^2.0.6: version "2.1.4" resolved "https://registry.yarnpkg.com/error-stack-parser/-/error-stack-parser-2.1.4.tgz#229cb01cdbfa84440bfa91876285b94680188286" @@ -7400,7 +7353,7 @@ http-errors@2.0.0: statuses "2.0.1" toidentifier "1.0.1" -http-errors@^1.3.1, http-errors@^1.6.3, http-errors@^1.7.3, http-errors@~1.8.0: +http-errors@^1.6.3, http-errors@^1.7.3, http-errors@~1.8.0: version "1.8.1" resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c" integrity sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g== @@ -9274,11 +9227,6 @@ jws@^4.0.0: jwa "^2.0.0" safe-buffer "^5.0.1" -keygrip@~1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/keygrip/-/keygrip-1.0.3.tgz#399d709f0aed2bab0a059e0cdd3a5023a053e1dc" - integrity sha512-/PpesirAIfaklxUzp4Yb7xBper9MwP6hNRA6BGGUFCgbJ+BM5CKBtsoxinNXkLHAr+GXS1/lSlF2rP7cv5Fl+g== - keygrip@~1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/keygrip/-/keygrip-1.1.0.tgz#871b1681d5e159c62a445b0c74b615e0917e7226" @@ -9364,13 +9312,6 @@ koa-body@4.2.0: co-body "^5.1.1" formidable "^1.1.1" -koa-compose@^3.0.0: - version "3.2.1" - resolved "https://registry.yarnpkg.com/koa-compose/-/koa-compose-3.2.1.tgz#a85ccb40b7d986d8e5a345b3a1ace8eabcf54de7" - integrity sha512-8gen2cvKHIZ35eDEik5WOo8zbVp9t4cP8p4hW4uE55waxolLRexKKrqfCpwhGVppnB40jWeF8bZeTVg99eZgPw== - dependencies: - any-promise "^1.1.0" - koa-compose@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/koa-compose/-/koa-compose-4.1.0.tgz#507306b9371901db41121c812e923d0d67d3e877" @@ -9392,14 +9333,6 @@ koa-connect@2.1.0: resolved "https://registry.yarnpkg.com/koa-connect/-/koa-connect-2.1.0.tgz#16bce0a917c4cb24233aaac83fbc5b83804b4a1c" integrity sha512-O9pcFafHk0oQsBevlbTBlB9co+2RUQJ4zCzu3qJPmGlGoeEZkne+7gWDkecqDPSbCtED6LmhlQladxs6NjOnMQ== -koa-convert@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/koa-convert/-/koa-convert-1.2.0.tgz#da40875df49de0539098d1700b50820cebcd21d0" - integrity sha512-K9XqjmEDStGX09v3oxR7t5uPRy0jqJdvodHa6wxWTHrTfDq0WUNnYTOOUZN6g8OM8oZQXprQASbiIXG2Ez8ehA== - dependencies: - co "^4.6.0" - koa-compose "^3.0.0" - koa-convert@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/koa-convert/-/koa-convert-2.0.0.tgz#86a0c44d81d40551bae22fee6709904573eea4f5" @@ -9501,37 +9434,7 @@ koa2-ratelimit@1.1.1: resolved "https://registry.yarnpkg.com/koa2-ratelimit/-/koa2-ratelimit-1.1.1.tgz#9c1d8257770e4a0a08063ba2ddcaf690fd457d23" integrity sha512-IpxGMdZqEhMykW0yYKGVB4vDEacPvSBH4hNpDL38ABj3W2KHNLujAljGEDg7eEjXvrRbXRSWXzANhV3c9v7nyg== -koa@2.7.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/koa/-/koa-2.7.0.tgz#7e00843506942b9d82c6cc33749f657c6e5e7adf" - integrity sha512-7ojD05s2Q+hFudF8tDLZ1CpCdVZw8JQELWSkcfG9bdtoTDzMmkRF6BQBU7JzIzCCOY3xd3tftiy/loHBUYaY2Q== - dependencies: - accepts "^1.3.5" - cache-content-type "^1.0.0" - content-disposition "~0.5.2" - content-type "^1.0.4" - cookies "~0.7.1" - debug "~3.1.0" - delegates "^1.0.0" - depd "^1.1.2" - destroy "^1.0.4" - error-inject "^1.0.0" - escape-html "^1.0.3" - fresh "~0.5.2" - http-assert "^1.3.0" - http-errors "^1.6.3" - is-generator-function "^1.0.7" - koa-compose "^4.1.0" - koa-convert "^1.2.0" - koa-is-json "^1.0.0" - on-finished "^2.3.0" - only "~0.0.2" - parseurl "^1.3.2" - statuses "^1.5.0" - type-is "^1.6.16" - vary "^1.1.2" - -koa@^2.13.1, koa@^2.13.4: +koa@2.13.4, koa@^2.13.1, koa@^2.13.4: version "2.13.4" resolved "https://registry.yarnpkg.com/koa/-/koa-2.13.4.tgz#ee5b0cb39e0b8069c38d115139c774833d32462e" integrity sha512-43zkIKubNbnrULWlHdN5h1g3SEKXOEzoAlRsHOTFpnlDu8JlAOZSMJBLULusuXRequboiwJcj5vtYXKB3k7+2g== @@ -10125,7 +10028,7 @@ merge2@^1.3.0, merge2@^1.4.1: resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== -methods@^1.0.1, methods@^1.1.1, methods@^1.1.2: +methods@^1.1.1, methods@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== @@ -11218,7 +11121,7 @@ path-parser@^6.1.0: search-params "3.0.0" tslib "^1.10.0" -path-to-regexp@1.x, path-to-regexp@^1.1.1: +path-to-regexp@1.x: version "1.8.0" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.8.0.tgz#887b3ba9d84393e87a0a0b9f4cb756198b53548a" integrity sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA== @@ -14226,7 +14129,7 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" -urijs@^1.19.0, urijs@^1.19.2: +urijs@^1.19.2: version "1.19.11" resolved "https://registry.yarnpkg.com/urijs/-/urijs-1.19.11.tgz#204b0d6b605ae80bea54bea39280cdb7c9f923cc" integrity sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ== diff --git a/packages/types/package.json b/packages/types/package.json index 0f989066cd..ac934941ef 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -16,6 +16,7 @@ "@types/koa": "2.13.4", "@types/node": "14.18.20", "rimraf": "3.0.2", - "typescript": "4.7.3" + "typescript": "4.7.3", + "@types/pouchdb": "6.4.0" } } diff --git a/packages/types/src/api/web/app/backup.ts b/packages/types/src/api/web/app/backup.ts new file mode 100644 index 0000000000..f16f8416ae --- /dev/null +++ b/packages/types/src/api/web/app/backup.ts @@ -0,0 +1,17 @@ +import { AppBackupTrigger, AppBackupType } from "../../../documents" + +export interface SearchAppBackupsRequest { + trigger: AppBackupTrigger + type: AppBackupType + startDate: string + endDate: string + page?: string +} + +export interface CreateAppBackupRequest { + name: string +} + +export interface UpdateAppBackupRequest { + name: string +} diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts new file mode 100644 index 0000000000..1d73755cb6 --- /dev/null +++ b/packages/types/src/api/web/app/index.ts @@ -0,0 +1 @@ +export * from "./backup" diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index 0129fb38d9..1dbe22aa46 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -1,3 +1,5 @@ export * from "./analytics" export * from "./user" export * from "./errors" +export * from "./schedule" +export * from "./app" diff --git a/packages/types/src/api/web/schedule.ts b/packages/types/src/api/web/schedule.ts new file mode 100644 index 0000000000..bf762b1603 --- /dev/null +++ b/packages/types/src/api/web/schedule.ts @@ -0,0 +1,15 @@ +import { + ScheduleMetadata, + ScheduleRepeatPeriod, + ScheduleType, +} from "../../documents" + +export interface CreateScheduleRequest { + type: ScheduleType + name: string + startDate: string + repeat: ScheduleRepeatPeriod + metadata: ScheduleMetadata +} + +export interface UpdateScheduleRequest extends CreateScheduleRequest {} diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts index c66d3203e8..98ffcdf360 100644 --- a/packages/types/src/api/web/user.ts +++ b/packages/types/src/api/web/user.ts @@ -44,3 +44,10 @@ export interface InviteUsersResponse { successful: { email: string }[] unsuccessful: { email: string; reason: string }[] } + +export interface SearchUsersRequest { + page?: string + email?: string + appId?: string + userIds?: string[] +} diff --git a/packages/types/src/documents/app/backup.ts b/packages/types/src/documents/app/backup.ts new file mode 100644 index 0000000000..1dd7b45287 --- /dev/null +++ b/packages/types/src/documents/app/backup.ts @@ -0,0 +1,68 @@ +import { Document } from "../document" +import { User } from "../../" + +export enum AppBackupType { + BACKUP = "backup", + RESTORE = "restore", +} + +export enum AppBackupStatus { + STARTED = "started", + PENDING = "pending", + COMPLETE = "complete", + FAILED = "failed", +} + +export enum AppBackupTrigger { + PUBLISH = "publish", + MANUAL = "manual", + SCHEDULED = "scheduled", + RESTORING = "restoring", +} + +export interface AppBackupContents { + datasources: string[] + screens: string[] + automations: string[] +} + +export interface AppBackupMetadata { + appId: string + trigger?: AppBackupTrigger + type: AppBackupType + status: AppBackupStatus + name?: string + createdBy?: string | User + timestamp: string + contents?: AppBackupContents +} + +export interface AppBackup extends Document, AppBackupMetadata { + filename?: string +} + +export type AppBackupFetchOpts = { + trigger?: AppBackupTrigger + type?: AppBackupType + limit?: number + page?: string + paginate?: boolean + startDate?: string + endDate?: string +} + +export interface AppBackupQueueData { + appId: string + docId: string + docRev: string + export?: { + trigger: AppBackupTrigger + name?: string + createdBy?: string + } + import?: { + backupId: string + nameForBackup: string + createdBy?: string + } +} diff --git a/packages/types/src/documents/app/index.ts b/packages/types/src/documents/app/index.ts index e8b29257fc..dad594b804 100644 --- a/packages/types/src/documents/app/index.ts +++ b/packages/types/src/documents/app/index.ts @@ -10,3 +10,4 @@ export * from "./view" export * from "../document" export * from "./row" export * from "./user" +export * from "./backup" diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts index ee5c0231e7..2cac32279b 100644 --- a/packages/types/src/documents/app/row.ts +++ b/packages/types/src/documents/app/row.ts @@ -16,6 +16,14 @@ export enum FieldType { INTERNAL = "internal", } +export interface RowAttachment { + size: number + name: string + url: string + extension: string + key: string +} + export interface Row extends Document { type?: string tableId?: string diff --git a/packages/types/src/documents/app/table.ts b/packages/types/src/documents/app/table.ts index 12135b60cf..1d559c5bef 100644 --- a/packages/types/src/documents/app/table.ts +++ b/packages/types/src/documents/app/table.ts @@ -49,6 +49,7 @@ export interface Table extends Document { sourceId?: string relatedFormula?: string[] constrained?: string[] + sql?: boolean indexes?: { [key: string]: any } dataImport?: { [key: string]: any } } diff --git a/packages/types/src/documents/global/index.ts b/packages/types/src/documents/global/index.ts index 84684df369..9f779d4937 100644 --- a/packages/types/src/documents/global/index.ts +++ b/packages/types/src/documents/global/index.ts @@ -3,3 +3,4 @@ export * from "./user" export * from "./userGroup" export * from "./plugin" export * from "./quotas" +export * from "./schedule" diff --git a/packages/types/src/documents/global/schedule.ts b/packages/types/src/documents/global/schedule.ts new file mode 100644 index 0000000000..63bb9284fe --- /dev/null +++ b/packages/types/src/documents/global/schedule.ts @@ -0,0 +1,32 @@ +import { Document } from "../document" + +export enum ScheduleType { + APP_BACKUP = "app_backup", +} + +export enum ScheduleRepeatPeriod { + DAILY = "daily", + WEEKLY = "weekly", + MONTHLY = "monthly", +} + +export interface Schedule extends Document { + type: ScheduleType + name: string + startDate: string + repeat: ScheduleRepeatPeriod + metadata: ScheduleMetadata +} + +export type ScheduleMetadata = AppBackupScheduleMetadata + +export const isAppBackupMetadata = ( + type: ScheduleType, + metadata: ScheduleMetadata +): metadata is AppBackupScheduleMetadata => { + return type === ScheduleType.APP_BACKUP +} + +export interface AppBackupScheduleMetadata { + apps: string[] +} diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts new file mode 100644 index 0000000000..6d47a5c36b --- /dev/null +++ b/packages/types/src/sdk/db.ts @@ -0,0 +1,22 @@ +export type PouchOptions = { + inMemory: boolean + replication: boolean + onDisk: boolean + find: boolean +} + +export enum SortOption { + ASCENDING = "asc", + DESCENDING = "desc", +} + +export type CouchFindOptions = { + selector: PouchDB.Find.Selector + fields?: string[] + sort?: { + [key: string]: SortOption + }[] + limit?: number + skip?: number + bookmark?: string +} diff --git a/packages/types/src/sdk/events/backup.ts b/packages/types/src/sdk/events/backup.ts new file mode 100644 index 0000000000..8d3e2b1afa --- /dev/null +++ b/packages/types/src/sdk/events/backup.ts @@ -0,0 +1,7 @@ +import { BaseEvent } from "./event" + +export interface AppBackupRestoreEvent extends BaseEvent { + appId: string + backupName: string + backupCreatedAt: string +} diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index 73e5315713..71caf2bf96 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -168,6 +168,9 @@ export enum Event { PLUGIN_INIT = "plugin:init", PLUGIN_IMPORTED = "plugin:imported", PLUGIN_DELETED = "plugin:deleted", + + // BACKUP + APP_BACKUP_RESTORED = "app:backup:restored", } // properties added at the final stage of the event pipeline diff --git a/packages/types/src/sdk/events/index.ts b/packages/types/src/sdk/events/index.ts index cc0c2b9aa1..5abc30f5b9 100644 --- a/packages/types/src/sdk/events/index.ts +++ b/packages/types/src/sdk/events/index.ts @@ -20,3 +20,4 @@ export * from "./backfill" export * from "./identification" export * from "./userGroup" export * from "./plugin" +export * from "./backup" diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index 0c374dd105..724b152303 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -8,3 +8,4 @@ export * from "./search" export * from "./koa" export * from "./auth" export * from "./locks" +export * from "./db" diff --git a/packages/types/src/sdk/koa.ts b/packages/types/src/sdk/koa.ts index 8d419d5cf1..8004ba72ae 100644 --- a/packages/types/src/sdk/koa.ts +++ b/packages/types/src/sdk/koa.ts @@ -1,4 +1,4 @@ -import { Context } from "koa" +import { Context, Request } from "koa" import { User } from "../documents" import { License } from "../sdk" @@ -7,7 +7,11 @@ export interface ContextUser extends User { license: License } -export interface BBContext extends Context { - user?: ContextUser +export interface BBRequest extends Request { body: any } + +export interface BBContext extends Context { + request: BBRequest + user?: ContextUser +} diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts index cbd1f4a50c..f06a8d1382 100644 --- a/packages/types/src/sdk/licensing/feature.ts +++ b/packages/types/src/sdk/licensing/feature.ts @@ -1,3 +1,4 @@ export enum Feature { USER_GROUPS = "userGroups", + APP_BACKUPS = "appBackups", } diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 74777d4590..86a092c6f5 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -25,6 +25,7 @@ export enum MonthlyQuotaName { export enum ConstantQuotaName { AUTOMATION_LOG_RETENTION_DAYS = "automationLogRetentionDays", + APP_BACKUPS_RETENTION_DAYS = "appBackupRetentionDays", } export type MeteredQuotaName = StaticQuotaName | MonthlyQuotaName @@ -76,6 +77,7 @@ export type StaticQuotas = { export type ConstantQuotas = { [ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota + [ConstantQuotaName.APP_BACKUPS_RETENTION_DAYS]: Quota } export type Quotas = { diff --git a/packages/types/yarn.lock b/packages/types/yarn.lock index c80ff652ba..f225ffc442 100644 --- a/packages/types/yarn.lock +++ b/packages/types/yarn.lock @@ -39,6 +39,13 @@ "@types/keygrip" "*" "@types/node" "*" +"@types/debug@*": + version "4.1.7" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.7.tgz#7cc0ea761509124709b8b2d1090d8f6c17aadb82" + integrity sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg== + dependencies: + "@types/ms" "*" + "@types/express-serve-static-core@^4.17.18": version "4.17.29" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.29.tgz#2a1795ea8e9e9c91b4a4bbe475034b20c1ec711c" @@ -113,6 +120,11 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw== +"@types/ms@*": + version "0.7.31" + resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197" + integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== + "@types/node@*": version "18.0.6" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.0.6.tgz#0ba49ac517ad69abe7a1508bc9b3a5483df9d5d7" @@ -123,6 +135,152 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650" integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA== +"@types/pouchdb-adapter-cordova-sqlite@*": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-cordova-sqlite/-/pouchdb-adapter-cordova-sqlite-1.0.1.tgz#49e5ee6df7cc0c23196fcb340f43a560e74eb1d6" + integrity sha512-nqlXpW1ho3KBg1mUQvZgH2755y3z/rw4UA7ZJCPMRTHofxGMY8izRVw5rHBL4/7P615or0J2udpRYxgkT3D02g== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-fruitdown@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-fruitdown/-/pouchdb-adapter-fruitdown-6.1.3.tgz#9b140ad9645cc56068728acf08ec19ac0046658e" + integrity sha512-Wz1Z1JLOW1hgmFQjqnSkmyyfH7by/iWb4abKn684WMvQfmxx6BxKJpJ4+eulkVPQzzgMMSgU1MpnQOm9FgRkbw== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-http@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-http/-/pouchdb-adapter-http-6.1.3.tgz#6e592d5f48deb6274a21ddac1498dd308096bcf3" + integrity sha512-9Z4TLbF/KJWy/D2sWRPBA+RNU0odQimfdvlDX+EY7rGcd3aVoH8qjD/X0Xcd/0dfBH5pKrNIMFFQgW/TylRCmA== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-idb@*": + version "6.1.4" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-idb/-/pouchdb-adapter-idb-6.1.4.tgz#cb9a18864585d600820cd325f007614c5c3989cd" + integrity sha512-KIAXbkF4uYUz0ZwfNEFLtEkK44mEWopAsD76UhucH92XnJloBysav+TjI4FFfYQyTjoW3S1s6V+Z14CUJZ0F6w== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-leveldb@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-leveldb/-/pouchdb-adapter-leveldb-6.1.3.tgz#17c7e75d75b992050bca15991e97fba575c61bb3" + integrity sha512-ex8NFqQGFwEpFi7AaZ5YofmuemfZNsL3nTFZBUCAKYMBkazQij1pe2ILLStSvJr0XS0qxgXjCEW19T5Wqiiskg== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-localstorage@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-localstorage/-/pouchdb-adapter-localstorage-6.1.3.tgz#0dde02ba6b9d6073a295a20196563942ba9a54bd" + integrity sha512-oor040tye1KKiGLWYtIy7rRT7C2yoyX3Tf6elEJRpjOA7Ja/H8lKc4LaSh9ATbptIcES6MRqZDxtp7ly9hsW3Q== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-memory@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-memory/-/pouchdb-adapter-memory-6.1.3.tgz#9eabdbc890fcf58960ee8b68b8685f837e75c844" + integrity sha512-gVbsIMzDzgZYThFVT4eVNsmuZwVm/4jDxP1sjlgc3qtDIxbtBhGgyNfcskwwz9Zu5Lv1avkDsIWvcxQhnvRlHg== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-node-websql@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-node-websql/-/pouchdb-adapter-node-websql-6.1.3.tgz#aa18bc68af8cf509acd12c400010dcd5fab2243d" + integrity sha512-F/P+os6Jsa7CgHtH64+Z0HfwIcj0hIRB5z8gNhF7L7dxPWoAfkopK5H2gydrP3sQrlGyN4WInF+UJW/Zu1+FKg== + dependencies: + "@types/pouchdb-adapter-websql" "*" + "@types/pouchdb-core" "*" + +"@types/pouchdb-adapter-websql@*": + version "6.1.4" + resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-websql/-/pouchdb-adapter-websql-6.1.4.tgz#359fbe42ccac0ac90b492ddb8c32fafd0aa96d79" + integrity sha512-zMJQCtXC40hBsIDRn0GhmpeGMK0f9l/OGWfLguvczROzxxcOD7REI+e6SEmX7gJKw5JuMvlfuHzkQwjmvSJbtg== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-browser@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-browser/-/pouchdb-browser-6.1.3.tgz#8f33d6ef58d6817d1f6d36979148a1c7f63244d8" + integrity sha512-EdYowrWxW9SWBMX/rux2eq7dbHi5Zeyzz+FF/IAsgQKnUxgeCO5VO2j4zTzos0SDyJvAQU+EYRc11r7xGn5tvA== + dependencies: + "@types/pouchdb-adapter-http" "*" + "@types/pouchdb-adapter-idb" "*" + "@types/pouchdb-adapter-websql" "*" + "@types/pouchdb-core" "*" + "@types/pouchdb-mapreduce" "*" + "@types/pouchdb-replication" "*" + +"@types/pouchdb-core@*": + version "7.0.10" + resolved "https://registry.yarnpkg.com/@types/pouchdb-core/-/pouchdb-core-7.0.10.tgz#d1ea1549e7fad6cb579f71459b1bc27252e06a5a" + integrity sha512-mKhjLlWWXyV3PTTjDhzDV1kc2dolO7VYFa75IoKM/hr8Er9eo8RIbS7mJLfC8r/C3p6ihZu9yZs1PWC1LQ0SOA== + dependencies: + "@types/debug" "*" + "@types/pouchdb-find" "*" + +"@types/pouchdb-find@*": + version "7.3.0" + resolved "https://registry.yarnpkg.com/@types/pouchdb-find/-/pouchdb-find-7.3.0.tgz#b917030e9f4bf6e56bf8c3b9fe4b2a25e989009a" + integrity sha512-sFPli5tBjGX9UfXioik1jUzPdcN84eV82n0lmEFuoPepWqkLjQcyri0eOa++HYOaNPyMDhKFBqEALEZivK2dRg== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-http@*": + version "6.1.3" + resolved "https://registry.yarnpkg.com/@types/pouchdb-http/-/pouchdb-http-6.1.3.tgz#09576c0d409da1f8dee34ec5b768415e2472ea52" + integrity sha512-0e9E5SqNOyPl/3FnEIbENssB4FlJsNYuOy131nxrZk36S+y1R/6qO7ZVRypWpGTqBWSuVd7gCsq2UDwO/285+w== + dependencies: + "@types/pouchdb-adapter-http" "*" + "@types/pouchdb-core" "*" + +"@types/pouchdb-mapreduce@*": + version "6.1.7" + resolved "https://registry.yarnpkg.com/@types/pouchdb-mapreduce/-/pouchdb-mapreduce-6.1.7.tgz#9ab32d1e0f234f1bf6d1e4c5d7e216e9e23ac0a3" + integrity sha512-WzBwm7tmO9QhfRzVaWT4v6JQSS/fG2OoUDrWrhX87rPe2Pn6laPvdK5li6myNRxCoI/l5e8Jd+oYBAFnaiFucA== + dependencies: + "@types/pouchdb-core" "*" + +"@types/pouchdb-node@*": + version "6.1.4" + resolved "https://registry.yarnpkg.com/@types/pouchdb-node/-/pouchdb-node-6.1.4.tgz#5214c0169fcfd2237d373380bbd65a934feb5dfb" + integrity sha512-wnTCH8X1JOPpNOfVhz8HW0AvmdHh6pt40MuRj0jQnK7QEHsHS79WujsKTKSOF8QXtPwpvCNSsI7ut7H7tfxxJQ== + dependencies: + "@types/pouchdb-adapter-http" "*" + "@types/pouchdb-adapter-leveldb" "*" + "@types/pouchdb-core" "*" + "@types/pouchdb-mapreduce" "*" + "@types/pouchdb-replication" "*" + +"@types/pouchdb-replication@*": + version "6.4.4" + resolved "https://registry.yarnpkg.com/@types/pouchdb-replication/-/pouchdb-replication-6.4.4.tgz#743406c90f13a988fa3e346ea74ce40acd170d00" + integrity sha512-BsE5LKpjJK4iAf6Fx5kyrMw+33V+Ip7uWldUnU2BYrrvtR+MLD22dcImm7DZN1st2wPPb91i0XEnQzvP0w1C/Q== + dependencies: + "@types/pouchdb-core" "*" + "@types/pouchdb-find" "*" + +"@types/pouchdb@6.4.0": + version "6.4.0" + resolved "https://registry.yarnpkg.com/@types/pouchdb/-/pouchdb-6.4.0.tgz#f9c41ca64b23029f9bf2eb4bf6956e6431cb79f8" + integrity sha512-eGCpX+NXhd5VLJuJMzwe3L79fa9+IDTrAG3CPaf4s/31PD56hOrhDJTSmRELSXuiqXr6+OHzzP0PldSaWsFt7w== + dependencies: + "@types/pouchdb-adapter-cordova-sqlite" "*" + "@types/pouchdb-adapter-fruitdown" "*" + "@types/pouchdb-adapter-http" "*" + "@types/pouchdb-adapter-idb" "*" + "@types/pouchdb-adapter-leveldb" "*" + "@types/pouchdb-adapter-localstorage" "*" + "@types/pouchdb-adapter-memory" "*" + "@types/pouchdb-adapter-node-websql" "*" + "@types/pouchdb-adapter-websql" "*" + "@types/pouchdb-browser" "*" + "@types/pouchdb-core" "*" + "@types/pouchdb-http" "*" + "@types/pouchdb-mapreduce" "*" + "@types/pouchdb-node" "*" + "@types/pouchdb-replication" "*" + "@types/qs@*": version "6.9.7" resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" diff --git a/packages/worker/package.json b/packages/worker/package.json index 9f60e99efc..5a0fe31425 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -72,7 +72,6 @@ "devDependencies": { "@types/jest": "26.0.23", "@types/koa": "2.13.4", - "@types/koa-router": "7.4.4", "@types/koa__router": "8.0.11", "@types/node": "14.18.20", "@types/uuid": "8.3.4", diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 8894330f67..ea1df5b45a 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -7,6 +7,7 @@ import { CloudAccount, InviteUserRequest, InviteUsersRequest, + SearchUsersRequest, User, } from "@budibase/types" import { @@ -144,7 +145,8 @@ export const destroy = async (ctx: any) => { } export const search = async (ctx: any) => { - const paginated = await sdk.users.paginatedUsers(ctx.request.body) + const body = ctx.request.body as SearchUsersRequest + const paginated = await sdk.users.paginatedUsers(body) // user hashed password shouldn't ever be returned for (let user of paginated.data) { if (user) { diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts index 775514ea5e..b0290507fc 100644 --- a/packages/worker/src/sdk/users/users.ts +++ b/packages/worker/src/sdk/users/users.ts @@ -27,6 +27,7 @@ import { MigrationType, PlatformUserByEmail, RowResponse, + SearchUsersRequest, User, } from "@budibase/types" import { sendEmail } from "../../utilities/email" @@ -56,7 +57,8 @@ export const paginatedUsers = async ({ page, email, appId, -}: { page?: string; email?: string; appId?: string } = {}) => { + userIds, +}: SearchUsersRequest = {}) => { const db = tenancy.getGlobalDB() // get one extra document, to have the next page const opts: any = { @@ -94,16 +96,7 @@ export const paginatedUsers = async ({ */ export const getUser = async (userId: string) => { const db = tenancy.getGlobalDB() - let user - try { - user = await db.get(userId) - } catch (err: any) { - // no user found, just return nothing - if (err.status === 404) { - return {} - } - throw err - } + let user = await db.get(userId) if (user) { delete user.password } diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index d2a138d69f..c6550296e1 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -1026,13 +1026,6 @@ dependencies: "@types/koa" "*" -"@types/koa-router@7.4.4": - version "7.4.4" - resolved "https://registry.yarnpkg.com/@types/koa-router/-/koa-router-7.4.4.tgz#db72bde3616365d74f00178d5f243c4fce7da572" - integrity sha512-3dHlZ6CkhgcWeF6wafEUvyyqjWYfKmev3vy1PtOmr0mBc3wpXPU5E8fBBd4YQo5bRpHPfmwC5yDaX7s4jhIN6A== - dependencies: - "@types/koa" "*" - "@types/koa@*", "@types/koa@2.13.4": version "2.13.4" resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.4.tgz#10620b3f24a8027ef5cbae88b393d1b31205726b" diff --git a/qa-core/src/config/internal-api/TestConfiguration/screens.ts b/qa-core/src/config/internal-api/TestConfiguration/screens.ts index 30c688022a..7814ffdfbc 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/screens.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/screens.ts @@ -1,23 +1,23 @@ -import { Screen } from "@budibase/types" +import { Screen } from "@budibase/types" import { Response } from "node-fetch" import InternalAPIClient from "./InternalAPIClient" - - export default class ScreenApi { - api: InternalAPIClient - - constructor(apiClient: InternalAPIClient) { - this.api = apiClient - } - - async create(body: any): Promise<[Response, Screen]> { - const response = await this.api.post(`/screens`, { body }) - const json = await response.json() - return [response, json] - } - async delete(screenId: string, rev: string): Promise<[Response, Screen]> { - const response = await this.api.del(`/screens/${screenId}/${rev}`) - const json = await response.json() - return [response, json] - } +export default class ScreenApi { + api: InternalAPIClient + + constructor(apiClient: InternalAPIClient) { + this.api = apiClient } + + async create(body: any): Promise<[Response, Screen]> { + const response = await this.api.post(`/screens`, { body }) + const json = await response.json() + return [response, json] + } + + async delete(screenId: string, rev: string): Promise<[Response, Screen]> { + const response = await this.api.del(`/screens/${screenId}/${rev}`) + const json = await response.json() + return [response, json] + } +} diff --git a/qa-core/src/config/internal-api/fixtures/screens.ts b/qa-core/src/config/internal-api/fixtures/screens.ts index 1ebc1eb5c8..044a35ba83 100644 --- a/qa-core/src/config/internal-api/fixtures/screens.ts +++ b/qa-core/src/config/internal-api/fixtures/screens.ts @@ -3,32 +3,32 @@ import generator from "../../generator" const randomId = generator.guid() const generateScreen = (roleId: string): any => ({ - showNavigation: true, - width: "Large", - name: randomId, - template: "createFromScratch", - props: { - _id: randomId, - _component: - "@budibase/standard-components/container", - _styles: { - normal: {}, - hover: {}, - active: {}, - selected: {} - }, - _children: [], - _instanceName: "New Screen", - direction: "column", - hAlign: "stretch", - vAlign: "top", - size: "grow", - gap: "M" - }, routing: { - route: "/test", - roleId: roleId, - homeScreen: false + showNavigation: true, + width: "Large", + name: randomId, + template: "createFromScratch", + props: { + _id: randomId, + _component: "@budibase/standard-components/container", + _styles: { + normal: {}, + hover: {}, + active: {}, + selected: {}, }, + _children: [], + _instanceName: "New Screen", + direction: "column", + hAlign: "stretch", + vAlign: "top", + size: "grow", + gap: "M", + }, + routing: { + route: "/test", + roleId: roleId, + homeScreen: false, + }, }) export default generateScreen diff --git a/qa-core/src/tests/internal-api/screens/screens.spec.ts b/qa-core/src/tests/internal-api/screens/screens.spec.ts index 0ef9fcf279..34266d6622 100644 --- a/qa-core/src/tests/internal-api/screens/screens.spec.ts +++ b/qa-core/src/tests/internal-api/screens/screens.spec.ts @@ -5,7 +5,6 @@ import generateApp from "../../../config/internal-api/fixtures/applications" import { Screen } from "@budibase/types" import generateScreen from "../../../config/internal-api/fixtures/screens" - describe("Internal API - /screens endpoints", () => { const api = new InternalAPIClient() const config = new TestConfiguration(api) @@ -27,7 +26,9 @@ describe("Internal API - /screens endpoints", () => { const roleArray = ["BASIC", "POWER", "ADMIN", "PUBLIC"] appConfig.applications.api.appId = app.appId for (let role in roleArray) { - const [response, screen] = await config.screen.create(generateScreen(roleArray[role])) + const [response, screen] = await config.screen.create( + generateScreen(roleArray[role]) + ) expect(response).toHaveStatusCode(200) expect(screen.routing.roleId).toEqual(roleArray[role]) } @@ -39,7 +40,9 @@ describe("Internal API - /screens endpoints", () => { // Create Screen appConfig.applications.api.appId = app.appId - const [response, screen] = await config.screen.create(generateScreen("BASIC")) + const [response, screen] = await config.screen.create( + generateScreen("BASIC") + ) // Check screen exists const [routesResponse, routes] = await appConfig.applications.getRoutes() @@ -53,7 +56,9 @@ describe("Internal API - /screens endpoints", () => { // Create Screen appConfig.applications.api.appId = app.appId - const [screenResponse, screen] = await config.screen.create(generateScreen("BASIC")) + const [screenResponse, screen] = await config.screen.create( + generateScreen("BASIC") + ) // Delete Screen const [response] = await config.screen.delete(screen._id!, screen._rev!)