diff --git a/hosting/proxy/10-listen-on-ipv6-by-default.sh b/hosting/proxy/10-listen-on-ipv6-by-default.sh new file mode 100644 index 0000000000..e2e89388a9 --- /dev/null +++ b/hosting/proxy/10-listen-on-ipv6-by-default.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# vim:sw=4:ts=4:et + +set -e + +ME=$(basename $0) +NGINX_CONF_FILE="/etc/nginx/nginx.conf" +DEFAULT_CONF_FILE="/etc/nginx/conf.d/default.conf" + +# check if we have ipv6 available +if [ ! -f "/proc/net/if_inet6" ]; then + # ipv6 not available so delete lines from nginx conf + if [ -f "$NGINX_CONF_FILE" ]; then + sed -i '/listen \[::\]/d' $NGINX_CONF_FILE + fi + if [ -f "$DEFAULT_CONF_FILE" ]; then + sed -i '/listen \[::\]/d' $DEFAULT_CONF_FILE + fi + echo "$ME: info: ipv6 not available so delete lines from nginx conf" +else + echo "$ME: info: ipv6 is available so no need to delete lines from nginx conf" +fi + +exit 0 diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile index 298762aaf1..5fd0dc7d11 100644 --- a/hosting/proxy/Dockerfile +++ b/hosting/proxy/Dockerfile @@ -5,7 +5,7 @@ FROM nginx:latest # override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template - +COPY 10-listen-on-ipv6-by-default.sh /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh # Error handling COPY error.html /usr/share/nginx/html/error.html diff --git a/lerna.json b/lerna.json index f35cc59762..81c44cd64a 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.0.24-alpha.2", + "version": "2.0.30-alpha.5", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 579e86802e..7733a6df95 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,6 @@ "private": true, "devDependencies": { "@rollup/plugin-json": "^4.0.2", - "@types/mongodb": "3.6.3", "@typescript-eslint/parser": "4.28.0", "babel-eslint": "^10.0.3", "eslint": "^7.28.0", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 2d7cfece80..54d77821d9 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.0.24-alpha.2", + "version": "2.0.30-alpha.5", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "2.0.24-alpha.2", + "@budibase/types": "2.0.30-alpha.5", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", @@ -62,6 +62,7 @@ ] }, "devDependencies": { + "@types/chance": "1.1.3", "@types/jest": "27.5.1", "@types/koa": "2.0.52", "@types/lodash": "4.14.180", @@ -72,6 +73,7 @@ "@types/semver": "7.3.7", "@types/tar-fs": "2.0.1", "@types/uuid": "8.3.4", + "chance": "1.1.3", "ioredis-mock": "5.8.0", "jest": "27.5.1", "koa": "2.7.0", diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 83b23b479d..42cad17620 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -37,6 +37,7 @@ const core = { db, ...dbConstants, redis, + locks: redis.redlock, objectStore, utils, users, diff --git a/packages/backend-core/src/pkg/redis.ts b/packages/backend-core/src/pkg/redis.ts index 65ab186d9a..297c2b54f4 100644 --- a/packages/backend-core/src/pkg/redis.ts +++ b/packages/backend-core/src/pkg/redis.ts @@ -3,9 +3,11 @@ import Client from "../redis" import utils from "../redis/utils" import clients from "../redis/init" +import * as redlock from "../redis/redlock" export = { Client, utils, clients, + redlock, } diff --git a/packages/backend-core/src/redis/index.ts b/packages/backend-core/src/redis/index.ts index 206110366f..8a15320ff3 100644 --- a/packages/backend-core/src/redis/index.ts +++ b/packages/backend-core/src/redis/index.ts @@ -214,6 +214,34 @@ export = class RedisWrapper { } } + async bulkGet(keys: string[]) { + const db = this._db + if (keys.length === 0) { + return {} + } + const prefixedKeys = keys.map(key => addDbPrefix(db, key)) + let response = await this.getClient().mget(prefixedKeys) + if (Array.isArray(response)) { + let final: any = {} + let count = 0 + for (let result of response) { + if (result) { + let parsed + try { + parsed = JSON.parse(result) + } catch (err) { + parsed = result + } + final[keys[count]] = parsed + } + count++ + } + return final + } else { + throw new Error(`Invalid response: ${response}`) + } + } + async store(key: string, value: any, expirySeconds: number | null = null) { const db = this._db if (typeof value === "object") { diff --git a/packages/backend-core/src/redis/init.js b/packages/backend-core/src/redis/init.js index 8e5d10f838..3150ef2c1c 100644 --- a/packages/backend-core/src/redis/init.js +++ b/packages/backend-core/src/redis/init.js @@ -1,27 +1,23 @@ const Client = require("./index") const utils = require("./utils") -const { getRedlock } = require("./redlock") -let userClient, sessionClient, appClient, cacheClient, writethroughClient -let migrationsRedlock - -// turn retry off so that only one instance can ever hold the lock -const migrationsRedlockConfig = { retryCount: 0 } +let userClient, + sessionClient, + appClient, + cacheClient, + writethroughClient, + lockClient async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() sessionClient = await new Client(utils.Databases.SESSIONS).init() appClient = await new Client(utils.Databases.APP_METADATA).init() cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() + lockClient = await new Client(utils.Databases.LOCKS).init() writethroughClient = await new Client( utils.Databases.WRITE_THROUGH, utils.SelectableDatabases.WRITE_THROUGH ).init() - // pass the underlying ioredis client to redlock - migrationsRedlock = getRedlock( - cacheClient.getClient(), - migrationsRedlockConfig - ) } process.on("exit", async () => { @@ -30,6 +26,7 @@ process.on("exit", async () => { if (appClient) await appClient.finish() if (cacheClient) await cacheClient.finish() if (writethroughClient) await writethroughClient.finish() + if (lockClient) await lockClient.finish() }) module.exports = { @@ -63,10 +60,10 @@ module.exports = { } return writethroughClient }, - getMigrationsRedlock: async () => { - if (!migrationsRedlock) { + getLockClient: async () => { + if (!lockClient) { await init() } - return migrationsRedlock + return lockClient }, } diff --git a/packages/backend-core/src/redis/redlock.ts b/packages/backend-core/src/redis/redlock.ts index beef375b55..abb13b2534 100644 --- a/packages/backend-core/src/redis/redlock.ts +++ b/packages/backend-core/src/redis/redlock.ts @@ -1,14 +1,37 @@ -import Redlock from "redlock" +import Redlock, { Options } from "redlock" +import { getLockClient } from "./init" +import { LockOptions, LockType } from "@budibase/types" +import * as tenancy from "../tenancy" -export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { - return new Redlock([redisClient], { +let noRetryRedlock: Redlock | undefined + +const getClient = async (type: LockType): Promise => { + switch (type) { + case LockType.TRY_ONCE: { + if (!noRetryRedlock) { + noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE) + } + return noRetryRedlock + } + default: { + throw new Error(`Could not get redlock client: ${type}`) + } + } +} + +export const OPTIONS = { + TRY_ONCE: { + // immediately throws an error if the lock is already held + retryCount: 0, + }, + DEFAULT: { // the expected clock drift; for more details // see http://redis.io/topics/distlock driftFactor: 0.01, // multiplied by lock ttl to determine drift time // the max number of times Redlock will attempt // to lock a resource before erroring - retryCount: opts.retryCount, + retryCount: 10, // the time in ms between attempts retryDelay: 200, // time in ms @@ -16,6 +39,45 @@ export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => { // the max time in ms randomly added to retries // to improve performance under high contention // see https://www.awsarchitectureblog.com/2015/03/backoff.html - retryJitter: 200, // time in ms - }) + retryJitter: 100, // time in ms + }, +} + +export const newRedlock = async (opts: Options = {}) => { + let options = { ...OPTIONS.DEFAULT, ...opts } + const redisWrapper = await getLockClient() + const client = redisWrapper.getClient() + return new Redlock([client], options) +} + +export const doWithLock = async (opts: LockOptions, task: any) => { + const redlock = await getClient(opts.type) + let lock + try { + // aquire lock + let name: string = `${tenancy.getTenantId()}_${opts.name}` + if (opts.nameSuffix) { + name = name + `_${opts.nameSuffix}` + } + lock = await redlock.lock(name, opts.ttl) + // perform locked task + return task() + } catch (e: any) { + // lock limit exceeded + if (e.name === "LockError") { + if (opts.type === LockType.TRY_ONCE) { + // don't throw for try-once locks, they will always error + // due to retry count (0) exceeded + return + } else { + throw e + } + } else { + throw e + } + } finally { + if (lock) { + await lock.unlock() + } + } } diff --git a/packages/backend-core/src/redis/utils.js b/packages/backend-core/src/redis/utils.js index 90b3561f31..af719197b5 100644 --- a/packages/backend-core/src/redis/utils.js +++ b/packages/backend-core/src/redis/utils.js @@ -28,6 +28,7 @@ exports.Databases = { LICENSES: "license", GENERIC_CACHE: "data_cache", WRITE_THROUGH: "writeThrough", + LOCKS: "locks", } /** diff --git a/packages/backend-core/tests/utilities/structures/accounts.ts b/packages/backend-core/tests/utilities/structures/accounts.ts new file mode 100644 index 0000000000..5d23962575 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/accounts.ts @@ -0,0 +1,23 @@ +import { generator, uuid } from "." +import { AuthType, CloudAccount, Hosting } from "@budibase/types" +import * as db from "../../../src/db/utils" + +export const cloudAccount = (): CloudAccount => { + return { + accountId: uuid(), + createdAt: Date.now(), + verified: true, + verificationSent: true, + tier: "", + email: generator.email(), + tenantId: generator.word(), + hosting: Hosting.CLOUD, + authType: AuthType.PASSWORD, + password: generator.word(), + tenantName: generator.word(), + name: generator.name(), + size: "10+", + profession: "Software Engineer", + budibaseUserId: db.generateGlobalUserID(), + } +} diff --git a/packages/backend-core/tests/utilities/structures/common.ts b/packages/backend-core/tests/utilities/structures/common.ts new file mode 100644 index 0000000000..51ae220254 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/common.ts @@ -0,0 +1 @@ +export { v4 as uuid } from "uuid" diff --git a/packages/backend-core/tests/utilities/structures/index.ts b/packages/backend-core/tests/utilities/structures/index.ts index 12b6ab7ad6..68064b9715 100644 --- a/packages/backend-core/tests/utilities/structures/index.ts +++ b/packages/backend-core/tests/utilities/structures/index.ts @@ -1 +1,8 @@ +export * from "./common" + +import Chance from "chance" +export const generator = new Chance() + export * as koa from "./koa" +export * as accounts from "./accounts" +export * as licenses from "./licenses" diff --git a/packages/backend-core/tests/utilities/structures/licenses.ts b/packages/backend-core/tests/utilities/structures/licenses.ts new file mode 100644 index 0000000000..a541e91860 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/licenses.ts @@ -0,0 +1,18 @@ +import { AccountPlan, License, PlanType, Quotas } from "@budibase/types" + +const newPlan = (type: PlanType = PlanType.FREE): AccountPlan => { + return { + type, + } +} + +export const newLicense = (opts: { + quotas: Quotas + planType?: PlanType +}): License => { + return { + features: [], + quotas: opts.quotas, + plan: newPlan(opts.planType), + } +} diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 2e62aea734..6bc9b63728 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -663,6 +663,11 @@ "@types/connect" "*" "@types/node" "*" +"@types/chance@1.1.3": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@types/chance/-/chance-1.1.3.tgz#d19fe9391288d60fdccd87632bfc9ab2b4523fea" + integrity sha512-X6c6ghhe4/sQh4XzcZWSFaTAUOda38GQHmq9BUanYkOE/EO7ZrkazwKmtsj3xzTjkLWmwULE++23g3d3CCWaWw== + "@types/connect@*": version "3.4.35" resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1" @@ -1555,6 +1560,11 @@ chalk@^4.0.0, chalk@^4.1.0: ansi-styles "^4.1.0" supports-color "^7.1.0" +chance@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.3.tgz#414f08634ee479c7a316b569050ea20751b82dd3" + integrity sha512-XeJsdoVAzDb1WRPRuMBesRSiWpW1uNTo5Fd7mYxPJsAfgX71+jfuCOHOdbyBz2uAUZ8TwKcXgWk3DMedFfJkbg== + char-regex@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" diff --git a/packages/bbui/package.json b/packages/bbui/package.json index c178a8c331..d3f9a10fdf 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.0.24-alpha.2", + "version": "2.0.30-alpha.5", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "2.0.24-alpha.2", + "@budibase/string-templates": "2.0.30-alpha.5", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/cypress/integration/autoScreensUI.spec.js b/packages/builder/cypress/integration/autoScreensUI.spec.js index a22dbb0a1e..581e5c431b 100644 --- a/packages/builder/cypress/integration/autoScreensUI.spec.js +++ b/packages/builder/cypress/integration/autoScreensUI.spec.js @@ -2,7 +2,7 @@ import filterTests from "../support/filterTests" const interact = require('../support/interact') filterTests(['smoke', 'all'], () => { - context("Auto Screens UI", () => { + xcontext("Auto Screens UI", () => { before(() => { cy.login() cy.deleteAllApps() diff --git a/packages/builder/cypress/integration/datasources/postgreSql.spec.js b/packages/builder/cypress/integration/datasources/postgreSql.spec.js index 8ef574566e..e1aa0ff128 100644 --- a/packages/builder/cypress/integration/datasources/postgreSql.spec.js +++ b/packages/builder/cypress/integration/datasources/postgreSql.spec.js @@ -1,7 +1,7 @@ import filterTests from "../../support/filterTests" filterTests(["all"], () => { - context("PostgreSQL Datasource Testing", () => { + xcontext("PostgreSQL Datasource Testing", () => { if (Cypress.env("TEST_ENV")) { before(() => { cy.login() diff --git a/packages/builder/package.json b/packages/builder/package.json index 3f2219482e..8bcc8c6d3a 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.0.24-alpha.2", + "version": "2.0.30-alpha.5", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "2.0.24-alpha.2", - "@budibase/client": "2.0.24-alpha.2", - "@budibase/frontend-core": "2.0.24-alpha.2", - "@budibase/string-templates": "2.0.24-alpha.2", + "@budibase/bbui": "2.0.30-alpha.5", + "@budibase/client": "2.0.30-alpha.5", + "@budibase/frontend-core": "2.0.30-alpha.5", + "@budibase/string-templates": "2.0.30-alpha.5", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/builder/src/builderStore/store/screenTemplates/newRowScreen.js b/packages/builder/src/builderStore/store/screenTemplates/newRowScreen.js index dd97c511e5..6564bf6050 100644 --- a/packages/builder/src/builderStore/store/screenTemplates/newRowScreen.js +++ b/packages/builder/src/builderStore/store/screenTemplates/newRowScreen.js @@ -1,13 +1,8 @@ import sanitizeUrl from "./utils/sanitizeUrl" import { Screen } from "./utils/Screen" import { Component } from "./utils/Component" -import { - makeBreadcrumbContainer, - makeMainForm, - makeTitleContainer, - makeSaveButton, - makeDatasourceFormComponents, -} from "./utils/commonComponents" +import { makeBreadcrumbContainer } from "./utils/commonComponents" +import { getSchemaForDatasource } from "../../dataBinding" export default function (tables) { return tables.map(table => { @@ -23,48 +18,55 @@ export default function (tables) { export const newRowUrl = table => sanitizeUrl(`/${table.name}/new/row`) export const NEW_ROW_TEMPLATE = "NEW_ROW_TEMPLATE" -function generateTitleContainer(table, formId) { - return makeTitleContainer("New Row").addChild(makeSaveButton(table, formId)) +const rowListUrl = table => sanitizeUrl(`/${table.name}`) + +const getFields = schema => { + let columns = [] + Object.entries(schema || {}).forEach(([field, fieldSchema]) => { + if (!field || !fieldSchema) { + return + } + if (!fieldSchema?.autocolumn) { + columns.push(field) + } + }) + return columns } -const createScreen = table => { - const screen = new Screen() - .instanceName(`${table.name} - New`) - .customProps({ - hAlign: "center", - }) - .route(newRowUrl(table)) - - const form = makeMainForm() - .instanceName("Form") +const generateFormBlock = table => { + const datasource = { type: "table", tableId: table._id } + const { schema } = getSchemaForDatasource(null, datasource, { + formSchema: true, + }) + const formBlock = new Component("@budibase/standard-components/formblock") + formBlock .customProps({ + title: "New row", actionType: "Create", + actionUrl: rowListUrl(table), + showDeleteButton: false, + showSaveButton: true, + fields: getFields(schema), dataSource: { label: table.name, tableId: table._id, type: "table", }, + labelPosition: "left", size: "spectrum--medium", }) - - const fieldGroup = new Component("@budibase/standard-components/fieldgroup") - .instanceName("Field Group") - .customProps({ - labelPosition: "left", - }) - - // Add all form fields from this schema to the field group - const datasource = { type: "table", tableId: table._id } - makeDatasourceFormComponents(datasource).forEach(component => { - fieldGroup.addChild(component) - }) - - // Add all children to the form - const formId = form._json._id - form - .addChild(makeBreadcrumbContainer(table.name, "New")) - .addChild(generateTitleContainer(table, formId)) - .addChild(fieldGroup) - - return screen.addChild(form).json() + .instanceName(`${table.name} - Form block`) + return formBlock +} + +const createScreen = table => { + const formBlock = generateFormBlock(table) + const screen = new Screen() + .instanceName(`${table.name} - New`) + .route(newRowUrl(table)) + + return screen + .addChild(makeBreadcrumbContainer(table.name, "New row")) + .addChild(formBlock) + .json() } diff --git a/packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js b/packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js index a1916769c9..22b39aba3e 100644 --- a/packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js +++ b/packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js @@ -1,15 +1,8 @@ import sanitizeUrl from "./utils/sanitizeUrl" -import { rowListUrl } from "./rowListScreen" import { Screen } from "./utils/Screen" import { Component } from "./utils/Component" -import { makePropSafe } from "@budibase/string-templates" -import { - makeBreadcrumbContainer, - makeTitleContainer, - makeSaveButton, - makeMainForm, - makeDatasourceFormComponents, -} from "./utils/commonComponents" +import { makeBreadcrumbContainer } from "./utils/commonComponents" +import { getSchemaForDatasource } from "../../dataBinding" export default function (tables) { return tables.map(table => { @@ -25,125 +18,53 @@ export default function (tables) { export const ROW_DETAIL_TEMPLATE = "ROW_DETAIL_TEMPLATE" export const rowDetailUrl = table => sanitizeUrl(`/${table.name}/:id`) -function generateTitleContainer(table, title, formId, repeaterId) { - const saveButton = makeSaveButton(table, formId) - const deleteButton = new Component("@budibase/standard-components/button") - .text("Delete") - .customProps({ - type: "secondary", - quiet: true, - size: "M", - onClick: [ - { - parameters: { - tableId: table._id, - rowId: `{{ ${makePropSafe(repeaterId)}.${makePropSafe("_id")} }}`, - revId: `{{ ${makePropSafe(repeaterId)}.${makePropSafe("_rev")} }}`, - confirm: true, - }, - "##eventHandlerType": "Delete Row", - }, - { - parameters: { - url: rowListUrl(table), - }, - "##eventHandlerType": "Navigate To", - }, - ], - }) - .instanceName("Delete Button") +const rowListUrl = table => sanitizeUrl(`/${table.name}`) - const buttons = new Component("@budibase/standard-components/container") - .instanceName("Button Container") - .customProps({ - direction: "row", - hAlign: "right", - vAlign: "middle", - size: "shrink", - gap: "M", - }) - .addChild(deleteButton) - .addChild(saveButton) +const getFields = schema => { + let columns = [] + Object.entries(schema || {}).forEach(([field, fieldSchema]) => { + if (!field || !fieldSchema) { + return + } + if (!fieldSchema?.autocolumn) { + columns.push(field) + } + }) + return columns +} - return makeTitleContainer(title).addChild(buttons) +const generateFormBlock = table => { + const datasource = { type: "table", tableId: table._id } + const { schema } = getSchemaForDatasource(null, datasource, { + formSchema: true, + }) + + const formBlock = new Component("@budibase/standard-components/formblock") + formBlock + .customProps({ + title: "Edit row", + actionType: "Update", + actionUrl: rowListUrl(table), + showDeleteButton: true, + showSaveButton: true, + fields: getFields(schema), + dataSource: { + label: table.name, + tableId: table._id, + type: "table", + }, + labelPosition: "left", + size: "spectrum--medium", + }) + .instanceName(`${table.name} - Form block`) + return formBlock } const createScreen = table => { - const provider = new Component("@budibase/standard-components/dataprovider") - .instanceName(`Data Provider`) - .customProps({ - dataSource: { - label: table.name, - name: table._id, - tableId: table._id, - type: "table", - }, - filter: [ - { - field: "_id", - operator: "equal", - type: "string", - value: `{{ ${makePropSafe("url")}.${makePropSafe("id")} }}`, - valueType: "Binding", - }, - ], - limit: 1, - paginate: false, - }) - - const repeater = new Component("@budibase/standard-components/repeater") - .instanceName("Repeater") - .customProps({ - dataProvider: `{{ literal ${makePropSafe(provider._json._id)} }}`, - noRowsMessage: "We couldn't find a row to display", - }) - - const form = makeMainForm() - .instanceName("Form") - .customProps({ - actionType: "Update", - size: "spectrum--medium", - dataSource: { - label: table.name, - tableId: table._id, - type: "table", - }, - }) - - const fieldGroup = new Component("@budibase/standard-components/fieldgroup") - .instanceName("Field Group") - .customProps({ - labelPosition: "left", - }) - - // Add all form fields from this schema to the field group - const datasource = { type: "table", tableId: table._id } - makeDatasourceFormComponents(datasource).forEach(component => { - fieldGroup.addChild(component) - }) - - // Add all children to the form - const formId = form._json._id - const repeaterId = repeater._json._id - const heading = table.primaryDisplay - ? `{{ ${makePropSafe(repeaterId)}.${makePropSafe(table.primaryDisplay)} }}` - : null - form - .addChild(makeBreadcrumbContainer(table.name, heading || "Edit")) - .addChild( - generateTitleContainer(table, heading || "Edit Row", formId, repeaterId) - ) - .addChild(fieldGroup) - - repeater.addChild(form) - provider.addChild(repeater) - return new Screen() .instanceName(`${table.name} - Detail`) .route(rowDetailUrl(table)) - .customProps({ - hAlign: "center", - }) - .addChild(provider) + .addChild(makeBreadcrumbContainer(table.name, "Edit row")) + .addChild(generateFormBlock(table)) .json() } diff --git a/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js b/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js index 39e88ae69e..b04d588ded 100644 --- a/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js +++ b/packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js @@ -2,7 +2,6 @@ import sanitizeUrl from "./utils/sanitizeUrl" import { newRowUrl } from "./newRowScreen" import { Screen } from "./utils/Screen" import { Component } from "./utils/Component" -import { makePropSafe } from "@budibase/string-templates" export default function (tables) { return tables.map(table => { @@ -18,48 +17,17 @@ export default function (tables) { export const ROW_LIST_TEMPLATE = "ROW_LIST_TEMPLATE" export const rowListUrl = table => sanitizeUrl(`/${table.name}`) -function generateTitleContainer(table) { - const newButton = new Component("@budibase/standard-components/button") - .text("Create New") - .customProps({ - size: "M", - type: "primary", - onClick: [ - { - parameters: { - url: newRowUrl(table), - }, - "##eventHandlerType": "Navigate To", - }, - ], - }) - .instanceName("New Button") - - const heading = new Component("@budibase/standard-components/heading") - .instanceName("Title") - .text(table.name) - .customProps({ - size: "M", - align: "left", - }) - - return new Component("@budibase/standard-components/container") - .customProps({ - direction: "row", - hAlign: "stretch", - vAlign: "middle", - size: "shrink", - gap: "M", - }) - .instanceName("Title Container") - .addChild(heading) - .addChild(newButton) -} - -const createScreen = table => { - const provider = new Component("@budibase/standard-components/dataprovider") - .instanceName(`Data Provider`) +const generateTableBlock = table => { + const tableBlock = new Component("@budibase/standard-components/tableblock") + tableBlock .customProps({ + linkRows: true, + linkURL: `${rowListUrl(table)}/:id`, + showAutoColumns: false, + showTitleButton: true, + titleButtonText: "Create new", + titleButtonURL: newRowUrl(table), + title: table.name, dataSource: { label: table.name, name: table._id, @@ -68,41 +36,16 @@ const createScreen = table => { }, size: "spectrum--medium", paginate: true, - limit: 8, - }) - - const spectrumTable = new Component("@budibase/standard-components/table") - .customProps({ - dataProvider: `{{ literal ${makePropSafe(provider._json._id)} }}`, - showAutoColumns: false, - quiet: false, rowCount: 8, }) - .instanceName(`${table.name} Table`) - - const safeTableId = makePropSafe(spectrumTable._json._id) - const safeRowId = makePropSafe("_id") - const viewLink = new Component("@budibase/standard-components/link") - .customProps({ - text: "View", - url: `${rowListUrl(table)}/{{ ${safeTableId}.${safeRowId} }}`, - size: "S", - color: "var(--spectrum-global-color-gray-600)", - align: "left", - }) - .normalStyle({ - ["margin-left"]: "16px", - ["margin-right"]: "16px", - }) - .instanceName("View Link") - - spectrumTable.addChild(viewLink) - provider.addChild(spectrumTable) + .instanceName(`${table.name} - Table block`) + return tableBlock +} +const createScreen = table => { return new Screen() .route(rowListUrl(table)) .instanceName(`${table.name} - List`) - .addChild(generateTitleContainer(table)) - .addChild(provider) + .addChild(generateTableBlock(table)) .json() } diff --git a/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js b/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js index 9176d535ab..f00cd9c215 100644 --- a/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js +++ b/packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js @@ -65,6 +65,11 @@ export function makeBreadcrumbContainer(tableName, text) { vAlign: "middle", size: "shrink", }) + .normalStyle({ + width: "600px", + "margin-right": "auto", + "margin-left": "auto", + }) .instanceName("Breadcrumbs") .addChild(link) .addChild(arrowText) @@ -138,6 +143,7 @@ const fieldTypeToComponentMap = { attachment: "attachmentfield", link: "relationshipfield", json: "jsonfield", + barcodeqr: "codescanner", } export function makeDatasourceFormComponents(datasource) { diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 3fd38bddeb..b7249ad60c 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -261,6 +261,7 @@ } else { return [ FIELDS.STRING, + FIELDS.BARCODEQR, FIELDS.LONGFORM, FIELDS.OPTIONS, FIELDS.DATETIME, diff --git a/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte b/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte index e2ccab11af..600e331d3e 100644 --- a/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte @@ -17,12 +17,21 @@ $: selectedRoleId = selectedRole._id $: otherRoles = editableRoles.filter(role => role._id !== selectedRoleId) $: isCreating = selectedRoleId == null || selectedRoleId === "" + + $: hasUniqueRoleName = !otherRoles + ?.map(role => role.name) + ?.includes(selectedRole.name) + $: valid = selectedRole.name && selectedRole.inherits && selectedRole.permissionId && !builtInRoles.includes(selectedRole.name) + $: shouldDisableRoleInput = + builtInRoles.includes(selectedRole.name) && + selectedRole.name?.toLowerCase() === selectedRoleId?.toLowerCase() + const fetchBasePermissions = async () => { try { basePermissions = await API.getBasePermissions() @@ -99,7 +108,7 @@ title="Edit Roles" confirmText={isCreating ? "Create" : "Save"} onConfirm={saveRole} - disabled={!valid} + disabled={!valid || !hasUniqueRoleName} > {#if errors.length} @@ -119,15 +128,16 @@ x._id} getOptionLabel={x => x.name} - disabled={builtInRoles.includes(selectedRole.name)} + disabled={shouldDisableRoleInput} /> {/if}
- {#if !isCreating} + {#if !isCreating && !builtInRoles.includes(selectedRole.name)} {/if}
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte index cef49d81a1..ff413094a0 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte @@ -209,27 +209,29 @@ {:else} No tables found. {/if} - -
- Relationships - -
- - Tell budibase how your tables are related to get even more smart features. - -{#if relationshipInfo && relationshipInfo.length > 0} - openRelationshipModal(detail.from, detail.to)} - schema={relationshipSchema} - data={relationshipInfo} - allowEditColumns={false} - allowEditRows={false} - allowSelectRows={false} - /> -{:else} - No relationships configured. +{#if integration.relationships !== false} + +
+ Relationships + +
+ + Tell budibase how your tables are related to get even more smart features. + + {#if relationshipInfo && relationshipInfo.length > 0} +
openRelationshipModal(detail.from, detail.to)} + schema={relationshipSchema} + data={relationshipInfo} + allowEditColumns={false} + allowEditRows={false} + allowSelectRows={false} + /> + {:else} + No relationships configured. + {/if} {/if} diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index 28ce35d9f7..427e8d80a1 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -8,6 +8,15 @@ export const FIELDS = { presence: false, }, }, + BARCODEQR: { + name: "Barcode/QR", + type: "barcodeqr", + constraints: { + type: "string", + length: {}, + presence: false, + }, + }, LONGFORM: { name: "Long Form Text", type: "longform", @@ -148,6 +157,7 @@ export const ALLOWABLE_STRING_OPTIONS = [ FIELDS.STRING, FIELDS.OPTIONS, FIELDS.LONGFORM, + FIELDS.BARCODEQR, ] export const ALLOWABLE_STRING_TYPES = ALLOWABLE_STRING_OPTIONS.map( opt => opt.type diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json index 088f0c0989..3d9cf7e2a3 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json @@ -66,7 +66,8 @@ "relationshipfield", "datetimefield", "multifieldselect", - "s3upload" + "s3upload", + "codescanner" ] }, { diff --git a/packages/builder/src/pages/builder/portal/overview/_components/AccessTab.svelte b/packages/builder/src/pages/builder/portal/overview/_components/AccessTab.svelte index a4046513fc..71a86f2fca 100644 --- a/packages/builder/src/pages/builder/portal/overview/_components/AccessTab.svelte +++ b/packages/builder/src/pages/builder/portal/overview/_components/AccessTab.svelte @@ -156,8 +156,8 @@ page={$usersFetch.pageNumber + 1} hasPrevPage={$usersFetch.hasPrevPage} hasNextPage={$usersFetch.hasNextPage} - goToPrevPage={$usersFetch.loading ? null : fetch.prevPage} - goToNextPage={$usersFetch.loading ? null : fetch.nextPage} + goToPrevPage={$usersFetch.loading ? null : usersFetch.prevPage} + goToNextPage={$usersFetch.loading ? null : usersFetch.nextPage} /> diff --git a/packages/builder/src/pages/builder/portal/settings/upgrade.svelte b/packages/builder/src/pages/builder/portal/settings/upgrade.svelte index 16bbc0b03d..5d10f22396 100644 --- a/packages/builder/src/pages/builder/portal/settings/upgrade.svelte +++ b/packages/builder/src/pages/builder/portal/settings/upgrade.svelte @@ -91,7 +91,6 @@ }) -{"is adming" + $auth.isAdmin} {#if $auth.isAdmin} @@ -129,6 +130,8 @@ props={{ dataProvider, noRowsMessage: "We couldn't find a row to display", + direction: "column", + hAlign: "center", }} > diff --git a/packages/client/src/components/app/forms/CodeScanner.svelte b/packages/client/src/components/app/forms/CodeScanner.svelte new file mode 100644 index 0000000000..5dff3a96fa --- /dev/null +++ b/packages/client/src/components/app/forms/CodeScanner.svelte @@ -0,0 +1,234 @@ + + +
+ {#if value && !manualMode} +
+ + {value} +
+ {/if} + + {#if allowManualEntry && manualMode} +
+ { + dispatch("change", value) + }} + /> +
+ {/if} + + {#if value} + { + dispatch("change", "") + }} + {disabled} + > + Clear + + {:else} + { + showReaderModal() + }} + {disabled} + > + {scanButtonText} + + {/if} +
+ +
+ + +
+
+ + {#if cameraEnabled === false} +
Your camera is disabled.
+ {/if} +
+
+ {#if cameraEnabled === true} +
+ {#if value} +
+ + {value} +
+ {:else} +
+ + Searching for code... +
+ {/if} +
+ {/if} + +
+
+ {#if allowManualEntry && !manualMode} + + {/if} + + +
+
+ + +
+ + diff --git a/packages/client/src/components/app/forms/CodeScannerField.svelte b/packages/client/src/components/app/forms/CodeScannerField.svelte new file mode 100644 index 0000000000..7e020aa9c7 --- /dev/null +++ b/packages/client/src/components/app/forms/CodeScannerField.svelte @@ -0,0 +1,47 @@ + + + + {#if fieldState} + + {/if} + diff --git a/packages/client/src/components/app/forms/index.js b/packages/client/src/components/app/forms/index.js index 0ff82cea94..44c1516885 100644 --- a/packages/client/src/components/app/forms/index.js +++ b/packages/client/src/components/app/forms/index.js @@ -13,3 +13,4 @@ export { default as passwordfield } from "./PasswordField.svelte" export { default as formstep } from "./FormStep.svelte" export { default as jsonfield } from "./JSONField.svelte" export { default as s3upload } from "./S3Upload.svelte" +export { default as codescanner } from "./CodeScannerField.svelte" diff --git a/packages/client/src/constants.js b/packages/client/src/constants.js index 23d44a34f5..bd387c7f9d 100644 --- a/packages/client/src/constants.js +++ b/packages/client/src/constants.js @@ -1,5 +1,6 @@ export const FieldTypes = { STRING: "string", + BARCODEQR: "barcodeqr", LONGFORM: "longform", OPTIONS: "options", NUMBER: "number", diff --git a/packages/client/src/stores/builder.js b/packages/client/src/stores/builder.js index 0404183c91..0bb5693edb 100644 --- a/packages/client/src/stores/builder.js +++ b/packages/client/src/stores/builder.js @@ -50,6 +50,9 @@ const createBuilderStore = () => { duplicateComponent: id => { dispatchEvent("duplicate-component", { id }) }, + deleteComponent: id => { + dispatchEvent("delete-component", { id }) + }, notifyLoaded: () => { dispatchEvent("preview-loaded") }, diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 4c22e3f219..77a8f3598f 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.0.24-alpha.2", + "version": "2.0.30-alpha.5", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "2.0.24-alpha.2", + "@budibase/bbui": "2.0.30-alpha.5", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index f7f7ab6d6a..eae290ecd9 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.0.24-alpha.2", + "version": "2.0.30-alpha.5", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/__mocks__/mongodb.ts b/packages/server/__mocks__/mongodb.ts index 4a1867f6f9..01b6e76fc4 100644 --- a/packages/server/__mocks__/mongodb.ts +++ b/packages/server/__mocks__/mongodb.ts @@ -33,7 +33,7 @@ module MongoMock { }) } - mongodb.ObjectID = jest.requireActual("mongodb").ObjectID + mongodb.ObjectId = jest.requireActual("mongodb").ObjectId module.exports = mongodb } diff --git a/packages/server/package.json b/packages/server/package.json index ec34d9024b..c46a68fba8 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.0.24-alpha.2", + "version": "2.0.30-alpha.5", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "2.0.24-alpha.2", - "@budibase/client": "2.0.24-alpha.2", - "@budibase/pro": "2.0.24-alpha.2", - "@budibase/string-templates": "2.0.24-alpha.2", - "@budibase/types": "2.0.24-alpha.2", + "@budibase/backend-core": "2.0.30-alpha.5", + "@budibase/client": "2.0.30-alpha.5", + "@budibase/pro": "2.0.30-alpha.5", + "@budibase/string-templates": "2.0.30-alpha.5", + "@budibase/types": "2.0.30-alpha.5", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", @@ -123,7 +123,7 @@ "koa2-ratelimit": "1.1.1", "lodash": "4.17.21", "memorystream": "0.3.1", - "mongodb": "3.6.3", + "mongodb": "4.9", "mssql": "6.2.3", "mysql2": "2.3.3", "node-fetch": "2.6.7", @@ -166,7 +166,6 @@ "@types/koa": "2.13.4", "@types/koa__router": "8.0.0", "@types/lodash": "4.14.180", - "@types/mongodb": "3.6.3", "@types/node": "14.18.20", "@types/node-fetch": "2.6.1", "@types/oracledb": "5.2.2", diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 9dde91b348..a7caf85e94 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -32,7 +32,7 @@ const { import { USERS_TABLE_SCHEMA } from "../../constants" import { removeAppFromUserRoles } from "../../utilities/workerRequests" import { clientLibraryPath, stringToReadStream } from "../../utilities" -import { getAllLocks } from "../../utilities/redis" +import { getLocksById } from "../../utilities/redis" import { updateClientLibrary, backupClientLibrary, @@ -45,11 +45,10 @@ import { cleanupAutomations } from "../../automations/utils" import { context } from "@budibase/backend-core" import { checkAppMetadata } from "../../automations/logging" import { getUniqueRows } from "../../utilities/usageQuota/rows" -import { quotas } from "@budibase/pro" +import { quotas, groups } from "@budibase/pro" import { errors, events, migrations } from "@budibase/backend-core" import { App, Layout, Screen, MigrationType } from "@budibase/types" import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts" -import { groups } from "@budibase/pro" import { enrichPluginURLs } from "../../utilities/plugins" const URL_REGEX_SLASH = /\/|\\/g @@ -172,16 +171,16 @@ export const fetch = async (ctx: any) => { const all = ctx.query && ctx.query.status === AppStatus.ALL const apps = await getAllApps({ dev, all }) + const appIds = apps + .filter((app: any) => app.status === "development") + .map((app: any) => app.appId) // get the locks for all the dev apps if (dev || all) { - const locks = await getAllLocks() + const locks = await getLocksById(appIds) for (let app of apps) { - if (app.status !== "development") { - continue - } - const lock = locks.find((lock: any) => lock.appId === app.appId) + const lock = locks[app.appId] if (lock) { - app.lockedBy = lock.user + app.lockedBy = lock } else { // make sure its definitely not present delete app.lockedBy diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 5edf862706..a51e7ad6ec 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -17,7 +17,6 @@ import { getProdAppDB, getDevAppDB, } from "@budibase/backend-core/context" -import { quotas } from "@budibase/pro" import { events } from "@budibase/backend-core" // the max time we can wait for an invalidation to complete before considering it failed diff --git a/packages/server/src/api/controllers/dev.js b/packages/server/src/api/controllers/dev.js index 8438175ca8..c8f134756b 100644 --- a/packages/server/src/api/controllers/dev.js +++ b/packages/server/src/api/controllers/dev.js @@ -103,7 +103,7 @@ exports.revert = async ctx => { target: appId, }) try { - if (!env.isTest()) { + if (env.COUCH_DB_URL) { // in-memory db stalls on rollback await replication.rollback() } diff --git a/packages/server/src/api/controllers/integration.js b/packages/server/src/api/controllers/integration.js index 3d1643601b..2f11ec19ed 100644 --- a/packages/server/src/api/controllers/integration.js +++ b/packages/server/src/api/controllers/integration.js @@ -1,17 +1,9 @@ const { getDefinitions } = require("../../integrations") -const { SourceName } = require("@budibase/types") -const googlesheets = require("../../integrations/googlesheets") -const { featureFlags } = require("@budibase/backend-core") exports.fetch = async function (ctx) { ctx.status = 200 const defs = await getDefinitions() - // for google sheets integration google verification - if (featureFlags.isEnabled(featureFlags.TenantFeatureFlag.GOOGLE_SHEETS)) { - defs[SourceName.GOOGLE_SHEETS] = googlesheets.schema - } - ctx.body = defs } diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 5c09a2f3b6..c92f942986 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -56,6 +56,7 @@ const _import = async (ctx: any) => { config: { url: info.url, defaultHeaders: [], + rejectUnauthorized: true, }, name: info.name, } diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 22c9b6dc55..7e55c71aea 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -18,6 +18,7 @@ import { Table } from "@budibase/types" import { quotas } from "@budibase/pro" import { isEqual } from "lodash" import { cloneDeep } from "lodash/fp" +import env from "../../../environment" function checkAutoColumns(table: Table, oldTable: Table) { if (!table.schema) { @@ -167,7 +168,7 @@ export async function destroy(ctx: any) { await db.remove(tableToDelete) // remove table search index - if (!isTest()) { + if (!isTest() || env.COUCH_DB_URL) { const currentIndexes = await db.getIndexes() const existingIndex = currentIndexes.indexes.find( (existing: any) => existing.name === `search:${ctx.params.tableId}` diff --git a/packages/server/src/api/routes/tests/application.spec.js b/packages/server/src/api/routes/tests/application.spec.js index dcfc2c6d9b..f62665d184 100644 --- a/packages/server/src/api/routes/tests/application.spec.js +++ b/packages/server/src/api/routes/tests/application.spec.js @@ -1,7 +1,7 @@ jest.mock("../../../utilities/redis", () => ({ init: jest.fn(), - getAllLocks: () => { - return [] + getLocksById: () => { + return {} }, doesUserHaveLock: () => { return true diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index c002c10f7b..2a92e87ff8 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -31,6 +31,7 @@ exports.NoEmptyFilterStrings = [ exports.FieldTypes = { STRING: "string", + BARCODEQR: "barcodeqr", LONGFORM: "longform", OPTIONS: "options", NUMBER: "number", @@ -51,6 +52,7 @@ exports.CanSwitchTypes = [ exports.FieldTypes.STRING, exports.FieldTypes.OPTIONS, exports.FieldTypes.LONGFORM, + exports.FieldTypes.BARCODEQR, ], [exports.FieldTypes.BOOLEAN, exports.FieldTypes.NUMBER], ] diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index 6e19d9d342..1ef5206fff 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -31,6 +31,7 @@ export interface BearerAuthConfig { export interface RestConfig { url: string + rejectUnauthorized: boolean defaultHeaders: { [key: string]: any } diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index d50837b273..fac8403c49 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -8,6 +8,7 @@ function runServer() { checkDevelopmentEnvironment() fixPath() // this will setup http and https proxies form env variables + process.env.GLOBAL_AGENT_FORCE_GLOBAL_AGENT = "false" bootstrap() require("./app") } diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts index af561e81c5..02f7ad8718 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/server/src/integrations/base/sqlTable.ts @@ -40,6 +40,7 @@ function generateSchema( case FieldTypes.STRING: case FieldTypes.OPTIONS: case FieldTypes.LONGFORM: + case FieldTypes.BARCODEQR: schema.text(key) break case FieldTypes.NUMBER: diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 240f68ed91..c926aeb992 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -33,6 +33,7 @@ const DEFINITIONS: { [key: string]: Integration } = { [SourceName.ARANGODB]: arangodb.schema, [SourceName.REST]: rest.schema, [SourceName.FIRESTORE]: firebase.schema, + [SourceName.GOOGLE_SHEETS]: googlesheets.schema, [SourceName.REDIS]: redis.schema, [SourceName.SNOWFLAKE]: snowflake.schema, } @@ -66,10 +67,6 @@ if ( INTEGRATIONS[SourceName.ORACLE] = oracle.integration } -if (environment.SELF_HOSTED) { - DEFINITIONS[SourceName.GOOGLE_SHEETS] = googlesheets.schema -} - module.exports = { getDefinitions: async () => { const pluginSchemas: { [key: string]: Integration } = {} diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 9f17028648..d7709a08c7 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -6,13 +6,12 @@ import { } from "@budibase/types" import { MongoClient, - ObjectID, - FilterQuery, - UpdateQuery, - FindOneAndUpdateOption, - UpdateOneOptions, - UpdateManyOptions, - CommonOptions, + ObjectId, + Filter, + UpdateFilter, + FindOneAndUpdateOptions, + UpdateOptions, + OperationOptions, } from "mongodb" interface MongoDBConfig { @@ -57,6 +56,232 @@ const SCHEMA: Integration = { delete: { type: QueryType.JSON, }, + aggregate: { + type: QueryType.JSON, + steps: [ + { + key: "$addFields", + template: "{\n\t\n}", + }, + { + key: "$bucket", + template: `{ + "groupBy": "", + "boundaries": [], + "default": "", + "output": {} +}`, + }, + { + key: "$bucketAuto", + template: `{ + "groupBy": "", + "buckets": 1, + "output": {}, + "granularity": "R5" +}`, + }, + { + key: "$changeStream", + template: `{ + "allChangesForCluster": true, + "fullDocument": "", + "fullDocumentBeforeChange": "", + "resumeAfter": 1, + "showExpandedEvents": true, + "startAfter": {}, + "startAtOperationTime": "" +}`, + }, + { + key: "$collStats", + template: `{ + "latencyStats": { "histograms": true } }, + "storageStats": { "scale": 1 } }, + "count": {}, + "queryExecStats": {} +}`, + }, + { + key: "$count", + template: ``, + }, + { + key: "$densify", + template: `{ + "field": "", + "partitionByFields": [], + "range": { + "step": 1, + "unit": 1, + "bounds": "full" + } +}`, + }, + { + key: "$documents", + template: `[]`, + }, + { + key: "$facet", + template: `{\n\t\n}`, + }, + { + key: "$fill", + template: `{ + "partitionBy": "", + "partitionByFields": [], + "sortBy": {}, + "output": {} +}`, + }, + { + key: "$geoNear", + template: `{ + "near": { + "type": "Point", + "coordinates": [ + -73.98142, 40.71782 + ] + }, + "key": "location", + "distanceField": "dist.calculated", + "query": { "category": "Parks" } +}`, + }, + { + key: "$graphLookup", + template: `{ + "from": "", + "startWith": "", + "connectFromField": "", + "connectToField": "", + "as": "", + "maxDepth": 1, + "depthField": "", + "restrictSearchWithMatch": {} +}`, + }, + { + key: "$group", + template: `{ + "_id": "" +}`, + }, + { + key: "$indexStats", + template: "{\n\t\n}", + }, + { + key: "$limit", + template: `1`, + }, + { + key: "$listLocalSessions", + template: `{\n\t\n}`, + }, + { + key: "$listSessions", + template: `{\n\t\n}`, + }, + { + key: "$lookup", + template: `{ + "from": "", + "localField": "", + "foreignField": "", + "as": "" +}`, + }, + { + key: "$match", + template: "{\n\t\n}", + }, + { + key: "$merge", + template: `{ + "into": {}, + "on": "_id", + "whenMatched": "replace", + "whenNotMatched": "insert" +}`, + }, + { + key: "$out", + template: `{ + "db": "", + "coll": "" +}`, + }, + { + key: "$planCacheStats", + template: "{\n\t\n}", + }, + { + key: "$project", + template: "{\n\t\n}", + }, + { + key: "$redact", + template: "", + }, + { + key: "$replaceRoot", + template: `{ "newRoot": "" }`, + }, + { + key: "$replaceWith", + template: ``, + }, + { + key: "$sample", + template: `{ "size": 3 }`, + }, + { + key: "$set", + template: "{\n\t\n}", + }, + { + key: "$setWindowFields", + template: `{ + "partitionBy": "", + "sortBy": {}, + "output": {} +}`, + }, + { + key: "$skip", + template: `1`, + }, + { + key: "$sort", + template: "{\n\t\n}", + }, + { + key: "$sortByCount", + template: "", + }, + { + key: "$unionWith", + template: `{ + "coll": "", + "pipeline": [] +}`, + }, + { + key: "$unset", + template: "", + }, + { + key: "$unwind", + template: `{ + "path": "", + "includeArrayIndex": "", + "preserveNullAndEmptyArrays": true +}`, + }, + ], + }, }, extra: { collection: { @@ -64,8 +289,8 @@ const SCHEMA: Integration = { type: DatasourceFieldType.STRING, required: true, }, - actionTypes: { - displayName: "Action Types", + actionType: { + displayName: "Query Type", type: DatasourceFieldType.LIST, required: true, data: { @@ -73,6 +298,7 @@ const SCHEMA: Integration = { create: ["insertOne", "insertMany"], update: ["updateOne", "updateMany"], delete: ["deleteOne", "deleteMany"], + aggregate: ["json", "pipeline"], }, }, }, @@ -104,7 +330,7 @@ class MongoIntegration implements IntegrationBase { ) { const id = json[field].match(/(?<=objectid\(['"]).*(?=['"]\))/gi)?.[0] if (id) { - json[field] = ObjectID.createFromHexString(id) + json[field] = ObjectId.createFromHexString(id) } } } @@ -168,7 +394,7 @@ class MongoIntegration implements IntegrationBase { // For mongodb we add an extra actionType to specify // which method we want to call on the collection - switch (query.extra.actionTypes) { + switch (query.extra.actionType) { case "insertOne": { return await collection.insertOne(json) } @@ -177,7 +403,7 @@ class MongoIntegration implements IntegrationBase { } default: { throw new Error( - `actionType ${query.extra.actionTypes} does not exist on DB for create` + `actionType ${query.extra.actionType} does not exist on DB for create` ) } } @@ -196,7 +422,7 @@ class MongoIntegration implements IntegrationBase { const collection = db.collection(query.extra.collection) let json = this.createObjectIds(query.json) - switch (query.extra.actionTypes) { + switch (query.extra.actionType) { case "find": { return await collection.find(json).toArray() } @@ -208,9 +434,9 @@ class MongoIntegration implements IntegrationBase { json = this.parseQueryParams(query.json, "update") } let findAndUpdateJson = this.createObjectIds(json) as { - filter: FilterQuery - update: UpdateQuery - options: FindOneAndUpdateOption + filter: Filter + update: UpdateFilter + options: FindOneAndUpdateOptions } return await collection.findOneAndUpdate( findAndUpdateJson.filter, @@ -226,7 +452,7 @@ class MongoIntegration implements IntegrationBase { } default: { throw new Error( - `actionType ${query.extra.actionTypes} does not exist on DB for read` + `actionType ${query.extra.actionType} does not exist on DB for read` ) } } @@ -248,29 +474,29 @@ class MongoIntegration implements IntegrationBase { queryJson = this.parseQueryParams(queryJson, "update") } let json = this.createObjectIds(queryJson) as { - filter: FilterQuery - update: UpdateQuery + filter: Filter + update: UpdateFilter options: object } - switch (query.extra.actionTypes) { + switch (query.extra.actionType) { case "updateOne": { return await collection.updateOne( json.filter, json.update, - json.options as UpdateOneOptions + json.options as UpdateOptions ) } case "updateMany": { return await collection.updateMany( json.filter, json.update, - json.options as UpdateManyOptions + json.options as UpdateOptions ) } default: { throw new Error( - `actionType ${query.extra.actionTypes} does not exist on DB for update` + `actionType ${query.extra.actionType} does not exist on DB for update` ) } } @@ -292,8 +518,8 @@ class MongoIntegration implements IntegrationBase { queryJson = this.parseQueryParams(queryJson, "delete") } let json = this.createObjectIds(queryJson) as { - filter: FilterQuery - options: CommonOptions + filter: Filter + options: OperationOptions } if (!json.options) { json = { @@ -302,7 +528,7 @@ class MongoIntegration implements IntegrationBase { } } - switch (query.extra.actionTypes) { + switch (query.extra.actionType) { case "deleteOne": { return await collection.deleteOne(json.filter, json.options) } @@ -311,7 +537,7 @@ class MongoIntegration implements IntegrationBase { } default: { throw new Error( - `actionType ${query.extra.actionTypes} does not exist on DB for delete` + `actionType ${query.extra.actionType} does not exist on DB for delete` ) } } @@ -322,6 +548,43 @@ class MongoIntegration implements IntegrationBase { await this.client.close() } } + + async aggregate(query: { + json: object + steps: any[] + extra: { [key: string]: string } + }) { + try { + await this.connect() + const db = this.client.db(this.config.db) + const collection = db.collection(query.extra.collection) + let response = [] + if (query.extra?.actionType === "pipeline") { + for await (const doc of collection.aggregate( + query.steps.map(({ key, value }) => { + let temp: any = {} + temp[key] = JSON.parse(value.value) + return this.createObjectIds(temp) + }) + )) { + response.push(doc) + } + } else { + const stages: Array = query.json as Array + for await (const doc of collection.aggregate( + stages ? this.createObjectIds(stages) : [] + )) { + response.push(doc) + } + } + return response + } catch (err) { + console.error("Error writing to mongodb", err) + throw err + } finally { + await this.client.close() + } + } } export default { diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index f0f254c0ea..08d2337593 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -14,6 +14,7 @@ import { BearerAuthConfig, } from "../definitions/datasource" import { get } from "lodash" +import * as https from "https" import qs from "querystring" const fetch = require("node-fetch") const { formatBytes } = require("../utilities") @@ -76,11 +77,11 @@ const SCHEMA: Integration = { required: false, default: {}, }, - legacyHttpParser: { - display: "Legacy HTTP Support", + rejectUnauthorized: { + display: "Reject Unauthorized", type: DatasourceFieldType.BOOLEAN, + default: true, required: false, - default: false, }, }, query: { @@ -218,8 +219,12 @@ class RestIntegration implements IntegrationBase { } } - // make sure the query string is fully encoded - const main = `${path}?${qs.encode(qs.decode(queryString))}` + if (queryString) { + // make sure the query string is fully encoded + queryString = "?" + qs.encode(qs.decode(queryString)) + } + const main = `${path}${queryString}` + let complete = main if (this.config.url && !main.startsWith("http")) { complete = !this.config.url ? main : `${this.config.url}/${main}` @@ -381,6 +386,13 @@ class RestIntegration implements IntegrationBase { paginationValues ) + if (this.config.rejectUnauthorized == false) { + input.agent = new https.Agent({ + rejectUnauthorized: false, + }) + } + + // Deprecated by rejectUnauthorized if (this.config.legacyHttpParser) { // https://github.com/nodejs/node/issues/43798 input.extraHttpOptions = { insecureHTTPParser: true } diff --git a/packages/server/src/integrations/tests/mongo.spec.ts b/packages/server/src/integrations/tests/mongo.spec.ts index a326d3ac2a..b941d7ae66 100644 --- a/packages/server/src/integrations/tests/mongo.spec.ts +++ b/packages/server/src/integrations/tests/mongo.spec.ts @@ -34,7 +34,7 @@ describe("MongoDB Integration", () => { await config.integration.create({ index: indexName, json: body, - extra: { collection: "testCollection", actionTypes: "insertOne" }, + extra: { collection: "testCollection", actionType: "insertOne" }, }) expect(config.integration.client.insertOne).toHaveBeenCalledWith(body) }) @@ -44,7 +44,7 @@ describe("MongoDB Integration", () => { json: { address: "test", }, - extra: { collection: "testCollection", actionTypes: "find" }, + extra: { collection: "testCollection", actionType: "find" }, } const response = await config.integration.read(query) expect(config.integration.client.find).toHaveBeenCalledWith(query.json) @@ -61,7 +61,7 @@ describe("MongoDB Integration", () => { opt: "option", }, }, - extra: { collection: "testCollection", actionTypes: "deleteOne" }, + extra: { collection: "testCollection", actionType: "deleteOne" }, } await config.integration.delete(query) expect(config.integration.client.deleteOne).toHaveBeenCalledWith( @@ -83,7 +83,7 @@ describe("MongoDB Integration", () => { upsert: false, }, }, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalledWith( @@ -97,7 +97,7 @@ describe("MongoDB Integration", () => { const restore = disableConsole() const query = { - extra: { collection: "testCollection", actionTypes: "deleteOne" }, + extra: { collection: "testCollection", actionType: "deleteOne" }, } let error = null @@ -125,19 +125,19 @@ describe("MongoDB Integration", () => { upsert: false, }, }, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalled() const args = config.integration.client.updateOne.mock.calls[0] expect(args[0]).toEqual({ - _id: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), - name: mongo.ObjectID.createFromHexString("BBBB12345678ABCD12345678"), + _id: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), + name: mongo.ObjectId.createFromHexString("BBBB12345678ABCD12345678"), }) expect(args[1]).toEqual({ - _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"), - name: mongo.ObjectID.createFromHexString("CCCC12345678ABCD12345678"), + _id: mongo.ObjectId.createFromHexString("FFFF12345678ABCD12345678"), + name: mongo.ObjectId.createFromHexString("CCCC12345678ABCD12345678"), }) expect(args[2]).toEqual({ upsert: false, @@ -161,7 +161,7 @@ describe("MongoDB Integration", () => { upsert: true, }, }, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalled() @@ -169,12 +169,12 @@ describe("MongoDB Integration", () => { const args = config.integration.client.updateOne.mock.calls[0] expect(args[0]).toEqual({ _id: { - $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), + $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), }, }) expect(args[1]).toEqual({ $set: { - _id: mongo.ObjectID.createFromHexString("FFFF12345678ABCD12345678"), + _id: mongo.ObjectId.createFromHexString("FFFF12345678ABCD12345678"), }, }) expect(args[2]).toEqual({ @@ -200,7 +200,7 @@ describe("MongoDB Integration", () => { upsert: false, }, }, - extra: { collection: "testCollection", actionTypes: "findOneAndUpdate" }, + extra: { collection: "testCollection", actionType: "findOneAndUpdate" }, } await config.integration.read(query) expect(config.integration.client.findOneAndUpdate).toHaveBeenCalled() @@ -208,7 +208,7 @@ describe("MongoDB Integration", () => { const args = config.integration.client.findOneAndUpdate.mock.calls[0] expect(args[0]).toEqual({ _id: { - $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), + $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), }, }) expect(args[1]).toEqual({ @@ -245,7 +245,7 @@ describe("MongoDB Integration", () => { { "upsert": true }`, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalled() @@ -253,7 +253,7 @@ describe("MongoDB Integration", () => { const args = config.integration.client.updateOne.mock.calls[0] expect(args[0]).toEqual({ _id: { - $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), + $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), }, }) expect(args[1]).toEqual({ @@ -300,7 +300,7 @@ describe("MongoDB Integration", () => { "upsert": true, "extra": "ad\\"{\\"d" }`, - extra: { collection: "testCollection", actionTypes: "updateOne" }, + extra: { collection: "testCollection", actionType: "updateOne" }, } await config.integration.update(query) expect(config.integration.client.updateOne).toHaveBeenCalled() @@ -308,7 +308,7 @@ describe("MongoDB Integration", () => { const args = config.integration.client.updateOne.mock.calls[0] expect(args[0]).toEqual({ _id: { - $eq: mongo.ObjectID.createFromHexString("ACBD12345678ABCD12345678"), + $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), }, }) expect(args[1]).toEqual({ diff --git a/packages/server/src/integrations/tests/rest.spec.ts b/packages/server/src/integrations/tests/rest.spec.ts index 175e6bce43..d53f9c7ce3 100644 --- a/packages/server/src/integrations/tests/rest.spec.ts +++ b/packages/server/src/integrations/tests/rest.spec.ts @@ -1,17 +1,16 @@ -jest.mock("node-fetch", () => - jest.fn(() => ({ - headers: { - raw: () => { - return { "content-type": ["application/json"] } - }, - get: () => ["application/json"], +const mockFetch = jest.fn(() => ({ + headers: { + raw: () => { + return { "content-type": ["application/json"] } }, - json: jest.fn(() => ({ - my_next_cursor: 123, - })), - text: jest.fn(), - })) -) + get: () => ["application/json"], + }, + json: jest.fn(() => ({ + my_next_cursor: 123, + })), + text: jest.fn(), +})) +jest.mock("node-fetch", () => mockFetch) import fetch from "node-fetch" import { default as RestIntegration } from "../rest" const FormData = require("form-data") @@ -256,7 +255,7 @@ describe("REST Integration", () => { authConfigId: "c59c14bd1898a43baa08da68959b24686", } await config.integration.read(query) - expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/?`, { + expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/`, { method: "GET", headers: { Authorization: "Basic dXNlcjpwYXNzd29yZA==", @@ -269,7 +268,7 @@ describe("REST Integration", () => { authConfigId: "0d91d732f34e4befabeff50b392a8ff3", } await config.integration.read(query) - expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/?`, { + expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/`, { method: "GET", headers: { Authorization: "Bearer mytoken", @@ -327,7 +326,7 @@ describe("REST Integration", () => { }, } await config.integration.create(query) - expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, { + expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, { body: JSON.stringify({ [pageParam]: pageValue, [sizeParam]: sizeValue, @@ -359,7 +358,7 @@ describe("REST Integration", () => { }, } await config.integration.create(query) - expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, { + expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, { body: expect.any(FormData), headers: {}, method: "POST", @@ -390,7 +389,7 @@ describe("REST Integration", () => { }, } await config.integration.create(query) - expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, { + expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, { body: expect.any(URLSearchParams), headers: {}, method: "POST", @@ -456,7 +455,7 @@ describe("REST Integration", () => { }, } const res = await config.integration.create(query) - expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, { + expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, { body: JSON.stringify({ [pageParam]: pageValue, [sizeParam]: sizeValue, @@ -490,7 +489,7 @@ describe("REST Integration", () => { }, } const res = await config.integration.create(query) - expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, { + expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, { body: expect.any(FormData), headers: {}, method: "POST", @@ -523,7 +522,7 @@ describe("REST Integration", () => { }, } const res = await config.integration.create(query) - expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?`, { + expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api`, { body: expect.any(URLSearchParams), headers: {}, method: "POST", @@ -563,7 +562,7 @@ describe("REST Integration", () => { legacyHttpParser: true, }) await config.integration.read({}) - expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/?`, { + expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/`, { method: "GET", headers: {}, extraHttpOptions: { @@ -572,4 +571,21 @@ describe("REST Integration", () => { }) }) }) + + it("Attaches custom agent when Reject Unauthorized option is false", async () => { + config = new TestConfiguration({ + url: BASE_URL, + rejectUnauthorized: false, + }) + await config.integration.read({}) + + const calls: any = mockFetch.mock.calls[0] + const url = calls[0] + expect(url).toBe(`${BASE_URL}/`) + + const calledConfig = calls[1] + expect(calledConfig.method).toBe("GET") + expect(calledConfig.headers).toEqual({}) + expect(calledConfig.agent.options.rejectUnauthorized).toBe(false) + }) }) diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts index cb1e6d1c82..275a954a78 100644 --- a/packages/server/src/migrations/index.ts +++ b/packages/server/src/migrations/index.ts @@ -1,5 +1,11 @@ -import { migrations, redis } from "@budibase/backend-core" -import { Migration, MigrationOptions, MigrationName } from "@budibase/types" +import { locks, migrations } from "@budibase/backend-core" +import { + Migration, + MigrationOptions, + MigrationName, + LockType, + LockName, +} from "@budibase/types" import env from "../environment" // migration functions @@ -86,33 +92,14 @@ export const migrate = async (options?: MigrationOptions) => { } const migrateWithLock = async (options?: MigrationOptions) => { - // get a new lock client - const redlock = await redis.clients.getMigrationsRedlock() - // lock for 15 minutes - const ttl = 1000 * 60 * 15 - - let migrationLock - - // acquire lock - try { - migrationLock = await redlock.lock("migrations", ttl) - } catch (e: any) { - if (e.name === "LockError") { - return - } else { - throw e + await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.MIGRATIONS, + ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes + }, + async () => { + await migrations.runMigrations(MIGRATIONS, options) } - } - - // run migrations - try { - await migrations.runMigrations(MIGRATIONS, options) - } finally { - // release lock - try { - await migrationLock.unlock() - } catch (e) { - console.error("unable to release migration lock") - } - } + ) } diff --git a/packages/server/src/utilities/csvParser.js b/packages/server/src/utilities/csvParser.js index 8f9b3373c9..09449f6fc1 100644 --- a/packages/server/src/utilities/csvParser.js +++ b/packages/server/src/utilities/csvParser.js @@ -4,6 +4,7 @@ const { FieldTypes } = require("../constants") const VALIDATORS = { [FieldTypes.STRING]: () => true, [FieldTypes.OPTIONS]: () => true, + [FieldTypes.BARCODEQR]: () => true, [FieldTypes.NUMBER]: attribute => { // allow not to be present if (!attribute) { diff --git a/packages/server/src/utilities/redis.js b/packages/server/src/utilities/redis.js index 4eddca6e4a..b39b7cae55 100644 --- a/packages/server/src/utilities/redis.js +++ b/packages/server/src/utilities/redis.js @@ -34,12 +34,8 @@ exports.doesUserHaveLock = async (devAppId, user) => { return expected === userId } -exports.getAllLocks = async () => { - const locks = await devAppClient.scan() - return locks.map(lock => ({ - appId: lock.key, - user: lock.value, - })) +exports.getLocksById = async appIds => { + return await devAppClient.bulkGet(appIds) } exports.updateLock = async (devAppId, user) => { diff --git a/packages/server/src/utilities/rowProcessor/index.js b/packages/server/src/utilities/rowProcessor/index.js index e4c364eaf3..91daa1b5a0 100644 --- a/packages/server/src/utilities/rowProcessor/index.js +++ b/packages/server/src/utilities/rowProcessor/index.js @@ -48,6 +48,11 @@ const TYPE_TRANSFORM_MAP = { [null]: "", [undefined]: undefined, }, + [FieldTypes.BARCODEQR]: { + "": "", + [null]: "", + [undefined]: undefined, + }, [FieldTypes.FORMULA]: { "": "", [null]: "", diff --git a/packages/server/src/utilities/users.js b/packages/server/src/utilities/users.js index 3fa222e677..498b934605 100644 --- a/packages/server/src/utilities/users.js +++ b/packages/server/src/utilities/users.js @@ -7,14 +7,17 @@ const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles") exports.getFullUser = async (ctx, userId) => { const global = await getGlobalUser(userId) let metadata = {} + + // always prefer the user metadata _id and _rev + delete global._id + delete global._rev + try { // this will throw an error if the db doesn't exist, or there is no appId const db = getAppDB() metadata = await db.get(userId) } catch (err) { - // it is fine if there is no user metadata, just remove global db info - delete global._id - delete global._rev + // it is fine if there is no user metadata yet } delete metadata.csrfToken return { diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 7cfb82a072..1f90321e71 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.24-alpha.2": - version "2.0.24-alpha.2" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.24-alpha.2.tgz#8677856bc4230c30c209ca3cf4abd6d72ac1712c" - integrity sha512-TVf6al/KtOdilwWnaU2ijhUZEZ9c2WNEIN03HZDjHTKM3ur8gNg5znma2Vd0YiBLmnA3a+cQqyu0UkJKxezbVg== +"@budibase/backend-core@2.0.30-alpha.5": + version "2.0.30-alpha.5" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.5.tgz#afa810c1fcb3d9424a6cb13bd6faf7d9ba52f4b8" + integrity sha512-mS3rqhqjzA8ExJFE5DLsdfSKs+NR5/JRNzl4RT6gxavZ0OJuQuYmsGHurkpBVO9sCaBbZpQEoiM/i1CXU0cAyw== dependencies: - "@budibase/types" "2.0.24-alpha.2" + "@budibase/types" "2.0.30-alpha.5" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -1180,13 +1180,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@2.0.24-alpha.2": - version "2.0.24-alpha.2" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.24-alpha.2.tgz#023b943d9b457f363b132e841bb6daa306cc2bbe" - integrity sha512-uAW93A0r2SyUl40gTUJoyAYGu0tgh9CikFFf5EddJytkOd+iAKQJbTjbpUTdBsyivBeZTNIVKCfIf8wxx4XYXQ== +"@budibase/pro@2.0.30-alpha.5": + version "2.0.30-alpha.5" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.5.tgz#bd73648765111f5119be17388d8776747a3ec1c9" + integrity sha512-2TIqb3dxI0SKwZVmZ/ChuSpMSBA2WeuZaozEDngxypuJkdHsrjVlmBmzD0C8iNtsplLVNq/Nz0THsVkXvCzbfQ== dependencies: - "@budibase/backend-core" "2.0.24-alpha.2" - "@budibase/types" "2.0.24-alpha.2" + "@budibase/backend-core" "2.0.30-alpha.5" + "@budibase/types" "2.0.30-alpha.5" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" @@ -1209,10 +1209,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@2.0.24-alpha.2": - version "2.0.24-alpha.2" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.24-alpha.2.tgz#3945eb9b869ff7a91fe32cf443a0aa7daf6c7a8d" - integrity sha512-Bna7JkI2UTPrWH63xrIjfBjgiD5T09n4ITcxxgPw3W81VZ6BJnqiBLCLP6wM5c5xIhKag3VAHQX9n8OREANYAA== +"@budibase/types@2.0.30-alpha.5": + version "2.0.30-alpha.5" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.5.tgz#0bcad9b621b9fd98edba7c3e85d22b8ca8accee5" + integrity sha512-2cW0Aa5KZ/V9Zhrp7W7O++pV8wFh9ejzAwodYqSMlBwNmsuKUxCZtfrQtEVuxNGXABq4cA2Mw0ElGEtcRjHOmw== "@bull-board/api@3.7.0": version "3.7.0" @@ -2661,7 +2661,7 @@ "@types/connect" "*" "@types/node" "*" -"@types/bson@*", "@types/bson@4.2.0": +"@types/bson@4.2.0": version "4.2.0" resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.2.0.tgz#a2f71e933ff54b2c3bf267b67fa221e295a33337" integrity sha512-ELCPqAdroMdcuxqwMgUpifQyRoTpyYCNr1V9xKyF40VsBobsj+BbWNRvwGchMgBPGqkw655ypkjj2MEF5ywVwg== @@ -2887,14 +2887,6 @@ resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw== -"@types/mongodb@3.6.3": - version "3.6.3" - resolved "https://registry.yarnpkg.com/@types/mongodb/-/mongodb-3.6.3.tgz#5655af409d9e32d5d5ae9a653abf3e5f9c83eb7a" - integrity sha512-6YNqGP1hk5bjUFaim+QoFFuI61WjHiHE1BNeB41TA00Xd2K7zG4lcWyLLq/XtIp36uMavvS5hoAUJ+1u/GcX2Q== - dependencies: - "@types/bson" "*" - "@types/node" "*" - "@types/node-fetch@2.6.1": version "2.6.1" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975" @@ -3034,6 +3026,19 @@ dependencies: "@types/node" "*" +"@types/webidl-conversions@*": + version "6.1.1" + resolved "https://registry.yarnpkg.com/@types/webidl-conversions/-/webidl-conversions-6.1.1.tgz#e33bc8ea812a01f63f90481c666334844b12a09e" + integrity sha512-XAahCdThVuCFDQLT7R7Pk/vqeObFNL3YqRyFZg+AqAP/W1/w3xHaIxuW7WszQqTbIBOPRcItYJIou3i/mppu3Q== + +"@types/whatwg-url@^8.2.1": + version "8.2.2" + resolved "https://registry.yarnpkg.com/@types/whatwg-url/-/whatwg-url-8.2.2.tgz#749d5b3873e845897ada99be4448041d4cc39e63" + integrity sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA== + dependencies: + "@types/node" "*" + "@types/webidl-conversions" "*" + "@types/yargs-parser@*": version "21.0.0" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.0.tgz#0c60e537fa790f5f9472ed2776c2b71ec117351b" @@ -4150,14 +4155,6 @@ bl@^1.0.0: readable-stream "^2.3.5" safe-buffer "^5.1.1" -bl@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/bl/-/bl-2.2.1.tgz#8c11a7b730655c5d56898cdc871224f40fd901d5" - integrity sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g== - dependencies: - readable-stream "^2.3.5" - safe-buffer "^5.1.1" - bl@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/bl/-/bl-3.0.1.tgz#1cbb439299609e419b5a74d7fce2f8b37d8e5c6f" @@ -4307,10 +4304,12 @@ bson@*: dependencies: buffer "^5.6.0" -bson@^1.1.4: - version "1.1.6" - resolved "https://registry.yarnpkg.com/bson/-/bson-1.1.6.tgz#fb819be9a60cd677e0853aee4ca712a785d6618a" - integrity sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg== +bson@^4.7.0: + version "4.7.0" + resolved "https://registry.yarnpkg.com/bson/-/bson-4.7.0.tgz#7874a60091ffc7a45c5dd2973b5cad7cded9718a" + integrity sha512-VrlEE4vuiO1WTpfof4VmaVolCVYkYTgB9iWgYNOrVlnifpME/06fhFRmONgBhClD5pFC1t9ZWqFUQEQAzY43bA== + dependencies: + buffer "^5.6.0" buffer-alloc-unsafe@^1.1.0: version "1.1.0" @@ -5318,7 +5317,7 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ== -denque@^1.1.0, denque@^1.4.1: +denque@^1.1.0: version "1.5.1" resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf" integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw== @@ -5328,6 +5327,11 @@ denque@^2.0.1: resolved "https://registry.yarnpkg.com/denque/-/denque-2.0.1.tgz#bcef4c1b80dc32efe97515744f21a4229ab8934a" integrity sha512-tfiWc6BQLXNLpNiR5iGd0Ocu3P3VpxfzFiqubLgMfhfOw9WyvgJBd46CClNn9k3qfbjvT//0cf7AlYRX/OslMQ== +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@2.0.0, depd@^2.0.0, depd@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" @@ -7694,6 +7698,11 @@ ip-regex@^2.1.0: resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" integrity sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw== +ip@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.0.tgz#4cf4ab182fee2314c75ede1276f8c80b479936da" + integrity sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ== + is-accessor-descriptor@^0.1.6: version "0.1.6" resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz#a9e12cb3ae8d876727eeef3843f8a0897b5c98d6" @@ -10284,18 +10293,25 @@ moment-timezone@^0.5.15: resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108" integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w== -mongodb@3.6.3: - version "3.6.3" - resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-3.6.3.tgz#eddaed0cc3598474d7a15f0f2a5b04848489fd05" - integrity sha512-rOZuR0QkodZiM+UbQE5kDsJykBqWi0CL4Ec2i1nrGrUI3KO11r6Fbxskqmq3JK2NH7aW4dcccBuUujAP0ERl5w== +mongodb-connection-string-url@^2.5.3: + version "2.5.3" + resolved "https://registry.yarnpkg.com/mongodb-connection-string-url/-/mongodb-connection-string-url-2.5.3.tgz#c0c572b71570e58be2bd52b33dffd1330cfb6990" + integrity sha512-f+/WsED+xF4B74l3k9V/XkTVj5/fxFH2o5ToKXd8Iyi5UhM+sO9u0Ape17Mvl/GkZaFtM0HQnzAG5OTmhKw+tQ== dependencies: - bl "^2.2.1" - bson "^1.1.4" - denque "^1.4.1" - require_optional "^1.0.1" - safe-buffer "^5.1.2" + "@types/whatwg-url" "^8.2.1" + whatwg-url "^11.0.0" + +mongodb@4.9: + version "4.9.0" + resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-4.9.0.tgz#58618439b721f2d6f7d38bb10a4612e29d7f1c8a" + integrity sha512-tJJEFJz7OQTQPZeVHZJIeSOjMRqc5eSyXTt86vSQENEErpkiG7279tM/GT5AVZ7TgXNh9HQxoa2ZkbrANz5GQw== + dependencies: + bson "^4.7.0" + denque "^2.1.0" + mongodb-connection-string-url "^2.5.3" + socks "^2.7.0" optionalDependencies: - saslprep "^1.0.0" + saslprep "^1.0.3" monitor-event-loop-delay@^1.0.0: version "1.0.0" @@ -12308,14 +12324,6 @@ require-main-filename@^2.0.0: resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg== -require_optional@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/require_optional/-/require_optional-1.0.1.tgz#4cf35a4247f64ca3df8c2ef208cc494b1ca8fc2e" - integrity sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g== - dependencies: - resolve-from "^2.0.0" - semver "^5.1.0" - requires-port@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" @@ -12328,11 +12336,6 @@ resolve-cwd@^3.0.0: dependencies: resolve-from "^5.0.0" -resolve-from@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-2.0.0.tgz#9480ab20e94ffa1d9e80a804c7ea147611966b57" - integrity sha512-qpFcKaXsq8+oRoLilkwyc7zHGF5i9Q2/25NIgLQQ/+VVv9rU4qvr6nXVAw1DsnXJyQkZsR4Ytfbtg5ehfcUssQ== - resolve-from@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" @@ -12519,7 +12522,7 @@ sanitize-s3-objectkey@0.0.1: resolved "https://registry.yarnpkg.com/sanitize-s3-objectkey/-/sanitize-s3-objectkey-0.0.1.tgz#efa9887cd45275b40234fb4bb12fc5754fe64e7e" integrity sha512-ZTk7aqLxy4sD40GWcYWoLfbe05XLmkKvh6vGKe13ADlei24xlezcvjgKy1qRArlaIbIMYaqK7PCalvZtulZlaQ== -saslprep@^1.0.0: +saslprep@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/saslprep/-/saslprep-1.0.3.tgz#4c02f946b56cf54297e347ba1093e7acac4cf226" integrity sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag== @@ -12581,7 +12584,7 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: version "5.7.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== @@ -12769,6 +12772,11 @@ slice-ansi@^2.1.0: astral-regex "^1.0.0" is-fullwidth-code-point "^2.0.0" +smart-buffer@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" + integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== + snapdragon-node@^2.0.1: version "2.1.1" resolved "https://registry.yarnpkg.com/snapdragon-node/-/snapdragon-node-2.1.1.tgz#6c175f86ff14bdb0724563e8f3c1b021a286853b" @@ -12869,6 +12877,14 @@ socket.io@^4.5.1: socket.io-adapter "~2.4.0" socket.io-parser "~4.2.0" +socks@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.7.0.tgz#f9225acdb841e874dca25f870e9130990f3913d0" + integrity sha512-scnOe9y4VuiNUULJN72GrM26BNOjVsfPXI+j+98PkyEfsIXroa5ofyjT+FzGvn/xHs73U2JtoBYAVx9Hl4quSA== + dependencies: + ip "^2.0.0" + smart-buffer "^4.2.0" + sonic-boom@^1.0.2: version "1.4.1" resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-1.4.1.tgz#d35d6a74076624f12e6f917ade7b9d75e918f53e" @@ -13873,6 +13889,13 @@ tr46@^2.1.0: dependencies: punycode "^2.1.1" +tr46@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-3.0.0.tgz#555c4e297a950617e8eeddef633c87d4d9d6cbf9" + integrity sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA== + dependencies: + punycode "^2.1.1" + tr46@~0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" @@ -14432,6 +14455,11 @@ webidl-conversions@^6.1.0: resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-6.1.0.tgz#9111b4d7ea80acd40f5270d666621afa78b69514" integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w== +webidl-conversions@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a" + integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g== + webpack-cli@^4.9.1: version "4.9.2" resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-4.9.2.tgz#77c1adaea020c3f9e2db8aad8ea78d235c83659d" @@ -14513,6 +14541,14 @@ whatwg-mimetype@^2.1.0, whatwg-mimetype@^2.2.0, whatwg-mimetype@^2.3.0: resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== +whatwg-url@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-11.0.0.tgz#0a849eebb5faf2119b901bb76fd795c2848d4018" + integrity sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ== + dependencies: + tr46 "^3.0.0" + webidl-conversions "^7.0.0" + whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index e8679e5a06..fb6c053f35 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.0.24-alpha.2", + "version": "2.0.30-alpha.5", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index c9c4607407..1d30e80535 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.0.24-alpha.2", + "version": "2.0.30-alpha.5", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/types/src/api/account/index.ts b/packages/types/src/api/account/index.ts index 50c6bf22c6..0cbc487bcc 100644 --- a/packages/types/src/api/account/index.ts +++ b/packages/types/src/api/account/index.ts @@ -1 +1,2 @@ export * from "./user" +export * from "./license" diff --git a/packages/types/src/api/account/license.ts b/packages/types/src/api/account/license.ts new file mode 100644 index 0000000000..40ee79c3e3 --- /dev/null +++ b/packages/types/src/api/account/license.ts @@ -0,0 +1,11 @@ +import { QuotaUsage } from "../../documents" + +export interface GetLicenseRequest { + quotaUsage: QuotaUsage +} + +export interface QuotaTriggeredRequest { + percentage: number + name: string + resetDate?: string +} diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts index e7dcf2d89f..70c3061c3f 100644 --- a/packages/types/src/documents/account/account.ts +++ b/packages/types/src/documents/account/account.ts @@ -1,4 +1,12 @@ -import { Feature, Hosting, PlanType, Quotas } from "../../sdk" +import { + Feature, + Hosting, + MonthlyQuotaName, + PlanType, + Quotas, + StaticQuotaName, +} from "../../sdk" +import { MonthlyUsage, QuotaUsage, StaticUsage } from "../global" export interface CreateAccount { email: string @@ -42,6 +50,7 @@ export interface Account extends CreateAccount { licenseKey?: string licenseKeyActivatedAt?: number licenseOverrides?: LicenseOverrides + quotaUsage?: QuotaUsage } export interface PasswordAccount extends Account { diff --git a/packages/types/src/documents/global/quotas.ts b/packages/types/src/documents/global/quotas.ts index eb1d77c228..84e5af3996 100644 --- a/packages/types/src/documents/global/quotas.ts +++ b/packages/types/src/documents/global/quotas.ts @@ -24,19 +24,34 @@ export interface UsageBreakdown { } } -export type MonthlyUsage = { +export type QuotaTriggers = { + [key: string]: string | undefined +} + +export interface StaticUsage { + [StaticQuotaName.APPS]: number + [StaticQuotaName.PLUGINS]: number + [StaticQuotaName.USER_GROUPS]: number + [StaticQuotaName.ROWS]: number + triggers: { + [key in StaticQuotaName]?: QuotaTriggers + } +} + +export interface MonthlyUsage { [MonthlyQuotaName.QUERIES]: number [MonthlyQuotaName.AUTOMATIONS]: number [MonthlyQuotaName.DAY_PASSES]: number + triggers: { + [key in MonthlyQuotaName]?: QuotaTriggers + } breakdown?: { [key in BreakdownQuotaName]?: UsageBreakdown } } export interface BaseQuotaUsage { - usageQuota: { - [key in StaticQuotaName]: number - } + usageQuota: StaticUsage monthly: { [key: string]: MonthlyUsage } @@ -51,6 +66,13 @@ export interface QuotaUsage extends BaseQuotaUsage { } } +export type SetUsageValues = { + total: number + app?: number + breakdown?: number + triggers?: QuotaTriggers +} + export type UsageValues = { total: number app?: number diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index d01d636b86..279a76446f 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -24,6 +24,7 @@ export enum QueryType { export enum DatasourceFieldType { STRING = "string", + CODE = "code", LONGFORM = "longForm", BOOLEAN = "boolean", NUMBER = "number", @@ -70,6 +71,11 @@ export enum FilterType { ONE_OF = "oneOf", } +export interface StepDefinition { + key: string + template: string +} + export interface QueryDefinition { type: QueryType displayName?: string @@ -77,6 +83,7 @@ export interface QueryDefinition { customisable?: boolean fields?: object urlDisplay?: boolean + steps?: Array } export interface ExtraQueryConfig { diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index bae566b42e..0c374dd105 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -7,3 +7,4 @@ export * from "./datasources" export * from "./search" export * from "./koa" export * from "./auth" +export * from "./locks" diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 49dd561db0..74777d4590 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -61,26 +61,40 @@ export type PlanQuotas = { [PlanType.ENTERPRISE]: Quotas } +export type MonthlyQuotas = { + [MonthlyQuotaName.QUERIES]: Quota + [MonthlyQuotaName.AUTOMATIONS]: Quota + [MonthlyQuotaName.DAY_PASSES]: Quota +} + +export type StaticQuotas = { + [StaticQuotaName.ROWS]: Quota + [StaticQuotaName.APPS]: Quota + [StaticQuotaName.USER_GROUPS]: Quota + [StaticQuotaName.PLUGINS]: Quota +} + +export type ConstantQuotas = { + [ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota +} + export type Quotas = { [QuotaType.USAGE]: { - [QuotaUsageType.MONTHLY]: { - [MonthlyQuotaName.QUERIES]: Quota - [MonthlyQuotaName.AUTOMATIONS]: Quota - [MonthlyQuotaName.DAY_PASSES]: Quota - } - [QuotaUsageType.STATIC]: { - [StaticQuotaName.ROWS]: Quota - [StaticQuotaName.APPS]: Quota - [StaticQuotaName.USER_GROUPS]: Quota - [StaticQuotaName.PLUGINS]: Quota - } - } - [QuotaType.CONSTANT]: { - [ConstantQuotaName.AUTOMATION_LOG_RETENTION_DAYS]: Quota + [QuotaUsageType.MONTHLY]: MonthlyQuotas + [QuotaUsageType.STATIC]: StaticQuotas } + [QuotaType.CONSTANT]: ConstantQuotas } export interface Quota { name: string value: number + /** + * Array of whole numbers (1-100) that dictate the percentage that this quota should trigger + * at in relation to the corresponding usage inside budibase. + * + * Triggering results in a budibase installation sending a request to account-portal, + * which can have subsequent effects such as sending emails to users. + */ + triggers: number[] } diff --git a/packages/types/src/sdk/locks.ts b/packages/types/src/sdk/locks.ts new file mode 100644 index 0000000000..3aa067bea1 --- /dev/null +++ b/packages/types/src/sdk/locks.ts @@ -0,0 +1,31 @@ +export enum LockType { + /** + * If this lock is already held the attempted operation will not be performed. + * No retries will take place and no error will be thrown. + */ + TRY_ONCE = "try_once", +} + +export enum LockName { + MIGRATIONS = "migrations", + TRIGGER_QUOTA = "trigger_quota", +} + +export interface LockOptions { + /** + * The lock type determines which client to use + */ + type: LockType + /** + * The name for the lock + */ + name: LockName + /** + * The ttl to auto-expire the lock if not unlocked manually + */ + ttl: number + /** + * The suffix to add to the lock name for additional uniqueness + */ + nameSuffix?: string +} diff --git a/packages/worker/package.json b/packages/worker/package.json index f9e4929cde..6922392043 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.0.24-alpha.2", + "version": "2.0.30-alpha.5", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "2.0.24-alpha.2", - "@budibase/pro": "2.0.24-alpha.2", - "@budibase/string-templates": "2.0.24-alpha.2", - "@budibase/types": "2.0.24-alpha.2", + "@budibase/backend-core": "2.0.30-alpha.5", + "@budibase/pro": "2.0.30-alpha.5", + "@budibase/string-templates": "2.0.30-alpha.5", + "@budibase/types": "2.0.30-alpha.5", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", diff --git a/packages/worker/src/api/controllers/system/tenants.js b/packages/worker/src/api/controllers/system/tenants.js deleted file mode 100644 index c54a3d9834..0000000000 --- a/packages/worker/src/api/controllers/system/tenants.js +++ /dev/null @@ -1,58 +0,0 @@ -const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db") -const { getTenantId } = require("@budibase/backend-core/tenancy") -const { deleteTenant } = require("@budibase/backend-core/deprovision") -const { quotas } = require("@budibase/pro") - -exports.exists = async ctx => { - const tenantId = ctx.request.params - ctx.body = { - exists: await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => { - let exists = false - try { - const tenantsDoc = await db.get( - StaticDatabases.PLATFORM_INFO.docs.tenants - ) - if (tenantsDoc) { - exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1 - } - } catch (err) { - // if error it doesn't exist - } - return exists - }), - } -} - -exports.fetch = async ctx => { - ctx.body = await doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => { - let tenants = [] - try { - const tenantsDoc = await db.get( - StaticDatabases.PLATFORM_INFO.docs.tenants - ) - if (tenantsDoc) { - tenants = tenantsDoc.tenantIds - } - } catch (err) { - // if error it doesn't exist - } - return tenants - }) -} - -exports.delete = async ctx => { - const tenantId = getTenantId() - - if (ctx.params.tenantId !== tenantId) { - ctx.throw(403, "Unauthorized") - } - - try { - await deleteTenant(tenantId) - await quotas.bustCache() - ctx.status = 204 - } catch (err) { - ctx.log.error(err) - throw err - } -} diff --git a/packages/worker/src/api/controllers/system/tenants.ts b/packages/worker/src/api/controllers/system/tenants.ts new file mode 100644 index 0000000000..d6e6261c22 --- /dev/null +++ b/packages/worker/src/api/controllers/system/tenants.ts @@ -0,0 +1,66 @@ +const { StaticDatabases, doWithDB } = require("@budibase/backend-core/db") +const { getTenantId } = require("@budibase/backend-core/tenancy") +const { deleteTenant } = require("@budibase/backend-core/deprovision") +import { quotas } from "@budibase/pro" + +export const exists = async (ctx: any) => { + const tenantId = ctx.request.params + ctx.body = { + exists: await doWithDB( + StaticDatabases.PLATFORM_INFO.name, + async (db: any) => { + let exists = false + try { + const tenantsDoc = await db.get( + StaticDatabases.PLATFORM_INFO.docs.tenants + ) + if (tenantsDoc) { + exists = tenantsDoc.tenantIds.indexOf(tenantId) !== -1 + } + } catch (err) { + // if error it doesn't exist + } + return exists + } + ), + } +} + +export const fetch = async (ctx: any) => { + ctx.body = await doWithDB( + StaticDatabases.PLATFORM_INFO.name, + async (db: any) => { + let tenants = [] + try { + const tenantsDoc = await db.get( + StaticDatabases.PLATFORM_INFO.docs.tenants + ) + if (tenantsDoc) { + tenants = tenantsDoc.tenantIds + } + } catch (err) { + // if error it doesn't exist + } + return tenants + } + ) +} + +const _delete = async (ctx: any) => { + const tenantId = getTenantId() + + if (ctx.params.tenantId !== tenantId) { + ctx.throw(403, "Unauthorized") + } + + try { + await deleteTenant(tenantId) + await quotas.bustCache() + ctx.status = 204 + } catch (err) { + ctx.log.error(err) + throw err + } +} + +export { _delete as delete } diff --git a/packages/worker/src/migrations/index.ts b/packages/worker/src/migrations/index.ts index 6900596216..19ef076a52 100644 --- a/packages/worker/src/migrations/index.ts +++ b/packages/worker/src/migrations/index.ts @@ -1,5 +1,11 @@ -import { migrations, redis } from "@budibase/backend-core" -import { Migration, MigrationOptions, MigrationName } from "@budibase/types" +import { migrations, locks } from "@budibase/backend-core" +import { + Migration, + MigrationOptions, + MigrationName, + LockType, + LockName, +} from "@budibase/types" import env from "../environment" // migration functions @@ -42,33 +48,14 @@ export const migrate = async (options?: MigrationOptions) => { } const migrateWithLock = async (options?: MigrationOptions) => { - // get a new lock client - const redlock = await redis.clients.getMigrationsRedlock() - // lock for 15 minutes - const ttl = 1000 * 60 * 15 - - let migrationLock - - // acquire lock - try { - migrationLock = await redlock.lock("migrations", ttl) - } catch (e: any) { - if (e.name === "LockError") { - return - } else { - throw e + await locks.doWithLock( + { + type: LockType.TRY_ONCE, + name: LockName.MIGRATIONS, + ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes + }, + async () => { + await migrations.runMigrations(MIGRATIONS, options) } - } - - // run migrations - try { - await migrations.runMigrations(MIGRATIONS, options) - } finally { - // release lock - try { - await migrationLock.unlock() - } catch (e) { - console.error("unable to release migration lock") - } - } + ) } diff --git a/packages/worker/src/sdk/users/events.ts b/packages/worker/src/sdk/users/events.ts index 0094c6fd84..3046442393 100644 --- a/packages/worker/src/sdk/users/events.ts +++ b/packages/worker/src/sdk/users/events.ts @@ -1,7 +1,6 @@ import env from "../../environment" import { events, accounts, tenancy } from "@budibase/backend-core" import { User, UserRoles, CloudAccount } from "@budibase/types" -import { users as pro } from "@budibase/pro" export const handleDeleteEvents = async (user: any) => { await events.user.deleted(user) diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index e33d793690..f1a75b22bb 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.0.24-alpha.2": - version "2.0.24-alpha.2" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.24-alpha.2.tgz#8677856bc4230c30c209ca3cf4abd6d72ac1712c" - integrity sha512-TVf6al/KtOdilwWnaU2ijhUZEZ9c2WNEIN03HZDjHTKM3ur8gNg5znma2Vd0YiBLmnA3a+cQqyu0UkJKxezbVg== +"@budibase/backend-core@2.0.30-alpha.5": + version "2.0.30-alpha.5" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.0.30-alpha.5.tgz#afa810c1fcb3d9424a6cb13bd6faf7d9ba52f4b8" + integrity sha512-mS3rqhqjzA8ExJFE5DLsdfSKs+NR5/JRNzl4RT6gxavZ0OJuQuYmsGHurkpBVO9sCaBbZpQEoiM/i1CXU0cAyw== dependencies: - "@budibase/types" "2.0.24-alpha.2" + "@budibase/types" "2.0.30-alpha.5" "@shopify/jest-koa-mocks" "5.0.1" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" @@ -327,21 +327,21 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@2.0.24-alpha.2": - version "2.0.24-alpha.2" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.24-alpha.2.tgz#023b943d9b457f363b132e841bb6daa306cc2bbe" - integrity sha512-uAW93A0r2SyUl40gTUJoyAYGu0tgh9CikFFf5EddJytkOd+iAKQJbTjbpUTdBsyivBeZTNIVKCfIf8wxx4XYXQ== +"@budibase/pro@2.0.30-alpha.5": + version "2.0.30-alpha.5" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.0.30-alpha.5.tgz#bd73648765111f5119be17388d8776747a3ec1c9" + integrity sha512-2TIqb3dxI0SKwZVmZ/ChuSpMSBA2WeuZaozEDngxypuJkdHsrjVlmBmzD0C8iNtsplLVNq/Nz0THsVkXvCzbfQ== dependencies: - "@budibase/backend-core" "2.0.24-alpha.2" - "@budibase/types" "2.0.24-alpha.2" + "@budibase/backend-core" "2.0.30-alpha.5" + "@budibase/types" "2.0.30-alpha.5" "@koa/router" "8.0.8" joi "17.6.0" node-fetch "^2.6.1" -"@budibase/types@2.0.24-alpha.2": - version "2.0.24-alpha.2" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.24-alpha.2.tgz#3945eb9b869ff7a91fe32cf443a0aa7daf6c7a8d" - integrity sha512-Bna7JkI2UTPrWH63xrIjfBjgiD5T09n4ITcxxgPw3W81VZ6BJnqiBLCLP6wM5c5xIhKag3VAHQX9n8OREANYAA== +"@budibase/types@2.0.30-alpha.5": + version "2.0.30-alpha.5" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.0.30-alpha.5.tgz#0bcad9b621b9fd98edba7c3e85d22b8ca8accee5" + integrity sha512-2cW0Aa5KZ/V9Zhrp7W7O++pV8wFh9ejzAwodYqSMlBwNmsuKUxCZtfrQtEVuxNGXABq4cA2Mw0ElGEtcRjHOmw== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" diff --git a/qa-core/src/config/internal-api/TestConfiguration/applications.ts b/qa-core/src/config/internal-api/TestConfiguration/applications.ts index 0c51487122..3b50a62781 100644 --- a/qa-core/src/config/internal-api/TestConfiguration/applications.ts +++ b/qa-core/src/config/internal-api/TestConfiguration/applications.ts @@ -3,6 +3,11 @@ import { App } from "@budibase/types" import { Response } from "node-fetch" import InternalAPIClient from "./InternalAPIClient" import FormData from "form-data" +import { RouteConfig } from "../fixtures/types/routing" +import { AppPackageResponse } from "../fixtures/types/appPackage" +import { DeployConfig } from "../fixtures/types/deploy" + +type messageResponse = { message: string } export default class AppApi { api: InternalAPIClient @@ -23,13 +28,13 @@ export default class AppApi { return [response, Object.keys(json.routes).length > 0] } - async getAppPackage(appId: string): Promise<[Response, any]> { + async getAppPackage(appId: string): Promise<[Response, AppPackageResponse]> { const response = await this.api.get(`/applications/${appId}/appPackage`) const json = await response.json() return [response, json] } - async publish(): Promise<[Response, string]> { + async publish(): Promise<[Response, DeployConfig]> { const response = await this.api.post("/deploy") const json = await response.json() return [response, json] @@ -46,4 +51,52 @@ export default class AppApi { const json = await response.json() return [response, json.data] } + + async sync(appId: string): Promise<[Response, messageResponse]> { + const response = await this.api.post(`/applications/${appId}/sync`) + const json = await response.json() + return [response, json] + } + + async updateClient( + appId: string, + body: any + ): Promise<[Response, Application]> { + const response = await this.api.put( + `/applications/${appId}/client/update`, + { body } + ) + const json = await response.json() + return [response, json] + } + + async revert(appId: string): Promise<[Response, messageResponse]> { + const response = await this.api.post(`/dev/${appId}/revert`) + const json = await response.json() + return [response, json] + } + + async delete(appId: string): Promise<[Response, any]> { + const response = await this.api.del(`/applications/${appId}`) + const json = await response.json() + return [response, json] + } + + async update(appId: string, body: any): Promise<[Response, Application]> { + const response = await this.api.put(`/applications/${appId}`, { body }) + const json = await response.json() + return [response, json] + } + + async addScreentoApp(body: any): Promise<[Response, Application]> { + const response = await this.api.post(`/screens`, { body }) + const json = await response.json() + return [response, json] + } + + async getRoutes(): Promise<[Response, RouteConfig]> { + const response = await this.api.get(`/routing`) + const json = await response.json() + return [response, json] + } } diff --git a/qa-core/src/config/internal-api/fixtures/screens.ts b/qa-core/src/config/internal-api/fixtures/screens.ts new file mode 100644 index 0000000000..28e58e8eb8 --- /dev/null +++ b/qa-core/src/config/internal-api/fixtures/screens.ts @@ -0,0 +1,34 @@ +import generator from "../../generator" + +const randomId = generator.guid() + +const generateScreen = (): any => ({ + showNavigation: true, + width: "Large", + props: { + _id: randomId, + _component: "@budibase/standard-components/container", + _styles: { + normal: {}, + hover: {}, + active: {}, + selected: {}, + }, + _children: [], + _instanceName: "New Screen", + direction: "column", + hAlign: "stretch", + vAlign: "top", + size: "grow", + gap: "M", + }, + routing: { + route: "/test", + roleId: "BASIC", + homeScreen: false, + }, + name: randomId, + template: "createFromScratch", +}) + +export default generateScreen diff --git a/qa-core/src/config/internal-api/fixtures/types/appPackage.ts b/qa-core/src/config/internal-api/fixtures/types/appPackage.ts new file mode 100644 index 0000000000..0285565dea --- /dev/null +++ b/qa-core/src/config/internal-api/fixtures/types/appPackage.ts @@ -0,0 +1,9 @@ +import { Application } from "@budibase/server/api/controllers/public/mapping/types" +import { Layout } from "@budibase/types" +import { Screen } from "@budibase/types" +// Create type for getAppPackage response +export interface AppPackageResponse { + application: Partial + layout: Layout + screens: Screen[] +} diff --git a/qa-core/src/config/internal-api/fixtures/types/deploy.ts b/qa-core/src/config/internal-api/fixtures/types/deploy.ts new file mode 100644 index 0000000000..77acb3bb52 --- /dev/null +++ b/qa-core/src/config/internal-api/fixtures/types/deploy.ts @@ -0,0 +1,5 @@ +export interface DeployConfig { + appUrl: string + status: string + _id: string +} diff --git a/qa-core/src/config/internal-api/fixtures/types/routing.ts b/qa-core/src/config/internal-api/fixtures/types/routing.ts new file mode 100644 index 0000000000..c898d7fecd --- /dev/null +++ b/qa-core/src/config/internal-api/fixtures/types/routing.ts @@ -0,0 +1,17 @@ +export interface RouteConfig { + routes: Record +} + +export interface Route { + subpaths: Record +} + +export interface Subpath { + screens: ScreenRouteConfig +} + +export interface ScreenRouteConfig { + BASIC?: string + POWER?: string + ADMIN?: string +} diff --git a/qa-core/src/tests/internal-api/applications/create.spec.ts b/qa-core/src/tests/internal-api/applications/create.spec.ts index 2c934e0bd7..c09af632de 100644 --- a/qa-core/src/tests/internal-api/applications/create.spec.ts +++ b/qa-core/src/tests/internal-api/applications/create.spec.ts @@ -4,6 +4,7 @@ import { db } from "@budibase/backend-core" import InternalAPIClient from "../../../config/internal-api/TestConfiguration/InternalAPIClient" import generateApp from "../../../config/internal-api/fixtures/applications" import generator from "../../../config/generator" +import generateScreen from "../../../config/internal-api/fixtures/screens" describe("Internal API - /applications endpoints", () => { const api = new InternalAPIClient() @@ -84,4 +85,111 @@ describe("Internal API - /applications endpoints", () => { await config.applications.canRender() expect(publishedAppRenders).toBe(true) }) + + it("POST - Sync application before deployment", async () => { + const [response, app] = await config.applications.create(generateApp()) + expect(response).toHaveStatusCode(200) + expect(app.appId).toBeDefined() + config.applications.api.appId = app.appId + + const [syncResponse, sync] = await config.applications.sync( + app.appId + ) + expect(syncResponse).toHaveStatusCode(200) + expect(sync).toEqual({ + message: "App sync not required, app not deployed.", + }) + }) + + it("POST - Sync application after deployment", async () => { + const [response, app] = await config.applications.create(generateApp()) + expect(response).toHaveStatusCode(200) + expect(app.appId).toBeDefined() + config.applications.api.appId = app.appId + + // publish app + await config.applications.publish() + + const [syncResponse, sync] = await config.applications.sync( + app.appId + ) + expect(syncResponse).toHaveStatusCode(200) + expect(sync).toEqual({ + message: "App sync completed successfully.", + }) + }) + + it("PUT - Update an application", async () => { + const [response, app] = await config.applications.create(generateApp()) + expect(response).toHaveStatusCode(200) + expect(app.appId).toBeDefined() + config.applications.api.appId = app.appId + + const [updateResponse, updatedApp] = await config.applications.update( + app.appId, + { + name: generator.word(), + } + ) + expect(updateResponse).toHaveStatusCode(200) + expect(updatedApp.name).not.toEqual(app.name) + }) + + it("POST - Revert Changes without changes", async () => { + const [response, app] = await config.applications.create(generateApp()) + expect(response).toHaveStatusCode(200) + expect(app.appId).toBeDefined() + config.applications.api.appId = app.appId + + const [revertResponse, revert] = await config.applications.revert( + app.appId + ) + expect(revertResponse).toHaveStatusCode(400) + expect(revert).toEqual({ + message: "App has not yet been deployed", + status: 400, + }) + }) + + it("POST - Revert Changes", async () => { + const [response, app] = await config.applications.create(generateApp()) + expect(response).toHaveStatusCode(200) + expect(app.appId).toBeDefined() + config.applications.api.appId = app.appId + + // publish app + const [publishResponse, publish] = await config.applications.publish() + expect(publishResponse).toHaveStatusCode(200) + expect(publish.status).toEqual("SUCCESS") + + // Change/add component to the app + const [screenResponse, screen] = await config.applications.addScreentoApp( + generateScreen() + ) + expect(screenResponse).toHaveStatusCode(200) + expect(screen._id).toBeDefined() + + // // Revert the app to published state + const [revertResponse, revert] = await config.applications.revert( + app.appId + ) + expect(revertResponse).toHaveStatusCode(200) + expect(revert).toEqual({ + message: "Reverted changes successfully.", + }) + + // Check screen is removed + const [routesResponse, routes] = await config.applications.getRoutes() + expect(routesResponse).toHaveStatusCode(200) + expect(routes.routes["/test"]).toBeUndefined() + }) + + it("DELETE - Delete an application", async () => { + const [response, app] = await config.applications.create(generateApp()) + expect(response).toHaveStatusCode(200) + expect(app.appId).toBeDefined() + + const [deleteResponse] = await config.applications.delete(app.appId) + expect(deleteResponse).toHaveStatusCode(200) + }) }) diff --git a/yarn.lock b/yarn.lock index 340aad3f08..a0c22d92bf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -986,13 +986,6 @@ estree-walker "^1.0.1" picomatch "^2.2.2" -"@types/bson@*": - version "4.2.0" - resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.2.0.tgz#a2f71e933ff54b2c3bf267b67fa221e295a33337" - integrity sha512-ELCPqAdroMdcuxqwMgUpifQyRoTpyYCNr1V9xKyF40VsBobsj+BbWNRvwGchMgBPGqkw655ypkjj2MEF5ywVwg== - dependencies: - bson "*" - "@types/estree@0.0.39": version "0.0.39" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f" @@ -1008,19 +1001,6 @@ resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c" integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ== -"@types/mongodb@3.6.3": - version "3.6.3" - resolved "https://registry.yarnpkg.com/@types/mongodb/-/mongodb-3.6.3.tgz#5655af409d9e32d5d5ae9a653abf3e5f9c83eb7a" - integrity sha512-6YNqGP1hk5bjUFaim+QoFFuI61WjHiHE1BNeB41TA00Xd2K7zG4lcWyLLq/XtIp36uMavvS5hoAUJ+1u/GcX2Q== - dependencies: - "@types/bson" "*" - "@types/node" "*" - -"@types/node@*": - version "17.0.33" - resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.33.tgz#3c1879b276dc63e73030bb91165e62a4509cd506" - integrity sha512-miWq2m2FiQZmaHfdZNcbpp9PuXg34W5JZ5CrJ/BaS70VuhoJENBEQybeiYSaPBRNq6KQGnjfEnc/F3PN++D+XQ== - "@types/node@>= 8": version "18.0.0" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.0.0.tgz#67c7b724e1bcdd7a8821ce0d5ee184d3b4dd525a" @@ -1466,13 +1446,6 @@ braces@^3.0.2: dependencies: fill-range "^7.0.1" -bson@*: - version "4.6.3" - resolved "https://registry.yarnpkg.com/bson/-/bson-4.6.3.tgz#d1a9a0b84b9e84b62390811fc5580f6a8b1d858c" - integrity sha512-rAqP5hcUVJhXP2MCSNVsf0oM2OGU1So6A9pVRDYayvJ5+hygXHQApf87wd5NlhPM1J9RJnbqxIG/f8QTzRoQ4A== - dependencies: - buffer "^5.6.0" - btoa-lite@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/btoa-lite/-/btoa-lite-1.0.0.tgz#337766da15801210fdd956c22e9c6891ab9d0337" @@ -1483,7 +1456,7 @@ buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== -buffer@^5.5.0, buffer@^5.6.0: +buffer@^5.5.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==