From 08a22f1525af1fc7225f4792a0183b7e07d3365c Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 23 Oct 2023 16:47:05 +0100 Subject: [PATCH 01/17] Show 'Creator' instead of 'Admin' for the global user role picker --- .../src/components/common/RoleSelect.svelte | 15 ++++++++++++--- .../_components/BuilderSidePanel.svelte | 11 +++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/components/common/RoleSelect.svelte b/packages/builder/src/components/common/RoleSelect.svelte index 82752554d5..2df61926e1 100644 --- a/packages/builder/src/components/common/RoleSelect.svelte +++ b/packages/builder/src/components/common/RoleSelect.svelte @@ -39,7 +39,15 @@ allowCreator ) => { if (allowedRoles?.length) { - return roles.filter(role => allowedRoles.includes(role._id)) + const filteredRoles = roles.filter(role => + allowedRoles.includes(role._id) + ) + return [ + ...filteredRoles, + ...(allowedRoles.includes(Constants.Roles.CREATOR) + ? [{ _id: Constants.Roles.CREATOR, name: "Creator", enabled: false }] + : []), + ] } let newRoles = [...roles] @@ -129,8 +137,9 @@ getOptionColour={getColor} getOptionIcon={getIcon} isOptionEnabled={option => - option._id !== Constants.Roles.CREATOR || - $licensing.perAppBuildersEnabled} + (option._id !== Constants.Roles.CREATOR || + $licensing.perAppBuildersEnabled) && + option.enabled !== false} {placeholder} {error} /> diff --git a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte index a7d9584330..f9a40b09a6 100644 --- a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte @@ -516,6 +516,13 @@ } return null } + + const parseRole = user => { + if (user.isAdminOrGlobalBuilder) { + return Constants.Roles.CREATOR + } + return user.role + } @@ -725,7 +732,7 @@ From cea1c04b73c508d96f1a1a828b964ec07b2858b5 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Tue, 24 Oct 2023 17:16:44 +0200 Subject: [PATCH 02/17] Creators count functionality --- .../backend-core/src/cache/writethrough.ts | 4 +- packages/backend-core/src/users/db.ts | 110 ++++++++++-------- packages/backend-core/src/users/users.ts | 2 +- .../tests/core/users/users.spec.js | 54 +++++++++ .../core/utilities/structures/licenses.ts | 8 ++ packages/types/src/sdk/featureFlag.ts | 3 + packages/types/src/sdk/licensing/billing.ts | 7 ++ packages/types/src/sdk/licensing/plan.ts | 4 + 8 files changed, 142 insertions(+), 50 deletions(-) create mode 100644 packages/backend-core/tests/core/users/users.spec.js diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index e64c116663..c331d791a6 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -119,8 +119,8 @@ export class Writethrough { this.writeRateMs = writeRateMs } - async put(doc: any) { - return put(this.db, doc, this.writeRateMs) + async put(doc: any, writeRateMs: number = this.writeRateMs) { + return put(this.db, doc, writeRateMs) } async get(id: string) { diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index a2539e836e..daa09bee6f 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -25,12 +25,17 @@ import { import { getAccountHolderFromUserIds, isAdmin, + isCreator, validateUniqueUser, } from "./utils" import { searchExistingEmails } from "./lookup" import { hash } from "../utils" -type QuotaUpdateFn = (change: number, cb?: () => Promise) => Promise +type QuotaUpdateFn = ( + change: number, + creatorsChange: number, + cb?: () => Promise +) => Promise type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise type FeatureFn = () => Promise type GroupGetFn = (ids: string[]) => Promise @@ -245,7 +250,8 @@ export class UserDB { } const change = dbUser ? 0 : 1 // no change if there is existing user - return UserDB.quotas.addUsers(change, async () => { + const creatorsChange = isCreator(dbUser) !== isCreator(user) ? 1 : 0 + return UserDB.quotas.addUsers(change, creatorsChange, async () => { await validateUniqueUser(email, tenantId) let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser) @@ -307,6 +313,7 @@ export class UserDB { let usersToSave: any[] = [] let newUsers: any[] = [] + let newCreators: any[] = [] const emails = newUsersRequested.map((user: User) => user.email) const existingEmails = await searchExistingEmails(emails) @@ -327,59 +334,66 @@ export class UserDB { } newUser.userGroups = groups newUsers.push(newUser) + if (isCreator(newUser)) { + newCreators.push(newUser) + } } const account = await accountSdk.getAccountByTenantId(tenantId) - return UserDB.quotas.addUsers(newUsers.length, async () => { - // create the promises array that will be called by bulkDocs - newUsers.forEach((user: any) => { - usersToSave.push( - UserDB.buildUser( - user, - { - hashPassword: true, - requirePassword: user.requirePassword, - }, - tenantId, - undefined, // no dbUser - account + return UserDB.quotas.addUsers( + newUsers.length, + newCreators.length, + async () => { + // create the promises array that will be called by bulkDocs + newUsers.forEach((user: any) => { + usersToSave.push( + UserDB.buildUser( + user, + { + hashPassword: true, + requirePassword: user.requirePassword, + }, + tenantId, + undefined, // no dbUser + account + ) ) - ) - }) + }) - const usersToBulkSave = await Promise.all(usersToSave) - await usersCore.bulkUpdateGlobalUsers(usersToBulkSave) + const usersToBulkSave = await Promise.all(usersToSave) + await usersCore.bulkUpdateGlobalUsers(usersToBulkSave) - // Post-processing of bulk added users, e.g. events and cache operations - for (const user of usersToBulkSave) { - // TODO: Refactor to bulk insert users into the info db - // instead of relying on looping tenant creation - await platform.users.addUser(tenantId, user._id, user.email) - await eventHelpers.handleSaveEvents(user, undefined) - } + // Post-processing of bulk added users, e.g. events and cache operations + for (const user of usersToBulkSave) { + // TODO: Refactor to bulk insert users into the info db + // instead of relying on looping tenant creation + await platform.users.addUser(tenantId, user._id, user.email) + await eventHelpers.handleSaveEvents(user, undefined) + } + + const saved = usersToBulkSave.map(user => { + return { + _id: user._id, + email: user.email, + } + }) + + // now update the groups + if (Array.isArray(saved) && groups) { + const groupPromises = [] + const createdUserIds = saved.map(user => user._id) + for (let groupId of groups) { + groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds)) + } + await Promise.all(groupPromises) + } - const saved = usersToBulkSave.map(user => { return { - _id: user._id, - email: user.email, + successful: saved, + unsuccessful, } - }) - - // now update the groups - if (Array.isArray(saved) && groups) { - const groupPromises = [] - const createdUserIds = saved.map(user => user._id) - for (let groupId of groups) { - groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds)) - } - await Promise.all(groupPromises) } - - return { - successful: saved, - unsuccessful, - } - }) + ) } static async bulkDelete(userIds: string[]): Promise { @@ -419,11 +433,12 @@ export class UserDB { _deleted: true, })) const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete) + const creatorsToDelete = usersToDelete.filter(isCreator) - await UserDB.quotas.removeUsers(toDelete.length) for (let user of usersToDelete) { await bulkDeleteProcessing(user) } + await UserDB.quotas.removeUsers(toDelete.length, creatorsToDelete.length) // Build Response // index users by id @@ -472,7 +487,8 @@ export class UserDB { await db.remove(userId, dbUser._rev) - await UserDB.quotas.removeUsers(1) + const creatorsToDelete = isCreator(dbUser) ? 1 : 0 + await UserDB.quotas.removeUsers(1, creatorsToDelete) await eventHelpers.handleDeleteEvents(dbUser) await cache.user.invalidateUser(userId) await sessions.invalidateSessions(userId, { reason: "deletion" }) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 6237c23972..bad108ab84 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -14,11 +14,11 @@ import { } from "../db" import { BulkDocsResponse, - ContextUser, SearchQuery, SearchQueryOperators, SearchUsersRequest, User, + ContextUser, DatabaseQueryOpts, } from "@budibase/types" import { getGlobalDB } from "../context" diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.js new file mode 100644 index 0000000000..ae7109344a --- /dev/null +++ b/packages/backend-core/tests/core/users/users.spec.js @@ -0,0 +1,54 @@ +const _ = require('lodash/fp') +const {structures} = require("../../../tests") + +jest.mock("../../../src/context") +jest.mock("../../../src/db") + +const context = require("../../../src/context") +const db = require("../../../src/db") + +const {getCreatorCount} = require('../../../src/users/users') + +describe("Users", () => { + + let getGlobalDBMock + let getGlobalUserParamsMock + let paginationMock + + beforeEach(() => { + jest.resetAllMocks() + + getGlobalDBMock = jest.spyOn(context, "getGlobalDB") + getGlobalUserParamsMock = jest.spyOn(db, "getGlobalUserParams") + paginationMock = jest.spyOn(db, "pagination") + }) + + it("Retrieves the number of creators", async () => { + const getUsers = (offset, limit, creators = false) => { + const range = _.range(offset, limit) + const opts = creators ? {builder: {global: true}} : undefined + return range.map(() => structures.users.user(opts)) + } + const page1Data = getUsers(0, 8) + const page2Data = getUsers(8, 12, true) + getGlobalDBMock.mockImplementation(() => ({ + name : "fake-db", + allDocs: () => ({ + rows: [...page1Data, ...page2Data] + }) + })) + paginationMock.mockImplementationOnce(() => ({ + data: page1Data, + hasNextPage: true, + nextPage: "1" + })) + paginationMock.mockImplementation(() => ({ + data: page2Data, + hasNextPage: false, + nextPage: undefined + })) + const creatorsCount = await getCreatorCount() + expect(creatorsCount).toBe(4) + expect(paginationMock).toHaveBeenCalledTimes(2) + }) +}) diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts index 0e34f2e9bb..bb452f9ad5 100644 --- a/packages/backend-core/tests/core/utilities/structures/licenses.ts +++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts @@ -123,6 +123,10 @@ export function customer(): Customer { export function subscription(): Subscription { return { amount: 10000, + amounts: { + user: 10000, + creator: 0, + }, cancelAt: undefined, currency: "usd", currentPeriodEnd: 0, @@ -131,6 +135,10 @@ export function subscription(): Subscription { duration: PriceDuration.MONTHLY, pastDueAt: undefined, quantity: 0, + quantities: { + user: 0, + creator: 0, + }, status: "active", } } diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts index 53aa4842c4..e3935bc7ee 100644 --- a/packages/types/src/sdk/featureFlag.ts +++ b/packages/types/src/sdk/featureFlag.ts @@ -1,5 +1,8 @@ export enum FeatureFlag { LICENSING = "LICENSING", + // Feature IDs in Posthog + PER_CREATOR_PER_USER_PRICE = "18873", + PER_CREATOR_PER_USER_PRICE_ALERT = "18530", } export interface TenantFeatureFlags { diff --git a/packages/types/src/sdk/licensing/billing.ts b/packages/types/src/sdk/licensing/billing.ts index 35f366c811..bcbc7abd18 100644 --- a/packages/types/src/sdk/licensing/billing.ts +++ b/packages/types/src/sdk/licensing/billing.ts @@ -5,10 +5,17 @@ export interface Customer { currency: string | null | undefined } +export interface SubscriptionItems { + user: number | undefined + creator: number | undefined +} + export interface Subscription { amount: number + amounts: SubscriptionItems | undefined currency: string quantity: number + quantities: SubscriptionItems | undefined duration: PriceDuration cancelAt: number | null | undefined currentPeriodStart: number diff --git a/packages/types/src/sdk/licensing/plan.ts b/packages/types/src/sdk/licensing/plan.ts index 3e214a01ff..1604dfb8af 100644 --- a/packages/types/src/sdk/licensing/plan.ts +++ b/packages/types/src/sdk/licensing/plan.ts @@ -4,7 +4,9 @@ export enum PlanType { PRO = "pro", /** @deprecated */ TEAM = "team", + /** @deprecated */ PREMIUM = "premium", + PREMIUM_PLUS = "premium_plus", BUSINESS = "business", ENTERPRISE = "enterprise", } @@ -26,10 +28,12 @@ export interface AvailablePrice { currency: string duration: PriceDuration priceId: string + type?: string } export enum PlanModel { PER_USER = "perUser", + PER_CREATOR_PER_USER = "per_creator_per_user", DAY_PASS = "dayPass", } From 102a0824844f58a51ba784290271ed7e1fcdd10c Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Tue, 24 Oct 2023 17:18:13 +0200 Subject: [PATCH 03/17] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index d24c0dc3a3..39bff12817 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit d24c0dc3a30014cbe61860252aa48104cad36376 +Subproject commit 39bff1281715c647f5d0c1db9bbf1d53c9fd4fc6 From 5dd61f8994c477a4c804c939557c959e543bef17 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 27 Oct 2023 11:47:01 +0100 Subject: [PATCH 04/17] Remove APIDoc comments. --- packages/server/src/api/routes/row.ts | 190 ------------------------ packages/server/src/api/routes/table.ts | 115 -------------- 2 files changed, 305 deletions(-) diff --git a/packages/server/src/api/routes/row.ts b/packages/server/src/api/routes/row.ts index c29cb65eac..516bfd20c6 100644 --- a/packages/server/src/api/routes/row.ts +++ b/packages/server/src/api/routes/row.ts @@ -11,128 +11,24 @@ const { PermissionType, PermissionLevel } = permissions const router: Router = new Router() router - /** - * @api {get} /api/:sourceId/:rowId/enrich Get an enriched row - * @apiName Get an enriched row - * @apiGroup rows - * @apiPermission table read access - * @apiDescription This API is only useful when dealing with rows that have relationships. - * Normally when a row is a returned from the API relationships will only have the structure - * `{ primaryDisplay: "name", _id: ... }` but this call will return the full related rows - * for each relationship instead. - * - * @apiParam {string} rowId The ID of the row which is to be retrieved and enriched. - * - * @apiSuccess {object} row The response body will be the enriched row. - */ .get( "/api/:sourceId/:rowId/enrich", paramSubResource("sourceId", "rowId"), authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.fetchEnrichedRow ) - /** - * @api {get} /api/:sourceId/rows Get all rows in a table - * @apiName Get all rows in a table - * @apiGroup rows - * @apiPermission table read access - * @apiDescription This is a deprecated endpoint that should not be used anymore, instead use the search endpoint. - * This endpoint gets all of the rows within the specified table - it is not heavily used - * due to its lack of support for pagination. With SQL tables this will retrieve up to a limit and then - * will simply stop. - * - * @apiParam {string} sourceId The ID of the table to retrieve all rows within. - * - * @apiSuccess {object[]} rows The response body will be an array of all rows found. - */ .get( "/api/:sourceId/rows", paramResource("sourceId"), authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.fetch ) - /** - * @api {get} /api/:sourceId/rows/:rowId Retrieve a single row - * @apiName Retrieve a single row - * @apiGroup rows - * @apiPermission table read access - * @apiDescription This endpoint retrieves only the specified row. If you wish to retrieve - * a row by anything other than its _id field, use the search endpoint. - * - * @apiParam {string} sourceId The ID of the table to retrieve a row from. - * @apiParam {string} rowId The ID of the row to retrieve. - * - * @apiSuccess {object} body The response body will be the row that was found. - */ .get( "/api/:sourceId/rows/:rowId", paramSubResource("sourceId", "rowId"), authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.find ) - /** - * @api {post} /api/:sourceId/search Search for rows in a table - * @apiName Search for rows in a table - * @apiGroup rows - * @apiPermission table read access - * @apiDescription This is the primary method of accessing rows in Budibase, the data provider - * and data UI in the builder are built atop this. All filtering, sorting and pagination is - * handled through this, for internal and external (datasource plus, e.g. SQL) tables. - * - * @apiParam {string} sourceId The ID of the table to retrieve rows from. - * - * @apiParam (Body) {boolean} [paginate] If pagination is required then this should be set to true, - * defaults to false. - * @apiParam (Body) {object} [query] This contains a set of filters which should be applied, if none - * specified then the request will be unfiltered. An example with all of the possible query - * options has been supplied below. - * @apiParam (Body) {number} [limit] This sets a limit for the number of rows that will be returned, - * this will be implemented at the database level if supported for performance reasons. This - * is useful when paginating to set exactly how many rows per page. - * @apiParam (Body) {string} [bookmark] If pagination is enabled then a bookmark will be returned - * with each successful search request, this should be supplied back to get the next page. - * @apiParam (Body) {object} [sort] If sort is desired this should contain the name of the column to - * sort on. - * @apiParam (Body) {string} [sortOrder] If sort is enabled then this can be either "descending" or - * "ascending" as required. - * @apiParam (Body) {string} [sortType] If sort is enabled then you must specify the type of search - * being used, either "string" or "number". This is only used for internal tables. - * - * @apiParamExample {json} Example: - * { - * "tableId": "ta_70260ff0b85c467ca74364aefc46f26d", - * "query": { - * "string": {}, - * "fuzzy": {}, - * "range": { - * "columnName": { - * "high": 20, - * "low": 10, - * } - * }, - * "equal": { - * "columnName": "someValue" - * }, - * "notEqual": {}, - * "empty": {}, - * "notEmpty": {}, - * "oneOf": { - * "columnName": ["value"] - * } - * }, - * "limit": 10, - * "sort": "name", - * "sortOrder": "descending", - * "sortType": "string", - * "paginate": true - * } - * - * @apiSuccess {object[]} rows An array of rows that was found based on the supplied parameters. - * @apiSuccess {boolean} hasNextPage If pagination was enabled then this specifies whether or - * not there is another page after this request. - * @apiSuccess {string} bookmark The bookmark to be sent with the next request to get the next - * page. - */ .post( "/api/:sourceId/search", internalSearchValidator(), @@ -148,30 +44,6 @@ router authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.search ) - /** - * @api {post} /api/:sourceId/rows Creates a new row - * @apiName Creates a new row - * @apiGroup rows - * @apiPermission table write access - * @apiDescription This API will create a new row based on the supplied body. If the - * body includes an "_id" field then it will update an existing row if the field - * links to one. Please note that "_id", "_rev" and "tableId" are fields that are - * already used by Budibase tables and cannot be used for columns. - * - * @apiParam {string} sourceId The ID of the table to save a row to. - * - * @apiParam (Body) {string} [_id] If the row exists already then an ID for the row must be provided. - * @apiParam (Body) {string} [_rev] If working with an existing row for an internal table its revision - * must also be provided. - * @apiParam (Body) {string} tableId The ID of the table should also be specified in the row body itself. - * @apiParam (Body) {any} [any] Any field supplied in the body will be assessed to see if it matches - * a column in the specified table. All other fields will be dropped and not stored. - * - * @apiSuccess {string} _id The ID of the row that was just saved, if it was just created this - * is the rows new ID. - * @apiSuccess {string} [_rev] If saving to an internal table a revision will also be returned. - * @apiSuccess {object} body The contents of the row that was saved will be returned as well. - */ .post( "/api/:sourceId/rows", paramResource("sourceId"), @@ -179,14 +51,6 @@ router trimViewRowInfo, rowController.save ) - /** - * @api {patch} /api/:sourceId/rows Updates a row - * @apiName Update a row - * @apiGroup rows - * @apiPermission table write access - * @apiDescription This endpoint is identical to the row creation endpoint but instead it will - * error if an _id isn't provided, it will only function for existing rows. - */ .patch( "/api/:sourceId/rows", paramResource("sourceId"), @@ -194,52 +58,12 @@ router trimViewRowInfo, rowController.patch ) - /** - * @api {post} /api/:sourceId/rows/validate Validate inputs for a row - * @apiName Validate inputs for a row - * @apiGroup rows - * @apiPermission table write access - * @apiDescription When attempting to save a row you may want to check if the row is valid - * given the table schema, this will iterate through all the constraints on the table and - * check if the request body is valid. - * - * @apiParam {string} sourceId The ID of the table the row is to be validated for. - * - * @apiParam (Body) {any} [any] Any fields provided in the request body will be tested - * against the table schema and constraints. - * - * @apiSuccess {boolean} valid If inputs provided are acceptable within the table schema this - * will be true, if it is not then then errors property will be populated. - * @apiSuccess {object} [errors] A key value map of information about fields on the input - * which do not match the table schema. The key name will be the column names that have breached - * the schema. - */ .post( "/api/:sourceId/rows/validate", paramResource("sourceId"), authorized(PermissionType.TABLE, PermissionLevel.WRITE), rowController.validate ) - /** - * @api {delete} /api/:sourceId/rows Delete rows - * @apiName Delete rows - * @apiGroup rows - * @apiPermission table write access - * @apiDescription This endpoint can delete a single row, or delete them in a bulk - * fashion. - * - * @apiParam {string} sourceId The ID of the table the row is to be deleted from. - * - * @apiParam (Body) {object[]} [rows] If bulk deletion is desired then provide the rows in this - * key of the request body that are to be deleted. - * @apiParam (Body) {string} [_id] If deleting a single row then provide its ID in this field. - * @apiParam (Body) {string} [_rev] If deleting a single row from an internal table then provide its - * revision here. - * - * @apiSuccess {object[]|object} body If deleting bulk then the response body will be an array - * of the deleted rows, if deleting a single row then the body will contain a "row" property which - * is the deleted row. - */ .delete( "/api/:sourceId/rows", paramResource("sourceId"), @@ -247,20 +71,6 @@ router trimViewRowInfo, rowController.destroy ) - - /** - * @api {post} /api/:sourceId/rows/exportRows Export Rows - * @apiName Export rows - * @apiGroup rows - * @apiPermission table write access - * @apiDescription This API can export a number of provided rows - * - * @apiParam {string} sourceId The ID of the table the row is to be deleted from. - * - * @apiParam (Body) {object[]} [rows] The row IDs which are to be exported - * - * @apiSuccess {object[]|object} - */ .post( "/api/:sourceId/rows/exportRows", paramResource("sourceId"), diff --git a/packages/server/src/api/routes/table.ts b/packages/server/src/api/routes/table.ts index 7ffa5acb3e..0172d9844d 100644 --- a/packages/server/src/api/routes/table.ts +++ b/packages/server/src/api/routes/table.ts @@ -9,99 +9,13 @@ const { BUILDER, PermissionLevel, PermissionType } = permissions const router: Router = new Router() router - /** - * @api {get} /api/tables Fetch all tables - * @apiName Fetch all tables - * @apiGroup tables - * @apiPermission table read access - * @apiDescription This endpoint retrieves all of the tables which have been created in - * an app. This includes all of the external and internal tables; to tell the difference - * between these look for the "type" property on each table, either being "internal" or "external". - * - * @apiSuccess {object[]} body The response body will be the list of tables that was found - as - * this does not take any parameters the only error scenario is no access. - */ .get("/api/tables", authorized(BUILDER), tableController.fetch) - /** - * @api {get} /api/tables/:id Fetch a single table - * @apiName Fetch a single table - * @apiGroup tables - * @apiPermission table read access - * @apiDescription Retrieves a single table this could be be internal or external based on - * the provided table ID. - * - * @apiParam {string} id The ID of the table which is to be retrieved. - * - * @apiSuccess {object[]} body The response body will be the table that was found. - */ .get( "/api/tables/:tableId", paramResource("tableId"), authorized(PermissionType.TABLE, PermissionLevel.READ, { schema: true }), tableController.find ) - /** - * @api {post} /api/tables Save a table - * @apiName Save a table - * @apiGroup tables - * @apiPermission builder - * @apiDescription Create or update a table with this endpoint, this will function for both internal - * external tables. - * - * @apiParam (Body) {string} [_id] If updating an existing table then the ID of the table must be specified. - * @apiParam (Body) {string} [_rev] If updating an existing internal table then the revision must also be specified. - * @apiParam (Body) {string} type] This should either be "internal" or "external" depending on the table type - - * this will default to internal. - * @apiParam (Body) {string} [sourceId] If creating an external table then this should be set to the datasource ID. If - * building an internal table this does not need to be set, although it will be returned as "bb_internal". - * @apiParam (Body) {string} name The name of the table, this will be used in the UI. To rename the table simply - * supply the table structure to this endpoint with the name changed. - * @apiParam (Body) {object} schema A key value object which has all of the columns in the table as the keys in this - * object. For each column a "type" and "constraints" must be specified, with some types requiring further information. - * More information about the schema structure can be found in the Typescript definitions. - * @apiParam (Body) {string} [primaryDisplay] The name of the column which should be used when displaying rows - * from this table as relationships. - * @apiParam (Body) {object[]} [indexes] Specifies the search indexes - this is deprecated behaviour with the introduction - * of lucene indexes. This functionality is only available for internal tables. - * @apiParam (Body) {object} [_rename] If a column is to be renamed then the "old" column name should be set in this - * structure, and the "updated", new column name should also be supplied. The schema should also be updated, this field - * lets the server know that a field hasn't just been deleted, that the data has moved to a new name, this will fix - * the rows in the table. This functionality is only available for internal tables. - * @apiParam (Body) {object[]} [rows] When creating a table using a compatible data source, an array of objects to be imported into the new table can be provided. - * - * @apiParamExample {json} Example: - * { - * "_id": "ta_05541307fa0f4044abee071ca2a82119", - * "_rev": "10-0fbe4e78f69b255d79f1017e2eeef807", - * "type": "internal", - * "views": {}, - * "name": "tableName", - * "schema": { - * "column": { - * "type": "string", - * "constraints": { - * "type": "string", - * "length": { - * "maximum": null - * }, - * "presence": false - * }, - * "name": "column" - * }, - * }, - * "primaryDisplay": "column", - * "indexes": [], - * "sourceId": "bb_internal", - * "_rename": { - * "old": "columnName", - * "updated": "newColumnName", - * }, - * "rows": [] - * } - * - * @apiSuccess {object} table The response body will contain the table structure after being cleaned up and - * saved to the database. - */ .post( "/api/tables", // allows control over updating a table @@ -125,41 +39,12 @@ router authorized(BUILDER), tableController.validateExistingTableImport ) - /** - * @api {post} /api/tables/:tableId/:revId Delete a table - * @apiName Delete a table - * @apiGroup tables - * @apiPermission builder - * @apiDescription This endpoint will delete a table and all of its associated data, for this reason it is - * quite dangerous - it will work for internal and external tables. - * - * @apiParam {string} tableId The ID of the table which is to be deleted. - * @apiParam {string} [revId] If deleting an internal table then the revision must also be supplied (_rev), for - * external tables this can simply be set to anything, e.g. "external". - * - * @apiSuccess {string} message A message stating that the table was deleted successfully. - */ .delete( "/api/tables/:tableId/:revId", paramResource("tableId"), authorized(BUILDER), tableController.destroy ) - /** - * @api {post} /api/tables/:tableId/:revId Import CSV to existing table - * @apiName Import CSV to existing table - * @apiGroup tables - * @apiPermission builder - * @apiDescription This endpoint will import data to existing tables, internal or external. It is used in combination - * with the CSV validation endpoint. Take the output of the CSV validation endpoint and pass it to this endpoint to - * import the data; please note this will only import fields that already exist on the table/match the type. - * - * @apiParam {string} tableId The ID of the table which the data should be imported to. - * - * @apiParam (Body) {object[]} rows An array of objects representing the rows to be imported, key-value pairs not matching the table schema will be ignored. - * - * @apiSuccess {string} message A message stating that the data was imported successfully. - */ .post( "/api/tables/:tableId/import", paramResource("tableId"), From 6ecf831f028c650d6df68fa54c9fa518e68f70ac Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 27 Oct 2023 17:10:03 +0100 Subject: [PATCH 05/17] Updating pro submodule reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 5ed0ee2aca..4506399e0d 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 5ed0ee2aca9d754d80cd46bae412b24621afa47e +Subproject commit 4506399e0d0297554cacbef1f436884aabdb9741 From d4929ea3b61e5774aabf4f7fe758a66b5e872f21 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 27 Oct 2023 18:03:06 +0100 Subject: [PATCH 06/17] Fixing an issue where unpublished apps with custom roles, when used in groups would cause users to be unable to login. --- packages/backend-core/src/security/roles.ts | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index b05cf79c8c..02421fd1d0 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -122,7 +122,9 @@ export async function roleToNumber(id?: string) { if (isBuiltin(id)) { return builtinRoleToNumber(id) } - const hierarchy = (await getUserRoleHierarchy(id)) as RoleDoc[] + const hierarchy = (await getUserRoleHierarchy(id, { + defaultPublic: true, + })) as RoleDoc[] for (let role of hierarchy) { if (isBuiltin(role?.inherits)) { return builtinRoleToNumber(role.inherits) + 1 @@ -177,7 +179,7 @@ export async function getRole( role = Object.assign(role, dbRole) // finalise the ID role._id = getExternalRoleID(role._id, role.version) - } catch (err) { + } catch (err: any) { if (!isBuiltin(roleId) && opts?.defaultPublic) { return cloneDeep(BUILTIN_ROLES.PUBLIC) } @@ -192,12 +194,15 @@ export async function getRole( /** * Simple function to get all the roles based on the top level user role ID. */ -async function getAllUserRoles(userRoleId?: string): Promise { +async function getAllUserRoles( + userRoleId?: string, + opts?: { defaultPublic?: boolean } +): Promise { // admins have access to all roles if (userRoleId === BUILTIN_IDS.ADMIN) { return getAllRoles() } - let currentRole = await getRole(userRoleId) + let currentRole = await getRole(userRoleId, opts) let roles = currentRole ? [currentRole] : [] let roleIds = [userRoleId] // get all the inherited roles @@ -226,12 +231,16 @@ export async function getUserRoleIdHierarchy( * Returns an ordered array of the user's inherited role IDs, this can be used * to determine if a user can access something that requires a specific role. * @param userRoleId The user's role ID, this can be found in their access token. + * @param opts optional - if want to default to public use this. * @returns returns an ordered array of the roles, with the first being their * highest level of access and the last being the lowest level. */ -export async function getUserRoleHierarchy(userRoleId?: string) { +export async function getUserRoleHierarchy( + userRoleId?: string, + opts?: { defaultPublic?: boolean } +) { // special case, if they don't have a role then they are a public user - return getAllUserRoles(userRoleId) + return getAllUserRoles(userRoleId, opts) } // this function checks that the provided permissions are in an array format From 330059991e0d6f6f0cf0af5c3ad3199c044b5509 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 27 Oct 2023 18:04:28 +0100 Subject: [PATCH 07/17] Removing any. --- packages/backend-core/src/security/roles.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 02421fd1d0..0d33031de5 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -179,7 +179,7 @@ export async function getRole( role = Object.assign(role, dbRole) // finalise the ID role._id = getExternalRoleID(role._id, role.version) - } catch (err: any) { + } catch (err) { if (!isBuiltin(roleId) && opts?.defaultPublic) { return cloneDeep(BUILTIN_ROLES.PUBLIC) } From 887383bdb6a4b54d7ab747571415f112f83f2e14 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Fri, 27 Oct 2023 18:12:21 +0100 Subject: [PATCH 08/17] Update README.md Removing dead link. --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index 9deb16cd4f..7827d4e48a 100644 --- a/README.md +++ b/README.md @@ -126,13 +126,6 @@ You can learn more about the Budibase API at the following places: - [Build an app with Budibase and Next.js](https://budibase.com/blog/building-a-crud-app-with-budibase-and-next.js/) -

- Budibase data -

-

- -


- ## 🏁 Get started Deploy Budibase self-hosted in your existing infrastructure, using Docker, Kubernetes, and Digital Ocean. From 1221808c67c65989b24a7540ab0b31f44989e9ff Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Mon, 30 Oct 2023 09:00:20 +0100 Subject: [PATCH 09/17] Updata pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 39bff12817..3820c0c93a 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 39bff1281715c647f5d0c1db9bbf1d53c9fd4fc6 +Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8 From 725e3aa4ef8431990fa9a54d9857f12f95da9a5d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 10:11:06 +0100 Subject: [PATCH 10/17] Use image v2 on build:docker --- packages/server/package.json | 2 +- packages/worker/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 4a858f3be9..b89fe86a84 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -18,7 +18,7 @@ "test": "bash scripts/test.sh", "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit", "test:watch": "jest --watch", - "build:docker": "yarn build && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION", + "build:docker": "yarn nx build && docker build ../.. -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2", "run:docker": "node dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", "dev:stack:up": "node scripts/dev/manage.js up", diff --git a/packages/worker/package.json b/packages/worker/package.json index 205bf3309a..dd847e6df4 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -20,7 +20,7 @@ "run:docker": "node dist/index.js", "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", - "build:docker": "yarn build && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION", + "build:docker": "yarn nx build && docker build ../.. -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2", "dev:stack:init": "node ./scripts/dev/manage.js init", "dev:builder": "npm run dev:stack:init && nodemon", "dev:built": "yarn run dev:stack:init && yarn run run:docker", From b542040ad4fa3d379842348633fbe15546c816b0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 10:27:59 +0100 Subject: [PATCH 11/17] Use v2 for single image --- .github/workflows/release-singleimage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml index f7f87f6e4c..4d35916f4d 100644 --- a/.github/workflows/release-singleimage.yml +++ b/.github/workflows/release-singleimage.yml @@ -67,7 +67,7 @@ jobs: push: true platforms: linux/amd64,linux/arm64 tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile + file: ./hosting/single/Dockerfile.v2 - name: Tag and release Budibase Azure App Service docker image uses: docker/build-push-action@v2 with: @@ -76,4 +76,4 @@ jobs: platforms: linux/amd64 build-args: TARGETBUILD=aas tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile + file: ./hosting/single/Dockerfile.v2 From b0ef79bbd9f9a7ab3a8bbe2f5513426ff2b61362 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 11:36:32 +0100 Subject: [PATCH 12/17] Build for both amd and arm platforms --- packages/server/package.json | 2 +- packages/worker/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index b89fe86a84..c37959d33d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -18,7 +18,7 @@ "test": "bash scripts/test.sh", "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit", "test:watch": "jest --watch", - "build:docker": "yarn nx build && docker build ../.. -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2", + "build:docker": "yarn nx build && docker buildx build ../.. -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2 --platform linux/amd64,linux/arm64", "run:docker": "node dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", "dev:stack:up": "node scripts/dev/manage.js up", diff --git a/packages/worker/package.json b/packages/worker/package.json index dd847e6df4..a391db533b 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -20,7 +20,7 @@ "run:docker": "node dist/index.js", "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", - "build:docker": "yarn nx build && docker build ../.. -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2", + "build:docker": "yarn nx build && docker buildx build ../.. -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2 --platform linux/amd64,linux/arm64", "dev:stack:init": "node ./scripts/dev/manage.js init", "dev:builder": "npm run dev:stack:init && nodemon", "dev:built": "yarn run dev:stack:init && yarn run run:docker", From 6267d9b601e7b350388b5adb139e6ef6c9cb2b16 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 11:39:25 +0100 Subject: [PATCH 13/17] Test building images --- .github/workflows/budibase_ci.yml | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 77867c8617..3ec87f7244 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -42,6 +42,39 @@ jobs: - run: yarn --frozen-lockfile - run: yarn lint + test-release-images: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + fetch-depth: 0 + + - uses: actions/setup-node@v1 + with: + node-version: 18.x + + - run: yarn install --frozen-lockfile + - name: Update versions + run: ./scripts/updateVersions.sh + - run: yarn lint + - run: yarn build + - run: yarn build:sdk + + - name: "Get Current tag" + id: currenttag + run: | + version=$(./scripts/getCurrentVersion.sh) + echo "Using tag $version" + echo "version=$version" >> "$GITHUB_OUTPUT" + + - name: Build/release Docker images + run: | + yarn lerna run --stream build:docker + env: + BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }} + build: runs-on: ubuntu-latest steps: From ac67a17b9393979b65f8bf4335a4f83232872e97 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 11:45:05 +0100 Subject: [PATCH 14/17] Use buildx --- .github/workflows/budibase_ci.yml | 6 +++++- .github/workflows/release-master.yml | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 3ec87f7244..13245a7fa1 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -51,9 +51,10 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} fetch-depth: 0 - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 with: node-version: 18.x + cache: "yarn" - run: yarn install --frozen-lockfile - name: Update versions @@ -69,6 +70,9 @@ jobs: echo "Using tag $version" echo "version=$version" >> "$GITHUB_OUTPUT" + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 - name: Build/release Docker images run: | yarn lerna run --stream build:docker diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 9ab8530341..4c5a3f4a1e 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -63,6 +63,9 @@ jobs: echo "Using tag $version" echo "version=$version" >> "$GITHUB_OUTPUT" + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 - name: Build/release Docker images run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD From 24eed537edc45d8febb9c119f5a4f54f7f296558 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 12:00:55 +0100 Subject: [PATCH 15/17] Fix timeouts --- packages/server/Dockerfile | 2 +- packages/worker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index e1b3b208c7..ea4c5b217a 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -38,7 +38,7 @@ RUN apt update && apt upgrade -y \ COPY package.json . COPY dist/yarn.lock . -RUN yarn install --production=true \ +RUN yarn install --production=true --network-timeout 1000000 \ # Remove unneeded data from file system to reduce image size && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \ && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile index 4230ee86f8..50f1bb78b9 100644 --- a/packages/worker/Dockerfile +++ b/packages/worker/Dockerfile @@ -14,7 +14,7 @@ RUN yarn global add pm2 COPY package.json . COPY dist/yarn.lock . -RUN yarn install --production=true +RUN yarn install --production=true --network-timeout 1000000 # Remove unneeded data from file system to reduce image size RUN apk del .gyp \ && yarn cache clean From d9c34f3f465476291b2cad1b539e0e5e3ceb649e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 12:40:05 +0100 Subject: [PATCH 16/17] Remove build docker in ci pipelines --- .github/workflows/budibase_ci.yml | 37 ------------------------------- 1 file changed, 37 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 13245a7fa1..77867c8617 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -42,43 +42,6 @@ jobs: - run: yarn --frozen-lockfile - run: yarn lint - test-release-images: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - uses: actions/setup-node@v3 - with: - node-version: 18.x - cache: "yarn" - - - run: yarn install --frozen-lockfile - - name: Update versions - run: ./scripts/updateVersions.sh - - run: yarn lint - - run: yarn build - - run: yarn build:sdk - - - name: "Get Current tag" - id: currenttag - run: | - version=$(./scripts/getCurrentVersion.sh) - echo "Using tag $version" - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - name: Build/release Docker images - run: | - yarn lerna run --stream build:docker - env: - BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }} - build: runs-on: ubuntu-latest steps: From 9229ab6896c81c0bc7d545ba3bb3ee5c70840a87 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 30 Oct 2023 12:12:56 +0000 Subject: [PATCH 17/17] Bump version to 2.12.0 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 384473120b..3179bc3b2e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.45", + "version": "2.12.0", "npmClient": "yarn", "packages": [ "packages/*"