From cea1c04b73c508d96f1a1a828b964ec07b2858b5 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Tue, 24 Oct 2023 17:16:44 +0200 Subject: [PATCH 01/36] Creators count functionality --- .../backend-core/src/cache/writethrough.ts | 4 +- packages/backend-core/src/users/db.ts | 110 ++++++++++-------- packages/backend-core/src/users/users.ts | 2 +- .../tests/core/users/users.spec.js | 54 +++++++++ .../core/utilities/structures/licenses.ts | 8 ++ packages/types/src/sdk/featureFlag.ts | 3 + packages/types/src/sdk/licensing/billing.ts | 7 ++ packages/types/src/sdk/licensing/plan.ts | 4 + 8 files changed, 142 insertions(+), 50 deletions(-) create mode 100644 packages/backend-core/tests/core/users/users.spec.js diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index e64c116663..c331d791a6 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -119,8 +119,8 @@ export class Writethrough { this.writeRateMs = writeRateMs } - async put(doc: any) { - return put(this.db, doc, this.writeRateMs) + async put(doc: any, writeRateMs: number = this.writeRateMs) { + return put(this.db, doc, writeRateMs) } async get(id: string) { diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index a2539e836e..daa09bee6f 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -25,12 +25,17 @@ import { import { getAccountHolderFromUserIds, isAdmin, + isCreator, validateUniqueUser, } from "./utils" import { searchExistingEmails } from "./lookup" import { hash } from "../utils" -type QuotaUpdateFn = (change: number, cb?: () => Promise) => Promise +type QuotaUpdateFn = ( + change: number, + creatorsChange: number, + cb?: () => Promise +) => Promise type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise type FeatureFn = () => Promise type GroupGetFn = (ids: string[]) => Promise @@ -245,7 +250,8 @@ export class UserDB { } const change = dbUser ? 0 : 1 // no change if there is existing user - return UserDB.quotas.addUsers(change, async () => { + const creatorsChange = isCreator(dbUser) !== isCreator(user) ? 1 : 0 + return UserDB.quotas.addUsers(change, creatorsChange, async () => { await validateUniqueUser(email, tenantId) let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser) @@ -307,6 +313,7 @@ export class UserDB { let usersToSave: any[] = [] let newUsers: any[] = [] + let newCreators: any[] = [] const emails = newUsersRequested.map((user: User) => user.email) const existingEmails = await searchExistingEmails(emails) @@ -327,59 +334,66 @@ export class UserDB { } newUser.userGroups = groups newUsers.push(newUser) + if (isCreator(newUser)) { + newCreators.push(newUser) + } } const account = await accountSdk.getAccountByTenantId(tenantId) - return UserDB.quotas.addUsers(newUsers.length, async () => { - // create the promises array that will be called by bulkDocs - newUsers.forEach((user: any) => { - usersToSave.push( - UserDB.buildUser( - user, - { - hashPassword: true, - requirePassword: user.requirePassword, - }, - tenantId, - undefined, // no dbUser - account + return UserDB.quotas.addUsers( + newUsers.length, + newCreators.length, + async () => { + // create the promises array that will be called by bulkDocs + newUsers.forEach((user: any) => { + usersToSave.push( + UserDB.buildUser( + user, + { + hashPassword: true, + requirePassword: user.requirePassword, + }, + tenantId, + undefined, // no dbUser + account + ) ) - ) - }) + }) - const usersToBulkSave = await Promise.all(usersToSave) - await usersCore.bulkUpdateGlobalUsers(usersToBulkSave) + const usersToBulkSave = await Promise.all(usersToSave) + await usersCore.bulkUpdateGlobalUsers(usersToBulkSave) - // Post-processing of bulk added users, e.g. events and cache operations - for (const user of usersToBulkSave) { - // TODO: Refactor to bulk insert users into the info db - // instead of relying on looping tenant creation - await platform.users.addUser(tenantId, user._id, user.email) - await eventHelpers.handleSaveEvents(user, undefined) - } + // Post-processing of bulk added users, e.g. events and cache operations + for (const user of usersToBulkSave) { + // TODO: Refactor to bulk insert users into the info db + // instead of relying on looping tenant creation + await platform.users.addUser(tenantId, user._id, user.email) + await eventHelpers.handleSaveEvents(user, undefined) + } + + const saved = usersToBulkSave.map(user => { + return { + _id: user._id, + email: user.email, + } + }) + + // now update the groups + if (Array.isArray(saved) && groups) { + const groupPromises = [] + const createdUserIds = saved.map(user => user._id) + for (let groupId of groups) { + groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds)) + } + await Promise.all(groupPromises) + } - const saved = usersToBulkSave.map(user => { return { - _id: user._id, - email: user.email, + successful: saved, + unsuccessful, } - }) - - // now update the groups - if (Array.isArray(saved) && groups) { - const groupPromises = [] - const createdUserIds = saved.map(user => user._id) - for (let groupId of groups) { - groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds)) - } - await Promise.all(groupPromises) } - - return { - successful: saved, - unsuccessful, - } - }) + ) } static async bulkDelete(userIds: string[]): Promise { @@ -419,11 +433,12 @@ export class UserDB { _deleted: true, })) const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete) + const creatorsToDelete = usersToDelete.filter(isCreator) - await UserDB.quotas.removeUsers(toDelete.length) for (let user of usersToDelete) { await bulkDeleteProcessing(user) } + await UserDB.quotas.removeUsers(toDelete.length, creatorsToDelete.length) // Build Response // index users by id @@ -472,7 +487,8 @@ export class UserDB { await db.remove(userId, dbUser._rev) - await UserDB.quotas.removeUsers(1) + const creatorsToDelete = isCreator(dbUser) ? 1 : 0 + await UserDB.quotas.removeUsers(1, creatorsToDelete) await eventHelpers.handleDeleteEvents(dbUser) await cache.user.invalidateUser(userId) await sessions.invalidateSessions(userId, { reason: "deletion" }) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 6237c23972..bad108ab84 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -14,11 +14,11 @@ import { } from "../db" import { BulkDocsResponse, - ContextUser, SearchQuery, SearchQueryOperators, SearchUsersRequest, User, + ContextUser, DatabaseQueryOpts, } from "@budibase/types" import { getGlobalDB } from "../context" diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.js new file mode 100644 index 0000000000..ae7109344a --- /dev/null +++ b/packages/backend-core/tests/core/users/users.spec.js @@ -0,0 +1,54 @@ +const _ = require('lodash/fp') +const {structures} = require("../../../tests") + +jest.mock("../../../src/context") +jest.mock("../../../src/db") + +const context = require("../../../src/context") +const db = require("../../../src/db") + +const {getCreatorCount} = require('../../../src/users/users') + +describe("Users", () => { + + let getGlobalDBMock + let getGlobalUserParamsMock + let paginationMock + + beforeEach(() => { + jest.resetAllMocks() + + getGlobalDBMock = jest.spyOn(context, "getGlobalDB") + getGlobalUserParamsMock = jest.spyOn(db, "getGlobalUserParams") + paginationMock = jest.spyOn(db, "pagination") + }) + + it("Retrieves the number of creators", async () => { + const getUsers = (offset, limit, creators = false) => { + const range = _.range(offset, limit) + const opts = creators ? {builder: {global: true}} : undefined + return range.map(() => structures.users.user(opts)) + } + const page1Data = getUsers(0, 8) + const page2Data = getUsers(8, 12, true) + getGlobalDBMock.mockImplementation(() => ({ + name : "fake-db", + allDocs: () => ({ + rows: [...page1Data, ...page2Data] + }) + })) + paginationMock.mockImplementationOnce(() => ({ + data: page1Data, + hasNextPage: true, + nextPage: "1" + })) + paginationMock.mockImplementation(() => ({ + data: page2Data, + hasNextPage: false, + nextPage: undefined + })) + const creatorsCount = await getCreatorCount() + expect(creatorsCount).toBe(4) + expect(paginationMock).toHaveBeenCalledTimes(2) + }) +}) diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts index 0e34f2e9bb..bb452f9ad5 100644 --- a/packages/backend-core/tests/core/utilities/structures/licenses.ts +++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts @@ -123,6 +123,10 @@ export function customer(): Customer { export function subscription(): Subscription { return { amount: 10000, + amounts: { + user: 10000, + creator: 0, + }, cancelAt: undefined, currency: "usd", currentPeriodEnd: 0, @@ -131,6 +135,10 @@ export function subscription(): Subscription { duration: PriceDuration.MONTHLY, pastDueAt: undefined, quantity: 0, + quantities: { + user: 0, + creator: 0, + }, status: "active", } } diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts index 53aa4842c4..e3935bc7ee 100644 --- a/packages/types/src/sdk/featureFlag.ts +++ b/packages/types/src/sdk/featureFlag.ts @@ -1,5 +1,8 @@ export enum FeatureFlag { LICENSING = "LICENSING", + // Feature IDs in Posthog + PER_CREATOR_PER_USER_PRICE = "18873", + PER_CREATOR_PER_USER_PRICE_ALERT = "18530", } export interface TenantFeatureFlags { diff --git a/packages/types/src/sdk/licensing/billing.ts b/packages/types/src/sdk/licensing/billing.ts index 35f366c811..bcbc7abd18 100644 --- a/packages/types/src/sdk/licensing/billing.ts +++ b/packages/types/src/sdk/licensing/billing.ts @@ -5,10 +5,17 @@ export interface Customer { currency: string | null | undefined } +export interface SubscriptionItems { + user: number | undefined + creator: number | undefined +} + export interface Subscription { amount: number + amounts: SubscriptionItems | undefined currency: string quantity: number + quantities: SubscriptionItems | undefined duration: PriceDuration cancelAt: number | null | undefined currentPeriodStart: number diff --git a/packages/types/src/sdk/licensing/plan.ts b/packages/types/src/sdk/licensing/plan.ts index 3e214a01ff..1604dfb8af 100644 --- a/packages/types/src/sdk/licensing/plan.ts +++ b/packages/types/src/sdk/licensing/plan.ts @@ -4,7 +4,9 @@ export enum PlanType { PRO = "pro", /** @deprecated */ TEAM = "team", + /** @deprecated */ PREMIUM = "premium", + PREMIUM_PLUS = "premium_plus", BUSINESS = "business", ENTERPRISE = "enterprise", } @@ -26,10 +28,12 @@ export interface AvailablePrice { currency: string duration: PriceDuration priceId: string + type?: string } export enum PlanModel { PER_USER = "perUser", + PER_CREATOR_PER_USER = "per_creator_per_user", DAY_PASS = "dayPass", } From 102a0824844f58a51ba784290271ed7e1fcdd10c Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Tue, 24 Oct 2023 17:18:13 +0200 Subject: [PATCH 02/36] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index d24c0dc3a3..39bff12817 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit d24c0dc3a30014cbe61860252aa48104cad36376 +Subproject commit 39bff1281715c647f5d0c1db9bbf1d53c9fd4fc6 From 5266d80afe559a3f964ee8e65a926c0e2dffcf1a Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 26 Oct 2023 16:09:01 +0100 Subject: [PATCH 03/36] Fix to ensure single integer ids are parsed correctly --- packages/client/src/utils/buttonActions.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js index 18d6b3de3c..9b4640dbb4 100644 --- a/packages/client/src/utils/buttonActions.js +++ b/packages/client/src/utils/buttonActions.js @@ -103,7 +103,6 @@ const fetchRowHandler = async action => { const deleteRowHandler = async action => { const { tableId, rowId: rowConfig, notificationOverride } = action.parameters - if (tableId && rowConfig) { try { let requestConfig @@ -129,9 +128,11 @@ const deleteRowHandler = async action => { requestConfig = [parsedRowConfig] } else if (Array.isArray(parsedRowConfig)) { requestConfig = parsedRowConfig + } else if (Number.isInteger(parsedRowConfig)) { + requestConfig = [String(parsedRowConfig)] } - if (!requestConfig.length) { + if (!requestConfig && !parsedRowConfig) { notificationStore.actions.warning("No valid rows were supplied") return false } From 2160f4e5e205729e1af2afce6cf586c3f537e6f0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 27 Oct 2023 16:24:02 +0100 Subject: [PATCH 04/36] Add valid extension list to shared-core. --- packages/shared-core/src/constants.ts | 43 +++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/shared-core/src/constants.ts b/packages/shared-core/src/constants.ts index 725c246e2f..312e69c896 100644 --- a/packages/shared-core/src/constants.ts +++ b/packages/shared-core/src/constants.ts @@ -96,3 +96,46 @@ export enum BuilderSocketEvent { export const SocketSessionTTL = 60 export const ValidQueryNameRegex = /^[^()]*$/ export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g + +export const ValidFileExtensions = [ + "avif", + "css", + "csv", + "docx", + "drawio", + "editorconfig", + "edl", + "enc", + "export", + "geojson", + "gif", + "htm", + "html", + "ics", + "iqy", + "jfif", + "jpeg", + "jpg", + "json", + "log", + "md", + "mid", + "odt", + "pdf", + "png", + "ris", + "rtf", + "svg", + "tex", + "toml", + "twig", + "txt", + "url", + "wav", + "webp", + "xls", + "xlsx", + "xml", + "yaml", + "yml", +] From 6bb6f106d53ee3e1f866e4317e73f62cba1d207b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 27 Oct 2023 16:46:30 +0100 Subject: [PATCH 05/36] Apply valid file types to AttachmentCell. --- packages/bbui/src/Form/Dropzone.svelte | 2 ++ .../src/components/grid/cells/AttachmentCell.svelte | 3 +++ packages/shared-core/src/constants.ts | 1 - 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/bbui/src/Form/Dropzone.svelte b/packages/bbui/src/Form/Dropzone.svelte index 2a6fa1c57b..94742ea08d 100644 --- a/packages/bbui/src/Form/Dropzone.svelte +++ b/packages/bbui/src/Form/Dropzone.svelte @@ -17,6 +17,7 @@ export let fileTags = [] export let maximum = undefined export let compact = false + export let extensions = undefined const dispatch = createEventDispatcher() const onChange = e => { @@ -39,6 +40,7 @@ {fileTags} {maximum} {compact} + {extensions} on:change={onChange} /> diff --git a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte index a27c31bbe5..4e2a6025e5 100644 --- a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte @@ -2,6 +2,7 @@ import { onMount } from "svelte" import { getContext } from "svelte" import { Dropzone } from "@budibase/bbui" + import { ValidFileExtensions } from "@budibase/shared-core" export let value export let focused = false @@ -13,6 +14,7 @@ const { API, notifications } = getContext("grid") const imageExtensions = ["png", "tiff", "gif", "raw", "jpg", "jpeg"] + const validExtensions = ValidFileExtensions.map(ext => `.${ext}`).join(", ") let isOpen = false @@ -96,6 +98,7 @@ {value} compact on:change={e => onChange(e.detail)} + extensions={validExtensions} {processFiles} {deleteAttachments} {handleFileTooLarge} diff --git a/packages/shared-core/src/constants.ts b/packages/shared-core/src/constants.ts index 312e69c896..e7c6feb20a 100644 --- a/packages/shared-core/src/constants.ts +++ b/packages/shared-core/src/constants.ts @@ -96,7 +96,6 @@ export enum BuilderSocketEvent { export const SocketSessionTTL = 60 export const ValidQueryNameRegex = /^[^()]*$/ export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g - export const ValidFileExtensions = [ "avif", "css", From 5539ff9c9ce292d844c7d88b4a8384e45310b259 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 27 Oct 2023 16:53:32 +0100 Subject: [PATCH 06/36] Apply valid file types to RowFieldControl and AttackmentField. --- .../src/components/backend/DataTable/RowFieldControl.svelte | 5 ++++- .../client/src/components/app/forms/AttachmentField.svelte | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte index 61b706e28e..9d52cb815e 100644 --- a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte +++ b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte @@ -13,6 +13,7 @@ import { capitalise } from "helpers" import LinkedRowSelector from "components/common/LinkedRowSelector.svelte" import Editor from "../../integration/QueryEditor.svelte" + import { ValidFileExtensions } from "@budibase/shared-core" export let defaultValue export let meta @@ -20,6 +21,8 @@ export let readonly export let error + let validExtensions = ValidFileExtensions.map(ext => `.${ext}`).join(", ") + const resolveTimeStamp = timestamp => { if (!timestamp) { return null @@ -59,7 +62,7 @@ bind:value /> {:else if type === "attachment"} - + {:else if type === "boolean"} {:else if type === "array" && meta.constraints.inclusion.length !== 0} diff --git a/packages/client/src/components/app/forms/AttachmentField.svelte b/packages/client/src/components/app/forms/AttachmentField.svelte index e24115ebc0..2effe607ae 100644 --- a/packages/client/src/components/app/forms/AttachmentField.svelte +++ b/packages/client/src/components/app/forms/AttachmentField.svelte @@ -2,13 +2,14 @@ import Field from "./Field.svelte" import { CoreDropzone } from "@budibase/bbui" import { getContext } from "svelte" + import { ValidFileExtensions } from "@budibase/shared-core" export let field export let label export let disabled = false export let compact = false export let validation - export let extensions + export let extensions = ValidFileExtensions.map(ext => `.${ext}`).join(", ") export let onChange export let maximum = undefined From 6ecf831f028c650d6df68fa54c9fa518e68f70ac Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 27 Oct 2023 17:10:03 +0100 Subject: [PATCH 07/36] Updating pro submodule reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 5ed0ee2aca..4506399e0d 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 5ed0ee2aca9d754d80cd46bae412b24621afa47e +Subproject commit 4506399e0d0297554cacbef1f436884aabdb9741 From f1aa32e4461b604c551915953940a990f0670404 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 27 Oct 2023 17:19:39 +0100 Subject: [PATCH 08/36] Truncate file size on the grid, validate extension in the attachment API. --- packages/bbui/src/Form/Core/Dropzone.svelte | 10 ++++++---- .../backend/DataTable/RowFieldControl.svelte | 5 ++++- .../server/src/api/controllers/static/index.ts | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte index e9ee75bd8b..0b6a9bb94f 100644 --- a/packages/bbui/src/Form/Core/Dropzone.svelte +++ b/packages/bbui/src/Form/Core/Dropzone.svelte @@ -159,8 +159,10 @@ {#if selectedImage.size}
{#if selectedImage.size <= BYTES_IN_MB} - {`${selectedImage.size / BYTES_IN_KB} KB`} - {:else}{`${selectedImage.size / BYTES_IN_MB} MB`}{/if} + {`${(selectedImage.size / BYTES_IN_KB).toFixed(1)} KB`} + {:else}{`${(selectedImage.size / BYTES_IN_MB).toFixed( + 1 + )} MB`}{/if}
{/if} {#if !disabled} @@ -203,8 +205,8 @@ {#if file.size}
{#if file.size <= BYTES_IN_MB} - {`${file.size / BYTES_IN_KB} KB`} - {:else}{`${file.size / BYTES_IN_MB} MB`}{/if} + {`${(file.size / BYTES_IN_KB).toFixed(1)} KB`} + {:else}{`${(file.size / BYTES_IN_MB).toFixed(1)} MB`}{/if}
{/if} {#if !disabled} diff --git a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte index 9d52cb815e..ea1161fe9b 100644 --- a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte +++ b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte @@ -14,6 +14,7 @@ import LinkedRowSelector from "components/common/LinkedRowSelector.svelte" import Editor from "../../integration/QueryEditor.svelte" import { ValidFileExtensions } from "@budibase/shared-core" + import { admin } from "stores/portal" export let defaultValue export let meta @@ -21,7 +22,9 @@ export let readonly export let error - let validExtensions = ValidFileExtensions.map(ext => `.${ext}`).join(", ") + let validExtensions = $admin.cloud + ? ValidFileExtensions.map(ext => `.${ext}`).join(", ") + : "*" const resolveTimeStamp = timestamp => { if (!timestamp) { diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 984cb16c06..e8d403ad12 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -1,3 +1,5 @@ +import { ValidFileExtensions } from "@budibase/shared-core" + require("svelte/register") import { join } from "../../../utilities/centralPath" @@ -17,6 +19,7 @@ import fs from "fs" import sdk from "../../../sdk" import * as pro from "@budibase/pro" import { App, Ctx } from "@budibase/types" +import environment from "../../../environment" const send = require("koa-send") @@ -78,6 +81,17 @@ export const uploadFile = async function (ctx: Ctx) { const uploads = files.map(async (file: any) => { const fileExtension = [...file.name.split(".")].pop() + if ( + !environment.SELF_HOSTED && + !ValidFileExtensions.includes(fileExtension) + ) { + ctx.throw( + 400, + `Invalid file extension. Valid extensions are: ${ValidFileExtensions.join( + ", " + )}` + ) + } // filenames converted to UUIDs so they are unique const processedFileName = `${uuid.v4()}.${fileExtension}` From 84ba840dbcfea6e059c3872999f6ac716783eee5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 27 Oct 2023 17:29:05 +0100 Subject: [PATCH 09/36] Apply valid file type change to AttachmentField in cloud only. --- .../client/src/components/app/forms/AttachmentField.svelte | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/client/src/components/app/forms/AttachmentField.svelte b/packages/client/src/components/app/forms/AttachmentField.svelte index 2effe607ae..e57510c770 100644 --- a/packages/client/src/components/app/forms/AttachmentField.svelte +++ b/packages/client/src/components/app/forms/AttachmentField.svelte @@ -3,13 +3,16 @@ import { CoreDropzone } from "@budibase/bbui" import { getContext } from "svelte" import { ValidFileExtensions } from "@budibase/shared-core" + import { environmentStore } from "../../../stores/index.js" export let field export let label export let disabled = false export let compact = false export let validation - export let extensions = ValidFileExtensions.map(ext => `.${ext}`).join(", ") + export let extensions = $environmentStore.cloud + ? ValidFileExtensions.map(ext => `.${ext}`).join(", ") + : "*" export let onChange export let maximum = undefined From d4929ea3b61e5774aabf4f7fe758a66b5e872f21 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 27 Oct 2023 18:03:06 +0100 Subject: [PATCH 10/36] Fixing an issue where unpublished apps with custom roles, when used in groups would cause users to be unable to login. --- packages/backend-core/src/security/roles.ts | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index b05cf79c8c..02421fd1d0 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -122,7 +122,9 @@ export async function roleToNumber(id?: string) { if (isBuiltin(id)) { return builtinRoleToNumber(id) } - const hierarchy = (await getUserRoleHierarchy(id)) as RoleDoc[] + const hierarchy = (await getUserRoleHierarchy(id, { + defaultPublic: true, + })) as RoleDoc[] for (let role of hierarchy) { if (isBuiltin(role?.inherits)) { return builtinRoleToNumber(role.inherits) + 1 @@ -177,7 +179,7 @@ export async function getRole( role = Object.assign(role, dbRole) // finalise the ID role._id = getExternalRoleID(role._id, role.version) - } catch (err) { + } catch (err: any) { if (!isBuiltin(roleId) && opts?.defaultPublic) { return cloneDeep(BUILTIN_ROLES.PUBLIC) } @@ -192,12 +194,15 @@ export async function getRole( /** * Simple function to get all the roles based on the top level user role ID. */ -async function getAllUserRoles(userRoleId?: string): Promise { +async function getAllUserRoles( + userRoleId?: string, + opts?: { defaultPublic?: boolean } +): Promise { // admins have access to all roles if (userRoleId === BUILTIN_IDS.ADMIN) { return getAllRoles() } - let currentRole = await getRole(userRoleId) + let currentRole = await getRole(userRoleId, opts) let roles = currentRole ? [currentRole] : [] let roleIds = [userRoleId] // get all the inherited roles @@ -226,12 +231,16 @@ export async function getUserRoleIdHierarchy( * Returns an ordered array of the user's inherited role IDs, this can be used * to determine if a user can access something that requires a specific role. * @param userRoleId The user's role ID, this can be found in their access token. + * @param opts optional - if want to default to public use this. * @returns returns an ordered array of the roles, with the first being their * highest level of access and the last being the lowest level. */ -export async function getUserRoleHierarchy(userRoleId?: string) { +export async function getUserRoleHierarchy( + userRoleId?: string, + opts?: { defaultPublic?: boolean } +) { // special case, if they don't have a role then they are a public user - return getAllUserRoles(userRoleId) + return getAllUserRoles(userRoleId, opts) } // this function checks that the provided permissions are in an array format From 330059991e0d6f6f0cf0af5c3ad3199c044b5509 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 27 Oct 2023 18:04:28 +0100 Subject: [PATCH 11/36] Removing any. --- packages/backend-core/src/security/roles.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 02421fd1d0..0d33031de5 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -179,7 +179,7 @@ export async function getRole( role = Object.assign(role, dbRole) // finalise the ID role._id = getExternalRoleID(role._id, role.version) - } catch (err: any) { + } catch (err) { if (!isBuiltin(roleId) && opts?.defaultPublic) { return cloneDeep(BUILTIN_ROLES.PUBLIC) } From 887383bdb6a4b54d7ab747571415f112f83f2e14 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Fri, 27 Oct 2023 18:12:21 +0100 Subject: [PATCH 12/36] Update README.md Removing dead link. --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index 9deb16cd4f..7827d4e48a 100644 --- a/README.md +++ b/README.md @@ -126,13 +126,6 @@ You can learn more about the Budibase API at the following places: - [Build an app with Budibase and Next.js](https://budibase.com/blog/building-a-crud-app-with-budibase-and-next.js/) -

- Budibase data -

-

- -


- ## 🏁 Get started Deploy Budibase self-hosted in your existing infrastructure, using Docker, Kubernetes, and Digital Ocean. From 1221808c67c65989b24a7540ab0b31f44989e9ff Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Mon, 30 Oct 2023 09:00:20 +0100 Subject: [PATCH 13/36] Updata pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 39bff12817..3820c0c93a 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 39bff1281715c647f5d0c1db9bbf1d53c9fd4fc6 +Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8 From 725e3aa4ef8431990fa9a54d9857f12f95da9a5d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 10:11:06 +0100 Subject: [PATCH 14/36] Use image v2 on build:docker --- packages/server/package.json | 2 +- packages/worker/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 4a858f3be9..b89fe86a84 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -18,7 +18,7 @@ "test": "bash scripts/test.sh", "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit", "test:watch": "jest --watch", - "build:docker": "yarn build && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION", + "build:docker": "yarn nx build && docker build ../.. -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2", "run:docker": "node dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", "dev:stack:up": "node scripts/dev/manage.js up", diff --git a/packages/worker/package.json b/packages/worker/package.json index 205bf3309a..dd847e6df4 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -20,7 +20,7 @@ "run:docker": "node dist/index.js", "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", - "build:docker": "yarn build && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION", + "build:docker": "yarn nx build && docker build ../.. -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2", "dev:stack:init": "node ./scripts/dev/manage.js init", "dev:builder": "npm run dev:stack:init && nodemon", "dev:built": "yarn run dev:stack:init && yarn run run:docker", From b542040ad4fa3d379842348633fbe15546c816b0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 10:27:59 +0100 Subject: [PATCH 15/36] Use v2 for single image --- .github/workflows/release-singleimage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml index f7f87f6e4c..4d35916f4d 100644 --- a/.github/workflows/release-singleimage.yml +++ b/.github/workflows/release-singleimage.yml @@ -67,7 +67,7 @@ jobs: push: true platforms: linux/amd64,linux/arm64 tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile + file: ./hosting/single/Dockerfile.v2 - name: Tag and release Budibase Azure App Service docker image uses: docker/build-push-action@v2 with: @@ -76,4 +76,4 @@ jobs: platforms: linux/amd64 build-args: TARGETBUILD=aas tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile + file: ./hosting/single/Dockerfile.v2 From b0ef79bbd9f9a7ab3a8bbe2f5513426ff2b61362 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 11:36:32 +0100 Subject: [PATCH 16/36] Build for both amd and arm platforms --- packages/server/package.json | 2 +- packages/worker/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index b89fe86a84..c37959d33d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -18,7 +18,7 @@ "test": "bash scripts/test.sh", "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit", "test:watch": "jest --watch", - "build:docker": "yarn nx build && docker build ../.. -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2", + "build:docker": "yarn nx build && docker buildx build ../.. -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2 --platform linux/amd64,linux/arm64", "run:docker": "node dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", "dev:stack:up": "node scripts/dev/manage.js up", diff --git a/packages/worker/package.json b/packages/worker/package.json index dd847e6df4..a391db533b 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -20,7 +20,7 @@ "run:docker": "node dist/index.js", "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", - "build:docker": "yarn nx build && docker build ../.. -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2", + "build:docker": "yarn nx build && docker buildx build ../.. -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2 --platform linux/amd64,linux/arm64", "dev:stack:init": "node ./scripts/dev/manage.js init", "dev:builder": "npm run dev:stack:init && nodemon", "dev:built": "yarn run dev:stack:init && yarn run run:docker", From 6267d9b601e7b350388b5adb139e6ef6c9cb2b16 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 11:39:25 +0100 Subject: [PATCH 17/36] Test building images --- .github/workflows/budibase_ci.yml | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 77867c8617..3ec87f7244 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -42,6 +42,39 @@ jobs: - run: yarn --frozen-lockfile - run: yarn lint + test-release-images: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + fetch-depth: 0 + + - uses: actions/setup-node@v1 + with: + node-version: 18.x + + - run: yarn install --frozen-lockfile + - name: Update versions + run: ./scripts/updateVersions.sh + - run: yarn lint + - run: yarn build + - run: yarn build:sdk + + - name: "Get Current tag" + id: currenttag + run: | + version=$(./scripts/getCurrentVersion.sh) + echo "Using tag $version" + echo "version=$version" >> "$GITHUB_OUTPUT" + + - name: Build/release Docker images + run: | + yarn lerna run --stream build:docker + env: + BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }} + build: runs-on: ubuntu-latest steps: From ac67a17b9393979b65f8bf4335a4f83232872e97 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 11:45:05 +0100 Subject: [PATCH 18/36] Use buildx --- .github/workflows/budibase_ci.yml | 6 +++++- .github/workflows/release-master.yml | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 3ec87f7244..13245a7fa1 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -51,9 +51,10 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} fetch-depth: 0 - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 with: node-version: 18.x + cache: "yarn" - run: yarn install --frozen-lockfile - name: Update versions @@ -69,6 +70,9 @@ jobs: echo "Using tag $version" echo "version=$version" >> "$GITHUB_OUTPUT" + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 - name: Build/release Docker images run: | yarn lerna run --stream build:docker diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 9ab8530341..4c5a3f4a1e 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -63,6 +63,9 @@ jobs: echo "Using tag $version" echo "version=$version" >> "$GITHUB_OUTPUT" + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 - name: Build/release Docker images run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD From 24eed537edc45d8febb9c119f5a4f54f7f296558 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 12:00:55 +0100 Subject: [PATCH 19/36] Fix timeouts --- packages/server/Dockerfile | 2 +- packages/worker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index e1b3b208c7..ea4c5b217a 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -38,7 +38,7 @@ RUN apt update && apt upgrade -y \ COPY package.json . COPY dist/yarn.lock . -RUN yarn install --production=true \ +RUN yarn install --production=true --network-timeout 1000000 \ # Remove unneeded data from file system to reduce image size && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \ && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile index 4230ee86f8..50f1bb78b9 100644 --- a/packages/worker/Dockerfile +++ b/packages/worker/Dockerfile @@ -14,7 +14,7 @@ RUN yarn global add pm2 COPY package.json . COPY dist/yarn.lock . -RUN yarn install --production=true +RUN yarn install --production=true --network-timeout 1000000 # Remove unneeded data from file system to reduce image size RUN apk del .gyp \ && yarn cache clean From d9c34f3f465476291b2cad1b539e0e5e3ceb649e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 12:40:05 +0100 Subject: [PATCH 20/36] Remove build docker in ci pipelines --- .github/workflows/budibase_ci.yml | 37 ------------------------------- 1 file changed, 37 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 13245a7fa1..77867c8617 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -42,43 +42,6 @@ jobs: - run: yarn --frozen-lockfile - run: yarn lint - test-release-images: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - uses: actions/setup-node@v3 - with: - node-version: 18.x - cache: "yarn" - - - run: yarn install --frozen-lockfile - - name: Update versions - run: ./scripts/updateVersions.sh - - run: yarn lint - - run: yarn build - - run: yarn build:sdk - - - name: "Get Current tag" - id: currenttag - run: | - version=$(./scripts/getCurrentVersion.sh) - echo "Using tag $version" - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - name: Build/release Docker images - run: | - yarn lerna run --stream build:docker - env: - BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }} - build: runs-on: ubuntu-latest steps: From 9229ab6896c81c0bc7d545ba3bb3ee5c70840a87 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 30 Oct 2023 12:12:56 +0000 Subject: [PATCH 21/36] Bump version to 2.12.0 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 384473120b..3179bc3b2e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.45", + "version": "2.12.0", "npmClient": "yarn", "packages": [ "packages/*" From a141598a1cb269ab5deb2b939a861a753d8b236d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 14:26:28 +0100 Subject: [PATCH 22/36] Remove unused scripts --- package.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/package.json b/package.json index d3f4903e6c..03257d2d06 100644 --- a/package.json +++ b/package.json @@ -56,8 +56,6 @@ "build:specs": "lerna run --stream specs", "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", - "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -", - "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", From 6631a7a11d8704a075bec9fc7516ccfb08aac141 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 14:34:55 +0100 Subject: [PATCH 23/36] Push docker images via docker/build-push-action --- .github/workflows/release-master.yml | 39 +++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 4c5a3f4a1e..a8af6a4da1 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -66,14 +66,47 @@ jobs: - name: Setup Docker Buildx id: buildx uses: docker/setup-buildx-action@v1 - - name: Build/release Docker images + + - name: Docker login run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - yarn build:docker env: DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }} + + - name: Build docker (budibase/worker) + uses: docker/build-push-action@v5 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64 + build-args: | + BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} + tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} + file: ./packages/worker/Dockerfile.v2 + cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest + cache-to: type=inline + env: + IMAGE_NAME: budibase/worker + IMAGE_TAG: ${{ inputs.image_tag }} + BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} + + - name: Build docker (budibase/apps) + uses: docker/build-push-action@v5 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64 + build-args: | + BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} + tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} + file: ./packages/server/Dockerfile.v2 + cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest + cache-to: type=inline + env: + IMAGE_NAME: budibase/apps + IMAGE_TAG: ${{ inputs.image_tag }} + BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} release-helm-chart: needs: [release-images] From 0cc978f86050ffb40082013fb788fd07e81fb9b5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 14:37:33 +0100 Subject: [PATCH 24/36] Push proxy dockerfile --- .github/workflows/release-master.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index a8af6a4da1..4a89d8462f 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -74,7 +74,7 @@ jobs: DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - - name: Build docker (budibase/worker) + - name: Build worker docker uses: docker/build-push-action@v5 with: context: . @@ -91,7 +91,7 @@ jobs: IMAGE_TAG: ${{ inputs.image_tag }} BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} - - name: Build docker (budibase/apps) + - name: Build server docker uses: docker/build-push-action@v5 with: context: . @@ -108,6 +108,19 @@ jobs: IMAGE_TAG: ${{ inputs.image_tag }} BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} + - name: Build proxy docker + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} + file: ./hosting/proxy/Dockerfile + cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest + cache-to: type=inline + env: + IMAGE_NAME: proxy-service + IMAGE_TAG: ${{ inputs.image_tag }} + release-helm-chart: needs: [release-images] runs-on: ubuntu-latest From a80ea2f2c05747fe4607e1bcafac5506b6d5b4f7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 14:38:26 +0100 Subject: [PATCH 25/36] Clean scripts --- package.json | 2 -- packages/server/package.json | 1 - packages/worker/package.json | 1 - 3 files changed, 4 deletions(-) diff --git a/package.json b/package.json index 03257d2d06..417fb31e0e 100644 --- a/package.json +++ b/package.json @@ -54,8 +54,6 @@ "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"", "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint", "build:specs": "lerna run --stream specs", - "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", - "build:docker:proxy": "docker build hosting/proxy -t proxy-service", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", diff --git a/packages/server/package.json b/packages/server/package.json index c37959d33d..c845f7889d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -18,7 +18,6 @@ "test": "bash scripts/test.sh", "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit", "test:watch": "jest --watch", - "build:docker": "yarn nx build && docker buildx build ../.. -t app-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2 --platform linux/amd64,linux/arm64", "run:docker": "node dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", "dev:stack:up": "node scripts/dev/manage.js up", diff --git a/packages/worker/package.json b/packages/worker/package.json index a391db533b..ec86575395 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -20,7 +20,6 @@ "run:docker": "node dist/index.js", "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", - "build:docker": "yarn nx build && docker buildx build ../.. -t worker-service --label version=$BUDIBASE_RELEASE_VERSION --build-arg BUDIBASE_VERSION=$BUDIBASE_RELEASE_VERSION -f Dockerfile.v2 --platform linux/amd64,linux/arm64", "dev:stack:init": "node ./scripts/dev/manage.js init", "dev:builder": "npm run dev:stack:init && nodemon", "dev:built": "yarn run dev:stack:init && yarn run run:docker", From 0dd655e75dec40415c0e660af74dc2ee72c0802b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 14:39:03 +0100 Subject: [PATCH 26/36] Remove sh --- .github/workflows/release-master.yml | 2 +- hosting/scripts/linux/release-to-docker-hub.sh | 18 ------------------ 2 files changed, 1 insertion(+), 19 deletions(-) delete mode 100755 hosting/scripts/linux/release-to-docker-hub.sh diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 4a89d8462f..24d71a35a6 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -118,7 +118,7 @@ jobs: cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest cache-to: type=inline env: - IMAGE_NAME: proxy-service + IMAGE_NAME: budibase/proxy-service IMAGE_TAG: ${{ inputs.image_tag }} release-helm-chart: diff --git a/hosting/scripts/linux/release-to-docker-hub.sh b/hosting/scripts/linux/release-to-docker-hub.sh deleted file mode 100755 index 599a10f914..0000000000 --- a/hosting/scripts/linux/release-to-docker-hub.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -tag=$1 - -if [[ ! "$tag" ]]; then - echo "No tag present. You must pass a tag to this script" - exit 1 -fi - -echo "Tagging images with tag: $tag" - -docker tag proxy-service budibase/proxy:$tag -docker tag app-service budibase/apps:$tag -docker tag worker-service budibase/worker:$tag - -docker push --all-tags budibase/apps -docker push --all-tags budibase/worker -docker push --all-tags budibase/proxy From 469114823600edb7f156fae9eef9514efed9b981 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 14:44:48 +0100 Subject: [PATCH 27/36] Cache yarn --- .github/workflows/release-master.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 24d71a35a6..c2bee8da40 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -36,6 +36,7 @@ jobs: - uses: actions/setup-node@v1 with: node-version: 18.x + cache: yarn - run: yarn install --frozen-lockfile - name: Update versions From b74f11e0e2d23a67c6c6f30d404b2f719d4cbd32 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 14:53:14 +0100 Subject: [PATCH 28/36] Fix proxy image name --- .github/workflows/release-master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index c2bee8da40..5d67101f4e 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -119,7 +119,7 @@ jobs: cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest cache-to: type=inline env: - IMAGE_NAME: budibase/proxy-service + IMAGE_NAME: budibase/proxy IMAGE_TAG: ${{ inputs.image_tag }} release-helm-chart: From ac3c9a374cbefa12e51e7dc2354128cc344f5240 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 15:02:36 +0100 Subject: [PATCH 29/36] Fix image tag --- .github/workflows/release-master.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 5d67101f4e..e0adddc6a8 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -89,7 +89,7 @@ jobs: cache-to: type=inline env: IMAGE_NAME: budibase/worker - IMAGE_TAG: ${{ inputs.image_tag }} + IMAGE_TAG: ${{ steps.currenttag.outputs.version }} BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} - name: Build server docker @@ -106,7 +106,7 @@ jobs: cache-to: type=inline env: IMAGE_NAME: budibase/apps - IMAGE_TAG: ${{ inputs.image_tag }} + IMAGE_TAG: ${{ steps.currenttag.outputs.version }} BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} - name: Build proxy docker @@ -120,7 +120,7 @@ jobs: cache-to: type=inline env: IMAGE_NAME: budibase/proxy - IMAGE_TAG: ${{ inputs.image_tag }} + IMAGE_TAG: ${{ steps.currenttag.outputs.version }} release-helm-chart: needs: [release-images] From 436d6a1585a9d904b0339e7331d24a43e3ba135f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 30 Oct 2023 14:39:12 +0000 Subject: [PATCH 30/36] Revert frontend changes to filter out file extensions in the upload box. --- packages/bbui/src/Form/Dropzone.svelte | 2 -- .../components/backend/DataTable/RowFieldControl.svelte | 8 +------- .../src/components/app/forms/AttachmentField.svelte | 6 +----- .../src/components/grid/cells/AttachmentCell.svelte | 3 --- 4 files changed, 2 insertions(+), 17 deletions(-) diff --git a/packages/bbui/src/Form/Dropzone.svelte b/packages/bbui/src/Form/Dropzone.svelte index 94742ea08d..2a6fa1c57b 100644 --- a/packages/bbui/src/Form/Dropzone.svelte +++ b/packages/bbui/src/Form/Dropzone.svelte @@ -17,7 +17,6 @@ export let fileTags = [] export let maximum = undefined export let compact = false - export let extensions = undefined const dispatch = createEventDispatcher() const onChange = e => { @@ -40,7 +39,6 @@ {fileTags} {maximum} {compact} - {extensions} on:change={onChange} /> diff --git a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte index ea1161fe9b..61b706e28e 100644 --- a/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte +++ b/packages/builder/src/components/backend/DataTable/RowFieldControl.svelte @@ -13,8 +13,6 @@ import { capitalise } from "helpers" import LinkedRowSelector from "components/common/LinkedRowSelector.svelte" import Editor from "../../integration/QueryEditor.svelte" - import { ValidFileExtensions } from "@budibase/shared-core" - import { admin } from "stores/portal" export let defaultValue export let meta @@ -22,10 +20,6 @@ export let readonly export let error - let validExtensions = $admin.cloud - ? ValidFileExtensions.map(ext => `.${ext}`).join(", ") - : "*" - const resolveTimeStamp = timestamp => { if (!timestamp) { return null @@ -65,7 +59,7 @@ bind:value /> {:else if type === "attachment"} - + {:else if type === "boolean"} {:else if type === "array" && meta.constraints.inclusion.length !== 0} diff --git a/packages/client/src/components/app/forms/AttachmentField.svelte b/packages/client/src/components/app/forms/AttachmentField.svelte index e57510c770..e24115ebc0 100644 --- a/packages/client/src/components/app/forms/AttachmentField.svelte +++ b/packages/client/src/components/app/forms/AttachmentField.svelte @@ -2,17 +2,13 @@ import Field from "./Field.svelte" import { CoreDropzone } from "@budibase/bbui" import { getContext } from "svelte" - import { ValidFileExtensions } from "@budibase/shared-core" - import { environmentStore } from "../../../stores/index.js" export let field export let label export let disabled = false export let compact = false export let validation - export let extensions = $environmentStore.cloud - ? ValidFileExtensions.map(ext => `.${ext}`).join(", ") - : "*" + export let extensions export let onChange export let maximum = undefined diff --git a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte index 4e2a6025e5..a27c31bbe5 100644 --- a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte @@ -2,7 +2,6 @@ import { onMount } from "svelte" import { getContext } from "svelte" import { Dropzone } from "@budibase/bbui" - import { ValidFileExtensions } from "@budibase/shared-core" export let value export let focused = false @@ -14,7 +13,6 @@ const { API, notifications } = getContext("grid") const imageExtensions = ["png", "tiff", "gif", "raw", "jpg", "jpeg"] - const validExtensions = ValidFileExtensions.map(ext => `.${ext}`).join(", ") let isOpen = false @@ -98,7 +96,6 @@ {value} compact on:change={e => onChange(e.detail)} - extensions={validExtensions} {processFiles} {deleteAttachments} {handleFileTooLarge} From 344256a80588543427987e93b32c67d25766783e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 15:57:40 +0100 Subject: [PATCH 31/36] Fix proxy build --- .github/workflows/release-master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index e0adddc6a8..e1feaa1e7c 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -112,7 +112,7 @@ jobs: - name: Build proxy docker uses: docker/build-push-action@v5 with: - context: . + context: ./hosting/proxy push: true tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} file: ./hosting/proxy/Dockerfile From 674f40a06ee49dc1ffa7d56e8b79466e8dab7903 Mon Sep 17 00:00:00 2001 From: Samuel-Martineau Date: Fri, 27 Oct 2023 17:21:30 -0400 Subject: [PATCH 32/36] Fix how attachment URLs are handled --- packages/server/src/utilities/rowProcessor/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index cf3875b2ea..604f872c81 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -241,7 +241,7 @@ export async function outputProcessing( continue } row[property].forEach((attachment: RowAttachment) => { - attachment.url = objectStore.getAppFileUrl(attachment.key) + attachment.url ??= objectStore.getAppFileUrl(attachment.key) }) } } else if ( From b3a4a921aa195aacf68c5dd8dfd84f27b45e1cf1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 30 Oct 2023 16:47:47 +0100 Subject: [PATCH 33/36] Build multi platform proxy --- .github/workflows/release-master.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index e1feaa1e7c..df25182cd6 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -114,6 +114,7 @@ jobs: with: context: ./hosting/proxy push: true + platforms: linux/amd64,linux/arm64 tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} file: ./hosting/proxy/Dockerfile cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest From af59039d1c8a5735f0325c73c1c0da846c845c31 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 30 Oct 2023 16:46:27 +0000 Subject: [PATCH 34/36] Add tests for attachment processing endpoint. --- .../src/api/controllers/static/index.ts | 100 ++++++++++-------- .../src/api/routes/tests/attachment.spec.ts | 49 +++++++++ .../src/api/routes/tests/static.spec.js | 8 +- .../src/api/routes/tests/webhook.spec.ts | 8 +- .../integrations/tests/googlesheets.spec.ts | 9 +- .../src/tests/utilities/TestConfiguration.ts | 47 ++++---- .../src/tests/utilities/api/attachment.ts | 35 ++++++ .../server/src/tests/utilities/api/index.ts | 3 + packages/types/src/api/web/app/attachment.ts | 9 ++ packages/types/src/api/web/app/index.ts | 1 + 10 files changed, 199 insertions(+), 70 deletions(-) create mode 100644 packages/server/src/api/routes/tests/attachment.spec.ts create mode 100644 packages/server/src/tests/utilities/api/attachment.ts create mode 100644 packages/types/src/api/web/app/attachment.ts diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index e8d403ad12..8fbc0db910 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -13,35 +13,21 @@ import { } from "../../../utilities/fileSystem" import env from "../../../environment" import { DocumentType } from "../../../db/utils" -import { context, objectStore, utils, configs } from "@budibase/backend-core" +import { + context, + objectStore, + utils, + configs, + BadRequestError, +} from "@budibase/backend-core" import AWS from "aws-sdk" import fs from "fs" import sdk from "../../../sdk" import * as pro from "@budibase/pro" -import { App, Ctx } from "@budibase/types" -import environment from "../../../environment" +import { App, Ctx, ProcessAttachmentResponse, Upload } from "@budibase/types" const send = require("koa-send") -async function prepareUpload({ s3Key, bucket, metadata, file }: any) { - const response = await objectStore.upload({ - bucket, - metadata, - filename: s3Key, - path: file.path, - type: file.type, - }) - - // don't store a URL, work this out on the way out as the URL could change - return { - size: file.size, - name: file.name, - url: objectStore.getAppFileUrl(s3Key), - extension: [...file.name.split(".")].pop(), - key: response.Key, - } -} - export const toggleBetaUiFeature = async function (ctx: Ctx) { const cookieName = `beta:${ctx.params.feature}` @@ -75,34 +61,58 @@ export const serveBuilder = async function (ctx: Ctx) { await send(ctx, ctx.file, { root: builderPath }) } -export const uploadFile = async function (ctx: Ctx) { +export const uploadFile = async function ( + ctx: Ctx<{}, ProcessAttachmentResponse> +) { const file = ctx.request?.files?.file + if (!file) { + throw new BadRequestError("No file provided") + } + let files = file && Array.isArray(file) ? Array.from(file) : [file] - const uploads = files.map(async (file: any) => { - const fileExtension = [...file.name.split(".")].pop() - if ( - !environment.SELF_HOSTED && - !ValidFileExtensions.includes(fileExtension) - ) { - ctx.throw( - 400, - `Invalid file extension. Valid extensions are: ${ValidFileExtensions.join( - ", " - )}` - ) - } - // filenames converted to UUIDs so they are unique - const processedFileName = `${uuid.v4()}.${fileExtension}` + ctx.body = await Promise.all( + files.map(async file => { + if (!file.name) { + throw new BadRequestError( + "Attempted to upload a file without a filename" + ) + } - return prepareUpload({ - file, - s3Key: `${context.getProdAppId()}/attachments/${processedFileName}`, - bucket: ObjectStoreBuckets.APPS, + const extension = [...file.name.split(".")].pop() + if (!extension) { + throw new BadRequestError( + `File "${file.name}" has no extension, an extension is required to upload a file` + ) + } + + if (!env.SELF_HOSTED && !ValidFileExtensions.includes(extension)) { + throw new BadRequestError( + `File "${file.name}" has an invalid extension: "${extension}"` + ) + } + + // filenames converted to UUIDs so they are unique + const processedFileName = `${uuid.v4()}.${extension}` + + const s3Key = `${context.getProdAppId()}/attachments/${processedFileName}` + + const response = await objectStore.upload({ + bucket: ObjectStoreBuckets.APPS, + filename: s3Key, + path: file.path, + type: file.type, + }) + + return { + size: file.size, + name: file.name, + url: objectStore.getAppFileUrl(s3Key), + extension, + key: response.Key, + } }) - }) - - ctx.body = await Promise.all(uploads) + ) } export const deleteObjects = async function (ctx: Ctx) { diff --git a/packages/server/src/api/routes/tests/attachment.spec.ts b/packages/server/src/api/routes/tests/attachment.spec.ts new file mode 100644 index 0000000000..14d2e845f6 --- /dev/null +++ b/packages/server/src/api/routes/tests/attachment.spec.ts @@ -0,0 +1,49 @@ +import * as setup from "./utilities" +import { APIError } from "@budibase/types" + +describe("/api/applications/:appId/sync", () => { + let config = setup.getConfig() + + afterAll(setup.afterAll) + beforeAll(async () => { + await config.init() + }) + + describe("/api/attachments/process", () => { + it("should accept an image file upload", async () => { + let resp = await config.api.attachment.process( + "1px.jpg", + Buffer.from([0]) + ) + expect(resp.length).toBe(1) + + let upload = resp[0] + expect(upload.url.endsWith(".jpg")).toBe(true) + expect(upload.extension).toBe("jpg") + expect(upload.size).toBe(1) + expect(upload.name).toBe("1px.jpg") + }) + + it("should reject an upload with a malicious file extension", async () => { + await config.withEnv({ SELF_HOSTED: undefined }, async () => { + let resp = (await config.api.attachment.process( + "ohno.exe", + Buffer.from([0]), + { expectStatus: 400 } + )) as unknown as APIError + expect(resp.message).toContain("invalid extension") + }) + }) + + it("should reject an upload with no file", async () => { + let resp = (await config.api.attachment.process( + undefined as any, + undefined as any, + { + expectStatus: 400, + } + )) as unknown as APIError + expect(resp.message).toContain("No file provided") + }) + }) +}) diff --git a/packages/server/src/api/routes/tests/static.spec.js b/packages/server/src/api/routes/tests/static.spec.js index 13d963d057..a28d9ecd79 100644 --- a/packages/server/src/api/routes/tests/static.spec.js +++ b/packages/server/src/api/routes/tests/static.spec.js @@ -5,11 +5,15 @@ describe("/static", () => { let request = setup.getRequest() let config = setup.getConfig() let app + let cleanupEnv - afterAll(setup.afterAll) + afterAll(() => { + setup.afterAll() + cleanupEnv() + }) beforeAll(async () => { - config.modeSelf() + cleanupEnv = config.setEnv({ SELF_HOSTED: "true" }) app = await config.init() }) diff --git a/packages/server/src/api/routes/tests/webhook.spec.ts b/packages/server/src/api/routes/tests/webhook.spec.ts index e7046d07c8..118bfca95f 100644 --- a/packages/server/src/api/routes/tests/webhook.spec.ts +++ b/packages/server/src/api/routes/tests/webhook.spec.ts @@ -8,11 +8,15 @@ describe("/webhooks", () => { let request = setup.getRequest() let config = setup.getConfig() let webhook: Webhook + let cleanupEnv: () => void - afterAll(setup.afterAll) + afterAll(() => { + setup.afterAll() + cleanupEnv() + }) const setupTest = async () => { - config.modeSelf() + cleanupEnv = config.setEnv({ SELF_HOSTED: "true" }) await config.init() const autoConfig = basicAutomation() autoConfig.definition.trigger.schema = { diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 748baddc39..a38c6bda45 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -35,13 +35,18 @@ import { FieldType, Table, TableSchema } from "@budibase/types" describe("Google Sheets Integration", () => { let integration: any, config = new TestConfiguration() + let cleanupEnv: () => void beforeAll(() => { - config.setGoogleAuth("test") + cleanupEnv = config.setEnv({ + GOOGLE_CLIENT_ID: "test", + GOOGLE_CLIENT_SECRET: "test", + }) }) afterAll(async () => { - await config.end() + cleanupEnv() + config.end() }) beforeEach(async () => { diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index cec8c8aa12..5096b054a6 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -58,6 +58,7 @@ import { } from "@budibase/types" import API from "./api" +import { cloneDeep } from "lodash" type DefaultUserValues = { globalUserId: string @@ -188,30 +189,38 @@ class TestConfiguration { } } - // MODES - setMultiTenancy = (value: boolean) => { - env._set("MULTI_TENANCY", value) - coreEnv._set("MULTI_TENANCY", value) + async withEnv(newEnvVars: Partial, f: () => Promise) { + let cleanup = this.setEnv(newEnvVars) + try { + await f() + } finally { + cleanup() + } } - setSelfHosted = (value: boolean) => { - env._set("SELF_HOSTED", value) - coreEnv._set("SELF_HOSTED", value) - } + /* + * Sets the environment variables to the given values and returns a function + * that can be called to reset the environment variables to their original values. + */ + setEnv(newEnvVars: Partial): () => void { + const oldEnv = cloneDeep(env) + const oldCoreEnv = cloneDeep(coreEnv) - setGoogleAuth = (value: string) => { - env._set("GOOGLE_CLIENT_ID", value) - env._set("GOOGLE_CLIENT_SECRET", value) - coreEnv._set("GOOGLE_CLIENT_ID", value) - coreEnv._set("GOOGLE_CLIENT_SECRET", value) - } + let key: keyof typeof newEnvVars + for (key in newEnvVars) { + env._set(key, newEnvVars[key]) + coreEnv._set(key, newEnvVars[key]) + } - modeCloud = () => { - this.setSelfHosted(false) - } + return () => { + for (const [key, value] of Object.entries(oldEnv)) { + env._set(key, value) + } - modeSelf = () => { - this.setSelfHosted(true) + for (const [key, value] of Object.entries(oldCoreEnv)) { + coreEnv._set(key, value) + } + } } // UTILS diff --git a/packages/server/src/tests/utilities/api/attachment.ts b/packages/server/src/tests/utilities/api/attachment.ts new file mode 100644 index 0000000000..a466f1a67e --- /dev/null +++ b/packages/server/src/tests/utilities/api/attachment.ts @@ -0,0 +1,35 @@ +import { + APIError, + Datasource, + ProcessAttachmentResponse, +} from "@budibase/types" +import TestConfiguration from "../TestConfiguration" +import { TestAPI } from "./base" +import fs from "fs" + +export class AttachmentAPI extends TestAPI { + constructor(config: TestConfiguration) { + super(config) + } + + process = async ( + name: string, + file: Buffer | fs.ReadStream | string, + { expectStatus } = { expectStatus: 200 } + ): Promise => { + const result = await this.request + .post(`/api/attachments/process`) + .attach("file", file, name) + .set(this.config.defaultHeaders()) + + if (result.statusCode !== expectStatus) { + throw new Error( + `Expected status ${expectStatus} but got ${ + result.statusCode + }, body: ${JSON.stringify(result.body)}` + ) + } + + return result.body + } +} diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts index fce8237760..30ef7c478d 100644 --- a/packages/server/src/tests/utilities/api/index.ts +++ b/packages/server/src/tests/utilities/api/index.ts @@ -7,6 +7,7 @@ import { DatasourceAPI } from "./datasource" import { LegacyViewAPI } from "./legacyView" import { ScreenAPI } from "./screen" import { ApplicationAPI } from "./application" +import { AttachmentAPI } from "./attachment" export default class API { table: TableAPI @@ -17,6 +18,7 @@ export default class API { datasource: DatasourceAPI screen: ScreenAPI application: ApplicationAPI + attachment: AttachmentAPI constructor(config: TestConfiguration) { this.table = new TableAPI(config) @@ -27,5 +29,6 @@ export default class API { this.datasource = new DatasourceAPI(config) this.screen = new ScreenAPI(config) this.application = new ApplicationAPI(config) + this.attachment = new AttachmentAPI(config) } } diff --git a/packages/types/src/api/web/app/attachment.ts b/packages/types/src/api/web/app/attachment.ts new file mode 100644 index 0000000000..792bdf3885 --- /dev/null +++ b/packages/types/src/api/web/app/attachment.ts @@ -0,0 +1,9 @@ +export interface Upload { + size: number + name: string + url: string + extension: string + key: string +} + +export type ProcessAttachmentResponse = Upload[] diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts index 276d7fa7c1..f5b876009b 100644 --- a/packages/types/src/api/web/app/index.ts +++ b/packages/types/src/api/web/app/index.ts @@ -5,3 +5,4 @@ export * from "./view" export * from "./rows" export * from "./table" export * from "./permission" +export * from "./attachment" From db398a839e056c1db9ecaf8176291f07b72d2831 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 30 Oct 2023 16:54:07 +0000 Subject: [PATCH 35/36] Bump version to 2.12.1 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 3179bc3b2e..6df4a4c4cd 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.12.0", + "version": "2.12.1", "npmClient": "yarn", "packages": [ "packages/*" From ca9491ce67694db85ff941f639310b5b1c9be56a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 30 Oct 2023 16:55:57 +0000 Subject: [PATCH 36/36] Surface error message from attachments API to user. --- packages/builder/src/components/common/Dropzone.svelte | 2 +- .../src/components/grid/cells/AttachmentCell.svelte | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/common/Dropzone.svelte b/packages/builder/src/components/common/Dropzone.svelte index fd2359fd91..daa6ad1807 100644 --- a/packages/builder/src/components/common/Dropzone.svelte +++ b/packages/builder/src/components/common/Dropzone.svelte @@ -23,7 +23,7 @@ try { return await API.uploadBuilderAttachment(data) } catch (error) { - notifications.error("Failed to upload attachment") + notifications.error(error.message || "Failed to upload attachment") return [] } } diff --git a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte index a27c31bbe5..fc0001d55e 100644 --- a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte @@ -55,7 +55,7 @@ try { return await API.uploadBuilderAttachment(data) } catch (error) { - $notifications.error("Failed to upload attachment") + $notifications.error(error.message || "Failed to upload attachment") return [] } }