diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 634bce18ac..365765ccbb 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -55,7 +55,7 @@ http { set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com"; set $csp_object "object-src 'none'"; set $csp_base_uri "base-uri 'self'"; - set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibase.qa https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com"; + set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com"; set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com https://js.intercomcdn.com https://fonts.intercomcdn.com"; set $csp_frame "frame-src 'self' https:"; set $csp_img "img-src http: https: data: blob:"; diff --git a/lerna.json b/lerna.json index be8d247eb1..b748b3fa36 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.10.12-alpha.10", + "version": "2.10.12-alpha.14", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts index 309f0fd159..758fd6bf9a 100644 --- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts @@ -86,8 +86,8 @@ export const useAuditLogs = () => { return useFeature(Feature.AUDIT_LOGS) } -export const usePublicApiUserRoles = () => { - return useFeature(Feature.USER_ROLE_PUBLIC_API) +export const useExpandedPublicApi = () => { + return useFeature(Feature.EXPANDED_PUBLIC_API) } export const useScimIntegration = () => { diff --git a/packages/bbui/src/Form/Core/Multiselect.svelte b/packages/bbui/src/Form/Core/Multiselect.svelte index 8816da33c4..8d72dd0652 100644 --- a/packages/bbui/src/Form/Core/Multiselect.svelte +++ b/packages/bbui/src/Form/Core/Multiselect.svelte @@ -14,12 +14,12 @@ export let autocomplete = false export let sort = false export let autoWidth = false - export let fetchTerm = null - export let useFetch = false + export let searchTerm = null export let customPopoverHeight export let customPopoverOffsetBelow export let customPopoverMaxHeight export let open = false + export let loading const dispatch = createEventDispatcher() @@ -82,6 +82,7 @@ diff --git a/packages/bbui/src/Form/Core/Picker.svelte b/packages/bbui/src/Form/Core/Picker.svelte index 9b90c1a865..aa06d5f748 100644 --- a/packages/bbui/src/Form/Core/Picker.svelte +++ b/packages/bbui/src/Form/Core/Picker.svelte @@ -2,7 +2,7 @@ import "@spectrum-css/picker/dist/index-vars.css" import "@spectrum-css/popover/dist/index-vars.css" import "@spectrum-css/menu/dist/index-vars.css" - import { createEventDispatcher } from "svelte" + import { createEventDispatcher, onDestroy } from "svelte" import clickOutside from "../../Actions/click_outside" import Search from "./Search.svelte" import Icon from "../../Icon/Icon.svelte" @@ -10,6 +10,7 @@ import Popover from "../../Popover/Popover.svelte" import Tags from "../../Tags/Tags.svelte" import Tag from "../../Tags/Tag.svelte" + import ProgressCircle from "../../ProgressCircle/ProgressCircle.svelte" export let id = null export let disabled = false @@ -35,19 +36,20 @@ export let autoWidth = false export let autocomplete = false export let sort = false - export let fetchTerm = null - export let useFetch = false + export let searchTerm = null export let customPopoverHeight export let customPopoverOffsetBelow export let customPopoverMaxHeight export let align = "left" export let footer = null export let customAnchor = null + export let loading + const dispatch = createEventDispatcher() - let searchTerm = null let button let popover + let component $: sortedOptions = getSortedOptions(options, getOptionLabel, sort) $: filteredOptions = getFilteredOptions( @@ -82,7 +84,7 @@ } const getFilteredOptions = (options, term, getLabel) => { - if (autocomplete && term && !fetchTerm) { + if (autocomplete && term) { const lowerCaseTerm = term.toLowerCase() return options.filter(option => { return `${getLabel(option)}`.toLowerCase().includes(lowerCaseTerm) @@ -90,6 +92,20 @@ } return options } + + const onScroll = e => { + const scrollPxThreshold = 100 + const scrollPositionFromBottom = + e.target.scrollHeight - e.target.clientHeight - e.target.scrollTop + if (scrollPositionFromBottom < scrollPxThreshold) { + dispatch("loadMore") + } + } + + $: component?.addEventListener("scroll", onScroll) + onDestroy(() => { + component?.removeEventListener("scroll", null) + }) + + + + + diff --git a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte index 1c264a5aaf..0cc61c69e6 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte @@ -180,7 +180,7 @@
selectTable(TableNames.USERS)} diff --git a/packages/builder/src/components/start/ImportAppModal.svelte b/packages/builder/src/components/start/ImportAppModal.svelte new file mode 100644 index 0000000000..7d30ded896 --- /dev/null +++ b/packages/builder/src/components/start/ImportAppModal.svelte @@ -0,0 +1,69 @@ + + + + Updating an app using an app export will replace all tables, datasources, + queries, screens and automations. It is recommended to perform a backup + before running this operation. + + { + file = e.detail?.[0] + }} + /> + + {#if encrypted} + + {/if} + + diff --git a/packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte index 519a9c0f45..801ddd1130 100644 --- a/packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/settings/_layout.svelte @@ -32,9 +32,9 @@ active={$isActive("./embed")} /> app.devId == $store.appId) $: app = filteredApps.length ? filteredApps[0] : {} $: appDeployed = app?.status === AppStatus.DEPLOYED - let exportModal + let exportModal, importModal let exportPublishedVersion = false const exportApp = opts => { exportPublishedVersion = !!opts?.published exportModal.show() } + + const importApp = () => { + importModal.show() + } + + + + Export your app Export your latest edited or published app - -
+
exportApp({ published: false })}> Export latest edited app @@ -47,10 +55,20 @@ Export latest published app
+ + + Import your app + Import an export to update this app + +
+ importApp()}> + Import app + +
diff --git a/packages/builder/src/pages/builder/portal/apps/_components/PortalSideBar.svelte b/packages/builder/src/pages/builder/portal/apps/_components/PortalSideBar.svelte new file mode 100644 index 0000000000..1e21bd7a9a --- /dev/null +++ b/packages/builder/src/pages/builder/portal/apps/_components/PortalSideBar.svelte @@ -0,0 +1,140 @@ + + +
+
+ {#if searching} + + {:else} + Apps + + {/if} +
+ $goto("./create")} + /> +
+
+
+ $goto("./")} + selected={!$params.appId} + /> + {#each filteredApps as app} + $goto(`./${app.appId}`)} + /> + {/each} +
+
+ + diff --git a/packages/builder/src/pages/builder/portal/apps/_layout.svelte b/packages/builder/src/pages/builder/portal/apps/_layout.svelte index bf0bca0df4..c4a0bfd913 100644 --- a/packages/builder/src/pages/builder/portal/apps/_layout.svelte +++ b/packages/builder/src/pages/builder/portal/apps/_layout.svelte @@ -11,6 +11,7 @@ import { onMount } from "svelte" import { redirect } from "@roxi/routify" import { sdk } from "@budibase/shared-core" + import PortalSideBar from "./_components/PortalSideBar.svelte" // Don't block loading if we've already hydrated state let loaded = $apps.length != null @@ -44,5 +45,18 @@ {#if loaded} - +
+ + +
{/if} + + diff --git a/packages/builder/src/stores/portal/index.js b/packages/builder/src/stores/portal/index.js index a7c430e621..e70df5c3ee 100644 --- a/packages/builder/src/stores/portal/index.js +++ b/packages/builder/src/stores/portal/index.js @@ -1,3 +1,5 @@ +import { writable } from "svelte/store" + export { organisation } from "./organisation" export { users } from "./users" export { admin } from "./admin" @@ -14,3 +16,5 @@ export { environment } from "./environment" export { menu } from "./menu" export { auditLogs } from "./auditLogs" export { features } from "./features" + +export const sideBarCollapsed = writable(false) diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index ecbbfcde6a..c9fe4a8549 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -1,11 +1,6 @@ {#if fieldState} -
- option._id} - {placeholder} - customPopoverOffsetBelow={autocomplete ? 32 : null} - customPopoverMaxHeight={autocomplete ? 240 : null} - sort={true} - /> - {#if autocomplete} -
- - {#if searching} -
- -
- {/if} -
- {/if} -
+ option._id} + {placeholder} + bind:searchTerm + loading={$fetch.loading} + bind:open + customPopoverMaxHeight={400} + /> {/if} - - diff --git a/packages/frontend-core/src/api/app.js b/packages/frontend-core/src/api/app.js index 982066f05a..49137cbecd 100644 --- a/packages/frontend-core/src/api/app.js +++ b/packages/frontend-core/src/api/app.js @@ -1,3 +1,5 @@ +import { sdk } from "@budibase/shared-core" + export const buildAppEndpoints = API => ({ /** * Fetches screen definition for an app. @@ -81,6 +83,22 @@ export const buildAppEndpoints = API => ({ }) }, + /** + * Update an application using an export - the body + * should be of type FormData, with a "file" and a "password" if encrypted. + * @param appId The ID of the app to update - this will always be + * converted to development ID. + * @param body a FormData body with a file and password. + */ + updateAppFromExport: async (appId, body) => { + const devId = sdk.applications.getDevAppID(appId) + return await API.post({ + url: `/api/applications/${devId}/import`, + body, + json: false, + }) + }, + /** * Imports an export of all apps. * @param apps the FormData containing the apps to import diff --git a/packages/pro b/packages/pro index 3c51e0938e..3038568214 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 3c51e0938e2226038f4456bb8c96d857310b8d0c +Subproject commit 30385682141e5ba9d98de7d71d5be1672109cd15 diff --git a/packages/server/scripts/integrations/postgres/init.sql b/packages/server/scripts/integrations/postgres/init.sql index f89ad2812d..b7ce1b7d5b 100644 --- a/packages/server/scripts/integrations/postgres/init.sql +++ b/packages/server/scripts/integrations/postgres/init.sql @@ -9,6 +9,7 @@ CREATE TABLE Persons ( Address varchar(255), City varchar(255) DEFAULT 'Belfast', Age INTEGER DEFAULT 20 NOT NULL, + Year INTEGER, Type person_job ); CREATE TABLE Tasks ( @@ -49,9 +50,10 @@ CREATE TABLE CompositeTable ( Name varchar(255), PRIMARY KEY (KeyPartOne, KeyPartTwo) ); -INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast', 'qa'); -INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('John', 'Smith', '64 Updown Road', 'Dublin', 'programmer'); -INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age) VALUES ('Foo', 'Bar', 'Foo Street', 'Bartown', 'support', 0); +INSERT INTO Persons (FirstName, LastName, Address, City, Type, Year) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast', 'qa', 1999); +INSERT INTO Persons (FirstName, LastName, Address, City, Type, Year) VALUES ('John', 'Smith', '64 Updown Road', 'Dublin', 'programmer', 1996); +INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Foo', 'Bar', 'Foo Street', 'Bartown', 'support', 0, 1993); +INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('Jonny', 'Muffin', 'Muffin Street', 'Cork', 'support'); INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (1, 2, 'assembling', TRUE); INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (2, 1, 'processing', FALSE); INSERT INTO Products (ProductName) VALUES ('Computers'); diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index 1e5718c5b5..18f9dd4245 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -613,6 +613,23 @@ "data" ] }, + "appExport": { + "type": "object", + "properties": { + "encryptPassword": { + "description": "An optional password used to encrypt the export.", + "type": "string" + }, + "excludeRows": { + "description": "Set whether the internal table rows should be excluded from the export.", + "type": "boolean" + } + }, + "required": [ + "encryptPassword", + "excludeRows" + ] + }, "row": { "description": "The row to be created/updated, based on the table schema.", "type": "object", @@ -2163,6 +2180,87 @@ } } }, + "/applications/{appId}/import": { + "post": { + "operationId": "appImport", + "summary": "Import an app to an existing app 🔒", + "description": "This endpoint is only available on a business or enterprise license.", + "tags": [ + "applications" + ], + "parameters": [ + { + "$ref": "#/components/parameters/appIdUrl" + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "encryptedPassword": { + "description": "Password for the export if it is encrypted.", + "type": "string" + }, + "appExport": { + "description": "The app export to import.", + "type": "string", + "format": "binary" + } + }, + "required": [ + "appExport" + ] + } + } + } + }, + "responses": { + "204": { + "description": "Application has been updated." + } + } + } + }, + "/applications/{appId}/export": { + "post": { + "operationId": "appExport", + "summary": "Export an app 🔒", + "description": "This endpoint is only available on a business or enterprise license.", + "tags": [ + "applications" + ], + "parameters": [ + { + "$ref": "#/components/parameters/appIdUrl" + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/appExport" + } + } + } + }, + "responses": { + "200": { + "description": "A gzip tarball containing the app export, encrypted if password provided.", + "content": { + "application/gzip": { + "schema": { + "type": "string", + "format": "binary", + "example": "Tarball containing database and object store contents..." + } + } + } + } + } + } + }, "/applications/search": { "post": { "operationId": "appSearch", diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index 07320917b8..4916141569 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -587,6 +587,19 @@ components: - appUrl required: - data + appExport: + type: object + properties: + encryptPassword: + description: An optional password used to encrypt the export. + type: string + excludeRows: + description: Set whether the internal table rows should be excluded from the + export. + type: boolean + required: + - encryptPassword + - excludeRows row: description: The row to be created/updated, based on the table schema. type: object @@ -1763,6 +1776,57 @@ paths: examples: deployment: $ref: "#/components/examples/deploymentOutput" + "/applications/{appId}/import": + post: + operationId: appImport + summary: Import an app to an existing app 🔒 + description: This endpoint is only available on a business or enterprise license. + tags: + - applications + parameters: + - $ref: "#/components/parameters/appIdUrl" + requestBody: + content: + multipart/form-data: + schema: + type: object + properties: + encryptedPassword: + description: Password for the export if it is encrypted. + type: string + appExport: + description: The app export to import. + type: string + format: binary + required: + - appExport + responses: + "204": + description: Application has been updated. + "/applications/{appId}/export": + post: + operationId: appExport + summary: Export an app 🔒 + description: This endpoint is only available on a business or enterprise license. + tags: + - applications + parameters: + - $ref: "#/components/parameters/appIdUrl" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/appExport" + responses: + "200": + description: A gzip tarball containing the app export, encrypted if password + provided. + content: + application/gzip: + schema: + type: string + format: binary + example: Tarball containing database and object store contents... /applications/search: post: operationId: appSearch diff --git a/packages/server/specs/resources/application.ts b/packages/server/specs/resources/application.ts index cd7a68c049..081dd9e72a 100644 --- a/packages/server/specs/resources/application.ts +++ b/packages/server/specs/resources/application.ts @@ -134,4 +134,15 @@ export default new Resource() deploymentOutput: object({ data: deploymentOutputSchema, }), + appExport: object({ + encryptPassword: { + description: "An optional password used to encrypt the export.", + type: "string", + }, + excludeRows: { + description: + "Set whether the internal table rows should be excluded from the export.", + type: "boolean", + }, + }), }) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 71bd034b9f..11f6e6c249 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -39,9 +39,8 @@ import { } from "../../db/defaultData/datasource_bb_default" import { removeAppFromUserRoles } from "../../utilities/workerRequests" import { stringToReadStream } from "../../utilities" -import { doesUserHaveLock, getLocksById } from "../../utilities/redis" +import { doesUserHaveLock } from "../../utilities/redis" import { cleanupAutomations } from "../../automations/utils" -import { checkAppMetadata } from "../../automations/logging" import { getUniqueRows } from "../../utilities/usageQuota/rows" import { groups, licensing, quotas } from "@budibase/pro" import { @@ -51,7 +50,6 @@ import { PlanType, Screen, UserCtx, - ContextUser, } from "@budibase/types" import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts" import sdk from "../../sdk" @@ -575,6 +573,28 @@ export async function sync(ctx: UserCtx) { } } +export async function importToApp(ctx: UserCtx) { + const { appId } = ctx.params + const appExport = ctx.request.files?.appExport + const password = ctx.request.body.encryptionPassword as string + if (!appExport) { + ctx.throw(400, "Must supply app export to import") + } + if (Array.isArray(appExport)) { + ctx.throw(400, "Must only supply one app export") + } + const fileAttributes = { type: appExport.type!, path: appExport.path! } + try { + await sdk.applications.updateWithExport(appId, fileAttributes, password) + } catch (err: any) { + ctx.throw( + 500, + `Unable to perform update, please retry - ${err?.message || err}` + ) + } + ctx.body = { message: "app updated" } +} + export async function updateAppPackage(appPackage: any, appId: any) { return context.doInAppContext(appId, async () => { const db = context.getAppDB() diff --git a/packages/server/src/api/controllers/public/applications.ts b/packages/server/src/api/controllers/public/applications.ts index fd72db95d3..316da72377 100644 --- a/packages/server/src/api/controllers/public/applications.ts +++ b/packages/server/src/api/controllers/public/applications.ts @@ -2,9 +2,11 @@ import { db as dbCore, context } from "@budibase/backend-core" import { search as stringSearch, addRev } from "./utils" import * as controller from "../application" import * as deployController from "../deploy" +import * as backupController from "../backup" import { Application } from "../../../definitions/common" import { UserCtx } from "@budibase/types" import { Next } from "koa" +import { sdk as proSdk } from "@budibase/pro" function fixAppID(app: Application, params: any) { if (!params) { @@ -80,6 +82,8 @@ export async function destroy(ctx: UserCtx, next: Next) { export async function unpublish(ctx: UserCtx, next: Next) { await context.doInAppContext(ctx.params.appId, async () => { await controller.unpublish(ctx) + ctx.body = undefined + ctx.status = 204 await next() }) } @@ -91,12 +95,22 @@ export async function publish(ctx: UserCtx, next: Next) { }) } +// get licensed endpoints from pro +export const importToApp = proSdk.publicApi.applications.buildImportFn( + controller.importToApp +) +export const exportApp = proSdk.publicApi.applications.buildExportFn( + backupController.exportAppDump +) + export default { create, update, read, destroy, search, - publish, unpublish, + publish, + importToApp, + exportApp, } diff --git a/packages/server/src/api/routes/application.ts b/packages/server/src/api/routes/application.ts index 18760d485a..a21d6a2153 100644 --- a/packages/server/src/api/routes/application.ts +++ b/packages/server/src/api/routes/application.ts @@ -4,6 +4,7 @@ import * as deploymentController from "../controllers/deploy" import authorized from "../../middleware/authorized" import { permissions } from "@budibase/backend-core" import { applicationValidator } from "./utils/validators" +import { importToApp } from "../controllers/application" const router: Router = new Router() @@ -58,5 +59,10 @@ router authorized(permissions.GLOBAL_BUILDER), controller.destroy ) + .post( + "/api/applications/:appId/import", + authorized(permissions.BUILDER), + controller.importToApp + ) export default router diff --git a/packages/server/src/api/routes/public/applications.ts b/packages/server/src/api/routes/public/applications.ts index 088d974e6c..5410eb7dcf 100644 --- a/packages/server/src/api/routes/public/applications.ts +++ b/packages/server/src/api/routes/public/applications.ts @@ -137,6 +137,70 @@ write.push( new Endpoint("post", "/applications/:appId/publish", controller.publish) ) +/** + * @openapi + * /applications/{appId}/import: + * post: + * operationId: appImport + * summary: Import an app to an existing app 🔒 + * description: This endpoint is only available on a business or enterprise license. + * tags: + * - applications + * parameters: + * - $ref: '#/components/parameters/appIdUrl' + * requestBody: + * content: + * multipart/form-data: + * schema: + * type: object + * properties: + * encryptedPassword: + * description: Password for the export if it is encrypted. + * type: string + * appExport: + * description: The app export to import. + * type: string + * format: binary + * required: + * - appExport + * responses: + * 204: + * description: Application has been updated. + */ +write.push( + new Endpoint("post", "/applications/:appId/import", controller.importToApp) +) + +/** + * @openapi + * /applications/{appId}/export: + * post: + * operationId: appExport + * summary: Export an app 🔒 + * description: This endpoint is only available on a business or enterprise license. + * tags: + * - applications + * parameters: + * - $ref: '#/components/parameters/appIdUrl' + * requestBody: + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/appExport' + * responses: + * 200: + * description: A gzip tarball containing the app export, encrypted if password provided. + * content: + * application/gzip: + * schema: + * type: string + * format: binary + * example: Tarball containing database and object store contents... + */ +read.push( + new Endpoint("post", "/applications/:appId/export", controller.exportApp) +) + /** * @openapi * /applications/{appId}: diff --git a/packages/server/src/api/routes/public/middleware/mapper.ts b/packages/server/src/api/routes/public/middleware/mapper.ts index 138a5ac23f..03feb6cc5c 100644 --- a/packages/server/src/api/routes/public/middleware/mapper.ts +++ b/packages/server/src/api/routes/public/middleware/mapper.ts @@ -1,3 +1,4 @@ +import { Ctx } from "@budibase/types" import mapping from "../../../controllers/public/mapping" enum Resources { @@ -9,11 +10,19 @@ enum Resources { SEARCH = "search", } -function isArrayResponse(ctx: any) { +function isAttachment(ctx: Ctx) { + return ctx.body?.path && ctx.body?.flags && ctx.body?.mode +} + +function isArrayResponse(ctx: Ctx) { return ctx.url.endsWith(Resources.SEARCH) || Array.isArray(ctx.body) } -function processApplications(ctx: any) { +function noResponse(ctx: Ctx) { + return !Array.isArray(ctx.body) && Object.keys(ctx.body).length === 0 +} + +function processApplications(ctx: Ctx) { if (isArrayResponse(ctx)) { return mapping.mapApplications(ctx) } else { @@ -21,7 +30,7 @@ function processApplications(ctx: any) { } } -function processTables(ctx: any) { +function processTables(ctx: Ctx) { if (isArrayResponse(ctx)) { return mapping.mapTables(ctx) } else { @@ -29,7 +38,7 @@ function processTables(ctx: any) { } } -function processRows(ctx: any) { +function processRows(ctx: Ctx) { if (isArrayResponse(ctx)) { return mapping.mapRowSearch(ctx) } else { @@ -37,7 +46,7 @@ function processRows(ctx: any) { } } -function processUsers(ctx: any) { +function processUsers(ctx: Ctx) { if (isArrayResponse(ctx)) { return mapping.mapUsers(ctx) } else { @@ -45,7 +54,7 @@ function processUsers(ctx: any) { } } -function processQueries(ctx: any) { +function processQueries(ctx: Ctx) { if (isArrayResponse(ctx)) { return mapping.mapQueries(ctx) } else { @@ -53,8 +62,8 @@ function processQueries(ctx: any) { } } -export default async (ctx: any, next: any) => { - if (!ctx.body) { +export default async (ctx: Ctx, next: any) => { + if (!ctx.body || noResponse(ctx) || isAttachment(ctx)) { return await next() } let urlParts = ctx.url.split("/") diff --git a/packages/server/src/api/routes/public/tests/applications.spec.ts b/packages/server/src/api/routes/public/tests/applications.spec.ts new file mode 100644 index 0000000000..0a2ffe9e95 --- /dev/null +++ b/packages/server/src/api/routes/public/tests/applications.spec.ts @@ -0,0 +1,91 @@ +import * as setup from "../../tests/utilities" +import { + generateMakeRequest, + generateMakeRequestWithFormData, + MakeRequestResponse, + MakeRequestWithFormDataResponse, +} from "./utils" +import { User } from "@budibase/types" +import { join } from "path" +import { mocks } from "@budibase/backend-core/tests" + +const PASSWORD = "testtest" +const NO_LICENSE_MSG = "Endpoint unavailable, license required." + +let config = setup.getConfig() +let apiKey: string, + globalUser: User, + makeRequest: MakeRequestResponse, + makeRequestFormData: MakeRequestWithFormDataResponse + +beforeAll(async () => { + await config.init() + globalUser = await config.globalUser() + apiKey = await config.generateApiKey(globalUser._id) + makeRequest = generateMakeRequest(apiKey) + makeRequestFormData = generateMakeRequestWithFormData(apiKey) +}) + +afterAll(setup.afterAll) + +describe("check export/import", () => { + async function runExport() { + return await makeRequest("post", `/applications/${config.appId}/export`, { + encryptionPassword: PASSWORD, + excludeRows: true, + }) + } + + async function runImport() { + const pathToExport = join( + __dirname, + "..", + "..", + "tests", + "assets", + "export.tar.gz" + ) + return await makeRequestFormData( + "post", + `/applications/${config.appId}/import`, + { + encryptionPassword: PASSWORD, + appExport: { path: pathToExport }, + } + ) + } + + it("check licensing for export", async () => { + const res = await runExport() + expect(res.status).toBe(403) + expect(res.body.message).toBe(NO_LICENSE_MSG) + }) + + it("check licensing for import", async () => { + const res = await runImport() + expect(res.status).toBe(403) + expect(res.body.message).toBe(NO_LICENSE_MSG) + }) + + it("should be able to export app", async () => { + mocks.licenses.useExpandedPublicApi() + const res = await runExport() + expect(res.headers["content-disposition"]).toMatch( + /attachment; filename=".*-export-.*\.tar.gz"/g + ) + expect(res.body instanceof Buffer).toBe(true) + expect(res.status).toBe(200) + }) + + it("should be able to import app", async () => { + mocks.licenses.useExpandedPublicApi() + const res = await runImport() + expect(Object.keys(res.body).length).toBe(0) + // check screens imported correctly + const screens = await config.api.screen.list() + expect(screens.length).toBe(2) + expect(screens[0].routing.route).toBe("/derp") + expect(screens[1].routing.route).toBe("/blank") + expect(res.status).toBe(204) + }) +}) diff --git a/packages/server/src/api/routes/public/tests/users.spec.ts b/packages/server/src/api/routes/public/tests/users.spec.ts index c81acca1df..9d38dc4791 100644 --- a/packages/server/src/api/routes/public/tests/users.spec.ts +++ b/packages/server/src/api/routes/public/tests/users.spec.ts @@ -92,7 +92,7 @@ describe("no user role update in free", () => { describe("no user role update in business", () => { beforeAll(() => { updateMock() - mocks.licenses.usePublicApiUserRoles() + mocks.licenses.useExpandedPublicApi() }) it("should allow 'roles' to be updated", async () => { @@ -105,7 +105,7 @@ describe("no user role update in business", () => { }) it("should allow 'admin' to be updated", async () => { - mocks.licenses.usePublicApiUserRoles() + mocks.licenses.useExpandedPublicApi() const res = await makeRequest("post", "/users", { ...base(), admin: { global: true }, @@ -115,7 +115,7 @@ describe("no user role update in business", () => { }) it("should allow 'builder' to be updated", async () => { - mocks.licenses.usePublicApiUserRoles() + mocks.licenses.useExpandedPublicApi() const res = await makeRequest("post", "/users", { ...base(), builder: { global: true }, diff --git a/packages/server/src/api/routes/public/tests/utils.ts b/packages/server/src/api/routes/public/tests/utils.ts index 755e2d659f..1b57682af9 100644 --- a/packages/server/src/api/routes/public/tests/utils.ts +++ b/packages/server/src/api/routes/public/tests/utils.ts @@ -11,6 +11,32 @@ export type MakeRequestResponse = ( intAppId?: string ) => Promise +export type MakeRequestWithFormDataResponse = ( + method: HttpMethod, + endpoint: string, + fields: Record, + intAppId?: string +) => Promise + +function base( + apiKey: string, + endpoint: string, + intAppId: string | null, + isInternal: boolean +) { + const extraHeaders: any = { + "x-budibase-api-key": apiKey, + } + if (intAppId) { + extraHeaders["x-budibase-app-id"] = intAppId + } + + const url = isInternal + ? endpoint + : checkSlashesInUrl(`/api/public/v1/${endpoint}`) + return { headers: extraHeaders, url } +} + export function generateMakeRequest( apiKey: string, isInternal = false @@ -23,18 +49,8 @@ export function generateMakeRequest( body?: any, intAppId: string | null = config.getAppId() ) => { - const extraHeaders: any = { - "x-budibase-api-key": apiKey, - } - if (intAppId) { - extraHeaders["x-budibase-app-id"] = intAppId - } - - const url = isInternal - ? endpoint - : checkSlashesInUrl(`/api/public/v1/${endpoint}`) - - const req = request[method](url).set(config.defaultHeaders(extraHeaders)) + const { headers, url } = base(apiKey, endpoint, intAppId, isInternal) + const req = request[method](url).set(config.defaultHeaders(headers)) if (body) { req.send(body) } @@ -43,3 +59,30 @@ export function generateMakeRequest( return res } } + +export function generateMakeRequestWithFormData( + apiKey: string, + isInternal = false +): MakeRequestWithFormDataResponse { + const request = setup.getRequest()! + const config = setup.getConfig()! + return async ( + method: HttpMethod, + endpoint: string, + fields: Record, + intAppId: string | null = config.getAppId() + ) => { + const { headers, url } = base(apiKey, endpoint, intAppId, isInternal) + const req = request[method](url).set(config.defaultHeaders(headers)) + for (let [field, value] of Object.entries(fields)) { + if (typeof value === "string") { + req.field(field, value) + } else { + req.attach(field, value.path) + } + } + const res = await req + expect(res.body).toBeDefined() + return res + } +} diff --git a/packages/server/src/api/routes/tests/appImport.spec.ts b/packages/server/src/api/routes/tests/appImport.spec.ts new file mode 100644 index 0000000000..ef3c739e72 --- /dev/null +++ b/packages/server/src/api/routes/tests/appImport.spec.ts @@ -0,0 +1,32 @@ +import * as setup from "./utilities" +import path from "path" + +jest.setTimeout(15000) +const PASSWORD = "testtest" + +describe("/applications/:appId/import", () => { + let request = setup.getRequest() + let config = setup.getConfig() + + afterAll(setup.afterAll) + + beforeAll(async () => { + await config.init() + }) + + it("should be able to perform import", async () => { + const appId = config.getAppId() + const res = await request + .post(`/api/applications/${appId}/import`) + .field("encryptionPassword", PASSWORD) + .attach("appExport", path.join(__dirname, "assets", "export.tar.gz")) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + expect(res.body.message).toBe("app updated") + const screens = await config.api.screen.list() + expect(screens.length).toBe(2) + expect(screens[0].routing.route).toBe("/derp") + expect(screens[1].routing.route).toBe("/blank") + }) +}) diff --git a/packages/server/src/api/routes/tests/assets/export.tar.gz b/packages/server/src/api/routes/tests/assets/export.tar.gz new file mode 100644 index 0000000000..af16873a78 Binary files /dev/null and b/packages/server/src/api/routes/tests/assets/export.tar.gz differ diff --git a/packages/server/src/definitions/openapi.ts b/packages/server/src/definitions/openapi.ts index fe5c17b218..52434494e5 100644 --- a/packages/server/src/definitions/openapi.ts +++ b/packages/server/src/definitions/openapi.ts @@ -18,6 +18,14 @@ export interface paths { "/applications/{appId}/publish": { post: operations["appPublish"]; }; + "/applications/{appId}/import": { + /** This endpoint is only available on a business or enterprise license. */ + post: operations["appImport"]; + }; + "/applications/{appId}/export": { + /** This endpoint is only available on a business or enterprise license. */ + post: operations["appExport"]; + }; "/applications/search": { /** Based on application properties (currently only name) search for applications. */ post: operations["appSearch"]; @@ -158,6 +166,12 @@ export interface components { appUrl: string; }; }; + appExport: { + /** @description An optional password used to encrypt the export. */ + encryptPassword: string; + /** @description Set whether the internal table rows should be excluded from the export. */ + excludeRows: boolean; + }; /** @description The row to be created/updated, based on the table schema. */ row: { [key: string]: unknown }; searchOutput: { @@ -889,6 +903,54 @@ export interface operations { }; }; }; + /** This endpoint is only available on a business or enterprise license. */ + appImport: { + parameters: { + path: { + /** The ID of the app which this request is targeting. */ + appId: components["parameters"]["appIdUrl"]; + }; + }; + responses: { + /** Application has been updated. */ + 204: never; + }; + requestBody: { + content: { + "multipart/form-data": { + /** @description Password for the export if it is encrypted. */ + encryptedPassword?: string; + /** + * Format: binary + * @description The app export to import. + */ + appExport: string; + }; + }; + }; + }; + /** This endpoint is only available on a business or enterprise license. */ + appExport: { + parameters: { + path: { + /** The ID of the app which this request is targeting. */ + appId: components["parameters"]["appIdUrl"]; + }; + }; + responses: { + /** A gzip tarball containing the app export, encrypted if password provided. */ + 200: { + content: { + "application/gzip": string; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["appExport"]; + }; + }; + }; /** Based on application properties (currently only name) search for applications. */ appSearch: { responses: { diff --git a/packages/server/src/sdk/app/applications/import.ts b/packages/server/src/sdk/app/applications/import.ts new file mode 100644 index 0000000000..a7788924d8 --- /dev/null +++ b/packages/server/src/sdk/app/applications/import.ts @@ -0,0 +1,102 @@ +import { db as dbCore } from "@budibase/backend-core" +import { + DocumentTypesToImport, + Document, + Database, + RowValue, +} from "@budibase/types" +import backups from "../backups" + +export type FileAttributes = { + type: string + path: string +} + +function mergeUpdateAndDeleteDocuments( + updateDocs: Document[], + deleteDocs: Document[] +) { + // compress the documents to create and to delete (if same ID, then just update the rev) + const finalToDelete = [] + for (let deleteDoc of deleteDocs) { + const found = updateDocs.find(doc => doc._id === deleteDoc._id) + if (found) { + found._rev = deleteDoc._rev + } else { + finalToDelete.push(deleteDoc) + } + } + return [...updateDocs, ...finalToDelete] +} + +async function removeImportableDocuments(db: Database) { + // get the references to the documents, not the whole document + const docPromises = [] + for (let docType of DocumentTypesToImport) { + docPromises.push(db.allDocs(dbCore.getDocParams(docType))) + } + let documentRefs: { _id: string; _rev: string }[] = [] + for (let response of await Promise.all(docPromises)) { + documentRefs = documentRefs.concat( + response.rows.map(row => ({ + _id: row.id, + _rev: (row.value as RowValue).rev, + })) + ) + } + // add deletion key + return documentRefs.map(ref => ({ _deleted: true, ...ref })) +} + +async function getImportableDocuments(db: Database) { + // get the whole document + const docPromises = [] + for (let docType of DocumentTypesToImport) { + docPromises.push( + db.allDocs(dbCore.getDocParams(docType, null, { include_docs: true })) + ) + } + // map the responses to the document itself + let documents: Document[] = [] + for (let response of await Promise.all(docPromises)) { + documents = documents.concat(response.rows.map(row => row.doc)) + } + // remove the _rev, stops it being written + documents.forEach(doc => { + delete doc._rev + }) + return documents +} + +export async function updateWithExport( + appId: string, + file: FileAttributes, + password?: string +) { + const devId = dbCore.getDevAppID(appId) + const tempAppName = `temp_${devId}` + const tempDb = dbCore.getDB(tempAppName) + const appDb = dbCore.getDB(devId) + try { + const template = { + file: { + type: file.type!, + path: file.path!, + password, + }, + } + // get a temporary version of the import + // don't need obj store, the existing app already has everything we need + await backups.importApp(devId, tempDb, template, { + importObjStoreContents: false, + }) + // get the documents to copy + const toUpdate = await getImportableDocuments(tempDb) + // clear out the old documents + const toDelete = await removeImportableDocuments(appDb) + // now bulk update documents - add new ones, delete old ones and update common ones + await appDb.bulkDocs(mergeUpdateAndDeleteDocuments(toUpdate, toDelete)) + } finally { + await tempDb.destroy() + } +} diff --git a/packages/server/src/sdk/app/applications/index.ts b/packages/server/src/sdk/app/applications/index.ts index 963d065ce2..04ed3b2919 100644 --- a/packages/server/src/sdk/app/applications/index.ts +++ b/packages/server/src/sdk/app/applications/index.ts @@ -1,9 +1,11 @@ import * as sync from "./sync" import * as utils from "./utils" import * as applications from "./applications" +import * as imports from "./import" export default { ...sync, ...utils, ...applications, + ...imports, } diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index 307cdf4015..fe875f0c3d 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -8,11 +8,7 @@ import { TABLE_ROW_PREFIX, USER_METDATA_PREFIX, } from "../../../db/utils" -import { - DB_EXPORT_FILE, - GLOBAL_DB_EXPORT_FILE, - STATIC_APP_FILES, -} from "./constants" +import { DB_EXPORT_FILE, STATIC_APP_FILES } from "./constants" import fs from "fs" import { join } from "path" import env from "../../../environment" diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index 619f888329..c8e54e9e1d 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -151,7 +151,8 @@ export function getListOfAppsInMulti(tmpPath: string) { export async function importApp( appId: string, db: Database, - template: TemplateType + template: TemplateType, + opts: { importObjStoreContents: boolean } = { importObjStoreContents: true } ) { let prodAppId = dbCore.getProdAppID(appId) let dbStream: any @@ -165,7 +166,7 @@ export async function importApp( } const contents = fs.readdirSync(tmpPath) // have to handle object import - if (contents.length) { + if (contents.length && opts.importObjStoreContents) { let promises = [] let excludedFiles = [GLOBAL_DB_EXPORT_FILE, DB_EXPORT_FILE] for (let filename of contents) { diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index da7af8acd7..799e6f34e9 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -53,7 +53,6 @@ import { View, FieldType, RelationshipType, - ViewV2, CreateViewRequest, } from "@budibase/types" diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts index 31c74a0e78..889133b847 100644 --- a/packages/server/src/tests/utilities/api/index.ts +++ b/packages/server/src/tests/utilities/api/index.ts @@ -5,6 +5,7 @@ import { TableAPI } from "./table" import { ViewV2API } from "./viewV2" import { DatasourceAPI } from "./datasource" import { LegacyViewAPI } from "./legacyView" +import { ScreenAPI } from "./screen" export default class API { table: TableAPI @@ -13,6 +14,7 @@ export default class API { row: RowAPI permission: PermissionAPI datasource: DatasourceAPI + screen: ScreenAPI constructor(config: TestConfiguration) { this.table = new TableAPI(config) @@ -21,5 +23,6 @@ export default class API { this.row = new RowAPI(config) this.permission = new PermissionAPI(config) this.datasource = new DatasourceAPI(config) + this.screen = new ScreenAPI(config) } } diff --git a/packages/server/src/tests/utilities/api/screen.ts b/packages/server/src/tests/utilities/api/screen.ts new file mode 100644 index 0000000000..9245ffe4ba --- /dev/null +++ b/packages/server/src/tests/utilities/api/screen.ts @@ -0,0 +1,18 @@ +import TestConfiguration from "../TestConfiguration" +import { Screen } from "@budibase/types" +import { TestAPI } from "./base" + +export class ScreenAPI extends TestAPI { + constructor(config: TestConfiguration) { + super(config) + } + + list = async (): Promise => { + const res = await this.request + .get(`/api/screens`) + .set(this.config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + return res.body as Screen[] + } +} diff --git a/packages/types/src/documents/document.ts b/packages/types/src/documents/document.ts index 03e01907b8..763da62d61 100644 --- a/packages/types/src/documents/document.ts +++ b/packages/types/src/documents/document.ts @@ -39,6 +39,25 @@ export enum DocumentType { AUDIT_LOG = "al", } +// these are the core documents that make up the data, design +// and automation sections of an app. This excludes any internal +// rows as we shouldn't import data. +export const DocumentTypesToImport: DocumentType[] = [ + DocumentType.ROLE, + DocumentType.DATASOURCE, + DocumentType.DATASOURCE_PLUS, + DocumentType.TABLE, + DocumentType.AUTOMATION, + DocumentType.WEBHOOK, + DocumentType.SCREEN, + DocumentType.QUERY, + DocumentType.METADATA, + DocumentType.MEM_VIEW, + // Deprecated but still copied + DocumentType.INSTANCE, + DocumentType.LAYOUT, +] + // these documents don't really exist, they are part of other // documents or enriched into existence as part of get requests export enum VirtualDocumentType { diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts index bd3a6583bf..732a4a6c77 100644 --- a/packages/types/src/sdk/licensing/feature.ts +++ b/packages/types/src/sdk/licensing/feature.ts @@ -11,7 +11,7 @@ export enum Feature { SYNC_AUTOMATIONS = "syncAutomations", APP_BUILDERS = "appBuilders", OFFLINE = "offline", - USER_ROLE_PUBLIC_API = "userRolePublicApi", + EXPANDED_PUBLIC_API = "expandedPublicApi", VIEW_PERMISSIONS = "viewPermissions", }