diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index a3e4790430..e955992c66 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -67,6 +67,8 @@ spec: - name: AWS_REGION value: {{ .Values.services.objectStore.region }} {{ end }} + - name: MINIO_ENABLED + value: {{ .Values.services.objectStore.minio | quote }} - name: MINIO_ACCESS_KEY valueFrom: secretKeyRef: @@ -77,13 +79,19 @@ spec: secretKeyRef: name: {{ template "budibase.fullname" . }} key: objectStoreSecret + - name: CLOUDFRONT_CDN + value: {{ .Values.services.objectStore.cloudfront.cdn | quote }} + - name: CLOUDFRONT_PUBLIC_KEY_ID + value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }} + - name: CLOUDFRONT_PRIVATE_KEY_64 + value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }} - name: MINIO_URL value: {{ .Values.services.objectStore.url }} - name: PLUGIN_BUCKET_NAME value: {{ .Values.services.objectStore.pluginBucketName | quote }} - name: APPS_BUCKET_NAME value: {{ .Values.services.objectStore.appsBucketName | quote }} - - name: GLOBAL_CLOUD_BUCKET_NAME + - name: GLOBAL_BUCKET_NAME value: {{ .Values.services.objectStore.globalBucketName | quote }} - name: BACKUPS_BUCKET_NAME value: {{ .Values.services.objectStore.backupsBucketName | quote }} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 44bbb8aa20..7168764d56 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -68,6 +68,8 @@ spec: - name: AWS_REGION value: {{ .Values.services.objectStore.region }} {{ end }} + - name: MINIO_ENABLED + value: {{ .Values.services.objectStore.minio | quote }} - name: MINIO_ACCESS_KEY valueFrom: secretKeyRef: @@ -80,11 +82,17 @@ spec: key: objectStoreSecret - name: MINIO_URL value: {{ .Values.services.objectStore.url }} + - name: CLOUDFRONT_CDN + value: {{ .Values.services.objectStore.cloudfront.cdn | quote }} + - name: CLOUDFRONT_PUBLIC_KEY_ID + value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }} + - name: CLOUDFRONT_PRIVATE_KEY_64 + value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }} - name: PLUGIN_BUCKET_NAME value: {{ .Values.services.objectStore.pluginBucketName | quote }} - name: APPS_BUCKET_NAME value: {{ .Values.services.objectStore.appsBucketName | quote }} - - name: GLOBAL_CLOUD_BUCKET_NAME + - name: GLOBAL_BUCKET_NAME value: {{ .Values.services.objectStore.globalBucketName | quote }} - name: BACKUPS_BUCKET_NAME value: {{ .Values.services.objectStore.backupsBucketName | quote }} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 726df7585b..1b2b1c3dcb 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -167,6 +167,7 @@ services: resources: {} objectStore: + # Set to false if using another object store such as S3 minio: true browser: true port: 9000 @@ -182,6 +183,13 @@ services: ## set, choosing the default provisioner. storageClass: "" resources: {} + cloudfront: + # Set the url of a distribution to enable cloudfront + cdn: "" + # ID of public key stored in cloudfront + publicKeyId: "" + # Base64 encoded private key for the above public key + privateKey64: "" # Override values in couchDB subchart couchdb: diff --git a/hosting/nginx.dev.conf.hbs b/hosting/nginx.dev.conf.hbs index 51678d23e8..4adee0d925 100644 --- a/hosting/nginx.dev.conf.hbs +++ b/hosting/nginx.dev.conf.hbs @@ -186,6 +186,26 @@ http { proxy_pass http://dev-service:9000; } + location /files/signed/ { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # IMPORTANT: Signed urls will inspect the host header of the request. + # Normally a signed url will need to be generated with a specified client host in mind. + # To support dynamic hosts, e.g. some unknown self-hosted installation url, + # use a predefined host header. The host 'minio-service' is also used at the time of url signing. + proxy_set_header Host minio-service; + + proxy_connect_timeout 300; + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio-service:9000; + rewrite ^/files/signed/(.*)$ /$1 break; + } + client_header_timeout 60; client_body_timeout 60; keepalive_timeout 60; diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs index 6f0f1b420d..cd70ce1ae2 100644 --- a/hosting/nginx.prod.conf.hbs +++ b/hosting/nginx.prod.conf.hbs @@ -208,6 +208,26 @@ http { proxy_pass http://$minio:9000; } + location /files/signed/ { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # IMPORTANT: Signed urls will inspect the host header of the request. + # Normally a signed url will need to be generated with a specified client host in mind. + # To support dynamic hosts, e.g. some unknown self-hosted installation url, + # use a predefined host header. The host 'minio-service' is also used at the time of url signing. + proxy_set_header Host minio-service; + + proxy_connect_timeout 300; + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://$minio:9000; + rewrite ^/files/signed/(.*)$ /$1 break; + } + client_header_timeout 60; client_body_timeout 60; keepalive_timeout 60; diff --git a/hosting/single/nginx/nginx-default-site.conf b/hosting/single/nginx/nginx-default-site.conf index acadb06250..3903c0647d 100644 --- a/hosting/single/nginx/nginx-default-site.conf +++ b/hosting/single/nginx/nginx-default-site.conf @@ -95,15 +95,37 @@ server { } location / { - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; - proxy_connect_timeout 300; - proxy_http_version 1.1; - proxy_set_header Connection ""; - chunked_transfer_encoding off; - proxy_pass http://127.0.0.1:9000; + proxy_connect_timeout 300; + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://127.0.0.1:9000; + } + + location /files/signed/ { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # IMPORTANT: Signed urls will inspect the host header of the request. + # Normally a signed url will need to be generated with a specified client host in mind. + # To support dynamic hosts, e.g. some unknown self-hosted installation url, + # use a predefined host header. The host 'minio-service' is also used at the time of url signing. + proxy_set_header Host minio-service; + + proxy_connect_timeout 300; + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://127.0.0.1:9000; + rewrite ^/files/signed/(.*)$ /$1 break; } client_header_timeout 60; diff --git a/lerna.json b/lerna.json index 0d02624756..439190499e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.1.46-alpha.6", + "version": "2.2.4-alpha.2", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 361bf99fe2..8ecd0a7ee2 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh", "build": "lerna run build", "build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput", + "build:backend": "lerna run build --ignore @budibase/client --ignore @budibase/bbui --ignore @budibase/builder --ignore @budibase/cli", "build:sdk": "lerna run build:sdk", "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular", "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro", diff --git a/packages/backend-core/__mocks__/aws-sdk.ts b/packages/backend-core/__mocks__/aws-sdk.ts index 7fac80faa9..b8d91dbaa9 100644 --- a/packages/backend-core/__mocks__/aws-sdk.ts +++ b/packages/backend-core/__mocks__/aws-sdk.ts @@ -3,7 +3,10 @@ const mockS3 = { deleteObject: jest.fn().mockReturnThis(), deleteObjects: jest.fn().mockReturnThis(), createBucket: jest.fn().mockReturnThis(), - listObjects: jest.fn().mockReturnThis(), + listObject: jest.fn().mockReturnThis(), + getSignedUrl: jest.fn((operation: string, params: any) => { + return `http://s3.example.com/${params.Bucket}/${params.Key}` + }), promise: jest.fn().mockReturnThis(), catch: jest.fn(), } diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index e3be11a056..dd36746391 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.1.46-alpha.6", + "version": "2.2.4-alpha.2", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,9 +20,11 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "2.1.46-alpha.6", + "@budibase/nano": "10.1.1", + "@budibase/types": "2.2.4-alpha.2", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", + "aws-cloudfront-sign": "2.2.0", "aws-sdk": "2.1030.0", "bcrypt": "5.0.1", "bcryptjs": "2.4.3", @@ -35,7 +37,6 @@ "koa-passport": "4.1.4", "lodash": "4.17.21", "lodash.isarguments": "3.1.0", - "nano": "^10.1.0", "node-fetch": "2.6.7", "passport-google-oauth": "2.0.0", "passport-jwt": "4.0.0", diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index d743d2f49b..c44ec4e767 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -2,7 +2,7 @@ // store an app ID to pretend there is a context import env from "../environment" import Context from "./Context" -import { getDevelopmentAppID, getProdAppID } from "../db/conversions" +import * as conversions from "../db/conversions" import { getDB } from "../db/db" import { DocumentType, @@ -181,6 +181,14 @@ export function getAppId(): string | undefined { } } +export const getProdAppId = () => { + const appId = getAppId() + if (!appId) { + throw new Error("Could not get appId") + } + return conversions.getProdAppID(appId) +} + export function updateTenantId(tenantId?: string) { let context: ContextMap = updateContext({ tenantId, @@ -229,7 +237,7 @@ export function getProdAppDB(opts?: any): Database { if (!appId) { throw new Error("Unable to retrieve prod DB - no app ID.") } - return getDB(getProdAppID(appId), opts) + return getDB(conversions.getProdAppID(appId), opts) } /** @@ -241,5 +249,5 @@ export function getDevAppDB(opts?: any): Database { if (!appId) { throw new Error("Unable to retrieve dev DB - no app ID.") } - return getDB(getDevelopmentAppID(appId), opts) + return getDB(conversions.getDevelopmentAppID(appId), opts) } diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index de06b4e8ee..9b4761d961 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -1,4 +1,4 @@ -import Nano from "nano" +import Nano from "@budibase/nano" import { AllDocsResponse, AnyDocument, diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 590c3eeef8..5e501c8d22 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -14,7 +14,7 @@ import { doWithDB, allDbs, directCouchAllDbs } from "./db" import { getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "./conversions" import * as events from "../events" -import { App, Database, ConfigType } from "@budibase/types" +import { App, Database, ConfigType, isSettingsConfig } from "@budibase/types" /** * Generates a new app ID. @@ -489,18 +489,12 @@ export const getScopedFullConfig = async function ( // custom logic for settings doc if (type === ConfigType.SETTINGS) { - if (scopedConfig && scopedConfig.doc) { - // overrides affected by environment variables - scopedConfig.doc.config.platformUrl = await getPlatformUrl({ - tenantAware: true, - }) - scopedConfig.doc.config.analyticsEnabled = - await events.analytics.enabled() - } else { + if (!scopedConfig || !scopedConfig.doc) { // defaults scopedConfig = { doc: { _id: generateConfigID({ type, user, workspace }), + type: ConfigType.SETTINGS, config: { platformUrl: await getPlatformUrl({ tenantAware: true }), analyticsEnabled: await events.analytics.enabled(), @@ -508,6 +502,16 @@ export const getScopedFullConfig = async function ( }, } } + + // will always be true - use assertion function to get type access + if (isSettingsConfig(scopedConfig.doc)) { + // overrides affected by environment + scopedConfig.doc.config.platformUrl = await getPlatformUrl({ + tenantAware: true, + }) + scopedConfig.doc.config.analyticsEnabled = + await events.analytics.enabled() + } } return scopedConfig && scopedConfig.doc diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 2377c8ceba..60cf5b7882 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -25,7 +25,6 @@ const DefaultBucketName = { APPS: "prod-budi-app-assets", TEMPLATES: "templates", GLOBAL: "global", - CLOUD: "prod-budi-tenant-uploads", PLUGINS: "plugins", } @@ -33,6 +32,9 @@ const environment = { isTest, isJest, isDev, + isProd: () => { + return !isDev() + }, JS_BCRYPT: process.env.JS_BCRYPT, JWT_SECRET: process.env.JWT_SECRET, COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", @@ -47,6 +49,7 @@ const environment = { MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, AWS_REGION: process.env.AWS_REGION, MINIO_URL: process.env.MINIO_URL, + MINIO_ENABLED: process.env.MINIO_ENABLED || 1, INTERNAL_API_KEY: process.env.INTERNAL_API_KEY, MULTI_TENANCY: process.env.MULTI_TENANCY, ACCOUNT_PORTAL_URL: @@ -59,6 +62,9 @@ const environment = { POSTHOG_TOKEN: process.env.POSTHOG_TOKEN, ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS, + CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN, + CLOUDFRONT_PRIVATE_KEY_64: process.env.CLOUDFRONT_PRIVATE_KEY_64, + CLOUDFRONT_PUBLIC_KEY_ID: process.env.CLOUDFRONT_PUBLIC_KEY_ID, BACKUPS_BUCKET_NAME: process.env.BACKUPS_BUCKET_NAME || DefaultBucketName.BACKUPS, APPS_BUCKET_NAME: process.env.APPS_BUCKET_NAME || DefaultBucketName.APPS, @@ -66,12 +72,9 @@ const environment = { process.env.TEMPLATES_BUCKET_NAME || DefaultBucketName.TEMPLATES, GLOBAL_BUCKET_NAME: process.env.GLOBAL_BUCKET_NAME || DefaultBucketName.GLOBAL, - GLOBAL_CLOUD_BUCKET_NAME: - process.env.GLOBAL_CLOUD_BUCKET_NAME || DefaultBucketName.CLOUD, PLUGIN_BUCKET_NAME: process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS, USE_COUCH: process.env.USE_COUCH || true, - DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, DEFAULT_LICENSE: process.env.DEFAULT_LICENSE, SERVICE: process.env.SERVICE || "budibase", LOG_LEVEL: process.env.LOG_LEVEL, @@ -92,6 +95,11 @@ for (let [key, value] of Object.entries(environment)) { // @ts-ignore environment[key] = 0 } + // handle the edge case of "false" to disable an environment variable + if (value === "false") { + // @ts-ignore + environment[key] = 0 + } } export = environment diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts new file mode 100644 index 0000000000..9951058d6a --- /dev/null +++ b/packages/backend-core/src/objectStore/buckets/app.ts @@ -0,0 +1,40 @@ +import env from "../../environment" +import * as objectStore from "../objectStore" +import * as cloudfront from "../cloudfront" + +/** + * In production the client library is stored in the object store, however in development + * we use the symlinked version produced by lerna, located in node modules. We link to this + * via a specific endpoint (under /api/assets/client). + * @param {string} appId In production we need the appId to look up the correct bucket, as the + * version of the client lib may differ between apps. + * @param {string} version The version to retrieve. + * @return {string} The URL to be inserted into appPackage response or server rendered + * app index file. + */ +export const clientLibraryUrl = (appId: string, version: string) => { + if (env.isProd()) { + let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js` + if (env.CLOUDFRONT_CDN) { + // append app version to bust the cache + if (version) { + file += `?v=${version}` + } + // don't need to use presigned for client with cloudfront + // file is public + return cloudfront.getUrl(file) + } else { + return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file) + } + } else { + return `/api/assets/client` + } +} + +export const getAppFileUrl = (s3Key: string) => { + if (env.CLOUDFRONT_CDN) { + return cloudfront.getPresignedUrl(s3Key) + } else { + return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, s3Key) + } +} diff --git a/packages/backend-core/src/objectStore/buckets/global.ts b/packages/backend-core/src/objectStore/buckets/global.ts new file mode 100644 index 0000000000..8bf883b11e --- /dev/null +++ b/packages/backend-core/src/objectStore/buckets/global.ts @@ -0,0 +1,29 @@ +import env from "../../environment" +import * as tenancy from "../../tenancy" +import * as objectStore from "../objectStore" +import * as cloudfront from "../cloudfront" + +// URLs + +export const getGlobalFileUrl = (type: string, name: string, etag?: string) => { + let file = getGlobalFileS3Key(type, name) + if (env.CLOUDFRONT_CDN) { + if (etag) { + file = `${file}?etag=${etag}` + } + return cloudfront.getPresignedUrl(file) + } else { + return objectStore.getPresignedUrl(env.GLOBAL_BUCKET_NAME, file) + } +} + +// KEYS + +export const getGlobalFileS3Key = (type: string, name: string) => { + let file = `${type}/${name}` + if (env.MULTI_TENANCY) { + const tenantId = tenancy.getTenantId() + file = `${tenantId}/${file}` + } + return file +} diff --git a/packages/backend-core/src/objectStore/buckets/index.ts b/packages/backend-core/src/objectStore/buckets/index.ts new file mode 100644 index 0000000000..8398242ee5 --- /dev/null +++ b/packages/backend-core/src/objectStore/buckets/index.ts @@ -0,0 +1,3 @@ +export * from "./app" +export * from "./global" +export * from "./plugins" diff --git a/packages/backend-core/src/objectStore/buckets/plugins.ts b/packages/backend-core/src/objectStore/buckets/plugins.ts new file mode 100644 index 0000000000..cd3bf77e87 --- /dev/null +++ b/packages/backend-core/src/objectStore/buckets/plugins.ts @@ -0,0 +1,71 @@ +import env from "../../environment" +import * as objectStore from "../objectStore" +import * as tenancy from "../../tenancy" +import * as cloudfront from "../cloudfront" +import { Plugin } from "@budibase/types" + +// URLS + +export const enrichPluginURLs = (plugins: Plugin[]) => { + if (!plugins || !plugins.length) { + return [] + } + return plugins.map(plugin => { + const jsUrl = getPluginJSUrl(plugin) + const iconUrl = getPluginIconUrl(plugin) + return { ...plugin, jsUrl, iconUrl } + }) +} + +const getPluginJSUrl = (plugin: Plugin) => { + const s3Key = getPluginJSKey(plugin) + return getPluginUrl(s3Key) +} + +const getPluginIconUrl = (plugin: Plugin): string | undefined => { + const s3Key = getPluginIconKey(plugin) + if (!s3Key) { + return + } + return getPluginUrl(s3Key) +} + +const getPluginUrl = (s3Key: string) => { + if (env.CLOUDFRONT_CDN) { + return cloudfront.getPresignedUrl(s3Key) + } else { + return objectStore.getPresignedUrl(env.PLUGIN_BUCKET_NAME, s3Key) + } +} + +// S3 KEYS + +export const getPluginJSKey = (plugin: Plugin) => { + return getPluginS3Key(plugin, "plugin.min.js") +} + +export const getPluginIconKey = (plugin: Plugin) => { + // stored iconUrl is deprecated - hardcode to icon.svg in this case + const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName + if (!iconFileName) { + return + } + return getPluginS3Key(plugin, iconFileName) +} + +const getPluginS3Key = (plugin: Plugin, fileName: string) => { + const s3Key = getPluginS3Dir(plugin.name) + return `${s3Key}/${fileName}` +} + +export const getPluginS3Dir = (pluginName: string) => { + let s3Key = `${pluginName}` + if (env.MULTI_TENANCY) { + const tenantId = tenancy.getTenantId() + s3Key = `${tenantId}/${s3Key}` + } + if (env.CLOUDFRONT_CDN) { + s3Key = `plugins/${s3Key}` + } + return s3Key +} diff --git a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts new file mode 100644 index 0000000000..0375e97cbc --- /dev/null +++ b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts @@ -0,0 +1,171 @@ +import * as app from "../app" +import { getAppFileUrl } from "../app" +import { testEnv } from "../../../../tests" + +describe("app", () => { + beforeEach(() => { + testEnv.nodeJest() + }) + + describe("clientLibraryUrl", () => { + function getClientUrl() { + return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0") + } + + describe("single tenant", () => { + beforeAll(() => { + testEnv.singleTenant() + }) + + it("gets url in dev", () => { + testEnv.nodeDev() + const url = getClientUrl() + expect(url).toBe("/api/assets/client") + }) + + it("gets url with embedded minio", () => { + testEnv.withMinio() + const url = getClientUrl() + expect(url).toBe( + "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" + ) + }) + + it("gets url with custom S3", () => { + testEnv.withS3() + const url = getClientUrl() + expect(url).toBe( + "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" + ) + }) + + it("gets url with cloudfront + s3", () => { + testEnv.withCloudfront() + const url = getClientUrl() + expect(url).toBe( + "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0" + ) + }) + }) + + describe("multi tenant", () => { + beforeAll(() => { + testEnv.multiTenant() + }) + + it("gets url in dev", async () => { + testEnv.nodeDev() + await testEnv.withTenant(tenantId => { + const url = getClientUrl() + expect(url).toBe("/api/assets/client") + }) + }) + + it("gets url with embedded minio", async () => { + await testEnv.withTenant(tenantId => { + testEnv.withMinio() + const url = getClientUrl() + expect(url).toBe( + "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" + ) + }) + }) + + it("gets url with custom S3", async () => { + await testEnv.withTenant(tenantId => { + testEnv.withS3() + const url = getClientUrl() + expect(url).toBe( + "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" + ) + }) + }) + + it("gets url with cloudfront + s3", async () => { + await testEnv.withTenant(tenantId => { + testEnv.withCloudfront() + const url = getClientUrl() + expect(url).toBe( + "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0" + ) + }) + }) + }) + }) + + describe("getAppFileUrl", () => { + function getAppFileUrl() { + return app.getAppFileUrl("app_123/attachments/image.jpeg") + } + + describe("single tenant", () => { + beforeAll(() => { + testEnv.multiTenant() + }) + + it("gets url with embedded minio", () => { + testEnv.withMinio() + const url = getAppFileUrl() + expect(url).toBe( + "/files/signed/prod-budi-app-assets/app_123/attachments/image.jpeg" + ) + }) + + it("gets url with custom S3", () => { + testEnv.withS3() + const url = getAppFileUrl() + expect(url).toBe( + "http://s3.example.com/prod-budi-app-assets/app_123/attachments/image.jpeg" + ) + }) + + it("gets url with cloudfront + s3", () => { + testEnv.withCloudfront() + const url = getAppFileUrl() + // omit rest of signed params + expect( + url.includes("http://cf.example.com/app_123/attachments/image.jpeg?") + ).toBe(true) + }) + }) + + describe("multi tenant", () => { + beforeAll(() => { + testEnv.multiTenant() + }) + + it("gets url with embedded minio", async () => { + testEnv.withMinio() + await testEnv.withTenant(tenantId => { + const url = getAppFileUrl() + expect(url).toBe( + "/files/signed/prod-budi-app-assets/app_123/attachments/image.jpeg" + ) + }) + }) + + it("gets url with custom S3", async () => { + testEnv.withS3() + await testEnv.withTenant(tenantId => { + const url = getAppFileUrl() + expect(url).toBe( + "http://s3.example.com/prod-budi-app-assets/app_123/attachments/image.jpeg" + ) + }) + }) + + it("gets url with cloudfront + s3", async () => { + testEnv.withCloudfront() + await testEnv.withTenant(tenantId => { + const url = getAppFileUrl() + // omit rest of signed params + expect( + url.includes( + "http://cf.example.com/app_123/attachments/image.jpeg?" + ) + ).toBe(true) + }) + }) + }) + }) +}) diff --git a/packages/backend-core/src/objectStore/buckets/tests/global.spec.ts b/packages/backend-core/src/objectStore/buckets/tests/global.spec.ts new file mode 100644 index 0000000000..b495812356 --- /dev/null +++ b/packages/backend-core/src/objectStore/buckets/tests/global.spec.ts @@ -0,0 +1,74 @@ +import * as global from "../global" +import { testEnv } from "../../../../tests" + +describe("global", () => { + describe("getGlobalFileUrl", () => { + function getGlobalFileUrl() { + return global.getGlobalFileUrl("settings", "logoUrl", "etag") + } + + describe("single tenant", () => { + beforeAll(() => { + testEnv.singleTenant() + }) + + it("gets url with embedded minio", () => { + testEnv.withMinio() + const url = getGlobalFileUrl() + expect(url).toBe("/files/signed/global/settings/logoUrl") + }) + + it("gets url with custom S3", () => { + testEnv.withS3() + const url = getGlobalFileUrl() + expect(url).toBe("http://s3.example.com/global/settings/logoUrl") + }) + + it("gets url with cloudfront + s3", () => { + testEnv.withCloudfront() + const url = getGlobalFileUrl() + // omit rest of signed params + expect( + url.includes("http://cf.example.com/settings/logoUrl?etag=etag&") + ).toBe(true) + }) + }) + + describe("multi tenant", () => { + beforeAll(() => { + testEnv.multiTenant() + }) + + it("gets url with embedded minio", async () => { + testEnv.withMinio() + await testEnv.withTenant(tenantId => { + const url = getGlobalFileUrl() + expect(url).toBe(`/files/signed/global/${tenantId}/settings/logoUrl`) + }) + }) + + it("gets url with custom S3", async () => { + testEnv.withS3() + await testEnv.withTenant(tenantId => { + const url = getGlobalFileUrl() + expect(url).toBe( + `http://s3.example.com/global/${tenantId}/settings/logoUrl` + ) + }) + }) + + it("gets url with cloudfront + s3", async () => { + testEnv.withCloudfront() + await testEnv.withTenant(tenantId => { + const url = getGlobalFileUrl() + // omit rest of signed params + expect( + url.includes( + `http://cf.example.com/${tenantId}/settings/logoUrl?etag=etag&` + ) + ).toBe(true) + }) + }) + }) + }) +}) diff --git a/packages/backend-core/src/objectStore/buckets/tests/plugins.spec.ts b/packages/backend-core/src/objectStore/buckets/tests/plugins.spec.ts new file mode 100644 index 0000000000..affb8d8318 --- /dev/null +++ b/packages/backend-core/src/objectStore/buckets/tests/plugins.spec.ts @@ -0,0 +1,110 @@ +import * as plugins from "../plugins" +import { structures, testEnv } from "../../../../tests" + +describe("plugins", () => { + describe("enrichPluginURLs", () => { + const plugin = structures.plugins.plugin() + + function getEnrichedPluginUrls() { + const enriched = plugins.enrichPluginURLs([plugin])[0] + return { + jsUrl: enriched.jsUrl!, + iconUrl: enriched.iconUrl!, + } + } + + describe("single tenant", () => { + beforeAll(() => { + testEnv.singleTenant() + }) + + it("gets url with embedded minio", () => { + testEnv.withMinio() + const urls = getEnrichedPluginUrls() + expect(urls.jsUrl).toBe( + `/files/signed/plugins/${plugin.name}/plugin.min.js` + ) + expect(urls.iconUrl).toBe( + `/files/signed/plugins/${plugin.name}/icon.svg` + ) + }) + + it("gets url with custom S3", () => { + testEnv.withS3() + const urls = getEnrichedPluginUrls() + expect(urls.jsUrl).toBe( + `http://s3.example.com/plugins/${plugin.name}/plugin.min.js` + ) + expect(urls.iconUrl).toBe( + `http://s3.example.com/plugins/${plugin.name}/icon.svg` + ) + }) + + it("gets url with cloudfront + s3", () => { + testEnv.withCloudfront() + const urls = getEnrichedPluginUrls() + // omit rest of signed params + expect( + urls.jsUrl.includes( + `http://cf.example.com/plugins/${plugin.name}/plugin.min.js?` + ) + ).toBe(true) + expect( + urls.iconUrl.includes( + `http://cf.example.com/plugins/${plugin.name}/icon.svg?` + ) + ).toBe(true) + }) + }) + + describe("multi tenant", () => { + beforeAll(() => { + testEnv.multiTenant() + }) + + it("gets url with embedded minio", async () => { + testEnv.withMinio() + await testEnv.withTenant(tenantId => { + const urls = getEnrichedPluginUrls() + expect(urls.jsUrl).toBe( + `/files/signed/plugins/${tenantId}/${plugin.name}/plugin.min.js` + ) + expect(urls.iconUrl).toBe( + `/files/signed/plugins/${tenantId}/${plugin.name}/icon.svg` + ) + }) + }) + + it("gets url with custom S3", async () => { + testEnv.withS3() + await testEnv.withTenant(tenantId => { + const urls = getEnrichedPluginUrls() + expect(urls.jsUrl).toBe( + `http://s3.example.com/plugins/${tenantId}/${plugin.name}/plugin.min.js` + ) + expect(urls.iconUrl).toBe( + `http://s3.example.com/plugins/${tenantId}/${plugin.name}/icon.svg` + ) + }) + }) + + it("gets url with cloudfront + s3", async () => { + testEnv.withCloudfront() + await testEnv.withTenant(tenantId => { + const urls = getEnrichedPluginUrls() + // omit rest of signed params + expect( + urls.jsUrl.includes( + `http://cf.example.com/plugins/${tenantId}/${plugin.name}/plugin.min.js?` + ) + ).toBe(true) + expect( + urls.iconUrl.includes( + `http://cf.example.com/plugins/${tenantId}/${plugin.name}/icon.svg?` + ) + ).toBe(true) + }) + }) + }) + }) +}) diff --git a/packages/backend-core/src/objectStore/cloudfront.ts b/packages/backend-core/src/objectStore/cloudfront.ts new file mode 100644 index 0000000000..a61ea7f583 --- /dev/null +++ b/packages/backend-core/src/objectStore/cloudfront.ts @@ -0,0 +1,41 @@ +import env from "../environment" +const cfsign = require("aws-cloudfront-sign") + +let PRIVATE_KEY: string | undefined + +function getPrivateKey() { + if (!env.CLOUDFRONT_PRIVATE_KEY_64) { + throw new Error("CLOUDFRONT_PRIVATE_KEY_64 is not set") + } + + if (PRIVATE_KEY) { + return PRIVATE_KEY + } + + PRIVATE_KEY = Buffer.from(env.CLOUDFRONT_PRIVATE_KEY_64, "base64").toString( + "utf-8" + ) + + return PRIVATE_KEY +} + +const getCloudfrontSignParams = () => { + return { + keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID, + privateKeyString: getPrivateKey(), + expireTime: new Date().getTime() + 1000 * 60 * 60, // 1 hour + } +} + +export const getPresignedUrl = (s3Key: string) => { + const url = getUrl(s3Key) + return cfsign.getSignedUrl(url, getCloudfrontSignParams()) +} + +export const getUrl = (s3Key: string) => { + let prefix = "/" + if (s3Key.startsWith("/")) { + prefix = "" + } + return `${env.CLOUDFRONT_CDN}${prefix}${s3Key}` +} diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts index 2971834f0e..02c99828dd 100644 --- a/packages/backend-core/src/objectStore/index.ts +++ b/packages/backend-core/src/objectStore/index.ts @@ -1,2 +1,3 @@ export * from "./objectStore" export * from "./utils" +export * from "./buckets" diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 2ae8848c53..89e1c88e10 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -8,7 +8,7 @@ import { promisify } from "util" import { join } from "path" import fs from "fs" import env from "../environment" -import { budibaseTempDir, ObjectStoreBuckets } from "./utils" +import { budibaseTempDir } from "./utils" import { v4 } from "uuid" import { APP_PREFIX, APP_DEV_PREFIX } from "../db" @@ -26,7 +26,7 @@ type UploadParams = { bucket: string filename: string path: string - type?: string + type?: string | null // can be undefined, we will remove it metadata?: { [key: string]: string | undefined @@ -41,6 +41,7 @@ const CONTENT_TYPE_MAP: any = { json: "application/json", gz: "application/gzip", } + const STRING_CONTENT_TYPES = [ CONTENT_TYPE_MAP.html, CONTENT_TYPE_MAP.css, @@ -58,35 +59,17 @@ export function sanitizeBucket(input: string) { return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX) } -function publicPolicy(bucketName: string) { - return { - Version: "2012-10-17", - Statement: [ - { - Effect: "Allow", - Principal: { - AWS: ["*"], - }, - Action: "s3:GetObject", - Resource: [`arn:aws:s3:::${bucketName}/*`], - }, - ], - } -} - -const PUBLIC_BUCKETS = [ - ObjectStoreBuckets.APPS, - ObjectStoreBuckets.GLOBAL, - ObjectStoreBuckets.PLUGINS, -] - /** * Gets a connection to the object store using the S3 SDK. * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from. + * @param {object} opts configuration for the object store. * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage. * @constructor */ -export const ObjectStore = (bucket: string) => { +export const ObjectStore = ( + bucket: string, + opts: { presigning: boolean } = { presigning: false } +) => { const config: any = { s3ForcePathStyle: true, signatureVersion: "v4", @@ -100,9 +83,20 @@ export const ObjectStore = (bucket: string) => { Bucket: sanitizeBucket(bucket), } } + + // custom S3 is in use i.e. minio if (env.MINIO_URL) { - config.endpoint = env.MINIO_URL + if (opts.presigning && !env.MINIO_ENABLED) { + // IMPORTANT: Signed urls will inspect the host header of the request. + // Normally a signed url will need to be generated with a specified host in mind. + // To support dynamic hosts, e.g. some unknown self-hosted installation url, + // use a predefined host. The host 'minio-service' is also forwarded to minio requests via nginx + config.endpoint = "minio-service" + } else { + config.endpoint = env.MINIO_URL + } } + return new AWS.S3(config) } @@ -135,16 +129,6 @@ export const makeSureBucketExists = async (client: any, bucketName: string) => { await promises[bucketName] delete promises[bucketName] } - // public buckets are quite hidden in the system, make sure - // no bucket is set accidentally - if (PUBLIC_BUCKETS.includes(bucketName)) { - await client - .putBucketPolicy({ - Bucket: bucketName, - Policy: JSON.stringify(publicPolicy(bucketName)), - }) - .promise() - } } else { throw new Error("Unable to write to object store bucket.") } @@ -274,6 +258,36 @@ export const listAllObjects = async (bucketName: string, path: string) => { return objects } +/** + * Generate a presigned url with a default TTL of 1 hour + */ +export const getPresignedUrl = ( + bucketName: string, + key: string, + durationSeconds: number = 3600 +) => { + const objectStore = ObjectStore(bucketName, { presigning: true }) + const params = { + Bucket: sanitizeBucket(bucketName), + Key: sanitizeKey(key), + Expires: durationSeconds, + } + const url = objectStore.getSignedUrl("getObject", params) + + if (!env.MINIO_ENABLED) { + // return the full URL to the client + return url + } else { + // return the path only to the client + // use the presigned url route to ensure the static + // hostname will be used in the request + const signedUrl = new URL(url) + const path = signedUrl.pathname + const query = signedUrl.search + return `/files/signed${path}${query}` + } +} + /** * Same as retrieval function but puts to a temporary file. */ diff --git a/packages/backend-core/src/objectStore/utils.ts b/packages/backend-core/src/objectStore/utils.ts index f3c9e93943..dba5f3d1c2 100644 --- a/packages/backend-core/src/objectStore/utils.ts +++ b/packages/backend-core/src/objectStore/utils.ts @@ -14,7 +14,6 @@ export const ObjectStoreBuckets = { APPS: env.APPS_BUCKET_NAME, TEMPLATES: env.TEMPLATES_BUCKET_NAME, GLOBAL: env.GLOBAL_BUCKET_NAME, - GLOBAL_CLOUD: env.GLOBAL_CLOUD_BUCKET_NAME, PLUGINS: env.PLUGIN_BUCKET_NAME, } diff --git a/packages/backend-core/src/tenancy/tenancy.ts b/packages/backend-core/src/tenancy/tenancy.ts index e0e0703433..732402bcb7 100644 --- a/packages/backend-core/src/tenancy/tenancy.ts +++ b/packages/backend-core/src/tenancy/tenancy.ts @@ -1,4 +1,4 @@ -import { doWithDB, queryPlatformView, getGlobalDBName } from "../db" +import { doWithDB, getGlobalDBName } from "../db" import { DEFAULT_TENANT_ID, getTenantId, @@ -8,11 +8,10 @@ import { import env from "../environment" import { BBContext, - PlatformUser, TenantResolutionStrategy, GetTenantIdOptions, } from "@budibase/types" -import { Header, StaticDatabases, ViewName } from "../constants" +import { Header, StaticDatabases } from "../constants" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name @@ -111,27 +110,7 @@ export async function lookupTenantId(userId: string) { }) } -// lookup, could be email or userId, either will return a doc -export async function getTenantUser( - identifier: string -): Promise { - // use the view here and allow to find anyone regardless of casing - // Use lowercase to ensure email login is case-insensitive - const users = await queryPlatformView( - ViewName.PLATFORM_USERS_LOWERCASE, - { - keys: [identifier.toLowerCase()], - include_docs: true, - } - ) - if (Array.isArray(users)) { - return users[0] - } else { - return users - } -} - -export function isUserInAppTenant(appId: string, user?: any) { +export const isUserInAppTenant = (appId: string, user?: any) => { let userTenantId if (user) { userTenantId = user.tenantId || DEFAULT_TENANT_ID diff --git a/packages/backend-core/src/tests/utils.spec.js b/packages/backend-core/src/utils/tests/utils.spec.ts similarity index 61% rename from packages/backend-core/src/tests/utils.spec.js rename to packages/backend-core/src/utils/tests/utils.spec.ts index fb3828921d..bb76a93653 100644 --- a/packages/backend-core/src/tests/utils.spec.js +++ b/packages/backend-core/src/utils/tests/utils.spec.ts @@ -1,8 +1,8 @@ -const { structures } = require("../../tests") -const utils = require("../utils") -const events = require("../events") -const { DEFAULT_TENANT_ID } = require("../constants") -const { doInTenant } = require("../context") +import { structures } from "../../../tests" +import * as utils from "../../utils" +import * as events from "../../events" +import { DEFAULT_TENANT_ID } from "../../constants" +import { doInTenant } from "../../context" describe("utils", () => { describe("platformLogout", () => { @@ -14,4 +14,4 @@ describe("utils", () => { }) }) }) -}) \ No newline at end of file +}) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 3e9fbb177a..fd8d31b13f 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -1,6 +1,13 @@ import { getAllApps, queryGlobalView } from "../db" import { options } from "../middleware/passport/jwt" -import { Header, Cookie, MAX_VALID_DATE } from "../constants" +import { + Header, + Cookie, + MAX_VALID_DATE, + DocumentType, + SEPARATOR, + ViewName, +} from "../constants" import env from "../environment" import * as userCache from "../cache/user" import { getSessionsForUser, invalidateSessions } from "../security/sessions" @@ -8,12 +15,11 @@ import * as events from "../events" import * as tenancy from "../tenancy" import { App, - BBContext, + Ctx, PlatformLogoutOpts, TenantResolutionStrategy, } from "@budibase/types" import { SetOption } from "cookies" -import { DocumentType, SEPARATOR, ViewName } from "../constants" const jwt = require("jsonwebtoken") const APP_PREFIX = DocumentType.APP + SEPARATOR @@ -25,7 +31,7 @@ function confirmAppId(possibleAppId: string | undefined) { : undefined } -async function resolveAppUrl(ctx: BBContext) { +async function resolveAppUrl(ctx: Ctx) { const appUrl = ctx.path.split("/")[2] let possibleAppUrl = `/${appUrl.toLowerCase()}` @@ -50,7 +56,7 @@ async function resolveAppUrl(ctx: BBContext) { return app && app.appId ? app.appId : undefined } -export function isServingApp(ctx: BBContext) { +export function isServingApp(ctx: Ctx) { // dev app if (ctx.path.startsWith(`/${APP_PREFIX}`)) { return true @@ -67,7 +73,7 @@ export function isServingApp(ctx: BBContext) { * @param {object} ctx The main request body to look through. * @returns {string|undefined} If an appId was found it will be returned. */ -export async function getAppIdFromCtx(ctx: BBContext) { +export async function getAppIdFromCtx(ctx: Ctx) { // look in headers const options = [ctx.headers[Header.APP_ID]] let appId @@ -83,12 +89,16 @@ export async function getAppIdFromCtx(ctx: BBContext) { appId = confirmAppId(ctx.request.body.appId) } - // look in the url - dev app - let appPath = - ctx.request.headers.referrer || - ctx.path.split("/").filter(subPath => subPath.startsWith(APP_PREFIX)) - if (!appId && appPath.length) { - appId = confirmAppId(appPath[0]) + // look in the path + const pathId = parseAppIdFromUrl(ctx.path) + if (!appId && pathId) { + appId = confirmAppId(pathId) + } + + // look in the referer + const refererId = parseAppIdFromUrl(ctx.request.headers.referer) + if (!appId && refererId) { + appId = confirmAppId(refererId) } // look in the url - prod app @@ -99,6 +109,13 @@ export async function getAppIdFromCtx(ctx: BBContext) { return appId } +function parseAppIdFromUrl(url?: string) { + if (!url) { + return + } + return url.split("/").find(subPath => subPath.startsWith(APP_PREFIX)) +} + /** * opens the contents of the specified encrypted JWT. * @return {object} the contents of the token. @@ -115,7 +132,7 @@ export function openJwt(token: string) { * @param {object} ctx The request which is to be manipulated. * @param {string} name The name of the cookie to get. */ -export function getCookie(ctx: BBContext, name: string) { +export function getCookie(ctx: Ctx, name: string) { const cookie = ctx.cookies.get(name) if (!cookie) { @@ -133,7 +150,7 @@ export function getCookie(ctx: BBContext, name: string) { * @param {object} opts options like whether to sign. */ export function setCookie( - ctx: BBContext, + ctx: Ctx, value: any, name = "builder", opts = { sign: true } @@ -159,7 +176,7 @@ export function setCookie( /** * Utility function, simply calls setCookie with an empty string for value */ -export function clearCookie(ctx: BBContext, name: string) { +export function clearCookie(ctx: Ctx, name: string) { setCookie(ctx, null, name) } @@ -169,7 +186,7 @@ export function clearCookie(ctx: BBContext, name: string) { * @param {object} ctx The koa context object to be tested. * @return {boolean} returns true if the call is from the client lib (a built app rather than the builder). */ -export function isClient(ctx: BBContext) { +export function isClient(ctx: Ctx) { return ctx.headers[Header.TYPE] === "client" } diff --git a/packages/backend-core/tests/jestSetup.ts b/packages/backend-core/tests/jestSetup.ts index 7870a721aa..b7ab5b49d9 100644 --- a/packages/backend-core/tests/jestSetup.ts +++ b/packages/backend-core/tests/jestSetup.ts @@ -17,7 +17,9 @@ env._set("MINIO_URL", "http://localhost") env._set("MINIO_ACCESS_KEY", "test") env._set("MINIO_SECRET_KEY", "test") -global.console.log = jest.fn() // console.log are ignored in tests +if (!process.env.DEBUG) { + global.console.log = jest.fn() // console.log are ignored in tests +} if (!process.env.CI) { // set a longer timeout in dev for debugging diff --git a/packages/backend-core/tests/utilities/index.ts b/packages/backend-core/tests/utilities/index.ts index 65578ff013..ee96a94152 100644 --- a/packages/backend-core/tests/utilities/index.ts +++ b/packages/backend-core/tests/utilities/index.ts @@ -1,6 +1,7 @@ export * as mocks from "./mocks" export * as structures from "./structures" export { generator } from "./structures" +export * as testEnv from "./testEnv" import * as dbConfig from "./db" dbConfig.init() diff --git a/packages/backend-core/tests/utilities/mocks/index.ts b/packages/backend-core/tests/utilities/mocks/index.ts index 931816be45..401fd7d7a7 100644 --- a/packages/backend-core/tests/utilities/mocks/index.ts +++ b/packages/backend-core/tests/utilities/mocks/index.ts @@ -1,6 +1,6 @@ -import "./posthog" -import "./events" export * as accounts from "./accounts" export * as date from "./date" export * as licenses from "./licenses" export { default as fetch } from "./fetch" +import "./posthog" +import "./events" diff --git a/packages/backend-core/tests/utilities/mocks/licenses.ts b/packages/backend-core/tests/utilities/mocks/licenses.ts index 0ef5eedb73..1fbda5655e 100644 --- a/packages/backend-core/tests/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/utilities/mocks/licenses.ts @@ -2,14 +2,14 @@ import { Feature, License, Quotas } from "@budibase/types" import _ from "lodash" let CLOUD_FREE_LICENSE: License -let TEST_LICENSE: License +let UNLIMITED_LICENSE: License let getCachedLicense: any // init for the packages other than pro export function init(proPkg: any) { initInternal({ CLOUD_FREE_LICENSE: proPkg.constants.licenses.CLOUD_FREE_LICENSE, - TEST_LICENSE: proPkg.constants.licenses.DEVELOPER_FREE_LICENSE, + UNLIMITED_LICENSE: proPkg.constants.licenses.UNLIMITED_LICENSE, getCachedLicense: proPkg.licensing.cache.getCachedLicense, }) } @@ -17,11 +17,11 @@ export function init(proPkg: any) { // init for the pro package export function initInternal(opts: { CLOUD_FREE_LICENSE: License - TEST_LICENSE: License + UNLIMITED_LICENSE: License getCachedLicense: any }) { CLOUD_FREE_LICENSE = opts.CLOUD_FREE_LICENSE - TEST_LICENSE = opts.TEST_LICENSE + UNLIMITED_LICENSE = opts.UNLIMITED_LICENSE getCachedLicense = opts.getCachedLicense } @@ -48,7 +48,7 @@ export const useLicense = (license: License, opts?: UseLicenseOpts) => { } export const useUnlimited = (opts?: UseLicenseOpts) => { - return useLicense(TEST_LICENSE, opts) + return useLicense(UNLIMITED_LICENSE, opts) } export const useCloudFree = () => { @@ -58,7 +58,7 @@ export const useCloudFree = () => { // FEATURES const useFeature = (feature: Feature) => { - const license = _.cloneDeep(TEST_LICENSE) + const license = _.cloneDeep(UNLIMITED_LICENSE) const opts: UseLicenseOpts = { features: [feature], } @@ -77,7 +77,7 @@ export const useGroups = () => { // QUOTAS export const setAutomationLogsQuota = (value: number) => { - const license = _.cloneDeep(TEST_LICENSE) + const license = _.cloneDeep(UNLIMITED_LICENSE) license.quotas.constant.automationLogRetentionDays.value = value return useLicense(license) } diff --git a/packages/backend-core/tests/utilities/structures/index.ts b/packages/backend-core/tests/utilities/structures/index.ts index 68064b9715..e0ed4df9c4 100644 --- a/packages/backend-core/tests/utilities/structures/index.ts +++ b/packages/backend-core/tests/utilities/structures/index.ts @@ -6,3 +6,4 @@ export const generator = new Chance() export * as koa from "./koa" export * as accounts from "./accounts" export * as licenses from "./licenses" +export * as plugins from "./plugins" diff --git a/packages/backend-core/tests/utilities/structures/plugins.ts b/packages/backend-core/tests/utilities/structures/plugins.ts new file mode 100644 index 0000000000..e2d92858d3 --- /dev/null +++ b/packages/backend-core/tests/utilities/structures/plugins.ts @@ -0,0 +1,19 @@ +import { generator } from "." +import { Plugin, PluginSource, PluginType } from "@budibase/types" + +export function plugin(): Plugin { + return { + description: generator.word(), + name: generator.word(), + version: "1.0.0", + source: PluginSource.FILE, + package: { + name: generator.word, + }, + hash: generator.hash(), + schema: { + type: PluginType.DATASOURCE, + }, + iconFileName: "icon.svg", + } +} diff --git a/packages/backend-core/tests/utilities/testEnv.ts b/packages/backend-core/tests/utilities/testEnv.ts new file mode 100644 index 0000000000..b4f06b5153 --- /dev/null +++ b/packages/backend-core/tests/utilities/testEnv.ts @@ -0,0 +1,87 @@ +import env from "../../src/environment" +import * as tenancy from "../../src/tenancy" +import { newid } from "../../src/utils" + +// TENANCY + +export async function withTenant(task: (tenantId: string) => any) { + const tenantId = newid() + return tenancy.doInTenant(tenantId, async () => { + await task(tenantId) + }) +} + +export function singleTenant() { + env._set("MULTI_TENANCY", 0) +} + +export function multiTenant() { + env._set("MULTI_TENANCY", 1) +} + +// NODE + +export function nodeDev() { + env._set("NODE_ENV", "dev") +} + +export function nodeJest() { + env._set("NODE_ENV", "jest") +} + +// FILES + +export function withS3() { + env._set("NODE_ENV", "production") + env._set("MINIO_ENABLED", 0) + env._set("MINIO_URL", "http://s3.example.com") + env._set("CLOUDFRONT_CDN", undefined) +} + +const CLOUDFRONT_TEST_KEY = + "-----BEGIN RSA PRIVATE KEY-----\n" + + "MIIEpAIBAAKCAQEAqXRsir/0Qba1xEnybUs7d7QEAE02GRc+4H7HD5l5VnAxkV1m\n" + + "tNTXTmoYkaIhLdebV1EwQs3T9knxoyd4cVcrDkDfDLZErfYWJsuE3/QYNknnZs4/\n" + + "Ai0cg+v9ZX3gcizvpYg9GQI3INM0uRG8lJwGP7FQ/kknhA2yVFVCSxX6kkNtOUh5\n" + + "dKSG7m6IwswcSwD++Z/94vsFkoZIGY0e1CD/drFJ6+1TFY2YgbDKT5wDFLJ9vHFx\n" + + "/5o4POwn3gz/ru2Db9jbRdfEAqRdy46nRKQgBGUmupAgSK1+BJEzafexp8RmCGb0\n" + + "WUffxOtj8/jNCeCF0JBgVHAe3crOQ8ySrtoaHQIDAQABAoIBAA+ipW07/u6dTDI7\n" + + "XHoHKgqGeqQIe8he47dVG0ruL0rxeTFfe92NkfwzP+cYHZWcQkIRRLG1Six8cCZM\n" + + "uwlCML/U7n++xaGDhlG4D5+WZzGDKi3LM/cgcHQfrzbRIYeHa+lLI9AN60ZFFqVI\n" + + "5KyVpOH1m3KLD3FYzi6H22EQOxmJpqWlt2uArny5LxlPJKmmGSFjvneb4N2ZAKGQ\n" + + "QfClJGz9tRjceWUUdJrpqmTmBQIosKmLPq8PEviUNAVG+6m4r8jiRbf8OKkAm+3L\n" + + "LVIsN8HfYB9jEuERYPnbuXdX0kDEkg0xEyTH5YbNZvfm5ptCU9Xn+Jz1trF+wCHD\n" + + "2RlxdQUCgYEA3U0nCf6NTmmeMCsAX6gvaPuM0iUfUfS3b3G57I6u46lLGNLsfJw6\n" + + "MTpVc164lKYQK9czw/ijKzb8e3mcyzbPorVkajMjUCNWGrMK+vFbOGmqQkhUi30U\n" + + "IJuuTktMd+21D/SpLlev4MLria23vUIKEqNenYpV6wkGLt/mKtISaPMCgYEAxAYx\n" + + "j+xJLTK9eN+rpekwjYE78hD9VoBkBnr/NBiGV302AsJRuq2+L4zcBnAsH+SidFim\n" + + "cwqoj3jeVT8ZQFXlK3fGVaEJsCXd6GWk8ZIWUTn9JZwi2KcCvCU/YiHfx8c7y7Gl\n" + + "SiPXUPsvvkcw6RRh2u4J5tHLIqJe3W58ENoBNK8CgYEApxTBDMKrXTBQxn0w4wfQ\n" + + "A6soPuDYLMBeXj226eswD6KZmDxnYA1zwgcQzPIO2ewm+XKZGrR2PQJezbqbrrHL\n" + + "QkVBcwz49GA5eh8Dg0MGZCki6rhBXK8qqxPfHi2rpkBKG6nUsbBykXeY7XHC75kU\n" + + "kc3WeYsgIzvE908EMAA69hECgYEAinbpiYVZh1DBH+G26MIYZswz4OB5YyHcBevZ\n" + + "2x27v48VmMtUWe4iWopAXVfdA0ZILrD0Gm0b9gRl4IdqudQyxgqcEZ5oLoIBBwjN\n" + + "g0oy83tnwqpQvwLx3p7c79+HqCGmrlK0s/MvQ+e6qMi21t1r5e6hFed5euSA6B8E\n" + + "Cg9ELMcCgYB9bGwlNAE+iuzMIhKev1s7h3TzqKtGw37TtHXvxcTQs3uawJQksQ2s\n" + + "K0Zy1Ta7vybbwAA5m+LxoMT04WUdJO7Cr8/3rBMrbKKO3H7IgC3G+nXnOBdshzn5\n" + + "ifMbhZslFThC/osD5ZV7snXZgTWyPexaINJhHmdrAWpmW1h+UFoiMw==\n" + + "-----END RSA PRIVATE KEY-----\n" + +const CLOUDFRONT_TEST_KEY_64 = Buffer.from( + CLOUDFRONT_TEST_KEY, + "utf-8" +).toString("base64") + +export function withCloudfront() { + withS3() + env._set("CLOUDFRONT_CDN", "http://cf.example.com") + env._set("CLOUDFRONT_PUBLIC_KEY_ID", "keypair_123") + env._set("CLOUDFRONT_PRIVATE_KEY_64", CLOUDFRONT_TEST_KEY_64) +} + +export function withMinio() { + env._set("NODE_ENV", "production") + env._set("MINIO_ENABLED", 1) + env._set("MINIO_URL", "http://minio.example.com") + env._set("CLOUDFRONT_CDN", undefined) +} diff --git a/packages/backend-core/tsconfig.json b/packages/backend-core/tsconfig.json index ccefd149a0..1d9da5f2ae 100644 --- a/packages/backend-core/tsconfig.json +++ b/packages/backend-core/tsconfig.json @@ -8,6 +8,10 @@ } }, "references": [ - { "path": "../types" }, + { "path": "../types" } + ], + "exclude": [ + "node_modules", + "dist" ] } \ No newline at end of file diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 0a25d5fb43..249c614d82 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -470,6 +470,18 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@budibase/nano@10.1.1": + version "10.1.1" + resolved "https://registry.yarnpkg.com/@budibase/nano/-/nano-10.1.1.tgz#36ccda4d9bb64b5ee14dd2b27a295b40739b1038" + integrity sha512-kbMIzMkjVtl+xI0UPwVU0/pn8/ccxTyfzwBz6Z+ZiN2oUSb0fJCe0qwA6o8dxwSa8nZu4MbGAeMJl3CJndmWtA== + dependencies: + "@types/tough-cookie" "^4.0.2" + axios "^1.1.3" + http-cookie-agent "^4.0.2" + node-abort-controller "^3.0.1" + qs "^6.11.0" + tough-cookie "^4.1.2" + "@cspotcode/source-map-support@^0.8.0": version "0.8.1" resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1" @@ -1526,6 +1538,13 @@ asynckit@^0.4.0: resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== +aws-cloudfront-sign@2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/aws-cloudfront-sign/-/aws-cloudfront-sign-2.2.0.tgz#3910f5a6d0d90fec07f2b4ef8ab07f3eefb5625d" + integrity sha512-qG+rwZMP3KRTPPbVmWY8DlrT56AkA4iVOeo23vkdK2EXeW/brJFN2haSNKzVz+oYhFMEIzVVloeAcrEzuRkuVQ== + dependencies: + lodash "^3.6.0" + aws-sdk@2.1030.0: version "2.1030.0" resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.1030.0.tgz#24a856af3d2b8b37c14a8f59974993661c66fd82" @@ -3827,6 +3846,11 @@ lodash@4.17.21, lodash@^4.17.21: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== +lodash@^3.6.0: + version "3.10.1" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6" + integrity sha512-9mDDwqVIma6OZX79ZlDACZl8sBm0TEnkf99zV3iMA4GzkIT/9hiqP5mY0HoT1iNLCrKc/R1HByV+yJfRWVJryQ== + lowercase-keys@^1.0.0, lowercase-keys@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f" @@ -4022,18 +4046,6 @@ msgpackr@^1.5.2: optionalDependencies: msgpackr-extract "^2.1.2" -nano@^10.1.0: - version "10.1.0" - resolved "https://registry.yarnpkg.com/nano/-/nano-10.1.0.tgz#afdd5a7440e62f09a8e23f41fcea328d27383922" - integrity sha512-COeN2TpLcHuSN44QLnPmfZCoCsKAg8/aelPOVqqm/2/MvRHDEA11/Kld5C4sLzDlWlhFZ3SO2WGJGevCsvcEzQ== - dependencies: - "@types/tough-cookie" "^4.0.2" - axios "^1.1.3" - http-cookie-agent "^4.0.2" - node-abort-controller "^3.0.1" - qs "^6.11.0" - tough-cookie "^4.1.2" - napi-macros@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.0.0.tgz#2b6bae421e7b96eb687aa6c77a7858640670001b" diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 18fc6dffcb..20e727f0c6 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.1.46-alpha.6", + "version": "2.2.4-alpha.2", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/string-templates": "2.1.46-alpha.6", + "@budibase/string-templates": "2.2.4-alpha.2", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", "@spectrum-css/avatar": "3.0.2", diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte index c64e69b201..e2ef4e65ce 100644 --- a/packages/bbui/src/Form/Core/Dropzone.svelte +++ b/packages/bbui/src/Form/Core/Dropzone.svelte @@ -15,6 +15,7 @@ export let value = [] export let id = null export let disabled = false + export let compact = false export let fileSizeLimit = BYTES_IN_MB * 20 export let processFiles = null export let deleteAttachments = null @@ -239,70 +240,72 @@ bind:this={fileInput} on:change={handleFile} /> - - - - - - - - - - - - - - - -

- Drag and drop your file -

+ {#if !compact} + + + + + + + + + + + + + + + +

+ Drag and drop your file +

+ {/if} {#if !disabled}

Select a file to upload -
- from your computer + {#if !compact} +
+ from your computer + {/if}

{#if fileTags.length} diff --git a/packages/builder/package.json b/packages/builder/package.json index 1320a538d1..b1d31891c7 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.1.46-alpha.6", + "version": "2.2.4-alpha.2", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "2.1.46-alpha.6", - "@budibase/client": "2.1.46-alpha.6", - "@budibase/frontend-core": "2.1.46-alpha.6", - "@budibase/string-templates": "2.1.46-alpha.6", + "@budibase/bbui": "2.2.4-alpha.2", + "@budibase/client": "2.2.4-alpha.2", + "@budibase/frontend-core": "2.2.4-alpha.2", + "@budibase/string-templates": "2.2.4-alpha.2", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index d09faa34c9..a73db5648b 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -232,6 +232,7 @@ {filters} {bindings} {schemaFields} + datasource={{ type: "table", tableId }} panel={AutomationBindingPanel} fillWidth on:change={e => (tempFilters = e.detail)} diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index 4f5c3375bd..bdf2f46b2c 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -190,6 +190,7 @@ {filters} on:change={onFilter} disabled={!hasCols} + tableId={id} /> {/key} diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte index be9c6259c6..5db4eb5288 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte @@ -6,6 +6,7 @@ export let schema export let filters export let disabled = false + export let tableId const dispatch = createEventDispatcher() @@ -37,6 +38,7 @@ allowBindings={false} {filters} {schemaFields} + datasource={{ type: "table", tableId }} on:change={e => (tempValue = e.detail)} /> diff --git a/packages/builder/src/components/backend/DatasourceNavigator/IntegrationIcon.svelte b/packages/builder/src/components/backend/DatasourceNavigator/IntegrationIcon.svelte index e6cfbf7db8..a0cdabc63c 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/IntegrationIcon.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/IntegrationIcon.svelte @@ -1,18 +1,15 @@ + {#each $builderStore.usedPlugins as plugin} + {/each} {/if} diff --git a/packages/client/src/components/app/Layout.svelte b/packages/client/src/components/app/Layout.svelte index 6582384569..1f0bed214a 100644 --- a/packages/client/src/components/app/Layout.svelte +++ b/packages/client/src/components/app/Layout.svelte @@ -275,7 +275,6 @@ justify-content: center; align-items: stretch; z-index: 1; - border-top: 1px solid var(--spectrum-global-color-gray-300); overflow: hidden; position: relative; } @@ -316,6 +315,12 @@ top: 0; left: 0; } + .layout--top .nav-wrapper { + border-bottom: 1px solid var(--spectrum-global-color-gray-300); + } + .layout--left .nav-wrapper { + border-right: 1px solid var(--spectrum-global-color-gray-300); + } .nav { display: flex; @@ -390,10 +395,6 @@ align-items: stretch; flex: 1 1 auto; z-index: 1; - border-top: 1px solid var(--spectrum-global-color-gray-300); - } - .layout--none .main-wrapper { - border-top: none; } .main { display: flex; @@ -487,7 +488,7 @@ } /* Desktop nav overrides */ - .desktop.layout--left { + .desktop.layout--left .layout-body { flex-direction: row; overflow: hidden; } @@ -523,6 +524,8 @@ top: 0; left: 0; box-shadow: 0 0 8px -1px rgba(0, 0, 0, 0.075); + border-bottom: 1px solid var(--spectrum-global-color-gray-300); + border-right: none; } /* Show close button in drawer */ diff --git a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte index df06979f48..199a6122ab 100644 --- a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte +++ b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte @@ -21,6 +21,7 @@ schema $: dataProviderId = dataProvider?.id + $: datasource = dataProvider?.datasource $: addExtension = getAction( dataProviderId, ActionTypes.AddDataProviderQueryExtension @@ -29,7 +30,7 @@ dataProviderId, ActionTypes.RemoveDataProviderQueryExtension ) - $: fetchSchema(dataProvider || {}) + $: fetchSchema(datasource) $: schemaFields = getSchemaFields(schema, allowedFields) // Add query extension to data provider @@ -42,8 +43,7 @@ } } - async function fetchSchema(dataProvider) { - const datasource = dataProvider?.datasource + async function fetchSchema(datasource) { if (datasource) { schema = await fetchDatasourceSchema(datasource, { enrichRelationships: true, @@ -102,7 +102,7 @@ - + {/if} diff --git a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte index 8d89d3a6ff..5bc3e500a1 100644 --- a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte +++ b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte @@ -15,6 +15,7 @@ export let schemaFields export let filters = [] + export let datasource const context = getContext("context") const BannedTypes = ["link", "attachment", "json"] @@ -59,7 +60,9 @@ // Ensure a valid operator is set const validOperators = LuceneUtils.getValidOperatorsForType( - expression.type + expression.type, + expression.field, + datasource ).map(x => x.value) if (!validOperators.includes(expression.operator)) { expression.operator = @@ -118,7 +121,11 @@ />