diff --git a/.github/workflows/smoke_test.yaml b/.github/workflows/smoke_test.yaml index 79c97b69af..d5a5f0b02a 100644 --- a/.github/workflows/smoke_test.yaml +++ b/.github/workflows/smoke_test.yaml @@ -31,6 +31,7 @@ jobs: continue-on-error: true uses: cypress-io/github-action@v2 with: + record: true install: false command: yarn test:e2e:ci:record env: @@ -48,7 +49,7 @@ jobs: uses: tsickert/discord-webhook@v4.0.0 with: webhook-url: ${{ secrets.BUDI_QA_WEBHOOK }} - content: "Smoke test run completed with ${{ steps.cypress.outcome }}. See results at ${{ steps.cypress.dashboardUrl }}" + content: "Smoke test run completed with ${{ steps.cypress.outcome }}. See results at ${{ steps.cypress.outputs.dashboardUrl }}" embed-title: ${{ steps.cypress.outcome }} embed-color: ${{ steps.cypress.outcome == 'success' && '3066993' || '15548997' }} diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 3bbe718d73..98a949418c 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -114,8 +114,8 @@ spec: value: {{ .Values.globals.google.secret | quote }} - name: AUTOMATION_MAX_ITERATIONS value: {{ .Values.globals.automationMaxIterations | quote }} - - name: EXCLUDE_QUOTAS_TENANTS - value: {{ .Values.globals.excludeQuotasTenants | quote }} + - name: TENANT_FEATURE_FLAGS + value: {{ .Values.globals.tenantFeatureFlags | quote }} image: budibase/apps:{{ .Values.globals.appVersion }} imagePullPolicy: Always diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs index cd5f7b4d48..a127dfbd5c 100644 --- a/hosting/nginx.prod.conf.hbs +++ b/hosting/nginx.prod.conf.hbs @@ -30,7 +30,7 @@ http { log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; - + map $http_upgrade $connection_upgrade { default "upgrade"; } @@ -42,13 +42,13 @@ http { client_max_body_size 1000m; ignore_invalid_headers off; proxy_buffering off; - + set $csp_default "default-src 'self'"; set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io"; set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com"; set $csp_object "object-src 'none'"; set $csp_base_uri "base-uri 'self'"; - set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.s3.*.amazonaws.com"; + set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com"; set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com https://js.intercomcdn.com https://fonts.intercomcdn.com"; set $csp_frame "frame-src 'self' https:"; set $csp_img "img-src http: https: data: blob:"; @@ -58,7 +58,7 @@ http { error_page 502 503 504 /error.html; location = /error.html { - root /usr/share/nginx/html; + root /usr/share/nginx/html; internal; } @@ -154,4 +154,4 @@ http { gzip_comp_level 6; gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml; } -} \ No newline at end of file +} diff --git a/lerna.json b/lerna.json index b373661fa5..0b0dae59bb 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.143-alpha.2", + "version": "1.0.148-alpha.0", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 65f376e216..eda57e22ef 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.143-alpha.2", + "version": "1.0.148-alpha.0", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/backend-core/src/db/conversions.js b/packages/backend-core/src/db/conversions.js index 50d896322f..455cc712d8 100644 --- a/packages/backend-core/src/db/conversions.js +++ b/packages/backend-core/src/db/conversions.js @@ -23,24 +23,30 @@ exports.isDevApp = app => { } /** - * Convert a development app ID to a deployed app ID. + * Generates a development app ID from a real app ID. + * @returns {string} the dev app ID which can be used for dev database. */ -exports.getProdAppID = appId => { - // if dev, convert it - if (appId.startsWith(APP_DEV_PREFIX)) { - const id = appId.split(APP_DEV_PREFIX)[1] - return `${APP_PREFIX}${id}` +exports.getDevelopmentAppID = appId => { + if (!appId || appId.startsWith(APP_DEV_PREFIX)) { + return appId } - return appId + // split to take off the app_ element, then join it together incase any other app_ exist + const split = appId.split(APP_PREFIX) + split.shift() + const rest = split.join(APP_PREFIX) + return `${APP_DEV_PREFIX}${rest}` } /** - * Convert a deployed app ID to a development app ID. + * Convert a development app ID to a deployed app ID. */ -exports.getDevelopmentAppID = appId => { - if (!appId.startsWith(APP_DEV_PREFIX)) { - const id = appId.split(APP_PREFIX)[1] - return `${APP_DEV_PREFIX}${id}` +exports.getProdAppID = appId => { + if (!appId || !appId.startsWith(APP_DEV_PREFIX)) { + return appId } - return appId + // split to take off the app_dev element, then join it together incase any other app_ exist + const split = appId.split(APP_DEV_PREFIX) + split.shift() + const rest = split.join(APP_DEV_PREFIX) + return `${APP_PREFIX}${rest}` } diff --git a/packages/backend-core/src/db/tests/utils.spec.js b/packages/backend-core/src/db/tests/utils.spec.js new file mode 100644 index 0000000000..ebef670a81 --- /dev/null +++ b/packages/backend-core/src/db/tests/utils.spec.js @@ -0,0 +1,61 @@ +const { + generateAppID, + getDevelopmentAppID, + getProdAppID, + isDevAppID, + isProdAppID, +} = require("../utils") + +function getID() { + const appId = generateAppID() + const split = appId.split("_") + const uuid = split[split.length - 1] + const devAppId = `app_dev_${uuid}` + return { appId, devAppId, split, uuid } +} + +describe("app ID manipulation", () => { + it("should be able to generate a new app ID", () => { + expect(generateAppID().startsWith("app_")).toEqual(true) + }) + + it("should be able to convert a production app ID to development", () => { + const { appId, uuid } = getID() + expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`) + }) + + it("should be able to convert a development app ID to development", () => { + const { devAppId, uuid } = getID() + expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`) + }) + + it("should be able to convert a development ID to a production", () => { + const { devAppId, uuid } = getID() + expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`) + }) + + it("should be able to convert a production ID to production", () => { + const { appId, uuid } = getID() + expect(getProdAppID(appId)).toEqual(`app_${uuid}`) + }) + + it("should be able to confirm dev app ID is development", () => { + const { devAppId } = getID() + expect(isDevAppID(devAppId)).toEqual(true) + }) + + it("should be able to confirm prod app ID is not development", () => { + const { appId } = getID() + expect(isDevAppID(appId)).toEqual(false) + }) + + it("should be able to confirm prod app ID is prod", () => { + const { appId } = getID() + expect(isProdAppID(appId)).toEqual(true) + }) + + it("should be able to confirm dev app ID is not prod", () => { + const { devAppId } = getID() + expect(isProdAppID(devAppId)).toEqual(false) + }) +}) \ No newline at end of file diff --git a/packages/backend-core/src/db/utils.js b/packages/backend-core/src/db/utils.js index 1e2ba846de..5f7bf794c2 100644 --- a/packages/backend-core/src/db/utils.js +++ b/packages/backend-core/src/db/utils.js @@ -43,6 +43,18 @@ exports.isDevAppID = isDevAppID exports.getDevelopmentAppID = getDevelopmentAppID exports.getProdAppID = getProdAppID +/** + * Generates a new app ID. + * @returns {string} The new app ID which the app doc can be stored under. + */ +exports.generateAppID = (tenantId = null) => { + let id = APP_PREFIX + if (tenantId) { + id += `${tenantId}${SEPARATOR}` + } + return `${id}${newid()}` +} + /** * If creating DB allDocs/query params with only a single top level ID this can be used, this * is usually the case as most of our docs are top level e.g. tables, automations, users and so on. diff --git a/packages/backend-core/src/environment.js b/packages/backend-core/src/environment.js index c4fc4a87c8..f628e899ad 100644 --- a/packages/backend-core/src/environment.js +++ b/packages/backend-core/src/environment.js @@ -34,6 +34,12 @@ module.exports = { COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, PLATFORM_URL: process.env.PLATFORM_URL, TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS, + BACKUPS_BUCKET_NAME: process.env.BACKUPS_BUCKET_NAME || "backups", + APPS_BUCKET_NAME: process.env.APPS_BUCKET_NAME || "prod-budi-app-assets", + TEMPLATES_BUCKET_NAME: process.env.TEMPLATES_BUCKET_NAME || "templates", + GLOBAL_BUCKET_NAME: process.env.GLOBAL_BUCKET_NAME || "global", + GLOBAL_CLOUD_BUCKET_NAME: + process.env.GLOBAL_CLOUD_BUCKET_NAME || "prod-budi-tenant-uploads", USE_COUCH: process.env.USE_COUCH || true, isTest, isDev, diff --git a/packages/backend-core/src/featureFlags/index.js b/packages/backend-core/src/featureFlags/index.js index 6d3d86978a..c050cbdfef 100644 --- a/packages/backend-core/src/featureFlags/index.js +++ b/packages/backend-core/src/featureFlags/index.js @@ -49,4 +49,5 @@ exports.getTenantFeatureFlags = tenantId => { exports.FeatureFlag = { LICENSING: "LICENSING", + GOOGLE_SHEETS: "GOOGLE_SHEETS", } diff --git a/packages/backend-core/src/objectStore/utils.js b/packages/backend-core/src/objectStore/utils.js index 1634a24981..a243553df8 100644 --- a/packages/backend-core/src/objectStore/utils.js +++ b/packages/backend-core/src/objectStore/utils.js @@ -1,12 +1,13 @@ const { join } = require("path") const { tmpdir } = require("os") +const env = require("../environment") exports.ObjectStoreBuckets = { - BACKUPS: "backups", - APPS: "prod-budi-app-assets", - TEMPLATES: "templates", - GLOBAL: "global", - GLOBAL_CLOUD: "prod-budi-tenant-uploads", + BACKUPS: env.BACKUPS_BUCKET_NAME, + APPS: env.APPS_BUCKET_NAME, + TEMPLATES: env.TEMPLATES_BUCKET_NAME, + GLOBAL: env.GLOBAL_BUCKET_NAME, + GLOBAL_CLOUD: env.GLOBAL_CLOUD_BUCKET_NAME, } exports.budibaseTempDir = function () { diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 515595dbd2..df3a503b2d 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.143-alpha.2", + "version": "1.0.148-alpha.0", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^1.0.143-alpha.2", + "@budibase/string-templates": "^1.0.148-alpha.0", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 230da9e7c8..924e0c9db1 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.143-alpha.2", + "version": "1.0.148-alpha.0", "license": "GPL-3.0", "private": true, "scripts": { @@ -67,10 +67,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.143-alpha.2", - "@budibase/client": "^1.0.143-alpha.2", - "@budibase/frontend-core": "^1.0.143-alpha.2", - "@budibase/string-templates": "^1.0.143-alpha.2", + "@budibase/bbui": "^1.0.148-alpha.0", + "@budibase/client": "^1.0.148-alpha.0", + "@budibase/frontend-core": "^1.0.148-alpha.0", + "@budibase/string-templates": "^1.0.148-alpha.0", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/builder/src/pages/builder/portal/manage/users/_components/BasicOnboardingModal.svelte b/packages/builder/src/pages/builder/portal/manage/users/_components/BasicOnboardingModal.svelte index dd3b37fce5..a4b06f45a2 100644 --- a/packages/builder/src/pages/builder/portal/manage/users/_components/BasicOnboardingModal.svelte +++ b/packages/builder/src/pages/builder/portal/manage/users/_components/BasicOnboardingModal.svelte @@ -47,7 +47,7 @@ diff --git a/packages/cli/package.json b/packages/cli/package.json index 2aa370516b..3a9a9db663 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.143-alpha.2", + "version": "1.0.148-alpha.0", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 8ede3d9f0e..00cc0894da 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.143-alpha.2", + "version": "1.0.148-alpha.0", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.143-alpha.2", - "@budibase/frontend-core": "^1.0.143-alpha.2", - "@budibase/string-templates": "^1.0.143-alpha.2", + "@budibase/bbui": "^1.0.148-alpha.0", + "@budibase/frontend-core": "^1.0.148-alpha.0", + "@budibase/string-templates": "^1.0.148-alpha.0", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/client/src/components/app/forms/FormStep.svelte b/packages/client/src/components/app/forms/FormStep.svelte index 22972c5c48..4441f515ee 100644 --- a/packages/client/src/components/app/forms/FormStep.svelte +++ b/packages/client/src/components/app/forms/FormStep.svelte @@ -22,7 +22,7 @@ if ( formContext && $builderStore.inBuilder && - $componentStore?.selectedComponentPath?.includes($component.id) + $componentStore.selectedComponentPath?.includes($component.id) ) { formContext.formApi.setStep(step) } diff --git a/packages/client/src/components/preview/SettingsBar.svelte b/packages/client/src/components/preview/SettingsBar.svelte index bf0b48250a..154a115c98 100644 --- a/packages/client/src/components/preview/SettingsBar.svelte +++ b/packages/client/src/components/preview/SettingsBar.svelte @@ -155,7 +155,7 @@ icon="Duplicate" on:click={() => { builderStore.actions.duplicateComponent( - $builderStore.selectedComponent._id + $builderStore.selectedComponentId ) }} title="Duplicate component" diff --git a/packages/client/src/sdk.js b/packages/client/src/sdk.js index 50ec07ba98..db31da7164 100644 --- a/packages/client/src/sdk.js +++ b/packages/client/src/sdk.js @@ -7,6 +7,7 @@ import { builderStore, uploadStore, rowSelectionStore, + componentStore, } from "stores" import { styleable } from "utils/styleable" import { linkable } from "utils/linkable" @@ -24,6 +25,7 @@ export default { screenStore, builderStore, uploadStore, + componentStore, styleable, linkable, getAction, diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 5fda245283..4ee38a0730 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "1.0.143-alpha.2", + "version": "1.0.148-alpha.0", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^1.0.143-alpha.2", + "@budibase/bbui": "^1.0.148-alpha.0", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/server/__mocks__/@google-cloud/firestore.ts b/packages/server/__mocks__/@google-cloud/firestore.ts new file mode 100644 index 0000000000..a438d6a7c5 --- /dev/null +++ b/packages/server/__mocks__/@google-cloud/firestore.ts @@ -0,0 +1,36 @@ +module FirebaseMock { + const firebase: any = {} + + firebase.Firestore = function () { + this.get = jest.fn(() => [ + { + data: jest.fn(() => ({ result: "test" })), + }, + ]) + + this.update = jest.fn() + this.set = jest.fn() + this.delete = jest.fn() + + this.doc = jest.fn(() => ({ + update: this.update, + set: this.set, + delete: this.delete, + get: jest.fn(() => ({ + data: jest.fn(() => ({ result: "test" })), + })), + id: "test_id", + })) + + this.where = jest.fn(() => ({ + get: this.get, + })) + + this.collection = jest.fn(() => ({ + doc: this.doc, + where: this.where, + })) + } + + module.exports = firebase +} diff --git a/packages/server/__mocks__/pg.ts b/packages/server/__mocks__/pg.ts index af2ae24a97..44aeabcb38 100644 --- a/packages/server/__mocks__/pg.ts +++ b/packages/server/__mocks__/pg.ts @@ -14,21 +14,13 @@ module PgMock { function Client() {} Client.prototype.query = query + Client.prototype.end = jest.fn() Client.prototype.connect = jest.fn() Client.prototype.release = jest.fn() - function Pool() {} - const on = jest.fn() - Pool.prototype.query = query - Pool.prototype.connect = jest.fn(() => { - // @ts-ignore - return new Client() - }) - Pool.prototype.on = on pg.Client = Client - pg.Pool = Pool pg.queryMock = query pg.on = on diff --git a/packages/server/package.json b/packages/server/package.json index 8ed5c5168b..dcbd7bfed1 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.143-alpha.2", + "version": "1.0.148-alpha.0", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -68,10 +68,10 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.143-alpha.2", - "@budibase/client": "^1.0.143-alpha.2", - "@budibase/pro": "1.0.143-alpha.2", - "@budibase/string-templates": "^1.0.143-alpha.2", + "@budibase/backend-core": "^1.0.148-alpha.0", + "@budibase/client": "^1.0.148-alpha.0", + "@budibase/pro": "1.0.148-alpha.0", + "@budibase/string-templates": "^1.0.148-alpha.0", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/src/api/controllers/integration.js b/packages/server/src/api/controllers/integration.js index f3f3309c02..28748541c4 100644 --- a/packages/server/src/api/controllers/integration.js +++ b/packages/server/src/api/controllers/integration.js @@ -1,21 +1,16 @@ const { cloneDeep } = require("lodash") const { definitions } = require("../../integrations") -const { getTenantId } = require("@budibase/backend-core/tenancy") const { SourceNames } = require("../../definitions/datasource") const googlesheets = require("../../integrations/googlesheets") -const env = require("../../environment") +const { featureFlags } = require("@budibase/backend-core") exports.fetch = async function (ctx) { ctx.status = 200 const defs = cloneDeep(definitions) // for google sheets integration google verification - if (env.EXCLUDE_QUOTAS_TENANTS) { - const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.split(",") - const tenantId = getTenantId() - if (excludedTenants.includes(tenantId)) { - defs[SourceNames.GOOGLE_SHEETS] = googlesheets.schema - } + if (featureFlags.isEnabled(featureFlags.FeatureFlag.GOOGLE_SHEETS)) { + defs[SourceNames.GOOGLE_SHEETS] = googlesheets.schema } ctx.body = defs diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index d7063e590b..c8c8ae8e58 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -175,9 +175,10 @@ module External { const thisRow: Row = {} // filter the row down to what is actually the row (not joined) for (let fieldName of Object.keys(table.schema)) { - const value = row[`${table.name}.${fieldName}`] || row[fieldName] + const pathValue = row[`${table.name}.${fieldName}`] + const value = pathValue != null ? pathValue : row[fieldName] // all responses include "select col as table.col" so that overlaps are handled - if (value) { + if (value != null) { thisRow[fieldName] = value } } diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js index e40773bd0e..280596928b 100644 --- a/packages/server/src/db/utils.js +++ b/packages/server/src/db/utils.js @@ -9,6 +9,8 @@ const { StaticDatabases, isDevAppID, isProdAppID, + getDevelopmentAppID, + generateAppID, } = require("@budibase/backend-core/db") const UNICODE_MAX = "\ufff0" @@ -80,6 +82,8 @@ exports.UNICODE_MAX = UNICODE_MAX exports.SearchIndexes = SearchIndexes exports.AppStatus = AppStatus exports.BudibaseInternalDB = BudibaseInternalDB +exports.generateAppID = generateAppID +exports.generateDevAppID = getDevelopmentAppID exports.generateRoleID = generateRoleID exports.getRoleParams = getRoleParams @@ -243,28 +247,6 @@ exports.getLinkParams = (otherProps = {}) => { return getDocParams(DocumentTypes.LINK, null, otherProps) } -/** - * Generates a new app ID. - * @returns {string} The new app ID which the app doc can be stored under. - */ -exports.generateAppID = (tenantId = null) => { - let id = `${DocumentTypes.APP}${SEPARATOR}` - if (tenantId) { - id += `${tenantId}${SEPARATOR}` - } - return `${id}${newid()}` -} - -/** - * Generates a development app ID from a real app ID. - * @returns {string} the dev app ID which can be used for dev database. - */ -exports.generateDevAppID = appId => { - const prefix = `${DocumentTypes.APP}${SEPARATOR}` - const rest = appId.split(prefix)[1] - return `${DocumentTypes.APP_DEV}${SEPARATOR}${rest}` -} - /** * Generates a new layout ID. * @returns {string} The new layout ID which the layout doc can be stored under. diff --git a/packages/server/src/integrations/firebase.ts b/packages/server/src/integrations/firebase.ts index 503dae5c95..b985797b4f 100644 --- a/packages/server/src/integrations/firebase.ts +++ b/packages/server/src/integrations/firebase.ts @@ -92,13 +92,13 @@ module Firebase { class FirebaseIntegration implements IntegrationBase { private config: FirebaseConfig - private db: Firestore + private client: Firestore constructor(config: FirebaseConfig) { this.config = config if (config.serviceAccount) { const serviceAccount = JSON.parse(config.serviceAccount) - this.db = new Firestore({ + this.client = new Firestore({ projectId: serviceAccount.project_id, credentials: { client_email: serviceAccount.client_email, @@ -106,7 +106,7 @@ module Firebase { }, }) } else { - this.db = new Firestore({ + this.client = new Firestore({ projectId: config.projectId, credentials: { client_email: config.email, @@ -118,7 +118,7 @@ module Firebase { async create(query: { json: object; extra: { [key: string]: string } }) { try { - const documentReference = this.db + const documentReference = this.client .collection(query.extra.collection) .doc() await documentReference.set({ ...query.json, id: documentReference.id }) @@ -133,7 +133,7 @@ module Firebase { async read(query: { json: object; extra: { [key: string]: string } }) { try { let snapshot - const collectionRef = this.db.collection(query.extra.collection) + const collectionRef = this.client.collection(query.extra.collection) if ( query.extra.filterField && query.extra.filter && @@ -164,19 +164,19 @@ module Firebase { extra: { [key: string]: string } }) { try { - await this.db + await this.client .collection(query.extra.collection) .doc(query.json.id) .update(query.json) return ( - await this.db + await this.client .collection(query.extra.collection) .doc(query.json.id) .get() ).data() } catch (err) { - console.error("Error writing to firebase", err) + console.error("Error writing to Firestore", err) throw err } } @@ -186,13 +186,13 @@ module Firebase { extra: { [key: string]: string } }) { try { - await this.db + await this.client .collection(query.extra.collection) .doc(query.json.id) .delete() return true } catch (err) { - console.error("Error writing to mongodb", err) + console.error("Error deleting from Firestore", err) throw err } } diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 711e9d2262..f3e7a5fc3a 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -46,6 +46,7 @@ const INTEGRATIONS = { [SourceNames.FIREBASE]: firebase.integration, [SourceNames.GOOGLE_SHEETS]: googlesheets.integration, [SourceNames.REDIS]: redis.integration, + [SourceNames.FIREBASE]: firebase.integration, } // optionally add oracle integration if the oracle binary can be installed diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 01257f3aa0..1da0ab31af 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -16,7 +16,7 @@ import { import { DatasourcePlus } from "./base/datasourcePlus" module PostgresModule { - const { Pool } = require("pg") + const { Client } = require("pg") const Sql = require("./base/sql") const { escapeDangerousCharacters } = require("../utilities") @@ -104,7 +104,6 @@ module PostgresModule { } class PostgresIntegration extends Sql implements DatasourcePlus { - static pool: any private readonly client: any private readonly config: PostgresConfig private index: number = 1 @@ -136,11 +135,7 @@ module PostgresModule { } : undefined, } - if (!this.pool) { - this.pool = new Pool(newConfig) - } - - this.client = this.pool + this.client = new Client(newConfig) this.setSchema() } @@ -171,16 +166,17 @@ module PostgresModule { } catch (err) { // @ts-ignore throw new Error(err) + } finally { + await this.client.end() } } - setSchema() { + async setSchema() { + await this.client.connect() if (!this.config.schema) { this.config.schema = "public" } - this.client.on("connect", (client: any) => { - client.query(`SET search_path TO ${this.config.schema}`) - }) + this.client.query(`SET search_path TO ${this.config.schema}`) this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'` } @@ -208,6 +204,8 @@ module PostgresModule { } } catch (err) { tableKeys = {} + } finally { + await this.client.close() } const columnsResponse = await this.client.query(this.COLUMNS_SQL) diff --git a/packages/server/src/integrations/tests/firebase.spec.js b/packages/server/src/integrations/tests/firebase.spec.js new file mode 100644 index 0000000000..97d3b2c0d7 --- /dev/null +++ b/packages/server/src/integrations/tests/firebase.spec.js @@ -0,0 +1,92 @@ +const firebase = require("@google-cloud/firestore") +const FirebaseIntegration = require("../firebase") +jest.mock("@google-cloud/firestore") + +class TestConfiguration { + constructor(config = {}) { + this.integration = new FirebaseIntegration.integration(config) + } +} + +describe("Firebase Integration", () => { + let config + let tableName = "Users" + + beforeEach(() => { + config = new TestConfiguration({ + serviceAccount: "{}" + }) + }) + + it("calls the create method with the correct params", async () => { + await config.integration.create({ + table: tableName, + json: { + Name: "Test Name" + }, + extra: { + collection: "test" + } + }) + expect(config.integration.client.collection).toHaveBeenCalledWith("test") + expect(config.integration.client.set).toHaveBeenCalledWith({ + Name: "Test Name", + id: "test_id" + }) + }) + + it("calls the read method with the correct params", async () => { + const response = await config.integration.read({ + table: tableName, + json: { + Name: "Test" + }, + extra: { + collection: "test", + filterField: "field", + filter: "==", + filterValue: "value", + } + }) + expect(config.integration.client.collection).toHaveBeenCalledWith("test") + expect(config.integration.client.where).toHaveBeenCalledWith("field", "==", "value") + expect(response).toEqual([{ result: "test"}]) + }) + + it("calls the update method with the correct params", async () => { + const response = await config.integration.update({ + table: tableName, + json: { + id: "test", + Name: "Test" + }, + extra: { + collection: "test" + } + }) + expect(config.integration.client.collection).toHaveBeenCalledWith("test") + expect(config.integration.client.update).toHaveBeenCalledWith({ + Name: "Test", + id: "test" + }) + expect(response).toEqual({ + result: "test" + }) + }) + + it("calls the delete method with the correct params", async () => { + const response = await config.integration.delete({ + table: tableName, + json: { + id: "test", + Name: "Test" + }, + extra: { + collection: "test" + } + }) + expect(config.integration.client.collection).toHaveBeenCalledWith("test") + expect(config.integration.client.doc).toHaveBeenCalledWith("test") + expect(config.integration.client.delete).toHaveBeenCalled() + }) +}) \ No newline at end of file diff --git a/packages/server/src/integrations/tests/postgres.spec.js b/packages/server/src/integrations/tests/postgres.spec.js index 5c0d086ce0..4ce5f12e96 100644 --- a/packages/server/src/integrations/tests/postgres.spec.js +++ b/packages/server/src/integrations/tests/postgres.spec.js @@ -15,10 +15,6 @@ describe("Postgres Integration", () => { config = new TestConfiguration() }) - it("calls the connection callback", async () => { - expect(pg.on).toHaveBeenCalledWith('connect', expect.anything()) - }) - it("calls the create method with the correct params", async () => { const sql = "insert into users (name, age) values ('Joe', 123);" await config.integration.create({ diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index e24b226eda..50c20c9f4c 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1014,10 +1014,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.138": - version "1.0.138" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.138.tgz#5297d6cf5b9ec8c15f0a6df4c7d8273b8ac900f0" - integrity sha512-1qN/5urKX8bBXwEz266Z94rco8dTI7VqIh75m8ZcqrAfoUpjvZJS76gZxfc5U/QWPwrgVFnLtYvnEjaLbGEflg== +"@budibase/backend-core@1.0.147": + version "1.0.147" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.147.tgz#fc93a803e97e304170b33bd28b4f5deda32bec18" + integrity sha512-0GcF9G/tTAsko9g352MB8K3EtMTVb7v7Av6RdsYyKBdVtOD42HKFzSOD0n/RQUL4v70YByiu99zJAB6z1m1Pcg== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -1091,12 +1091,12 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@1.0.138": - version "1.0.138" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.138.tgz#cacbebe5ce93eb533af62a794a638944c2c61544" - integrity sha512-4ABlUZvl2h8sd8awJATf3KJeoFWV/8SoqdbKiH1ICdUcM/6dad7nhbJ15QqJL+Uuh/+mN2yEbr8V6Un2+yF+CA== +"@budibase/pro@1.0.147": + version "1.0.147" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.147.tgz#c1ec9d92ffaa40285a83c57ddbec1f32802cd9a1" + integrity sha512-e4im6Byqeeit/QFHkVhVdRmgIlJJY/W06cQrOuiG/1ngO4PGnXeJqtjQjHVfvchD5Xi3y1MgQ4AtH2Bzb5LEhw== dependencies: - "@budibase/backend-core" "1.0.138" + "@budibase/backend-core" "1.0.147" node-fetch "^2.6.1" "@budibase/standard-components@^0.9.139": diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 17c0cde2f3..6970474c12 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.143-alpha.2", + "version": "1.0.148-alpha.0", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 19e419bc7f..5cb3f800ee 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.143-alpha.2", + "version": "1.0.148-alpha.0", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -31,9 +31,9 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.143-alpha.2", - "@budibase/pro": "1.0.143-alpha.2", - "@budibase/string-templates": "^1.0.143-alpha.2", + "@budibase/backend-core": "^1.0.148-alpha.0", + "@budibase/pro": "1.0.148-alpha.0", + "@budibase/string-templates": "^1.0.148-alpha.0", "@koa/router": "^8.0.0", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "^0.3.0", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index a2a29f5f77..80f2996bec 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -293,10 +293,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.138": - version "1.0.138" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.138.tgz#5297d6cf5b9ec8c15f0a6df4c7d8273b8ac900f0" - integrity sha512-1qN/5urKX8bBXwEz266Z94rco8dTI7VqIh75m8ZcqrAfoUpjvZJS76gZxfc5U/QWPwrgVFnLtYvnEjaLbGEflg== +"@budibase/backend-core@1.0.147": + version "1.0.147" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.147.tgz#fc93a803e97e304170b33bd28b4f5deda32bec18" + integrity sha512-0GcF9G/tTAsko9g352MB8K3EtMTVb7v7Av6RdsYyKBdVtOD42HKFzSOD0n/RQUL4v70YByiu99zJAB6z1m1Pcg== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -321,12 +321,12 @@ uuid "^8.3.2" zlib "^1.0.5" -"@budibase/pro@1.0.138": - version "1.0.138" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.138.tgz#cacbebe5ce93eb533af62a794a638944c2c61544" - integrity sha512-4ABlUZvl2h8sd8awJATf3KJeoFWV/8SoqdbKiH1ICdUcM/6dad7nhbJ15QqJL+Uuh/+mN2yEbr8V6Un2+yF+CA== +"@budibase/pro@1.0.147": + version "1.0.147" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.147.tgz#c1ec9d92ffaa40285a83c57ddbec1f32802cd9a1" + integrity sha512-e4im6Byqeeit/QFHkVhVdRmgIlJJY/W06cQrOuiG/1ngO4PGnXeJqtjQjHVfvchD5Xi3y1MgQ4AtH2Bzb5LEhw== dependencies: - "@budibase/backend-core" "1.0.138" + "@budibase/backend-core" "1.0.147" node-fetch "^2.6.1" "@cspotcode/source-map-consumer@0.8.0":