From ac944e532bfb176b3ed63ea4e3e675976267c649 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Sep 2021 18:24:09 +0100 Subject: [PATCH 01/55] Moving views into a different location so they don't trigger tree creation and attempting to use in memory pouchDB to run views on the fly. --- .../src/api/controllers/row/internal.js | 78 ++++++++--- .../server/src/api/controllers/view/index.js | 122 ++++++++++++++---- packages/server/src/db/inMemoryView.js | 38 ++++++ packages/server/src/db/utils.js | 9 ++ packages/server/src/environment.js | 7 + 5 files changed, 207 insertions(+), 47 deletions(-) create mode 100644 packages/server/src/db/inMemoryView.js diff --git a/packages/server/src/api/controllers/row/internal.js b/packages/server/src/api/controllers/row/internal.js index 2299a20580..9cfca69fa8 100644 --- a/packages/server/src/api/controllers/row/internal.js +++ b/packages/server/src/api/controllers/row/internal.js @@ -5,6 +5,7 @@ const { generateRowID, DocumentTypes, InternalTables, + generateMemoryViewID, } = require("../../../db/utils") const userController = require("../user") const { @@ -16,6 +17,8 @@ const { isEqual } = require("lodash") const { validate, findRow } = require("./utils") const { fullSearch, paginatedSearch } = require("./internalSearch") const { getGlobalUsersFromMetadata } = require("../../../utilities/global") +const inMemoryViews = require("../../../db/inMemoryView") +const env = require("../../../environment") const CALCULATION_TYPES = { SUM: "sum", @@ -36,6 +39,40 @@ async function storeResponse(ctx, db, row, oldTable, table) { return { row, table } } +// doesn't do the outputProcessing +async function getRawTableData(ctx, db, tableId) { + let rows + if (tableId === InternalTables.USER_METADATA) { + await userController.fetchMetadata(ctx) + rows = ctx.body + } else { + const response = await db.allDocs( + getRowParams(tableId, null, { + include_docs: true, + }) + ) + rows = response.rows.map(row => row.doc) + } + return rows +} + +async function getView(db, viewName) { + let viewInfo + if (env.SELF_HOSTED) { + const designDoc = await db.get("_design/database") + viewInfo = designDoc.views[viewName] + } else { + viewInfo = await db.get(generateMemoryViewID(viewName)) + if (viewInfo) { + viewInfo = viewInfo.view + } + } + if (!viewInfo) { + throw "View does not exist." + } + return viewInfo +} + exports.patch = async ctx => { const appId = ctx.appId const db = new CouchDB(appId) @@ -139,15 +176,28 @@ exports.fetchView = async ctx => { const db = new CouchDB(appId) const { calculation, group, field } = ctx.query - const designDoc = await db.get("_design/database") - const viewInfo = designDoc.views[viewName] + const viewInfo = await getView(db, viewName) if (!viewInfo) { throw "View does not exist." } - const response = await db.query(`database/${viewName}`, { - include_docs: !calculation, - group: !!group, - }) + let response + // TODO: make sure not self hosted in Cloud + if (!env.SELF_HOSTED) { + response = await db.query(`database/${viewName}`, { + include_docs: !calculation, + group: !!group, + }) + } else { + const tableId = viewInfo.meta.tableId + const data = await getRawTableData(ctx, db, tableId) + response = await inMemoryViews.runView( + appId, + viewInfo, + calculation, + group, + data + ) + } let rows if (!calculation) { @@ -191,19 +241,9 @@ exports.fetch = async ctx => { const appId = ctx.appId const db = new CouchDB(appId) - let rows, - table = await db.get(ctx.params.tableId) - if (ctx.params.tableId === InternalTables.USER_METADATA) { - await userController.fetchMetadata(ctx) - rows = ctx.body - } else { - const response = await db.allDocs( - getRowParams(ctx.params.tableId, null, { - include_docs: true, - }) - ) - rows = response.rows.map(row => row.doc) - } + const tableId = ctx.params.tableId + let table = await db.get(tableId) + let rows = await getRawTableData(ctx, db, tableId) return outputProcessing(ctx, table, rows) } diff --git a/packages/server/src/api/controllers/view/index.js b/packages/server/src/api/controllers/view/index.js index 3d0f236fce..4c2e6ee08b 100644 --- a/packages/server/src/api/controllers/view/index.js +++ b/packages/server/src/api/controllers/view/index.js @@ -3,14 +3,27 @@ const viewTemplate = require("./viewBuilder") const { apiFileReturn } = require("../../../utilities/fileSystem") const exporters = require("./exporters") const { fetchView } = require("../row") -const { ViewNames } = require("../../../db/utils") +const { + ViewNames, + generateMemoryViewID, + getMemoryViewParams, +} = require("../../../db/utils") +const env = require("../../../environment") -const controller = { - fetch: async ctx => { - const db = new CouchDB(ctx.appId) +async function getView(db, viewName) { + if (env.SELF_HOSTED) { const designDoc = await db.get("_design/database") - const response = [] + return designDoc.views[viewName] + } else { + const viewDoc = await db.get(generateMemoryViewID(viewName)) + return viewDoc.view + } +} +async function getViews(db) { + const response = [] + if (env.SELF_HOSTED) { + const designDoc = await db.get("_design/database") for (let name of Object.keys(designDoc.views)) { // Only return custom views, not built ins if (Object.values(ViewNames).indexOf(name) !== -1) { @@ -21,30 +34,91 @@ const controller = { ...designDoc.views[name], }) } + } else { + const views = ( + await db.allDocs( + getMemoryViewParams({ + include_docs: true, + }) + ) + ).rows.map(row => row.doc) + for (let viewDoc of views) { + response.push({ + name: viewDoc.name, + ...viewDoc.view, + }) + } + } + return response +} - ctx.body = response +async function saveView(db, originalName, viewToSave, viewTemplate) { + if (env.SELF_HOSTED) { + const designDoc = await db.get("_design/database") + designDoc.views = { + ...designDoc.views, + [viewToSave.name]: viewTemplate, + } + // view has been renamed + if (originalName) { + delete designDoc.views[originalName] + } + await db.put(designDoc) + } else { + const id = generateMemoryViewID(viewToSave.name) + const originalId = originalName ? generateMemoryViewID(originalName) : null + const viewDoc = { + _id: id, + view: viewTemplate, + name: viewToSave.name, + tableId: viewTemplate.meta.tableId, + } + try { + const old = await db.get(id) + if (originalId) { + const originalDoc = await db.get(originalId) + await db.remove(originalDoc._id, originalDoc._rev) + } + if (old && old._rev) { + viewDoc._rev = old._rev + } + } catch (err) { + // didn't exist, just skip + } + await db.put(viewDoc) + } +} + +async function deleteView(db, viewName) { + if (env.SELF_HOSTED) { + const designDoc = await db.get("_design/database") + const view = designDoc.views[viewName] + delete designDoc.views[viewName] + await db.put(designDoc) + return view + } else { + const id = generateMemoryViewID(viewName) + const viewDoc = await db.get(id) + await db.remove(viewDoc._id, viewDoc._rev) + return viewDoc.view + } +} + +const controller = { + fetch: async ctx => { + const db = new CouchDB(ctx.appId) + ctx.body = await getViews(db) }, save: async ctx => { const db = new CouchDB(ctx.appId) const { originalName, ...viewToSave } = ctx.request.body - const designDoc = await db.get("_design/database") const view = viewTemplate(viewToSave) if (!viewToSave.name) { ctx.throw(400, "Cannot create view without a name") } - designDoc.views = { - ...designDoc.views, - [viewToSave.name]: view, - } - - // view has been renamed - if (originalName) { - delete designDoc.views[originalName] - } - - await db.put(designDoc) + await saveView(db, originalName, viewToSave, view) // add views to table document const table = await db.get(ctx.request.body.tableId) @@ -53,11 +127,9 @@ const controller = { view.meta.schema = table.schema } table.views[viewToSave.name] = view.meta - if (originalName) { delete table.views[originalName] } - await db.put(table) ctx.body = { @@ -67,13 +139,8 @@ const controller = { }, destroy: async ctx => { const db = new CouchDB(ctx.appId) - const designDoc = await db.get("_design/database") const viewName = decodeURI(ctx.params.viewName) - const view = designDoc.views[viewName] - delete designDoc.views[viewName] - - await db.put(designDoc) - + const view = await deleteView(db, viewName) const table = await db.get(view.meta.tableId) delete table.views[viewName] await db.put(table) @@ -82,10 +149,9 @@ const controller = { }, exportView: async ctx => { const db = new CouchDB(ctx.appId) - const designDoc = await db.get("_design/database") const viewName = decodeURI(ctx.query.view) + const view = await getView(db, viewName) - const view = designDoc.views[viewName] const format = ctx.query.format if (!format) { ctx.throw(400, "Format must be specified, either csv or json") diff --git a/packages/server/src/db/inMemoryView.js b/packages/server/src/db/inMemoryView.js new file mode 100644 index 0000000000..f82b418b7f --- /dev/null +++ b/packages/server/src/db/inMemoryView.js @@ -0,0 +1,38 @@ +const PouchDB = require("pouchdb") +const memory = require("pouchdb-adapter-memory") + +PouchDB.plugin(memory) +const Pouch = PouchDB.defaults({ + prefix: undefined, + adapter: "memory", +}) + +exports.runView = async (appId, view, calculation, group, data) => { + // appId doesn't really do anything since its all in memory + // use it just incase multiple databases at the same time + const db = new Pouch(appId) + await db.put({ + _id: "_design/database", + views: { + runner: view, + }, + }) + // write all the docs to the in memory Pouch + await db.bulkDocs(data) + const response = await db.query(`database/runner`, { + include_docs: !calculation, + group: !!group, + }) + // need to fix the revs to be totally accurate + for (let row of response.rows) { + if (!row._rev || !row._id) { + continue + } + const found = data.find(possible => possible._id === row._id) + if (found) { + row._rev = found._rev + } + } + await db.destroy() + return response +} diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js index ec1c267fa2..3e20b30869 100644 --- a/packages/server/src/db/utils.js +++ b/packages/server/src/db/utils.js @@ -39,6 +39,7 @@ const DocumentTypes = { QUERY: "query", DEPLOYMENTS: "deployments", METADATA: "metadata", + MEM_VIEW: "view", } const ViewNames = { @@ -348,6 +349,14 @@ exports.getMetadataParams = (type, entityId = null, otherProps = {}) => { return getDocParams(DocumentTypes.METADATA, docId, otherProps) } +exports.generateMemoryViewID = viewName => { + return `${DocumentTypes.MEM_VIEW}${SEPARATOR}${viewName}` +} + +exports.getMemoryViewParams = (otherProps = {}) => { + return getDocParams(DocumentTypes.MEM_VIEW, null, otherProps) +} + /** * This can be used with the db.allDocs to get a list of IDs */ diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 9e029e440a..89e015b6f5 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -66,3 +66,10 @@ module.exports = { return !isDev() }, } + +// convert any strings to numbers if required, like "0" would be true otherwise +for (let [key, value] of Object.entries(module.exports)) { + if (typeof value === "string" && !isNaN(parseInt(value))) { + module.exports[key] = parseInt(value) + } +} From 0651b874ac54130a0186268e3730c4786e22491f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 20 Sep 2021 19:21:04 +0100 Subject: [PATCH 02/55] Fixing issues with automation steps found during testing. --- .../src/api/controllers/row/internal.js | 22 +++++++++++++--- .../server/src/automations/steps/createRow.js | 13 ++++------ .../server/src/automations/steps/deleteRow.js | 18 ++++++------- .../server/src/automations/steps/queryRows.js | 25 ++++++++----------- .../server/src/automations/steps/updateRow.js | 17 ++++++------- packages/server/src/db/inMemoryView.js | 11 +++++--- .../src/utilities/rowProcessor/index.js | 10 +++++--- 7 files changed, 65 insertions(+), 51 deletions(-) diff --git a/packages/server/src/api/controllers/row/internal.js b/packages/server/src/api/controllers/row/internal.js index 9cfca69fa8..77e3069239 100644 --- a/packages/server/src/api/controllers/row/internal.js +++ b/packages/server/src/api/controllers/row/internal.js @@ -11,6 +11,7 @@ const userController = require("../user") const { inputProcessing, outputProcessing, + processAutoColumn, } = require("../../../utilities/rowProcessor") const { FieldTypes } = require("../../../constants") const { isEqual } = require("lodash") @@ -28,11 +29,26 @@ const CALCULATION_TYPES = { async function storeResponse(ctx, db, row, oldTable, table) { row.type = "row" - const response = await db.put(row) // don't worry about rev, tables handle rev/lastID updates + // if another row has been written since processing this will + // handle the auto ID clash if (!isEqual(oldTable, table)) { - await db.put(table) + try { + await db.put(table) + } catch (err) { + if (err.status === 409) { + const updatedTable = await db.get(table._id) + let response = processAutoColumn(null, updatedTable, row, { + reprocessing: true, + }) + await db.put(response.table) + row = response.row + } else { + throw err + } + } } + const response = await db.put(row) row._rev = response.rev // process the row before return, to include relationships row = await outputProcessing(ctx, table, row, { squash: false }) @@ -182,7 +198,7 @@ exports.fetchView = async ctx => { } let response // TODO: make sure not self hosted in Cloud - if (!env.SELF_HOSTED) { + if (env.SELF_HOSTED) { response = await db.query(`database/${viewName}`, { include_docs: !calculation, group: !!group, diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js index 9706126438..9033004578 100644 --- a/packages/server/src/automations/steps/createRow.js +++ b/packages/server/src/automations/steps/createRow.js @@ -2,6 +2,7 @@ const rowController = require("../../api/controllers/row") const automationUtils = require("../automationUtils") const env = require("../../environment") const usage = require("../../utilities/usageQuota") +const { buildCtx } = require("./utils") exports.definition = { name: "Create Row", @@ -69,16 +70,12 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) { } } // have to clean up the row, remove the table from it - const ctx = { + const ctx = buildCtx(appId, emitter, { + body: inputs.row, params: { tableId: inputs.row.tableId, }, - request: { - body: inputs.row, - }, - appId, - eventEmitter: emitter, - } + }) try { inputs.row = await automationUtils.cleanUpRow( @@ -86,7 +83,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) { inputs.row.tableId, inputs.row ) - if (env.isProd()) { + if (env.USE_QUOTAS) { await usage.update(apiKey, usage.Properties.ROW, 1) } await rowController.save(ctx) diff --git a/packages/server/src/automations/steps/deleteRow.js b/packages/server/src/automations/steps/deleteRow.js index 26623d628b..0f9648cc51 100644 --- a/packages/server/src/automations/steps/deleteRow.js +++ b/packages/server/src/automations/steps/deleteRow.js @@ -1,6 +1,7 @@ const rowController = require("../../api/controllers/row") const env = require("../../environment") const usage = require("../../utilities/usageQuota") +const { buildCtx } = require("./utils") exports.definition = { description: "Delete a row from your database", @@ -60,19 +61,16 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) { }, } } - let ctx = { + + let ctx = buildCtx(appId, emitter, { + body: { + _id: inputs.id, + _rev: inputs.revision, + }, params: { tableId: inputs.tableId, }, - request: { - body: { - _id: inputs.id, - _rev: inputs.revision, - }, - }, - appId, - eventEmitter: emitter, - } + }) try { if (env.isProd()) { diff --git a/packages/server/src/automations/steps/queryRows.js b/packages/server/src/automations/steps/queryRows.js index 64b757418e..3c4bb422a0 100644 --- a/packages/server/src/automations/steps/queryRows.js +++ b/packages/server/src/automations/steps/queryRows.js @@ -1,6 +1,7 @@ const rowController = require("../../api/controllers/row") const tableController = require("../../api/controllers/table") const { FieldTypes } = require("../../constants") +const { buildCtx } = require("./utils") const SortOrders = { ASCENDING: "ascending", @@ -70,12 +71,11 @@ exports.definition = { } async function getTable(appId, tableId) { - const ctx = { + const ctx = buildCtx(appId, null, { params: { id: tableId, }, - appId, - } + }) await tableController.find(ctx) return ctx.body } @@ -89,21 +89,18 @@ exports.run = async function ({ inputs, appId }) { sortType = fieldType === FieldTypes.NUMBER ? FieldTypes.NUMBER : FieldTypes.STRING } - const ctx = { + const ctx = buildCtx(appId, null, { params: { tableId, }, - request: { - body: { - sortOrder, - sortType, - sort: sortColumn, - query: filters || {}, - limit, - }, + body: { + sortOrder, + sortType, + sort: sortColumn, + query: filters || {}, + limit, }, - appId, - } + }) try { await rowController.search(ctx) return { diff --git a/packages/server/src/automations/steps/updateRow.js b/packages/server/src/automations/steps/updateRow.js index ac5eb16fcd..94f77bc801 100644 --- a/packages/server/src/automations/steps/updateRow.js +++ b/packages/server/src/automations/steps/updateRow.js @@ -1,5 +1,6 @@ const rowController = require("../../api/controllers/row") const automationUtils = require("../automationUtils") +const { buildCtx } = require("./utils") exports.definition = { name: "Update Row", @@ -72,19 +73,15 @@ exports.run = async function ({ inputs, appId, emitter }) { } // have to clean up the row, remove the table from it - const ctx = { + const ctx = buildCtx(appId, emitter, { + body: { + ...inputs.row, + _id: inputs.rowId, + }, params: { rowId: inputs.rowId, }, - request: { - body: { - ...inputs.row, - _id: inputs.rowId, - }, - }, - appId, - eventEmitter: emitter, - } + }) try { inputs.row = await automationUtils.cleanUpRowById( diff --git a/packages/server/src/db/inMemoryView.js b/packages/server/src/db/inMemoryView.js index f82b418b7f..d2dd22161e 100644 --- a/packages/server/src/db/inMemoryView.js +++ b/packages/server/src/db/inMemoryView.js @@ -17,9 +17,14 @@ exports.runView = async (appId, view, calculation, group, data) => { runner: view, }, }) - // write all the docs to the in memory Pouch - await db.bulkDocs(data) - const response = await db.query(`database/runner`, { + // write all the docs to the in memory Pouch (remove revs) + await db.bulkDocs( + data.map(row => ({ + ...row, + _rev: undefined, + })) + ) + const response = await db.query("database/runner", { include_docs: !calculation, group: !!group, }) diff --git a/packages/server/src/utilities/rowProcessor/index.js b/packages/server/src/utilities/rowProcessor/index.js index bb4ac98bb7..4e1571892d 100644 --- a/packages/server/src/utilities/rowProcessor/index.js +++ b/packages/server/src/utilities/rowProcessor/index.js @@ -89,10 +89,11 @@ const TYPE_TRANSFORM_MAP = { * @param {Object} user The user to be used for an appId as well as the createdBy and createdAt fields. * @param {Object} table The table which is to be used for the schema, as well as handling auto IDs incrementing. * @param {Object} row The row which is to be updated with information for the auto columns. + * @param {Object} opts specific options for function to carry out optional features. * @returns {{row: Object, table: Object}} The updated row and table, the table may need to be updated * for automatic ID purposes. */ -function processAutoColumn(user, table, row) { +function processAutoColumn(user, table, row, opts = { reprocessing: false }) { let now = new Date().toISOString() // if a row doesn't have a revision then it doesn't exist yet const creating = !row._rev @@ -102,7 +103,7 @@ function processAutoColumn(user, table, row) { } switch (schema.subtype) { case AutoFieldSubTypes.CREATED_BY: - if (creating) { + if (creating && !opts.reprocessing) { row[key] = [user.userId] } break @@ -112,7 +113,9 @@ function processAutoColumn(user, table, row) { } break case AutoFieldSubTypes.UPDATED_BY: - row[key] = [user.userId] + if (!opts.reprocessing) { + row[key] = [user.userId] + } break case AutoFieldSubTypes.UPDATED_AT: row[key] = now @@ -127,6 +130,7 @@ function processAutoColumn(user, table, row) { } return { table, row } } +exports.processAutoColumn = processAutoColumn /** * This will coerce a value to the correct types based on the type transform map From 0993825286780f07287b7f18239d69fa4444e7d4 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 21 Sep 2021 11:47:14 +0100 Subject: [PATCH 03/55] analytics clients --- .github/workflows/release-develop.yml | 1 + .github/workflows/release.yml | 1 + packages/auth/src/objectStore/index.js | 1 + packages/builder/src/analytics.js | 139 ----------------- .../builder/src/analytics/IntercomClient.js | 66 ++++++++ .../builder/src/analytics/PosthogClient.js | 144 ++++++++++++++++++ .../builder/src/analytics/SentryClient.js | 39 +++++ packages/builder/src/analytics/constants.js | 49 ++++++ packages/builder/src/analytics/index.js | 76 +++++++++ packages/builder/src/builderStore/index.js | 4 +- .../src/components/deploy/DeployModal.svelte | 6 +- .../components/start/CreateAppModal.svelte | 6 +- .../src/pages/builder/auth/login.svelte | 13 +- .../pages/builder/portal/apps/index.svelte | 6 +- .../builder/portal/manage/auth/index.svelte | 2 + .../builder/portal/manage/email/index.svelte | 2 + .../users/_components/AddUserModal.svelte | 2 + .../portal/settings/organisation.svelte | 2 +- packages/builder/vite.config.js | 3 + 19 files changed, 412 insertions(+), 150 deletions(-) delete mode 100644 packages/builder/src/analytics.js create mode 100644 packages/builder/src/analytics/IntercomClient.js create mode 100644 packages/builder/src/analytics/PosthogClient.js create mode 100644 packages/builder/src/analytics/SentryClient.js create mode 100644 packages/builder/src/analytics/constants.js create mode 100644 packages/builder/src/analytics/index.js diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml index ce41fcc3e6..cf0d6f848c 100644 --- a/.github/workflows/release-develop.yml +++ b/.github/workflows/release-develop.yml @@ -7,6 +7,7 @@ on: env: POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }} + INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} POSTHOG_URL: ${{ secrets.POSTHOG_URL }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aaee3923ef..00e7f9a795 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,7 @@ on: env: POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }} + INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} POSTHOG_URL: ${{ secrets.POSTHOG_URL }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} diff --git a/packages/auth/src/objectStore/index.js b/packages/auth/src/objectStore/index.js index 81bdd06b62..0f1307c39d 100644 --- a/packages/auth/src/objectStore/index.js +++ b/packages/auth/src/objectStore/index.js @@ -73,6 +73,7 @@ exports.ObjectStore = bucket => { AWS.config.update({ accessKeyId: env.MINIO_ACCESS_KEY, secretAccessKey: env.MINIO_SECRET_KEY, + region: env.AWS_REGION }) const config = { s3ForcePathStyle: true, diff --git a/packages/builder/src/analytics.js b/packages/builder/src/analytics.js deleted file mode 100644 index 5b130a8e6b..0000000000 --- a/packages/builder/src/analytics.js +++ /dev/null @@ -1,139 +0,0 @@ -import * as Sentry from "@sentry/browser" -import posthog from "posthog-js" -import api from "builderStore/api" - -let analyticsEnabled -const posthogConfigured = process.env.POSTHOG_TOKEN && process.env.POSTHOG_URL -const sentryConfigured = process.env.SENTRY_DSN - -const FEEDBACK_SUBMITTED_KEY = "budibase:feedback_submitted" -const APP_FIRST_STARTED_KEY = "budibase:first_run" -const feedbackHours = 12 - -async function activate() { - if (analyticsEnabled === undefined) { - // only the server knows the true NODE_ENV - // this was an issue as NODE_ENV = 'cypress' on the server, - // but 'production' on the client - const response = await api.get("/api/analytics") - analyticsEnabled = (await response.json()).enabled === true - } - if (!analyticsEnabled) return - if (sentryConfigured) Sentry.init({ dsn: process.env.SENTRY_DSN }) - if (posthogConfigured) { - posthog.init(process.env.POSTHOG_TOKEN, { - autocapture: false, - capture_pageview: false, - api_host: process.env.POSTHOG_URL, - }) - posthog.set_config({ persistence: "cookie" }) - } -} - -function identify(id) { - if (!analyticsEnabled || !id) return - if (posthogConfigured) posthog.identify(id) - if (sentryConfigured) - Sentry.configureScope(scope => { - scope.setUser({ id: id }) - }) -} - -async function identifyByApiKey(apiKey) { - if (!analyticsEnabled) return true - try { - const response = await fetch( - `https://03gaine137.execute-api.eu-west-1.amazonaws.com/prod/account/id?api_key=${apiKey.trim()}` - ) - if (response.status === 200) { - const id = await response.json() - - await api.put("/api/keys/userId", { value: id }) - identify(id) - return true - } - - return false - } catch (error) { - console.log(error) - } -} - -function captureException(err) { - if (!analyticsEnabled) return - Sentry.captureException(err) - captureEvent("Error", { error: err.message ? err.message : err }) -} - -function captureEvent(eventName, props = {}) { - if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return - props.sourceApp = "builder" - posthog.capture(eventName, props) -} - -if (!localStorage.getItem(APP_FIRST_STARTED_KEY)) { - localStorage.setItem(APP_FIRST_STARTED_KEY, Date.now()) -} - -const isFeedbackTimeElapsed = sinceDateStr => { - const sinceDate = parseFloat(sinceDateStr) - const feedbackMilliseconds = feedbackHours * 60 * 60 * 1000 - return Date.now() > sinceDate + feedbackMilliseconds -} - -function submitFeedback(values) { - if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return - localStorage.setItem(FEEDBACK_SUBMITTED_KEY, Date.now()) - - const prefixedValues = Object.entries(values).reduce((obj, [key, value]) => { - obj[`feedback_${key}`] = value - return obj - }, {}) - - posthog.capture("Feedback Submitted", prefixedValues) -} - -function requestFeedbackOnDeploy() { - if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return false - const lastSubmittedStr = localStorage.getItem(FEEDBACK_SUBMITTED_KEY) - if (!lastSubmittedStr) return true - return isFeedbackTimeElapsed(lastSubmittedStr) -} - -function highlightFeedbackIcon() { - if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return false - const lastSubmittedStr = localStorage.getItem(FEEDBACK_SUBMITTED_KEY) - if (lastSubmittedStr) return isFeedbackTimeElapsed(lastSubmittedStr) - const firstRunStr = localStorage.getItem(APP_FIRST_STARTED_KEY) - if (!firstRunStr) return false - return isFeedbackTimeElapsed(firstRunStr) -} - -// Opt In/Out -const ifAnalyticsEnabled = func => () => { - if (analyticsEnabled && process.env.POSTHOG_TOKEN) { - return func() - } -} -const disabled = () => posthog.has_opted_out_capturing() -const optIn = () => posthog.opt_in_capturing() -const optOut = () => posthog.opt_out_capturing() - -export default { - activate, - identify, - identifyByApiKey, - captureException, - captureEvent, - requestFeedbackOnDeploy, - submitFeedback, - highlightFeedbackIcon, - disabled: () => { - if (analyticsEnabled == null) { - return true - } - return ifAnalyticsEnabled(disabled) - }, - optIn: ifAnalyticsEnabled(optIn), - optOut: ifAnalyticsEnabled(optOut), -} diff --git a/packages/builder/src/analytics/IntercomClient.js b/packages/builder/src/analytics/IntercomClient.js new file mode 100644 index 0000000000..2cabeb97df --- /dev/null +++ b/packages/builder/src/analytics/IntercomClient.js @@ -0,0 +1,66 @@ +export default class IntercomClient { + constructor(token) { + this.token = token + } + + init() { + if (!this.token) return + + const token = this.token + + var w = window + var ic = w.Intercom + if (typeof ic === "function") { + ic("reattach_activator") + ic("update", w.intercomSettings) + } else { + var d = document + var i = function () { + i.c(arguments) + } + i.q = [] + i.c = function (args) { + i.q.push(args) + } + w.Intercom = i + var l = function () { + var s = d.createElement("script") + s.type = "text/javascript" + s.async = true + s.src = "https://widget.intercom.io/widget/" + token + var x = d.getElementsByTagName("script")[0] + x.parentNode.insertBefore(s, x) + } + if (document.readyState === "complete") { + l() + } else if (w.attachEvent) { + w.attachEvent("onload", l) + } else { + w.addEventListener("load", l, false) + } + + this.initialised = true + } + } + + show(user = {}) { + if (!this.initialised) return + + return window.Intercom("boot", { + app_id: this.token, + ...user + }) + } + + update() { + if (!this.initialised) return + + return window.Intercom("update") + } + + captureEvent(event, props = {}) { + if (!this.initialised) return + + window.Intercom("trackEvent", event, props); + } +} diff --git a/packages/builder/src/analytics/PosthogClient.js b/packages/builder/src/analytics/PosthogClient.js new file mode 100644 index 0000000000..9a98520c91 --- /dev/null +++ b/packages/builder/src/analytics/PosthogClient.js @@ -0,0 +1,144 @@ +import posthog from "posthog-js" +import { Events } from "./constants" + +// let analyticsEnabled +// const posthogConfigured = process.env.POSTHOG_TOKEN && process.env.POSTHOG_URL + +// const FEEDBACK_SUBMITTED_KEY = "budibase:feedback_submitted" +// const APP_FIRST_STARTED_KEY = "budibase:first_run" +// const feedbackHours = 12 + +export default class PosthogClient { + constructor(token, url) { + this.token = token + this.url = url + } + + init() { + if (!this.token || !this.url) return + + posthog.init(this.token, { + autocapture: false, + capture_pageview: false, + api_host: this.url, + }) + posthog.set_config({ persistence: "cookie" }) + + this.initialised = true + } + + disabled() { + return posthog.has_opted_out_capturing() + } + + optIn() { + return posthog.opt_in_capturing() + } + + optOut() { + return posthog.opt_out_capturing() + } + + identify(id) { + if (!this.initialised) return + + posthog.identify(id) + } + + updateUser(meta) { + if (!this.initialised) return + + posthog.people.set(meta) + } + + captureException(err) { + if (!this.initialised) return + + this.captureEvent("Error", { error: err.message ? err.message : err }) + } + + captureEvent(eventName, props) { + if (!this.initialised) return + + props.sourceApp = "builder" + posthog.capture(eventName, props) + } + + npsFeedback(values) { + if (!this.initialised) return + + localStorage.setItem(Events.NPS.SUBMITTED, Date.now()) + + const prefixedFeedback = {} + for (let key in values) { + prefixedFeedback[`feedback_${key}`] = values[key] + } + + posthog.capture(Events.NPS.SUBMITTED, prefixedFeedback) + } +} + +// function captureEvent(eventName, props = {}) { +// if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return +// props.sourceApp = "builder" +// posthog.capture(eventName, props) +// } + +// if (!localStorage.getItem(APP_FIRST_STARTED_KEY)) { +// localStorage.setItem(APP_FIRST_STARTED_KEY, Date.now()) +// } + +// function submitFeedback(values) { +// if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return +// localStorage.setItem(FEEDBACK_SUBMITTED_KEY, Date.now()) + +// const prefixedValues = Object.entries(values).reduce((obj, [key, value]) => { +// obj[`feedback_${key}`] = value +// return obj +// }, {}) + +// posthog.capture("Feedback Submitted", prefixedValues) +// } + +// function requestFeedbackOnDeploy() { +// if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return false +// const lastSubmittedStr = localStorage.getItem(FEEDBACK_SUBMITTED_KEY) +// if (!lastSubmittedStr) return true +// return isFeedbackTimeElapsed(lastSubmittedStr) +// } + +// function highlightFeedbackIcon() { +// if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return false +// const lastSubmittedStr = localStorage.getItem(FEEDBACK_SUBMITTED_KEY) +// if (lastSubmittedStr) return isFeedbackTimeElapsed(lastSubmittedStr) +// const firstRunStr = localStorage.getItem(APP_FIRST_STARTED_KEY) +// if (!firstRunStr) return false +// return isFeedbackTimeElapsed(firstRunStr) +// } + +// Opt In/Out +// const ifAnalyticsEnabled = func => () => { +// if (analyticsEnabled && process.env.POSTHOG_TOKEN) { +// return func() +// } +// } +// const disabled = () => posthog.has_opted_out_capturing() +// const optIn = () => posthog.opt_in_capturing() +// const optOut = () => posthog.opt_out_capturing() + +// export default { +// init, +// identify, +// captureException, +// captureEvent, +// submitFeedback, +// highlightFeedbackIcon, +// disabled: () => { +// if (analyticsEnabled == null) { +// return true +// } +// return ifAnalyticsEnabled(disabled) +// }, +// optIn: ifAnalyticsEnabled(optIn), +// optOut: ifAnalyticsEnabled(optOut), +// } diff --git a/packages/builder/src/analytics/SentryClient.js b/packages/builder/src/analytics/SentryClient.js new file mode 100644 index 0000000000..11377d254f --- /dev/null +++ b/packages/builder/src/analytics/SentryClient.js @@ -0,0 +1,39 @@ +import * as Sentry from "@sentry/browser" + +export default class SentryClient { + constructor(dsn) { + this.dsn = dsn + } + + init() { + if (this.dsn) { + Sentry.init({ dsn: this.dsn }) + + this.initalised = true + } + } + + captureException(err) { + if (!this.initalised) return + + Sentry.captureException(err) + } + + identify(id) { + Sentry.configureScope(scope => { + scope.setUser({ id }) + }) + } +} + +// export function init() { +// if (process.env.SENTRY_DSN) { +// Sentry.init({ dsn: process.env.SENTRY_DSN }) +// } +// } + +// export function captureException(err) { +// // if (!analyticsEnabled) return +// Sentry.captureException(err) +// // captureEvent("Error", { error: err.message ? err.message : err }) +// } diff --git a/packages/builder/src/analytics/constants.js b/packages/builder/src/analytics/constants.js new file mode 100644 index 0000000000..1102ad3ce4 --- /dev/null +++ b/packages/builder/src/analytics/constants.js @@ -0,0 +1,49 @@ +export const Events = { + BUILDER: { + STARTED: "Builder Started" + }, + COMPONENT: { + CREATED: "Added Component" + }, + DATASOURCE: { + CREATED: "Datasource Created", + UPDATED: "Datasource Updated", + }, + TABLE: { + CREATED: "Table Created", + }, + VIEW: { + CREATED: "View Created", + ADDED_FILTER: "Added View Filter", + ADDED_CALCULATE: "Added View Calculate" + }, + SCREEN: { + CREATED: "Screen Created" + }, + AUTOMATION: { + CREATED: "Added Component", + SAVED: "Automation Saved", + BLOCK_ADDED: "Added Automation Block", + }, + NPS: { + SUBMITTED: "budibase:feedback_submitted" + }, + APP: { + CREATED: "budibase:app_created", + PUBLISHED: "budibase:app_published", + UNPUBLISHED: "budibase:app_unpublished" + }, + ANALYTICS: { + OPT_IN: "budibase:analytics_opt_in", + OPT_OUT: "budibase:analytics_opt_out" + }, + USER: { + INVITE: "budibase:portal_user_invite" + }, + SMTP: { + SAVED: "budibase:smtp_saved" + }, + SSO: { + SAVED: "budibase:sso_saved" + } +} diff --git a/packages/builder/src/analytics/index.js b/packages/builder/src/analytics/index.js new file mode 100644 index 0000000000..9fd8f82668 --- /dev/null +++ b/packages/builder/src/analytics/index.js @@ -0,0 +1,76 @@ +import api from "builderStore/api" +import PosthogClient from "./PosthogClient" +import IntercomClient from "./IntercomClient" +import SentryClient from "./SentryClient" +import { Events } from "./constants" + +// const posthog = new PosthogClient( +// process.env.POSTHOG_TOKEN, +// process.env.POSTHOG_URL +// ) +const posthog = new PosthogClient( + "phc_yGOn4i7jWKaCTapdGR6lfA4AvmuEQ2ijn5zAVSFYPlS", + "https://app.posthog.com" +) +// const sentry = new SentryClient(process.env.SENTRY_DSN) +const sentry = new SentryClient("https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131") +// const intercom = new IntercomClient(process.env.INTERCOM_TOKEN) +const intercom = new IntercomClient("qz2sxfuv") + + +class AnalyticsHub { + constructor() { + this.clients = [posthog, sentry, intercom] + } + + async activate() { + const analyticsStatus = await api.get("/api/analytics") + const json = await analyticsStatus.json() + + if (json.enabled) { + this.clients.forEach(client => client.init()) + } + + this.enabled = json.enabled + } + + optIn() { + this.captureEvent(Events.ANALYTICS.OPT_IN) + this.clients.forEach(client => client.optIn()) + } + + optOut() { + this.captureEvent(Events.ANALYTICS.OPT_OUT) + this.clients.forEach(client => client.optOut()) + } + + identify(id, metadata) { + posthog.identify(id) + if (metadata) { + posthog.updateUser(metadata) + } + sentry.identify(id) + } + + captureException(err) { + sentry.captureException(err) + } + + captureEvent(eventName, props = {}) { + posthog.captureEvent(eventName, props) + intercom.captureEvent(eventName, props) + } + + showChat(user) { + intercom.show(user) + } + + submitFeedback(values) { + posthog.npsFeedback(values) + } +} + +const analytics = new AnalyticsHub() + +export { Events } +export default analytics \ No newline at end of file diff --git a/packages/builder/src/builderStore/index.js b/packages/builder/src/builderStore/index.js index 6fecda84c0..fb888b6fc9 100644 --- a/packages/builder/src/builderStore/index.js +++ b/packages/builder/src/builderStore/index.js @@ -3,7 +3,7 @@ import { getAutomationStore } from "./store/automation" import { getHostingStore } from "./store/hosting" import { getThemeStore } from "./store/theme" import { derived, writable } from "svelte/store" -import analytics from "analytics" +import analytics, { Events } from "analytics" import { FrontendTypes, LAYOUT_NAMES } from "../constants" import { findComponent } from "./storeUtils" @@ -58,7 +58,7 @@ export const selectedAccessRole = writable("BASIC") export const initialise = async () => { try { await analytics.activate() - analytics.captureEvent("Builder Started") + analytics.captureEvent(Events.BUILDER.STARTED) } catch (err) { console.log(err) } diff --git a/packages/builder/src/components/deploy/DeployModal.svelte b/packages/builder/src/components/deploy/DeployModal.svelte index 4daa16c7c4..5500aff866 100644 --- a/packages/builder/src/components/deploy/DeployModal.svelte +++ b/packages/builder/src/components/deploy/DeployModal.svelte @@ -2,7 +2,8 @@ import { onMount, onDestroy } from "svelte" import { Button, Modal, notifications, ModalContent } from "@budibase/bbui" import api from "builderStore/api" - import analytics from "analytics" + import analytics, { Events } from "analytics" + import { store } from "builderStore" const DeploymentStatus = { SUCCESS: "SUCCESS", @@ -23,6 +24,9 @@ if (response.status !== 200) { throw new Error(`status ${response.status}`) } else { + analytics.captureEvent(Events.APP.PUBLISHED, { + appId: $store.appId + }) notifications.success(`Application published successfully`) } } catch (err) { diff --git a/packages/builder/src/components/start/CreateAppModal.svelte b/packages/builder/src/components/start/CreateAppModal.svelte index 4310d3322e..9ce9d746d7 100644 --- a/packages/builder/src/components/start/CreateAppModal.svelte +++ b/packages/builder/src/components/start/CreateAppModal.svelte @@ -12,7 +12,7 @@ import { admin } from "stores/portal" import { string, mixed, object } from "yup" import api, { get, post } from "builderStore/api" - import analytics from "analytics" + import analytics, { Events } from "analytics" import { onMount } from "svelte" import { capitalise } from "helpers" import { goto } from "@roxi/routify" @@ -98,9 +98,9 @@ throw new Error(appJson.message) } - analytics.captureEvent("App Created", { + analytics.captureEvent(Events.APP.CREATED, { name: $values.name, - appId: appJson._id, + appId: appJson.instance._id, template, }) diff --git a/packages/builder/src/pages/builder/auth/login.svelte b/packages/builder/src/pages/builder/auth/login.svelte index 783e5a4903..568a0550a3 100644 --- a/packages/builder/src/pages/builder/auth/login.svelte +++ b/packages/builder/src/pages/builder/auth/login.svelte @@ -15,6 +15,7 @@ import OIDCButton from "./_components/OIDCButton.svelte" import Logo from "assets/bb-emblem.svg" import { onMount } from "svelte" + import analytics from "analytics" let username = "" let password = "" @@ -25,10 +26,20 @@ async function login() { try { - await auth.login({ + const { user } = await auth.login({ username, password, }) + + analytics.identify(user._id, user) + analytics.showChat({ + email: user.email, + created_at: Date.now(), + name: user.name, + user_id: user._id, + tenant: user.tenantId + }) + if ($auth?.user?.forceResetPassword) { $goto("./reset") } else { diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte index a18ec6a8bd..319a00f471 100644 --- a/packages/builder/src/pages/builder/portal/apps/index.svelte +++ b/packages/builder/src/pages/builder/portal/apps/index.svelte @@ -69,9 +69,9 @@ const checkKeys = async () => { const response = await api.get(`/api/keys/`) const keys = await response.json() - if (keys.userId) { - analytics.identify(keys.userId) - } + // if (keys.userId) { + // analytics.identify(keys.userId) + // } } const initiateAppCreation = () => { diff --git a/packages/builder/src/pages/builder/portal/manage/auth/index.svelte b/packages/builder/src/pages/builder/portal/manage/auth/index.svelte index 48d9da18f9..c2445e14ae 100644 --- a/packages/builder/src/pages/builder/portal/manage/auth/index.svelte +++ b/packages/builder/src/pages/builder/portal/manage/auth/index.svelte @@ -23,6 +23,7 @@ import api from "builderStore/api" import { organisation, auth, admin } from "stores/portal" import { uuid } from "builderStore/uuid" + import analytics, { Events } from "analytics" $: tenantId = $auth.tenantId $: multiTenancyEnabled = $admin.multiTenancy @@ -209,6 +210,7 @@ providers[res.type]._id = res._id }) notifications.success(`Settings saved.`) + analytics.captureEvent(Events.SSO.SAVED) }) .catch(err => { notifications.error(`Failed to update auth settings. ${err}`) diff --git a/packages/builder/src/pages/builder/portal/manage/email/index.svelte b/packages/builder/src/pages/builder/portal/manage/email/index.svelte index 76d98ed545..5a78623b81 100644 --- a/packages/builder/src/pages/builder/portal/manage/email/index.svelte +++ b/packages/builder/src/pages/builder/portal/manage/email/index.svelte @@ -16,6 +16,7 @@ import { email } from "stores/portal" import api from "builderStore/api" import { cloneDeep } from "lodash/fp" + import analytics, { Events } from "analytics" const ConfigTypes = { SMTP: "smtp", @@ -69,6 +70,7 @@ smtpConfig._rev = json._rev smtpConfig._id = json._id notifications.success(`Settings saved.`) + analytics.captureEvent(Events.SMTP.SAVED) } } diff --git a/packages/builder/src/pages/builder/portal/manage/users/_components/AddUserModal.svelte b/packages/builder/src/pages/builder/portal/manage/users/_components/AddUserModal.svelte index 9504f73b68..be5d6db746 100644 --- a/packages/builder/src/pages/builder/portal/manage/users/_components/AddUserModal.svelte +++ b/packages/builder/src/pages/builder/portal/manage/users/_components/AddUserModal.svelte @@ -10,6 +10,7 @@ } from "@budibase/bbui" import { createValidationStore, emailValidator } from "helpers/validation" import { users } from "stores/portal" + import analytics, { Events } from "analytics" export let disabled @@ -25,6 +26,7 @@ notifications.error(res.message) } else { notifications.success(res.message) + analytics.captureEvent(Events.USER.INVITE) } } diff --git a/packages/builder/src/pages/builder/portal/settings/organisation.svelte b/packages/builder/src/pages/builder/portal/settings/organisation.svelte index be8b60e6e7..3d149e6e60 100644 --- a/packages/builder/src/pages/builder/portal/settings/organisation.svelte +++ b/packages/builder/src/pages/builder/portal/settings/organisation.svelte @@ -25,7 +25,7 @@ } const values = writable({ - analytics: !analytics.disabled(), + analytics: analytics.enabled, company: $organisation.company, platformUrl: $organisation.platformUrl, logo: $organisation.logoUrl diff --git a/packages/builder/vite.config.js b/packages/builder/vite.config.js index d8b8dbba1d..12b45e7cf8 100644 --- a/packages/builder/vite.config.js +++ b/packages/builder/vite.config.js @@ -22,6 +22,9 @@ export default ({ mode }) => { isProduction ? "production" : "development" ), "process.env.POSTHOG_TOKEN": JSON.stringify(process.env.POSTHOG_TOKEN), + "process.env.INTERCOM_TOKEN": JSON.stringify( + process.env.INTERCOM_TOKEN + ), "process.env.POSTHOG_URL": JSON.stringify(process.env.POSTHOG_URL), "process.env.SENTRY_DSN": JSON.stringify(process.env.SENTRY_DSN), }), From 9aecdedbfc6b4cb5766bb4ea5c1d66bb5ae50b2c Mon Sep 17 00:00:00 2001 From: Maurits Lourens Date: Tue, 21 Sep 2021 15:16:10 +0200 Subject: [PATCH 04/55] remove the link from navigation bar when removing the screen --- .../src/builderStore/store/frontend.js | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 603fa88b09..84c975edc5 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -215,6 +215,13 @@ export const getFrontendStore = () => { if (screenToDelete._id === state.selectedScreenId) { state.selectedScreenId = null } + //remove the link for this screen + screenDeletePromises.push( + store.actions.components.links.delete( + screenToDelete.routing.route, + screenToDelete.props._instanceName + ) + ) } return state }) @@ -646,6 +653,36 @@ export const getFrontendStore = () => { // Save layout await store.actions.layouts.save(layout) }, + delete: async (url, title) => { + const layout = get(mainLayout) + if (!layout) { + return + } + + // Add link setting to main layout + if (layout.props._component.endsWith("layout")) { + // If using a new SDK, add to the layout component settings + layout.props.links = layout.props.links.filter( + link => !(link.text === title && link.url === url) + ) + } else { + // If using an old SDK, add to the navigation component + // TODO: remove this when we can assume everyone has updated + const nav = findComponentType( + layout.props, + "@budibase/standard-components/navigation" + ) + if (!nav) { + return + } + + nav._children = nav._children.filter( + child => !(child.url === url && child.text === title) + ) + } + // Save layout + await store.actions.layouts.save(layout) + }, }, }, } From 8f70ad9cdcd24242718134f89430f08e826e1a1e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 21 Sep 2021 15:59:50 +0100 Subject: [PATCH 05/55] Performance improvements after testing - switching to temporary in memory queries for cloud views. --- packages/server/package.json | 2 +- .../src/api/controllers/row/internal.js | 8 +----- .../server/src/api/controllers/table/utils.js | 28 +++++++++++-------- packages/server/src/db/inMemoryView.js | 27 ++++++++++-------- packages/server/src/db/linkedRows/index.js | 7 +++-- .../src/utilities/rowProcessor/index.js | 21 ++++++++++---- 6 files changed, 55 insertions(+), 38 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 6d6b05752e..321dabe272 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -107,6 +107,7 @@ "pouchdb-all-dbs": "1.0.2", "pouchdb-find": "^7.2.2", "pouchdb-replication-stream": "1.2.9", + "pouchdb-adapter-memory": "^7.2.1", "server-destroy": "1.0.1", "svelte": "^3.38.2", "to-json-schema": "0.2.5", @@ -132,7 +133,6 @@ "express": "^4.17.1", "jest": "^27.0.5", "nodemon": "^2.0.4", - "pouchdb-adapter-memory": "^7.2.1", "prettier": "^2.3.1", "rimraf": "^3.0.2", "supertest": "^4.0.2", diff --git a/packages/server/src/api/controllers/row/internal.js b/packages/server/src/api/controllers/row/internal.js index 77e3069239..b663cc9e32 100644 --- a/packages/server/src/api/controllers/row/internal.js +++ b/packages/server/src/api/controllers/row/internal.js @@ -206,13 +206,7 @@ exports.fetchView = async ctx => { } else { const tableId = viewInfo.meta.tableId const data = await getRawTableData(ctx, db, tableId) - response = await inMemoryViews.runView( - appId, - viewInfo, - calculation, - group, - data - ) + response = await inMemoryViews.runView(viewInfo, calculation, group, data) } let rows diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index 154a9ba8f5..d263002da6 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -68,23 +68,17 @@ exports.handleDataImport = async (appId, user, table, dataImport) => { // Populate the table with rows imported from CSV in a bulk update const data = await csvParser.transform(dataImport) + let finalData = [] for (let i = 0; i < data.length; i++) { let row = data[i] row._id = generateRowID(table._id) row.tableId = table._id - const processed = inputProcessing(user, table, row) + const processed = inputProcessing(user, table, row, { + noAutoRelationships: true, + }) table = processed.table row = processed.row - // make sure link rows are up to date - row = await linkRows.updateLinks({ - appId, - eventType: linkRows.EventType.ROW_SAVE, - row, - tableId: row.tableId, - table, - }) - for (let [fieldName, schema] of Object.entries(table.schema)) { // check whether the options need to be updated for inclusion as part of the data import if ( @@ -98,10 +92,20 @@ exports.handleDataImport = async (appId, user, table, dataImport) => { ] } } - data[i] = row + + // make sure link rows are up to date + finalData.push( + linkRows.updateLinks({ + appId, + eventType: linkRows.EventType.ROW_SAVE, + row, + tableId: row.tableId, + table, + }) + ) } - await db.bulkDocs(data) + await db.bulkDocs(await Promise.all(finalData)) let response = await db.put(table) table._rev = response._rev } diff --git a/packages/server/src/db/inMemoryView.js b/packages/server/src/db/inMemoryView.js index d2dd22161e..892617e068 100644 --- a/packages/server/src/db/inMemoryView.js +++ b/packages/server/src/db/inMemoryView.js @@ -1,5 +1,6 @@ const PouchDB = require("pouchdb") const memory = require("pouchdb-adapter-memory") +const newid = require("./newid") PouchDB.plugin(memory) const Pouch = PouchDB.defaults({ @@ -7,16 +8,11 @@ const Pouch = PouchDB.defaults({ adapter: "memory", }) -exports.runView = async (appId, view, calculation, group, data) => { - // appId doesn't really do anything since its all in memory - // use it just incase multiple databases at the same time - const db = new Pouch(appId) - await db.put({ - _id: "_design/database", - views: { - runner: view, - }, - }) +exports.runView = async (view, calculation, group, data) => { + // use a different ID each time for the DB, make sure they + // are always unique for each query, don't want overlap + // which could cause 409s + const db = new Pouch(newid()) // write all the docs to the in memory Pouch (remove revs) await db.bulkDocs( data.map(row => ({ @@ -24,7 +20,16 @@ exports.runView = async (appId, view, calculation, group, data) => { _rev: undefined, })) ) - const response = await db.query("database/runner", { + let fn = (doc, emit) => emit(doc._id) + eval("fn = " + view.map.replace("function (doc)", "function (doc, emit)")) + const queryFns = { + meta: view.meta, + map: fn, + } + if (view.reduce) { + queryFns.reduce = view.reduce + } + const response = await db.query(queryFns, { include_docs: !calculation, group: !!group, }) diff --git a/packages/server/src/db/linkedRows/index.js b/packages/server/src/db/linkedRows/index.js index 67412e7e89..303cd085c1 100644 --- a/packages/server/src/db/linkedRows/index.js +++ b/packages/server/src/db/linkedRows/index.js @@ -76,9 +76,12 @@ async function getFullLinkedDocs(ctx, appId, links) { // create DBs const db = new CouchDB(appId) const linkedRowIds = links.map(link => link.id) - let linked = (await db.allDocs(getMultiIDParams(linkedRowIds))).rows.map( + const uniqueRowIds = [...new Set(linkedRowIds)] + let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map( row => row.doc ) + // convert the unique db rows back to a full list of linked rows + const linked = linkedRowIds.map(id => dbRows.find(row => row._id === id)) // need to handle users as specific cases let [users, other] = partition(linked, linkRow => linkRow._id.startsWith(USER_METDATA_PREFIX) @@ -112,7 +115,7 @@ exports.updateLinks = async function (args) { let linkController = new LinkController(args) try { if ( - !(await linkController.doesTableHaveLinkedFields()) && + !(await linkController.doesTableHaveLinkedFields(table)) && (oldTable == null || !(await linkController.doesTableHaveLinkedFields(oldTable))) ) { diff --git a/packages/server/src/utilities/rowProcessor/index.js b/packages/server/src/utilities/rowProcessor/index.js index 4e1571892d..07549dd8a8 100644 --- a/packages/server/src/utilities/rowProcessor/index.js +++ b/packages/server/src/utilities/rowProcessor/index.js @@ -93,7 +93,12 @@ const TYPE_TRANSFORM_MAP = { * @returns {{row: Object, table: Object}} The updated row and table, the table may need to be updated * for automatic ID purposes. */ -function processAutoColumn(user, table, row, opts = { reprocessing: false }) { +function processAutoColumn( + user, + table, + row, + opts = { reprocessing: false, noAutoRelationships: false } +) { let now = new Date().toISOString() // if a row doesn't have a revision then it doesn't exist yet const creating = !row._rev @@ -103,7 +108,7 @@ function processAutoColumn(user, table, row, opts = { reprocessing: false }) { } switch (schema.subtype) { case AutoFieldSubTypes.CREATED_BY: - if (creating && !opts.reprocessing) { + if (creating && !opts.reprocessing && !opts.noAutoRelationships) { row[key] = [user.userId] } break @@ -113,7 +118,7 @@ function processAutoColumn(user, table, row, opts = { reprocessing: false }) { } break case AutoFieldSubTypes.UPDATED_BY: - if (!opts.reprocessing) { + if (!opts.reprocessing && !opts.noAutoRelationships) { row[key] = [user.userId] } break @@ -155,9 +160,15 @@ exports.coerce = (row, type) => { * @param {object} user the user which is performing the input. * @param {object} row the row which is being created/updated. * @param {object} table the table which the row is being saved to. + * @param {object} opts some input processing options (like disabling auto-column relationships). * @returns {object} the row which has been prepared to be written to the DB. */ -exports.inputProcessing = (user = {}, table, row) => { +exports.inputProcessing = ( + user = {}, + table, + row, + opts = { noAutoRelationships: false } +) => { let clonedRow = cloneDeep(row) // need to copy the table so it can be differenced on way out const copiedTable = cloneDeep(table) @@ -180,7 +191,7 @@ exports.inputProcessing = (user = {}, table, row) => { } } // handle auto columns - this returns an object like {table, row} - return processAutoColumn(user, copiedTable, clonedRow) + return processAutoColumn(user, copiedTable, clonedRow, opts) } /** From efdd78c7fe3971909e266ca7fe81eeec785697a2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 21 Sep 2021 16:07:04 +0100 Subject: [PATCH 06/55] Removing use of db.post as its dangerous to allow the possibility of _id being auto-generated, these docs would be lost as we couldn't query them out by the usual id mechanism. --- .../src/middleware/passport/tests/third-party-common.spec.js | 2 +- packages/auth/src/middleware/passport/third-party-common.js | 2 +- packages/server/src/api/controllers/datasource.js | 4 ++-- packages/server/src/api/controllers/row/internal.js | 1 - packages/server/src/api/controllers/table/index.js | 2 +- packages/worker/src/api/controllers/global/workspaces.js | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/auth/src/middleware/passport/tests/third-party-common.spec.js b/packages/auth/src/middleware/passport/tests/third-party-common.spec.js index ff38a01fbb..1ace65ba40 100644 --- a/packages/auth/src/middleware/passport/tests/third-party-common.spec.js +++ b/packages/auth/src/middleware/passport/tests/third-party-common.spec.js @@ -104,7 +104,7 @@ describe("third party common", () => { _id: id, email: email, } - const response = await db.post(dbUser) + const response = await db.put(dbUser) dbUser._rev = response.rev } diff --git a/packages/auth/src/middleware/passport/third-party-common.js b/packages/auth/src/middleware/passport/third-party-common.js index 7c03944232..c25aa3e0b0 100644 --- a/packages/auth/src/middleware/passport/third-party-common.js +++ b/packages/auth/src/middleware/passport/third-party-common.js @@ -71,7 +71,7 @@ exports.authenticateThirdParty = async function ( dbUser = await syncUser(dbUser, thirdPartyUser) // create or sync the user - const response = await db.post(dbUser) + const response = await db.put(dbUser) dbUser._rev = response.rev // authenticate diff --git a/packages/server/src/api/controllers/datasource.js b/packages/server/src/api/controllers/datasource.js index 38b6e68932..4a2fd7d86a 100644 --- a/packages/server/src/api/controllers/datasource.js +++ b/packages/server/src/api/controllers/datasource.js @@ -51,7 +51,7 @@ exports.buildSchemaFromDb = async function (ctx) { await connector.buildSchema(datasource._id, datasource.entities) datasource.entities = connector.tables - const response = await db.post(datasource) + const response = await db.put(datasource) datasource._rev = response.rev ctx.body = datasource @@ -89,7 +89,7 @@ exports.save = async function (ctx) { ...ctx.request.body, } - const response = await db.post(datasource) + const response = await db.put(datasource) datasource._rev = response.rev // Drain connection pools when configuration is changed diff --git a/packages/server/src/api/controllers/row/internal.js b/packages/server/src/api/controllers/row/internal.js index b663cc9e32..f8960129c2 100644 --- a/packages/server/src/api/controllers/row/internal.js +++ b/packages/server/src/api/controllers/row/internal.js @@ -197,7 +197,6 @@ exports.fetchView = async ctx => { throw "View does not exist." } let response - // TODO: make sure not self hosted in Cloud if (env.SELF_HOSTED) { response = await db.query(`database/${viewName}`, { include_docs: !calculation, diff --git a/packages/server/src/api/controllers/table/index.js b/packages/server/src/api/controllers/table/index.js index 60b5167f66..c7b72cf1c8 100644 --- a/packages/server/src/api/controllers/table/index.js +++ b/packages/server/src/api/controllers/table/index.js @@ -145,7 +145,7 @@ exports.save = async function (ctx) { if (updatedRows && updatedRows.length !== 0) { await db.bulkDocs(updatedRows) } - const result = await db.post(tableToSave) + const result = await db.put(tableToSave) tableToSave._rev = result.rev tableToSave = await tableSaveFunctions.after(tableToSave) diff --git a/packages/worker/src/api/controllers/global/workspaces.js b/packages/worker/src/api/controllers/global/workspaces.js index 95a1ec296d..48a710c92d 100644 --- a/packages/worker/src/api/controllers/global/workspaces.js +++ b/packages/worker/src/api/controllers/global/workspaces.js @@ -11,7 +11,7 @@ exports.save = async function (ctx) { } try { - const response = await db.post(workspaceDoc) + const response = await db.put(workspaceDoc) ctx.body = { _id: response.id, _rev: response.rev, From 1ad2687f4f9cda6f45cfb017cc7b6ce5cc633e90 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 21 Sep 2021 17:21:01 +0100 Subject: [PATCH 07/55] Set datasource to internal datasource upon loading internal datasource page --- .../app/[application]/data/datasource/bb_internal/_layout.svelte | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte new file mode 100644 index 0000000000..e69de29bb2 From 216a7c7a3f81c32fe3342eca12b8308e86ca1bea Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 21 Sep 2021 17:21:25 +0100 Subject: [PATCH 08/55] Ensure datasource containing the currently selected table is always expanded --- .../DatasourceNavigator.svelte | 30 +++++++++++++++---- .../datasource/bb_internal/_layout.svelte | 7 +++++ .../builder/src/stores/backend/datasources.js | 15 ++++++++-- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte index 84c737eb67..193e778300 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte @@ -1,8 +1,9 @@ {#if $database?._id}
- {#each $datasources.list as datasource, idx} + {#each enrichedDataSources as datasource, idx} 0} text={datasource.name} - opened={openDataSources.includes(datasource._id)} - selected={$datasources.selected === datasource._id} + opened={datasource.open} + selected={datasource.selected} withArrow={true} on:click={() => selectDatasource(datasource)} on:iconClick={() => toggleNode(datasource)} @@ -61,7 +81,7 @@ {/if} - {#if openDataSources.includes(datasource._id)} + {#if datasource.open} {/if} diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte index e69de29bb2..ed271aae34 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte @@ -0,0 +1,7 @@ + + + diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index 5c6ed3f2cb..5e42315948 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -1,4 +1,4 @@ -import { writable } from "svelte/store" +import { writable, get } from "svelte/store" import { queries, tables, views } from "./" import api from "../../builderStore/api" @@ -8,7 +8,8 @@ export const INITIAL_DATASOURCE_VALUES = { } export function createDatasourcesStore() { - const { subscribe, update, set } = writable(INITIAL_DATASOURCE_VALUES) + const store = writable(INITIAL_DATASOURCE_VALUES) + const { subscribe, update, set } = store return { subscribe, @@ -21,7 +22,15 @@ export function createDatasourcesStore() { fetch: async () => { const response = await api.get(`/api/datasources`) const json = await response.json() - update(state => ({ ...state, list: json, selected: null })) + + // Clear selected if it no longer exists, otherwise keep it + const selected = get(store).selected + let nextSelected = null + if (selected && json.find(source => source._id === selected)) { + nextSelected = selected + } + + update(state => ({ ...state, list: json, selected: nextSelected })) return json }, select: async datasourceId => { From d8b2dd035a24055c855083a49c6003f0ad8970b5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 21 Sep 2021 17:37:26 +0100 Subject: [PATCH 09/55] Adding the ability to migrate from existing in db views to in memory views. --- .../src/api/controllers/row/internal.js | 30 +- .../server/src/api/controllers/view/index.js | 258 ++++++------------ .../server/src/api/controllers/view/utils.js | 109 ++++++++ .../server/src/api/routes/tests/view.spec.js | 2 +- packages/server/src/environment.js | 2 +- 5 files changed, 213 insertions(+), 188 deletions(-) create mode 100644 packages/server/src/api/controllers/view/utils.js diff --git a/packages/server/src/api/controllers/row/internal.js b/packages/server/src/api/controllers/row/internal.js index f8960129c2..d429c14cc7 100644 --- a/packages/server/src/api/controllers/row/internal.js +++ b/packages/server/src/api/controllers/row/internal.js @@ -20,6 +20,7 @@ const { fullSearch, paginatedSearch } = require("./internalSearch") const { getGlobalUsersFromMetadata } = require("../../../utilities/global") const inMemoryViews = require("../../../db/inMemoryView") const env = require("../../../environment") +const { migrateToInMemoryView } = require("../view/utils") const CALCULATION_TYPES = { SUM: "sum", @@ -74,15 +75,33 @@ async function getRawTableData(ctx, db, tableId) { async function getView(db, viewName) { let viewInfo - if (env.SELF_HOSTED) { + async function getFromDesignDoc() { const designDoc = await db.get("_design/database") viewInfo = designDoc.views[viewName] + return viewInfo + } + let migrate = false + if (env.SELF_HOSTED) { + viewInfo = await getFromDesignDoc() } else { - viewInfo = await db.get(generateMemoryViewID(viewName)) - if (viewInfo) { - viewInfo = viewInfo.view + try { + viewInfo = await db.get(generateMemoryViewID(viewName)) + if (viewInfo) { + viewInfo = viewInfo.view + } + } catch (err) { + // check if it can be retrieved from design doc (needs migrated) + if (err.status !== 404) { + viewInfo = null + } else { + viewInfo = await getFromDesignDoc() + migrate = !!viewInfo + } } } + if (migrate) { + await migrateToInMemoryView(db, viewName) + } if (!viewInfo) { throw "View does not exist." } @@ -193,9 +212,6 @@ exports.fetchView = async ctx => { const db = new CouchDB(appId) const { calculation, group, field } = ctx.query const viewInfo = await getView(db, viewName) - if (!viewInfo) { - throw "View does not exist." - } let response if (env.SELF_HOSTED) { response = await db.query(`database/${viewName}`, { diff --git a/packages/server/src/api/controllers/view/index.js b/packages/server/src/api/controllers/view/index.js index 4c2e6ee08b..ecaee0f32f 100644 --- a/packages/server/src/api/controllers/view/index.js +++ b/packages/server/src/api/controllers/view/index.js @@ -2,193 +2,93 @@ const CouchDB = require("../../../db") const viewTemplate = require("./viewBuilder") const { apiFileReturn } = require("../../../utilities/fileSystem") const exporters = require("./exporters") +const { saveView, getView, getViews, deleteView } = require("./utils") const { fetchView } = require("../row") -const { - ViewNames, - generateMemoryViewID, - getMemoryViewParams, -} = require("../../../db/utils") -const env = require("../../../environment") -async function getView(db, viewName) { - if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") - return designDoc.views[viewName] - } else { - const viewDoc = await db.get(generateMemoryViewID(viewName)) - return viewDoc.view +exports.fetch = async ctx => { + const db = new CouchDB(ctx.appId) + ctx.body = await getViews(db) +} + +exports.save = async ctx => { + const db = new CouchDB(ctx.appId) + const { originalName, ...viewToSave } = ctx.request.body + const view = viewTemplate(viewToSave) + + if (!viewToSave.name) { + ctx.throw(400, "Cannot create view without a name") + } + + await saveView(db, originalName, viewToSave.name, view) + + // add views to table document + const table = await db.get(ctx.request.body.tableId) + if (!table.views) table.views = {} + if (!view.meta.schema) { + view.meta.schema = table.schema + } + table.views[viewToSave.name] = view.meta + if (originalName) { + delete table.views[originalName] + } + await db.put(table) + + ctx.body = { + ...table.views[viewToSave.name], + name: viewToSave.name, } } -async function getViews(db) { - const response = [] - if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") - for (let name of Object.keys(designDoc.views)) { - // Only return custom views, not built ins - if (Object.values(ViewNames).indexOf(name) !== -1) { - continue - } - response.push({ - name, - ...designDoc.views[name], - }) +exports.destroy = async ctx => { + const db = new CouchDB(ctx.appId) + const viewName = decodeURI(ctx.params.viewName) + const view = await deleteView(db, viewName) + const table = await db.get(view.meta.tableId) + delete table.views[viewName] + await db.put(table) + + ctx.body = view +} + +exports.exportView = async ctx => { + const db = new CouchDB(ctx.appId) + const viewName = decodeURI(ctx.query.view) + const view = await getView(db, viewName) + + const format = ctx.query.format + if (!format) { + ctx.throw(400, "Format must be specified, either csv or json") + } + + if (view) { + ctx.params.viewName = viewName + // Fetch view rows + ctx.query = { + group: view.meta.groupBy, + calculation: view.meta.calculation, + stats: !!view.meta.field, + field: view.meta.field, } } else { - const views = ( - await db.allDocs( - getMemoryViewParams({ - include_docs: true, - }) - ) - ).rows.map(row => row.doc) - for (let viewDoc of views) { - response.push({ - name: viewDoc.name, - ...viewDoc.view, - }) - } + // table all_ view + /* istanbul ignore next */ + ctx.params.viewName = viewName } - return response -} -async function saveView(db, originalName, viewToSave, viewTemplate) { - if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") - designDoc.views = { - ...designDoc.views, - [viewToSave.name]: viewTemplate, - } - // view has been renamed - if (originalName) { - delete designDoc.views[originalName] - } - await db.put(designDoc) - } else { - const id = generateMemoryViewID(viewToSave.name) - const originalId = originalName ? generateMemoryViewID(originalName) : null - const viewDoc = { - _id: id, - view: viewTemplate, - name: viewToSave.name, - tableId: viewTemplate.meta.tableId, - } - try { - const old = await db.get(id) - if (originalId) { - const originalDoc = await db.get(originalId) - await db.remove(originalDoc._id, originalDoc._rev) - } - if (old && old._rev) { - viewDoc._rev = old._rev - } - } catch (err) { - // didn't exist, just skip - } - await db.put(viewDoc) + await fetchView(ctx) + + let schema = view && view.meta && view.meta.schema + if (!schema) { + const tableId = ctx.params.tableId || view.meta.tableId + const table = await db.get(tableId) + schema = table.schema } + + // Export part + let headers = Object.keys(schema) + const exporter = exporters[format] + const filename = `${viewName}.${format}` + // send down the file + ctx.attachment(filename) + ctx.body = apiFileReturn(exporter(headers, ctx.body)) } - -async function deleteView(db, viewName) { - if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") - const view = designDoc.views[viewName] - delete designDoc.views[viewName] - await db.put(designDoc) - return view - } else { - const id = generateMemoryViewID(viewName) - const viewDoc = await db.get(id) - await db.remove(viewDoc._id, viewDoc._rev) - return viewDoc.view - } -} - -const controller = { - fetch: async ctx => { - const db = new CouchDB(ctx.appId) - ctx.body = await getViews(db) - }, - save: async ctx => { - const db = new CouchDB(ctx.appId) - const { originalName, ...viewToSave } = ctx.request.body - const view = viewTemplate(viewToSave) - - if (!viewToSave.name) { - ctx.throw(400, "Cannot create view without a name") - } - - await saveView(db, originalName, viewToSave, view) - - // add views to table document - const table = await db.get(ctx.request.body.tableId) - if (!table.views) table.views = {} - if (!view.meta.schema) { - view.meta.schema = table.schema - } - table.views[viewToSave.name] = view.meta - if (originalName) { - delete table.views[originalName] - } - await db.put(table) - - ctx.body = { - ...table.views[viewToSave.name], - name: viewToSave.name, - } - }, - destroy: async ctx => { - const db = new CouchDB(ctx.appId) - const viewName = decodeURI(ctx.params.viewName) - const view = await deleteView(db, viewName) - const table = await db.get(view.meta.tableId) - delete table.views[viewName] - await db.put(table) - - ctx.body = view - }, - exportView: async ctx => { - const db = new CouchDB(ctx.appId) - const viewName = decodeURI(ctx.query.view) - const view = await getView(db, viewName) - - const format = ctx.query.format - if (!format) { - ctx.throw(400, "Format must be specified, either csv or json") - } - - if (view) { - ctx.params.viewName = viewName - // Fetch view rows - ctx.query = { - group: view.meta.groupBy, - calculation: view.meta.calculation, - stats: !!view.meta.field, - field: view.meta.field, - } - } else { - // table all_ view - /* istanbul ignore next */ - ctx.params.viewName = viewName - } - - await fetchView(ctx) - - let schema = view && view.meta && view.meta.schema - if (!schema) { - const tableId = ctx.params.tableId || view.meta.tableId - const table = await db.get(tableId) - schema = table.schema - } - - // Export part - let headers = Object.keys(schema) - const exporter = exporters[format] - const filename = `${viewName}.${format}` - // send down the file - ctx.attachment(filename) - ctx.body = apiFileReturn(exporter(headers, ctx.body)) - }, -} - -module.exports = controller diff --git a/packages/server/src/api/controllers/view/utils.js b/packages/server/src/api/controllers/view/utils.js new file mode 100644 index 0000000000..c93604177f --- /dev/null +++ b/packages/server/src/api/controllers/view/utils.js @@ -0,0 +1,109 @@ +const { + ViewNames, + generateMemoryViewID, + getMemoryViewParams, +} = require("../../../db/utils") +const env = require("../../../environment") + +exports.getView = async (db, viewName) => { + if (env.SELF_HOSTED) { + const designDoc = await db.get("_design/database") + return designDoc.views[viewName] + } else { + const viewDoc = await db.get(generateMemoryViewID(viewName)) + return viewDoc.view + } +} + +exports.getViews = async db => { + const response = [] + if (env.SELF_HOSTED) { + const designDoc = await db.get("_design/database") + for (let name of Object.keys(designDoc.views)) { + // Only return custom views, not built ins + if (Object.values(ViewNames).indexOf(name) !== -1) { + continue + } + response.push({ + name, + ...designDoc.views[name], + }) + } + } else { + const views = ( + await db.allDocs( + getMemoryViewParams({ + include_docs: true, + }) + ) + ).rows.map(row => row.doc) + for (let viewDoc of views) { + response.push({ + name: viewDoc.name, + ...viewDoc.view, + }) + } + } + return response +} + +exports.saveView = async (db, originalName, viewName, viewTemplate) => { + if (env.SELF_HOSTED) { + const designDoc = await db.get("_design/database") + designDoc.views = { + ...designDoc.views, + [viewName]: viewTemplate, + } + // view has been renamed + if (originalName) { + delete designDoc.views[originalName] + } + await db.put(designDoc) + } else { + const id = generateMemoryViewID(viewName) + const originalId = originalName ? generateMemoryViewID(originalName) : null + const viewDoc = { + _id: id, + view: viewTemplate, + name: viewName, + tableId: viewTemplate.meta.tableId, + } + try { + const old = await db.get(id) + if (originalId) { + const originalDoc = await db.get(originalId) + await db.remove(originalDoc._id, originalDoc._rev) + } + if (old && old._rev) { + viewDoc._rev = old._rev + } + } catch (err) { + // didn't exist, just skip + } + await db.put(viewDoc) + } +} + +exports.deleteView = async (db, viewName) => { + if (env.SELF_HOSTED) { + const designDoc = await db.get("_design/database") + const view = designDoc.views[viewName] + delete designDoc.views[viewName] + await db.put(designDoc) + return view + } else { + const id = generateMemoryViewID(viewName) + const viewDoc = await db.get(id) + await db.remove(viewDoc._id, viewDoc._rev) + return viewDoc.view + } +} + +exports.migrateToInMemoryView = async (db, viewName) => { + // delete the view initially + const designDoc = await db.get("_design/database") + const view = designDoc.views[viewName] + delete designDoc.views[viewName] + await db.put(designDoc) + await exports.saveView(db, null, viewName, view) +} diff --git a/packages/server/src/api/routes/tests/view.spec.js b/packages/server/src/api/routes/tests/view.spec.js index 458da6e023..b1c5f655c6 100644 --- a/packages/server/src/api/routes/tests/view.spec.js +++ b/packages/server/src/api/routes/tests/view.spec.js @@ -205,7 +205,7 @@ describe("/views", () => { }) describe("exportView", () => { - it("should be able to delete a view", async () => { + it("should be able to export a view", async () => { await config.createTable(priceTable()) await config.createRow() const view = await config.createView() diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 98ca904b62..89e015b6f5 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -26,7 +26,7 @@ module.exports = { COUCH_DB_URL: process.env.COUCH_DB_URL, MINIO_URL: process.env.MINIO_URL, WORKER_URL: process.env.WORKER_URL, - SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED), + SELF_HOSTED: process.env.SELF_HOSTED, AWS_REGION: process.env.AWS_REGION, ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, From b4f9fd4f12050a028ad9c8264461724c89fbb478 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 21 Sep 2021 20:21:15 +0100 Subject: [PATCH 10/55] changed events to constants, instantiated analytics client at right time --- .github/workflows/release.yml | 14 ++- packages/auth/src/objectStore/index.js | 2 +- packages/builder/src/App.svelte | 6 -- .../builder/src/analytics/IntercomClient.js | 10 +- .../builder/src/analytics/PosthogClient.js | 90 ++---------------- .../builder/src/analytics/SentryClient.js | 14 +-- packages/builder/src/analytics/constants.js | 94 +++++++++---------- packages/builder/src/analytics/index.js | 53 ++++++----- packages/builder/src/builderStore/index.js | 10 -- .../builderStore/store/automation/index.js | 4 +- .../src/builderStore/store/frontend.js | 4 +- .../CreateAutomationModal.svelte | 4 +- .../UpdateAutomationModal.svelte | 4 +- .../DataTable/modals/CalculateModal.svelte | 4 +- .../DataTable/modals/CreateViewModal.svelte | 4 +- .../DataTable/modals/FilterModal.svelte | 4 +- .../modals/CreateDatasourceModal.svelte | 4 +- .../modals/UpdateDatasourceModal.svelte | 4 +- .../modals/CreateTableModal.svelte | 4 +- .../src/components/deploy/DeployModal.svelte | 2 +- .../NavigationPanel/NewScreenModal.svelte | 4 +- .../src/pages/builder/auth/login.svelte | 12 +-- .../pages/builder/portal/apps/index.svelte | 12 +-- .../portal/settings/organisation.svelte | 8 +- packages/builder/src/stores/portal/auth.js | 14 +++ packages/server/package.json | 22 ++--- 26 files changed, 156 insertions(+), 251 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 00e7f9a795..7e0685c2c1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,7 +48,19 @@ jobs: uses: "WyriHaximus/github-action-get-previous-tag@v1" - name: Build/release Docker images - run: | + if: ${{ github.event.inputs.release_self_host != 'Y' }} + run: | + docker login -u $DOCKER_USER -p $DOCKER_PASSWORD + yarn build + yarn build:docker + env: + DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} + BUDIBASE_RELEASE_VERSION: ${{ steps.previoustag.outputs.tag }} + + - name: Build/release Docker images (Self Host) + if: ${{ github.event.inputs.release_self_host == 'Y' }} + run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD yarn build yarn build:docker diff --git a/packages/auth/src/objectStore/index.js b/packages/auth/src/objectStore/index.js index 0f1307c39d..9f271ad80e 100644 --- a/packages/auth/src/objectStore/index.js +++ b/packages/auth/src/objectStore/index.js @@ -73,7 +73,7 @@ exports.ObjectStore = bucket => { AWS.config.update({ accessKeyId: env.MINIO_ACCESS_KEY, secretAccessKey: env.MINIO_SECRET_KEY, - region: env.AWS_REGION + region: env.AWS_REGION, }) const config = { s3ForcePathStyle: true, diff --git a/packages/builder/src/App.svelte b/packages/builder/src/App.svelte index 0624690b27..60051ea043 100644 --- a/packages/builder/src/App.svelte +++ b/packages/builder/src/App.svelte @@ -1,16 +1,10 @@ diff --git a/packages/builder/src/analytics/IntercomClient.js b/packages/builder/src/analytics/IntercomClient.js index 2cabeb97df..10c2f99695 100644 --- a/packages/builder/src/analytics/IntercomClient.js +++ b/packages/builder/src/analytics/IntercomClient.js @@ -48,7 +48,7 @@ export default class IntercomClient { return window.Intercom("boot", { app_id: this.token, - ...user + ...user, }) } @@ -61,6 +61,12 @@ export default class IntercomClient { captureEvent(event, props = {}) { if (!this.initialised) return - window.Intercom("trackEvent", event, props); + window.Intercom("trackEvent", event, props) + } + + logout() { + if (!this.initialised) return + + window.Intercom("shutdown") } } diff --git a/packages/builder/src/analytics/PosthogClient.js b/packages/builder/src/analytics/PosthogClient.js index 9a98520c91..8387c310d3 100644 --- a/packages/builder/src/analytics/PosthogClient.js +++ b/packages/builder/src/analytics/PosthogClient.js @@ -1,13 +1,6 @@ import posthog from "posthog-js" import { Events } from "./constants" -// let analyticsEnabled -// const posthogConfigured = process.env.POSTHOG_TOKEN && process.env.POSTHOG_URL - -// const FEEDBACK_SUBMITTED_KEY = "budibase:feedback_submitted" -// const APP_FIRST_STARTED_KEY = "budibase:first_run" -// const feedbackHours = 12 - export default class PosthogClient { constructor(token, url) { this.token = token @@ -27,18 +20,6 @@ export default class PosthogClient { this.initialised = true } - disabled() { - return posthog.has_opted_out_capturing() - } - - optIn() { - return posthog.opt_in_capturing() - } - - optOut() { - return posthog.opt_out_capturing() - } - identify(id) { if (!this.initialised) return @@ -76,69 +57,10 @@ export default class PosthogClient { posthog.capture(Events.NPS.SUBMITTED, prefixedFeedback) } + + logout() { + if (!this.initialised) return + + posthog.reset() + } } - -// function captureEvent(eventName, props = {}) { -// if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return -// props.sourceApp = "builder" -// posthog.capture(eventName, props) -// } - -// if (!localStorage.getItem(APP_FIRST_STARTED_KEY)) { -// localStorage.setItem(APP_FIRST_STARTED_KEY, Date.now()) -// } - -// function submitFeedback(values) { -// if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return -// localStorage.setItem(FEEDBACK_SUBMITTED_KEY, Date.now()) - -// const prefixedValues = Object.entries(values).reduce((obj, [key, value]) => { -// obj[`feedback_${key}`] = value -// return obj -// }, {}) - -// posthog.capture("Feedback Submitted", prefixedValues) -// } - -// function requestFeedbackOnDeploy() { -// if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return false -// const lastSubmittedStr = localStorage.getItem(FEEDBACK_SUBMITTED_KEY) -// if (!lastSubmittedStr) return true -// return isFeedbackTimeElapsed(lastSubmittedStr) -// } - -// function highlightFeedbackIcon() { -// if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return false -// const lastSubmittedStr = localStorage.getItem(FEEDBACK_SUBMITTED_KEY) -// if (lastSubmittedStr) return isFeedbackTimeElapsed(lastSubmittedStr) -// const firstRunStr = localStorage.getItem(APP_FIRST_STARTED_KEY) -// if (!firstRunStr) return false -// return isFeedbackTimeElapsed(firstRunStr) -// } - -// Opt In/Out -// const ifAnalyticsEnabled = func => () => { -// if (analyticsEnabled && process.env.POSTHOG_TOKEN) { -// return func() -// } -// } -// const disabled = () => posthog.has_opted_out_capturing() -// const optIn = () => posthog.opt_in_capturing() -// const optOut = () => posthog.opt_out_capturing() - -// export default { -// init, -// identify, -// captureException, -// captureEvent, -// submitFeedback, -// highlightFeedbackIcon, -// disabled: () => { -// if (analyticsEnabled == null) { -// return true -// } -// return ifAnalyticsEnabled(disabled) -// }, -// optIn: ifAnalyticsEnabled(optIn), -// optOut: ifAnalyticsEnabled(optOut), -// } diff --git a/packages/builder/src/analytics/SentryClient.js b/packages/builder/src/analytics/SentryClient.js index 11377d254f..b871ea15d9 100644 --- a/packages/builder/src/analytics/SentryClient.js +++ b/packages/builder/src/analytics/SentryClient.js @@ -20,20 +20,10 @@ export default class SentryClient { } identify(id) { + if (!this.initalised) return + Sentry.configureScope(scope => { scope.setUser({ id }) }) } } - -// export function init() { -// if (process.env.SENTRY_DSN) { -// Sentry.init({ dsn: process.env.SENTRY_DSN }) -// } -// } - -// export function captureException(err) { -// // if (!analyticsEnabled) return -// Sentry.captureException(err) -// // captureEvent("Error", { error: err.message ? err.message : err }) -// } diff --git a/packages/builder/src/analytics/constants.js b/packages/builder/src/analytics/constants.js index 1102ad3ce4..d38b7bba4f 100644 --- a/packages/builder/src/analytics/constants.js +++ b/packages/builder/src/analytics/constants.js @@ -1,49 +1,49 @@ export const Events = { - BUILDER: { - STARTED: "Builder Started" - }, - COMPONENT: { - CREATED: "Added Component" - }, - DATASOURCE: { - CREATED: "Datasource Created", - UPDATED: "Datasource Updated", - }, - TABLE: { - CREATED: "Table Created", - }, - VIEW: { - CREATED: "View Created", - ADDED_FILTER: "Added View Filter", - ADDED_CALCULATE: "Added View Calculate" - }, - SCREEN: { - CREATED: "Screen Created" - }, - AUTOMATION: { - CREATED: "Added Component", - SAVED: "Automation Saved", - BLOCK_ADDED: "Added Automation Block", - }, - NPS: { - SUBMITTED: "budibase:feedback_submitted" - }, - APP: { - CREATED: "budibase:app_created", - PUBLISHED: "budibase:app_published", - UNPUBLISHED: "budibase:app_unpublished" - }, - ANALYTICS: { - OPT_IN: "budibase:analytics_opt_in", - OPT_OUT: "budibase:analytics_opt_out" - }, - USER: { - INVITE: "budibase:portal_user_invite" - }, - SMTP: { - SAVED: "budibase:smtp_saved" - }, - SSO: { - SAVED: "budibase:sso_saved" - } + BUILDER: { + STARTED: "Builder Started", + }, + COMPONENT: { + CREATED: "Added Component", + }, + DATASOURCE: { + CREATED: "Datasource Created", + UPDATED: "Datasource Updated", + }, + TABLE: { + CREATED: "Table Created", + }, + VIEW: { + CREATED: "View Created", + ADDED_FILTER: "Added View Filter", + ADDED_CALCULATE: "Added View Calculate", + }, + SCREEN: { + CREATED: "Screen Created", + }, + AUTOMATION: { + CREATED: "Automation Created", + SAVED: "Automation Saved", + BLOCK_ADDED: "Added Automation Block", + }, + NPS: { + SUBMITTED: "budibase:feedback_submitted", + }, + APP: { + CREATED: "budibase:app_created", + PUBLISHED: "budibase:app_published", + UNPUBLISHED: "budibase:app_unpublished", + }, + ANALYTICS: { + OPT_IN: "budibase:analytics_opt_in", + OPT_OUT: "budibase:analytics_opt_out", + }, + USER: { + INVITE: "budibase:portal_user_invite", + }, + SMTP: { + SAVED: "budibase:smtp_saved", + }, + SSO: { + SAVED: "budibase:sso_saved", + }, } diff --git a/packages/builder/src/analytics/index.js b/packages/builder/src/analytics/index.js index 9fd8f82668..b79ab67e0c 100644 --- a/packages/builder/src/analytics/index.js +++ b/packages/builder/src/analytics/index.js @@ -3,20 +3,15 @@ import PosthogClient from "./PosthogClient" import IntercomClient from "./IntercomClient" import SentryClient from "./SentryClient" import { Events } from "./constants" +import { auth } from "stores/portal" +import { get } from "svelte/store" -// const posthog = new PosthogClient( -// process.env.POSTHOG_TOKEN, -// process.env.POSTHOG_URL -// ) const posthog = new PosthogClient( - "phc_yGOn4i7jWKaCTapdGR6lfA4AvmuEQ2ijn5zAVSFYPlS", - "https://app.posthog.com" + process.env.POSTHOG_TOKEN, + process.env.POSTHOG_URL ) -// const sentry = new SentryClient(process.env.SENTRY_DSN) -const sentry = new SentryClient("https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131") -// const intercom = new IntercomClient(process.env.INTERCOM_TOKEN) -const intercom = new IntercomClient("qz2sxfuv") - +const sentry = new SentryClient(process.env.SENTRY_DSN) +const intercom = new IntercomClient(process.env.INTERCOM_TOKEN) class AnalyticsHub { constructor() { @@ -24,24 +19,27 @@ class AnalyticsHub { } async activate() { + // Setting the analytics env var off in the backend overrides org/tenant settings const analyticsStatus = await api.get("/api/analytics") const json = await analyticsStatus.json() - if (json.enabled) { - this.clients.forEach(client => client.init()) + // Multitenancy disabled on the backend + if (!json.enabled) return + + const tenantId = get(auth).tenantId + + if (tenantId) { + const res = await api.get( + `/api/global/configs/public?tenantId=${tenantId}` + ) + const orgJson = await res.json() + + // analytics opted out for the tenant + if (orgJson.config?.analytics === false) return } - this.enabled = json.enabled - } - - optIn() { - this.captureEvent(Events.ANALYTICS.OPT_IN) - this.clients.forEach(client => client.optIn()) - } - - optOut() { - this.captureEvent(Events.ANALYTICS.OPT_OUT) - this.clients.forEach(client => client.optOut()) + this.clients.forEach(client => client.init()) + this.enabled = true } identify(id, metadata) { @@ -68,9 +66,14 @@ class AnalyticsHub { submitFeedback(values) { posthog.npsFeedback(values) } + + async logout() { + posthog.logout() + intercom.logout() + } } const analytics = new AnalyticsHub() export { Events } -export default analytics \ No newline at end of file +export default analytics diff --git a/packages/builder/src/builderStore/index.js b/packages/builder/src/builderStore/index.js index fb888b6fc9..f32dedd47e 100644 --- a/packages/builder/src/builderStore/index.js +++ b/packages/builder/src/builderStore/index.js @@ -3,7 +3,6 @@ import { getAutomationStore } from "./store/automation" import { getHostingStore } from "./store/hosting" import { getThemeStore } from "./store/theme" import { derived, writable } from "svelte/store" -import analytics, { Events } from "analytics" import { FrontendTypes, LAYOUT_NAMES } from "../constants" import { findComponent } from "./storeUtils" @@ -55,13 +54,4 @@ export const mainLayout = derived(store, $store => { export const selectedAccessRole = writable("BASIC") -export const initialise = async () => { - try { - await analytics.activate() - analytics.captureEvent(Events.BUILDER.STARTED) - } catch (err) { - console.log(err) - } -} - export const screenSearchString = writable(null) diff --git a/packages/builder/src/builderStore/store/automation/index.js b/packages/builder/src/builderStore/store/automation/index.js index e60553070b..0a47970d28 100644 --- a/packages/builder/src/builderStore/store/automation/index.js +++ b/packages/builder/src/builderStore/store/automation/index.js @@ -2,7 +2,7 @@ import { writable } from "svelte/store" import api from "../../api" import Automation from "./Automation" import { cloneDeep } from "lodash/fp" -import analytics from "analytics" +import analytics, { Events } from "analytics" const automationActions = store => ({ fetch: async () => { @@ -110,7 +110,7 @@ const automationActions = store => ({ state.selectedBlock = newBlock return state }) - analytics.captureEvent("Added Automation Block", { + analytics.captureEvent(Events.AUTOMATION.BLOCK_ADDED, { name: block.name, }) }, diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 603fa88b09..b7c42003da 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -19,7 +19,7 @@ import { import { fetchComponentLibDefinitions } from "../loadComponentLibraries" import api from "../api" import { FrontendTypes } from "constants" -import analytics from "analytics" +import analytics, { Events } from "analytics" import { findComponentType, findComponentParent, @@ -443,7 +443,7 @@ export const getFrontendStore = () => { }) // Log event - analytics.captureEvent("Added Component", { + analytics.captureEvent(Events.COMPONENT.CREATED, { name: componentInstance._component, }) diff --git a/packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte b/packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte index 6580cd0b87..33a2f79b6f 100644 --- a/packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte +++ b/packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte @@ -4,7 +4,7 @@ import { automationStore } from "builderStore" import { notifications } from "@budibase/bbui" import { Input, ModalContent, Layout, Body, Icon } from "@budibase/bbui" - import analytics from "analytics" + import analytics, { Events } from "analytics" let name let selectedTrigger @@ -36,7 +36,7 @@ notifications.success(`Automation ${name} created.`) $goto(`./${$automationStore.selectedAutomation.automation._id}`) - analytics.captureEvent("Automation Created", { name }) + analytics.captureEvent(Events.AUTOMATION.CREATED, { name }) } $: triggers = Object.entries($automationStore.blockDefinitions.TRIGGER) diff --git a/packages/builder/src/components/automation/AutomationPanel/UpdateAutomationModal.svelte b/packages/builder/src/components/automation/AutomationPanel/UpdateAutomationModal.svelte index 29966ec372..64197c3a77 100644 --- a/packages/builder/src/components/automation/AutomationPanel/UpdateAutomationModal.svelte +++ b/packages/builder/src/components/automation/AutomationPanel/UpdateAutomationModal.svelte @@ -2,7 +2,7 @@ import { automationStore } from "builderStore" import { notifications } from "@budibase/bbui" import { Icon, Input, ModalContent, Modal } from "@budibase/bbui" - import analytics from "analytics" + import analytics, { Events } from "analytics" let name let error = "" @@ -26,7 +26,7 @@ } await automationStore.actions.save(updatedAutomation) notifications.success(`Automation ${name} updated successfully.`) - analytics.captureEvent("Automation Saved", { name }) + analytics.captureEvent(Events.AUTOMATION.SAVED, { name }) hide() } diff --git a/packages/builder/src/components/backend/DataTable/modals/CalculateModal.svelte b/packages/builder/src/components/backend/DataTable/modals/CalculateModal.svelte index 660a822898..50d44eca88 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CalculateModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CalculateModal.svelte @@ -1,7 +1,7 @@ diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateViewModal.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateViewModal.svelte index 61777c0b7e..2f6ec51233 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateViewModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateViewModal.svelte @@ -3,7 +3,7 @@ import { goto } from "@roxi/routify" import { views as viewsStore } from "stores/backend" import { tables } from "stores/backend" - import analytics from "analytics" + import analytics, { Events } from "analytics" let name let field @@ -21,7 +21,7 @@ field, }) notifications.success(`View ${name} created`) - analytics.captureEvent("View Created", { name }) + analytics.captureEvent(Events.VIEW.CREATED, { name }) $goto(`../../view/${name}`) } diff --git a/packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte b/packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte index 170bb75142..9c6f4956b0 100644 --- a/packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte @@ -11,7 +11,7 @@ Icon, } from "@budibase/bbui" import { tables, views } from "stores/backend" - import analytics from "analytics" + import analytics, { Events } from "analytics" const CONDITIONS = [ { @@ -65,7 +65,7 @@ function saveView() { views.save(view) notifications.success(`View ${view.name} saved.`) - analytics.captureEvent("Added View Filter", { + analytics.captureEvent(Events.VIEW.ADDED_FILTER, { filters: JSON.stringify(view.filters), }) } diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte index 9cdd893230..e7affb30c4 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte @@ -5,7 +5,7 @@ import { Input, Label, ModalContent, Modal, Context } from "@budibase/bbui" import TableIntegrationMenu from "../TableIntegrationMenu/index.svelte" import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte" - import analytics from "analytics" + import analytics, { Events } from "analytics" import { getContext } from "svelte" const modalContext = getContext(Context.Modal) @@ -45,7 +45,7 @@ plus, }) notifications.success(`Datasource ${name} created successfully.`) - analytics.captureEvent("Datasource Created", { name, type }) + analytics.captureEvent(Events.DATASOURCE.CREATED, { name, type }) // Navigate to new datasource $goto(`./datasource/${response._id}`) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/UpdateDatasourceModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/UpdateDatasourceModal.svelte index f93af59a38..28625aa86e 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/UpdateDatasourceModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/UpdateDatasourceModal.svelte @@ -2,7 +2,7 @@ import { datasources } from "stores/backend" import { notifications } from "@budibase/bbui" import { Input, ModalContent, Modal } from "@budibase/bbui" - import analytics from "analytics" + import analytics, { Events } from "analytics" let error = "" let modal @@ -35,7 +35,7 @@ } await datasources.save(updatedDatasource) notifications.success(`Datasource ${name} updated successfully.`) - analytics.captureEvent("Datasource Updated", updatedDatasource) + analytics.captureEvent(Events.DATASOURCE.UPDATED, updatedDatasource) hide() } diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte index b59e5cda5e..dd8876be27 100644 --- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte +++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte @@ -12,7 +12,7 @@ Layout, } from "@budibase/bbui" import TableDataImport from "../TableDataImport.svelte" - import analytics from "analytics" + import analytics, { Events } from "analytics" import screenTemplates from "builderStore/store/screenTemplates" import { buildAutoColumn, getAutoColumnInformation } from "builderStore/utils" import { NEW_ROW_TEMPLATE } from "builderStore/store/screenTemplates/newRowScreen" @@ -67,7 +67,7 @@ // Create table const table = await tables.save(newTable) notifications.success(`Table ${name} created successfully.`) - analytics.captureEvent("Table Created", { name }) + analytics.captureEvent(Events.TABLE.CREATED, { name }) // Create auto screens if (createAutoscreens) { diff --git a/packages/builder/src/components/deploy/DeployModal.svelte b/packages/builder/src/components/deploy/DeployModal.svelte index 5500aff866..3dcf0c27b1 100644 --- a/packages/builder/src/components/deploy/DeployModal.svelte +++ b/packages/builder/src/components/deploy/DeployModal.svelte @@ -25,7 +25,7 @@ throw new Error(`status ${response.status}`) } else { analytics.captureEvent(Events.APP.PUBLISHED, { - appId: $store.appId + appId: $store.appId, }) notifications.success(`Application published successfully`) } diff --git a/packages/builder/src/components/design/NavigationPanel/NewScreenModal.svelte b/packages/builder/src/components/design/NavigationPanel/NewScreenModal.svelte index ed0c764956..e02f9d87e5 100644 --- a/packages/builder/src/components/design/NavigationPanel/NewScreenModal.svelte +++ b/packages/builder/src/components/design/NavigationPanel/NewScreenModal.svelte @@ -4,7 +4,7 @@ import { roles } from "stores/backend" import { Input, Select, ModalContent, Toggle } from "@budibase/bbui" import getTemplates from "builderStore/store/screenTemplates" - import analytics from "analytics" + import analytics, { Events } from "analytics" const CONTAINER = "@budibase/standard-components/container" @@ -66,7 +66,7 @@ if (templateIndex !== undefined) { const template = templates[templateIndex] - analytics.captureEvent("Screen Created", { + analytics.captureEvent(Events.SCREEN.CREATED, { template: template.id || template.name, }) } diff --git a/packages/builder/src/pages/builder/auth/login.svelte b/packages/builder/src/pages/builder/auth/login.svelte index 568a0550a3..f9f2b34578 100644 --- a/packages/builder/src/pages/builder/auth/login.svelte +++ b/packages/builder/src/pages/builder/auth/login.svelte @@ -15,7 +15,6 @@ import OIDCButton from "./_components/OIDCButton.svelte" import Logo from "assets/bb-emblem.svg" import { onMount } from "svelte" - import analytics from "analytics" let username = "" let password = "" @@ -26,20 +25,11 @@ async function login() { try { - const { user } = await auth.login({ + await auth.login({ username, password, }) - analytics.identify(user._id, user) - analytics.showChat({ - email: user.email, - created_at: Date.now(), - name: user.name, - user_id: user._id, - tenant: user.tenantId - }) - if ($auth?.user?.forceResetPassword) { $goto("./reset") } else { diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte index 319a00f471..d84b327e90 100644 --- a/packages/builder/src/pages/builder/portal/apps/index.svelte +++ b/packages/builder/src/pages/builder/portal/apps/index.svelte @@ -15,8 +15,7 @@ } from "@budibase/bbui" import CreateAppModal from "components/start/CreateAppModal.svelte" import UpdateAppModal from "components/start/UpdateAppModal.svelte" - import api, { del } from "builderStore/api" - import analytics from "analytics" + import { del } from "builderStore/api" import { onMount } from "svelte" import { apps, auth, admin } from "stores/portal" import download from "downloadjs" @@ -66,14 +65,6 @@ } } - const checkKeys = async () => { - const response = await api.get(`/api/keys/`) - const keys = await response.json() - // if (keys.userId) { - // analytics.identify(keys.userId) - // } - } - const initiateAppCreation = () => { creationModal.show() creatingApp = true @@ -188,7 +179,6 @@ } onMount(async () => { - checkKeys() await apps.load() loaded = true }) diff --git a/packages/builder/src/pages/builder/portal/settings/organisation.svelte b/packages/builder/src/pages/builder/portal/settings/organisation.svelte index 3d149e6e60..79eaebb28b 100644 --- a/packages/builder/src/pages/builder/portal/settings/organisation.svelte +++ b/packages/builder/src/pages/builder/portal/settings/organisation.svelte @@ -48,13 +48,6 @@ async function saveConfig() { loading = true - // Set analytics preference - if ($values.analytics) { - analytics.optIn() - } else { - analytics.optOut() - } - // Upload logo if required if ($values.logo && !$values.logo.url) { await uploadLogo($values.logo) @@ -64,6 +57,7 @@ const config = { company: $values.company ?? "", platformUrl: $values.platformUrl ?? "", + analytics: $values.analytics, } // remove logo if required if (!$values.logo) { diff --git a/packages/builder/src/stores/portal/auth.js b/packages/builder/src/stores/portal/auth.js index fe8f87cfb2..22f7fedca4 100644 --- a/packages/builder/src/stores/portal/auth.js +++ b/packages/builder/src/stores/portal/auth.js @@ -1,6 +1,7 @@ import { derived, writable, get } from "svelte/store" import api from "../../builderStore/api" import { admin } from "stores/portal" +import analytics from "analytics" export function createAuthStore() { const auth = writable({ @@ -49,6 +50,19 @@ export function createAuthStore() { } return store }) + + if (user) { + analytics.activate().then(() => { + analytics.identify(user._id, user) + // TODO: check if the user is a certain org size + // analytics.showChat({ + // email: user.email, + // name: user.name, + // user_id: user._id, + // tenant: user.tenantId + // }) + }) + } } async function setOrganisation(tenantId) { diff --git a/packages/server/package.json b/packages/server/package.json index 6d6b05752e..fb2e522b87 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.125-alpha.18", + "version": "0.9.136", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -13,7 +13,7 @@ "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/", "test": "jest --coverage --maxWorkers=2", "test:watch": "jest --watch", - "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client", + "predocker": "copyfiles -f ../client/dist/budibase-client.js ../standard-components/manifest.json client", "build:docker": "yarn run predocker && docker build . -t app-service", "run:docker": "node dist/index.js", "dev:stack:up": "node scripts/dev/manage.js up", @@ -23,10 +23,9 @@ "format": "prettier --config ../../.prettierrc.json 'src/**/*.ts' --write", "lint": "eslint --fix src/", "lint:fix": "yarn run format && yarn run lint", + "initialise": "node scripts/initialise.js", "multi:enable": "node scripts/multiTenancy.js enable", - "multi:disable": "node scripts/multiTenancy.js disable", - "selfhost:enable": "node scripts/selfhost.js enable", - "selfhost:disable": "node scripts/selfhost.js disable" + "multi:disable": "node scripts/multiTenancy.js disable" }, "jest": { "preset": "ts-jest", @@ -49,8 +48,7 @@ "!src/automations/tests/**/*", "!src/utilities/fileProcessor.js", "!src/utilities/fileSystem/**/*", - "!src/utilities/redis.js", - "!src/api/controllers/row/internalSearch.js" + "!src/utilities/redis.js" ], "coverageReporters": [ "lcov", @@ -64,9 +62,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.125-alpha.18", - "@budibase/client": "^0.9.125-alpha.18", - "@budibase/string-templates": "^0.9.125-alpha.18", + "@budibase/auth": "^0.9.136", + "@budibase/client": "^0.9.136", + "@budibase/string-templates": "^0.9.136", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", @@ -98,7 +96,7 @@ "lodash": "4.17.21", "mongodb": "3.6.3", "mssql": "6.2.3", - "mysql": "^2.18.1", + "mysql": "2.18.1", "node-fetch": "2.6.0", "open": "7.3.0", "pg": "8.5.1", @@ -112,12 +110,14 @@ "to-json-schema": "0.2.5", "uuid": "3.3.2", "validate.js": "0.13.1", + "worker-farm": "1.7.0", "yargs": "13.2.4", "zlib": "1.0.5" }, "devDependencies": { "@babel/core": "^7.14.3", "@babel/preset-env": "^7.14.4", + "@budibase/standard-components": "^0.9.136", "@jest/test-sequencer": "^24.8.0", "@types/bull": "^3.15.1", "@types/jest": "^26.0.23", From a00598ce38e97eea407ef540730f0891f2ae8703 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 21 Sep 2021 20:28:33 +0100 Subject: [PATCH 11/55] JSdoc --- .../builder/src/analytics/IntercomClient.js | 26 +++++++++++++++++-- .../builder/src/analytics/PosthogClient.js | 26 ++++++++++++++----- .../builder/src/analytics/SentryClient.js | 8 ++++++ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/analytics/IntercomClient.js b/packages/builder/src/analytics/IntercomClient.js index 10c2f99695..8cc7e35bbf 100644 --- a/packages/builder/src/analytics/IntercomClient.js +++ b/packages/builder/src/analytics/IntercomClient.js @@ -3,6 +3,9 @@ export default class IntercomClient { this.token = token } + /** + * Instantiate intercom using their provided script. + */ init() { if (!this.token) return @@ -43,6 +46,11 @@ export default class IntercomClient { } } + /** + * Show the intercom chat bubble. + * @param {Object} user - user to identify + * @returns Intercom global object + */ show(user = {}) { if (!this.initialised) return @@ -52,21 +60,35 @@ export default class IntercomClient { }) } + /** + * Update intercom user details and messages. + * @returns Intercom global object + */ update() { if (!this.initialised) return return window.Intercom("update") } + /** + * Capture analytics events and send them to intercom. + * @param {String} event - event identifier + * @param {Object} props - properties for the event + * @returns Intercom global object + */ captureEvent(event, props = {}) { if (!this.initialised) return - window.Intercom("trackEvent", event, props) + return window.Intercom("trackEvent", event, props) } + /** + * Disassociate the user from the current session. + * @returns Intercom global object + */ logout() { if (!this.initialised) return - window.Intercom("shutdown") + return window.Intercom("shutdown") } } diff --git a/packages/builder/src/analytics/PosthogClient.js b/packages/builder/src/analytics/PosthogClient.js index 8387c310d3..4c45c10216 100644 --- a/packages/builder/src/analytics/PosthogClient.js +++ b/packages/builder/src/analytics/PosthogClient.js @@ -20,24 +20,31 @@ export default class PosthogClient { this.initialised = true } + /** + * Set the posthog context to the current user + * @param {String} id - unique user id + */ identify(id) { if (!this.initialised) return posthog.identify(id) } + /** + * Update user metadata associated with current user in posthog + * @param {Object} meta - user fields + */ updateUser(meta) { if (!this.initialised) return posthog.people.set(meta) } - captureException(err) { - if (!this.initialised) return - - this.captureEvent("Error", { error: err.message ? err.message : err }) - } - + /** + * Capture analytics events and send them to posthog. + * @param {String} event - event identifier + * @param {Object} props - properties for the event + */ captureEvent(eventName, props) { if (!this.initialised) return @@ -45,6 +52,10 @@ export default class PosthogClient { posthog.capture(eventName, props) } + /** + * Submit NPS feedback to posthog. + * @param {Object} values - NPS Values + */ npsFeedback(values) { if (!this.initialised) return @@ -58,6 +69,9 @@ export default class PosthogClient { posthog.capture(Events.NPS.SUBMITTED, prefixedFeedback) } + /** + * Reset posthog user back to initial state on logout. + */ logout() { if (!this.initialised) return diff --git a/packages/builder/src/analytics/SentryClient.js b/packages/builder/src/analytics/SentryClient.js index b871ea15d9..25e4c37690 100644 --- a/packages/builder/src/analytics/SentryClient.js +++ b/packages/builder/src/analytics/SentryClient.js @@ -13,12 +13,20 @@ export default class SentryClient { } } + /** + * Capture an exception and send it to sentry. + * @param {Error} err - JS error object + */ captureException(err) { if (!this.initalised) return Sentry.captureException(err) } + /** + * Identify user in sentry. + * @param {String} id - Unique user id + */ identify(id) { if (!this.initalised) return From 447f1b7d902d2a9c2379dda3997a61d10075b87a Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 21 Sep 2021 20:38:53 +0100 Subject: [PATCH 12/55] remove superfluous deps --- packages/server/package.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index fb2e522b87..8c967f8bf7 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -110,14 +110,12 @@ "to-json-schema": "0.2.5", "uuid": "3.3.2", "validate.js": "0.13.1", - "worker-farm": "1.7.0", "yargs": "13.2.4", "zlib": "1.0.5" }, "devDependencies": { "@babel/core": "^7.14.3", "@babel/preset-env": "^7.14.4", - "@budibase/standard-components": "^0.9.136", "@jest/test-sequencer": "^24.8.0", "@types/bull": "^3.15.1", "@types/jest": "^26.0.23", From b7ff1a2464391db56b00486b0f0be7a7a7540ee8 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 21 Sep 2021 20:39:56 +0100 Subject: [PATCH 13/55] turn analytics off for self hosted --- packages/server/src/api/controllers/analytics.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/analytics.js b/packages/server/src/api/controllers/analytics.js index d6e1a9ce5b..eb64bc87b9 100644 --- a/packages/server/src/api/controllers/analytics.js +++ b/packages/server/src/api/controllers/analytics.js @@ -2,6 +2,6 @@ const env = require("../../environment") exports.isEnabled = async function (ctx) { ctx.body = { - enabled: env.ENABLE_ANALYTICS === "true", + enabled: !env.SELF_HOSTED && env.ENABLE_ANALYTICS === "true", } } From 45981c62ef7ec0423d1cd4a06789cc5083c6cda0 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 21 Sep 2021 20:47:04 +0100 Subject: [PATCH 14/55] lint --- packages/builder/src/analytics/PosthogClient.js | 4 ++-- packages/builder/src/analytics/SentryClient.js | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/builder/src/analytics/PosthogClient.js b/packages/builder/src/analytics/PosthogClient.js index 4c45c10216..0a1fde42ea 100644 --- a/packages/builder/src/analytics/PosthogClient.js +++ b/packages/builder/src/analytics/PosthogClient.js @@ -21,7 +21,7 @@ export default class PosthogClient { } /** - * Set the posthog context to the current user + * Set the posthog context to the current user * @param {String} id - unique user id */ identify(id) { @@ -70,7 +70,7 @@ export default class PosthogClient { } /** - * Reset posthog user back to initial state on logout. + * Reset posthog user back to initial state on logout. */ logout() { if (!this.initialised) return diff --git a/packages/builder/src/analytics/SentryClient.js b/packages/builder/src/analytics/SentryClient.js index 25e4c37690..2a1f8732e3 100644 --- a/packages/builder/src/analytics/SentryClient.js +++ b/packages/builder/src/analytics/SentryClient.js @@ -13,20 +13,20 @@ export default class SentryClient { } } - /** - * Capture an exception and send it to sentry. - * @param {Error} err - JS error object - */ + /** + * Capture an exception and send it to sentry. + * @param {Error} err - JS error object + */ captureException(err) { if (!this.initalised) return Sentry.captureException(err) } - /** - * Identify user in sentry. - * @param {String} id - Unique user id - */ + /** + * Identify user in sentry. + * @param {String} id - Unique user id + */ identify(id) { if (!this.initalised) return From 1790d13bb59a0712d6fb46468dd624445b386758 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 22 Sep 2021 09:29:51 +0100 Subject: [PATCH 15/55] added user created at timestamp --- .../builder/portal/manage/users/_components/AddUserModal.svelte | 2 +- packages/worker/src/api/controllers/global/users.js | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/portal/manage/users/_components/AddUserModal.svelte b/packages/builder/src/pages/builder/portal/manage/users/_components/AddUserModal.svelte index be5d6db746..25a69af1c8 100644 --- a/packages/builder/src/pages/builder/portal/manage/users/_components/AddUserModal.svelte +++ b/packages/builder/src/pages/builder/portal/manage/users/_components/AddUserModal.svelte @@ -26,7 +26,7 @@ notifications.error(res.message) } else { notifications.success(res.message) - analytics.captureEvent(Events.USER.INVITE) + analytics.captureEvent(Events.USER.INVITE, { type: selected }) } } diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index 8f754e2922..ac92f3902c 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -63,6 +63,7 @@ async function saveUser(user, tenantId, hashPassword = true) { _id = _id || generateGlobalUserID() user = { + createdAt: Date.now(), ...dbUser, ...user, _id, @@ -138,6 +139,7 @@ exports.adminUser = async ctx => { const user = { email: email, password: password, + createdAt: Date.now(), roles: {}, builder: { global: true, From 9196ae8edb153207ab1e9d60b603a83e3ae37749 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 22 Sep 2021 09:38:32 +0100 Subject: [PATCH 16/55] account for user size in intercom --- packages/builder/src/stores/portal/auth.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/builder/src/stores/portal/auth.js b/packages/builder/src/stores/portal/auth.js index 22f7fedca4..cb4e14c4a9 100644 --- a/packages/builder/src/stores/portal/auth.js +++ b/packages/builder/src/stores/portal/auth.js @@ -54,13 +54,15 @@ export function createAuthStore() { if (user) { analytics.activate().then(() => { analytics.identify(user._id, user) - // TODO: check if the user is a certain org size - // analytics.showChat({ - // email: user.email, - // name: user.name, - // user_id: user._id, - // tenant: user.tenantId - // }) + if (user.size === "100+" || user.size === "10000+") { + analytics.showChat({ + email: user.email, + created_at: user.createdAt || Date.now(), + name: user.name, + user_id: user._id, + tenant: user.tenantId + }) + } }) } } From 398a82f1a724b9a91a1fb06c3752bcfbc6bc1800 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 22 Sep 2021 12:26:02 +0100 Subject: [PATCH 17/55] lint --- packages/builder/src/stores/portal/auth.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/stores/portal/auth.js b/packages/builder/src/stores/portal/auth.js index cb4e14c4a9..e33a1f22ac 100644 --- a/packages/builder/src/stores/portal/auth.js +++ b/packages/builder/src/stores/portal/auth.js @@ -60,7 +60,7 @@ export function createAuthStore() { created_at: user.createdAt || Date.now(), name: user.name, user_id: user._id, - tenant: user.tenantId + tenant: user.tenantId, }) } }) From 22e75b8154d4c15070663f379ec4f76fc78755df Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 22 Sep 2021 14:50:52 +0100 Subject: [PATCH 18/55] Only create default home screen and layouts when not importing an app --- packages/server/src/api/controllers/application.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index da0014c5f8..c3142e9733 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -230,7 +230,12 @@ exports.create = async function (ctx) { const response = await db.put(newApplication, { force: true }) newApplication._rev = response.rev - await createEmptyAppPackage(ctx, newApplication) + // Only create the default home screens and layout if we aren't importing + // an app + if (!useTemplate) { + await createEmptyAppPackage(ctx, newApplication) + } + /* istanbul ignore next */ if (!env.isTest()) { await createApp(appId) From 0c9203d539e1f542fab6f2671a2cf612c9e609c1 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 22 Sep 2021 15:03:22 +0000 Subject: [PATCH 19/55] v0.9.140-alpha.4 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index 850cb6aeaa..5bb17217e8 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.140-alpha.3", + "version": "0.9.140-alpha.4", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index 4ce04ce99f..1cc258bb2f 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.140-alpha.3", + "version": "0.9.140-alpha.4", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index b92d51e5b2..24581b203f 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.140-alpha.3", + "version": "0.9.140-alpha.4", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 19c8064e9f..e0f510578d 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.140-alpha.3", + "version": "0.9.140-alpha.4", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.3", - "@budibase/client": "^0.9.140-alpha.3", + "@budibase/bbui": "^0.9.140-alpha.4", + "@budibase/client": "^0.9.140-alpha.4", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.140-alpha.3", + "@budibase/string-templates": "^0.9.140-alpha.4", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 25229d44c4..7bfca5855d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.140-alpha.3", + "version": "0.9.140-alpha.4", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 9d2557b939..62ace6f4aa 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.140-alpha.3", + "version": "0.9.140-alpha.4", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.3", + "@budibase/bbui": "^0.9.140-alpha.4", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.140-alpha.3", + "@budibase/string-templates": "^0.9.140-alpha.4", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index bacadf10b5..420e7082a1 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.140-alpha.3", + "version": "0.9.140-alpha.4", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -62,9 +62,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.3", - "@budibase/client": "^0.9.140-alpha.3", - "@budibase/string-templates": "^0.9.140-alpha.3", + "@budibase/auth": "^0.9.140-alpha.4", + "@budibase/client": "^0.9.140-alpha.4", + "@budibase/string-templates": "^0.9.140-alpha.4", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 1f85a336c6..8dee560de9 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.140-alpha.3", + "version": "0.9.140-alpha.4", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 7d76c6419c..8baa4a321e 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.140-alpha.3", + "version": "0.9.140-alpha.4", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.3", - "@budibase/string-templates": "^0.9.140-alpha.3", + "@budibase/auth": "^0.9.140-alpha.4", + "@budibase/string-templates": "^0.9.140-alpha.4", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From fe3902a33c4f422385a7d9837e0a5854202fd5f6 Mon Sep 17 00:00:00 2001 From: Maurits Lourens Date: Wed, 22 Sep 2021 17:27:07 +0200 Subject: [PATCH 20/55] Check if the formula validates against its constraints --- packages/server/src/api/controllers/row/utils.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/server/src/api/controllers/row/utils.js b/packages/server/src/api/controllers/row/utils.js index cb9a5e166c..ca6c782713 100644 --- a/packages/server/src/api/controllers/row/utils.js +++ b/packages/server/src/api/controllers/row/utils.js @@ -5,6 +5,7 @@ const { InternalTables } = require("../../../db/utils") const userController = require("../user") const { FieldTypes } = require("../../../constants") const { integrations } = require("../../../integrations") +const { processStringSync } = require("@budibase/string-templates") validateJs.extend(validateJs.validators.datetime, { parse: function (value) { @@ -73,6 +74,11 @@ exports.validate = async ({ appId, tableId, row, table }) => { errors[fieldName] = "Field not in list" } }) + } else if (table.schema[fieldName].type === FieldTypes.FORMULA) { + res = validateJs.single( + processStringSync(table.schema[fieldName].formula, row), + constraints + ) } else { res = validateJs.single(row[fieldName], constraints) } From f70cee3c52ee581a6cf090b031f378ff01babb47 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 22 Sep 2021 17:46:54 +0100 Subject: [PATCH 21/55] Fixing issue #2673 - discovered issues with some column types not being worked out to be auto-columns in postgres. --- .../service-vehicles/docker-compose.yml | 28 ++++++++++ .../integrations/service-vehicles/init.sql | 52 +++++++++++++++++++ .../integrations/service-vehicles/reset.sh | 3 ++ .../api/controllers/row/ExternalRequest.ts | 6 ++- packages/server/src/integrations/mysql.ts | 15 +----- packages/server/src/integrations/postgres.ts | 25 ++++----- packages/server/src/integrations/utils.ts | 18 +++++++ 7 files changed, 117 insertions(+), 30 deletions(-) create mode 100644 packages/server/scripts/integrations/service-vehicles/docker-compose.yml create mode 100644 packages/server/scripts/integrations/service-vehicles/init.sql create mode 100755 packages/server/scripts/integrations/service-vehicles/reset.sh diff --git a/packages/server/scripts/integrations/service-vehicles/docker-compose.yml b/packages/server/scripts/integrations/service-vehicles/docker-compose.yml new file mode 100644 index 0000000000..7473e540db --- /dev/null +++ b/packages/server/scripts/integrations/service-vehicles/docker-compose.yml @@ -0,0 +1,28 @@ +version: "3.8" +services: + db: + container_name: postgres-vehicle + image: postgres + restart: always + environment: + POSTGRES_USER: root + POSTGRES_PASSWORD: root + POSTGRES_DB: main + ports: + - "5432:5432" + volumes: + #- pg_data:/var/lib/postgresql/data/ + - ./init.sql:/docker-entrypoint-initdb.d/init.sql + + pgadmin: + container_name: pgadmin + image: dpage/pgadmin4 + restart: always + environment: + PGADMIN_DEFAULT_EMAIL: root@root.com + PGADMIN_DEFAULT_PASSWORD: root + ports: + - "5050:80" + +#volumes: +# pg_data: diff --git a/packages/server/scripts/integrations/service-vehicles/init.sql b/packages/server/scripts/integrations/service-vehicles/init.sql new file mode 100644 index 0000000000..3e0485313e --- /dev/null +++ b/packages/server/scripts/integrations/service-vehicles/init.sql @@ -0,0 +1,52 @@ +SELECT 'CREATE DATABASE main' +WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec +CREATE TABLE Vehicles ( + id bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ), + Registration text COLLATE pg_catalog."default", + Make text COLLATE pg_catalog."default", + Model text COLLATE pg_catalog."default", + Colour text COLLATE pg_catalog."default", + Year smallint, + CONSTRAINT Vehicles_pkey PRIMARY KEY (id) +); + +CREATE TABLE ServiceLog ( + id bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ), + Description text COLLATE pg_catalog."default", + VehicleId bigint, + ServiceDate timestamp without time zone, + Category text COLLATE pg_catalog."default", + Mileage bigint, + CONSTRAINT ServiceLog_pkey PRIMARY KEY (id), + CONSTRAINT vehicle_foreign_key FOREIGN KEY (VehicleId) + REFERENCES Vehicles (id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE NO ACTION +); + +INSERT INTO Vehicles (Registration, Make, Model, Colour, Year) +VALUES ('FAZ 9837','Volkswagen','Polo','White',2002); +INSERT INTO Vehicles (Registration, Make, Model, Colour, Year) +VALUES ('JHI 8827','BMW','M3','Black',2013); +INSERT INTO Vehicles (Registration, Make, Model, Colour, Year) +VALUES ('D903PI','Volvo','XC40','Grey',2014); +INSERT INTO Vehicles (Registration, Make, Model, Colour, Year) +VALUES ('YFI002','Volkswagen','Golf','Dark Blue',2018); +INSERT INTO Vehicles (Registration, Make, Model, Colour, Year) +VALUES ('HGT5677','Skoda','Octavia','Graphite',2009); +INSERT INTO Vehicles (Registration, Make, Model, Colour, Year) +VALUES ('PPF9276','Skoda','Octavia','Graphite',2021); +INSERT INTO Vehicles (Registration, Make, Model, Colour, Year) +VALUES ('J893FT','Toyota','Corolla','Red',2015); +INSERT INTO Vehicles (Registration, Make, Model, Colour, Year) +VALUES ('MJK776','Honda','HR-V','Silver',2015); + + +INSERT INTO ServiceLog (Description, VehicleId, ServiceDate, Category, Mileage) +VALUES ('Change front brakes', 1, '2021-05-04', 'Brakes', 20667); +INSERT INTO ServiceLog (Description, VehicleId, ServiceDate, Category, Mileage) +VALUES ('Tyres - full set', 1, '2021-05-04', 'Brakes', 20667); +INSERT INTO ServiceLog (Description, VehicleId, ServiceDate, Category, Mileage) +VALUES ('Engine tune up', 2, '2021-07-14', 'Brakes', 50889); +INSERT INTO ServiceLog (Description, VehicleId, ServiceDate, Category, Mileage) +VALUES ('Replace transmission', 3, '2021-09-26', 'Transmission', 98002); diff --git a/packages/server/scripts/integrations/service-vehicles/reset.sh b/packages/server/scripts/integrations/service-vehicles/reset.sh new file mode 100755 index 0000000000..32778bd11f --- /dev/null +++ b/packages/server/scripts/integrations/service-vehicles/reset.sh @@ -0,0 +1,3 @@ +#!/bin/bash +docker-compose down +docker volume prune -f diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index eced518604..b809e597e4 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -437,7 +437,11 @@ module External { for (let [colName, { isMany, rows, tableId }] of Object.entries( related )) { - const table = this.getTable(tableId) + const table: Table = this.getTable(tableId) + // if its not the foreign key skip it, nothing to do + if (table.primary && table.primary.indexOf(colName) !== -1) { + continue + } for (let row of rows) { const filters = buildFilters(generateIdForRow(row, table), {}, table) // safety check, if there are no filters on deletion bad things happen diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 3ce21675d9..c5db35ed2a 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -12,7 +12,7 @@ import { getSqlQuery } from "./utils" module MySQLModule { const mysql = require("mysql") const Sql = require("./base/sql") - const { buildExternalTableId, convertType } = require("./utils") + const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") const { FieldTypes } = require("../constants") interface MySQLConfig { @@ -194,18 +194,7 @@ module MySQLModule { } } - // add the existing relationships from the entities if they exist, to prevent them from being overridden - if (entities && entities[tableName]) { - const existingTableSchema = entities[tableName].schema - for (let key in existingTableSchema) { - if (!existingTableSchema.hasOwnProperty(key)) { - continue - } - if (existingTableSchema[key].type === "link") { - tables[tableName].schema[key] = existingTableSchema[key] - } - } - } + copyExistingPropsOver(tableName, tables, entities) } this.client.end() diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index dd46652871..63719980fb 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -12,7 +12,7 @@ module PostgresModule { const { Pool } = require("pg") const Sql = require("./base/sql") const { FieldTypes } = require("../constants") - const { buildExternalTableId, convertType } = require("./utils") + const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils") interface PostgresConfig { host: string @@ -173,31 +173,24 @@ module PostgresModule { name: tableName, schema: {}, } - - // add the existing relationships from the entities if they exist, to prevent them from being overridden - if (entities && entities[tableName]) { - const existingTableSchema = entities[tableName].schema - for (let key in existingTableSchema) { - if (!existingTableSchema.hasOwnProperty(key)) { - continue - } - if (existingTableSchema[key].type === "link") { - tables[tableName].schema[key] = existingTableSchema[key] - } - } - } } const type: string = convertType(column.data_type, TYPE_MAP) - const isAuto: boolean = - typeof column.column_default === "string" && + const identity = !!(column.identity_generation || column.identity_start || column.identity_increment) + const hasDefault = typeof column.column_default === "string" && column.column_default.startsWith("nextval") + const isGenerated = column.is_generated && column.is_generated !== "NEVER" + const isAuto: boolean = hasDefault || identity || isGenerated tables[tableName].schema[columnName] = { autocolumn: isAuto, name: columnName, type, } } + + for (let tableName of Object.keys(tables)) { + copyExistingPropsOver(tableName, tables, entities) + } this.tables = tables } diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 5b247213c0..82c35bc2d9 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -82,3 +82,21 @@ export function isIsoDateString(str: string) { let d = new Date(str) return d.toISOString() === str } + +// add the existing relationships from the entities if they exist, to prevent them from being overridden +export function copyExistingPropsOver(tableName: string, tables: { [key: string]: any }, entities: { [key: string]: any }) { + if (entities && entities[tableName]) { + if (entities[tableName].primaryDisplay) { + tables[tableName].primaryDisplay = entities[tableName].primaryDisplay + } + const existingTableSchema = entities[tableName].schema + for (let key in existingTableSchema) { + if (!existingTableSchema.hasOwnProperty(key)) { + continue + } + if (existingTableSchema[key].type === "link") { + tables[tableName].schema[key] = existingTableSchema[key] + } + } + } +} From 15cd5e45144b43db3af75d58bbe695dc497b7cbd Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 22 Sep 2021 17:31:55 +0000 Subject: [PATCH 22/55] v0.9.140-alpha.5 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 10 +++++----- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 20 insertions(+), 20 deletions(-) diff --git a/lerna.json b/lerna.json index 5bb17217e8..e2e4d027b5 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.140-alpha.4", + "version": "0.9.140-alpha.5", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index 1cc258bb2f..8c1f38d64c 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.140-alpha.4", + "version": "0.9.140-alpha.5", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 24581b203f..9ddedfd28f 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.140-alpha.4", + "version": "0.9.140-alpha.5", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index e0f510578d..01dfc08f2c 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.140-alpha.4", + "version": "0.9.140-alpha.5", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.4", - "@budibase/client": "^0.9.140-alpha.4", + "@budibase/bbui": "^0.9.140-alpha.5", + "@budibase/client": "^0.9.140-alpha.5", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.140-alpha.4", + "@budibase/string-templates": "^0.9.140-alpha.5", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 7bfca5855d..147a0ffe2b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.140-alpha.4", + "version": "0.9.140-alpha.5", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 62ace6f4aa..c739c5b80d 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.140-alpha.4", + "version": "0.9.140-alpha.5", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.4", + "@budibase/bbui": "^0.9.140-alpha.5", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.140-alpha.4", + "@budibase/string-templates": "^0.9.140-alpha.5", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 671c062fcd..4d32021ff4 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.140-alpha.4", + "version": "0.9.140-alpha.5", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -62,9 +62,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.4", - "@budibase/client": "^0.9.140-alpha.4", - "@budibase/string-templates": "^0.9.140-alpha.4", + "@budibase/auth": "^0.9.140-alpha.5", + "@budibase/client": "^0.9.140-alpha.5", + "@budibase/string-templates": "^0.9.140-alpha.5", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", @@ -102,10 +102,10 @@ "pg": "8.5.1", "pino-pretty": "4.0.0", "pouchdb": "7.2.1", + "pouchdb-adapter-memory": "^7.2.1", "pouchdb-all-dbs": "1.0.2", "pouchdb-find": "^7.2.2", "pouchdb-replication-stream": "1.2.9", - "pouchdb-adapter-memory": "^7.2.1", "server-destroy": "1.0.1", "svelte": "^3.38.2", "to-json-schema": "0.2.5", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 8dee560de9..f78a7cdf72 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.140-alpha.4", + "version": "0.9.140-alpha.5", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 8baa4a321e..689e7d8569 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.140-alpha.4", + "version": "0.9.140-alpha.5", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.4", - "@budibase/string-templates": "^0.9.140-alpha.4", + "@budibase/auth": "^0.9.140-alpha.5", + "@budibase/string-templates": "^0.9.140-alpha.5", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From 9eb5b46bc89790732f689013002fb415f24725d7 Mon Sep 17 00:00:00 2001 From: Maurits Lourens Date: Thu, 23 Sep 2021 08:48:11 +0200 Subject: [PATCH 23/55] Collapse queries with their datasource --- .../DatasourceNavigator.svelte | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte index 84c737eb67..19713595ce 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte @@ -63,20 +63,19 @@ {#if openDataSources.includes(datasource._id)} + {#each $queries.list.filter(query => query.datasourceId === datasource._id) as query} + onClickQuery(query)} + > + + + {/each} {/if} - - {#each $queries.list.filter(query => query.datasourceId === datasource._id) as query} - onClickQuery(query)} - > - - - {/each} {/each}
{/if} From 2b2a78e1554e792730ad56f133424e4016b6f53d Mon Sep 17 00:00:00 2001 From: Maurits Lourens Date: Thu, 23 Sep 2021 09:54:58 +0200 Subject: [PATCH 24/55] Extend Postgres data type map --- packages/server/src/integrations/postgres.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index dd46652871..961dbec4e2 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -84,10 +84,13 @@ module PostgresModule { bigint: FieldTypes.NUMBER, decimal: FieldTypes.NUMBER, smallint: FieldTypes.NUMBER, + real: FieldTypes.NUMBER, + "double precision": FieldTypes.NUMBER, timestamp: FieldTypes.DATETIME, time: FieldTypes.DATETIME, boolean: FieldTypes.BOOLEAN, json: FieldTypes.JSON, + date: FieldTypes.DATETIME, } async function internalQuery(client: any, query: SqlQuery) { From aab5a1f1de1bc87b7a3b42fe8f3a186b0c96d25b Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 23 Sep 2021 08:49:16 +0000 Subject: [PATCH 25/55] v0.9.140-alpha.6 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index e2e4d027b5..a5f9e0ded9 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.140-alpha.5", + "version": "0.9.140-alpha.6", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index 8c1f38d64c..012051d267 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.140-alpha.5", + "version": "0.9.140-alpha.6", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 9ddedfd28f..f781cc7c39 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.140-alpha.5", + "version": "0.9.140-alpha.6", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 01dfc08f2c..52756145a2 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.140-alpha.5", + "version": "0.9.140-alpha.6", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.5", - "@budibase/client": "^0.9.140-alpha.5", + "@budibase/bbui": "^0.9.140-alpha.6", + "@budibase/client": "^0.9.140-alpha.6", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.140-alpha.5", + "@budibase/string-templates": "^0.9.140-alpha.6", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 147a0ffe2b..cfe403edc9 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.140-alpha.5", + "version": "0.9.140-alpha.6", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index c739c5b80d..a0df02e392 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.140-alpha.5", + "version": "0.9.140-alpha.6", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.5", + "@budibase/bbui": "^0.9.140-alpha.6", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.140-alpha.5", + "@budibase/string-templates": "^0.9.140-alpha.6", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 4d32021ff4..653300aa8f 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.140-alpha.5", + "version": "0.9.140-alpha.6", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -62,9 +62,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.5", - "@budibase/client": "^0.9.140-alpha.5", - "@budibase/string-templates": "^0.9.140-alpha.5", + "@budibase/auth": "^0.9.140-alpha.6", + "@budibase/client": "^0.9.140-alpha.6", + "@budibase/string-templates": "^0.9.140-alpha.6", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index f78a7cdf72..a244fab9ba 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.140-alpha.5", + "version": "0.9.140-alpha.6", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 689e7d8569..01fd3f0f8e 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.140-alpha.5", + "version": "0.9.140-alpha.6", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.5", - "@budibase/string-templates": "^0.9.140-alpha.5", + "@budibase/auth": "^0.9.140-alpha.6", + "@budibase/string-templates": "^0.9.140-alpha.6", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From 5dce298b44ac596c58db2c25286a167209b76972 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 23 Sep 2021 11:33:58 +0000 Subject: [PATCH 26/55] v0.9.140-alpha.7 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index a5f9e0ded9..34b61f435d 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.140-alpha.6", + "version": "0.9.140-alpha.7", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index 012051d267..50cdd91a27 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.140-alpha.6", + "version": "0.9.140-alpha.7", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index f781cc7c39..8236cfb83a 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.140-alpha.6", + "version": "0.9.140-alpha.7", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 52756145a2..8d4643b2c3 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.140-alpha.6", + "version": "0.9.140-alpha.7", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.6", - "@budibase/client": "^0.9.140-alpha.6", + "@budibase/bbui": "^0.9.140-alpha.7", + "@budibase/client": "^0.9.140-alpha.7", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.140-alpha.6", + "@budibase/string-templates": "^0.9.140-alpha.7", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index cfe403edc9..343bccf430 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.140-alpha.6", + "version": "0.9.140-alpha.7", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index a0df02e392..147cfc78c4 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.140-alpha.6", + "version": "0.9.140-alpha.7", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.6", + "@budibase/bbui": "^0.9.140-alpha.7", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.140-alpha.6", + "@budibase/string-templates": "^0.9.140-alpha.7", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 653300aa8f..38cf1184d9 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.140-alpha.6", + "version": "0.9.140-alpha.7", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -62,9 +62,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.6", - "@budibase/client": "^0.9.140-alpha.6", - "@budibase/string-templates": "^0.9.140-alpha.6", + "@budibase/auth": "^0.9.140-alpha.7", + "@budibase/client": "^0.9.140-alpha.7", + "@budibase/string-templates": "^0.9.140-alpha.7", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index a244fab9ba..7bb0e92378 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.140-alpha.6", + "version": "0.9.140-alpha.7", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 01fd3f0f8e..35e05cbef2 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.140-alpha.6", + "version": "0.9.140-alpha.7", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.6", - "@budibase/string-templates": "^0.9.140-alpha.6", + "@budibase/auth": "^0.9.140-alpha.7", + "@budibase/string-templates": "^0.9.140-alpha.7", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From 1952dc308eff355e249d1bff123e056cc602cc8e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 Sep 2021 16:17:23 +0100 Subject: [PATCH 27/55] Fixes issue #2616 - this is a slightly complex fix and handles a few other issues with mysql (around returning on creation of a row and relationships) - a new mechanism is now used for pagination and limiting which makes sure the limits are applied to the outer table rather than the combination of the outer and the joined. --- .../integrations/postgres/docker-compose.yml | 2 +- .../api/controllers/row/ExternalRequest.ts | 3 ++ packages/server/src/definitions/datasource.ts | 5 ++ packages/server/src/integrations/base/sql.ts | 52 +++++++++---------- packages/server/src/integrations/mysql.ts | 21 +++++++- 5 files changed, 54 insertions(+), 29 deletions(-) diff --git a/packages/server/scripts/integrations/postgres/docker-compose.yml b/packages/server/scripts/integrations/postgres/docker-compose.yml index e2bba9f38e..4dfcb0e1ad 100644 --- a/packages/server/scripts/integrations/postgres/docker-compose.yml +++ b/packages/server/scripts/integrations/postgres/docker-compose.yml @@ -15,7 +15,7 @@ services: - ./init.sql:/docker-entrypoint-initdb.d/init.sql pgadmin: - container_name: pgadmin + container_name: pgadmin-pg image: dpage/pgadmin4 restart: always environment: diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index b809e597e4..12db55efdc 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -544,6 +544,9 @@ module External { extra: { idFilter: buildFilters(id || generateIdForRow(row, table), {}, table), }, + meta: { + table, + } } // can't really use response right now const response = await makeExternalQuery(appId, json) diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index 48fd24e1cf..d7d4e77961 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -1,3 +1,5 @@ +import {Table} from "./common"; + export enum Operation { CREATE = "CREATE", READ = "READ", @@ -136,6 +138,9 @@ export interface QueryJson { sort?: SortJson paginate?: PaginationJson body?: object + meta?: { + table?: Table, + } extra?: { idFilter?: SearchFilters } diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index b59bac5a5a..91af3e1a85 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -1,7 +1,5 @@ import { Knex, knex } from "knex" const BASE_LIMIT = 5000 -// if requesting a single row then need to up the limit for the sake of joins -const SINGLE_ROW_LIMIT = 100 import { QueryJson, SearchFilters, @@ -146,46 +144,48 @@ function buildCreate( function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery { let { endpoint, resource, filters, sort, paginate, relationships } = json const tableName = endpoint.entityId - let query: KnexQuery = knex(tableName) // select all if not specified if (!resource) { resource = { fields: [] } } + let selectStatement: string|string[] = "*" // handle select if (resource.fields && resource.fields.length > 0) { // select the resources as the format "table.columnName" - this is what is provided // by the resource builder further up - query = query.select(resource.fields.map(field => `${field} as ${field}`)) - } else { - query = query.select("*") + selectStatement = resource.fields.map(field => `${field} as ${field}`) + } + let foundLimit = limit || BASE_LIMIT + // handle pagination + let foundOffset: number | null = null + if (paginate && paginate.page && paginate.limit) { + // @ts-ignore + const page = paginate.page <= 1 ? 0 : paginate.page - 1 + const offset = page * paginate.limit + foundLimit = paginate.limit + foundOffset = offset + } else if (paginate && paginate.limit) { + foundLimit = paginate.limit + } + // start building the query + let query: KnexQuery = knex(tableName).limit(foundLimit) + if (foundOffset) { + query = query.offset(foundOffset) } - // handle where - query = addFilters(tableName, query, filters) - // handle join - query = addRelationships(query, tableName, relationships) - // handle sorting if (sort) { for (let [key, value] of Object.entries(sort)) { const direction = value === SortDirection.ASCENDING ? "asc" : "desc" query = query.orderBy(key, direction) } } - let foundLimit = limit || BASE_LIMIT - // handle pagination - if (paginate && paginate.page && paginate.limit) { + query = addFilters(tableName, query, filters) + // @ts-ignore + let preQuery: KnexQuery = knex({ // @ts-ignore - const page = paginate.page <= 1 ? 0 : paginate.page - 1 - const offset = page * paginate.limit - foundLimit = paginate.limit - query = query.offset(offset) - } else if (paginate && paginate.limit) { - foundLimit = paginate.limit - } - if (foundLimit === 1) { - foundLimit = SINGLE_ROW_LIMIT - } - query = query.limit(foundLimit) - return query + [tableName]: query, + }).select(selectStatement) + // handle joins + return addRelationships(preQuery, tableName, relationships) } function buildUpdate( diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index c5db35ed2a..11220afb46 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -104,7 +104,7 @@ module MySQLModule { client: any, query: SqlQuery, connect: boolean = true - ): Promise { + ): Promise { // Node MySQL is callback based, so we must wrap our call in a promise return new Promise((resolve, reject) => { if (connect) { @@ -238,6 +238,23 @@ module MySQLModule { return internalQuery(this.client, input, false) } + // when creating if an ID has been inserted need to make sure + // the id filter is enriched with it before trying to retrieve the row + checkLookupKeys(results: any, json: QueryJson) { + if (!results?.insertId || !json.meta?.table || !json.meta.table.primary) { + return json + } + const primaryKey = json.meta.table.primary?.[0] + json.extra = { + idFilter: { + equal: { + [primaryKey]: results.insertId + }, + } + } + return json + } + async query(json: QueryJson) { const operation = this._operation(json) this.client.connect() @@ -250,7 +267,7 @@ module MySQLModule { const results = await internalQuery(this.client, input, false) // same as delete, manage returning if (operation === Operation.CREATE || operation === Operation.UPDATE) { - row = this.getReturningRow(json) + row = this.getReturningRow(this.checkLookupKeys(results, json)) } this.client.end() if (operation !== Operation.READ) { From 7c7266a54776fadd804c3fdf8541684ed716b878 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 Sep 2021 16:56:13 +0100 Subject: [PATCH 28/55] Fixing SQL test cases. --- packages/server/src/integrations/tests/sql.spec.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/server/src/integrations/tests/sql.spec.js b/packages/server/src/integrations/tests/sql.spec.js index fa8bcd1d86..64cdda215f 100644 --- a/packages/server/src/integrations/tests/sql.spec.js +++ b/packages/server/src/integrations/tests/sql.spec.js @@ -57,7 +57,7 @@ describe("SQL query builder", () => { const query = sql._query(generateReadJson()) expect(query).toEqual({ bindings: [limit], - sql: `select * from "${TABLE_NAME}" limit $1` + sql: `select * from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"` }) }) @@ -68,7 +68,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [limit], - sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from "${TABLE_NAME}" limit $1` + sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"` }) }) @@ -82,7 +82,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: ["John%", limit], - sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2` + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2) as "${TABLE_NAME}"` }) }) @@ -99,7 +99,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [2, 10, limit], - sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3` + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3) as "${TABLE_NAME}"` }) }) @@ -115,7 +115,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [10, "John", limit], - sql: `select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3` + sql: `select * from (select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3) as "${TABLE_NAME}"` }) }) @@ -160,7 +160,7 @@ describe("SQL query builder", () => { const query = new Sql("mssql", 10)._query(generateReadJson()) expect(query).toEqual({ bindings: [10], - sql: `select top (@p0) * from [${TABLE_NAME}]` + sql: `select * from (select top (@p0) * from [${TABLE_NAME}]) as [${TABLE_NAME}]` }) }) @@ -168,7 +168,7 @@ describe("SQL query builder", () => { const query = new Sql("mysql", 10)._query(generateReadJson()) expect(query).toEqual({ bindings: [10], - sql: `select * from \`${TABLE_NAME}\` limit ?` + sql: `select * from (select * from \`${TABLE_NAME}\` limit ?) as \`${TABLE_NAME}\`` }) }) }) From 9cbf830a92172bbfe88c6968223829d627cf9b71 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 23 Sep 2021 17:03:03 +0100 Subject: [PATCH 29/55] fixing path for client manifest --- packages/server/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/package.json b/packages/server/package.json index 38cf1184d9..7a29669433 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -13,7 +13,7 @@ "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/", "test": "jest --coverage --maxWorkers=2", "test:watch": "jest --watch", - "predocker": "copyfiles -f ../client/dist/budibase-client.js ../standard-components/manifest.json client", + "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client", "build:docker": "yarn run predocker && docker build . -t app-service", "run:docker": "node dist/index.js", "dev:stack:up": "node scripts/dev/manage.js up", From 2fa6f2aa54185ccf8a58f0e43abb50dc2ca586b6 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 23 Sep 2021 16:11:26 +0000 Subject: [PATCH 30/55] v0.9.140-alpha.8 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index 34b61f435d..c3ee9351fe 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.140-alpha.7", + "version": "0.9.140-alpha.8", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index 50cdd91a27..22f96811e8 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.140-alpha.7", + "version": "0.9.140-alpha.8", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 8236cfb83a..82ee4b7d31 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.140-alpha.7", + "version": "0.9.140-alpha.8", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 8d4643b2c3..c4d57c9709 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.140-alpha.7", + "version": "0.9.140-alpha.8", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.7", - "@budibase/client": "^0.9.140-alpha.7", + "@budibase/bbui": "^0.9.140-alpha.8", + "@budibase/client": "^0.9.140-alpha.8", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.140-alpha.7", + "@budibase/string-templates": "^0.9.140-alpha.8", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 343bccf430..459df417c3 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.140-alpha.7", + "version": "0.9.140-alpha.8", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 147cfc78c4..cc585ebac5 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.140-alpha.7", + "version": "0.9.140-alpha.8", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.7", + "@budibase/bbui": "^0.9.140-alpha.8", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.140-alpha.7", + "@budibase/string-templates": "^0.9.140-alpha.8", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 7a29669433..e0e7a8c8f8 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.140-alpha.7", + "version": "0.9.140-alpha.8", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -62,9 +62,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.7", - "@budibase/client": "^0.9.140-alpha.7", - "@budibase/string-templates": "^0.9.140-alpha.7", + "@budibase/auth": "^0.9.140-alpha.8", + "@budibase/client": "^0.9.140-alpha.8", + "@budibase/string-templates": "^0.9.140-alpha.8", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 7bb0e92378..d57515dbcc 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.140-alpha.7", + "version": "0.9.140-alpha.8", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 35e05cbef2..81b1129918 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.140-alpha.7", + "version": "0.9.140-alpha.8", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.7", - "@budibase/string-templates": "^0.9.140-alpha.7", + "@budibase/auth": "^0.9.140-alpha.8", + "@budibase/string-templates": "^0.9.140-alpha.8", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From a8641056492f1c99bf15eae3a9b42814dc291ed2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 Sep 2021 17:43:06 +0100 Subject: [PATCH 31/55] Fixing postgres datasource test. --- packages/server/src/api/routes/tests/datasource.spec.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/datasource.spec.js b/packages/server/src/api/routes/tests/datasource.spec.js index 98a99717fd..b6d94f714d 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.js +++ b/packages/server/src/api/routes/tests/datasource.spec.js @@ -94,7 +94,8 @@ describe("/datasources", () => { .expect(200) // this is mock data, can't test it expect(res.body).toBeDefined() - expect(pg.queryMock).toHaveBeenCalledWith(`select "users"."name" as "users.name", "users"."age" as "users.age" from "users" where "users"."name" ilike $1 limit $2`, ["John%", 5000]) + const expSql = `select "users"."name" as "users.name", "users"."age" as "users.age" from (select * from "users" where "users"."name" ilike $1 limit $2) as "users"` + expect(pg.queryMock).toHaveBeenCalledWith(expSql, ["John%", 5000]) }) }) From 649a33d3c8f800233b45934467cd1db020fcab2e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 Sep 2021 19:04:53 +0100 Subject: [PATCH 32/55] Getting rid of the concept of permissions hierarchy, roles still have a hierarchy and base permissions still follow the old system, but resources can be given a stack of separate permissions which don't override each other. --- packages/auth/src/security/permissions.js | 7 +- packages/auth/src/security/roles.js | 15 +++- .../server/src/api/controllers/permission.js | 68 ++++++++----------- .../server/src/api/routes/tests/role.spec.js | 2 +- packages/server/src/utilities/index.js | 8 +++ 5 files changed, 50 insertions(+), 50 deletions(-) diff --git a/packages/auth/src/security/permissions.js b/packages/auth/src/security/permissions.js index 03fa5fa562..d0308d783e 100644 --- a/packages/auth/src/security/permissions.js +++ b/packages/auth/src/security/permissions.js @@ -139,8 +139,7 @@ exports.doesHaveResourcePermission = ( // set foundSub to not subResourceId, incase there is no subResource let foundMain = false, foundSub = false - for (let [resource, level] of Object.entries(permissions)) { - const levels = getAllowedLevels(level) + for (let [resource, levels] of Object.entries(permissions)) { if (resource === resourceId && levels.indexOf(permLevel) !== -1) { foundMain = true } @@ -177,10 +176,6 @@ exports.doesHaveBasePermission = (permType, permLevel, permissionIds) => { return false } -exports.higherPermission = (perm1, perm2) => { - return levelToNumber(perm1) > levelToNumber(perm2) ? perm1 : perm2 -} - exports.isPermissionLevelHigherThanRead = level => { return levelToNumber(level) > 1 } diff --git a/packages/auth/src/security/roles.js b/packages/auth/src/security/roles.js index baa8fc40dc..71fbc10132 100644 --- a/packages/auth/src/security/roles.js +++ b/packages/auth/src/security/roles.js @@ -1,6 +1,6 @@ const { getDB } = require("../db") const { cloneDeep } = require("lodash/fp") -const { BUILTIN_PERMISSION_IDS, higherPermission } = require("./permissions") +const { BUILTIN_PERMISSION_IDS } = require("./permissions") const { generateRoleID, getRoleParams, @@ -193,8 +193,17 @@ exports.getUserPermissions = async (appId, userRoleId) => { const permissions = {} for (let role of rolesHierarchy) { if (role.permissions) { - for (let [resource, level] of Object.entries(role.permissions)) { - permissions[resource] = higherPermission(permissions[resource], level) + for (let [resource, levels] of Object.entries(role.permissions)) { + if (!permissions[resource]) { + permissions[resource] = [] + } + const permsSet = new Set(permissions[resource]) + if (Array.isArray(levels)) { + levels.forEach(level => permsSet.add(level)) + } else { + permsSet.add(levels) + } + permissions[resource] = [...permsSet] } } } diff --git a/packages/server/src/api/controllers/permission.js b/packages/server/src/api/controllers/permission.js index e269f8c41d..6c02663649 100644 --- a/packages/server/src/api/controllers/permission.js +++ b/packages/server/src/api/controllers/permission.js @@ -1,9 +1,4 @@ -const { - getBuiltinPermissions, - PermissionLevels, - isPermissionLevelHigherThanRead, - higherPermission, -} = require("@budibase/auth/permissions") +const { getBuiltinPermissions } = require("@budibase/auth/permissions") const { isBuiltin, getDBRoleID, @@ -16,6 +11,7 @@ const { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, } = require("../../utilities/security") +const { removeFromArray } = require("../../utilities") const PermissionUpdateType = { REMOVE: "remove", @@ -24,22 +20,6 @@ const PermissionUpdateType = { const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS -// quick function to perform a bit of weird logic, make sure fetch calls -// always say a write role also has read permission -function fetchLevelPerms(permissions, level, roleId) { - if (!permissions) { - permissions = {} - } - permissions[level] = roleId - if ( - isPermissionLevelHigherThanRead(level) && - !permissions[PermissionLevels.READ] - ) { - permissions[PermissionLevels.READ] = roleId - } - return permissions -} - // utility function to stop this repetition - permissions always stored under roles async function getAllDBRoles(db) { const body = await db.allDocs( @@ -74,23 +54,31 @@ async function updatePermissionOnRole( for (let role of dbRoles) { let updated = false const rolePermissions = role.permissions ? role.permissions : {} + // make sure its an array, also handle migrating + if ( + !rolePermissions[resourceId] || + !Array.isArray(rolePermissions[resourceId]) + ) { + rolePermissions[resourceId] = + typeof rolePermissions[resourceId] === "string" + ? [rolePermissions[resourceId]] + : [] + } // handle the removal/updating the role which has this permission first // the updating (role._id !== dbRoleId) is required because a resource/level can // only be permitted in a single role (this reduces hierarchy confusion and simplifies // the general UI for this, rather than needing to show everywhere it is used) if ( (role._id !== dbRoleId || remove) && - rolePermissions[resourceId] === level + rolePermissions[resourceId].indexOf(level) !== -1 ) { - delete rolePermissions[resourceId] + removeFromArray(rolePermissions[resourceId], level) updated = true } // handle the adding, we're on the correct role, at it to this if (!remove && role._id === dbRoleId) { - rolePermissions[resourceId] = higherPermission( - rolePermissions[resourceId], - level - ) + const set = new Set(rolePermissions[resourceId]) + rolePermissions[resourceId] = [...set.add(level)] updated = true } // handle the update, add it to bulk docs to perform at end @@ -127,12 +115,11 @@ exports.fetch = async function (ctx) { continue } const roleId = getExternalRoleID(role._id) - for (let [resource, level] of Object.entries(role.permissions)) { - permissions[resource] = fetchLevelPerms( - permissions[resource], - level, - roleId - ) + for (let [resource, levelArr] of Object.entries(role.permissions)) { + const levels = Array.isArray(levelArr) ? [levelArr] : levelArr + const perms = {} + levels.forEach(level => (perms[level] = roleId)) + permissions[resource] = perms } } // apply the base permissions @@ -157,12 +144,13 @@ exports.getResourcePerms = async function (ctx) { for (let level of SUPPORTED_LEVELS) { // update the various roleIds in the resource permissions for (let role of roles) { - if (role.permissions && role.permissions[resourceId] === level) { - permissions = fetchLevelPerms( - permissions, - level, - getExternalRoleID(role._id) - ) + const rolePerms = role.permissions + if ( + rolePerms && + (rolePerms[resourceId] === level || + rolePerms[resourceId].indexOf(level) !== -1) + ) { + permissions[level] = getExternalRoleID(role._id) } } } diff --git a/packages/server/src/api/routes/tests/role.spec.js b/packages/server/src/api/routes/tests/role.spec.js index ad42ef180a..d74a84b2b2 100644 --- a/packages/server/src/api/routes/tests/role.spec.js +++ b/packages/server/src/api/routes/tests/role.spec.js @@ -72,7 +72,7 @@ describe("/roles", () => { .expect(200) expect(res.body.length).toBeGreaterThan(0) const power = res.body.find(role => role._id === BUILTIN_ROLE_IDS.POWER) - expect(power.permissions[table._id]).toEqual("read") + expect(power.permissions[table._id]).toEqual(["read"]) }) }) diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index a81f9ddcf5..e24d04c581 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -10,6 +10,14 @@ exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms)) exports.isDev = env.isDev +exports.removeFromArray = (array, element) => { + const index = array.indexOf(element) + if (index !== -1) { + array.splice(index, 1) + } + return array +} + /** * Makes sure that a URL has the correct number of slashes, while maintaining the * http(s):// double slashes. From 78e4f00e11c12ea7705ae9ef00528aa1648fb95f Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 23 Sep 2021 19:34:01 +0100 Subject: [PATCH 33/55] Add button action for manually refreshing a data provider --- .../actions/RefreshDataProvider.svelte | 35 +++++++++++++++++++ .../EventsEditor/actions/index.js | 5 +++ packages/client/manifest.json | 1 + packages/client/src/utils/buttonActions.js | 4 +-- 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/RefreshDataProvider.svelte diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/RefreshDataProvider.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/RefreshDataProvider.svelte new file mode 100644 index 0000000000..4b445e7b21 --- /dev/null +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/RefreshDataProvider.svelte @@ -0,0 +1,35 @@ + + +
+ +