From 295c3ef1a3418f8e87cfbf33a159efbf710afe67 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 16 Jun 2021 18:38:00 +0100 Subject: [PATCH 1/2] Fixing automation integration and various components around forms. --- .../src/api/controllers/row/external.js | 48 ++++++++++++++----- .../server/src/automations/automationUtils.js | 10 +++- packages/server/src/integrations/postgres.js | 1 - packages/server/src/integrations/utils.js | 16 +++++++ 4 files changed, 60 insertions(+), 15 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index f567a90678..3dd638d326 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -1,7 +1,11 @@ const { makeExternalQuery } = require("./utils") const { DataSourceOperation, SortDirection } = require("../../../constants") const { getExternalTable } = require("../table/utils") -const { breakExternalTableId } = require("../../../integrations/utils") +const { + breakExternalTableId, + generateRowIdField, + breakRowIdField +} = require("../../../integrations/utils") function inputProcessing(row, table) { if (!row) { @@ -29,20 +33,35 @@ function outputProcessing(rows, table) { for (let field of primary) { idParts.push(row[field]) } - row._id = idParts + row._id = generateRowIdField(idParts) + row.tableId = table._id } return rows } -function buildIDFilter(id, table) { +function buildFilters(id, filters, table) { + const primary = table.primary + if (filters) { + // need to map over the filters and make sure the _id field isn't present + for (let filter of Object.values(filters)) { + if (filter._id) { + const parts = breakRowIdField(filter._id) + for (let field of primary) { + filter[field] = parts.shift() + } + } + // make sure this field doesn't exist on any filter + delete filter._id + } + } + // there is no id, just use the user provided filters if (!id || !table) { - return null + return filters } // if used as URL parameter it will have been joined if (typeof id === "string") { - id = id.split(",") + id = breakRowIdField(id) } - const primary = table.primary const equal = {} for (let field of primary) { // work through the ID and get the parts @@ -65,10 +84,9 @@ async function handleRequest( throw `Unable to process query, table "${tableName}" not defined.` } // clean up row on ingress using schema + filters = buildFilters(id, filters, table) row = inputProcessing(row, table) - // try and build an id filter if required - let idFilters = buildIDFilter(id, table) - if (operation === DataSourceOperation.DELETE && Object.keys(idFilters).length === 0) { + if (operation === DataSourceOperation.DELETE && Object.keys(filters).length === 0) { throw "Deletion must be filtered in someway" } let json = { @@ -81,7 +99,7 @@ async function handleRequest( // not specifying any fields means "*" fields: [], }, - filters: idFilters != null ? idFilters : filters, + filters, sort, paginate, body: row, @@ -103,7 +121,7 @@ exports.patch = async ctx => { const appId = ctx.appId const inputs = ctx.request.body const tableId = ctx.params.tableId - const id = inputs._id + const id = breakRowIdField(inputs._id) // don't save the ID to db delete inputs._id return handleRequest(appId, DataSourceOperation.UPDATE, tableId, { @@ -150,7 +168,7 @@ exports.destroy = async ctx => { const appId = ctx.appId const tableId = ctx.params.tableId return handleRequest(appId, DataSourceOperation.DELETE, tableId, { - id: ctx.request.body._id, + id: breakRowIdField(ctx.request.body._id), }) } @@ -163,7 +181,7 @@ exports.bulkDestroy = async ctx => { for (let row of rows) { promises.push( handleRequest(appId, DataSourceOperation.DELETE, tableId, { - id: row._id, + id: breakRowIdField(row._id), }) ) } @@ -182,6 +200,10 @@ exports.search = async ctx => { // todo: need to handle bookmarks page: params.bookmark, } + } else if (params && params.limit) { + paginateObj = { + limit: params.limit, + } } let sort if (params.sort) { diff --git a/packages/server/src/automations/automationUtils.js b/packages/server/src/automations/automationUtils.js index b86107f7e9..83c50eb71a 100644 --- a/packages/server/src/automations/automationUtils.js +++ b/packages/server/src/automations/automationUtils.js @@ -1,4 +1,6 @@ const CouchDB = require("../db") +const { isExternalTable, breakExternalTableId } = require("../integrations/utils") +const { getExternalTable } = require("../api/controllers/table/utils") /** * When values are input to the system generally they will be of type string as this is required for template strings. @@ -60,7 +62,13 @@ module.exports.cleanInputValues = (inputs, schema) => { */ module.exports.cleanUpRow = async (appId, tableId, row) => { const db = new CouchDB(appId) - const table = await db.get(tableId) + let table + if (isExternalTable(tableId)) { + const { datasourceId, tableName } = breakExternalTableId(tableId) + table = await getExternalTable(appId, datasourceId, tableName) + } else { + table = await db.get(tableId) + } return module.exports.cleanInputValues(row, { properties: table.schema }) } diff --git a/packages/server/src/integrations/postgres.js b/packages/server/src/integrations/postgres.js index 45c41d0d05..d40529defd 100644 --- a/packages/server/src/integrations/postgres.js +++ b/packages/server/src/integrations/postgres.js @@ -2,7 +2,6 @@ const { Pool } = require("pg") const { FIELD_TYPES } = require("./Integration") const Sql = require("./base/sql") const { FieldTypes } = require("../constants") -const { SEPARATOR } = require("@budibase/auth/db") const { buildExternalTableId } = require("./utils") const SCHEMA = { diff --git a/packages/server/src/integrations/utils.js b/packages/server/src/integrations/utils.js index 52fe04cfdc..efc0c359d3 100644 --- a/packages/server/src/integrations/utils.js +++ b/packages/server/src/integrations/utils.js @@ -16,3 +16,19 @@ exports.breakExternalTableId = tableId => { let datasourceId = parts.join(DOUBLE_SEPARATOR) return { datasourceId, tableName } } + +exports.generateRowIdField = (keyProps = []) => { + if (!Array.isArray(keyProps)) { + keyProps = [keyProps] + } + // this conserves order and types + return encodeURIComponent(JSON.stringify(keyProps)) +} + +// should always return an array +exports.breakRowIdField = _id => { + if (!_id) { + return null + } + return JSON.parse(decodeURIComponent(_id)) +} From 615b207fbb831f1986d1bb6001a3ed174f2aa9c5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 16 Jun 2021 18:39:59 +0100 Subject: [PATCH 2/2] Formatting. --- packages/builder/src/builderStore/api.js | 30 +++--- .../builderStore/store/automation/index.js | 7 +- .../common/bindings/BindingPanel.svelte | 8 +- .../common/bindings/ServerBindingPanel.svelte | 4 +- .../PropertyControls/DataSourceSelect.svelte | 5 +- .../EventsEditor/actions/ExecuteQuery.svelte | 5 +- .../stores/backend/tests/fixtures/queries.js | 6 +- .../src/api/controllers/row/external.js | 2 +- .../server/src/automations/automationUtils.js | 5 +- .../server/src/integrations/elasticsearch.js | 3 +- packages/server/src/middleware/authorized.js | 92 ++++++++++--------- packages/server/src/middleware/currentapp.js | 8 +- .../standard-components/src/Button.svelte | 2 +- .../src/forms/validation.js | 16 ++-- .../worker/src/api/controllers/admin/email.js | 11 +-- .../src/api/controllers/admin/groups.js | 7 +- .../worker/src/api/controllers/admin/users.js | 7 +- 17 files changed, 100 insertions(+), 118 deletions(-) diff --git a/packages/builder/src/builderStore/api.js b/packages/builder/src/builderStore/api.js index 2e683238bc..ae81df10f6 100644 --- a/packages/builder/src/builderStore/api.js +++ b/packages/builder/src/builderStore/api.js @@ -2,23 +2,21 @@ import { store } from "./index" import { get as svelteGet } from "svelte/store" import { removeCookie, Cookies } from "./cookies" -const apiCall = method => async ( - url, - body, - headers = { "Content-Type": "application/json" } -) => { - headers["x-budibase-app-id"] = svelteGet(store).appId - const json = headers["Content-Type"] === "application/json" - const resp = await fetch(url, { - method: method, - body: json ? JSON.stringify(body) : body, - headers, - }) - if (resp.status === 403) { - removeCookie(Cookies.Auth) +const apiCall = + method => + async (url, body, headers = { "Content-Type": "application/json" }) => { + headers["x-budibase-app-id"] = svelteGet(store).appId + const json = headers["Content-Type"] === "application/json" + const resp = await fetch(url, { + method: method, + body: json ? JSON.stringify(body) : body, + headers, + }) + if (resp.status === 403) { + removeCookie(Cookies.Auth) + } + return resp } - return resp -} export const post = apiCall("POST") export const get = apiCall("GET") diff --git a/packages/builder/src/builderStore/store/automation/index.js b/packages/builder/src/builderStore/store/automation/index.js index 7a01bccfab..c372f27bb7 100644 --- a/packages/builder/src/builderStore/store/automation/index.js +++ b/packages/builder/src/builderStore/store/automation/index.js @@ -100,9 +100,10 @@ const automationActions = store => ({ }, deleteAutomationBlock: block => { store.update(state => { - const idx = state.selectedAutomation.automation.definition.steps.findIndex( - x => x.id === block.id - ) + const idx = + state.selectedAutomation.automation.definition.steps.findIndex( + x => x.id === block.id + ) state.selectedAutomation.deleteBlock(block.id) // Select next closest step diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte index 37e7f520f3..01e0c5918b 100644 --- a/packages/builder/src/components/common/bindings/BindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte @@ -59,9 +59,7 @@
Columns
    - {#each context.filter(context => - context.readableBinding.match(searchRgx) - ) as { readableBinding }} + {#each context.filter( context => context.readableBinding.match(searchRgx) ) as { readableBinding }}
  • { value = addToText(value, getCaretPosition(), readableBinding) @@ -77,9 +75,7 @@
    Components
      - {#each instance.filter(instance => - instance.readableBinding.match(searchRgx) - ) as { readableBinding }} + {#each instance.filter( instance => instance.readableBinding.match(searchRgx) ) as { readableBinding }}
    • addToText(readableBinding)}> {readableBinding}
      • diff --git a/packages/builder/src/components/common/bindings/ServerBindingPanel.svelte b/packages/builder/src/components/common/bindings/ServerBindingPanel.svelte index dc921dfb27..2c0e33227b 100644 --- a/packages/builder/src/components/common/bindings/ServerBindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/ServerBindingPanel.svelte @@ -49,9 +49,7 @@
      {#each categories as [categoryName, bindings]} {categoryName} - {#each bindings.filter(binding => - binding.label.match(searchRgx) - ) as binding} + {#each bindings.filter( binding => binding.label.match(searchRgx) ) as binding}
      { diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte index 57c64ca0ed..43f9837232 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte @@ -103,8 +103,9 @@ } function fetchQueryDefinition(query) { - const source = $datasources.list.find(ds => ds._id === query.datasourceId) - .source + const source = $datasources.list.find( + ds => ds._id === query.datasourceId + ).source return $integrations[source].query[query.queryVerb] } diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/ExecuteQuery.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/ExecuteQuery.svelte index 8c10b47a4d..44f349a324 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/ExecuteQuery.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/ExecuteQuery.svelte @@ -18,8 +18,9 @@ ) function fetchQueryDefinition(query) { - const source = $datasources.list.find(ds => ds._id === query.datasourceId) - .source + const source = $datasources.list.find( + ds => ds._id === query.datasourceId + ).source return $integrations[source].query[query.queryVerb] } diff --git a/packages/builder/src/stores/backend/tests/fixtures/queries.js b/packages/builder/src/stores/backend/tests/fixtures/queries.js index 7815d61ea5..f5e595249a 100644 --- a/packages/builder/src/stores/backend/tests/fixtures/queries.js +++ b/packages/builder/src/stores/backend/tests/fixtures/queries.js @@ -9,8 +9,7 @@ export const SOME_QUERY = { queryVerb: "read", schema: {}, name: "Speakers", - _id: - "query_datasource_04b003a7b4a8428eadd3bb2f7eae0255_bcb8ffc6fcbc484e8d63121fc0bf986f", + _id: "query_datasource_04b003a7b4a8428eadd3bb2f7eae0255_bcb8ffc6fcbc484e8d63121fc0bf986f", _rev: "2-941f8699eb0adf995f8bd59c99203b26", readable: true, } @@ -75,8 +74,7 @@ export const SAVE_QUERY_RESPONSE = { }, }, name: "Speakers", - _id: - "query_datasource_04b003a7b4a8428eadd3bb2f7eae0255_bcb8ffc6fcbc484e8d63121fc0bf986f", + _id: "query_datasource_04b003a7b4a8428eadd3bb2f7eae0255_bcb8ffc6fcbc484e8d63121fc0bf986f", _rev: "3-5a64adef494b1e9c793dc91b51ce73c6", readable: true, } diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index 940bb7bb00..cd053b036c 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -4,7 +4,7 @@ const { getExternalTable } = require("../table/utils") const { breakExternalTableId, generateRowIdField, - breakRowIdField + breakRowIdField, } = require("../../../integrations/utils") function inputProcessing(row, table) { diff --git a/packages/server/src/automations/automationUtils.js b/packages/server/src/automations/automationUtils.js index 83c50eb71a..5cb84c63b1 100644 --- a/packages/server/src/automations/automationUtils.js +++ b/packages/server/src/automations/automationUtils.js @@ -1,5 +1,8 @@ const CouchDB = require("../db") -const { isExternalTable, breakExternalTableId } = require("../integrations/utils") +const { + isExternalTable, + breakExternalTableId, +} = require("../integrations/utils") const { getExternalTable } = require("../api/controllers/table/utils") /** diff --git a/packages/server/src/integrations/elasticsearch.js b/packages/server/src/integrations/elasticsearch.js index be1b945893..3a52a0278b 100644 --- a/packages/server/src/integrations/elasticsearch.js +++ b/packages/server/src/integrations/elasticsearch.js @@ -2,8 +2,7 @@ const { Client } = require("@elastic/elasticsearch") const { QUERY_TYPES, FIELD_TYPES } = require("./Integration") const SCHEMA = { - docs: - "https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/index.html", + docs: "https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/index.html", description: "Elasticsearch is a search engine based on the Lucene library. It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.", friendlyName: "ElasticSearch", diff --git a/packages/server/src/middleware/authorized.js b/packages/server/src/middleware/authorized.js index 8ed4c41db7..bd064f7e66 100644 --- a/packages/server/src/middleware/authorized.js +++ b/packages/server/src/middleware/authorized.js @@ -14,50 +14,52 @@ const WEBHOOK_ENDPOINTS = new RegExp( ["webhooks/trigger", "webhooks/schema"].join("|") ) -module.exports = (permType, permLevel = null) => async (ctx, next) => { - // webhooks don't need authentication, each webhook unique - if (WEBHOOK_ENDPOINTS.test(ctx.request.url)) { +module.exports = + (permType, permLevel = null) => + async (ctx, next) => { + // webhooks don't need authentication, each webhook unique + if (WEBHOOK_ENDPOINTS.test(ctx.request.url)) { + return next() + } + + if (!ctx.user) { + return ctx.throw(403, "No user info found") + } + + // check general builder stuff, this middleware is a good way + // to find API endpoints which are builder focused + await builderMiddleware(ctx, permType) + + const isAuthed = ctx.isAuthenticated + const { basePermissions, permissions } = await getUserPermissions( + ctx.appId, + ctx.roleId + ) + + // builders for now have permission to do anything + // TODO: in future should consider separating permissions with an require("@budibase/auth").isClient check + let isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global + const isBuilderApi = permType === PermissionTypes.BUILDER + if (isBuilder) { + return next() + } else if (isBuilderApi && !isBuilder) { + return ctx.throw(403, "Not Authorized") + } + + if ( + hasResource(ctx) && + doesHaveResourcePermission(permissions, permLevel, ctx) + ) { + return next() + } + + if (!isAuthed) { + ctx.throw(403, "Session not authenticated") + } + + if (!doesHaveBasePermission(permType, permLevel, basePermissions)) { + ctx.throw(403, "User does not have permission") + } + return next() } - - if (!ctx.user) { - return ctx.throw(403, "No user info found") - } - - // check general builder stuff, this middleware is a good way - // to find API endpoints which are builder focused - await builderMiddleware(ctx, permType) - - const isAuthed = ctx.isAuthenticated - const { basePermissions, permissions } = await getUserPermissions( - ctx.appId, - ctx.roleId - ) - - // builders for now have permission to do anything - // TODO: in future should consider separating permissions with an require("@budibase/auth").isClient check - let isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global - const isBuilderApi = permType === PermissionTypes.BUILDER - if (isBuilder) { - return next() - } else if (isBuilderApi && !isBuilder) { - return ctx.throw(403, "Not Authorized") - } - - if ( - hasResource(ctx) && - doesHaveResourcePermission(permissions, permLevel, ctx) - ) { - return next() - } - - if (!isAuthed) { - ctx.throw(403, "Session not authenticated") - } - - if (!doesHaveBasePermission(permType, permLevel, basePermissions)) { - ctx.throw(403, "User does not have permission") - } - - return next() -} diff --git a/packages/server/src/middleware/currentapp.js b/packages/server/src/middleware/currentapp.js index 19d0afd560..683b7f8ef3 100644 --- a/packages/server/src/middleware/currentapp.js +++ b/packages/server/src/middleware/currentapp.js @@ -1,9 +1,5 @@ -const { - getAppId, - setCookie, - getCookie, - clearCookie, -} = require("@budibase/auth").utils +const { getAppId, setCookie, getCookie, clearCookie } = + require("@budibase/auth").utils const { Cookies } = require("@budibase/auth").constants const { getRole } = require("@budibase/auth/roles") const { getGlobalSelf } = require("../utilities/workerRequests") diff --git a/packages/standard-components/src/Button.svelte b/packages/standard-components/src/Button.svelte index d307b25256..3739f5e3af 100644 --- a/packages/standard-components/src/Button.svelte +++ b/packages/standard-components/src/Button.svelte @@ -18,4 +18,4 @@ on:click={onClick} > {text || ""} - \ No newline at end of file + diff --git a/packages/standard-components/src/forms/validation.js b/packages/standard-components/src/forms/validation.js index ebdb3b3dab..b1df80509f 100644 --- a/packages/standard-components/src/forms/validation.js +++ b/packages/standard-components/src/forms/validation.js @@ -90,15 +90,17 @@ const numericalConstraint = (constraint, error) => value => { return null } -const inclusionConstraint = (options = []) => value => { - if (value == null || value === "") { +const inclusionConstraint = + (options = []) => + value => { + if (value == null || value === "") { + return null + } + if (!options.includes(value)) { + return "Invalid value" + } return null } - if (!options.includes(value)) { - return "Invalid value" - } - return null -} const dateConstraint = (dateString, isEarliest) => { const dateLimit = Date.parse(dateString) diff --git a/packages/worker/src/api/controllers/admin/email.js b/packages/worker/src/api/controllers/admin/email.js index c4b17ee980..6e16fd060c 100644 --- a/packages/worker/src/api/controllers/admin/email.js +++ b/packages/worker/src/api/controllers/admin/email.js @@ -5,15 +5,8 @@ const authPkg = require("@budibase/auth") const GLOBAL_DB = authPkg.StaticDatabases.GLOBAL.name exports.sendEmail = async ctx => { - const { - groupId, - email, - userId, - purpose, - contents, - from, - subject, - } = ctx.request.body + const { groupId, email, userId, purpose, contents, from, subject } = + ctx.request.body let user if (userId) { const db = new CouchDB(GLOBAL_DB) diff --git a/packages/worker/src/api/controllers/admin/groups.js b/packages/worker/src/api/controllers/admin/groups.js index 8a70721431..330fb38282 100644 --- a/packages/worker/src/api/controllers/admin/groups.js +++ b/packages/worker/src/api/controllers/admin/groups.js @@ -1,9 +1,6 @@ const CouchDB = require("../../../db") -const { - getGroupParams, - generateGroupID, - StaticDatabases, -} = require("@budibase/auth").db +const { getGroupParams, generateGroupID, StaticDatabases } = + require("@budibase/auth").db const GLOBAL_DB = StaticDatabases.GLOBAL.name diff --git a/packages/worker/src/api/controllers/admin/users.js b/packages/worker/src/api/controllers/admin/users.js index 6afd4ecbfd..5b95b2808f 100644 --- a/packages/worker/src/api/controllers/admin/users.js +++ b/packages/worker/src/api/controllers/admin/users.js @@ -1,9 +1,6 @@ const CouchDB = require("../../../db") -const { - generateGlobalUserID, - getGlobalUserParams, - StaticDatabases, -} = require("@budibase/auth").db +const { generateGlobalUserID, getGlobalUserParams, StaticDatabases } = + require("@budibase/auth").db const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils const { UserStatus, EmailTemplatePurpose } = require("../../../constants") const { checkInviteCode } = require("../../../utilities/redis")