From 1ad2687f4f9cda6f45cfb017cc7b6ce5cc633e90 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 21 Sep 2021 17:21:01 +0100 Subject: [PATCH 01/42] Set datasource to internal datasource upon loading internal datasource page --- .../app/[application]/data/datasource/bb_internal/_layout.svelte | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte new file mode 100644 index 0000000000..e69de29bb2 From 216a7c7a3f81c32fe3342eca12b8308e86ca1bea Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 21 Sep 2021 17:21:25 +0100 Subject: [PATCH 02/42] Ensure datasource containing the currently selected table is always expanded --- .../DatasourceNavigator.svelte | 30 +++++++++++++++---- .../datasource/bb_internal/_layout.svelte | 7 +++++ .../builder/src/stores/backend/datasources.js | 15 ++++++++-- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte index 84c737eb67..193e778300 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte @@ -1,8 +1,9 @@ {#if $database?._id}
- {#each $datasources.list as datasource, idx} + {#each enrichedDataSources as datasource, idx} 0} text={datasource.name} - opened={openDataSources.includes(datasource._id)} - selected={$datasources.selected === datasource._id} + opened={datasource.open} + selected={datasource.selected} withArrow={true} on:click={() => selectDatasource(datasource)} on:iconClick={() => toggleNode(datasource)} @@ -61,7 +81,7 @@ {/if} - {#if openDataSources.includes(datasource._id)} + {#if datasource.open} {/if} diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte index e69de29bb2..ed271aae34 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte @@ -0,0 +1,7 @@ + + + diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index 5c6ed3f2cb..5e42315948 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -1,4 +1,4 @@ -import { writable } from "svelte/store" +import { writable, get } from "svelte/store" import { queries, tables, views } from "./" import api from "../../builderStore/api" @@ -8,7 +8,8 @@ export const INITIAL_DATASOURCE_VALUES = { } export function createDatasourcesStore() { - const { subscribe, update, set } = writable(INITIAL_DATASOURCE_VALUES) + const store = writable(INITIAL_DATASOURCE_VALUES) + const { subscribe, update, set } = store return { subscribe, @@ -21,7 +22,15 @@ export function createDatasourcesStore() { fetch: async () => { const response = await api.get(`/api/datasources`) const json = await response.json() - update(state => ({ ...state, list: json, selected: null })) + + // Clear selected if it no longer exists, otherwise keep it + const selected = get(store).selected + let nextSelected = null + if (selected && json.find(source => source._id === selected)) { + nextSelected = selected + } + + update(state => ({ ...state, list: json, selected: nextSelected })) return json }, select: async datasourceId => { From 795295aa9082a80cce744eebdab837753ed95548 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Wed, 22 Sep 2021 08:42:24 +0000 Subject: [PATCH 03/42] v0.9.140 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index 9b2b1cac6d..a21b86e032 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.140-alpha.0", + "version": "0.9.140", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index 448b408742..b02625ef29 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 123d168fee..b542df3bb8 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index fb4d050392..10c7572743 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.0", - "@budibase/client": "^0.9.140-alpha.0", + "@budibase/bbui": "^0.9.140", + "@budibase/client": "^0.9.140", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.140-alpha.0", + "@budibase/string-templates": "^0.9.140", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 7bdf1a394f..f6fadeff6b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 98b0daf581..5ab814a4ac 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.140-alpha.0", + "@budibase/bbui": "^0.9.140", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.140-alpha.0", + "@budibase/string-templates": "^0.9.140", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index ad6e55dd6c..743c2d8ff4 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -64,9 +64,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.0", - "@budibase/client": "^0.9.140-alpha.0", - "@budibase/string-templates": "^0.9.140-alpha.0", + "@budibase/auth": "^0.9.140", + "@budibase/client": "^0.9.140", + "@budibase/string-templates": "^0.9.140", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 0ec109eca5..8686a43f51 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index eecd3d6959..2d2bf0df80 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.140-alpha.0", + "version": "0.9.140", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140-alpha.0", - "@budibase/string-templates": "^0.9.140-alpha.0", + "@budibase/auth": "^0.9.140", + "@budibase/string-templates": "^0.9.140", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From 5f5154b08474d92d9481290cc455665cb0b455cf Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Wed, 22 Sep 2021 11:36:15 +0000 Subject: [PATCH 04/42] v0.9.141 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index a21b86e032..483a1abd82 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.140", + "version": "0.9.141", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index b02625ef29..911163deed 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.140", + "version": "0.9.141", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index b542df3bb8..b45081b890 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.140", + "version": "0.9.141", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 10c7572743..6f9ae796f0 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.140", + "version": "0.9.141", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.140", - "@budibase/client": "^0.9.140", + "@budibase/bbui": "^0.9.141", + "@budibase/client": "^0.9.141", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.140", + "@budibase/string-templates": "^0.9.141", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index f6fadeff6b..9f9f0e7454 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.140", + "version": "0.9.141", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 5ab814a4ac..3cfdf8e2f3 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.140", + "version": "0.9.141", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.140", + "@budibase/bbui": "^0.9.141", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.140", + "@budibase/string-templates": "^0.9.141", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 743c2d8ff4..1bbb871f1b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.140", + "version": "0.9.141", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -64,9 +64,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140", - "@budibase/client": "^0.9.140", - "@budibase/string-templates": "^0.9.140", + "@budibase/auth": "^0.9.141", + "@budibase/client": "^0.9.141", + "@budibase/string-templates": "^0.9.141", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 8686a43f51..e0e73955d3 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.140", + "version": "0.9.141", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 2d2bf0df80..8849499786 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.140", + "version": "0.9.141", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.140", - "@budibase/string-templates": "^0.9.140", + "@budibase/auth": "^0.9.141", + "@budibase/string-templates": "^0.9.141", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From 22e75b8154d4c15070663f379ec4f76fc78755df Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 22 Sep 2021 14:50:52 +0100 Subject: [PATCH 05/42] Only create default home screen and layouts when not importing an app --- packages/server/src/api/controllers/application.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index da0014c5f8..c3142e9733 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -230,7 +230,12 @@ exports.create = async function (ctx) { const response = await db.put(newApplication, { force: true }) newApplication._rev = response.rev - await createEmptyAppPackage(ctx, newApplication) + // Only create the default home screens and layout if we aren't importing + // an app + if (!useTemplate) { + await createEmptyAppPackage(ctx, newApplication) + } + /* istanbul ignore next */ if (!env.isTest()) { await createApp(appId) From fb7a70054216d074385d3e9d4e9bb7c9bc2af9a8 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 23 Sep 2021 16:22:12 +0100 Subject: [PATCH 06/42] point logo upload to S3 bucket when not self hosted --- packages/worker/src/api/controllers/global/configs.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/worker/src/api/controllers/global/configs.js b/packages/worker/src/api/controllers/global/configs.js index aa83fd695f..c0c300e4db 100644 --- a/packages/worker/src/api/controllers/global/configs.js +++ b/packages/worker/src/api/controllers/global/configs.js @@ -10,6 +10,7 @@ const email = require("../../../utilities/email") const { upload, ObjectStoreBuckets } = require("@budibase/auth").objectStore const CouchDB = require("../../../db") const { getGlobalDB } = require("@budibase/auth/tenancy") +const env = require("../../../environment") exports.save = async function (ctx) { const db = getGlobalDB() @@ -174,7 +175,13 @@ exports.upload = async function (ctx) { const file = ctx.request.files.file const { type, name } = ctx.params - const bucket = ObjectStoreBuckets.GLOBAL + let bucket + if (env.SELF_HOSTED) { + bucket = ObjectStoreBuckets.GLOBAL + } else { + bucket = ObjectStoreBuckets.GLOBAL_CLOUD + } + const key = `${type}/${name}` await upload({ bucket, From 1952dc308eff355e249d1bff123e056cc602cc8e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 Sep 2021 16:17:23 +0100 Subject: [PATCH 07/42] Fixes issue #2616 - this is a slightly complex fix and handles a few other issues with mysql (around returning on creation of a row and relationships) - a new mechanism is now used for pagination and limiting which makes sure the limits are applied to the outer table rather than the combination of the outer and the joined. --- .../integrations/postgres/docker-compose.yml | 2 +- .../api/controllers/row/ExternalRequest.ts | 3 ++ packages/server/src/definitions/datasource.ts | 5 ++ packages/server/src/integrations/base/sql.ts | 52 +++++++++---------- packages/server/src/integrations/mysql.ts | 21 +++++++- 5 files changed, 54 insertions(+), 29 deletions(-) diff --git a/packages/server/scripts/integrations/postgres/docker-compose.yml b/packages/server/scripts/integrations/postgres/docker-compose.yml index e2bba9f38e..4dfcb0e1ad 100644 --- a/packages/server/scripts/integrations/postgres/docker-compose.yml +++ b/packages/server/scripts/integrations/postgres/docker-compose.yml @@ -15,7 +15,7 @@ services: - ./init.sql:/docker-entrypoint-initdb.d/init.sql pgadmin: - container_name: pgadmin + container_name: pgadmin-pg image: dpage/pgadmin4 restart: always environment: diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index b809e597e4..12db55efdc 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -544,6 +544,9 @@ module External { extra: { idFilter: buildFilters(id || generateIdForRow(row, table), {}, table), }, + meta: { + table, + } } // can't really use response right now const response = await makeExternalQuery(appId, json) diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index 48fd24e1cf..d7d4e77961 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -1,3 +1,5 @@ +import {Table} from "./common"; + export enum Operation { CREATE = "CREATE", READ = "READ", @@ -136,6 +138,9 @@ export interface QueryJson { sort?: SortJson paginate?: PaginationJson body?: object + meta?: { + table?: Table, + } extra?: { idFilter?: SearchFilters } diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index b59bac5a5a..91af3e1a85 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -1,7 +1,5 @@ import { Knex, knex } from "knex" const BASE_LIMIT = 5000 -// if requesting a single row then need to up the limit for the sake of joins -const SINGLE_ROW_LIMIT = 100 import { QueryJson, SearchFilters, @@ -146,46 +144,48 @@ function buildCreate( function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery { let { endpoint, resource, filters, sort, paginate, relationships } = json const tableName = endpoint.entityId - let query: KnexQuery = knex(tableName) // select all if not specified if (!resource) { resource = { fields: [] } } + let selectStatement: string|string[] = "*" // handle select if (resource.fields && resource.fields.length > 0) { // select the resources as the format "table.columnName" - this is what is provided // by the resource builder further up - query = query.select(resource.fields.map(field => `${field} as ${field}`)) - } else { - query = query.select("*") + selectStatement = resource.fields.map(field => `${field} as ${field}`) + } + let foundLimit = limit || BASE_LIMIT + // handle pagination + let foundOffset: number | null = null + if (paginate && paginate.page && paginate.limit) { + // @ts-ignore + const page = paginate.page <= 1 ? 0 : paginate.page - 1 + const offset = page * paginate.limit + foundLimit = paginate.limit + foundOffset = offset + } else if (paginate && paginate.limit) { + foundLimit = paginate.limit + } + // start building the query + let query: KnexQuery = knex(tableName).limit(foundLimit) + if (foundOffset) { + query = query.offset(foundOffset) } - // handle where - query = addFilters(tableName, query, filters) - // handle join - query = addRelationships(query, tableName, relationships) - // handle sorting if (sort) { for (let [key, value] of Object.entries(sort)) { const direction = value === SortDirection.ASCENDING ? "asc" : "desc" query = query.orderBy(key, direction) } } - let foundLimit = limit || BASE_LIMIT - // handle pagination - if (paginate && paginate.page && paginate.limit) { + query = addFilters(tableName, query, filters) + // @ts-ignore + let preQuery: KnexQuery = knex({ // @ts-ignore - const page = paginate.page <= 1 ? 0 : paginate.page - 1 - const offset = page * paginate.limit - foundLimit = paginate.limit - query = query.offset(offset) - } else if (paginate && paginate.limit) { - foundLimit = paginate.limit - } - if (foundLimit === 1) { - foundLimit = SINGLE_ROW_LIMIT - } - query = query.limit(foundLimit) - return query + [tableName]: query, + }).select(selectStatement) + // handle joins + return addRelationships(preQuery, tableName, relationships) } function buildUpdate( diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index c5db35ed2a..11220afb46 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -104,7 +104,7 @@ module MySQLModule { client: any, query: SqlQuery, connect: boolean = true - ): Promise { + ): Promise { // Node MySQL is callback based, so we must wrap our call in a promise return new Promise((resolve, reject) => { if (connect) { @@ -238,6 +238,23 @@ module MySQLModule { return internalQuery(this.client, input, false) } + // when creating if an ID has been inserted need to make sure + // the id filter is enriched with it before trying to retrieve the row + checkLookupKeys(results: any, json: QueryJson) { + if (!results?.insertId || !json.meta?.table || !json.meta.table.primary) { + return json + } + const primaryKey = json.meta.table.primary?.[0] + json.extra = { + idFilter: { + equal: { + [primaryKey]: results.insertId + }, + } + } + return json + } + async query(json: QueryJson) { const operation = this._operation(json) this.client.connect() @@ -250,7 +267,7 @@ module MySQLModule { const results = await internalQuery(this.client, input, false) // same as delete, manage returning if (operation === Operation.CREATE || operation === Operation.UPDATE) { - row = this.getReturningRow(json) + row = this.getReturningRow(this.checkLookupKeys(results, json)) } this.client.end() if (operation !== Operation.READ) { From 266c235c9864d68aa081bb6b5bd1c3e37bcd3f3e Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Thu, 23 Sep 2021 15:48:36 +0000 Subject: [PATCH 08/42] v0.9.142 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index 483a1abd82..a8cda22a63 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.141", + "version": "0.9.142", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index 911163deed..f62e4cb5e6 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.141", + "version": "0.9.142", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index b45081b890..ddede26a86 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.141", + "version": "0.9.142", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 6f9ae796f0..7ab313974b 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.141", + "version": "0.9.142", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.141", - "@budibase/client": "^0.9.141", + "@budibase/bbui": "^0.9.142", + "@budibase/client": "^0.9.142", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.141", + "@budibase/string-templates": "^0.9.142", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 9f9f0e7454..0340759e45 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.141", + "version": "0.9.142", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 3cfdf8e2f3..3d25a1b5d6 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.141", + "version": "0.9.142", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.141", + "@budibase/bbui": "^0.9.142", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^0.9.141", + "@budibase/string-templates": "^0.9.142", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 1bbb871f1b..9397ab16f3 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.141", + "version": "0.9.142", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -64,9 +64,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.141", - "@budibase/client": "^0.9.141", - "@budibase/string-templates": "^0.9.141", + "@budibase/auth": "^0.9.142", + "@budibase/client": "^0.9.142", + "@budibase/string-templates": "^0.9.142", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index e0e73955d3..44a3290696 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.141", + "version": "0.9.142", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 8849499786..83e53e9b45 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.141", + "version": "0.9.142", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -25,8 +25,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.141", - "@budibase/string-templates": "^0.9.141", + "@budibase/auth": "^0.9.142", + "@budibase/string-templates": "^0.9.142", "@koa/router": "^8.0.0", "@techpass/passport-openidconnect": "^0.3.0", "aws-sdk": "^2.811.0", From 7c7266a54776fadd804c3fdf8541684ed716b878 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 Sep 2021 16:56:13 +0100 Subject: [PATCH 09/42] Fixing SQL test cases. --- packages/server/src/integrations/tests/sql.spec.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/server/src/integrations/tests/sql.spec.js b/packages/server/src/integrations/tests/sql.spec.js index fa8bcd1d86..64cdda215f 100644 --- a/packages/server/src/integrations/tests/sql.spec.js +++ b/packages/server/src/integrations/tests/sql.spec.js @@ -57,7 +57,7 @@ describe("SQL query builder", () => { const query = sql._query(generateReadJson()) expect(query).toEqual({ bindings: [limit], - sql: `select * from "${TABLE_NAME}" limit $1` + sql: `select * from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"` }) }) @@ -68,7 +68,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [limit], - sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from "${TABLE_NAME}" limit $1` + sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"` }) }) @@ -82,7 +82,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: ["John%", limit], - sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2` + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2) as "${TABLE_NAME}"` }) }) @@ -99,7 +99,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [2, 10, limit], - sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3` + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3) as "${TABLE_NAME}"` }) }) @@ -115,7 +115,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [10, "John", limit], - sql: `select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3` + sql: `select * from (select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3) as "${TABLE_NAME}"` }) }) @@ -160,7 +160,7 @@ describe("SQL query builder", () => { const query = new Sql("mssql", 10)._query(generateReadJson()) expect(query).toEqual({ bindings: [10], - sql: `select top (@p0) * from [${TABLE_NAME}]` + sql: `select * from (select top (@p0) * from [${TABLE_NAME}]) as [${TABLE_NAME}]` }) }) @@ -168,7 +168,7 @@ describe("SQL query builder", () => { const query = new Sql("mysql", 10)._query(generateReadJson()) expect(query).toEqual({ bindings: [10], - sql: `select * from \`${TABLE_NAME}\` limit ?` + sql: `select * from (select * from \`${TABLE_NAME}\` limit ?) as \`${TABLE_NAME}\`` }) }) }) From a8641056492f1c99bf15eae3a9b42814dc291ed2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 Sep 2021 17:43:06 +0100 Subject: [PATCH 10/42] Fixing postgres datasource test. --- packages/server/src/api/routes/tests/datasource.spec.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/datasource.spec.js b/packages/server/src/api/routes/tests/datasource.spec.js index 98a99717fd..b6d94f714d 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.js +++ b/packages/server/src/api/routes/tests/datasource.spec.js @@ -94,7 +94,8 @@ describe("/datasources", () => { .expect(200) // this is mock data, can't test it expect(res.body).toBeDefined() - expect(pg.queryMock).toHaveBeenCalledWith(`select "users"."name" as "users.name", "users"."age" as "users.age" from "users" where "users"."name" ilike $1 limit $2`, ["John%", 5000]) + const expSql = `select "users"."name" as "users.name", "users"."age" as "users.age" from (select * from "users" where "users"."name" ilike $1 limit $2) as "users"` + expect(pg.queryMock).toHaveBeenCalledWith(expSql, ["John%", 5000]) }) }) From 649a33d3c8f800233b45934467cd1db020fcab2e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 Sep 2021 19:04:53 +0100 Subject: [PATCH 11/42] Getting rid of the concept of permissions hierarchy, roles still have a hierarchy and base permissions still follow the old system, but resources can be given a stack of separate permissions which don't override each other. --- packages/auth/src/security/permissions.js | 7 +- packages/auth/src/security/roles.js | 15 +++- .../server/src/api/controllers/permission.js | 68 ++++++++----------- .../server/src/api/routes/tests/role.spec.js | 2 +- packages/server/src/utilities/index.js | 8 +++ 5 files changed, 50 insertions(+), 50 deletions(-) diff --git a/packages/auth/src/security/permissions.js b/packages/auth/src/security/permissions.js index 03fa5fa562..d0308d783e 100644 --- a/packages/auth/src/security/permissions.js +++ b/packages/auth/src/security/permissions.js @@ -139,8 +139,7 @@ exports.doesHaveResourcePermission = ( // set foundSub to not subResourceId, incase there is no subResource let foundMain = false, foundSub = false - for (let [resource, level] of Object.entries(permissions)) { - const levels = getAllowedLevels(level) + for (let [resource, levels] of Object.entries(permissions)) { if (resource === resourceId && levels.indexOf(permLevel) !== -1) { foundMain = true } @@ -177,10 +176,6 @@ exports.doesHaveBasePermission = (permType, permLevel, permissionIds) => { return false } -exports.higherPermission = (perm1, perm2) => { - return levelToNumber(perm1) > levelToNumber(perm2) ? perm1 : perm2 -} - exports.isPermissionLevelHigherThanRead = level => { return levelToNumber(level) > 1 } diff --git a/packages/auth/src/security/roles.js b/packages/auth/src/security/roles.js index baa8fc40dc..71fbc10132 100644 --- a/packages/auth/src/security/roles.js +++ b/packages/auth/src/security/roles.js @@ -1,6 +1,6 @@ const { getDB } = require("../db") const { cloneDeep } = require("lodash/fp") -const { BUILTIN_PERMISSION_IDS, higherPermission } = require("./permissions") +const { BUILTIN_PERMISSION_IDS } = require("./permissions") const { generateRoleID, getRoleParams, @@ -193,8 +193,17 @@ exports.getUserPermissions = async (appId, userRoleId) => { const permissions = {} for (let role of rolesHierarchy) { if (role.permissions) { - for (let [resource, level] of Object.entries(role.permissions)) { - permissions[resource] = higherPermission(permissions[resource], level) + for (let [resource, levels] of Object.entries(role.permissions)) { + if (!permissions[resource]) { + permissions[resource] = [] + } + const permsSet = new Set(permissions[resource]) + if (Array.isArray(levels)) { + levels.forEach(level => permsSet.add(level)) + } else { + permsSet.add(levels) + } + permissions[resource] = [...permsSet] } } } diff --git a/packages/server/src/api/controllers/permission.js b/packages/server/src/api/controllers/permission.js index e269f8c41d..6c02663649 100644 --- a/packages/server/src/api/controllers/permission.js +++ b/packages/server/src/api/controllers/permission.js @@ -1,9 +1,4 @@ -const { - getBuiltinPermissions, - PermissionLevels, - isPermissionLevelHigherThanRead, - higherPermission, -} = require("@budibase/auth/permissions") +const { getBuiltinPermissions } = require("@budibase/auth/permissions") const { isBuiltin, getDBRoleID, @@ -16,6 +11,7 @@ const { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, } = require("../../utilities/security") +const { removeFromArray } = require("../../utilities") const PermissionUpdateType = { REMOVE: "remove", @@ -24,22 +20,6 @@ const PermissionUpdateType = { const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS -// quick function to perform a bit of weird logic, make sure fetch calls -// always say a write role also has read permission -function fetchLevelPerms(permissions, level, roleId) { - if (!permissions) { - permissions = {} - } - permissions[level] = roleId - if ( - isPermissionLevelHigherThanRead(level) && - !permissions[PermissionLevels.READ] - ) { - permissions[PermissionLevels.READ] = roleId - } - return permissions -} - // utility function to stop this repetition - permissions always stored under roles async function getAllDBRoles(db) { const body = await db.allDocs( @@ -74,23 +54,31 @@ async function updatePermissionOnRole( for (let role of dbRoles) { let updated = false const rolePermissions = role.permissions ? role.permissions : {} + // make sure its an array, also handle migrating + if ( + !rolePermissions[resourceId] || + !Array.isArray(rolePermissions[resourceId]) + ) { + rolePermissions[resourceId] = + typeof rolePermissions[resourceId] === "string" + ? [rolePermissions[resourceId]] + : [] + } // handle the removal/updating the role which has this permission first // the updating (role._id !== dbRoleId) is required because a resource/level can // only be permitted in a single role (this reduces hierarchy confusion and simplifies // the general UI for this, rather than needing to show everywhere it is used) if ( (role._id !== dbRoleId || remove) && - rolePermissions[resourceId] === level + rolePermissions[resourceId].indexOf(level) !== -1 ) { - delete rolePermissions[resourceId] + removeFromArray(rolePermissions[resourceId], level) updated = true } // handle the adding, we're on the correct role, at it to this if (!remove && role._id === dbRoleId) { - rolePermissions[resourceId] = higherPermission( - rolePermissions[resourceId], - level - ) + const set = new Set(rolePermissions[resourceId]) + rolePermissions[resourceId] = [...set.add(level)] updated = true } // handle the update, add it to bulk docs to perform at end @@ -127,12 +115,11 @@ exports.fetch = async function (ctx) { continue } const roleId = getExternalRoleID(role._id) - for (let [resource, level] of Object.entries(role.permissions)) { - permissions[resource] = fetchLevelPerms( - permissions[resource], - level, - roleId - ) + for (let [resource, levelArr] of Object.entries(role.permissions)) { + const levels = Array.isArray(levelArr) ? [levelArr] : levelArr + const perms = {} + levels.forEach(level => (perms[level] = roleId)) + permissions[resource] = perms } } // apply the base permissions @@ -157,12 +144,13 @@ exports.getResourcePerms = async function (ctx) { for (let level of SUPPORTED_LEVELS) { // update the various roleIds in the resource permissions for (let role of roles) { - if (role.permissions && role.permissions[resourceId] === level) { - permissions = fetchLevelPerms( - permissions, - level, - getExternalRoleID(role._id) - ) + const rolePerms = role.permissions + if ( + rolePerms && + (rolePerms[resourceId] === level || + rolePerms[resourceId].indexOf(level) !== -1) + ) { + permissions[level] = getExternalRoleID(role._id) } } } diff --git a/packages/server/src/api/routes/tests/role.spec.js b/packages/server/src/api/routes/tests/role.spec.js index ad42ef180a..d74a84b2b2 100644 --- a/packages/server/src/api/routes/tests/role.spec.js +++ b/packages/server/src/api/routes/tests/role.spec.js @@ -72,7 +72,7 @@ describe("/roles", () => { .expect(200) expect(res.body.length).toBeGreaterThan(0) const power = res.body.find(role => role._id === BUILTIN_ROLE_IDS.POWER) - expect(power.permissions[table._id]).toEqual("read") + expect(power.permissions[table._id]).toEqual(["read"]) }) }) diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index a81f9ddcf5..e24d04c581 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -10,6 +10,14 @@ exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms)) exports.isDev = env.isDev +exports.removeFromArray = (array, element) => { + const index = array.indexOf(element) + if (index !== -1) { + array.splice(index, 1) + } + return array +} + /** * Makes sure that a URL has the correct number of slashes, while maintaining the * http(s):// double slashes. From 78e4f00e11c12ea7705ae9ef00528aa1648fb95f Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 23 Sep 2021 19:34:01 +0100 Subject: [PATCH 12/42] Add button action for manually refreshing a data provider --- .../actions/RefreshDataProvider.svelte | 35 +++++++++++++++++++ .../EventsEditor/actions/index.js | 5 +++ packages/client/manifest.json | 1 + packages/client/src/utils/buttonActions.js | 4 +-- 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/RefreshDataProvider.svelte diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/RefreshDataProvider.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/RefreshDataProvider.svelte new file mode 100644 index 0000000000..4b445e7b21 --- /dev/null +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/RefreshDataProvider.svelte @@ -0,0 +1,35 @@ + + +
+ +