MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' of github.com:Budibase/budibase into feature/auth-core

This commit is contained in:
mike12345567 2022-01-11 15:16:07 +00:00
parent 3543c02f6d 337eb07422
commit b33523a73d
24 changed files with 322 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "1.0.27-alpha.7",
"version": "1.0.27-alpha.11",
"npmClient": "yarn",
"packages": [
"packages/*"

2
packages/backend-core/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
"version": "1.0.27-alpha.7",
"version": "1.0.27-alpha.11",
"description": "Budibase backend core libraries used in server and worker",
"main": "src/index.js",
"author": "Budibase",

12
packages/backend-core/src/db/utils.js
View File

@ -224,8 +224,15 @@ exports.getAllDbs = async () => {
}
}
let couchUrl = `${exports.getCouchUrl()}/_all_dbs`
if (env.MULTI_TENANCY) {
let tenantId = getTenantId()
if (!env.MULTI_TENANCY || tenantId == DEFAULT_TENANT_ID) {
// just get all DBs when:
// - single tenancy
// - default tenant
// - apps dbs don't contain tenant id
// - non-default tenant dbs are filtered out application side in getAllApps
await addDbs(couchUrl)
} else {
// get prod apps
await addDbs(
exports.getStartEndKeyURL(couchUrl, DocumentTypes.APP, tenantId)
@ -236,9 +243,6 @@ exports.getAllDbs = async () => {
)
// add global db name
dbs.push(getGlobalDBName(tenantId))
} else {
// just get all DBs in self host
await addDbs(couchUrl)
}
return dbs
}

28
packages/backend-core/src/objectStore/index.js
View File

@ -206,6 +206,34 @@ exports.retrieveToTmp = async (bucketName, filepath) => {
return outputPath
}
/**
* Delete a single file.
*/
exports.deleteFile = async (bucketName, filepath) => {
const objectStore = exports.ObjectStore(bucketName)
await exports.makeSureBucketExists(objectStore, bucketName)
const params = {
Bucket: bucketName,
Key: filepath,
}
return objectStore.deleteObject(params)
}
exports.deleteFiles = async (bucketName, filepaths) => {
const objectStore = exports.ObjectStore(bucketName)
await exports.makeSureBucketExists(objectStore, bucketName)
const params = {
Bucket: bucketName,
Delete: {
Objects: filepaths.map(path => ({ Key: path })),
},
}
return objectStore.deleteObjects(params).promise()
}
/**
* Delete a path, including everything within.
*/
exports.deleteFolder = async (bucketName, folder) => {
bucketName = sanitizeBucket(bucketName)
folder = sanitizeKey(folder)

2
packages/bbui/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "1.0.27-alpha.7",
"version": "1.0.27-alpha.11",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",

6
packages/bbui/yarn.lock
View File

@ -2076,9 +2076,9 @@ postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.27:
supports-color "^6.1.0"
postcss@^8.2.9:
version "8.2.10"
resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.2.10.tgz#ca7a042aa8aff494b334d0ff3e9e77079f6f702b"
integrity sha512-b/h7CPV7QEdrqIxtAf2j31U5ef05uBDuvoXv6L51Q4rcS1jdlXAVKJv+atCFdUXYl9dyTHGyoMzIepwowRJjFw==
version "8.2.13"
resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.2.13.tgz#dbe043e26e3c068e45113b1ed6375d2d37e2129f"
integrity sha512-FCE5xLH+hjbzRdpbRb1IMCvPv9yZx2QnDarBEYSN0N0HYk+TcXsEhwdFcFb+SRWOKzKGErhIEbBK2ogyLdTtfQ==
dependencies:
colorette "^1.2.2"
nanoid "^3.1.22"

8
packages/builder/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "1.0.27-alpha.7",
"version": "1.0.27-alpha.11",
"license": "GPL-3.0",
"private": true,
"scripts": {
@ -65,10 +65,10 @@
}
},
"dependencies": {
"@budibase/bbui": "^1.0.27-alpha.7",
"@budibase/client": "^1.0.27-alpha.7",
"@budibase/bbui": "^1.0.27-alpha.11",
"@budibase/client": "^1.0.27-alpha.11",
"@budibase/colorpicker": "1.1.2",
"@budibase/string-templates": "^1.0.27-alpha.7",
"@budibase/string-templates": "^1.0.27-alpha.11",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",

4
packages/builder/src/builderStore/datasource.js
View File

@ -23,10 +23,10 @@ function prepareData(config) {
return datasource
}
export async function saveDatasource(config) {
export async function saveDatasource(config, skipFetch = false) {
const datasource = prepareData(config)
// Create datasource
const resp = await datasources.save(datasource, datasource.plus)
const resp = await datasources.save(datasource, !skipFetch && datasource.plus)
// update the tables incase data source plus
await tables.fetch()

2
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte
View File

@ -199,7 +199,6 @@
<Body>
Tell budibase how your tables are related to get even more smart features.
</Body>
{/if}
{#if relationshipInfo && relationshipInfo.length > 0}
<Table
on:click={({ detail }) => openRelationshipModal(detail.from, detail.to)}
@ -212,6 +211,7 @@
{:else}
<Body size="S"><i>No relationships configured.</i></Body>
{/if}
{/if}
<style>
.query-header {

17
packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte
View File

@ -5,22 +5,28 @@
import { IntegrationNames } from "constants/backend"
import cloneDeep from "lodash/cloneDeepWith"
import { saveDatasource as save } from "builderStore/datasource"
import { onMount } from "svelte"
export let integration
export let modal
// kill the reference so the input isn't saved
let datasource = cloneDeep(integration)
let skipFetch = false
async function saveDatasource() {
try {
const resp = await save(datasource)
const resp = await save(datasource, skipFetch)
$goto(`./datasource/${resp._id}`)
notifications.success(`Datasource updated successfully.`)
} catch (err) {
notifications.error(`Error saving datasource: ${err}`)
}
}
onMount(() => {
skipFetch = false
})
</script>
<ModalContent
@ -28,9 +34,16 @@
onConfirm={() => saveDatasource()}
onCancel={() => modal.show()}
confirmText={datasource.plus
? "Fetch tables from database"
? "Save and fetch tables"
: "Save and continue to query"}
cancelText="Back"
showSecondaryButton={datasource.plus}
secondaryButtonText={datasource.plus ? "Skip table fetch" : undefined}
secondaryAction={() => {
skipFetch = true
saveDatasource()
return true
}}
size="L"
>
<Layout noPadding>

2
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "1.0.27-alpha.7",
"version": "1.0.27-alpha.11",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {

6
packages/client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "1.0.27-alpha.7",
"version": "1.0.27-alpha.11",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "^1.0.27-alpha.7",
"@budibase/bbui": "^1.0.27-alpha.11",
"@budibase/standard-components": "^0.9.139",
"@budibase/string-templates": "^1.0.27-alpha.7",
"@budibase/string-templates": "^1.0.27-alpha.11",
"regexparam": "^1.3.0",
"shortid": "^2.2.15",
"svelte-spa-router": "^3.0.5"

8
packages/server/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "1.0.27-alpha.7",
"version": "1.0.27-alpha.11",
"description": "Budibase Web Server",
"main": "src/index.ts",
"repository": {
@ -70,9 +70,9 @@
"license": "GPL-3.0",
"dependencies": {
"@apidevtools/swagger-parser": "^10.0.3",
"@budibase/backend-core": "^1.0.27-alpha.7",
"@budibase/client": "^1.0.27-alpha.7",
"@budibase/string-templates": "^1.0.27-alpha.7",
"@budibase/backend-core": "^1.0.27-alpha.11",
"@budibase/client": "^1.0.27-alpha.11",
"@budibase/string-templates": "^1.0.27-alpha.11",
"@bull-board/api": "^3.7.0",
"@bull-board/koa": "^3.7.0",
"@elastic/elasticsearch": "7.10.0",

34
packages/server/src/api/controllers/datasource.js
View File

@ -10,6 +10,7 @@ const {
const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
const { integrations } = require("../../integrations")
const { getDatasourceAndQuery } = require("./row/utils")
const { invalidateDynamicVariables } = require("../../threads/utils")
exports.fetch = async function (ctx) {
const database = new CouchDB(ctx.appId)
@ -57,10 +58,43 @@ exports.buildSchemaFromDb = async function (ctx) {
ctx.body = response
}
/**
* Check for variables that have been updated or removed and invalidate them.
*/
const invalidateVariables = async (existingDatasource, updatedDatasource) => {
const existingVariables = existingDatasource.config.dynamicVariables
const updatedVariables = updatedDatasource.config.dynamicVariables
const toInvalidate = []
if (!existingVariables) {
return
}
if (!updatedVariables) {
// invalidate all
toInvalidate.push(...existingVariables)
} else {
// invaldate changed / removed
existingVariables.forEach(existing => {
const unchanged = updatedVariables.find(
updated =>
existing.name === updated.name &&
existing.queryId === updated.queryId &&
existing.value === updated.value
)
if (!unchanged) {
toInvalidate.push(existing)
}
})
}
await invalidateDynamicVariables(toInvalidate)
}
exports.update = async function (ctx) {
const db = new CouchDB(ctx.appId)
const datasourceId = ctx.params.datasourceId
let datasource = await db.get(datasourceId)
await invalidateVariables(datasource, ctx.request.body)
datasource = { ...datasource, ...ctx.request.body }
const response = await db.put(datasource)

19
packages/server/src/api/controllers/row/internal.js
View File

@ -11,6 +11,7 @@ const {
inputProcessing,
outputProcessing,
processAutoColumn,
cleanupAttachments,
} = require("../../../utilities/rowProcessor")
const { FieldTypes } = require("../../../constants")
const { isEqual } = require("lodash")
@ -25,6 +26,7 @@ const {
getFromDesignDoc,
getFromMemoryDoc,
} = require("../view/utils")
const { cloneDeep } = require("lodash/fp")
const CALCULATION_TYPES = {
SUM: "sum",
@ -109,14 +111,14 @@ exports.patch = async ctx => {
const inputs = ctx.request.body
const tableId = inputs.tableId
const isUserTable = tableId === InternalTables.USER_METADATA
let dbRow
let oldRow
try {
dbRow = await db.get(inputs._id)
oldRow = await db.get(inputs._id)
} catch (err) {
if (isUserTable) {
// don't include the rev, it'll be the global rev
// this time
dbRow = {
oldRow = {
_id: inputs._id,
}
} else {
@ -125,13 +127,14 @@ exports.patch = async ctx => {
}
let dbTable = await db.get(tableId)
// need to build up full patch fields before coerce
let combinedRow = cloneDeep(oldRow)
for (let key of Object.keys(inputs)) {
if (!dbTable.schema[key]) continue
dbRow[key] = inputs[key]
combinedRow[key] = inputs[key]
}
// this returns the table and row incase they have been updated
let { table, row } = inputProcessing(ctx.user, dbTable, dbRow)
let { table, row } = inputProcessing(ctx.user, dbTable, combinedRow)
const validateResult = await validate({
row,
table,
@ -149,6 +152,8 @@ exports.patch = async ctx => {
tableId: row.tableId,
table,
})
// check if any attachments removed
await cleanupAttachments(appId, table, { oldRow, row })
if (isUserTable) {
// the row has been updated, need to put it into the ctx
@ -295,6 +300,8 @@ exports.destroy = async function (ctx) {
row,
tableId: row.tableId,
})
// remove any attachments that were on the row from object storage
await cleanupAttachments(appId, table, { row })
let response
if (ctx.params.tableId === InternalTables.USER_METADATA) {
@ -341,6 +348,8 @@ exports.bulkDestroy = async ctx => {
} else {
await db.bulkDocs(rows.map(row => ({ ...row, _deleted: true })))
}
// remove any attachments that were on the rows from object storage
await cleanupAttachments(appId, table, { rows })
await Promise.all(updates)
return { response: { ok: true }, rows }
}

45
packages/server/src/api/routes/tests/datasource.spec.js
View File

@ -4,6 +4,7 @@ let setup = require("./utilities")
let { basicDatasource } = setup.structures
let { checkBuilderEndpoint } = require("./utilities/TestFunctions")
const pg = require("pg")
const { checkCacheForDynamicVariable } = require("../../../threads/utils")
describe("/datasources", () => {
let request = setup.getRequest()
@ -31,6 +32,50 @@ describe("/datasources", () => {
})
})
describe("update", () => {
it("should update an existing datasource", async () => {
datasource.name = "Updated Test"
const res = await request
.put(`/api/datasources/${datasource._id}`)
.send(datasource)
.set(config.defaultHeaders())
.expect('Content-Type', /json/)
.expect(200)
expect(res.body.datasource.name).toEqual("Updated Test")
expect(res.body.errors).toBeUndefined()
})
describe("dynamic variables", () => {
async function preview(datasource, fields) {
return config.previewQuery(request, config, datasource, fields)
}
it("should invalidate changed or removed variables", async () => {
const { datasource, query } = await config.dynamicVariableDatasource()
// preview once to cache variables
await preview(datasource, { path: "www.test.com", queryString: "test={{ variable3 }}" })
// check variables in cache
let contents = await checkCacheForDynamicVariable(query._id, "variable3")
expect(contents.rows.length).toEqual(1)
// update the datasource to remove the variables
datasource.config.dynamicVariables = []
const res = await request
.put(`/api/datasources/${datasource._id}`)
.send(datasource)
.set(config.defaultHeaders())
.expect('Content-Type', /json/)
.expect(200)
expect(res.body.errors).toBeUndefined()
// check variables no longer in cache
contents = await checkCacheForDynamicVariable(query._id, "variable3")
expect(contents).toBe(null)
})
})
})
describe("fetch", () => {
it("returns all the datasources from the server", async () => {
const res = await request

67
packages/server/src/api/routes/tests/query.spec.js
View File

@ -229,52 +229,14 @@ describe("/queries", () => {
})
})
describe("test variables", () => {
async function restDatasource(cfg) {
return await config.createDatasource({
datasource: {
...basicDatasource().datasource,
source: "REST",
config: cfg || {},
},
})
}
async function dynamicVariableDatasource() {
const datasource = await restDatasource()
const basedOnQuery = await config.createQuery({
...basicQuery(datasource._id),
fields: {
path: "www.google.com",
},
})
await config.updateDatasource({
...datasource,
config: {
dynamicVariables: [
{ queryId: basedOnQuery._id, name: "variable3", value: "{{ data.0.[value] }}" }
]
}
})
return { datasource, query: basedOnQuery }
}
describe("variables", () => {
async function preview(datasource, fields) {
return await request
.post(`/api/queries/preview`)
.send({
datasourceId: datasource._id,
parameters: {},
fields,
queryVerb: "read",
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
return config.previewQuery(request, config, datasource, fields)
}
it("should work with static variables", async () => {
const datasource = await restDatasource({
const datasource = await config.restDatasource({
staticVariables: {
variable: "google",
variable2: "1",
@ -290,7 +252,7 @@ describe("/queries", () => {
})
it("should work with dynamic variables", async () => {
const { datasource } = await dynamicVariableDatasource()
const { datasource } = await config.dynamicVariableDatasource()
const res = await preview(datasource, {
path: "www.google.com",
queryString: "test={{ variable3 }}",
@ -300,7 +262,7 @@ describe("/queries", () => {
})
it("check that it automatically retries on fail with cached dynamics", async () => {
const { datasource, query: base } = await dynamicVariableDatasource()
const { datasource, query: base } = await config.dynamicVariableDatasource()
// preview once to cache
await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
// check its in cache
@ -313,5 +275,24 @@ describe("/queries", () => {
expect(res.body.schemaFields).toEqual(["fails", "url", "opts"])
expect(res.body.rows[0].fails).toEqual(1)
})
it("deletes variables when linked query is deleted", async () => {
const { datasource, query: base } = await config.dynamicVariableDatasource()
// preview once to cache
await preview(datasource, { path: "www.google.com", queryString: "test={{ variable3 }}" })
// check its in cache
let contents = await checkCacheForDynamicVariable(base._id, "variable3")
expect(contents.rows.length).toEqual(1)
// delete the query
await request
.delete(`/api/queries/${base._id}/${base._rev}`)
.set(config.defaultHeaders())
.expect(200)
// check variables no longer in cache
contents = await checkCacheForDynamicVariable(base._id, "variable3")
expect(contents).toBe(null)
})
})
})

2
packages/server/src/integrations/base/sqlTable.ts
View File

@ -36,7 +36,7 @@ function generateSchema(
case FieldTypes.STRING:
case FieldTypes.OPTIONS:
case FieldTypes.LONGFORM:
schema.string(key)
schema.text(key)
break
case FieldTypes.NUMBER:
// if meta is specified then this is a junction table entry

47
packages/server/src/tests/utilities/TestConfiguration.js
View File

@ -326,6 +326,53 @@ class TestConfiguration {
return this.datasource
}
async restDatasource(cfg) {
return this.createDatasource({
datasource: {
...basicDatasource().datasource,
source: "REST",
config: cfg || {},
},
})
}
async dynamicVariableDatasource() {
let datasource = await this.restDatasource()
const basedOnQuery = await this.createQuery({
...basicQuery(datasource._id),
fields: {
path: "www.google.com",
},
})
datasource = await this.updateDatasource({
...datasource,
config: {
dynamicVariables: [
{
queryId: basedOnQuery._id,
name: "variable3",
value: "{{ data.0.[value] }}",
},
],
},
})
return { datasource, query: basedOnQuery }
}
async previewQuery(request, config, datasource, fields) {
return request
.post(`/api/queries/preview`)
.send({
datasourceId: datasource._id,
parameters: {},
fields,
queryVerb: "read",
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
}
async createQuery(config = null) {
if (!this.datasource && !config) {
throw "No data source created for query."

5
packages/server/src/threads/query.js
View File

@ -46,7 +46,10 @@ class QueryRunner {
// transform as required
if (transformer) {
const runner = new ScriptRunner(transformer, { data: rows })
const runner = new ScriptRunner(transformer, {
data: rows,
params: parameters,
})
rows = runner.execute()
}

2
packages/server/src/utilities/fileSystem/utilities.js
View File

@ -2,6 +2,7 @@ const {
ObjectStore,
makeSureBucketExists,
upload,
deleteFiles,
streamUpload,
retrieve,
retrieveToTmp,
@ -28,3 +29,4 @@ exports.retrieveToTmp = retrieveToTmp
exports.deleteFolder = deleteFolder
exports.uploadDirectory = uploadDirectory
exports.downloadTarball = downloadTarball
exports.deleteFiles = deleteFiles

63
packages/server/src/utilities/rowProcessor/index.js
View File

@ -3,6 +3,10 @@ const { cloneDeep } = require("lodash/fp")
const { FieldTypes, AutoFieldSubTypes } = require("../../constants")
const { attachmentsRelativeURL } = require("../index")
const { processFormulas } = require("./utils")
const { deleteFiles } = require("../../utilities/fileSystem/utilities")
const { ObjectStoreBuckets } = require("../../constants")
const { isProdAppID, getDeployedAppID, dbExists } = require("@budibase/auth/db")
const CouchDB = require("../../db")
const BASE_AUTO_ID = 1
@ -95,6 +99,23 @@ const TYPE_TRANSFORM_MAP = {
},
}
/**
* Given the old state of the row and the new one after an update, this will
* find the keys that have been removed in the updated row.
*/
function getRemovedAttachmentKeys(oldRow, row, attachmentKey) {
if (!oldRow[attachmentKey]) {
return []
}
const oldKeys = oldRow[attachmentKey].map(attachment => attachment.key)
// no attachments in new row, all removed
if (!row[attachmentKey]) {
return oldKeys
}
const newKeys = row[attachmentKey].map(attachment => attachment.key)
return oldKeys.filter(key => newKeys.indexOf(key) === -1)
}
/**
* This will update any auto columns that are found on the row/table with the correct information based on
* time now and the current logged in user making the request.
@ -272,3 +293,45 @@ exports.outputProcessing = async (
}
return wasArray ? enriched : enriched[0]
}
/**
* Clean up any attachments that were attached to a row.
* @param {string} appId The ID of the app from which a row is being deleted.
* @param {object} table The table from which a row is being removed.
* @param {any} row optional - the row being removed.
* @param {any} rows optional - if multiple rows being deleted can do this in bulk.
* @param {any} oldRow optional - if updating a row this will determine the difference.
* @return {Promise<void>} When all attachments have been removed this will return.
*/
exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => {
if (!isProdAppID(appId)) {
const prodAppId = getDeployedAppID(appId)
// if prod exists, then don't allow deleting
const exists = await dbExists(CouchDB, prodAppId)
if (exists) {
return
}
}
let files = []
function addFiles(row, key) {
if (row[key]) {
files = files.concat(row[key].map(attachment => attachment.key))
}
}
for (let [key, schema] of Object.entries(table.schema)) {
if (schema.type !== FieldTypes.ATTACHMENT) {
continue
}
// if updating, need to manage the differences
if (oldRow && row) {
files = files.concat(getRemovedAttachmentKeys(oldRow, row, key))
} else if (row) {
addFiles(row, key)
} else if (rows) {
rows.forEach(row => addFiles(row, key))
}
}
if (files.length > 0) {
return deleteFiles(ObjectStoreBuckets.APPS, files)
}
}

2
packages/string-templates/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "1.0.27-alpha.7",
"version": "1.0.27-alpha.11",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

6
packages/worker/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "1.0.27-alpha.7",
"version": "1.0.27-alpha.11",
"description": "Budibase background service",
"main": "src/index.js",
"repository": {
@ -29,8 +29,8 @@
"author": "Budibase",
"license": "GPL-3.0",
"dependencies": {
"@budibase/backend-core": "^1.0.27-alpha.7",
"@budibase/string-templates": "^1.0.27-alpha.7",
"@budibase/backend-core": "^1.0.27-alpha.11",
"@budibase/string-templates": "^1.0.27-alpha.11",
"@koa/router": "^8.0.0",
"@sentry/node": "^6.0.0",
"@techpass/passport-openidconnect": "^0.3.0",