MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' of github.com:Budibase/budibase into develop

This commit is contained in:
Andrew Kingston 2023-01-31 15:50:00 +00:00
parent ca399568e2 6cde4aa67b
commit c280e44b26
30 changed files with 454 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.2.12-alpha.46",
"version": "2.2.12-alpha.49",
"npmClient": "yarn",
"packages": [
"packages/*"

4
packages/backend-core/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
"version": "2.2.12-alpha.46",
"version": "2.2.12-alpha.49",
"description": "Budibase backend core libraries used in server and worker",
"main": "dist/src/index.js",
"types": "dist/src/index.d.ts",
@ -23,7 +23,7 @@
},
"dependencies": {
"@budibase/nano": "10.1.1",
"@budibase/types": "2.2.12-alpha.46",
"@budibase/types": "2.2.12-alpha.49",
"@shopify/jest-koa-mocks": "5.0.1",
"@techpass/passport-openidconnect": "0.3.2",
"aws-cloudfront-sign": "2.2.0",

4
packages/bbui/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "2.2.12-alpha.46",
"version": "2.2.12-alpha.49",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",
@ -38,7 +38,7 @@
],
"dependencies": {
"@adobe/spectrum-css-workflow-icons": "1.2.1",
"@budibase/string-templates": "2.2.12-alpha.46",
"@budibase/string-templates": "2.2.12-alpha.49",
"@spectrum-css/accordion": "3.0.24",
"@spectrum-css/actionbutton": "1.0.1",
"@spectrum-css/actiongroup": "1.0.1",

2
packages/bbui/src/Form/Core/EnvDropdown.svelte
View File

@ -134,7 +134,7 @@
on:blur={onBlur}
on:focus={onFocus}
on:input={onInput}
{type}
type={hbsValue.length ? "text" : type}
style={align ? `text-align: ${align};` : ""}
class="spectrum-Textfield-input"
inputmode={type === "number" ? "decimal" : "text"}

10
packages/builder/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "2.2.12-alpha.46",
"version": "2.2.12-alpha.49",
"license": "GPL-3.0",
"private": true,
"scripts": {
@ -71,10 +71,10 @@
}
},
"dependencies": {
"@budibase/bbui": "2.2.12-alpha.46",
"@budibase/client": "2.2.12-alpha.46",
"@budibase/frontend-core": "2.2.12-alpha.46",
"@budibase/string-templates": "2.2.12-alpha.46",
"@budibase/bbui": "2.2.12-alpha.49",
"@budibase/client": "2.2.12-alpha.49",
"@budibase/frontend-core": "2.2.12-alpha.49",
"@budibase/string-templates": "2.2.12-alpha.49",
"@sentry/browser": "5.19.1",
"@spectrum-css/accordion": "^3.0.24",
"@spectrum-css/page": "^3.0.1",

36
packages/builder/src/components/backend/DataTable/DataTable.svelte
View File

@ -39,6 +39,23 @@
$: showError($fetch.error)
$: id, (filters = null)
let appliedFilter
let rawFilter
let appliedSort
let selectedRows = []
$: enrichedSchema,
() => {
appliedFilter = null
rawFilter = null
appliedSort = null
selectedRows = []
}
$: if (Number.isInteger($fetch.pageNumber)) {
selectedRows = []
}
const showError = error => {
if (error) {
notifications.error(error?.message || "Unable to fetch data.")
@ -95,11 +112,15 @@
}
// Fetch data whenever sorting option changes
const onSort = e => {
fetch.update({
const onSort = async e => {
const sort = {
sortColumn: e.detail.column,
sortOrder: e.detail.order,
})
}
await fetch.update(sort)
appliedSort = { ...sort }
appliedSort.sortOrder = appliedSort.sortOrder.toLowerCase()
selectedRows = []
}
// Fetch data whenever filters change
@ -108,16 +129,19 @@
fetch.update({
filter: filters,
})
appliedFilter = e.detail
}
// Fetch data whenever schema changes
const onUpdateColumns = () => {
selectedRows = []
fetch.refresh()
}
// Fetch data whenever rows are modified. Unfortunately we have to lose
// our pagination place, as our bookmarks will have shifted.
const onUpdateRows = () => {
selectedRows = []
fetch.refresh()
}
@ -142,6 +166,9 @@
disableSorting
on:updatecolumns={onUpdateColumns}
on:updaterows={onUpdateRows}
on:selectionUpdated={e => {
selectedRows = e.detail
}}
customPlaceholder
>
<div class="buttons">
@ -183,6 +210,9 @@
<ExportButton
disabled={!hasRows || !hasCols}
view={$tables.selected?._id}
filters={appliedFilter}
sorting={appliedSort}
{selectedRows}
/>
{#key id}
<TableFilterButton

4
packages/builder/src/components/backend/DataTable/Table.svelte
View File

@ -16,6 +16,7 @@
UNSORTABLE_TYPES,
} from "constants"
import RoleCell from "./cells/RoleCell.svelte"
import { createEventDispatcher } from "svelte"
export let schema = {}
export let data = []
@ -28,6 +29,8 @@
export let disableSorting = false
export let customPlaceholder = false
const dispatch = createEventDispatcher()
let selectedRows = []
let editableColumn
let editableRow
@ -36,6 +39,7 @@
let customRenderers = []
let confirmDelete
$: selectedRows, dispatch("selectionUpdated", selectedRows)
$: isUsersTable = tableId === TableNames.USERS
$: data && resetSelectedRows()
$: editRowComponent = isUsersTable ? CreateEditUser : CreateEditRow

5
packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte
View File

@ -3,7 +3,10 @@
import ExportModal from "../modals/ExportModal.svelte"
export let view
export let filters
export let sorting
export let disabled = false
export let selectedRows
let modal
</script>
@ -18,5 +21,5 @@
Export
</ActionButton>
<Modal bind:this={modal}>
<ExportModal {view} />
<ExportModal {view} {filters} {sorting} {selectedRows} />
</Modal>

145
packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte
View File

@ -1,7 +1,14 @@
<script>
import { Select, ModalContent, notifications } from "@budibase/bbui"
import {
Select,
ModalContent,
notifications,
Body,
Table,
} from "@budibase/bbui"
import download from "downloadjs"
import { API } from "api"
import { Constants, LuceneUtils } from "@budibase/frontend-core"
const FORMATS = [
{
@ -19,8 +26,71 @@
]
export let view
export let filters
export let sorting
export let selectedRows = []
let exportFormat = FORMATS[0].key
let filterLookup
$: luceneFilter = LuceneUtils.buildLuceneQuery(filters)
$: exportOpDisplay = buildExportOpDisplay(sorting, filterDisplay, filters)
const buildFilterLookup = () => {
return Object.keys(Constants.OperatorOptions).reduce((acc, key) => {
const op = Constants.OperatorOptions[key]
acc[op.value] = op.label
return acc
}, {})
}
filterLookup = buildFilterLookup()
const filterDisplay = () => {
if (!filters) {
return []
}
return filters.map(filter => {
let newFieldName = filter.field + ""
const parts = newFieldName.split(":")
parts.shift()
newFieldName = parts.join(":")
return {
Field: newFieldName,
Operation: filterLookup[filter.operator],
"Field Value": filter.value || "",
}
})
}
const buildExportOpDisplay = (sorting, filterDisplay) => {
let filterDisplayConfig = filterDisplay()
if (sorting) {
filterDisplayConfig = [
...filterDisplayConfig,
{
Field: sorting.sortColumn,
Operation: "Order By",
"Field Value": sorting.sortOrder,
},
]
}
return filterDisplayConfig
}
const displaySchema = {
Field: {
type: "string",
fieldName: "Field",
},
Operation: {
type: "string",
fieldName: "Operation",
},
"Field Value": {
type: "string",
fieldName: "Value",
},
}
async function exportView() {
try {
@ -33,9 +103,74 @@
notifications.error(`Unable to export ${exportFormat.toUpperCase()} data`)
}
}
async function exportRows() {
if (selectedRows?.length) {
const data = await API.exportRows({
tableId: view,
rows: selectedRows.map(row => row._id),
format: exportFormat,
})
download(data, `export.${exportFormat}`)
} else if (filters || sorting) {
const data = await API.exportRows({
tableId: view,
format: exportFormat,
search: {
query: luceneFilter,
sort: sorting?.sortColumn,
sortOrder: sorting?.sortOrder,
paginate: false,
},
})
download(data, `export.${exportFormat}`)
} else {
await exportView()
}
}
</script>
<ModalContent title="Export Data" confirmText="Export" onConfirm={exportView}>
<ModalContent
title="Export Data"
confirmText="Export"
onConfirm={exportRows}
size={filters?.length || sorting ? "M" : "S"}
>
{#if selectedRows?.length}
<Body size="S">
<strong>{selectedRows?.length}</strong>
{`row${selectedRows?.length > 1 ? "s" : ""} will be exported`}
</Body>
{:else if filters || (sorting?.sortOrder && sorting?.sortColumn)}
<Body size="S">
{#if !filters}
Exporting <strong>all</strong> rows
{:else}
Filters applied
{/if}
</Body>
<div class="table-wrap">
<Table
schema={displaySchema}
data={exportOpDisplay}
{filters}
loading={false}
rowCount={filters?.length + 1}
disableSorting={true}
allowSelectRows={false}
allowEditRows={false}
allowEditColumns={false}
quiet={true}
compact={true}
/>
</div>
{:else}
<Body size="S">
Exporting <strong>all</strong> rows
</Body>
{/if}
<Select
label="Format"
bind:value={exportFormat}
@ -45,3 +180,9 @@
getOptionValue={x => x.key}
/>
</ModalContent>
<style>
.table-wrap :global(.wrapper) {
max-width: 400px;
}
</style>

1
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationModal.svelte
View File

@ -248,6 +248,7 @@
/>
<EnvDropdown
label="Password"
type="password"
bind:value={form.basic.password}
on:change={onFieldChange}
on:blur={() => (blurred.basic.password = true)}

8
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "2.2.12-alpha.46",
"version": "2.2.12-alpha.49",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {
@ -26,9 +26,9 @@
"outputPath": "build"
},
"dependencies": {
"@budibase/backend-core": "2.2.12-alpha.46",
"@budibase/string-templates": "2.2.12-alpha.46",
"@budibase/types": "2.2.12-alpha.46",
"@budibase/backend-core": "2.2.12-alpha.49",
"@budibase/string-templates": "2.2.12-alpha.49",
"@budibase/types": "2.2.12-alpha.49",
"axios": "0.21.2",
"chalk": "4.1.0",
"cli-progress": "3.11.2",

8
packages/client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "2.2.12-alpha.46",
"version": "2.2.12-alpha.49",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "2.2.12-alpha.46",
"@budibase/frontend-core": "2.2.12-alpha.46",
"@budibase/string-templates": "2.2.12-alpha.46",
"@budibase/bbui": "2.2.12-alpha.49",
"@budibase/frontend-core": "2.2.12-alpha.49",
"@budibase/string-templates": "2.2.12-alpha.49",
"@spectrum-css/button": "^3.0.3",
"@spectrum-css/card": "^3.0.3",
"@spectrum-css/divider": "^1.0.3",

4
packages/frontend-core/package.json
View File

@ -1,12 +1,12 @@
{
"name": "@budibase/frontend-core",
"version": "2.2.12-alpha.46",
"version": "2.2.12-alpha.49",
"description": "Budibase frontend core libraries used in builder and client",
"author": "Budibase",
"license": "MPL-2.0",
"svelte": "src/index.js",
"dependencies": {
"@budibase/bbui": "2.2.12-alpha.46",
"@budibase/bbui": "2.2.12-alpha.49",
"lodash": "^4.17.21",
"svelte": "^3.46.2"
}

3
packages/frontend-core/src/api/rows.js
View File

@ -67,12 +67,13 @@ export const buildRowEndpoints = API => ({
* @param format the format to export (csv or json)
* @param columns which columns to export (all if undefined)
*/
exportRows: async ({ tableId, rows, format, columns }) => {
exportRows: async ({ tableId, rows, format, columns, search }) => {
return await API.post({
url: `/api/${tableId}/rows/exportRows?format=${format}`,
body: {
rows,
columns,
...search,
},
parseResponse: async response => {
return await response.text()

2
packages/sdk/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/sdk",
"version": "2.2.12-alpha.46",
"version": "2.2.12-alpha.49",
"description": "Budibase Public API SDK",
"author": "Budibase",
"license": "MPL-2.0",

12
packages/server/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "2.2.12-alpha.46",
"version": "2.2.12-alpha.49",
"description": "Budibase Web Server",
"main": "src/index.ts",
"repository": {
@ -43,11 +43,11 @@
"license": "GPL-3.0",
"dependencies": {
"@apidevtools/swagger-parser": "10.0.3",
"@budibase/backend-core": "2.2.12-alpha.46",
"@budibase/client": "2.2.12-alpha.46",
"@budibase/pro": "2.2.12-alpha.46",
"@budibase/string-templates": "2.2.12-alpha.46",
"@budibase/types": "2.2.12-alpha.46",
"@budibase/backend-core": "2.2.12-alpha.49",
"@budibase/client": "2.2.12-alpha.49",
"@budibase/pro": "2.2.12-alpha.48",
"@budibase/string-templates": "2.2.12-alpha.49",
"@budibase/types": "2.2.12-alpha.49",
"@bull-board/api": "3.7.0",
"@bull-board/koa": "3.9.4",
"@elastic/elasticsearch": "7.10.0",

31
packages/server/src/api/controllers/row/external.ts
View File

@ -21,6 +21,8 @@ import {
} from "@budibase/types"
import sdk from "../../../sdk"
const { cleanExportRows } = require("./utils")
export async function handleRequest(
operation: Operation,
tableId: string,
@ -100,7 +102,7 @@ export async function destroy(ctx: BBContext) {
export async function bulkDestroy(ctx: BBContext) {
const { rows } = ctx.request.body
const tableId = ctx.params.tableId
let promises = []
let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
for (let row of rows) {
promises.push(
handleRequest(Operation.DELETE, tableId, {
@ -186,20 +188,24 @@ export async function exportRows(ctx: BBContext) {
if (!datasource || !datasource.entities) {
ctx.throw(400, "Datasource has not been configured for plus API.")
}
ctx.request.body = {
query: {
oneOf: {
_id: ctx.request.body.rows.map(
(row: string) => JSON.parse(decodeURI(row))[0]
),
if (ctx.request.body.rows) {
ctx.request.body = {
query: {
oneOf: {
_id: ctx.request.body.rows.map(
(row: string) => JSON.parse(decodeURI(row))[0]
),
},
},
},
}
}
let result = await search(ctx)
let rows: Row[] = []
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.rows.length; i++) {
rows[i] = {}
@ -211,14 +217,19 @@ export async function exportRows(ctx: BBContext) {
rows = result.rows
}
let headers = Object.keys(rows[0])
// @ts-ignore
let schema = datasource.entities[tableName].schema
let exportRows = cleanExportRows(rows, schema, format, columns)
let headers = Object.keys(schema)
// @ts-ignore
const exporter = exporters[format]
const filename = `export.${format}`
// send down the file
ctx.attachment(filename)
return apiFileReturn(exporter(headers, rows))
return apiFileReturn(exporter(headers, exportRows))
}
export async function fetchEnrichedRow(ctx: BBContext) {

47
packages/server/src/api/controllers/row/internal.ts
View File

@ -27,7 +27,7 @@ import {
import { cloneDeep } from "lodash/fp"
import { context, db as dbCore } from "@budibase/backend-core"
import { finaliseRow, updateRelatedFormula } from "./staticFormula"
import { csv, json, jsonWithSchema, Format, isFormat } from "../view/exporters"
import { csv, json, jsonWithSchema, Format } from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
import {
Ctx,
@ -38,6 +38,8 @@ import {
Table,
} from "@budibase/types"
const { cleanExportRows } = require("./utils")
const CALCULATION_TYPES = {
SUM: "sum",
COUNT: "count",
@ -357,6 +359,14 @@ export async function search(ctx: Ctx) {
params.version = ctx.version
params.tableId = tableId
let table
if (params.sort && !params.sortType) {
table = await db.get(tableId)
const schema = table.schema
const sortField = schema[params.sort]
params.sortType = sortField.type == "number" ? "number" : "string"
}
let response
if (paginate) {
response = await paginatedSearch(query, params)
@ -370,7 +380,7 @@ export async function search(ctx: Ctx) {
if (tableId === InternalTables.USER_METADATA) {
response.rows = await getGlobalUsersFromMetadata(response.rows)
}
const table = await db.get(tableId)
table = table || (await db.get(tableId))
response.rows = await outputProcessing(table, response.rows)
}
@ -389,16 +399,25 @@ export async function exportRows(ctx: Ctx) {
const table = await db.get(ctx.params.tableId)
const rowIds = ctx.request.body.rows
let format = ctx.query.format
const { columns } = ctx.request.body
let response = (
await db.allDocs({
include_docs: true,
keys: rowIds,
})
).rows.map(row => row.doc)
const { columns, query } = ctx.request.body
let result
if (rowIds) {
let response = (
await db.allDocs({
include_docs: true,
keys: rowIds,
})
).rows.map(row => row.doc)
result = await outputProcessing(table, response)
} else if (query) {
let searchResponse = await exports.search(ctx)
result = searchResponse.rows
}
let result = (await outputProcessing(table, response)) as Row[]
let rows: Row[] = []
let schema = table.schema
// Filter data to only specified columns if required
if (columns && columns.length) {
@ -412,12 +431,16 @@ export async function exportRows(ctx: Ctx) {
rows = result
}
let exportRows = cleanExportRows(rows, schema, format, columns)
if (format === Format.CSV) {
ctx.attachment("export.csv")
return apiFileReturn(csv(Object.keys(rows[0]), rows))
return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
} else if (format === Format.JSON) {
ctx.attachment("export.json")
return apiFileReturn(json(rows))
return apiFileReturn(json(exportRows))
} else if (format === Format.JSON_WITH_SCHEMA) {
ctx.attachment("export.json")
return apiFileReturn(jsonWithSchema(schema, exportRows))
} else {
throw "Format not recognised"
}

38
packages/server/src/api/controllers/row/utils.ts
View File

@ -7,6 +7,7 @@ import { BBContext, Row, Table } from "@budibase/types"
export { removeKeyNumbering } from "../../../integrations/base/utils"
const validateJs = require("validate.js")
const { cloneDeep } = require("lodash/fp")
import { Format } from "../view/exporters"
import { Ctx } from "@budibase/types"
import sdk from "../../../sdk"
@ -117,3 +118,40 @@ export async function validate({
}
return { valid: Object.keys(errors).length === 0, errors }
}
export function cleanExportRows(
rows: any[],
schema: any,
format: string,
columns: string[]
) {
let cleanRows = [...rows]
const relationships = Object.entries(schema)
.filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
.map(entry => entry[0])
relationships.forEach(column => {
cleanRows.forEach(row => {
delete row[column]
})
delete schema[column]
})
// Intended to avoid 'undefined' in export
if (format === Format.CSV) {
const schemaKeys = Object.keys(schema)
for (let key of schemaKeys) {
if (columns?.length && columns.indexOf(key) > 0) {
continue
}
for (let row of cleanRows) {
if (row[key] == null) {
row[key] = ""
}
}
}
}
return cleanRows
}

32
packages/server/src/api/controllers/view/index.ts
View File

@ -3,7 +3,6 @@ import { apiFileReturn } from "../../../utilities/fileSystem"
import { csv, json, jsonWithSchema, Format, isFormat } from "./exporters"
import { deleteView, getView, getViews, saveView } from "./utils"
import { fetchView } from "../row"
import { FieldTypes } from "../../../constants"
import { context, events } from "@budibase/backend-core"
import { DocumentType } from "../../../db/utils"
import sdk from "../../../sdk"
@ -15,6 +14,7 @@ import {
TableSchema,
View,
} from "@budibase/types"
import { cleanExportRows } from "../row/utils"
const { cloneDeep, isEqual } = require("lodash")
@ -162,39 +162,17 @@ export async function exportView(ctx: BBContext) {
schema = table.schema
}
// remove any relationships
const relationships = Object.entries(schema)
.filter(entry => entry[1].type === FieldTypes.LINK)
.map(entry => entry[0])
// iterate relationship columns and remove from and row and schema
relationships.forEach(column => {
rows.forEach(row => {
delete row[column]
})
delete schema[column]
})
// make sure no "undefined" entries appear in the CSV
if (format === Format.CSV) {
const schemaKeys = Object.keys(schema)
for (let key of schemaKeys) {
for (let row of rows) {
if (row[key] == null) {
row[key] = ""
}
}
}
}
let exportRows = cleanExportRows(rows, schema, format, [])
if (format === Format.CSV) {
ctx.attachment(`${viewName}.csv`)
ctx.body = apiFileReturn(csv(Object.keys(schema), rows))
ctx.body = apiFileReturn(csv(Object.keys(schema), exportRows))
} else if (format === Format.JSON) {
ctx.attachment(`${viewName}.json`)
ctx.body = apiFileReturn(json(rows))
ctx.body = apiFileReturn(json(exportRows))
} else if (format === Format.JSON_WITH_SCHEMA) {
ctx.attachment(`${viewName}.json`)
ctx.body = apiFileReturn(jsonWithSchema(schema, rows))
ctx.body = apiFileReturn(jsonWithSchema(schema, exportRows))
} else {
throw "Format not recognised"
}

41
packages/server/src/definitions/datasource.ts
View File

@ -7,44 +7,3 @@
export interface QueryOptions {
disableReturning?: boolean
}
export enum AuthType {
BASIC = "basic",
BEARER = "bearer",
}
interface AuthConfig {
_id: string
name: string
type: AuthType
config: BasicAuthConfig | BearerAuthConfig
}
export interface BasicAuthConfig {
username: string
password: string
}
export interface BearerAuthConfig {
token: string
}
export interface RestConfig {
url: string
rejectUnauthorized: boolean
defaultHeaders: {
[key: string]: any
}
legacyHttpParser: boolean
authConfigs: AuthConfig[]
staticVariables: {
[key: string]: string
}
dynamicVariables: [
{
name: string
queryId: string
value: string
}
]
}

19
packages/server/src/integrations/rest.ts
View File

@ -6,13 +6,11 @@ import {
IntegrationBase,
PaginationValues,
RestQueryFields as RestQuery,
} from "@budibase/types"
import {
RestConfig,
AuthType,
BasicAuthConfig,
BearerAuthConfig,
} from "../definitions/datasource"
RestAuthType,
RestBasicAuthConfig,
RestBearerAuthConfig,
} from "@budibase/types"
import { get } from "lodash"
import * as https from "https"
import qs from "querystring"
@ -331,14 +329,14 @@ class RestIntegration implements IntegrationBase {
if (authConfig) {
let config
switch (authConfig.type) {
case AuthType.BASIC:
config = authConfig.config as BasicAuthConfig
case RestAuthType.BASIC:
config = authConfig.config as RestBasicAuthConfig
headers.Authorization = `Basic ${Buffer.from(
`${config.username}:${config.password}`
).toString("base64")}`
break
case AuthType.BEARER:
config = authConfig.config as BearerAuthConfig
case RestAuthType.BEARER:
config = authConfig.config as RestBearerAuthConfig
headers.Authorization = `Bearer ${config.token}`
break
}
@ -428,5 +426,4 @@ class RestIntegration implements IntegrationBase {
export default {
schema: SCHEMA,
integration: RestIntegration,
AuthType,
}

6
packages/server/src/integrations/tests/rest.spec.ts
View File

@ -15,6 +15,7 @@ jest.mock("node-fetch", () => {
import fetch from "node-fetch"
import { default as RestIntegration } from "../rest"
import { RestAuthType } from "@budibase/types"
const FormData = require("form-data")
const { URLSearchParams } = require("url")
@ -229,7 +230,7 @@ describe("REST Integration", () => {
const basicAuth = {
_id: "c59c14bd1898a43baa08da68959b24686",
name: "basic-1",
type: RestIntegration.AuthType.BASIC,
type: RestAuthType.BASIC,
config: {
username: "user",
password: "password",
@ -239,7 +240,7 @@ describe("REST Integration", () => {
const bearerAuth = {
_id: "0d91d732f34e4befabeff50b392a8ff3",
name: "bearer-1",
type: RestIntegration.AuthType.BEARER,
type: RestAuthType.BEARER,
config: {
token: "mytoken",
},
@ -581,6 +582,7 @@ describe("REST Integration", () => {
})
await config.integration.read({})
// @ts-ignore
const calls: any = fetch.mock.calls[0]
const url = calls[0]
expect(url).toBe(`${BASE_URL}/`)

62
packages/server/src/sdk/app/datasources/datasources.ts
View File

@ -1,17 +1,19 @@
import { context } from "@budibase/backend-core"
import { processObjectSync, findHBSBlocks } from "@budibase/string-templates"
import { findHBSBlocks, processObjectSync } from "@budibase/string-templates"
import {
Datasource,
DatasourceFieldType,
Integration,
PASSWORD_REPLACEMENT,
RestAuthConfig,
RestAuthType,
RestBasicAuthConfig,
SourceName,
} from "@budibase/types"
import { cloneDeep } from "lodash/fp"
import { getEnvironmentVariables } from "../../utils"
import { getDefinitions } from "../../../integrations"
const ENV_VAR_PREFIX = "env."
const USER_PREFIX = "user"
async function enrichDatasourceWithValues(datasource: Datasource) {
const cloned = cloneDeep(datasource)
@ -47,6 +49,18 @@ export async function getWithEnvVars(datasourceId: string) {
return enrichDatasourceWithValues(datasource)
}
function hasAuthConfigs(datasource: Datasource) {
return datasource.source === SourceName.REST && datasource.config?.authConfigs
}
function useEnvVars(str: any) {
if (typeof str !== "string") {
return false
}
const blocks = findHBSBlocks(str)
return blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
}
export async function removeSecrets(datasources: Datasource[]) {
const definitions = await getDefinitions()
for (let datasource of datasources) {
@ -56,17 +70,24 @@ export async function removeSecrets(datasources: Datasource[]) {
if (datasource.config.auth) {
delete datasource.config.auth
}
// remove passwords
for (let key of Object.keys(datasource.config)) {
if (typeof datasource.config[key] !== "string") {
continue
// specific to REST datasources, contains passwords
if (hasAuthConfigs(datasource)) {
const configs = datasource.config.authConfigs as RestAuthConfig[]
for (let config of configs) {
if (config.type !== RestAuthType.BASIC) {
continue
}
const basic = config.config as RestBasicAuthConfig
if (!useEnvVars(basic.password)) {
basic.password = PASSWORD_REPLACEMENT
}
}
const blocks = findHBSBlocks(datasource.config[key] as string)
const usesEnvVars =
blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
}
// remove general passwords
for (let key of Object.keys(datasource.config)) {
if (
!usesEnvVars &&
schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD
schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD &&
!useEnvVars(datasource.config[key])
) {
datasource.config[key] = PASSWORD_REPLACEMENT
}
@ -84,6 +105,23 @@ export function mergeConfigs(update: Datasource, old: Datasource) {
if (!update.config) {
return update
}
// specific to REST datasources, fix the auth configs again if required
if (hasAuthConfigs(update)) {
const configs = update.config.authConfigs as RestAuthConfig[]
const oldConfigs = old.config?.authConfigs as RestAuthConfig[]
for (let config of configs) {
if (config.type !== RestAuthType.BASIC) {
continue
}
const basic = config.config as RestBasicAuthConfig
const oldBasic = oldConfigs.find(old => old.name === config.name)
?.config as RestBasicAuthConfig
if (basic.password === PASSWORD_REPLACEMENT) {
basic.password = oldBasic.password
}
}
}
// update back to actual passwords for everything else
for (let [key, value] of Object.entries(update.config)) {
if (value !== PASSWORD_REPLACEMENT) {
continue

30
packages/server/yarn.lock
View File

@ -1273,13 +1273,13 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.2.12-alpha.46":
version "2.2.12-alpha.46"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.46.tgz#ac2c8168c4083e34be69e36e1b898f2b4e60a9bf"
integrity sha512-YNXF3/B17CrnOXfv27mzY1zEbBD/j5Nvr6ADJbakw7yrc3G58diLkJBo4kxzeEH4ujG3ftLDGLcbBP+zzoASZQ==
"@budibase/backend-core@2.2.12-alpha.48":
version "2.2.12-alpha.48"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.48.tgz#d211474e8ba57f2aa8d9c7b263cfbe96f7051ab9"
integrity sha512-7UuMHuV7jcaq5Y9fz7PM+YgT5bexuhm2UOoTivAtKOJwUR5Au7lGunVAZqLB5BPp55B9ncD/mQ5fev16vrep0Q==
dependencies:
"@budibase/nano" "10.1.1"
"@budibase/types" "2.2.12-alpha.46"
"@budibase/types" "2.2.12-alpha.48"
"@shopify/jest-koa-mocks" "5.0.1"
"@techpass/passport-openidconnect" "0.3.2"
aws-cloudfront-sign "2.2.0"
@ -1374,13 +1374,13 @@
qs "^6.11.0"
tough-cookie "^4.1.2"
"@budibase/pro@2.2.12-alpha.46":
version "2.2.12-alpha.46"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.46.tgz#bdd89640eea306a2fa7980387696c75b60ea0cc0"
integrity sha512-baiejsUGZfE+VSCj1Pb7sG5MDNqwzUTZ915t+YJ8tGg05TbR+/RaBLGfKkONbl9iaXK9Nl4KgiNytXvZQCZULw==
"@budibase/pro@2.2.12-alpha.48":
version "2.2.12-alpha.48"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.48.tgz#639431a654dbe55c316a94e4220adae76e5f7540"
integrity sha512-QMilBTDN/y5ieTnIjSlovhbmBLfTQV11B3YapJcdtpWKALyGpuZ7eYBY2DIjZtxTx5BE1Ib+BAJZmBxyXlkcHg==
dependencies:
"@budibase/backend-core" "2.2.12-alpha.46"
"@budibase/types" "2.2.12-alpha.46"
"@budibase/backend-core" "2.2.12-alpha.48"
"@budibase/types" "2.2.12-alpha.48"
"@koa/router" "8.0.8"
bull "4.10.1"
joi "17.6.0"
@ -1406,10 +1406,10 @@
svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0"
"@budibase/types@2.2.12-alpha.46":
version "2.2.12-alpha.46"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.46.tgz#ff305f4b5dc505380f1f075a5a54b1794f24175c"
integrity sha512-zMEIdD1/ylTu/EKbZ7v0Ulh31T6RR7HCa0ClmDjAdjVokeTC7pLQuUKg3OTKGNiJDwIPAPBlX45c1hCB4PZA8g==
"@budibase/types@2.2.12-alpha.48":
version "2.2.12-alpha.48"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.48.tgz#5036062395c0290e6d7d83c2438945a4fbe33432"
integrity sha512-DrGGGaJL7QIwM5W40jXFWHdVU7g3mTqqhqSmciglitaapngYnkVAztkS2itjimStB/6WuxE+V2dRKAJgkvKeGw==
"@bull-board/api@3.7.0":
version "3.7.0"

2
packages/string-templates/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "2.2.12-alpha.46",
"version": "2.2.12-alpha.49",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

2
packages/types/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/types",
"version": "2.2.12-alpha.46",
"version": "2.2.12-alpha.49",
"description": "Budibase types",
"main": "dist/index.js",
"types": "dist/index.d.ts",

41
packages/types/src/documents/app/datasource.ts
View File

@ -15,3 +15,44 @@ export interface Datasource extends Document {
[key: string]: Table
}
}
export enum RestAuthType {
BASIC = "basic",
BEARER = "bearer",
}
export interface RestBasicAuthConfig {
username: string
password: string
}
export interface RestBearerAuthConfig {
token: string
}
export interface RestAuthConfig {
_id: string
name: string
type: RestAuthType
config: RestBasicAuthConfig | RestBearerAuthConfig
}
export interface RestConfig {
url: string
rejectUnauthorized: boolean
defaultHeaders: {
[key: string]: any
}
legacyHttpParser: boolean
authConfigs: RestAuthConfig[]
staticVariables: {
[key: string]: string
}
dynamicVariables: [
{
name: string
queryId: string
value: string
}
]
}

10
packages/worker/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "2.2.12-alpha.46",
"version": "2.2.12-alpha.49",
"description": "Budibase background service",
"main": "src/index.ts",
"repository": {
@ -36,10 +36,10 @@
"author": "Budibase",
"license": "GPL-3.0",
"dependencies": {
"@budibase/backend-core": "2.2.12-alpha.46",
"@budibase/pro": "2.2.12-alpha.46",
"@budibase/string-templates": "2.2.12-alpha.46",
"@budibase/types": "2.2.12-alpha.46",
"@budibase/backend-core": "2.2.12-alpha.49",
"@budibase/pro": "2.2.12-alpha.48",
"@budibase/string-templates": "2.2.12-alpha.49",
"@budibase/types": "2.2.12-alpha.49",
"@koa/router": "8.0.8",
"@sentry/node": "6.17.7",
"@techpass/passport-openidconnect": "0.3.2",

30
packages/worker/yarn.lock
View File

@ -470,13 +470,13 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.2.12-alpha.46":
version "2.2.12-alpha.46"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.46.tgz#ac2c8168c4083e34be69e36e1b898f2b4e60a9bf"
integrity sha512-YNXF3/B17CrnOXfv27mzY1zEbBD/j5Nvr6ADJbakw7yrc3G58diLkJBo4kxzeEH4ujG3ftLDGLcbBP+zzoASZQ==
"@budibase/backend-core@2.2.12-alpha.48":
version "2.2.12-alpha.48"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.48.tgz#d211474e8ba57f2aa8d9c7b263cfbe96f7051ab9"
integrity sha512-7UuMHuV7jcaq5Y9fz7PM+YgT5bexuhm2UOoTivAtKOJwUR5Au7lGunVAZqLB5BPp55B9ncD/mQ5fev16vrep0Q==
dependencies:
"@budibase/nano" "10.1.1"
"@budibase/types" "2.2.12-alpha.46"
"@budibase/types" "2.2.12-alpha.48"
"@shopify/jest-koa-mocks" "5.0.1"
"@techpass/passport-openidconnect" "0.3.2"
aws-cloudfront-sign "2.2.0"
@ -521,13 +521,13 @@
qs "^6.11.0"
tough-cookie "^4.1.2"
"@budibase/pro@2.2.12-alpha.46":
version "2.2.12-alpha.46"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.46.tgz#bdd89640eea306a2fa7980387696c75b60ea0cc0"
integrity sha512-baiejsUGZfE+VSCj1Pb7sG5MDNqwzUTZ915t+YJ8tGg05TbR+/RaBLGfKkONbl9iaXK9Nl4KgiNytXvZQCZULw==
"@budibase/pro@2.2.12-alpha.48":
version "2.2.12-alpha.48"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.48.tgz#639431a654dbe55c316a94e4220adae76e5f7540"
integrity sha512-QMilBTDN/y5ieTnIjSlovhbmBLfTQV11B3YapJcdtpWKALyGpuZ7eYBY2DIjZtxTx5BE1Ib+BAJZmBxyXlkcHg==
dependencies:
"@budibase/backend-core" "2.2.12-alpha.46"
"@budibase/types" "2.2.12-alpha.46"
"@budibase/backend-core" "2.2.12-alpha.48"
"@budibase/types" "2.2.12-alpha.48"
"@koa/router" "8.0.8"
bull "4.10.1"
joi "17.6.0"
@ -535,10 +535,10 @@
lru-cache "^7.14.1"
node-fetch "^2.6.1"
"@budibase/types@2.2.12-alpha.46":
version "2.2.12-alpha.46"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.46.tgz#ff305f4b5dc505380f1f075a5a54b1794f24175c"
integrity sha512-zMEIdD1/ylTu/EKbZ7v0Ulh31T6RR7HCa0ClmDjAdjVokeTC7pLQuUKg3OTKGNiJDwIPAPBlX45c1hCB4PZA8g==
"@budibase/types@2.2.12-alpha.48":
version "2.2.12-alpha.48"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.48.tgz#5036062395c0290e6d7d83c2438945a4fbe33432"
integrity sha512-DrGGGaJL7QIwM5W40jXFWHdVU7g3mTqqhqSmciglitaapngYnkVAztkS2itjimStB/6WuxE+V2dRKAJgkvKeGw==
"@cspotcode/source-map-support@^0.8.0":
version "0.8.1"