Merge branch 'develop' of github.com:Budibase/budibase into develop

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.2.12-alpha.46",
+  "version": "2.2.12-alpha.49",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.2.12-alpha.46",
+  "version": "2.2.12-alpha.49",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -23,7 +23,7 @@
   },
   "dependencies": {
     "@budibase/nano": "10.1.1",
-    "@budibase/types": "2.2.12-alpha.46",
+    "@budibase/types": "2.2.12-alpha.49",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.2.12-alpha.46",
+  "version": "2.2.12-alpha.49",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.2.12-alpha.46",
+    "@budibase/string-templates": "2.2.12-alpha.49",
     "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",

packages/bbui/src/Form/Core/EnvDropdown.svelte

@@ -134,7 +134,7 @@
       on:blur={onBlur}
       on:focus={onFocus}
       on:input={onInput}
-      {type}
+      type={hbsValue.length ? "text" : type}
       style={align ? `text-align: ${align};` : ""}
       class="spectrum-Textfield-input"
       inputmode={type === "number" ? "decimal" : "text"}

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.2.12-alpha.46",
+  "version": "2.2.12-alpha.49",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -71,10 +71,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.46",
+    "@budibase/bbui": "2.2.12-alpha.49",
-    "@budibase/client": "2.2.12-alpha.46",
+    "@budibase/client": "2.2.12-alpha.49",
-    "@budibase/frontend-core": "2.2.12-alpha.46",
+    "@budibase/frontend-core": "2.2.12-alpha.49",
-    "@budibase/string-templates": "2.2.12-alpha.46",
+    "@budibase/string-templates": "2.2.12-alpha.49",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/accordion": "^3.0.24",
     "@spectrum-css/page": "^3.0.1",

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -39,6 +39,23 @@
   $: showError($fetch.error)
   $: id, (filters = null)
 
+  let appliedFilter
+  let rawFilter
+  let appliedSort
+  let selectedRows = []
+
+  $: enrichedSchema,
+    () => {
+      appliedFilter = null
+      rawFilter = null
+      appliedSort = null
+      selectedRows = []
+    }
+
+  $: if (Number.isInteger($fetch.pageNumber)) {
+    selectedRows = []
+  }
+
   const showError = error => {
     if (error) {
       notifications.error(error?.message || "Unable to fetch data.")
@@ -95,11 +112,15 @@
   }
 
   // Fetch data whenever sorting option changes
-  const onSort = e => {
+  const onSort = async e => {
-    fetch.update({
+    const sort = {
       sortColumn: e.detail.column,
       sortOrder: e.detail.order,
-    })
+    }
+    await fetch.update(sort)
+    appliedSort = { ...sort }
+    appliedSort.sortOrder = appliedSort.sortOrder.toLowerCase()
+    selectedRows = []
   }
 
   // Fetch data whenever filters change
@@ -108,16 +129,19 @@
     fetch.update({
       filter: filters,
     })
+    appliedFilter = e.detail
   }
 
   // Fetch data whenever schema changes
   const onUpdateColumns = () => {
+    selectedRows = []
     fetch.refresh()
   }
 
   // Fetch data whenever rows are modified. Unfortunately we have to lose
   // our pagination place, as our bookmarks will have shifted.
   const onUpdateRows = () => {
+    selectedRows = []
     fetch.refresh()
   }
 
@@ -142,6 +166,9 @@
     disableSorting
     on:updatecolumns={onUpdateColumns}
     on:updaterows={onUpdateRows}
+    on:selectionUpdated={e => {
+      selectedRows = e.detail
+    }}
     customPlaceholder
   >
     <div class="buttons">
@@ -183,6 +210,9 @@
         <ExportButton
           disabled={!hasRows || !hasCols}
           view={$tables.selected?._id}
+          filters={appliedFilter}
+          sorting={appliedSort}
+          {selectedRows}
         />
         {#key id}
           <TableFilterButton

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -16,6 +16,7 @@
     UNSORTABLE_TYPES,
   } from "constants"
   import RoleCell from "./cells/RoleCell.svelte"
+  import { createEventDispatcher } from "svelte"
 
   export let schema = {}
   export let data = []
@@ -28,6 +29,8 @@
   export let disableSorting = false
   export let customPlaceholder = false
 
+  const dispatch = createEventDispatcher()
+
   let selectedRows = []
   let editableColumn
   let editableRow
@@ -36,6 +39,7 @@
   let customRenderers = []
   let confirmDelete
 
+  $: selectedRows, dispatch("selectionUpdated", selectedRows)
   $: isUsersTable = tableId === TableNames.USERS
   $: data && resetSelectedRows()
   $: editRowComponent = isUsersTable ? CreateEditUser : CreateEditRow

packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte

@@ -3,7 +3,10 @@
   import ExportModal from "../modals/ExportModal.svelte"
 
   export let view
+  export let filters
+  export let sorting
   export let disabled = false
+  export let selectedRows
 
   let modal
 </script>
@@ -18,5 +21,5 @@
   Export
 </ActionButton>
 <Modal bind:this={modal}>
-  <ExportModal {view} />
+  <ExportModal {view} {filters} {sorting} {selectedRows} />
 </Modal>

packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte

@@ -1,7 +1,14 @@
 <script>
-  import { Select, ModalContent, notifications } from "@budibase/bbui"
+  import {
+    Select,
+    ModalContent,
+    notifications,
+    Body,
+    Table,
+  } from "@budibase/bbui"
   import download from "downloadjs"
   import { API } from "api"
+  import { Constants, LuceneUtils } from "@budibase/frontend-core"
 
   const FORMATS = [
     {
@@ -19,8 +26,71 @@
   ]
 
   export let view
+  export let filters
+  export let sorting
+  export let selectedRows = []
 
   let exportFormat = FORMATS[0].key
+  let filterLookup
+
+  $: luceneFilter = LuceneUtils.buildLuceneQuery(filters)
+  $: exportOpDisplay = buildExportOpDisplay(sorting, filterDisplay, filters)
+
+  const buildFilterLookup = () => {
+    return Object.keys(Constants.OperatorOptions).reduce((acc, key) => {
+      const op = Constants.OperatorOptions[key]
+      acc[op.value] = op.label
+      return acc
+    }, {})
+  }
+  filterLookup = buildFilterLookup()
+
+  const filterDisplay = () => {
+    if (!filters) {
+      return []
+    }
+    return filters.map(filter => {
+      let newFieldName = filter.field + ""
+      const parts = newFieldName.split(":")
+      parts.shift()
+      newFieldName = parts.join(":")
+      return {
+        Field: newFieldName,
+        Operation: filterLookup[filter.operator],
+        "Field Value": filter.value || "",
+      }
+    })
+  }
+
+  const buildExportOpDisplay = (sorting, filterDisplay) => {
+    let filterDisplayConfig = filterDisplay()
+    if (sorting) {
+      filterDisplayConfig = [
+        ...filterDisplayConfig,
+        {
+          Field: sorting.sortColumn,
+          Operation: "Order By",
+          "Field Value": sorting.sortOrder,
+        },
+      ]
+    }
+    return filterDisplayConfig
+  }
+
+  const displaySchema = {
+    Field: {
+      type: "string",
+      fieldName: "Field",
+    },
+    Operation: {
+      type: "string",
+      fieldName: "Operation",
+    },
+    "Field Value": {
+      type: "string",
+      fieldName: "Value",
+    },
+  }
 
   async function exportView() {
     try {
@@ -33,9 +103,74 @@
       notifications.error(`Unable to export ${exportFormat.toUpperCase()} data`)
     }
   }
+
+  async function exportRows() {
+    if (selectedRows?.length) {
+      const data = await API.exportRows({
+        tableId: view,
+        rows: selectedRows.map(row => row._id),
+        format: exportFormat,
+      })
+      download(data, `export.${exportFormat}`)
+    } else if (filters || sorting) {
+      const data = await API.exportRows({
+        tableId: view,
+        format: exportFormat,
+        search: {
+          query: luceneFilter,
+          sort: sorting?.sortColumn,
+          sortOrder: sorting?.sortOrder,
+          paginate: false,
+        },
+      })
+      download(data, `export.${exportFormat}`)
+    } else {
+      await exportView()
+    }
+  }
 </script>
 
-<ModalContent title="Export Data" confirmText="Export" onConfirm={exportView}>
+<ModalContent
+  title="Export Data"
+  confirmText="Export"
+  onConfirm={exportRows}
+  size={filters?.length || sorting ? "M" : "S"}
+>
+  {#if selectedRows?.length}
+    <Body size="S">
+      <strong>{selectedRows?.length}</strong>
+      {`row${selectedRows?.length > 1 ? "s" : ""} will be exported`}
+    </Body>
+  {:else if filters || (sorting?.sortOrder && sorting?.sortColumn)}
+    <Body size="S">
+      {#if !filters}
+        Exporting <strong>all</strong> rows
+      {:else}
+        Filters applied
+      {/if}
+    </Body>
+
+    <div class="table-wrap">
+      <Table
+        schema={displaySchema}
+        data={exportOpDisplay}
+        {filters}
+        loading={false}
+        rowCount={filters?.length + 1}
+        disableSorting={true}
+        allowSelectRows={false}
+        allowEditRows={false}
+        allowEditColumns={false}
+        quiet={true}
+        compact={true}
+      />
+    </div>
+  {:else}
+    <Body size="S">
+      Exporting <strong>all</strong> rows
+    </Body>
+  {/if}
+
   <Select
     label="Format"
     bind:value={exportFormat}
@@ -45,3 +180,9 @@
     getOptionValue={x => x.key}
   />
 </ModalContent>
+
+<style>
+  .table-wrap :global(.wrapper) {
+    max-width: 400px;
+  }
+</style>

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationModal.svelte

@@ -248,6 +248,7 @@
       />
       <EnvDropdown
         label="Password"
+        type="password"
         bind:value={form.basic.password}
         on:change={onFieldChange}
         on:blur={() => (blurred.basic.password = true)}

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.2.12-alpha.46",
+  "version": "2.2.12-alpha.49",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.2.12-alpha.46",
+    "@budibase/backend-core": "2.2.12-alpha.49",
-    "@budibase/string-templates": "2.2.12-alpha.46",
+    "@budibase/string-templates": "2.2.12-alpha.49",
-    "@budibase/types": "2.2.12-alpha.46",
+    "@budibase/types": "2.2.12-alpha.49",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.2.12-alpha.46",
+  "version": "2.2.12-alpha.49",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.46",
+    "@budibase/bbui": "2.2.12-alpha.49",
-    "@budibase/frontend-core": "2.2.12-alpha.46",
+    "@budibase/frontend-core": "2.2.12-alpha.49",
-    "@budibase/string-templates": "2.2.12-alpha.46",
+    "@budibase/string-templates": "2.2.12-alpha.49",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.2.12-alpha.46",
+  "version": "2.2.12-alpha.49",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.46",
+    "@budibase/bbui": "2.2.12-alpha.49",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/frontend-core/src/api/rows.js

@@ -67,12 +67,13 @@ export const buildRowEndpoints = API => ({
    * @param format the format to export (csv or json)
    * @param columns which columns to export (all if undefined)
    */
-  exportRows: async ({ tableId, rows, format, columns }) => {
+  exportRows: async ({ tableId, rows, format, columns, search }) => {
     return await API.post({
       url: `/api/${tableId}/rows/exportRows?format=${format}`,
       body: {
         rows,
         columns,
+        ...search,
       },
       parseResponse: async response => {
         return await response.text()

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.2.12-alpha.46",
+  "version": "2.2.12-alpha.49",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.2.12-alpha.46",
+  "version": "2.2.12-alpha.49",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.2.12-alpha.46",
+    "@budibase/backend-core": "2.2.12-alpha.49",
-    "@budibase/client": "2.2.12-alpha.46",
+    "@budibase/client": "2.2.12-alpha.49",
-    "@budibase/pro": "2.2.12-alpha.46",
+    "@budibase/pro": "2.2.12-alpha.48",
-    "@budibase/string-templates": "2.2.12-alpha.46",
+    "@budibase/string-templates": "2.2.12-alpha.49",
-    "@budibase/types": "2.2.12-alpha.46",
+    "@budibase/types": "2.2.12-alpha.49",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/controllers/row/external.ts

@@ -21,6 +21,8 @@ import {
 } from "@budibase/types"
 import sdk from "../../../sdk"
 
+const { cleanExportRows } = require("./utils")
+
 export async function handleRequest(
   operation: Operation,
   tableId: string,
@@ -100,7 +102,7 @@ export async function destroy(ctx: BBContext) {
 export async function bulkDestroy(ctx: BBContext) {
   const { rows } = ctx.request.body
   const tableId = ctx.params.tableId
-  let promises = []
+  let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
   for (let row of rows) {
     promises.push(
       handleRequest(Operation.DELETE, tableId, {
@@ -186,6 +188,8 @@ export async function exportRows(ctx: BBContext) {
   if (!datasource || !datasource.entities) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
+
+  if (ctx.request.body.rows) {
     ctx.request.body = {
       query: {
         oneOf: {
@@ -195,11 +199,13 @@ export async function exportRows(ctx: BBContext) {
         },
       },
     }
+  }
 
   let result = await search(ctx)
   let rows: Row[] = []
 
   // Filter data to only specified columns if required
+
   if (columns && columns.length) {
     for (let i = 0; i < result.rows.length; i++) {
       rows[i] = {}
@@ -211,14 +217,19 @@ export async function exportRows(ctx: BBContext) {
     rows = result.rows
   }
 
-  let headers = Object.keys(rows[0])
+  // @ts-ignore
+  let schema = datasource.entities[tableName].schema
+  let exportRows = cleanExportRows(rows, schema, format, columns)
+
+  let headers = Object.keys(schema)
+
   // @ts-ignore
   const exporter = exporters[format]
   const filename = `export.${format}`
 
   // send down the file
   ctx.attachment(filename)
-  return apiFileReturn(exporter(headers, rows))
+  return apiFileReturn(exporter(headers, exportRows))
 }
 
 export async function fetchEnrichedRow(ctx: BBContext) {

packages/server/src/api/controllers/row/internal.ts

@@ -27,7 +27,7 @@ import {
 import { cloneDeep } from "lodash/fp"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
-import { csv, json, jsonWithSchema, Format, isFormat } from "../view/exporters"
+import { csv, json, jsonWithSchema, Format } from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
   Ctx,
@@ -38,6 +38,8 @@ import {
   Table,
 } from "@budibase/types"
 
+const { cleanExportRows } = require("./utils")
+
 const CALCULATION_TYPES = {
   SUM: "sum",
   COUNT: "count",
@@ -357,6 +359,14 @@ export async function search(ctx: Ctx) {
   params.version = ctx.version
   params.tableId = tableId
 
+  let table
+  if (params.sort && !params.sortType) {
+    table = await db.get(tableId)
+    const schema = table.schema
+    const sortField = schema[params.sort]
+    params.sortType = sortField.type == "number" ? "number" : "string"
+  }
+
   let response
   if (paginate) {
     response = await paginatedSearch(query, params)
@@ -370,7 +380,7 @@ export async function search(ctx: Ctx) {
     if (tableId === InternalTables.USER_METADATA) {
       response.rows = await getGlobalUsersFromMetadata(response.rows)
     }
-    const table = await db.get(tableId)
+    table = table || (await db.get(tableId))
     response.rows = await outputProcessing(table, response.rows)
   }
 
@@ -389,7 +399,10 @@ export async function exportRows(ctx: Ctx) {
   const table = await db.get(ctx.params.tableId)
   const rowIds = ctx.request.body.rows
   let format = ctx.query.format
-  const { columns } = ctx.request.body
+  const { columns, query } = ctx.request.body
+
+  let result
+  if (rowIds) {
     let response = (
       await db.allDocs({
         include_docs: true,
@@ -397,8 +410,14 @@ export async function exportRows(ctx: Ctx) {
       })
     ).rows.map(row => row.doc)
 
-  let result = (await outputProcessing(table, response)) as Row[]
+    result = await outputProcessing(table, response)
+  } else if (query) {
+    let searchResponse = await exports.search(ctx)
+    result = searchResponse.rows
+  }
+
   let rows: Row[] = []
+  let schema = table.schema
 
   // Filter data to only specified columns if required
   if (columns && columns.length) {
@@ -412,12 +431,16 @@ export async function exportRows(ctx: Ctx) {
     rows = result
   }
 
+  let exportRows = cleanExportRows(rows, schema, format, columns)
   if (format === Format.CSV) {
     ctx.attachment("export.csv")
-    return apiFileReturn(csv(Object.keys(rows[0]), rows))
+    return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
   } else if (format === Format.JSON) {
     ctx.attachment("export.json")
-    return apiFileReturn(json(rows))
+    return apiFileReturn(json(exportRows))
+  } else if (format === Format.JSON_WITH_SCHEMA) {
+    ctx.attachment("export.json")
+    return apiFileReturn(jsonWithSchema(schema, exportRows))
   } else {
     throw "Format not recognised"
   }

packages/server/src/api/controllers/row/utils.ts

@@ -7,6 +7,7 @@ import { BBContext, Row, Table } from "@budibase/types"
 export { removeKeyNumbering } from "../../../integrations/base/utils"
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
+import { Format } from "../view/exporters"
 import { Ctx } from "@budibase/types"
 import sdk from "../../../sdk"
 
@@ -117,3 +118,40 @@ export async function validate({
   }
   return { valid: Object.keys(errors).length === 0, errors }
 }
+
+export function cleanExportRows(
+  rows: any[],
+  schema: any,
+  format: string,
+  columns: string[]
+) {
+  let cleanRows = [...rows]
+
+  const relationships = Object.entries(schema)
+    .filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
+    .map(entry => entry[0])
+
+  relationships.forEach(column => {
+    cleanRows.forEach(row => {
+      delete row[column]
+    })
+    delete schema[column]
+  })
+
+  // Intended to avoid 'undefined' in export
+  if (format === Format.CSV) {
+    const schemaKeys = Object.keys(schema)
+    for (let key of schemaKeys) {
+      if (columns?.length && columns.indexOf(key) > 0) {
+        continue
+      }
+      for (let row of cleanRows) {
+        if (row[key] == null) {
+          row[key] = ""
+        }
+      }
+    }
+  }
+
+  return cleanRows
+}

packages/server/src/api/controllers/view/index.ts

@@ -3,7 +3,6 @@ import { apiFileReturn } from "../../../utilities/fileSystem"
 import { csv, json, jsonWithSchema, Format, isFormat } from "./exporters"
 import { deleteView, getView, getViews, saveView } from "./utils"
 import { fetchView } from "../row"
-import { FieldTypes } from "../../../constants"
 import { context, events } from "@budibase/backend-core"
 import { DocumentType } from "../../../db/utils"
 import sdk from "../../../sdk"
@@ -15,6 +14,7 @@ import {
   TableSchema,
   View,
 } from "@budibase/types"
+import { cleanExportRows } from "../row/utils"
 
 const { cloneDeep, isEqual } = require("lodash")
 
@@ -162,39 +162,17 @@ export async function exportView(ctx: BBContext) {
     schema = table.schema
   }
 
-  // remove any relationships
+  let exportRows = cleanExportRows(rows, schema, format, [])
-  const relationships = Object.entries(schema)
-    .filter(entry => entry[1].type === FieldTypes.LINK)
-    .map(entry => entry[0])
-  // iterate relationship columns and remove from and row and schema
-  relationships.forEach(column => {
-    rows.forEach(row => {
-      delete row[column]
-    })
-    delete schema[column]
-  })
-
-  // make sure no "undefined" entries appear in the CSV
-  if (format === Format.CSV) {
-    const schemaKeys = Object.keys(schema)
-    for (let key of schemaKeys) {
-      for (let row of rows) {
-        if (row[key] == null) {
-          row[key] = ""
-        }
-      }
-    }
-  }
 
   if (format === Format.CSV) {
     ctx.attachment(`${viewName}.csv`)
-    ctx.body = apiFileReturn(csv(Object.keys(schema), rows))
+    ctx.body = apiFileReturn(csv(Object.keys(schema), exportRows))
   } else if (format === Format.JSON) {
     ctx.attachment(`${viewName}.json`)
-    ctx.body = apiFileReturn(json(rows))
+    ctx.body = apiFileReturn(json(exportRows))
   } else if (format === Format.JSON_WITH_SCHEMA) {
     ctx.attachment(`${viewName}.json`)
-    ctx.body = apiFileReturn(jsonWithSchema(schema, rows))
+    ctx.body = apiFileReturn(jsonWithSchema(schema, exportRows))
   } else {
     throw "Format not recognised"
   }

packages/server/src/definitions/datasource.ts

@@ -7,44 +7,3 @@
 export interface QueryOptions {
   disableReturning?: boolean
 }
-
-export enum AuthType {
-  BASIC = "basic",
-  BEARER = "bearer",
-}
-
-interface AuthConfig {
-  _id: string
-  name: string
-  type: AuthType
-  config: BasicAuthConfig | BearerAuthConfig
-}
-
-export interface BasicAuthConfig {
-  username: string
-  password: string
-}
-
-export interface BearerAuthConfig {
-  token: string
-}
-
-export interface RestConfig {
-  url: string
-  rejectUnauthorized: boolean
-  defaultHeaders: {
-    [key: string]: any
-  }
-  legacyHttpParser: boolean
-  authConfigs: AuthConfig[]
-  staticVariables: {
-    [key: string]: string
-  }
-  dynamicVariables: [
-    {
-      name: string
-      queryId: string
-      value: string
-    }
-  ]
-}

packages/server/src/integrations/rest.ts

@@ -6,13 +6,11 @@ import {
   IntegrationBase,
   PaginationValues,
   RestQueryFields as RestQuery,
-} from "@budibase/types"
-import {
   RestConfig,
-  AuthType,
+  RestAuthType,
-  BasicAuthConfig,
+  RestBasicAuthConfig,
-  BearerAuthConfig,
+  RestBearerAuthConfig,
-} from "../definitions/datasource"
+} from "@budibase/types"
 import { get } from "lodash"
 import * as https from "https"
 import qs from "querystring"
@@ -331,14 +329,14 @@ class RestIntegration implements IntegrationBase {
       if (authConfig) {
         let config
         switch (authConfig.type) {
-          case AuthType.BASIC:
+          case RestAuthType.BASIC:
-            config = authConfig.config as BasicAuthConfig
+            config = authConfig.config as RestBasicAuthConfig
             headers.Authorization = `Basic ${Buffer.from(
               `${config.username}:${config.password}`
             ).toString("base64")}`
             break
-          case AuthType.BEARER:
+          case RestAuthType.BEARER:
-            config = authConfig.config as BearerAuthConfig
+            config = authConfig.config as RestBearerAuthConfig
             headers.Authorization = `Bearer ${config.token}`
             break
         }
@@ -428,5 +426,4 @@ class RestIntegration implements IntegrationBase {
 export default {
   schema: SCHEMA,
   integration: RestIntegration,
-  AuthType,
 }

packages/server/src/integrations/tests/rest.spec.ts

@@ -15,6 +15,7 @@ jest.mock("node-fetch", () => {
 
 import fetch from "node-fetch"
 import { default as RestIntegration } from "../rest"
+import { RestAuthType } from "@budibase/types"
 const FormData = require("form-data")
 const { URLSearchParams } = require("url")
 
@@ -229,7 +230,7 @@ describe("REST Integration", () => {
     const basicAuth = {
       _id: "c59c14bd1898a43baa08da68959b24686",
       name: "basic-1",
-      type: RestIntegration.AuthType.BASIC,
+      type: RestAuthType.BASIC,
       config: {
         username: "user",
         password: "password",
@@ -239,7 +240,7 @@ describe("REST Integration", () => {
     const bearerAuth = {
       _id: "0d91d732f34e4befabeff50b392a8ff3",
       name: "bearer-1",
-      type: RestIntegration.AuthType.BEARER,
+      type: RestAuthType.BEARER,
       config: {
         token: "mytoken",
       },
@@ -581,6 +582,7 @@ describe("REST Integration", () => {
     })
     await config.integration.read({})
 
+    // @ts-ignore
     const calls: any = fetch.mock.calls[0]
     const url = calls[0]
     expect(url).toBe(`${BASE_URL}/`)

packages/server/src/sdk/app/datasources/datasources.ts

@@ -1,17 +1,19 @@
 import { context } from "@budibase/backend-core"
-import { processObjectSync, findHBSBlocks } from "@budibase/string-templates"
+import { findHBSBlocks, processObjectSync } from "@budibase/string-templates"
 import {
   Datasource,
   DatasourceFieldType,
-  Integration,
   PASSWORD_REPLACEMENT,
+  RestAuthConfig,
+  RestAuthType,
+  RestBasicAuthConfig,
+  SourceName,
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import { getEnvironmentVariables } from "../../utils"
 import { getDefinitions } from "../../../integrations"
 
 const ENV_VAR_PREFIX = "env."
-const USER_PREFIX = "user"
 
 async function enrichDatasourceWithValues(datasource: Datasource) {
   const cloned = cloneDeep(datasource)
@@ -47,6 +49,18 @@ export async function getWithEnvVars(datasourceId: string) {
   return enrichDatasourceWithValues(datasource)
 }
 
+function hasAuthConfigs(datasource: Datasource) {
+  return datasource.source === SourceName.REST && datasource.config?.authConfigs
+}
+
+function useEnvVars(str: any) {
+  if (typeof str !== "string") {
+    return false
+  }
+  const blocks = findHBSBlocks(str)
+  return blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
+}
+
 export async function removeSecrets(datasources: Datasource[]) {
   const definitions = await getDefinitions()
   for (let datasource of datasources) {
@@ -56,17 +70,24 @@ export async function removeSecrets(datasources: Datasource[]) {
       if (datasource.config.auth) {
         delete datasource.config.auth
       }
-      // remove passwords
+      // specific to REST datasources, contains passwords
-      for (let key of Object.keys(datasource.config)) {
+      if (hasAuthConfigs(datasource)) {
-        if (typeof datasource.config[key] !== "string") {
+        const configs = datasource.config.authConfigs as RestAuthConfig[]
+        for (let config of configs) {
+          if (config.type !== RestAuthType.BASIC) {
             continue
           }
-        const blocks = findHBSBlocks(datasource.config[key] as string)
+          const basic = config.config as RestBasicAuthConfig
-        const usesEnvVars =
+          if (!useEnvVars(basic.password)) {
-          blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
+            basic.password = PASSWORD_REPLACEMENT
+          }
+        }
+      }
+      // remove general passwords
+      for (let key of Object.keys(datasource.config)) {
         if (
-          !usesEnvVars &&
+          schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD &&
-          schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD
+          !useEnvVars(datasource.config[key])
         ) {
           datasource.config[key] = PASSWORD_REPLACEMENT
         }
@@ -84,6 +105,23 @@ export function mergeConfigs(update: Datasource, old: Datasource) {
   if (!update.config) {
     return update
   }
+  // specific to REST datasources, fix the auth configs again if required
+  if (hasAuthConfigs(update)) {
+    const configs = update.config.authConfigs as RestAuthConfig[]
+    const oldConfigs = old.config?.authConfigs as RestAuthConfig[]
+    for (let config of configs) {
+      if (config.type !== RestAuthType.BASIC) {
+        continue
+      }
+      const basic = config.config as RestBasicAuthConfig
+      const oldBasic = oldConfigs.find(old => old.name === config.name)
+        ?.config as RestBasicAuthConfig
+      if (basic.password === PASSWORD_REPLACEMENT) {
+        basic.password = oldBasic.password
+      }
+    }
+  }
+  // update back to actual passwords for everything else
   for (let [key, value] of Object.entries(update.config)) {
     if (value !== PASSWORD_REPLACEMENT) {
       continue

packages/server/yarn.lock

@@ -1273,13 +1273,13 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.2.12-alpha.46":
+"@budibase/backend-core@2.2.12-alpha.48":
-  version "2.2.12-alpha.46"
+  version "2.2.12-alpha.48"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.46.tgz#ac2c8168c4083e34be69e36e1b898f2b4e60a9bf"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.48.tgz#d211474e8ba57f2aa8d9c7b263cfbe96f7051ab9"
-  integrity sha512-YNXF3/B17CrnOXfv27mzY1zEbBD/j5Nvr6ADJbakw7yrc3G58diLkJBo4kxzeEH4ujG3ftLDGLcbBP+zzoASZQ==
+  integrity sha512-7UuMHuV7jcaq5Y9fz7PM+YgT5bexuhm2UOoTivAtKOJwUR5Au7lGunVAZqLB5BPp55B9ncD/mQ5fev16vrep0Q==
   dependencies:
     "@budibase/nano" "10.1.1"
-    "@budibase/types" "2.2.12-alpha.46"
+    "@budibase/types" "2.2.12-alpha.48"
     "@shopify/jest-koa-mocks" "5.0.1"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-cloudfront-sign "2.2.0"
@@ -1374,13 +1374,13 @@
     qs "^6.11.0"
     tough-cookie "^4.1.2"
 
-"@budibase/pro@2.2.12-alpha.46":
+"@budibase/pro@2.2.12-alpha.48":
-  version "2.2.12-alpha.46"
+  version "2.2.12-alpha.48"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.46.tgz#bdd89640eea306a2fa7980387696c75b60ea0cc0"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.48.tgz#639431a654dbe55c316a94e4220adae76e5f7540"
-  integrity sha512-baiejsUGZfE+VSCj1Pb7sG5MDNqwzUTZ915t+YJ8tGg05TbR+/RaBLGfKkONbl9iaXK9Nl4KgiNytXvZQCZULw==
+  integrity sha512-QMilBTDN/y5ieTnIjSlovhbmBLfTQV11B3YapJcdtpWKALyGpuZ7eYBY2DIjZtxTx5BE1Ib+BAJZmBxyXlkcHg==
   dependencies:
-    "@budibase/backend-core" "2.2.12-alpha.46"
+    "@budibase/backend-core" "2.2.12-alpha.48"
-    "@budibase/types" "2.2.12-alpha.46"
+    "@budibase/types" "2.2.12-alpha.48"
     "@koa/router" "8.0.8"
     bull "4.10.1"
     joi "17.6.0"
@@ -1406,10 +1406,10 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/types@2.2.12-alpha.46":
+"@budibase/types@2.2.12-alpha.48":
-  version "2.2.12-alpha.46"
+  version "2.2.12-alpha.48"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.46.tgz#ff305f4b5dc505380f1f075a5a54b1794f24175c"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.48.tgz#5036062395c0290e6d7d83c2438945a4fbe33432"
-  integrity sha512-zMEIdD1/ylTu/EKbZ7v0Ulh31T6RR7HCa0ClmDjAdjVokeTC7pLQuUKg3OTKGNiJDwIPAPBlX45c1hCB4PZA8g==
+  integrity sha512-DrGGGaJL7QIwM5W40jXFWHdVU7g3mTqqhqSmciglitaapngYnkVAztkS2itjimStB/6WuxE+V2dRKAJgkvKeGw==
 
 "@bull-board/api@3.7.0":
   version "3.7.0"

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "2.2.12-alpha.46",
+  "version": "2.2.12-alpha.49",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/types",
-  "version": "2.2.12-alpha.46",
+  "version": "2.2.12-alpha.49",
   "description": "Budibase types",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/types/src/documents/app/datasource.ts

@@ -15,3 +15,44 @@ export interface Datasource extends Document {
     [key: string]: Table
   }
 }
+
+export enum RestAuthType {
+  BASIC = "basic",
+  BEARER = "bearer",
+}
+
+export interface RestBasicAuthConfig {
+  username: string
+  password: string
+}
+
+export interface RestBearerAuthConfig {
+  token: string
+}
+
+export interface RestAuthConfig {
+  _id: string
+  name: string
+  type: RestAuthType
+  config: RestBasicAuthConfig | RestBearerAuthConfig
+}
+
+export interface RestConfig {
+  url: string
+  rejectUnauthorized: boolean
+  defaultHeaders: {
+    [key: string]: any
+  }
+  legacyHttpParser: boolean
+  authConfigs: RestAuthConfig[]
+  staticVariables: {
+    [key: string]: string
+  }
+  dynamicVariables: [
+    {
+      name: string
+      queryId: string
+      value: string
+    }
+  ]
+}

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.2.12-alpha.46",
+  "version": "2.2.12-alpha.49",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -36,10 +36,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "2.2.12-alpha.46",
+    "@budibase/backend-core": "2.2.12-alpha.49",
-    "@budibase/pro": "2.2.12-alpha.46",
+    "@budibase/pro": "2.2.12-alpha.48",
-    "@budibase/string-templates": "2.2.12-alpha.46",
+    "@budibase/string-templates": "2.2.12-alpha.49",
-    "@budibase/types": "2.2.12-alpha.46",
+    "@budibase/types": "2.2.12-alpha.49",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",

packages/worker/yarn.lock

@@ -470,13 +470,13 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.2.12-alpha.46":
+"@budibase/backend-core@2.2.12-alpha.48":
-  version "2.2.12-alpha.46"
+  version "2.2.12-alpha.48"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.46.tgz#ac2c8168c4083e34be69e36e1b898f2b4e60a9bf"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.48.tgz#d211474e8ba57f2aa8d9c7b263cfbe96f7051ab9"
-  integrity sha512-YNXF3/B17CrnOXfv27mzY1zEbBD/j5Nvr6ADJbakw7yrc3G58diLkJBo4kxzeEH4ujG3ftLDGLcbBP+zzoASZQ==
+  integrity sha512-7UuMHuV7jcaq5Y9fz7PM+YgT5bexuhm2UOoTivAtKOJwUR5Au7lGunVAZqLB5BPp55B9ncD/mQ5fev16vrep0Q==
   dependencies:
     "@budibase/nano" "10.1.1"
-    "@budibase/types" "2.2.12-alpha.46"
+    "@budibase/types" "2.2.12-alpha.48"
     "@shopify/jest-koa-mocks" "5.0.1"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-cloudfront-sign "2.2.0"
@@ -521,13 +521,13 @@
     qs "^6.11.0"
     tough-cookie "^4.1.2"
 
-"@budibase/pro@2.2.12-alpha.46":
+"@budibase/pro@2.2.12-alpha.48":
-  version "2.2.12-alpha.46"
+  version "2.2.12-alpha.48"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.46.tgz#bdd89640eea306a2fa7980387696c75b60ea0cc0"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.48.tgz#639431a654dbe55c316a94e4220adae76e5f7540"
-  integrity sha512-baiejsUGZfE+VSCj1Pb7sG5MDNqwzUTZ915t+YJ8tGg05TbR+/RaBLGfKkONbl9iaXK9Nl4KgiNytXvZQCZULw==
+  integrity sha512-QMilBTDN/y5ieTnIjSlovhbmBLfTQV11B3YapJcdtpWKALyGpuZ7eYBY2DIjZtxTx5BE1Ib+BAJZmBxyXlkcHg==
   dependencies:
-    "@budibase/backend-core" "2.2.12-alpha.46"
+    "@budibase/backend-core" "2.2.12-alpha.48"
-    "@budibase/types" "2.2.12-alpha.46"
+    "@budibase/types" "2.2.12-alpha.48"
     "@koa/router" "8.0.8"
     bull "4.10.1"
     joi "17.6.0"
@@ -535,10 +535,10 @@
     lru-cache "^7.14.1"
     node-fetch "^2.6.1"
 
-"@budibase/types@2.2.12-alpha.46":
+"@budibase/types@2.2.12-alpha.48":
-  version "2.2.12-alpha.46"
+  version "2.2.12-alpha.48"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.46.tgz#ff305f4b5dc505380f1f075a5a54b1794f24175c"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.48.tgz#5036062395c0290e6d7d83c2438945a4fbe33432"
-  integrity sha512-zMEIdD1/ylTu/EKbZ7v0Ulh31T6RR7HCa0ClmDjAdjVokeTC7pLQuUKg3OTKGNiJDwIPAPBlX45c1hCB4PZA8g==
+  integrity sha512-DrGGGaJL7QIwM5W40jXFWHdVU7g3mTqqhqSmciglitaapngYnkVAztkS2itjimStB/6WuxE+V2dRKAJgkvKeGw==
 
 "@cspotcode/source-map-support@^0.8.0":
   version "0.8.1"