Merge remote-tracking branch 'origin/master' into feature/add-buttongroup-to-formblock

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.13.6",
+  "version": "2.13.9",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -10,6 +10,7 @@ import {
   DatabaseDeleteIndexOpts,
   Document,
   isDocument,
+  RowResponse,
 } from "@budibase/types"
 import { getCouchInfo } from "./connections"
 import { directCouchUrlCall } from "./utils"
@@ -109,7 +110,7 @@ export class DatabaseImpl implements Database {
     }
   }
 
-  async get<T>(id?: string): Promise<T | any> {
+  async get<T extends Document>(id?: string): Promise<T> {
     const db = await this.checkSetup()
     if (!id) {
       throw new Error("Unable to get doc without a valid _id.")
@@ -117,6 +118,35 @@ export class DatabaseImpl implements Database {
     return this.updateOutput(() => db.get(id))
   }
 
+  async getMultiple<T extends Document>(
+    ids: string[],
+    opts?: { allowMissing?: boolean }
+  ): Promise<T[]> {
+    // get unique
+    ids = [...new Set(ids)]
+    const response = await this.allDocs<T>({
+      keys: ids,
+      include_docs: true,
+    })
+    const rowUnavailable = (row: RowResponse<T>) => {
+      // row is deleted - key lookup can return this
+      if (row.doc == null || ("deleted" in row.value && row.value.deleted)) {
+        return true
+      }
+      return row.error === "not_found"
+    }
+
+    const rows = response.rows.filter(row => !rowUnavailable(row))
+    const someMissing = rows.length !== response.rows.length
+    // some were filtered out - means some missing
+    if (!opts?.allowMissing && someMissing) {
+      const missing = response.rows.filter(row => rowUnavailable(row))
+      const missingIds = missing.map(row => row.key).join(", ")
+      throw new Error(`Unable to get documents: ${missingIds}`)
+    }
+    return rows.map(row => row.doc!)
+  }
+
   async remove(idOrDoc: string | Document, rev?: string) {
     const db = await this.checkSetup()
     let _id: string

packages/backend-core/src/redis/redis.ts

@@ -28,7 +28,6 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT
 // for testing just generate the client once
 let CLOSED = false
 let CLIENTS: { [key: number]: any } = {}
-0
 let CONNECTED = false
 
 // mock redis always connected

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -33,6 +33,10 @@
       part1: PrettyRelationshipDefinitions.MANY,
       part2: PrettyRelationshipDefinitions.ONE,
     },
+    [RelationshipType.ONE_TO_MANY]: {
+      part1: PrettyRelationshipDefinitions.ONE,
+      part2: PrettyRelationshipDefinitions.MANY,
+    },
   }
   let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions)
   let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions)
@@ -58,7 +62,7 @@
   let fromPrimary, fromForeign, fromColumn, toColumn
 
   let throughId, throughToKey, throughFromKey
-  let isManyToMany, isManyToOne, relationshipType
+  let relationshipType
   let hasValidated = false
 
   $: fromId = null
@@ -85,8 +89,9 @@
   $: valid =
     getErrorCount(errors) === 0 && allRequiredAttributesSet(relationshipType)
   $: isManyToMany = relationshipType === RelationshipType.MANY_TO_MANY
-  $: isManyToOne = relationshipType === RelationshipType.MANY_TO_ONE
+  $: isManyToOne =
-
+    relationshipType === RelationshipType.MANY_TO_ONE ||
+    relationshipType === RelationshipType.ONE_TO_MANY
   function getTable(id) {
     return plusTables.find(table => table._id === id)
   }

packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte

@@ -11,6 +11,7 @@
   export let componentBindings
   export let bindings
   export let parseSettings
+  export let disabled
 
   const draggable = getContext("draggable")
   const dispatch = createEventDispatcher()

packages/builder/src/components/design/settings/controls/FieldConfiguration/CellEditor.svelte

@@ -1,26 +0,0 @@
-<script>
-  import { DrawerContent, Drawer, Button, Icon } from "@budibase/bbui"
-  import ValidationDrawer from "components/design/settings/controls/ValidationEditor/ValidationDrawer.svelte"
-  export let column
-  export let type
-
-  let drawer
-</script>
-
-<Icon name="Settings" hoverable size="S" on:click={drawer.show} />
-<Drawer bind:this={drawer} title="Field Validation">
-  <svelte:fragment slot="description">
-    "{column.name}" field validation
-  </svelte:fragment>
-  <Button cta slot="buttons" on:click={drawer.hide}>Save</Button>
-  <DrawerContent slot="body">
-    <div class="container">
-      <ValidationDrawer
-        slot="body"
-        bind:rules={column.validation}
-        fieldName={column.name}
-        {type}
-      />
-    </div>
-  </DrawerContent>
-</Drawer>

packages/builder/src/components/design/settings/controls/FieldConfiguration/ColumnDrawer.svelte

@@ -1,202 +0,0 @@
-<script>
-  import {
-    Button,
-    Icon,
-    DrawerContent,
-    Layout,
-    Select,
-    Label,
-    Body,
-    Input,
-  } from "@budibase/bbui"
-  import { flip } from "svelte/animate"
-  import { dndzone } from "svelte-dnd-action"
-  import { generate } from "shortid"
-  import CellEditor from "./CellEditor.svelte"
-
-  export let columns = []
-  export let options = []
-  export let schema = {}
-
-  const flipDurationMs = 150
-  let dragDisabled = true
-
-  $: unselectedColumns = getUnselectedColumns(options, columns)
-  $: columns.forEach(column => {
-    if (!column.id) {
-      column.id = generate()
-    }
-  })
-
-  const getUnselectedColumns = (allColumns, selectedColumns) => {
-    let optionsObj = {}
-    allColumns.forEach(option => {
-      optionsObj[option] = true
-    })
-    selectedColumns?.forEach(column => {
-      delete optionsObj[column.name]
-    })
-    return Object.keys(optionsObj)
-  }
-
-  const getRemainingColumnOptions = selectedColumn => {
-    if (!selectedColumn || unselectedColumns.includes(selectedColumn)) {
-      return unselectedColumns
-    }
-    return [selectedColumn, ...unselectedColumns]
-  }
-
-  const addColumn = () => {
-    columns = [...columns, {}]
-  }
-
-  const removeColumn = id => {
-    columns = columns.filter(column => column.id !== id)
-  }
-
-  const updateColumnOrder = e => {
-    columns = e.detail.items
-  }
-
-  const handleFinalize = e => {
-    updateColumnOrder(e)
-    dragDisabled = true
-  }
-
-  const addAllColumns = () => {
-    let newColumns = columns || []
-    options.forEach(field => {
-      const fieldSchema = schema[field]
-      const hasCol = columns && columns.findIndex(x => x.name === field) !== -1
-      if (!fieldSchema?.autocolumn && !hasCol) {
-        newColumns.push({
-          name: field,
-          displayName: field,
-        })
-      }
-    })
-    columns = newColumns
-  }
-
-  const reset = () => {
-    columns = []
-  }
-
-  const getFieldType = column => {
-    return `validation/${schema[column.name]?.type}`
-  }
-</script>
-
-<DrawerContent>
-  <div class="container">
-    <Layout noPadding gap="S">
-      {#if columns?.length}
-        <Layout noPadding gap="XS">
-          <div class="column">
-            <div />
-            <Label size="L">Column</Label>
-            <Label size="L">Label</Label>
-            <div />
-            <div />
-          </div>
-          <div
-            class="columns"
-            use:dndzone={{
-              items: columns,
-              flipDurationMs,
-              dropTargetStyle: { outline: "none" },
-              dragDisabled,
-            }}
-            on:finalize={handleFinalize}
-            on:consider={updateColumnOrder}
-          >
-            {#each columns as column (column.id)}
-              <div class="column" animate:flip={{ duration: flipDurationMs }}>
-                <div
-                  class="handle"
-                  aria-label="drag-handle"
-                  style={dragDisabled ? "cursor: grab" : "cursor: grabbing"}
-                  on:mousedown={() => (dragDisabled = false)}
-                >
-                  <Icon name="DragHandle" size="XL" />
-                </div>
-                <Select
-                  bind:value={column.name}
-                  placeholder="Column"
-                  options={getRemainingColumnOptions(column.name)}
-                  on:change={e => (column.displayName = e.detail)}
-                />
-                <Input bind:value={column.displayName} placeholder="Label" />
-                <CellEditor type={getFieldType(column)} bind:column />
-                <Icon
-                  name="Close"
-                  hoverable
-                  size="S"
-                  on:click={() => removeColumn(column.id)}
-                  disabled={columns.length === 1}
-                />
-              </div>
-            {/each}
-          </div>
-        </Layout>
-      {:else}
-        <div class="column">
-          <div class="wide">
-            <Body size="S">Add columns to be included in your form below.</Body>
-          </div>
-        </div>
-      {/if}
-      <div class="column">
-        <div class="buttons wide">
-          <Button secondary icon="Add" on:click={addColumn}>Add column</Button>
-          <Button secondary quiet on:click={addAllColumns}>
-            Add all columns
-          </Button>
-          {#if columns?.length}
-            <Button secondary quiet on:click={reset}>Reset columns</Button>
-          {/if}
-        </div>
-      </div>
-    </Layout>
-  </div>
-</DrawerContent>
-
-<style>
-  .container {
-    width: 100%;
-    max-width: 600px;
-    margin: 0 auto;
-  }
-  .columns {
-    display: flex;
-    flex-direction: column;
-    justify-content: flex-start;
-    align-items: stretch;
-    gap: var(--spacing-m);
-  }
-  .column {
-    gap: var(--spacing-l);
-    display: grid;
-    grid-template-columns: 20px 1fr 1fr 16px 16px;
-    align-items: center;
-    border-radius: var(--border-radius-s);
-    transition: background-color ease-in-out 130ms;
-  }
-  .column:hover {
-    background-color: var(--spectrum-global-color-gray-100);
-  }
-  .handle {
-    display: grid;
-    place-items: center;
-  }
-  .wide {
-    grid-column: 2 / -1;
-  }
-  .buttons {
-    display: flex;
-    flex-direction: row;
-    justify-content: flex-start;
-    align-items: center;
-    gap: var(--spacing-m);
-  }
-</style>

packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldConfiguration.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { Toggle } from "@budibase/bbui"
   import { cloneDeep, isEqual } from "lodash/fp"
   import {
     getDatasourceForProvider,
@@ -26,6 +27,8 @@
   let sanitisedValue
   let unconfigured
 
+  let selectAll = true
+
   $: bindings = getBindableProperties($selectedScreen, componentInstance._id)
   $: actionType = componentInstance.actionType
   let componentBindings = []
@@ -146,16 +149,31 @@
     dispatch("change", getValidColumns(parentFieldsUpdated, options))
   }
 
-  const listUpdated = e => {
+  const listUpdated = columns => {
-    const parsedColumns = getValidColumns(e.detail, options)
+    const parsedColumns = getValidColumns(columns, options)
     dispatch("change", parsedColumns)
   }
 </script>
 
 <div class="field-configuration">
+  <div class="toggle-all">
+    <span />
+    <Toggle
+      on:change={() => {
+        let update = fieldList.map(field => ({
+          ...field,
+          active: selectAll,
+        }))
+        listUpdated(update)
+      }}
+      text=""
+      bind:value={selectAll}
+      thin
+    />
+  </div>
   {#if fieldList?.length}
     <DraggableList
-      on:change={listUpdated}
+      on:change={e => listUpdated(e.detail)}
       on:itemChange={processItemUpdate}
       items={fieldList}
       listItemKey={"_id"}
@@ -172,4 +190,21 @@
   .field-configuration :global(.spectrum-ActionButton) {
     width: 100%;
   }
+  .toggle-all {
+    display: flex;
+    justify-content: space-between;
+  }
+  .toggle-all :global(.spectrum-Switch) {
+    margin-right: 0px;
+    padding-right: calc(var(--spacing-s) - 1px);
+    min-height: unset;
+  }
+  .toggle-all :global(.spectrum-Switch .spectrum-Switch-switch) {
+    margin-top: 0px;
+  }
+  .toggle-all span {
+    color: var(--spectrum-global-color-gray-700);
+    font-size: 12px;
+    margin-left: calc(var(--spacing-s) - 1px);
+  }
 </style>

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -404,7 +404,7 @@
     datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
     const datasourceUrl = datasource?.config.url
     const qs = query?.fields.queryString
-    breakQs = restUtils.breakQueryString(qs)
+    breakQs = restUtils.breakQueryString(encodeURI(qs))
     breakQs = runtimeToReadableMap(mergedBindings, breakQs)
 
     const path = query.fields.path
@@ -652,7 +652,7 @@
     <div class="bottom">
       <Layout paddingY="S" gap="S">
         <Divider />
-        {#if !response && Object.keys(schema).length === 0}
+        {#if !response && Object.keys(schema || {}).length === 0}
           <Heading size="M">Response</Heading>
           <div class="placeholder">
             <div class="placeholder-internal">

packages/client/src/components/app/Text.svelte

@@ -94,7 +94,7 @@
   .align--right {
     text-align: right;
   }
-  .align-justify {
+  .align--justify {
     text-align: justify;
   }
 </style>

packages/server/src/api/controllers/row/internal.ts

@@ -1,9 +1,5 @@
 import * as linkRows from "../../../db/linkedRows"
-import {
+import { generateRowID, InternalTables } from "../../../db/utils"
-  generateRowID,
-  getMultiIDParams,
-  InternalTables,
-} from "../../../db/utils"
 import * as userController from "../user"
 import {
   cleanupAttachments,
@@ -240,8 +236,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
   const linkVals = links as LinkDocumentValue[]
 
   // look up the actual rows based on the ids
-  const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
+  let linkedRows = await db.getMultiple<Row>(
-  let linkedRows = (await db.allDocs<Row>(params)).rows.map(row => row.doc!)
+    linkVals.map(linkVal => linkVal.id),
+    { allowMissing: true }
+  )
 
   // get the linked tables
   const linkTableIds = getLinkedTableIDs(table as Table)

packages/server/src/api/controllers/row/utils.ts

@@ -1,21 +1,9 @@
 import { InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import { context } from "@budibase/backend-core"
-import {
+import { Ctx, Row, UserCtx } from "@budibase/types"
-  Ctx,
-  FieldType,
-  ManyToOneRelationshipFieldMetadata,
-  OneToManyRelationshipFieldMetadata,
-  Row,
-  SearchFilters,
-  Table,
-  UserCtx,
-} from "@budibase/types"
-import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
-import sdk from "../../../sdk"
 
 import validateJs from "validate.js"
-import { cloneDeep } from "lodash/fp"
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {

packages/server/src/db/linkedRows/index.ts

@@ -8,7 +8,7 @@ import {
   getLinkedTable,
 } from "./linkUtils"
 import flatten from "lodash/flatten"
-import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils"
+import { USER_METDATA_PREFIX } from "../utils"
 import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
 import { processFormulas } from "../../utilities/rowProcessor"
@@ -79,9 +79,7 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) {
   const db = context.getAppDB()
   const linkedRowIds = links.map(link => link.id)
   const uniqueRowIds = [...new Set(linkedRowIds)]
-  let dbRows = (await db.allDocs<Row>(getMultiIDParams(uniqueRowIds))).rows.map(
+  let dbRows = await db.getMultiple<Row>(uniqueRowIds, { allowMissing: true })
-    row => row.doc!
-  )
   // convert the unique db rows back to a full list of linked rows
   const linked = linkedRowIds
     .map(id => dbRows.find(row => row && row._id === id))

packages/server/src/db/utils.ts

@@ -283,16 +283,6 @@ export function generatePluginID(name: string) {
   return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
 }
 
-/**
- * This can be used with the db.allDocs to get a list of IDs
- */
-export function getMultiIDParams(ids: string[]) {
-  return {
-    keys: ids,
-    include_docs: true,
-  }
-}
-
 /**
  * Generates a new view ID.
  * @returns The new view ID which the view doc can be stored under.

packages/server/src/integrations/redis.ts

@@ -165,10 +165,22 @@ class RedisIntegration {
       // commands split line by line
       const commands = query.json.trim().split("\n")
       let pipelineCommands = []
+      let tokenised
 
       // process each command separately
       for (let command of commands) {
-        const tokenised = command.trim().split(" ")
+        const valueToken = command.trim().match(/".*"/)
+        if (valueToken?.[0]) {
+          tokenised = [
+            ...command
+              .substring(0, command.indexOf(valueToken[0]) - 1)
+              .trim()
+              .split(" "),
+            valueToken?.[0],
+          ]
+        } else {
+          tokenised = command.trim().split(" ")
+        }
         // Pipeline only accepts lower case commands
         tokenised[0] = tokenised[0].toLowerCase()
         pipelineCommands.push(tokenised)

packages/server/src/integrations/tests/redis.spec.ts

@@ -85,4 +85,21 @@ describe("Redis Integration", () => {
       ["get", "foo"],
     ])
   })
+
+  it("calls the pipeline method with double quoted phrase values", async () => {
+    const body = {
+      json: 'SET foo "What a wonderful world!"\nGET foo',
+    }
+
+    // ioredis-mock doesn't support pipelines
+    config.integration.client.pipeline = jest.fn(() => ({
+      exec: jest.fn(() => [[]]),
+    }))
+
+    await config.integration.command(body)
+    expect(config.integration.client.pipeline).toHaveBeenCalledWith([
+      ["set", "foo", '"What a wonderful world!"'],
+      ["get", "foo"],
+    ])
+  })
 })

packages/server/src/sdk/app/tables/getters.ts

@@ -1,5 +1,5 @@
 import { context } from "@budibase/backend-core"
-import { getMultiIDParams, getTableParams } from "../../../db/utils"
+import { getTableParams } from "../../../db/utils"
 import {
   breakExternalTableId,
   isExternalTableID,
@@ -17,6 +17,9 @@ import datasources from "../datasources"
 import sdk from "../../../sdk"
 
 export function processTable(table: Table): Table {
+  if (!table) {
+    return table
+  }
   if (table._id && isExternalTableID(table._id)) {
     return {
       ...table,
@@ -73,6 +76,9 @@ export async function getExternalTable(
   tableName: string
 ): Promise<Table> {
   const entities = await getExternalTablesInDatasource(datasourceId)
+  if (!entities[tableName]) {
+    throw new Error(`Unable to find table named "${tableName}"`)
+  }
   return processTable(entities[tableName])
 }
 
@@ -124,10 +130,10 @@ export async function getTables(tableIds: string[]): Promise<Table[]> {
   }
   if (internalTableIds.length) {
     const db = context.getAppDB()
-    const internalTableDocs = await db.allDocs<Table>(
+    const internalTables = await db.getMultiple<Table>(internalTableIds, {
-      getMultiIDParams(internalTableIds)
+      allowMissing: true,
-    )
+    })
-    tables = tables.concat(internalTableDocs.rows.map(row => row.doc!))
+    tables = tables.concat(internalTables)
   }
   return processTables(tables)
 }

packages/server/src/sdk/tests/tables.spec.ts

@@ -0,0 +1,39 @@
+import TestConfig from "../../tests/utilities/TestConfiguration"
+import { basicTable } from "../../tests/utilities/structures"
+import { Table } from "@budibase/types"
+import sdk from "../"
+
+describe("tables", () => {
+  const config = new TestConfig()
+  let table: Table
+
+  beforeAll(async () => {
+    await config.init()
+    table = await config.api.table.create(basicTable())
+  })
+
+  describe("getTables", () => {
+    it("should be able to retrieve tables", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tables = await sdk.tables.getTables([table._id!])
+        expect(tables.length).toBe(1)
+        expect(tables[0]._id).toBe(table._id)
+        expect(tables[0].name).toBe(table.name)
+      })
+    })
+
+    it("shouldn't fail when retrieving tables that don't exist", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tables = await sdk.tables.getTables(["unknown"])
+        expect(tables.length).toBe(0)
+      })
+    })
+
+    it("should de-duplicate the IDs", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tables = await sdk.tables.getTables([table._id!, table._id!])
+        expect(tables.length).toBe(1)
+      })
+    })
+  })
+})

packages/server/src/utilities/global.ts

@@ -1,4 +1,4 @@
-import { getMultiIDParams, getGlobalIDFromUserMetadataID } from "../db/utils"
+import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import {
   roles,
   db as dbCore,
@@ -96,9 +96,7 @@ export async function getRawGlobalUsers(userIds?: string[]): Promise<User[]> {
   const db = tenancy.getGlobalDB()
   let globalUsers: User[]
   if (userIds) {
-    globalUsers = (await db.allDocs<User>(getMultiIDParams(userIds))).rows.map(
+    globalUsers = await db.getMultiple<User>(userIds, { allowMissing: true })
-      row => row.doc!
-    )
   } else {
     globalUsers = (
       await db.allDocs<User>(

packages/types/src/sdk/db.ts

@@ -122,7 +122,11 @@ export interface Database {
 
   exists(): Promise<boolean>
   checkSetup(): Promise<Nano.DocumentScope<any>>
-  get<T>(id?: string): Promise<T>
+  get<T extends Document>(id?: string): Promise<T>
+  getMultiple<T extends Document>(
+    ids: string[],
+    opts?: { allowMissing?: boolean }
+  ): Promise<T[]>
   remove(
     id: string | Document,
     rev?: string