Merge pull request #12000 from Budibase/feature/multi-user-type-column

User type column improvements

.github/workflows/budibase_ci.yml

@@ -264,18 +264,23 @@ jobs:
 
           if [[ $branch == "master" ]]; then
             base_commit=$(git rev-parse origin/master)
-          else
+          elif [[ $branch == "develop" ]]; then
             base_commit=$(git rev-parse origin/develop)
           fi
 
+          if [[ ! -z $base_commit ]]; then
             echo "target_branch=$branch"
             echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
             echo "pro_commit=$pro_commit"
             echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT"
             echo "base_commit=$base_commit"
             echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
+          else
+            echo "Nothing to do - branch to branch merge."
+          fi
 
-      - name: Check submodule merged to develop
+      - name: Check submodule merged to base branch
+        if: ${{ steps.get_pro_commits.outputs.base_commit != '' }}
         uses: actions/github-script@v4
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -284,7 +289,7 @@ jobs:
             const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}';
 
             if (submoduleCommit !== baseCommit) {
-              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}"" branch.');
+              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.');
               console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md')
               process.exit(1);
             } else {

packages/backend-core/package.json

@@ -62,7 +62,7 @@
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
     "@types/cookies": "0.7.8",
-    "@types/jest": "29.5.3",
+    "@types/jest": "29.5.5",
     "@types/lodash": "4.14.180",
     "@types/node": "18.17.0",
     "@types/node-fetch": "2.6.4",

packages/backend-core/src/constants/db.ts

@@ -1,5 +1,10 @@
 import { prefixed, DocumentType } from "@budibase/types"
-export { SEPARATOR, UNICODE_MAX, DocumentType } from "@budibase/types"
+export {
+  SEPARATOR,
+  UNICODE_MAX,
+  DocumentType,
+  InternalTable,
+} from "@budibase/types"
 
 /**
  * Can be used to create a few different forms of querying a view.
@@ -30,10 +35,6 @@ export const DeprecatedViews = {
   ],
 }
 
-export enum InternalTable {
-  USER_METADATA = "ta_users",
-}
-
 export const StaticDatabases = {
   GLOBAL: {
     name: "global-db",

packages/backend-core/src/docIds/ids.ts

@@ -45,6 +45,11 @@ export function generateGlobalUserID(id?: any) {
   return `${DocumentType.USER}${SEPARATOR}${id || newid()}`
 }
 
+const isGlobalUserIDRegex = new RegExp(`^${DocumentType.USER}${SEPARATOR}.+`)
+export function isGlobalUserID(id: string) {
+  return isGlobalUserIDRegex.test(id)
+}
+
 /**
  * Generates a new user ID based on the passed in global ID.
  * @param {string} globalId The ID of the global user.

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -33,7 +33,7 @@
   import { getBindings } from "components/backend/DataTable/formula"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
-  import { FieldType } from "@budibase/types"
+  import { FieldType, FieldSubtype, SourceName } from "@budibase/types"
   import RelationshipSelector from "components/common/RelationshipSelector.svelte"
 
   const AUTO_TYPE = "auto"
@@ -43,7 +43,6 @@
   const NUMBER_TYPE = FIELDS.NUMBER.type
   const JSON_TYPE = FIELDS.JSON.type
   const DATE_TYPE = FIELDS.DATETIME.type
-  const USER_REFRENCE_TYPE = FIELDS.BB_REFERENCE_USER.compositeType
 
   const dispatch = createEventDispatcher()
   const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@@ -52,7 +51,19 @@
   export let field
 
   let mounted = false
-  let fieldDefinitions = cloneDeep(FIELDS)
+  const fieldDefinitions = Object.values(FIELDS).reduce(
+    // Storing the fields by complex field id
+    (acc, field) => ({
+      ...acc,
+      [makeFieldId(field.type, field.subtype)]: field,
+    }),
+    {}
+  )
+
+  function makeFieldId(type, subtype) {
+    return `${type}${subtype || ""}`.toUpperCase()
+  }
+
   let originalName
   let linkEditDisabled
   let primaryDisplay
@@ -72,8 +83,8 @@
   let jsonSchemaModal
   let allowedTypes = []
   let editableColumn = {
-    type: fieldDefinitions.STRING.type,
+    type: FIELDS.STRING.type,
-    constraints: fieldDefinitions.STRING.constraints,
+    constraints: FIELDS.STRING.constraints,
     // Initial value for column name in other table for linked records
     fieldName: $tables.selected.name,
   }
@@ -139,9 +150,6 @@
         $tables.selected.primaryDisplay == null ||
         $tables.selected.primaryDisplay === editableColumn.name
 
-      if (editableColumn.type === FieldType.BB_REFERENCE) {
-        editableColumn.type = `${editableColumn.type}_${editableColumn.subtype}`
-      }
       // Here we are setting the relationship values based on the editableColumn
       // This part of the code is used when viewing an existing field hence the check
       // for the tableId
@@ -172,7 +180,17 @@
       }
     }
 
-    allowedTypes = getAllowedTypes()
+    if (!savingColumn) {
+      editableColumn.fieldId = makeFieldId(
+        editableColumn.type,
+        editableColumn.subtype
+      )
+
+      allowedTypes = getAllowedTypes().map(t => ({
+        fieldId: makeFieldId(t.type, t.subtype),
+        ...t,
+      }))
+    }
   }
 
   $: initialiseField(field, savingColumn)
@@ -249,13 +267,7 @@
 
     let saveColumn = cloneDeep(editableColumn)
 
-    // Handle types on composite types
+    delete saveColumn.fieldId
-    const definition = fieldDefinitions[saveColumn.type.toUpperCase()]
-    if (definition && saveColumn.type === definition.compositeType) {
-      saveColumn.type = definition.type
-      saveColumn.subtype = definition.subtype
-      delete saveColumn.compositeType
-    }
 
     if (saveColumn.type === AUTO_TYPE) {
       saveColumn = buildAutoColumn(
@@ -320,27 +332,33 @@
     }
   }
 
-  function handleTypeChange(event) {
+  function onHandleTypeChange(event) {
+    handleTypeChange(event.detail)
+  }
+
+  function handleTypeChange(type) {
     // remove any extra fields that may not be related to this type
     delete editableColumn.autocolumn
     delete editableColumn.subtype
     delete editableColumn.tableId
     delete editableColumn.relationshipType
     delete editableColumn.formulaType
+    delete editableColumn.constraints
 
     // Add in defaults and initial definition
-    const definition = fieldDefinitions[event.detail?.toUpperCase()]
+    const definition = fieldDefinitions[type?.toUpperCase()]
     if (definition?.constraints) {
       editableColumn.constraints = definition.constraints
     }
 
+    editableColumn.type = definition.type
+    editableColumn.subtype = definition.subtype
+
     // Default relationships many to many
     if (editableColumn.type === LINK_TYPE) {
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
     } else if (editableColumn.type === FORMULA_TYPE) {
       editableColumn.formulaType = "dynamic"
-    } else if (editableColumn.type === USER_REFRENCE_TYPE) {
-      editableColumn.relationshipType = RelationshipType.ONE_TO_MANY
     }
   }
 
@@ -381,9 +399,26 @@
       return ALLOWABLE_NUMBER_OPTIONS
     }
 
+    const isUsers =
+      editableColumn.type === FieldType.BB_REFERENCE &&
+      editableColumn.subtype === FieldSubtype.USERS
+
     if (!external) {
       return [
-        ...Object.values(fieldDefinitions),
+        FIELDS.STRING,
+        FIELDS.BARCODEQR,
+        FIELDS.LONGFORM,
+        FIELDS.OPTIONS,
+        FIELDS.ARRAY,
+        FIELDS.NUMBER,
+        FIELDS.BIGINT,
+        FIELDS.BOOLEAN,
+        FIELDS.DATETIME,
+        FIELDS.ATTACHMENT,
+        FIELDS.LINK,
+        FIELDS.FORMULA,
+        FIELDS.JSON,
+        isUsers ? FIELDS.USERS : FIELDS.USER,
         { name: "Auto Column", type: AUTO_TYPE },
       ]
     } else {
@@ -397,7 +432,7 @@
         FIELDS.BOOLEAN,
         FIELDS.FORMULA,
         FIELDS.BIGINT,
-        FIELDS.BB_REFERENCE_USER,
+        isUsers ? FIELDS.USERS : FIELDS.USER,
       ]
       // no-sql or a spreadsheet
       if (!external || table.sql) {
@@ -472,6 +507,13 @@
     return newError
   }
 
+  function isUsersColumn(column) {
+    return (
+      column.type === FieldType.BB_REFERENCE &&
+      [FieldSubtype.USER, FieldSubtype.USERS].includes(column.subtype)
+    )
+  }
+
   onMount(() => {
     mounted = true
   })
@@ -489,11 +531,11 @@
   {/if}
   <Select
     disabled={!typeEnabled}
-    bind:value={editableColumn.type}
+    bind:value={editableColumn.fieldId}
-    on:change={handleTypeChange}
+    on:change={onHandleTypeChange}
     options={allowedTypes}
     getOptionLabel={field => field.name}
-    getOptionValue={field => field.compositeType || field.type}
+    getOptionValue={field => field.fieldId}
     getOptionIcon={field => field.icon}
     isOptionEnabled={option => {
       if (option.type == AUTO_TYPE) {
@@ -555,7 +597,7 @@
         <DatePicker bind:value={editableColumn.constraints.datetime.latest} />
       </div>
     </div>
-    {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER" && !editableColumn.dateOnly}
+    {#if datasource?.source !== SourceName.ORACLE && datasource?.source !== SourceName.SQL_SERVER && !editableColumn.dateOnly}
       <div>
         <div class="row">
           <Label>Time zones</Label>
@@ -659,18 +701,20 @@
     <Button primary text on:click={openJsonSchemaEditor}
       >Open schema editor</Button
     >
-  {:else if editableColumn.type === USER_REFRENCE_TYPE}
+  {:else if isUsersColumn(editableColumn) && datasource?.source !== SourceName.GOOGLE_SHEETS}
-    <!-- Disabled temporally -->
+    <Toggle
-    <!-- <Toggle
+      value={editableColumn.subtype === FieldSubtype.USERS}
-      value={editableColumn.relationshipType === RelationshipType.MANY_TO_MANY}
       on:change={e =>
-        (editableColumn.relationshipType = e.detail
+        handleTypeChange(
-          ? RelationshipType.MANY_TO_MANY
+          makeFieldId(
-          : RelationshipType.ONE_TO_MANY)}
+            FieldType.BB_REFERENCE,
+            e.detail ? FieldSubtype.USERS : FieldSubtype.USER
+          )
+        )}
       disabled={!isCreating}
       thin
       text="Allow multiple users"
-    /> -->
+    />
   {/if}
   {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
     <Select

packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte

@@ -49,6 +49,15 @@
       label: "Long Form Text",
       value: FIELDS.LONGFORM.type,
     },
+
+    {
+      label: "User",
+      value: `${FIELDS.USER.type}${FIELDS.USER.subtype}`,
+    },
+    {
+      label: "Users",
+      value: `${FIELDS.USERS.type}${FIELDS.USERS.subtype}`,
+    },
   ]
 
   $: {
@@ -143,7 +152,7 @@
       <div class="field">
         <span>{name}</span>
         <Select
-          value={schema[name]?.type}
+          value={`${schema[name]?.type}${schema[name]?.subtype || ""}`}
           options={typeOptions}
           placeholder={null}
           getOptionLabel={option => option.label}

packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte

@@ -3,6 +3,7 @@
   import { FIELDS } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
+  import { canBeDisplayColumn } from "@budibase/shared-core"
 
   export let rows = []
   export let schema = {}
@@ -10,36 +11,82 @@
   export let displayColumn = null
   export let promptUpload = false
 
-  const typeOptions = [
+  const typeOptions = {
-    {
+    [FIELDS.STRING.type]: {
       label: "Text",
       value: FIELDS.STRING.type,
+      config: {
+        type: FIELDS.STRING.type,
+        constraints: FIELDS.STRING.constraints,
       },
-    {
+    },
+    [FIELDS.NUMBER.type]: {
       label: "Number",
       value: FIELDS.NUMBER.type,
+      config: {
+        type: FIELDS.NUMBER.type,
+        constraints: FIELDS.NUMBER.constraints,
       },
-    {
+    },
+    [FIELDS.DATETIME.type]: {
       label: "Date",
       value: FIELDS.DATETIME.type,
+      config: {
+        type: FIELDS.DATETIME.type,
+        constraints: FIELDS.DATETIME.constraints,
       },
-    {
+    },
+    [FIELDS.OPTIONS.type]: {
       label: "Options",
       value: FIELDS.OPTIONS.type,
+      config: {
+        type: FIELDS.OPTIONS.type,
+        constraints: FIELDS.OPTIONS.constraints,
       },
-    {
+    },
+    [FIELDS.ARRAY.type]: {
       label: "Multi-select",
       value: FIELDS.ARRAY.type,
+      config: {
+        type: FIELDS.ARRAY.type,
+        constraints: FIELDS.ARRAY.constraints,
       },
-    {
+    },
+    [FIELDS.BARCODEQR.type]: {
       label: "Barcode/QR",
       value: FIELDS.BARCODEQR.type,
+      config: {
+        type: FIELDS.BARCODEQR.type,
+        constraints: FIELDS.BARCODEQR.constraints,
       },
-    {
+    },
+    [FIELDS.LONGFORM.type]: {
       label: "Long Form Text",
       value: FIELDS.LONGFORM.type,
+      config: {
+        type: FIELDS.LONGFORM.type,
+        constraints: FIELDS.LONGFORM.constraints,
       },
-  ]
+    },
+    user: {
+      label: "User",
+      value: "user",
+      config: {
+        type: FIELDS.USER.type,
+        subtype: FIELDS.USER.subtype,
+        constraints: FIELDS.USER.constraints,
+      },
+    },
+    users: {
+      label: "Users",
+      value: "users",
+      config: {
+        type: FIELDS.USERS.type,
+        subtype: FIELDS.USERS.subtype,
+        constraints: FIELDS.USERS.constraints,
+      },
+    },
+  }
 
   let fileInput
   let error = null
@@ -48,10 +95,16 @@
   let validation = {}
   let validateHash = ""
   let errors = {}
+  let selectedColumnTypes = {}
 
   $: displayColumnOptions = Object.keys(schema || {}).filter(column => {
-    return validation[column]
+    return validation[column] && canBeDisplayColumn(schema[column].type)
   })
+
+  $: if (displayColumn && !canBeDisplayColumn(schema[displayColumn].type)) {
+    displayColumn = null
+  }
+
   $: {
     // binding in consumer is causing double renders here
     const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
@@ -72,6 +125,13 @@
       rows = response.rows
       schema = response.schema
       fileName = response.fileName
+      selectedColumnTypes = Object.entries(response.schema).reduce(
+        (acc, [colName, fieldConfig]) => ({
+          ...acc,
+          [colName]: fieldConfig.type,
+        }),
+        {}
+      )
     } catch (e) {
       loading = false
       error = e
@@ -98,8 +158,10 @@
   }
 
   const handleChange = (name, e) => {
-    schema[name].type = e.detail
+    const { config } = typeOptions[e.detail]
-    schema[name].constraints = FIELDS[e.detail.toUpperCase()].constraints
+    schema[name].type = config.type
+    schema[name].subtype = config.subtype
+    schema[name].constraints = config.constraints
   }
 
   const openFileUpload = (promptUpload, fileInput) => {
@@ -142,9 +204,9 @@
       <div class="field">
         <span>{column.name}</span>
         <Select
-          bind:value={column.type}
+          bind:value={selectedColumnTypes[column.name]}
           on:change={e => handleChange(name, e)}
-          options={typeOptions}
+          options={Object.values(typeOptions)}
           placeholder={null}
           getOptionLabel={option => option.label}
           getOptionValue={option => option.value}

packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte

@@ -20,7 +20,6 @@
   import { FieldType } from "@budibase/types"
   import { createEventDispatcher, onMount } from "svelte"
   import FilterUsers from "./FilterUsers.svelte"
-  import { RelationshipType } from "constants/backend"
 
   export let schemaFields
   export let filters = []
@@ -126,6 +125,7 @@
     // Update type based on field
     const fieldSchema = enrichedSchemaFields.find(x => x.name === filter.field)
     filter.type = fieldSchema?.type
+    filter.subtype = fieldSchema?.subtype
 
     // Update external type based on field
     filter.externalType = getSchema(filter)?.externalType
@@ -196,7 +196,7 @@
     }
 
     return LuceneUtils.getValidOperatorsForType(
-      filter.type,
+      { type: filter.type, subtype: filter.subtype },
       filter.field,
       datasource
     )
@@ -301,9 +301,10 @@
               {:else if filter.type === FieldType.BB_REFERENCE}
                 <FilterUsers
                   bind:value={filter.value}
-                  multiselect={getSchema(filter).relationshipType ===
+                  multiselect={[
-                    RelationshipType.MANY_TO_MANY ||
+                    OperatorOptions.In.value,
-                    filter.operator === OperatorOptions.In.value}
+                    OperatorOptions.ContainsAny.value,
+                  ].includes(filter.operator)}
                   disabled={filter.noValue}
                 />
               {:else}

packages/builder/src/constants/backend/index.js

@@ -1,7 +1,9 @@
+import { FieldType, FieldSubtype } from "@budibase/types"
+
 export const FIELDS = {
   STRING: {
     name: "Text",
-    type: "string",
+    type: FieldType.STRING,
     icon: "Text",
     constraints: {
       type: "string",
@@ -11,7 +13,7 @@ export const FIELDS = {
   },
   BARCODEQR: {
     name: "Barcode/QR",
-    type: "barcodeqr",
+    type: FieldType.BARCODEQR,
     icon: "Camera",
     constraints: {
       type: "string",
@@ -21,7 +23,7 @@ export const FIELDS = {
   },
   LONGFORM: {
     name: "Long Form Text",
-    type: "longform",
+    type: FieldType.LONGFORM,
     icon: "TextAlignLeft",
     constraints: {
       type: "string",
@@ -31,7 +33,7 @@ export const FIELDS = {
   },
   OPTIONS: {
     name: "Options",
-    type: "options",
+    type: FieldType.OPTIONS,
     icon: "Dropdown",
     constraints: {
       type: "string",
@@ -41,7 +43,7 @@ export const FIELDS = {
   },
   ARRAY: {
     name: "Multi-select",
-    type: "array",
+    type: FieldType.ARRAY,
     icon: "Duplicate",
     constraints: {
       type: "array",
@@ -51,7 +53,7 @@ export const FIELDS = {
   },
   NUMBER: {
     name: "Number",
-    type: "number",
+    type: FieldType.NUMBER,
     icon: "123",
     constraints: {
       type: "number",
@@ -61,12 +63,12 @@ export const FIELDS = {
   },
   BIGINT: {
     name: "BigInt",
-    type: "bigint",
+    type: FieldType.BIGINT,
     icon: "TagBold",
   },
   BOOLEAN: {
     name: "Boolean",
-    type: "boolean",
+    type: FieldType.BOOLEAN,
     icon: "Boolean",
     constraints: {
       type: "boolean",
@@ -75,7 +77,7 @@ export const FIELDS = {
   },
   DATETIME: {
     name: "Date/Time",
-    type: "datetime",
+    type: FieldType.DATETIME,
     icon: "Calendar",
     constraints: {
       type: "string",
@@ -89,7 +91,7 @@ export const FIELDS = {
   },
   ATTACHMENT: {
     name: "Attachment",
-    type: "attachment",
+    type: FieldType.ATTACHMENT,
     icon: "Folder",
     constraints: {
       type: "array",
@@ -98,7 +100,7 @@ export const FIELDS = {
   },
   LINK: {
     name: "Relationship",
-    type: "link",
+    type: FieldType.LINK,
     icon: "Link",
     constraints: {
       type: "array",
@@ -107,26 +109,34 @@ export const FIELDS = {
   },
   FORMULA: {
     name: "Formula",
-    type: "formula",
+    type: FieldType.FORMULA,
     icon: "Calculator",
     constraints: {},
   },
   JSON: {
     name: "JSON",
-    type: "json",
+    type: FieldType.JSON,
     icon: "Brackets",
     constraints: {
       type: "object",
       presence: false,
     },
   },
-  BB_REFERENCE_USER: {
+  USER: {
     name: "User",
-    type: "bb_reference",
+    type: FieldType.BB_REFERENCE,
-    subtype: "user",
+    subtype: FieldSubtype.USER,
-    compositeType: "bb_reference_user", // Used for working with the subtype on CreateEditColumn as is it was a primary type
     icon: "User",
   },
+  USERS: {
+    name: "Users",
+    type: FieldType.BB_REFERENCE,
+    subtype: FieldSubtype.USERS,
+    icon: "User",
+    constraints: {
+      type: "array",
+    },
+  },
 }
 
 export const AUTO_COLUMN_SUB_TYPES = {

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte

@@ -118,7 +118,7 @@
   }
 
   const getOperatorOptions = condition => {
-    return LuceneUtils.getValidOperatorsForType(condition.valueType)
+    return LuceneUtils.getValidOperatorsForType({ type: condition.valueType })
   }
 
   const onOperatorChange = (condition, newOperator) => {
@@ -137,9 +137,9 @@
     condition.referenceValue = null
 
     // Ensure a valid operator is set
-    const validOperators = LuceneUtils.getValidOperatorsForType(newType).map(
+    const validOperators = LuceneUtils.getValidOperatorsForType({
-      x => x.value
+      type: newType,
-    )
+    }).map(x => x.value)
     if (!validOperators.includes(condition.operator)) {
       condition.operator =
         validOperators[0] ?? Constants.OperatorOptions.Equals.value

packages/client/manifest.json

@@ -5673,11 +5673,6 @@
         "label": "Validation",
         "key": "validation"
       },
-      {
-        "type": "filter/relationship",
-        "label": "Filtering",
-        "key": "filter"
-      },
       {
         "type": "boolean",
         "label": "Search",

packages/client/src/components/app/dynamic-filter/FilterModal.svelte

@@ -63,7 +63,7 @@
 
     // Ensure a valid operator is set
     const validOperators = LuceneUtils.getValidOperatorsForType(
-      expression.type,
+      { type: expression.type },
       expression.field,
       datasource
     ).map(x => x.value)
@@ -125,7 +125,7 @@
           <Select
             disabled={!filter.field}
             options={LuceneUtils.getValidOperatorsForType(
-              filter.type,
+              { type: filter.type, subtype: filter.subtype },
               filter.field,
               datasource
             )}

packages/client/src/components/app/forms/BBReferenceField.svelte

@@ -1,9 +1,28 @@
 <script>
   import RelationshipField from "./RelationshipField.svelte"
+  import { sdk } from "@budibase/shared-core"
+
+  export let defaultValue
+
+  function updateUserIDs(value) {
+    if (Array.isArray(value)) {
+      return value.map(val => sdk.users.getGlobalUserID(val))
+    } else {
+      return sdk.users.getGlobalUserID(value)
+    }
+  }
+
+  function updateReferences(value) {
+    if (sdk.users.containsUserID(value)) {
+      return updateUserIDs(value)
+    }
+    return value
+  }
 </script>
 
 <RelationshipField
   {...$$props}
   datasourceType={"user"}
   primaryDisplay={"email"}
+  defaultValue={updateReferences(defaultValue)}
 />

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -160,7 +160,9 @@
   const handleChange = value => {
     const changed = fieldApi.setValue(value)
     if (onChange && changed) {
-      onChange({ value })
+      onChange({
+        value,
+      })
     }
   }
 

packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte

@@ -1,7 +1,7 @@
 <script>
   import { getContext } from "svelte"
   import RelationshipCell from "./RelationshipCell.svelte"
-  import { FieldSubtype } from "@budibase/types"
+  import { FieldSubtype, RelationshipType } from "@budibase/types"
 
   export let api
 
@@ -12,10 +12,14 @@
     ...$$props.schema,
     // This is not really used, just adding some content to be able to render the relationship cell
     tableId: "external",
+    relationshipType:
+      subtype === FieldSubtype.USER
+        ? RelationshipType.ONE_TO_MANY
+        : RelationshipType.MANY_TO_MANY,
   }
 
   async function searchFunction(searchParams) {
-    if (subtype !== FieldSubtype.USER) {
+    if (subtype !== FieldSubtype.USER && subtype !== FieldSubtype.USERS) {
       throw `Search for '${subtype}' not implemented`
     }
 

packages/frontend-core/src/components/grid/cells/HeaderCell.svelte

@@ -1,7 +1,8 @@
 <script>
   import { getContext, onMount, tick } from "svelte"
-  import GridCell from "./GridCell.svelte"
+  import { canBeDisplayColumn } from "@budibase/shared-core"
   import { Icon, Popover, Menu, MenuItem, clickOutside } from "@budibase/bbui"
+  import GridCell from "./GridCell.svelte"
   import { getColumnIcon } from "../lib/utils"
 
   export let column
@@ -24,14 +25,6 @@
     datasource,
   } = getContext("grid")
 
-  const bannedDisplayColumnTypes = [
-    "link",
-    "array",
-    "attachment",
-    "boolean",
-    "json",
-  ]
-
   let anchor
   let open = false
   let editIsOpen = false
@@ -231,8 +224,7 @@
       <MenuItem
         icon="Label"
         on:click={makeDisplayColumn}
-        disabled={idx === "sticky" ||
+        disabled={idx === "sticky" || !canBeDisplayColumn(column.schema.type)}
-          bannedDisplayColumnTypes.includes(column.schema.type)}
       >
         Use as display column
       </MenuItem>

packages/frontend-core/src/components/grid/lib/utils.js

@@ -21,6 +21,7 @@ const TypeIconMap = {
   bigint: "TagBold",
   bb_reference: {
     user: "User",
+    users: "UserGroup",
   },
 }
 

packages/server/package.json

@@ -125,7 +125,7 @@
     "@trendyol/jest-testcontainers": "2.1.1",
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
-    "@types/jest": "29.5.3",
+    "@types/jest": "29.5.5",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
     "@types/lodash": "4.14.180",

packages/server/src/api/controllers/row/internal.ts

@@ -156,7 +156,10 @@ export async function destroy(ctx: UserCtx) {
   }
   const table = await sdk.tables.getTable(row.tableId)
   // update the row to include full relationships before deleting them
-  row = await outputProcessing(table, row, { squash: false })
+  row = await outputProcessing(table, row, {
+    squash: false,
+    skipBBReferences: true,
+  })
   // now remove the relationships
   await linkRows.updateLinks({
     eventType: linkRows.EventType.ROW_DELETE,
@@ -190,6 +193,7 @@ export async function bulkDestroy(ctx: UserCtx) {
   // they need to be the full rows (including previous relationships) for automations
   const processedRows = (await outputProcessing(table, rows, {
     squash: false,
+    skipBBReferences: true,
   })) as Row[]
 
   // remove the relationships first

packages/server/src/api/controllers/table/external.ts

@@ -15,8 +15,8 @@ import { handleRequest } from "../row/external"
 import { context, events } from "@budibase/backend-core"
 import { isRows, isSchema, parse } from "../../../utilities/schema"
 import {
+  BulkImportRequest,
   Datasource,
-  FieldSchema,
   ManyToManyRelationshipFieldMetadata,
   ManyToOneRelationshipFieldMetadata,
   OneToManyRelationshipFieldMetadata,
@@ -385,10 +385,10 @@ export async function destroy(ctx: UserCtx) {
   return tableToDelete
 }
 
-export async function bulkImport(ctx: UserCtx) {
+export async function bulkImport(ctx: UserCtx<BulkImportRequest>) {
   const table = await sdk.tables.getTable(ctx.params.tableId)
-  const { rows }: { rows: unknown } = ctx.request.body
+  const { rows } = ctx.request.body
-  const schema: unknown = table.schema
+  const schema = table.schema
 
   if (!rows || !isRows(rows) || !isSchema(schema)) {
     ctx.throw(400, "Provided data import information is invalid.")

packages/server/src/api/controllers/table/index.ts

@@ -8,6 +8,7 @@ import {
 import { isExternalTable, isSQL } from "../../../integrations/utils"
 import { events } from "@budibase/backend-core"
 import {
+  BulkImportRequest,
   FetchTablesResponse,
   SaveTableRequest,
   SaveTableResponse,
@@ -97,7 +98,7 @@ export async function destroy(ctx: UserCtx) {
   builderSocket?.emitTableDeletion(ctx, deletedTable)
 }
 
-export async function bulkImport(ctx: UserCtx) {
+export async function bulkImport(ctx: UserCtx<BulkImportRequest>) {
   const tableId = ctx.params.tableId
   await pickApi({ tableId }).bulkImport(ctx)
   // right now we don't trigger anything for bulk import because it

packages/server/src/api/controllers/table/internal.ts

@@ -10,7 +10,7 @@ import {
 } from "../../../utilities/rowProcessor"
 import { runStaticFormulaChecks } from "./bulkFormula"
 import {
-  AutoColumnFieldMetadata,
+  BulkImportRequest,
   RenameColumn,
   SaveTableRequest,
   SaveTableResponse,
@@ -207,7 +207,7 @@ export async function destroy(ctx: any) {
   return tableToDelete
 }
 
-export async function bulkImport(ctx: any) {
+export async function bulkImport(ctx: UserCtx<BulkImportRequest>) {
   const table = await sdk.tables.getTable(ctx.params.tableId)
   const { rows, identifierFields } = ctx.request.body
   await handleDataImport(ctx.user, table, rows, identifierFields)

packages/server/src/api/controllers/table/utils.ts

@@ -20,7 +20,13 @@ import viewTemplate from "../view/viewBuilder"
 import { cloneDeep } from "lodash/fp"
 import { quotas } from "@budibase/pro"
 import { events, context } from "@budibase/backend-core"
-import { ContextUser, Datasource, SourceName, Table } from "@budibase/types"
+import {
+  ContextUser,
+  Datasource,
+  Row,
+  SourceName,
+  Table,
+} from "@budibase/types"
 
 export async function clearColumns(table: any, columnNames: any) {
   const db = context.getAppDB()
@@ -144,12 +150,12 @@ export async function importToRows(
 }
 
 export async function handleDataImport(
-  user: any,
+  user: ContextUser,
-  table: any,
+  table: Table,
-  rows: any,
+  rows: Row[],
   identifierFields: Array<string> = []
 ) {
-  const schema: unknown = table.schema
+  const schema = table.schema
 
   if (!rows || !isRows(rows) || !isSchema(schema)) {
     return table

packages/server/src/api/controllers/view/exporters.ts

@@ -43,3 +43,7 @@ export enum Format {
 export function isFormat(format: any): format is Format {
   return Object.values(Format).includes(format as Format)
 }
+
+export function parseCsvExport<T>(value: string) {
+  return JSON.parse(value?.replace(/'/g, '"')) as T
+}

packages/server/src/api/routes/tests/row.spec.ts

@@ -1573,16 +1573,12 @@ describe.each([
         users: {
           name: "users",
           type: FieldType.BB_REFERENCE,
-          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+          subtype: FieldTypeSubtypes.BB_REFERENCE.USERS,
-          // TODO: users when all merged
         },
       }),
       () => config.createUser(),
       (row: Row) => ({
         _id: row._id,
-        email: row.email,
-        firstName: row.firstName,
-        lastName: row.lastName,
         primaryDisplay: row.email,
       }),
     ],

packages/server/src/api/routes/tests/table.spec.ts

@@ -2,6 +2,7 @@ import { generator } from "@budibase/backend-core/tests"
 import { events, context } from "@budibase/backend-core"
 import {
   FieldType,
+  SaveTableRequest,
   RelationshipType,
   Table,
   ViewCalculation,
@@ -52,7 +53,7 @@ describe("/tables", () => {
     })
 
     it("creates a table via data import", async () => {
-      const table = basicTable()
+      const table: SaveTableRequest = basicTable()
       table.rows = [{ name: "test-name", description: "test-desc" }]
 
       const res = await createTable(table)

packages/server/src/integrations/base/sqlTable.ts

@@ -1,5 +1,6 @@
 import { Knex, knex } from "knex"
 import {
+  FieldSubtype,
   NumberFieldMetadata,
   Operation,
   QueryJson,
@@ -10,6 +11,7 @@ import { breakExternalTableId } from "../utils"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
 import { FieldTypes, RelationshipType } from "../../constants"
+import { utils } from "@budibase/shared-core"
 
 function generateSchema(
   schema: CreateTableBuilder,
@@ -49,9 +51,21 @@ function generateSchema(
       case FieldTypes.OPTIONS:
       case FieldTypes.LONGFORM:
       case FieldTypes.BARCODEQR:
-      case FieldTypes.BB_REFERENCE:
         schema.text(key)
         break
+      case FieldTypes.BB_REFERENCE:
+        const subtype = column.subtype as FieldSubtype
+        switch (subtype) {
+          case FieldSubtype.USER:
+            schema.text(key)
+            break
+          case FieldSubtype.USERS:
+            schema.json(key)
+            break
+          default:
+            throw utils.unreachable(subtype)
+        }
+        break
       case FieldTypes.NUMBER:
         // if meta is specified then this is a junction table entry
         if (column.meta && column.meta.toKey && column.meta.toTable) {

packages/server/src/integrations/utils.ts

@@ -1,7 +1,12 @@
-import { SqlQuery, Table, SearchFilters, Datasource } from "@budibase/types"
+import {
+  SqlQuery,
+  Table,
+  SearchFilters,
+  Datasource,
+  FieldType,
+} from "@budibase/types"
 import { DocumentType, SEPARATOR } from "../db/utils"
 import {
-  FieldTypes,
   BuildSchemaErrors,
   InvalidColumns,
   NoEmptyFilterStrings,
@@ -13,57 +18,57 @@ const ROW_ID_REGEX = /^\[.*]$/g
 const ENCODED_SPACE = encodeURIComponent(" ")
 
 const SQL_NUMBER_TYPE_MAP = {
-  integer: FieldTypes.NUMBER,
+  integer: FieldType.NUMBER,
-  int: FieldTypes.NUMBER,
+  int: FieldType.NUMBER,
-  decimal: FieldTypes.NUMBER,
+  decimal: FieldType.NUMBER,
-  smallint: FieldTypes.NUMBER,
+  smallint: FieldType.NUMBER,
-  real: FieldTypes.NUMBER,
+  real: FieldType.NUMBER,
-  float: FieldTypes.NUMBER,
+  float: FieldType.NUMBER,
-  numeric: FieldTypes.NUMBER,
+  numeric: FieldType.NUMBER,
-  mediumint: FieldTypes.NUMBER,
+  mediumint: FieldType.NUMBER,
-  dec: FieldTypes.NUMBER,
+  dec: FieldType.NUMBER,
-  double: FieldTypes.NUMBER,
+  double: FieldType.NUMBER,
-  fixed: FieldTypes.NUMBER,
+  fixed: FieldType.NUMBER,
-  "double precision": FieldTypes.NUMBER,
+  "double precision": FieldType.NUMBER,
-  number: FieldTypes.NUMBER,
+  number: FieldType.NUMBER,
-  binary_float: FieldTypes.NUMBER,
+  binary_float: FieldType.NUMBER,
-  binary_double: FieldTypes.NUMBER,
+  binary_double: FieldType.NUMBER,
-  money: FieldTypes.NUMBER,
+  money: FieldType.NUMBER,
-  smallmoney: FieldTypes.NUMBER,
+  smallmoney: FieldType.NUMBER,
 }
 
 const SQL_DATE_TYPE_MAP = {
-  timestamp: FieldTypes.DATETIME,
+  timestamp: FieldType.DATETIME,
-  time: FieldTypes.DATETIME,
+  time: FieldType.DATETIME,
-  datetime: FieldTypes.DATETIME,
+  datetime: FieldType.DATETIME,
-  smalldatetime: FieldTypes.DATETIME,
+  smalldatetime: FieldType.DATETIME,
-  date: FieldTypes.DATETIME,
+  date: FieldType.DATETIME,
 }
 
 const SQL_DATE_ONLY_TYPES = ["date"]
 const SQL_TIME_ONLY_TYPES = ["time"]
 
 const SQL_STRING_TYPE_MAP = {
-  varchar: FieldTypes.STRING,
+  varchar: FieldType.STRING,
-  char: FieldTypes.STRING,
+  char: FieldType.STRING,
-  nchar: FieldTypes.STRING,
+  nchar: FieldType.STRING,
-  nvarchar: FieldTypes.STRING,
+  nvarchar: FieldType.STRING,
-  ntext: FieldTypes.STRING,
+  ntext: FieldType.STRING,
-  enum: FieldTypes.STRING,
+  enum: FieldType.STRING,
-  blob: FieldTypes.STRING,
+  blob: FieldType.STRING,
-  long: FieldTypes.STRING,
+  long: FieldType.STRING,
-  text: FieldTypes.STRING,
+  text: FieldType.STRING,
 }
 
 const SQL_BOOLEAN_TYPE_MAP = {
-  boolean: FieldTypes.BOOLEAN,
+  boolean: FieldType.BOOLEAN,
-  bit: FieldTypes.BOOLEAN,
+  bit: FieldType.BOOLEAN,
-  tinyint: FieldTypes.BOOLEAN,
+  tinyint: FieldType.BOOLEAN,
 }
 
 const SQL_MISC_TYPE_MAP = {
-  json: FieldTypes.JSON,
+  json: FieldType.JSON,
-  bigint: FieldTypes.BIGINT,
+  bigint: FieldType.BIGINT,
 }
 
 const SQL_TYPE_MAP = {
@@ -154,7 +159,7 @@ export function breakRowIdField(_id: string | { _id: string }): any[] {
 }
 
 export function convertSqlType(type: string) {
-  let foundType = FieldTypes.STRING
+  let foundType = FieldType.STRING
   const lcType = type.toLowerCase()
   let matchingTypes = []
   for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) {
@@ -169,7 +174,7 @@ export function convertSqlType(type: string) {
     }).internal
   }
   const schema: any = { type: foundType }
-  if (foundType === FieldTypes.DATETIME) {
+  if (foundType === FieldType.DATETIME) {
     schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lcType)
     schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lcType)
   }
@@ -212,7 +217,7 @@ export function shouldCopyRelationship(
   tableIds: string[]
 ) {
   return (
-    column.type === FieldTypes.LINK &&
+    column.type === FieldType.LINK &&
     column.tableId &&
     tableIds.includes(column.tableId)
   )
@@ -230,22 +235,23 @@ export function shouldCopySpecialColumn(
   column: { type: string },
   fetchedColumn: { type: string } | undefined
 ) {
-  const isFormula = column.type === FieldTypes.FORMULA
+  const isFormula = column.type === FieldType.FORMULA
   const specialTypes = [
-    FieldTypes.OPTIONS,
+    FieldType.OPTIONS,
-    FieldTypes.LONGFORM,
+    FieldType.LONGFORM,
-    FieldTypes.ARRAY,
+    FieldType.ARRAY,
-    FieldTypes.FORMULA,
+    FieldType.FORMULA,
+    FieldType.BB_REFERENCE,
   ]
   // column has been deleted, remove - formulas will never exist, always copy
   if (!isFormula && column && !fetchedColumn) {
     return false
   }
   const fetchedIsNumber =
-    !fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER
+    !fetchedColumn || fetchedColumn.type === FieldType.NUMBER
   return (
-    specialTypes.indexOf(column.type as FieldTypes) !== -1 ||
+    specialTypes.indexOf(column.type as FieldType) !== -1 ||
-    (fetchedIsNumber && column.type === FieldTypes.BOOLEAN)
+    (fetchedIsNumber && column.type === FieldType.BOOLEAN)
   )
 }
 

packages/server/src/sdk/app/rows/search/tests/utils.spec.ts

@@ -20,7 +20,21 @@ const tableWithUserCol: Table = {
   },
 }
 
-describe("searchInputMapping", () => {
+const tableWithUsersCol: Table = {
+  _id: tableId,
+  name: "table",
+  schema: {
+    user: {
+      name: "user",
+      type: FieldType.BB_REFERENCE,
+      subtype: FieldTypeSubtypes.BB_REFERENCE.USERS,
+    },
+  },
+}
+
+describe.each([tableWithUserCol, tableWithUsersCol])(
+  "searchInputMapping",
+  col => {
     const globalUserId = dbCore.generateGlobalUserID()
     const userMedataId = dbCore.generateUserMetadataID(globalUserId)
 
@@ -33,7 +47,7 @@ describe("searchInputMapping", () => {
           },
         },
       }
-    const output = searchInputMapping(tableWithUserCol, params)
+      const output = searchInputMapping(col, params)
       expect(output.query.equal!["1:user"]).toBe(globalUserId)
     })
 
@@ -46,7 +60,7 @@ describe("searchInputMapping", () => {
           },
         },
       }
-    const output = searchInputMapping(tableWithUserCol, params)
+      const output = searchInputMapping(col, params)
       expect(output.query.oneOf!["1:user"]).toStrictEqual([
         globalUserId,
         globalUserId,
@@ -63,7 +77,7 @@ describe("searchInputMapping", () => {
           },
         },
       }
-    const output = searchInputMapping(tableWithUserCol, params)
+      const output = searchInputMapping(col, params)
       expect(output.query.equal!["1:user"]).toBe(email)
     })
 
@@ -71,7 +85,8 @@ describe("searchInputMapping", () => {
       const params: any = {
         tableId,
       }
-    const output = searchInputMapping(tableWithUserCol, params)
+      const output = searchInputMapping(col, params)
       expect(output.query).toBeUndefined()
     })
-})
+  }
+)

packages/server/src/sdk/app/rows/search/utils.ts

@@ -5,8 +5,10 @@ import {
   Table,
   DocumentType,
   SEPARATOR,
+  FieldSubtype,
 } from "@budibase/types"
 import { db as dbCore } from "@budibase/backend-core"
+import { utils } from "@budibase/shared-core"
 
 function findColumnInQueries(
   column: string,
@@ -66,8 +68,14 @@ export function searchInputMapping(table: Table, options: SearchParams) {
   for (let [key, column] of Object.entries(table.schema)) {
     switch (column.type) {
       case FieldType.BB_REFERENCE:
-        if (column.subtype === FieldTypeSubtypes.BB_REFERENCE.USER) {
+        const subtype = column.subtype as FieldSubtype
+        switch (subtype) {
+          case FieldSubtype.USER:
+          case FieldSubtype.USERS:
             userColumnMapping(key, options)
+            break
+          default:
+            utils.unreachable(subtype)
         }
         break
     }

packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts

@@ -8,7 +8,7 @@ const ROW_PREFIX = DocumentType.ROW + SEPARATOR
 export async function processInputBBReferences(
   value: string | string[] | { _id: string } | { _id: string }[],
   subtype: FieldSubtype
-): Promise<string | null> {
+): Promise<string | string[] | null> {
   let referenceIds: string[] = []
 
   if (Array.isArray(value)) {
@@ -41,33 +41,39 @@ export async function processInputBBReferences(
 
   switch (subtype) {
     case FieldSubtype.USER:
+    case FieldSubtype.USERS:
       const { notFoundIds } = await cache.user.getUsers(referenceIds)
 
       if (notFoundIds?.length) {
         throw new InvalidBBRefError(notFoundIds[0], FieldSubtype.USER)
       }
 
-      break
+      if (subtype === FieldSubtype.USERS) {
-    default:
+        return referenceIds
-      throw utils.unreachable(subtype)
       }
 
       return referenceIds.join(",") || null
+
+    default:
+      throw utils.unreachable(subtype)
+  }
 }
 
 export async function processOutputBBReferences(
-  value: string,
+  value: string | string[],
   subtype: FieldSubtype
 ) {
-  if (typeof value !== "string") {
+  if (value === null || value === undefined) {
     // Already processed or nothing to process
     return value || undefined
   }
 
-  const ids = value.split(",").filter(id => !!id)
+  const ids =
+    typeof value === "string" ? value.split(",").filter(id => !!id) : value
 
   switch (subtype) {
     case FieldSubtype.USER:
+    case FieldSubtype.USERS:
       const { users } = await cache.user.getUsers(ids)
       if (!users.length) {
         return undefined
@@ -76,9 +82,6 @@ export async function processOutputBBReferences(
       return users.map(u => ({
         _id: u._id,
         primaryDisplay: u.email,
-        email: u.email,
-        firstName: u.firstName,
-        lastName: u.lastName,
       }))
 
     default:

packages/server/src/utilities/rowProcessor/index.ts

@@ -207,9 +207,14 @@ export async function inputProcessing(
 export async function outputProcessing<T extends Row[] | Row>(
   table: Table,
   rows: T,
-  opts: { squash?: boolean; preserveLinks?: boolean } = {
+  opts: {
+    squash?: boolean
+    preserveLinks?: boolean
+    skipBBReferences?: boolean
+  } = {
     squash: true,
     preserveLinks: false,
+    skipBBReferences: false,
   }
 ): Promise<T> {
   let safeRows: Row[]
@@ -225,10 +230,7 @@ export async function outputProcessing<T extends Row[] | Row>(
     ? await linkRows.attachFullLinkedDocs(table, safeRows)
     : safeRows
 
-  // process formulas
+  // process complex types: attachements, bb references...
-  enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
-
-  // set the attachments URLs
   for (let [property, column] of Object.entries(table.schema)) {
     if (column.type === FieldTypes.ATTACHMENT) {
       for (let row of enriched) {
@@ -239,7 +241,10 @@ export async function outputProcessing<T extends Row[] | Row>(
           attachment.url = objectStore.getAppFileUrl(attachment.key)
         })
       }
-    } else if (column.type == FieldTypes.BB_REFERENCE) {
+    } else if (
+      !opts.skipBBReferences &&
+      column.type == FieldTypes.BB_REFERENCE
+    ) {
       for (let row of enriched) {
         row[property] = await processOutputBBReferences(
           row[property],
@@ -248,6 +253,10 @@ export async function outputProcessing<T extends Row[] | Row>(
       }
     }
   }
+
+  // process formulas after the complex types had been processed
+  enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
+
   if (opts.squash) {
     enriched = (await linkRows.squashLinksToPrimaryDisplay(
       table,

packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts

@@ -180,9 +180,6 @@ describe("bbReferenceProcessor", () => {
           {
             _id: user._id,
             primaryDisplay: user.email,
-            email: user.email,
-            firstName: user.firstName,
-            lastName: user.lastName,
           },
         ])
         expect(cacheGetUsersSpy).toBeCalledTimes(1)
@@ -207,9 +204,6 @@ describe("bbReferenceProcessor", () => {
             [user1, user2].map(u => ({
               _id: u._id,
               primaryDisplay: u.email,
-              email: u.email,
-              firstName: u.firstName,
-              lastName: u.lastName,
             }))
           )
         )

packages/server/src/utilities/schema.ts

@@ -1,9 +1,13 @@
+import { FieldSubtype } from "@budibase/types"
 import { FieldTypes } from "../constants"
-import { ValidColumnNameRegex } from "@budibase/shared-core"
+import { ValidColumnNameRegex, utils } from "@budibase/shared-core"
+import { db } from "@budibase/backend-core"
+import { parseCsvExport } from "../api/controllers/view/exporters"
 
 interface SchemaColumn {
   readonly name: string
   readonly type: FieldTypes
+  readonly subtype: FieldSubtype
   readonly autocolumn?: boolean
   readonly constraints?: {
     presence: boolean
@@ -77,8 +81,14 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
   rows.forEach(row => {
     Object.entries(row).forEach(([columnName, columnData]) => {
       const columnType = schema[columnName]?.type
+      const columnSubtype = schema[columnName]?.subtype
       const isAutoColumn = schema[columnName]?.autocolumn
 
+      // If the column had an invalid value we don't want to override it
+      if (results.schemaValidation[columnName] === false) {
+        return
+      }
+
       // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
       if (typeof columnType !== "string") {
         results.invalidColumns.push(columnName)
@@ -112,6 +122,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
         isNaN(new Date(columnData).getTime())
       ) {
         results.schemaValidation[columnName] = false
+      } else if (
+        columnType === FieldTypes.BB_REFERENCE &&
+        !isValidBBReference(columnData, columnSubtype)
+      ) {
+        results.schemaValidation[columnName] = false
       } else {
         results.schemaValidation[columnName] = true
       }
@@ -138,6 +153,7 @@ export function parse(rows: Rows, schema: Schema): Rows {
       }
 
       const columnType = schema[columnName].type
+      const columnSubtype = schema[columnName].subtype
 
       if (columnType === FieldTypes.NUMBER) {
         // If provided must be a valid number
@@ -147,6 +163,23 @@ export function parse(rows: Rows, schema: Schema): Rows {
         parsedRow[columnName] = columnData
           ? new Date(columnData).toISOString()
           : columnData
+      } else if (columnType === FieldTypes.BB_REFERENCE) {
+        const parsedValues =
+          !!columnData && parseCsvExport<{ _id: string }[]>(columnData)
+        if (!parsedValues) {
+          parsedRow[columnName] = undefined
+        } else {
+          switch (columnSubtype) {
+            case FieldSubtype.USER:
+              parsedRow[columnName] = parsedValues[0]?._id
+              break
+            case FieldSubtype.USERS:
+              parsedRow[columnName] = parsedValues.map(u => u._id)
+              break
+            default:
+              utils.unreachable(columnSubtype)
+          }
+        }
       } else {
         parsedRow[columnName] = columnData
       }
@@ -155,3 +188,32 @@ export function parse(rows: Rows, schema: Schema): Rows {
     return parsedRow
   })
 }
+
+function isValidBBReference(
+  columnData: any,
+  columnSubtype: FieldSubtype
+): boolean {
+  switch (columnSubtype) {
+    case FieldSubtype.USER:
+    case FieldSubtype.USERS:
+      if (typeof columnData !== "string") {
+        return false
+      }
+      const userArray = parseCsvExport<{ _id: string }[]>(columnData)
+      if (!Array.isArray(userArray)) {
+        return false
+      }
+
+      if (columnSubtype === FieldSubtype.USER && userArray.length > 1) {
+        return false
+      }
+
+      const constainsWrongId = userArray.find(
+        user => !db.isGlobalUserID(user._id)
+      )
+      return !constainsWrongId
+
+    default:
+      throw utils.unreachable(columnSubtype)
+  }
+}

packages/shared-core/src/filters.ts

@@ -6,6 +6,7 @@ import {
   SearchFilter,
   SearchQuery,
   SearchQueryFields,
+  FieldSubtype,
 } from "@budibase/types"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
 import { deepGet } from "./helpers"
@@ -16,7 +17,7 @@ const HBS_REGEX = /{{([^{].*?)}}/g
  * Returns the valid operator options for a certain data type
  */
 export const getValidOperatorsForType = (
-  type: FieldType,
+  fieldType: { type: FieldType; subtype?: FieldSubtype },
   field: string,
   datasource: Datasource & { tableId: any } // TODO: is this table id ever populated?
 ) => {
@@ -43,6 +44,7 @@ export const getValidOperatorsForType = (
     value: string
     label: string
   }[] = []
+  const { type, subtype } = fieldType
   if (type === FieldType.STRING) {
     ops = stringOps
   } else if (type === FieldType.NUMBER || type === FieldType.BIGINT) {
@@ -59,8 +61,10 @@ export const getValidOperatorsForType = (
     ops = numOps
   } else if (type === FieldType.FORMULA) {
     ops = stringOps.concat([Op.MoreThan, Op.LessThan])
-  } else if (type === FieldType.BB_REFERENCE) {
+  } else if (type === FieldType.BB_REFERENCE && subtype == FieldSubtype.USER) {
     ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
+  } else if (type === FieldType.BB_REFERENCE && subtype == FieldSubtype.USERS) {
+    ops = [Op.Contains, Op.NotContains, Op.ContainsAny, Op.Empty, Op.NotEmpty]
   }
 
   // Only allow equal/not equal for _id in SQL tables

packages/shared-core/src/index.ts

@@ -3,3 +3,4 @@ export * as dataFilters from "./filters"
 export * as helpers from "./helpers"
 export * as utils from "./utils"
 export * as sdk from "./sdk"
+export * from "./table"

packages/shared-core/src/sdk/documents/users.ts

@@ -1,4 +1,10 @@
-import { ContextUser, User } from "@budibase/types"
+import {
+  ContextUser,
+  DocumentType,
+  SEPARATOR,
+  User,
+  InternalTable,
+} from "@budibase/types"
 import { getProdAppID } from "./applications"
 
 // checks if a user is specifically a builder, given an app ID
@@ -67,3 +73,21 @@ export function hasAdminPermissions(user?: User | ContextUser): boolean {
   }
   return !!user.admin?.global
 }
+
+export function getGlobalUserID(userId?: string): string | undefined {
+  if (typeof userId !== "string") {
+    return userId
+  }
+  const prefix = `${DocumentType.ROW}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}`
+  if (!userId.startsWith(prefix)) {
+    return userId
+  }
+  return userId.split(prefix)[1]
+}
+
+export function containsUserID(value: string | undefined): boolean {
+  if (typeof value !== "string") {
+    return false
+  }
+  return value.includes(`${DocumentType.USER}${SEPARATOR}`)
+}

packages/shared-core/src/table.ts

@@ -0,0 +1,25 @@
+import { FieldType } from "@budibase/types"
+
+const allowDisplayColumnByType: Record<FieldType, boolean> = {
+  [FieldType.STRING]: true,
+  [FieldType.LONGFORM]: true,
+  [FieldType.OPTIONS]: true,
+  [FieldType.NUMBER]: true,
+  [FieldType.DATETIME]: true,
+  [FieldType.FORMULA]: true,
+  [FieldType.AUTO]: true,
+  [FieldType.INTERNAL]: true,
+  [FieldType.BARCODEQR]: true,
+  [FieldType.BIGINT]: true,
+
+  [FieldType.BOOLEAN]: false,
+  [FieldType.ARRAY]: false,
+  [FieldType.ATTACHMENT]: false,
+  [FieldType.LINK]: false,
+  [FieldType.JSON]: false,
+  [FieldType.BB_REFERENCE]: false,
+}
+
+export function canBeDisplayColumn(type: FieldType): boolean {
+  return !!allowDisplayColumnByType[type]
+}

packages/shared-core/tsconfig.build.json

@@ -15,7 +15,8 @@
     "skipLibCheck": true,
     "paths": {
       "@budibase/types": ["../types/src"]
-    }
+    },
+    "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo"
   },
   "include": ["**/*.js", "**/*.ts"],
   "exclude": [

packages/shared-core/tsconfig.json

@@ -3,8 +3,7 @@
   "compilerOptions": {
     "baseUrl": "..",
     "rootDir": "src",
-    "composite": true,
+    "composite": true
-    "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo"
   },
   "exclude": ["node_modules", "dist"]
 }

packages/types/src/api/web/app/table.ts

@@ -1,4 +1,5 @@
 import {
+  Row,
   Table,
   TableRequest,
   TableSchema,
@@ -18,6 +19,13 @@ export interface TableResponse extends Table {
 
 export type FetchTablesResponse = TableResponse[]
 
-export interface SaveTableRequest extends TableRequest {}
+export interface SaveTableRequest extends TableRequest {
+  rows?: Row[]
+}
 
 export type SaveTableResponse = Table
+
+export interface BulkImportRequest {
+  rows: Row[]
+  identifierFields?: Array<string>
+}

packages/types/src/documents/app/row.ts

@@ -37,10 +37,12 @@ export interface Row extends Document {
 
 export enum FieldSubtype {
   USER = "user",
+  USERS = "users",
 }
 
 export const FieldTypeSubtypes = {
   BB_REFERENCE: {
     USER: FieldSubtype.USER,
+    USERS: FieldSubtype.USERS,
   },
 }

packages/types/src/documents/app/table/schema.ts

@@ -101,7 +101,7 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
 export interface BBReferenceFieldMetadata
   extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.BB_REFERENCE
-  subtype: FieldSubtype.USER
+  subtype: FieldSubtype.USER | FieldSubtype.USERS
 }
 
 export interface FieldConstraints {

packages/types/src/documents/app/table/table.ts

@@ -15,7 +15,6 @@ export interface Table extends Document {
   constrained?: string[]
   sql?: boolean
   indexes?: { [key: string]: any }
-  rows?: { [key: string]: any }
   created?: boolean
   rowHeight?: number
 }

packages/types/src/documents/document.ts

@@ -58,6 +58,10 @@ export const DocumentTypesToImport: DocumentType[] = [
   DocumentType.LAYOUT,
 ]
 
+export enum InternalTable {
+  USER_METADATA = "ta_users",
+}
+
 // these documents don't really exist, they are part of other
 // documents or enriched into existence as part of get requests
 export enum VirtualDocumentType {

packages/types/tsconfig.build.json

@@ -11,7 +11,8 @@
     "sourceMap": true,
     "declaration": true,
     "skipLibCheck": true,
-    "outDir": "dist"
+    "outDir": "dist",
+    "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo"
   },
   "include": ["src/**/*"],
   "exclude": ["node_modules", "dist", "**/*.spec.ts", "**/*.spec.js"]

packages/types/tsconfig.json

@@ -3,8 +3,7 @@
   "compilerOptions": {
     "baseUrl": ".",
     "rootDir": "./src",
-    "composite": true,
+    "composite": true
-    "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo"
   },
   "exclude": ["node_modules", "dist"]
 }

packages/worker/package.json

@@ -76,7 +76,7 @@
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@trendyol/jest-testcontainers": "2.1.1",
-    "@types/jest": "29.5.3",
+    "@types/jest": "29.5.5",
     "@types/jsonwebtoken": "8.5.1",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",

yarn.lock

@@ -4660,6 +4660,14 @@
     expect "^29.0.0"
     pretty-format "^29.0.0"
 
+"@types/jest@29.5.5":
+  version "29.5.5"
+  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-29.5.5.tgz#727204e06228fe24373df9bae76b90f3e8236a2a"
+  integrity sha512-ebylz2hnsWR9mYvmBFbXJXr+33UPc4+ZdxyDXh5w0FlPBTfCVN3wPL+kuOiQt3xvrK419v7XWeAs+AeOksafXg==
+  dependencies:
+    expect "^29.0.0"
+    pretty-format "^29.0.0"
+
 "@types/json-schema@*", "@types/json-schema@^7.0.6", "@types/json-schema@^7.0.8":
   version "7.0.11"
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3"