Merge branch 'master' of github.com:Budibase/budibase into develop

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.7.26-alpha.5",
+  "version": "2.7.33",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/bbui/src/Markdown/SpectrumMDE.svelte

@@ -87,7 +87,7 @@
     border-color: var(--spectrum-global-color-gray-400);
   }
   /* Toolbar button color */
-  :global(.EasyMDEContainer .editor-toolbar button i) {
+  :global(.EasyMDEContainer .editor-toolbar button) {
     color: var(--spectrum-global-color-gray-800);
   }
   /* Separator between toolbar buttons*/

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { tables } from "stores/backend"
+  import { datasources, tables } from "stores/backend"
   import EditRolesButton from "./buttons/EditRolesButton.svelte"
   import { TableNames } from "constants"
   import { Grid } from "@budibase/frontend-core"
@@ -26,6 +26,18 @@
   $: id = $tables.selected?._id
   $: isUsersTable = id === TableNames.USERS
   $: isInternal = $tables.selected?.type !== "external"
+
+  const handleGridTableUpdate = async e => {
+    tables.replaceTable(id, e.detail)
+
+    // We need to refresh datasources when an external table changes.
+    // Type "external" may exist - sometimes type is "table" and sometimes it
+    // is "external" - it has different meanings in different endpoints.
+    // If we check both these then we hopefully catch all external tables.
+    if (e.detail?.type === "external" || e.detail?.sql) {
+      await datasources.fetch()
+    }
+  }
 </script>
 
 <div class="wrapper">
@@ -37,7 +49,7 @@
     allowDeleteRows={!isUsersTable}
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
-    on:updatetable={e => tables.replaceTable(id, e.detail)}
+    on:updatetable={handleGridTableUpdate}
   >
     <svelte:fragment slot="controls">
       {#if isInternal}

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -59,7 +59,6 @@
   $: valid = getErrorCount(errors) === 0 && allRequiredAttributesSet()
   $: isManyToMany = relationshipType === RelationshipTypes.MANY_TO_MANY
   $: isManyToOne = relationshipType === RelationshipTypes.MANY_TO_ONE
-  $: toRelationship.relationshipType = fromRelationship?.relationshipType
 
   function getTable(id) {
     return plusTables.find(table => table._id === id)
@@ -180,6 +179,16 @@
     return getErrorCount(errors) === 0
   }
 
+  function otherRelationshipType(type) {
+    if (type === RelationshipTypes.MANY_TO_ONE) {
+      return RelationshipTypes.ONE_TO_MANY
+    } else if (type === RelationshipTypes.ONE_TO_MANY) {
+      return RelationshipTypes.MANY_TO_ONE
+    } else if (type === RelationshipTypes.MANY_TO_MANY) {
+      return RelationshipTypes.MANY_TO_MANY
+    }
+  }
+
   function buildRelationships() {
     const id = Helpers.uuid()
     //Map temporary variables
@@ -200,6 +209,7 @@
       ...toRelationship,
       tableId: fromId,
       name: fromColumn,
+      relationshipType: otherRelationshipType(relationshipType),
       through: throughId,
       type: "link",
       _id: id,

packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte

@@ -93,6 +93,7 @@
     try {
       await beforeSave()
       table = await tables.save(newTable)
+      await datasources.fetch()
       await afterSave(table)
     } catch (e) {
       notifications.error(e)

packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte

@@ -65,6 +65,7 @@
     const updatedTable = cloneDeep(table)
     updatedTable.name = updatedName
     await tables.save(updatedTable)
+    await datasources.fetch()
     notifications.success("Table renamed successfully")
   }
 

packages/builder/src/components/common/FontAwesomeIcon.svelte

@@ -9,6 +9,18 @@
     faFileArrowUp,
     faChevronLeft,
     faCircleInfo,
+    faBold,
+    faItalic,
+    faHeading,
+    faQuoteLeft,
+    faListUl,
+    faListOl,
+    faLink,
+    faImage,
+    faEye,
+    faColumns,
+    faArrowsAlt,
+    faQuestionCircle,
   } from "@fortawesome/free-solid-svg-icons"
   import { faGithub, faDiscord } from "@fortawesome/free-brands-svg-icons"
 
@@ -22,7 +34,22 @@
     faEnvelope,
     faFileArrowUp,
     faChevronLeft,
-    faCircleInfo
+    faCircleInfo,
+
+    // -- Required for easyMDE use in the builder.
+    faBold,
+    faItalic,
+    faHeading,
+    faQuoteLeft,
+    faListUl,
+    faListOl,
+    faLink,
+    faImage,
+    faEye,
+    faColumns,
+    faArrowsAlt,
+    faQuestionCircle
+    // --
   )
   dom.watch()
 </script>

packages/builder/src/stores/backend/datasources.js

@@ -117,6 +117,10 @@ export function createDatasourcesStore() {
         ...state,
         list: [...state.list, datasource],
       }))
+
+      // If this is a new datasource then we should refresh the tables list,
+      // because otherwise we'll never see the new tables
+      tables.fetch()
     }
 
     // Update existing datasource

packages/builder/src/stores/backend/tables.js

@@ -1,5 +1,4 @@
 import { get, writable, derived } from "svelte/store"
-import { datasources } from "./"
 import { cloneDeep } from "lodash/fp"
 import { API } from "api"
 import { SWITCHABLE_TYPES } from "constants/backend"
@@ -63,7 +62,6 @@ export function createTablesStore() {
 
     const savedTable = await API.saveTable(updatedTable)
     replaceTable(savedTable._id, savedTable)
-    await datasources.fetch()
     select(savedTable._id)
     return savedTable
   }

packages/client/src/components/app/forms/InnerForm.svelte

@@ -283,7 +283,7 @@
 
       // Skip if the value is the same
       if (!skipCheck && fieldState.value === value) {
-        return true
+        return false
       }
 
       // Update field state
@@ -295,7 +295,7 @@
         return state
       })
 
-      return !error
+      return true
     }
 
     // Clears the value of a certain field back to the default value
@@ -376,8 +376,9 @@
       deregister,
       validate: () => {
         // Validate the field by force setting the same value again
-        const { fieldState } = get(getField(field))
+        const fieldInfo = getField(field)
-        return setValue(fieldState.value, true)
+        setValue(get(fieldInfo).fieldState.value, true)
+        return !get(fieldInfo).fieldState.error
       },
     }
   }

packages/frontend-core/src/components/grid/stores/columns.js

@@ -90,12 +90,12 @@ export const deriveStores = context => {
     // Update local state
     table.set(newTable)
 
+    // Update server
+    await API.saveTable(newTable)
+
     // Broadcast change to external state can be updated, as this change
     // will not be received by the builder websocket because we caused it ourselves
     dispatch("updatetable", newTable)
-
-    // Update server
-    await API.saveTable(newTable)
   }
 
   return {

packages/frontend-core/src/components/grid/stores/rows.js

@@ -2,6 +2,7 @@ import { writable, derived, get } from "svelte/store"
 import { fetchData } from "../../../fetch/fetchData"
 import { notifications } from "@budibase/bbui"
 import { NewRowID, RowPageSize } from "../lib/constants"
+import { tick } from "svelte"
 
 const initialSortState = {
   column: null,
@@ -124,13 +125,22 @@ export const deriveStores = context => {
     })
 
     // Subscribe to changes of this fetch model
-    unsubscribe = newFetch.subscribe($fetch => {
+    unsubscribe = newFetch.subscribe(async $fetch => {
       if ($fetch.loaded && !$fetch.loading) {
         hasNextPage.set($fetch.hasNextPage)
         const $instanceLoaded = get(instanceLoaded)
         const resetRows = $fetch.resetKey !== lastResetKey
+        const previousResetKey = lastResetKey
         lastResetKey = $fetch.resetKey
 
+        // If resetting rows due to a table change, wipe data and wait for
+        // derived stores to compute. This prevents stale data being passed
+        // to cells when we save the new schema.
+        if (!$instanceLoaded && previousResetKey) {
+          rows.set([])
+          await tick()
+        }
+
         // Reset state properties when dataset changes
         if (!$instanceLoaded || resetRows) {
           table.set($fetch.definition)

packages/server/src/api/controllers/row/utils.ts

@@ -3,10 +3,10 @@ import * as userController from "../user"
 import { FieldTypes } from "../../../constants"
 import { context } from "@budibase/backend-core"
 import { makeExternalQuery } from "../../../integrations/base/query"
-import { Row, Table } from "@budibase/types"
+import { FieldType, Row, Table, UserCtx } from "@budibase/types"
 import { Format } from "../view/exporters"
-import { UserCtx } from "@budibase/types"
 import sdk from "../../../sdk"
+
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
 
@@ -20,6 +20,13 @@ validateJs.extend(validateJs.validators.datetime, {
   },
 })
 
+function isForeignKey(key: string, table: Table) {
+  const relationships = Object.values(table.schema).filter(
+    column => column.type === FieldType.LINK
+  )
+  return relationships.some(relationship => relationship.foreignKey === key)
+}
+
 export async function getDatasourceAndQuery(json: any) {
   const datasourceId = json.endpoint.datasourceId
   const datasource = await sdk.datasources.get(datasourceId)
@@ -65,6 +72,10 @@ export async function validate({
     const column = fetchedTable.schema[fieldName]
     const constraints = cloneDeep(column.constraints)
     const type = column.type
+    // foreign keys are likely to be enriched
+    if (isForeignKey(fieldName, fetchedTable)) {
+      continue
+    }
     // formulas shouldn't validated, data will be deleted anyway
     if (type === FieldTypes.FORMULA || column.autocolumn) {
       continue

packages/server/src/api/controllers/table/external.ts

@@ -26,6 +26,7 @@ import {
   RelationshipTypes,
 } from "@budibase/types"
 import sdk from "../../../sdk"
+import { builderSocket } from "../../../websockets"
 const { cloneDeep } = require("lodash/fp")
 
 async function makeTableRequest(
@@ -318,6 +319,11 @@ export async function save(ctx: UserCtx) {
   datasource.entities[tableToSave.name] = tableToSave
   await db.put(datasource)
 
+  // Since tables are stored inside datasources, we need to notify clients
+  // that the datasource definition changed
+  const updatedDatasource = await db.get(datasource._id)
+  builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource)
+
   return tableToSave
 }
 
@@ -344,6 +350,11 @@ export async function destroy(ctx: UserCtx) {
 
   await db.put(datasource)
 
+  // Since tables are stored inside datasources, we need to notify clients
+  // that the datasource definition changed
+  const updatedDatasource = await db.get(datasource._id)
+  builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource)
+
   return tableToDelete
 }
 

packages/server/src/integrations/mongodb.ts

@@ -385,7 +385,7 @@ class MongoIntegration implements IntegrationBase {
   createObjectIds(json: any) {
     const self = this
     function interpolateObjectIds(json: any) {
-      for (let field of Object.keys(json)) {
+      for (let field of Object.keys(json || {})) {
         if (json[field] instanceof Object) {
           json[field] = self.createObjectIds(json[field])
         }

packages/server/src/sdk/app/datasources/datasources.ts

@@ -36,11 +36,14 @@ export function checkDatasourceTypes(schema: Integration, config: any) {
 async function enrichDatasourceWithValues(datasource: Datasource) {
   const cloned = cloneDeep(datasource)
   const env = await getEnvironmentVariables()
+  //Do not process entities, as we do not want to process formulas
+  const { entities, ...clonedWithoutEntities } = cloned
   const processed = processObjectSync(
-    cloned,
+    clonedWithoutEntities,
     { env },
     { onlyFound: true }
   ) as Datasource
+  processed.entities = entities
   const definition = await getDefinition(processed.source)
   processed.config = checkDatasourceTypes(definition!, processed.config)
   return {