Merge branch 'master' into BUDI-8885/only-select-required-columns-from-sql-databases

packages/builder/package.json

@@ -7,7 +7,7 @@
     "build": "routify -b && NODE_OPTIONS=\"--max_old_space_size=4096\" vite build --emptyOutDir",
     "start": "routify -c rollup",
     "dev": "routify -c dev:vite",
-    "dev:vite": "vite --host 0.0.0.0",
+    "dev:vite": "vite --host 0.0.0.0 --mode=dev",
     "rollup": "rollup -c -w",
     "test": "vitest run",
     "test:watch": "vitest",

packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { Heading, Body, Layout, Button, Modal, Icon } from "@budibase/bbui"
+  import { Heading, Body, Layout, Button, Modal } from "@budibase/bbui"
   import AutomationPanel from "components/automation/AutomationPanel/AutomationPanel.svelte"
   import CreateAutomationModal from "components/automation/AutomationPanel/CreateAutomationModal.svelte"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
@@ -12,13 +12,10 @@
     automationStore,
     selectedAutomation,
   } from "stores/builder"
-  import { createLocalStorageStore } from "@budibase/frontend-core"
-  import { fly } from "svelte/transition"
 
   $: automationId = $selectedAutomation?.data?._id
   $: builderStore.selectResource(automationId)
 
-  const surveyDismissed = createLocalStorageStore("automation-survey", false)
   const stopSyncing = syncURLToState({
     urlParam: "automationId",
     stateKey: "selectedAutomationId",
@@ -31,11 +28,9 @@
 
   let modal
   let webhookModal
-  let mounted = false
 
   onMount(() => {
     $automationStore.showTestPanel = false
-    mounted = true
   })
 
   onDestroy(stopSyncing)
@@ -83,43 +78,6 @@
   </Modal>
 </div>
 
-{#if !$surveyDismissed && mounted}
-  <div
-    class="survey"
-    in:fly={{ x: 600, duration: 260, delay: 1000 }}
-    out:fly={{ x: 600, duration: 260 }}
-  >
-    <div class="survey__body">
-      <div class="survey__title">We value your feedback!</div>
-      <div class="survey__text">
-        <a
-          href="https://t.maze.co/310149185"
-          target="_blank"
-          rel="noopener noreferrer"
-          on:click={() => surveyDismissed.set(true)}
-        >
-          Complete our survey on Automations</a
-        >
-        and receive a $20 thank-you gift.
-        <a
-          href="https://drive.google.com/file/d/12-qk_2F9g5PdbM6wuKoz2KkIyLI-feMX/view?usp=sharing"
-          target="_blank"
-          rel="noopener noreferrer"
-        >
-          Terms apply.
-        </a>
-      </div>
-    </div>
-    <Icon
-      name="Close"
-      hoverable
-      color="var(--spectrum-global-color-static-gray-300)"
-      hoverColor="var(--spectrum-global-color-static-gray-100)"
-      on:click={() => surveyDismissed.set(true)}
-    />
-  </div>
-{/if}
-
 <style>
   .root {
     flex: 1 1 auto;
@@ -164,39 +122,4 @@
     grid-column: 3;
     overflow: auto;
   }
-
-  /* Survey */
-  .survey {
-    position: absolute;
-    bottom: 32px;
-    right: 32px;
-    background: var(--spectrum-semantic-positive-color-background);
-    display: flex;
-    flex-direction: row;
-    padding: var(--spacing-l) var(--spacing-xl);
-    border-radius: 4px;
-    gap: var(--spacing-xl);
-  }
-  .survey * {
-    color: var(--spectrum-global-color-static-gray-300);
-    white-space: nowrap;
-  }
-  .survey a {
-    text-decoration: underline;
-    transition: color 130ms ease-out;
-  }
-  .survey a:hover {
-    color: var(--spectrum-global-color-static-gray-100);
-    cursor: pointer;
-  }
-  .survey__body {
-    flex: 1 1 auto;
-    display: flex;
-    flex-direction: column;
-    gap: 2px;
-  }
-  .survey__title {
-    font-weight: 600;
-    font-size: 15px;
-  }
 </style>

packages/builder/src/stores/builder/datasources.ts

@@ -7,11 +7,26 @@ import {
 import { tables } from "./tables"
 import { queries } from "./queries"
 import { API } from "api"
-import { DatasourceFeature } from "@budibase/types"
+import {
+  DatasourceFeature,
+  Datasource,
+  Table,
+  Integration,
+  UIIntegration,
+  SourceName,
+} from "@budibase/types"
+// @ts-ignore
 import { TableNames } from "constants"
 
+// when building the internal DS - seems to represent it slightly differently to the backend typing of a DS
+interface InternalDatasource extends Omit<Datasource, "entities"> {
+  entities: Table[]
+}
+
 class TableImportError extends Error {
-  constructor(errors) {
+  errors: Record<string, string>
+
+  constructor(errors: Record<string, string>) {
     super()
     this.name = "TableImportError"
     this.errors = errors
@@ -26,8 +41,13 @@ class TableImportError extends Error {
   }
 }
 
+interface DatasourceStore {
+  list: Datasource[]
+  selectedDatasourceId: null | string
+}
+
 export function createDatasourcesStore() {
-  const store = writable({
+  const store = writable<DatasourceStore>({
     list: [],
     selectedDatasourceId: null,
   })
@@ -36,23 +56,25 @@ export function createDatasourcesStore() {
     // Set the internal datasource entities from the table list, which we're
     // able to keep updated unlike the egress generated definition of the
     // internal datasource
-    let internalDS = $store.list?.find(ds => ds._id === BUDIBASE_INTERNAL_DB_ID)
+    let internalDS: Datasource | InternalDatasource | undefined =
+      $store.list?.find(ds => ds._id === BUDIBASE_INTERNAL_DB_ID)
     let otherDS = $store.list?.filter(ds => ds._id !== BUDIBASE_INTERNAL_DB_ID)
     if (internalDS) {
+      const tables: Table[] = $tables.list?.filter((table: Table) => {
+        return (
+          table.sourceId === BUDIBASE_INTERNAL_DB_ID &&
+          table._id !== TableNames.USERS
+        )
+      })
       internalDS = {
         ...internalDS,
-        entities: $tables.list?.filter(table => {
+        entities: tables,
-          return (
-            table.sourceId === BUDIBASE_INTERNAL_DB_ID &&
-            table._id !== TableNames.USERS
-          )
-        }),
       }
     }
 
     // Build up enriched DS list
     // Only add the internal DS if we have at least one non-users table
-    let list = []
+    let list: (InternalDatasource | Datasource)[] = []
     if (internalDS?.entities?.length) {
       list.push(internalDS)
     }
@@ -75,62 +97,82 @@ export function createDatasourcesStore() {
     }))
   }
 
-  const select = id => {
+  const select = (id: string) => {
     store.update(state => ({
       ...state,
       selectedDatasourceId: id,
     }))
   }
 
-  const updateDatasource = (response, { ignoreErrors } = {}) => {
+  const updateDatasource = (
+    response: { datasource: Datasource; errors?: Record<string, string> },
+    { ignoreErrors }: { ignoreErrors?: boolean } = {}
+  ) => {
     const { datasource, errors } = response
     if (!ignoreErrors && errors && Object.keys(errors).length > 0) {
       throw new TableImportError(errors)
     }
-    replaceDatasource(datasource._id, datasource)
+    replaceDatasource(datasource._id!, datasource)
-    select(datasource._id)
+    select(datasource._id!)
     return datasource
   }
 
-  const updateSchema = async (datasource, tablesFilter) => {
+  const updateSchema = async (
+    datasource: Datasource,
+    tablesFilter: string[]
+  ) => {
     const response = await API.buildDatasourceSchema(
-      datasource?._id,
+      datasource?._id!,
       tablesFilter
     )
     updateDatasource(response)
   }
 
-  const sourceCount = source => {
+  const sourceCount = (source: string) => {
     return get(store).list.filter(datasource => datasource.source === source)
       .length
   }
 
-  const checkDatasourceValidity = async (integration, datasource) => {
+  const checkDatasourceValidity = async (
+    integration: Integration,
+    datasource: Datasource
+  ): Promise<{ valid: boolean; error?: string }> => {
     if (integration.features?.[DatasourceFeature.CONNECTION_CHECKING]) {
       const { connected, error } = await API.validateDatasource(datasource)
       if (connected) {
-        return
+        return { valid: true }
+      } else {
+        return { valid: false, error }
       }
-
-      throw new Error(`Unable to connect: ${error}`)
     }
+    return { valid: true }
   }
 
-  const create = async ({ integration, config }) => {
+  const create = async ({
+    integration,
+    config,
+  }: {
+    integration: UIIntegration
+    config: Record<string, any>
+  }) => {
     const count = sourceCount(integration.name)
     const nameModifier = count === 0 ? "" : ` ${count + 1}`
 
-    const datasource = {
+    const datasource: Datasource = {
       type: "datasource",
-      source: integration.name,
+      source: integration.name as SourceName,
       config,
       name: `${integration.friendlyName}${nameModifier}`,
       plus: integration.plus && integration.name !== IntegrationTypes.REST,
       isSQL: integration.isSQL,
     }
 
-    if (await checkDatasourceValidity(integration, datasource)) {
+    const { valid, error } = await checkDatasourceValidity(
-      throw new Error("Unable to connect")
+      integration,
+      datasource
+    )
+    if (!valid) {
+      throw new Error(`Unable to connect - ${error}`)
     }
 
     const response = await API.createDatasource({
@@ -141,7 +183,13 @@ export function createDatasourcesStore() {
     return updateDatasource(response, { ignoreErrors: true })
   }
 
-  const update = async ({ integration, datasource }) => {
+  const update = async ({
+    integration,
+    datasource,
+  }: {
+    integration: Integration
+    datasource: Datasource
+  }) => {
     if (await checkDatasourceValidity(integration, datasource)) {
       throw new Error("Unable to connect")
     }
@@ -151,15 +199,15 @@ export function createDatasourcesStore() {
     return updateDatasource(response)
   }
 
-  const deleteDatasource = async datasource => {
+  const deleteDatasource = async (datasource: Datasource) => {
     if (!datasource?._id || !datasource?._rev) {
       return
     }
     await API.deleteDatasource(datasource._id, datasource._rev)
-    replaceDatasource(datasource._id, null)
+    replaceDatasource(datasource._id)
   }
 
-  const replaceDatasource = (datasourceId, datasource) => {
+  const replaceDatasource = (datasourceId: string, datasource?: Datasource) => {
     if (!datasourceId) {
       return
     }
@@ -197,7 +245,7 @@ export function createDatasourcesStore() {
     }
   }
 
-  const getTableNames = async datasource => {
+  const getTableNames = async (datasource: Datasource) => {
     const info = await API.fetchInfoForDatasource(datasource)
     return info.tableNames || []
   }

packages/builder/src/stores/builder/sortedIntegrations.js

@@ -1,39 +0,0 @@
-import { integrations } from "./integrations"
-import { derived } from "svelte/store"
-
-import { DatasourceTypes } from "constants/backend"
-
-const getIntegrationOrder = type => {
-  if (type === DatasourceTypes.API) return 1
-  if (type === DatasourceTypes.RELATIONAL) return 2
-  if (type === DatasourceTypes.NON_RELATIONAL) return 3
-
-  // Sort all others arbitrarily by the first character of their name.
-  // Character codes can technically be as low as 0, so make sure the number is at least 4
-  return type.charCodeAt(0) + 4
-}
-
-export const createSortedIntegrationsStore = () => {
-  return derived(integrations, $integrations => {
-    const integrationsAsArray = Object.entries($integrations).map(
-      ([name, integration]) => ({
-        name,
-        ...integration,
-      })
-    )
-
-    return integrationsAsArray.sort((integrationA, integrationB) => {
-      const integrationASortOrder = getIntegrationOrder(integrationA.type)
-      const integrationBSortOrder = getIntegrationOrder(integrationB.type)
-      if (integrationASortOrder === integrationBSortOrder) {
-        return integrationA.friendlyName.localeCompare(
-          integrationB.friendlyName
-        )
-      }
-
-      return integrationASortOrder < integrationBSortOrder ? -1 : 1
-    })
-  })
-}
-
-export const sortedIntegrations = createSortedIntegrationsStore()

packages/builder/src/stores/builder/sortedIntegrations.ts

@@ -0,0 +1,46 @@
+import { integrations } from "./integrations"
+import { derived } from "svelte/store"
+
+import { DatasourceTypes } from "constants/backend"
+import { UIIntegration, Integration } from "@budibase/types"
+
+const getIntegrationOrder = (type: string | undefined) => {
+  // if type is not known, sort to end
+  if (!type) {
+    return Number.MAX_SAFE_INTEGER
+  }
+  if (type === DatasourceTypes.API) return 1
+  if (type === DatasourceTypes.RELATIONAL) return 2
+  if (type === DatasourceTypes.NON_RELATIONAL) return 3
+
+  // Sort all others arbitrarily by the first character of their name.
+  // Character codes can technically be as low as 0, so make sure the number is at least 4
+  return type.charCodeAt(0) + 4
+}
+
+export const createSortedIntegrationsStore = () => {
+  return derived<typeof integrations, UIIntegration[]>(
+    integrations,
+    $integrations => {
+      const entries: [string, Integration][] = Object.entries($integrations)
+      const integrationsAsArray = entries.map(([name, integration]) => ({
+        name,
+        ...integration,
+      }))
+
+      return integrationsAsArray.sort((integrationA, integrationB) => {
+        const integrationASortOrder = getIntegrationOrder(integrationA.type)
+        const integrationBSortOrder = getIntegrationOrder(integrationB.type)
+        if (integrationASortOrder === integrationBSortOrder) {
+          return integrationA.friendlyName.localeCompare(
+            integrationB.friendlyName
+          )
+        }
+
+        return integrationASortOrder < integrationBSortOrder ? -1 : 1
+      })
+    }
+  )
+}
+
+export const sortedIntegrations = createSortedIntegrationsStore()

packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte

@@ -102,9 +102,8 @@
     lastSearchId = Math.random()
     searching = true
     const thisSearchId = lastSearchId
-    const results = await searchFunction({
+    const results = await searchFunction(schema.tableId, {
       paginate: false,
-      tableId: schema.tableId,
       limit: 20,
       query: {
         string: {

packages/server/scripts/integrations/mysql/init.sql

@@ -30,6 +30,10 @@ CREATE TABLE Products (
     name text,
     updated time
 );
+CREATE TABLE `table with space` (
+    id serial primary key,
+    name text
+);
 INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Mike', 'Hughes', 28.2, '123 Fake Street', 'Belfast', '2021-01-19 03:14:07');
 INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Dave', 'Johnson', 29, '124 Fake Street', 'Belfast', '2022-04-01 00:11:11');
 INSERT INTO Person (Name) VALUES ('Elf');

packages/server/src/api/controllers/row/external.ts

@@ -55,10 +55,22 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const table = await utils.getTableFromSource(source)
   const { _id, ...rowData } = ctx.request.body
 
-  const dataToUpdate = await inputProcessing(
+  const beforeRow = await sdk.rows.external.getRow(table._id!, _id, {
+    relationships: true,
+  })
+
+  let dataToUpdate = cloneDeep(beforeRow)
+  const allowedField = utils.getSourceFields(source)
+  for (const key of Object.keys(rowData)) {
+    if (!allowedField.includes(key)) continue
+
+    dataToUpdate[key] = rowData[key]
+  }
+
+  dataToUpdate = await inputProcessing(
     ctx.user?._id,
     cloneDeep(source),
-    rowData
+    dataToUpdate
   )
 
   const validateResult = await sdk.rows.utils.validate({
@@ -69,10 +81,6 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
     throw { validation: validateResult.errors }
   }
 
-  const beforeRow = await sdk.rows.external.getRow(table._id!, _id, {
-    relationships: true,
-  })
-
   const response = await handleRequest(Operation.UPDATE, source, {
     id: breakRowIdField(_id),
     row: dataToUpdate,

packages/server/src/api/controllers/row/utils/utils.ts

@@ -110,6 +110,21 @@ function fixBooleanFields(row: Row, table: Table) {
   return row
 }
 
+export function getSourceFields(source: Table | ViewV2): string[] {
+  const isView = sdk.views.isView(source)
+  if (isView) {
+    const fields = Object.keys(
+      helpers.views.basicFields(source, { visible: true })
+    )
+    return fields
+  }
+
+  const fields = Object.entries(source.schema)
+    .filter(([_, field]) => field.visible !== false)
+    .map(([columnName]) => columnName)
+  return fields
+}
+
 export async function sqlOutputProcessing(
   rows: DatasourcePlusQueryResponse,
   source: Table | ViewV2,

packages/server/src/api/routes/tests/row.spec.ts

@@ -1333,6 +1333,62 @@ if (descriptions.length) {
           expect(resp.relationship.length).toBe(1)
         })
 
+        it("should be able to keep linked data when updating from views that trims links from the main table", async () => {
+          let row = await config.api.row.save(table._id!, {
+            name: "main",
+            description: "main description",
+          })
+          const row2 = await config.api.row.save(otherTable._id!, {
+            name: "link",
+            description: "link description",
+            relationship: [row._id],
+          })
+
+          const view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: "view",
+            schema: {
+              name: { visible: true },
+            },
+          })
+          const resp = await config.api.row.patch(view.id, {
+            _id: row._id!,
+            _rev: row._rev!,
+            tableId: row.tableId!,
+            name: "test2",
+            relationship: [row2._id],
+          })
+          expect(resp.relationship).toBeUndefined()
+
+          const updatedRow = await config.api.row.get(table._id!, row._id!)
+          expect(updatedRow.relationship.length).toBe(1)
+        })
+
+        it("should be able to keep linked data when updating from views that trims links from the foreign table", async () => {
+          let row = await config.api.row.save(table._id!, {
+            name: "main",
+            description: "main description",
+          })
+          const row2 = await config.api.row.save(otherTable._id!, {
+            name: "link",
+            description: "link description",
+            relationship: [row._id],
+          })
+
+          const view = await config.api.viewV2.create({
+            tableId: otherTable._id!,
+            name: "view",
+          })
+          await config.api.row.patch(view.id, {
+            _id: row2._id!,
+            _rev: row2._rev!,
+            tableId: row2.tableId!,
+          })
+
+          const updatedRow = await config.api.row.get(table._id!, row._id!)
+          expect(updatedRow.relationship.length).toBe(1)
+        })
+
         !isInternal &&
           // MSSQL needs a setting called IDENTITY_INSERT to be set to ON to allow writing
           // to identity columns. This is not something Budibase does currently.

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -55,7 +55,7 @@ if (descriptions.length) {
       let datasource: Datasource | undefined
 
       function saveTableRequest(
-        ...overrides: Partial<Omit<SaveTableRequest, "name">>[]
+        ...overrides: Partial<SaveTableRequest>[]
       ): SaveTableRequest {
         const req: SaveTableRequest = {
           name: generator.guid().replaceAll("-", "").substring(0, 16),
@@ -1898,6 +1898,36 @@ if (descriptions.length) {
             }
             expect(view.queryUI).toEqual(expected)
           })
+
+          it("tables and views can contain whitespaces", async () => {
+            const table = await config.api.table.save(
+              saveTableRequest({
+                name: `table with spaces ${generator.hash()}`,
+                schema: {
+                  name: {
+                    type: FieldType.STRING,
+                    name: "name",
+                  },
+                },
+              })
+            )
+
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: `view name with spaces`,
+              schema: {
+                name: { visible: true },
+              },
+            })
+
+            expect(await getDelegate(view)).toEqual({
+              ...view,
+              schema: {
+                id: { ...table.schema["id"], visible: false },
+                name: { ...table.schema["name"], visible: true },
+              },
+            })
+          })
         })
 
         describe("updating table schema", () => {

packages/server/src/tests/utilities/api/viewV2.ts

@@ -46,8 +46,11 @@ export class ViewV2API extends TestAPI {
   }
 
   get = async (viewId: string) => {
-    return (await this._get<ViewResponseEnriched>(`/api/v2/views/${viewId}`))
+    return (
-      .data
+      await this._get<ViewResponseEnriched>(
+        `/api/v2/views/${encodeURIComponent(viewId)}`
+      )
+    ).data
   }
 
   fetch = async (expectations?: Expectations) => {

packages/types/src/api/web/app/datasource.ts

@@ -12,7 +12,7 @@ export interface UpdateDatasourceResponse {
 export interface CreateDatasourceRequest {
   datasource: Datasource
   fetchSchema?: boolean
-  tablesFilter: string[]
+  tablesFilter?: string[]
 }
 
 export interface VerifyDatasourceRequest {

packages/types/src/index.ts

@@ -3,3 +3,4 @@ export * from "./sdk"
 export * from "./api"
 export * from "./core"
 export * from "./shared"
+export * from "./ui"

packages/types/src/ui/index.ts

@@ -0,0 +1 @@
+export * from "./stores"

packages/types/src/ui/stores/index.ts

@@ -0,0 +1 @@
+export * from "./integration"

packages/types/src/ui/stores/integration.ts

@@ -0,0 +1,5 @@
+import { Integration } from "@budibase/types"
+
+export interface UIIntegration extends Integration {
+  name: string
+}