Merge branch 'develop' of github.com:Budibase/budibase into feature/table-fetching-frontend

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.6.19-alpha.52",
+  "version": "2.6.24-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/backend-core",

packages/backend-core/src/events/identification.ts

@@ -86,6 +86,7 @@ const getCurrentIdentity = async (): Promise<Identity> => {
       installationId,
       tenantId,
       environment,
+      realTenantId: context.getTenantId(),
       hostInfo: userContext.hostInfo,
     }
   } else {

packages/builder/src/builderStore/websocket.js

@@ -1,15 +1,18 @@
 import { createWebsocket } from "@budibase/frontend-core"
-import { userStore } from "builderStore"
+import { userStore, store } from "builderStore"
 import { datasources, tables } from "stores/backend"
+import { get } from "svelte/store"
+import { auth } from "stores/portal"
 import { SocketEvent, BuilderSocketEvent } from "@budibase/shared-core"
+import { notifications } from "@budibase/bbui"
 
 export const createBuilderWebsocket = appId => {
   const socket = createWebsocket("/socket/builder")
 
   // Built-in events
   socket.on("connect", () => {
-    socket.emit(BuilderSocketEvent.SelectApp, appId, response => {
+    socket.emit(BuilderSocketEvent.SelectApp, { appId }, ({ users }) => {
-      userStore.actions.init(response.users)
+      userStore.actions.init(users)
     })
   })
   socket.on("connect_error", err => {
@@ -20,8 +23,21 @@ export const createBuilderWebsocket = appId => {
   })
 
   // User events
-  socket.onOther(SocketEvent.UserUpdate, userStore.actions.updateUser)
+  socket.onOther(SocketEvent.UserUpdate, ({ user }) => {
-  socket.onOther(SocketEvent.UserDisconnect, userStore.actions.removeUser)
+    userStore.actions.updateUser(user)
+  })
+  socket.onOther(SocketEvent.UserDisconnect, ({ sessionId }) => {
+    userStore.actions.removeUser(sessionId)
+  })
+  socket.onOther(BuilderSocketEvent.LockTransfer, ({ userId }) => {
+    if (userId === get(auth)?.user?._id) {
+      notifications.success("You can now edit screens and automations")
+      store.update(state => ({
+        ...state,
+        hasLock: true,
+      }))
+    }
+  })
 
   // Table events
   socket.onOther(BuilderSocketEvent.TableChange, ({ id, table }) => {

packages/builder/src/helpers/userInitials.js

@@ -1,13 +0,0 @@
-const getUserInitials = user => {
-  if (user.firstName && user.lastName) {
-    return user.firstName[0] + user.lastName[0]
-  } else if (user.firstName) {
-    return user.firstName[0]
-  } else if (user.email) {
-    return user.email[0]
-  }
-
-  return "U"
-}
-
-export default getUserInitials

packages/builder/src/pages/builder/portal/settings/branding.svelte

@@ -44,14 +44,14 @@
   let config = {}
   let updated = false
 
-  $: onConfigUpdate(config, mounted)
+  $: onConfigUpdate(config)
-  $: init = Object.keys(config).length > 0
+  $: initialised = Object.keys(config).length > 0
 
   $: isCloud = $admin.cloud
   $: brandingEnabled = $licensing.brandingEnabled
 
   const onConfigUpdate = () => {
-    if (!mounted || updated || !init) {
+    if (!mounted || updated || !initialised) {
       return
     }
     updated = true
@@ -122,34 +122,27 @@
     return response
   }
 
-  async function saveConfig() {
+  async function saveFiles() {
-    saving = true
-
     if (logoFile) {
       const logoResp = await uploadLogo(logoFile)
       if (logoResp.url) {
-        config = {
-          ...config,
-          logoUrl: logoResp.url,
-        }
         logoFile = null
         logoPreview = null
       }
+      config.logoUrl = undefined
     }
 
     if (faviconFile) {
       const faviconResp = await uploadFavicon(faviconFile)
       if (faviconResp.url) {
-        config = {
-          ...config,
-          faviconUrl: faviconResp.url,
-        }
         faviconFile = null
         faviconPreview = null
       }
+      config.faviconUrl = undefined
     }
+  }
 
-    // Trim
+  function trimFields() {
     const userStrings = [
       "metaTitle",
       "platformTitle",
@@ -168,11 +161,18 @@
       ...config,
       ...trimmed,
     }
+  }
+
+  async function saveConfig() {
+    saving = true
+
+    await saveFiles()
+    trimFields()
 
     try {
       // Update settings
       await organisation.save(config)
-      await organisation.init()
+      await init()
       notifications.success("Branding settings updated")
     } catch (e) {
       console.error("Branding updated failed", e)
@@ -182,9 +182,10 @@
     saving = false
   }
 
-  onMount(async () => {
+  async function init() {
-    await organisation.init()
+    if (!$organisation.loaded) {
-
+      await organisation.init()
+    }
     config = {
       faviconUrl: $organisation.faviconUrl,
       logoUrl: $organisation.logoUrl,
@@ -197,6 +198,10 @@
       metaImageUrl: $organisation.metaImageUrl,
       metaTitle: $organisation.metaTitle,
     }
+  }
+
+  onMount(async () => {
+    await init()
     mounted = true
   })
 </script>
@@ -262,6 +267,7 @@
               faviconFile = e.detail
               faviconPreview = null
             } else {
+              faviconFile = null
               clone.faviconUrl = ""
             }
             config = clone
@@ -408,7 +414,11 @@
           Upgrade
         </Button>
       {/if}
-      <Button on:click={saveConfig} cta disabled={saving || !updated || !init}>
+      <Button
+        on:click={saveConfig}
+        cta
+        disabled={saving || !updated || !$organisation.loaded}
+      >
         Save
       </Button>
     </div>

packages/builder/src/pages/builder/portal/users/users/[userId].svelte

@@ -31,7 +31,6 @@
   import AppNameTableRenderer from "./_components/AppNameTableRenderer.svelte"
   import AppRoleTableRenderer from "./_components/AppRoleTableRenderer.svelte"
   import ScimBanner from "../_components/SCIMBanner.svelte"
-  import { helpers } from "@budibase/shared-core"
 
   export let userId
 
@@ -91,7 +90,6 @@
   $: readonly = !$auth.isAdmin || scimEnabled
   $: privileged = user?.admin?.global || user?.builder?.global
   $: nameLabel = getNameLabel(user)
-  $: initials = helpers.getUserInitials(user)
   $: filteredGroups = getFilteredGroups($groups, searchTerm)
   $: availableApps = getAvailableApps($apps, privileged, user?.roles)
   $: userGroups = $groups.filter(x => {

packages/builder/src/stores/portal/auth.js

@@ -2,7 +2,6 @@ import { derived, writable, get } from "svelte/store"
 import { API } from "api"
 import { admin } from "stores/portal"
 import analytics from "analytics"
-import getUserInitials from "helpers/userInitials.js"
 
 export function createAuthStore() {
   const auth = writable({
@@ -14,12 +13,10 @@ export function createAuthStore() {
     postLogout: false,
   })
   const store = derived(auth, $store => {
-    let initials = null
     let isAdmin = false
     let isBuilder = false
     if ($store.user) {
       const user = $store.user
-      initials = getUserInitials(user)
       isAdmin = !!user.admin?.global
       isBuilder = !!user.builder?.global
     }
@@ -30,7 +27,6 @@ export function createAuthStore() {
       tenantSet: $store.tenantSet,
       loaded: $store.loaded,
       postLogout: $store.postLogout,
-      initials,
       isAdmin,
       isBuilder,
       isSSO: !!$store.user?.provider,

packages/builder/src/stores/portal/organisation.js

@@ -23,6 +23,7 @@ const DEFAULT_CONFIG = {
   oidcCallbackUrl: "",
   googleCallbackUrl: "",
   isSSOEnforced: false,
+  loaded: false,
 }
 
 export function createOrganisationStore() {
@@ -32,7 +33,7 @@ export function createOrganisationStore() {
   async function init() {
     const tenantId = get(auth).tenantId
     const settingsConfigDoc = await API.getTenantConfig(tenantId)
-    set({ ...DEFAULT_CONFIG, ...settingsConfigDoc.config })
+    set({ ...DEFAULT_CONFIG, ...settingsConfigDoc.config, loaded: true })
   }
 
   async function save(config) {
@@ -43,6 +44,10 @@ export function createOrganisationStore() {
     delete storeConfig.googleDatasourceConfigured
     delete storeConfig.oidcCallbackUrl
     delete storeConfig.googleCallbackUrl
+
+    // delete internal store field
+    delete storeConfig.loaded
+
     await API.saveConfig({
       type: "settings",
       config: { ...storeConfig, ...config },

packages/frontend-core/src/components/grid/lib/websocket.js

@@ -11,10 +11,13 @@ export const createGridWebsocket = context => {
       return
     }
     // Identify which table we are editing
-    socket.emit(GridSocketEvent.SelectTable, tableId, response => {
+    socket.emit(
-      // handle initial connection info
+      GridSocketEvent.SelectTable,
-      users.set(response.users)
+      { tableId },
-    })
+      ({ users: gridUsers }) => {
+        users.set(gridUsers)
+      }
+    )
   }
 
   // Built-in events
@@ -26,29 +29,29 @@ export const createGridWebsocket = context => {
   })
 
   // User events
-  socket.onOther(SocketEvent.UserUpdate, user => {
+  socket.onOther(SocketEvent.UserUpdate, ({ user }) => {
     users.actions.updateUser(user)
   })
-  socket.onOther(SocketEvent.UserDisconnect, user => {
+  socket.onOther(SocketEvent.UserDisconnect, ({ sessionId }) => {
-    users.actions.removeUser(user)
+    users.actions.removeUser(sessionId)
   })
 
   // Row events
-  socket.onOther(GridSocketEvent.RowChange, async data => {
+  socket.onOther(GridSocketEvent.RowChange, async ({ id, row }) => {
-    if (data.id) {
+    if (id) {
-      rows.actions.replaceRow(data.id, data.row)
+      rows.actions.replaceRow(id, row)
-    } else if (data.row.id) {
+    } else if (row.id) {
       // Handle users table edge cased
-      await rows.actions.refreshRow(data.row.id)
+      await rows.actions.refreshRow(row.id)
     }
   })
 
   // Table events
-  socket.onOther(GridSocketEvent.TableChange, data => {
+  socket.onOther(GridSocketEvent.TableChange, ({ table: newTable }) => {
     // Only update table if one exists. If the table was deleted then we don't
     // want to know - let the builder navigate away
-    if (data.table) {
+    if (newTable) {
-      table.set(data.table)
+      table.set(newTable)
     }
   })
 
@@ -57,7 +60,7 @@ export const createGridWebsocket = context => {
 
   // Notify selected cell changes
   focusedCellId.subscribe($focusedCellId => {
-    socket.emit(GridSocketEvent.SelectCell, $focusedCellId)
+    socket.emit(GridSocketEvent.SelectCell, { cellId: $focusedCellId })
   })
 
   return () => socket?.disconnect()

packages/pro

@@ -1 +1 @@
-Subproject commit cd06642b860111aa1bd3443ee10076ca3abf03c3
+Subproject commit 01fbc8670021c5a275c2a1a36ee18b984eeafad5

packages/server/src/automations/steps/queryRows.ts

@@ -14,6 +14,7 @@ import {
   SearchFilters,
   Table,
 } from "@budibase/types"
+import { db as dbCore } from "@budibase/backend-core"
 
 enum SortOrder {
   ASCENDING = "ascending",
@@ -121,7 +122,11 @@ function typeCoercion(filters: SearchFilters, table: Table) {
     const searchParam = filters[key]
     if (typeof searchParam === "object") {
       for (let [property, value] of Object.entries(searchParam)) {
-        const column = table.schema[property]
+        // We need to strip numerical prefixes here, so that we can look up
+        // the correct field name in the schema
+        const columnName = dbCore.removeKeyNumbering(property)
+        const column = table.schema[columnName]
+
         // convert string inputs
         if (!column || typeof value !== "string") {
           continue

packages/server/src/integrations/snowflake.ts

@@ -38,6 +38,9 @@ const SCHEMA: Integration = {
       type: "password",
       required: true,
     },
+    role: {
+      type: "string",
+    },
     warehouse: {
       type: "string",
       required: true,

packages/server/src/sdk/users/utils.ts

@@ -7,7 +7,7 @@ import {
   InternalTables,
 } from "../../db/utils"
 import { isEqual } from "lodash"
-import { ContextUser, UserMetadata, User } from "@budibase/types"
+import { ContextUser, UserMetadata, User, Database } from "@budibase/types"
 
 export function combineMetadataAndUser(
   user: ContextUser,
@@ -51,8 +51,10 @@ export function combineMetadataAndUser(
   return null
 }
 
-export async function rawUserMetadata() {
+export async function rawUserMetadata(db?: Database) {
-  const db = context.getAppDB()
+  if (!db) {
+    db = context.getAppDB()
+  }
   return (
     await db.allDocs(
       getUserMetadataParams(null, {
@@ -64,30 +66,36 @@ export async function rawUserMetadata() {
 
 export async function syncGlobalUsers() {
   // sync user metadata
-  const db = context.getAppDB()
+  const dbs = [context.getDevAppDB(), context.getProdAppDB()]
-  const resp = await Promise.all([getGlobalUsers(), rawUserMetadata()])
+  for (let db of dbs) {
-  const users = resp[0] as User[]
+    if (!(await db.exists())) {
-  const metadata = resp[1] as UserMetadata[]
-  const toWrite = []
-  for (let user of users) {
-    const combined = combineMetadataAndUser(user, metadata)
-    if (combined) {
-      toWrite.push(combined)
-    }
-  }
-  let foundEmails: string[] = []
-  for (let data of metadata) {
-    if (!data._id) {
       continue
     }
-    const alreadyExisting = data.email && foundEmails.indexOf(data.email) !== -1
+    const resp = await Promise.all([getGlobalUsers(), rawUserMetadata(db)])
-    const globalId = getGlobalIDFromUserMetadataID(data._id)
+    const users = resp[0] as User[]
-    if (!users.find(user => user._id === globalId) || alreadyExisting) {
+    const metadata = resp[1] as UserMetadata[]
-      toWrite.push({ ...data, _deleted: true })
+    const toWrite = []
+    for (let user of users) {
+      const combined = combineMetadataAndUser(user, metadata)
+      if (combined) {
+        toWrite.push(combined)
+      }
     }
-    if (data.email) {
+    let foundEmails: string[] = []
-      foundEmails.push(data.email)
+    for (let data of metadata) {
+      if (!data._id) {
+        continue
+      }
+      const alreadyExisting =
+        data.email && foundEmails.indexOf(data.email) !== -1
+      const globalId = getGlobalIDFromUserMetadataID(data._id)
+      if (!users.find(user => user._id === globalId) || alreadyExisting) {
+        toWrite.push({ ...data, _deleted: true })
+      }
+      if (data.email) {
+        foundEmails.push(data.email)
+      }
     }
+    await db.bulkDocs(toWrite)
   }
-  await db.bulkDocs(toWrite)
 }

packages/server/src/utilities/global.ts

@@ -122,11 +122,8 @@ export async function getGlobalUsers(
       delete user.forceResetPassword
       return user
     })
-  if (!appId) {
-    return globalUsers
-  }
 
-  if (opts?.noProcessing) {
+  if (opts?.noProcessing || !appId) {
     return globalUsers
   } else {
     // pass in the groups, meaning we don't actually need to retrieve them for

packages/server/src/websockets/builder.ts

@@ -5,7 +5,7 @@ import http from "http"
 import Koa from "koa"
 import { Datasource, Table, SocketSession, ContextUser } from "@budibase/types"
 import { gridSocket } from "./index"
-import { clearLock } from "../utilities/redis"
+import { clearLock, updateLock } from "../utilities/redis"
 import { Socket } from "socket.io"
 import { BuilderSocketEvent } from "@budibase/shared-core"
 
@@ -16,7 +16,7 @@ export default class BuilderSocket extends BaseSocket {
 
   async onConnect(socket?: Socket) {
     // Initial identification of selected app
-    socket?.on(BuilderSocketEvent.SelectApp, async (appId, callback) => {
+    socket?.on(BuilderSocketEvent.SelectApp, async ({ appId }, callback) => {
       await this.joinRoom(socket, appId)
 
       // Reply with all users in current room
@@ -26,7 +26,8 @@ export default class BuilderSocket extends BaseSocket {
   }
 
   async onDisconnect(socket: Socket) {
-    // Remove app lock from this user if they have no other connections
+    // Remove app lock from this user if they have no other connections,
+    // and transfer it to someone else if possible
     try {
       // @ts-ignore
       const session: SocketSession = socket.data
@@ -36,9 +37,26 @@ export default class BuilderSocket extends BaseSocket {
         return _id === otherSession._id && sessionId !== otherSession.sessionId
       })
       if (!hasOtherSession && room) {
+        // Clear the lock from this user since they had no other sessions
         // @ts-ignore
         const user: ContextUser = { _id: socket.data._id }
         await clearLock(room, user)
+
+        // Transfer lock ownership to the next oldest user
+        let otherSessions = sessions.filter(x => x._id !== _id).slice()
+        otherSessions.sort((a, b) => {
+          return a.connectedAt < b.connectedAt ? -1 : 1
+        })
+        const nextSession = otherSessions[0]
+        if (nextSession) {
+          const { _id, email, firstName, lastName } = nextSession
+          // @ts-ignore
+          const nextUser: ContextUser = { _id, email, firstName, lastName }
+          await updateLock(room, nextUser)
+          this.io.to(room).emit(BuilderSocketEvent.LockTransfer, {
+            userId: _id,
+          })
+        }
       }
     } catch (e) {
       // This is fine, just means this user didn't hold the lock

packages/server/src/websockets/grid.ts

@@ -15,7 +15,7 @@ export default class GridSocket extends BaseSocket {
 
   async onConnect(socket: Socket) {
     // Initial identification of connected spreadsheet
-    socket.on(GridSocketEvent.SelectTable, async (tableId, callback) => {
+    socket.on(GridSocketEvent.SelectTable, async ({ tableId }, callback) => {
       await this.joinRoom(socket, tableId)
 
       // Reply with all users in current room
@@ -24,7 +24,7 @@ export default class GridSocket extends BaseSocket {
     })
 
     // Handle users selecting a new cell
-    socket.on(GridSocketEvent.SelectCell, cellId => {
+    socket.on(GridSocketEvent.SelectCell, ({ cellId }) => {
       this.updateUser(socket, { focusedCellId: cellId })
     })
   }

packages/server/src/websockets/websocket.ts

@@ -77,6 +77,7 @@ export class BaseSocket {
                 firstName,
                 lastName,
                 sessionId: socket.id,
+                connectedAt: Date.now(),
               }
               next()
             }
@@ -173,7 +174,9 @@ export class BaseSocket {
     )
     const prunedSessionIds = sessionIds.filter((id, idx) => {
       if (!sessionsExist[idx]) {
-        this.io.to(room).emit(SocketEvent.UserDisconnect, sessionIds[idx])
+        this.io.to(room).emit(SocketEvent.UserDisconnect, {
+          sessionId: sessionIds[idx],
+        })
         return false
       }
       return true
@@ -216,7 +219,9 @@ export class BaseSocket {
     }
 
     // Notify other users
-    socket.to(room).emit(SocketEvent.UserUpdate, user)
+    socket.to(room).emit(SocketEvent.UserUpdate, {
+      user,
+    })
   }
 
   // Disconnects a socket from its current room
@@ -242,7 +247,7 @@ export class BaseSocket {
     )
 
     // Notify other users
-    socket.to(room).emit(SocketEvent.UserDisconnect, sessionId)
+    socket.to(room).emit(SocketEvent.UserDisconnect, { sessionId })
   }
 
   // Updates a connected user's metadata, assuming a room change is not required.

packages/shared-core/src/constants.ts

@@ -85,6 +85,7 @@ export enum BuilderSocketEvent {
   SelectApp = "SelectApp",
   TableChange = "TableChange",
   DatasourceChange = "DatasourceChange",
+  LockTransfer = "LockTransfer",
 }
 
 export const SocketSessionTTL = 60

packages/shared-core/src/helpers/helpers.ts

@@ -35,7 +35,10 @@ export const getUserInitials = (user: User) => {
   let initials = ""
   initials += user.firstName ? user.firstName[0] : ""
   initials += user.lastName ? user.lastName[0] : ""
-  return initials === "" ? user.email[0] : initials
+  if (initials !== "") {
+    return initials
+  }
+  return user.email?.[0] || "U"
 }
 
 /**

packages/types/src/sdk/websocket.ts

@@ -5,4 +5,5 @@ export interface SocketSession {
   lastName?: string
   sessionId: string
   room?: string
+  connectedAt: number
 }

packages/worker/src/api/controllers/global/configs.ts

@@ -23,6 +23,7 @@ import {
   isSettingsConfig,
   isSMTPConfig,
   OIDCConfigs,
+  SettingsBrandingConfig,
   SettingsInnerConfig,
   SSOConfig,
   SSOConfigType,
@@ -142,13 +143,29 @@ async function hasActivatedConfig(ssoConfigs?: SSOConfigs) {
   return !!Object.values(ssoConfigs).find(c => c?.activated)
 }
 
-async function verifySettingsConfig(config: SettingsInnerConfig) {
+async function verifySettingsConfig(
+  config: SettingsInnerConfig & SettingsBrandingConfig,
+  existingConfig?: SettingsInnerConfig & SettingsBrandingConfig
+) {
   if (config.isSSOEnforced) {
     const valid = await hasActivatedConfig()
     if (!valid) {
       throw new Error("Cannot enforce SSO without an activated configuration")
     }
   }
+
+  // always preserve file attributes
+  // these should be set via upload instead
+  // only allow for deletion by checking empty string to bypass this behaviour
+
+  if (existingConfig && config.logoUrl !== "") {
+    config.logoUrl = existingConfig.logoUrl
+    config.logoUrlEtag = existingConfig.logoUrlEtag
+  }
+  if (existingConfig && config.faviconUrl !== "") {
+    config.faviconUrl = existingConfig.faviconUrl
+    config.faviconUrlEtag = existingConfig.faviconUrlEtag
+  }
 }
 
 async function verifySSOConfig(type: SSOConfigType, config: SSOConfig) {
@@ -198,7 +215,7 @@ export async function save(ctx: UserCtx<Config>) {
         await email.verifyConfig(config)
         break
       case ConfigType.SETTINGS:
-        await verifySettingsConfig(config)
+        await verifySettingsConfig(config, existingConfig?.config)
         break
       case ConfigType.GOOGLE:
         await verifyGoogleConfig(config)
@@ -320,14 +337,15 @@ export async function publicSettings(
       )
     }
 
-    if (branding.faviconUrl && branding.faviconUrl !== "") {
+    // enrich the favicon url - empty url means deleted
-      // @ts-ignore
+    const faviconUrl =
-      config.faviconUrl = objectStore.getGlobalFileUrl(
+      branding.faviconUrl && branding.faviconUrl !== ""
-        "settings",
+        ? objectStore.getGlobalFileUrl(
-        "faviconUrl",
+            "settings",
-        branding.faviconUrl
+            "faviconUrl",
-      )
+            branding.faviconUrlEtag
-    }
+          )
+        : undefined
 
     // google
     const googleConfig = await configs.getGoogleConfig()
@@ -352,6 +370,7 @@ export async function publicSettings(
       config: {
         ...config,
         ...branding,
+        ...{ faviconUrl },
         google,
         googleDatasourceConfigured,
         oidc,